1 #define TRACE_NAME "MulticastTransfer"
3 #include "artdaq/TransferPlugins/TransferInterface.hh"
5 #include "artdaq-core/Data/Fragment.hh"
6 #include "artdaq-core/Utilities/ExceptionHandler.hh"
8 #include "fhiclcpp/ParameterSet.h"
9 #include "cetlib_except/exception.h"
11 #include <boost/asio.hpp>
12 #include <boost/bind.hpp>
18 #include <type_traits>
21 #pragma GCC diagnostic push
22 #pragma GCC diagnostic ignored "-Wunused-parameter"
34 using byte_t = artdaq::Fragment::byte_t;
67 size_t receiveTimeout)
override;
101 void fill_staging_memory(
const artdaq::Fragment& frag);
103 template <
typename T>
104 void book_container_of_buffers(std::vector<T>& buffers,
105 const size_t fragment_size,
106 const size_t total_subfragments,
107 const size_t first_subfragment_num,
108 const size_t last_subfragment_num);
110 void get_fragment_quantities(
const boost::asio::mutable_buffer& buf,
size_t& payload_size,
size_t& fragment_size,
111 size_t& expected_subfragments);
113 void set_receive_buffer_size(
size_t recv_buff_size);
115 class subfragment_identifier
119 subfragment_identifier(
size_t sequenceID,
size_t fragmentID,
size_t subfragment_number) :
120 sequenceID_(sequenceID)
121 , fragmentID_(fragmentID)
122 , subfragment_number_(subfragment_number) { }
124 size_t sequenceID()
const {
return sequenceID_; }
125 size_t fragmentID()
const {
return fragmentID_; }
126 size_t subfragment_number()
const {
return subfragment_number_; }
131 size_t subfragment_number_;
134 std::unique_ptr<boost::asio::io_service> io_service_;
136 std::unique_ptr<boost::asio::ip::udp::endpoint> local_endpoint_;
137 std::unique_ptr<boost::asio::ip::udp::endpoint> multicast_endpoint_;
138 std::unique_ptr<boost::asio::ip::udp::endpoint> opposite_endpoint_;
140 std::unique_ptr<boost::asio::ip::udp::socket> socket_;
142 size_t subfragment_size_;
143 size_t subfragments_per_send_;
145 size_t pause_on_copy_usecs_;
146 Fragment fragment_buffer_;
148 std::vector<byte_t> staging_memory_;
150 std::vector<boost::asio::mutable_buffer> receive_buffers_;
156 , io_service_(std::make_unique<std::remove_reference<decltype(*io_service_)>::type>())
157 , local_endpoint_(nullptr)
158 , multicast_endpoint_(nullptr)
159 , opposite_endpoint_(std::make_unique<std::remove_reference<decltype(*opposite_endpoint_)>::type>())
161 , subfragment_size_(pset.get<size_t>(
"subfragment_size"))
162 , subfragments_per_send_(pset.get<size_t>(
"subfragments_per_send"))
163 , pause_on_copy_usecs_(pset.get<size_t>(
"pause_on_copy_usecs", 0))
167 auto port = pset.get<
unsigned short>(
"multicast_port");
168 auto multicast_address = boost::asio::ip::address::from_string(pset.get<std::string>(
"multicast_address"));
169 auto local_address = boost::asio::ip::address::from_string(pset.get<std::string>(
"local_address"));
171 TLOG(TLVL_DEBUG) <<
GetTraceName() <<
": multicast address is set to " << multicast_address ;
172 TLOG(TLVL_DEBUG) <<
GetTraceName() <<
": local address is set to " << local_address ;
176 local_endpoint_ = std::make_unique<std::remove_reference<decltype(*local_endpoint_)>::type>(local_address, 0);
177 multicast_endpoint_ = std::make_unique<std::remove_reference<decltype(*multicast_endpoint_)>::type>(multicast_address, port);
179 socket_ = std::make_unique<std::remove_reference<decltype(*socket_)>::type>(*io_service_,
180 multicast_endpoint_->protocol());
181 socket_->bind(*local_endpoint_);
188 local_endpoint_ = std::make_unique<std::remove_reference<decltype(*local_endpoint_)>::type>(local_address, port);
189 socket_ = std::make_unique<std::remove_reference<decltype(*socket_)>::type>(*io_service_,
190 local_endpoint_->protocol());
192 boost::system::error_code ec;
194 socket_->set_option(boost::asio::ip::udp::socket::reuse_address(
true), ec);
198 std::cerr <<
"boost::system::error_code with value " << ec <<
" was found in setting reuse_address option" << std::endl;
201 set_receive_buffer_size(pset.get<
size_t>(
"receive_buffer_size", 0));
203 socket_->bind(boost::asio::ip::udp::endpoint(multicast_address, port));
207 socket_->set_option(boost::asio::ip::multicast::join_group(multicast_address), ec);
211 std::cerr <<
"boost::system::error_code with value " << ec <<
" was found in attempt to join multicast group" << std::endl;
217 ExceptionHandler(ExceptionHandlerRethrow::yes,
"Problem setting up the socket in MulticastTransfer");
220 auto max_subfragments =
223 staging_memory_.resize(max_subfragments * (
sizeof(subfragment_identifier) + subfragment_size_));
230 TLOG(TLVL_DEBUG) <<
GetTraceName() <<
": max_subfragments is " << max_subfragments ;
231 TLOG(TLVL_DEBUG) <<
GetTraceName() <<
": Staging buffer size is " << staging_memory_.size() ;
234 #pragma GCC diagnostic push
235 #pragma GCC diagnostic ignored "-Wunused-variable"
238 size_t receiveTimeout)
242 if (fragment.dataSizeBytes() > 0)
244 throw cet::exception(
"MulticastTransfer") <<
"Error in MulticastTransfer::receiveFragmentFrom: " <<
245 "nonzero payload found in fragment passed as argument";
248 static bool print_warning =
true;
252 std::cerr <<
"Please note that MulticastTransfer::receiveFragmentFrom does not use its receiveTimeout argument" << std::endl;
253 print_warning =
false;
256 fragment.resizeBytes(max_fragment_size_words_ -
sizeof(artdaq::detail::RawFragmentHeader));
258 static auto current_sequenceID = std::numeric_limits<Fragment::sequence_id_t>::max();
259 static auto current_fragmentID = std::numeric_limits<Fragment::fragment_id_t>::max();
261 size_t fragment_size = 0;
262 size_t expected_subfragments = 0;
263 size_t current_subfragments = 0;
264 bool fragment_complete =
false;
265 bool last_fragment_truncated =
false;
269 auto bytes_received = socket_->receive_from(receive_buffers_, *opposite_endpoint_);
271 size_t bytes_processed = 0;
273 for (
auto& buf : receive_buffers_)
275 auto buf_size = boost::asio::buffer_size(buf);
276 auto size_t_ptr = boost::asio::buffer_cast<
const size_t*>(buf);
277 auto seqID = *size_t_ptr;
278 auto fragID = *(size_t_ptr + 1);
279 auto subfragID = *(size_t_ptr + 2);
281 if (seqID != current_sequenceID || fragID != current_fragmentID)
286 assert(bytes_processed == 0);
288 if (current_subfragments < expected_subfragments)
290 last_fragment_truncated =
true;
292 if (expected_subfragments != std::numeric_limits<size_t>::max())
294 std::cerr <<
"Warning: only received " << current_subfragments <<
" subfragments for fragment with seqID = " <<
295 current_sequenceID <<
", fragID = " << current_fragmentID <<
" (expected " << expected_subfragments <<
")\n"
300 std::cerr <<
"Warning: only received " << current_subfragments <<
301 " subfragments for fragment with seqID = " <<
302 current_sequenceID <<
", fragID = " << current_fragmentID <<
303 ", # of expected subfragments is unknown as fragment header was not received)\n"
308 current_subfragments = 0;
309 fragment_size = std::numeric_limits<size_t>::max();
310 expected_subfragments = std::numeric_limits<size_t>::max();
311 current_sequenceID = seqID;
312 current_fragmentID = fragID;
315 auto ptr_into_fragment = fragment.headerBeginBytes() + subfragID * subfragment_size_;
317 auto ptr_into_buffer = boost::asio::buffer_cast<
const byte_t*>(buf) +
sizeof(subfragment_identifier);
319 std::copy(ptr_into_buffer, ptr_into_buffer + buf_size -
sizeof(subfragment_identifier), ptr_into_fragment);
323 if (buf_size >=
sizeof(subfragment_identifier) +
sizeof(artdaq::detail::RawFragmentHeader))
325 auto payload_size = std::numeric_limits<size_t>::max();
326 get_fragment_quantities(buf, payload_size, fragment_size, expected_subfragments);
328 fragment.resizeBytes(payload_size);
332 throw cet::exception(
"MulticastTransfer") <<
"Buffer size is too small to completely contain an artdaq::Fragment header; " <<
333 "please increase the default size";
337 current_subfragments++;
339 if (current_subfragments == expected_subfragments)
341 fragment_complete =
true;
344 bytes_processed += buf_size;
346 if (bytes_processed >= bytes_received)
352 if (last_fragment_truncated)
361 assert(!fragment_complete);
362 TLOG(TLVL_WARNING) << GetTraceName() <<
": Got an incomplete fragment" ;
366 if (fragment_complete)
368 return source_rank();
375 #pragma GCC diagnostic pop
379 auto ret = receiveFragment(fragment_buffer_, receiveTimeout);
380 if (ret == source_rank())
382 header = *
reinterpret_cast<detail::RawFragmentHeader*
>(fragment_buffer_.headerAddress());
383 return source_rank();
390 if (fragment_buffer_.size() > detail::RawFragmentHeader::num_words()) {
391 auto dataSize = (fragment_buffer_.size() - detail::RawFragmentHeader::num_words()) *
sizeof(RawDataType);
392 memcpy(destination, fragment_buffer_.headerAddress() + detail::RawFragmentHeader::num_words(), dataSize);
393 return source_rank();
403 return copyFragment(f, 100000000);
408 size_t send_timeout_usec)
412 if (fragment.sizeBytes() > max_fragment_size_words_)
414 throw cet::exception(
"MulticastTransfer") <<
"Error in MulticastTransfer::copyFragmentTo: " <<
415 fragment.sizeBytes() <<
" byte fragment exceeds max_fragment_size of " << max_fragment_size_words_;
418 static size_t ncalls = 1;
419 auto num_subfragments =
static_cast<size_t>(std::ceil(fragment.sizeBytes() /
static_cast<float>(subfragment_size_)));
423 fill_staging_memory(fragment);
425 for (
size_t batch_index = 0; ; batch_index++)
427 auto first_subfragment = batch_index * subfragments_per_send_;
428 auto last_subfragment = (batch_index + 1) * subfragments_per_send_ >= num_subfragments ?
429 num_subfragments - 1 :
430 (batch_index + 1) * subfragments_per_send_ - 1;
432 std::vector<boost::asio::const_buffer> buffers;
434 book_container_of_buffers(buffers, fragment.sizeBytes(), num_subfragments, first_subfragment, last_subfragment);
436 socket_->send_to(buffers, *multicast_endpoint_);
438 usleep(pause_on_copy_usecs_);
440 if (last_subfragment == num_subfragments - 1)
445 return CopyStatus::kSuccess;
448 #pragma GCC diagnostic push
449 #pragma GCC diagnostic ignored "-Wsign-compare"
451 void artdaq::MulticastTransfer::fill_staging_memory(
const artdaq::Fragment& fragment)
453 auto num_subfragments =
static_cast<size_t>(std::ceil(fragment.sizeBytes() /
static_cast<float>(subfragment_size_)));
454 TLOG(TLVL_DEBUG) << GetTraceName() <<
": # of subfragments to use is " << num_subfragments ;
456 for (
auto i_s = 0; i_s < num_subfragments; ++i_s)
458 auto staging_memory_copyto = &staging_memory_.at(i_s * (
sizeof(subfragment_identifier) + subfragment_size_));
460 subfragment_identifier sfi(fragment.sequenceID(), fragment.fragmentID(), i_s);
462 std::copy(reinterpret_cast<byte_t*>(&sfi),
463 reinterpret_cast<byte_t*>(&sfi) +
sizeof(subfragment_identifier),
464 staging_memory_copyto);
466 auto low_ptr_into_fragment = fragment.headerBeginBytes() + subfragment_size_ * i_s;
468 auto high_ptr_into_fragment = (i_s == num_subfragments - 1) ?
469 fragment.dataEndBytes() :
470 fragment.headerBeginBytes() + subfragment_size_ * (i_s + 1);
472 std::copy(low_ptr_into_fragment,
473 high_ptr_into_fragment,
474 staging_memory_copyto +
sizeof(subfragment_identifier));
478 #pragma GCC diagnostic pop
485 template <
typename T>
486 void artdaq::MulticastTransfer::book_container_of_buffers(std::vector<T>& buffers,
487 const size_t fragment_size,
488 const size_t total_subfragments,
489 const size_t first_subfragment_num,
490 const size_t last_subfragment_num)
492 assert(staging_memory_.size() >= total_subfragments * (
sizeof(subfragment_identifier) + subfragment_size_));
493 assert(buffers.size() == 0);
494 assert(last_subfragment_num < total_subfragments);
496 for (
auto i_f = first_subfragment_num; i_f <= last_subfragment_num; ++i_f)
498 auto bytes_to_store = (i_f == total_subfragments - 1) ?
499 sizeof(subfragment_identifier) + (fragment_size - (total_subfragments - 1) * subfragment_size_) :
500 sizeof(subfragment_identifier) + subfragment_size_;
502 buffers.emplace_back(&staging_memory_.at(i_f * (
sizeof(subfragment_identifier) + subfragment_size_)),
508 #pragma GCC diagnostic push // Needed since profile builds will ignore the assert
509 #pragma GCC diagnostic ignored "-Wunused-variable"
511 void artdaq::MulticastTransfer::get_fragment_quantities(
const boost::asio::mutable_buffer& buf,
size_t& payload_size,
512 size_t& fragment_size,
513 size_t& expected_subfragments)
515 byte_t* buffer_ptr = boost::asio::buffer_cast<byte_t*>(buf);
517 auto subfragment_num = *(
reinterpret_cast<size_t*
>(buffer_ptr) + 2);
519 assert(subfragment_num == 0);
521 artdaq::detail::RawFragmentHeader* header =
522 reinterpret_cast<artdaq::detail::RawFragmentHeader*
>(buffer_ptr +
sizeof(subfragment_identifier));
524 fragment_size = header->word_count *
sizeof(artdaq::RawDataType);
526 auto metadata_size = header->metadata_word_count *
sizeof(artdaq::RawDataType);
527 payload_size = fragment_size - metadata_size - artdaq::detail::RawFragmentHeader::num_words() *
528 sizeof(artdaq::RawDataType);
530 assert(fragment_size ==
531 artdaq::detail::RawFragmentHeader::num_words() *
sizeof(artdaq::RawDataType) +
535 expected_subfragments =
static_cast<size_t>(std::ceil(fragment_size / static_cast<float>(subfragment_size_)));
537 #pragma GCC diagnostic pop
539 void artdaq::MulticastTransfer::set_receive_buffer_size(
size_t recv_buff_size)
541 if (recv_buff_size == 0)
return;
542 boost::asio::socket_base::receive_buffer_size actual_recv_buff_size;
543 socket_->get_option(actual_recv_buff_size);
545 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Receive buffer size is currently " << actual_recv_buff_size.value() <<
546 " bytes, will try to change it to " << recv_buff_size ;
548 boost::asio::socket_base::receive_buffer_size recv_buff_option(recv_buff_size);
550 boost::system::error_code ec;
551 socket_->set_option(recv_buff_option, ec);
555 std::cerr <<
"boost::system::error_code with value " << ec <<
556 " was found in attempt to change receive buffer" << std::endl;
559 socket_->get_option(actual_recv_buff_size);
560 TLOG(TLVL_DEBUG) << GetTraceName() <<
": After attempted change, receive buffer size is now " << actual_recv_buff_size.value() ;
563 #pragma GCC diagnostic pop
int receiveFragment(artdaq::Fragment &fragment, size_t receiveTimeout) override
Receive a Fragment using Multicast.
Value to be returned upon receive timeout.
Role role() const
Get the TransferInterface::Role of this TransferInterface.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
std::string GetTraceName() const
Constructs a name suitable for TRACE messages.
CopyStatus copyFragment(artdaq::Fragment &fragment, size_t send_timeout_usec) override
Copy a Fragment to the destination. Multicast is always unreliable.
MulticastTransfer is a TransferInterface implementation plugin that transfers data using Multicast...
This TransferInterface is a Receiver.
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
This TransferInterface is a Sender.
virtual ~MulticastTransfer()=default
Default destructor.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
MulticastTransfer(fhicl::ParameterSet const &ps, Role role)
MulticastTransfer Constructor.
CopyStatus moveFragment(artdaq::Fragment &&fragment) override
Move a Fragment to the destination. Multicast is always unreliable.
This interface defines the functions used to transfer data between artdaq applications.
artdaq::Fragment::byte_t byte_t
Copy Fragment::byte_t into local scope.
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.
const size_t max_fragment_size_words_
The maximum size of the transferred Fragment objects, in artdaq::Fragment::RawDataType words...