1 #define TRACE_NAME "MulticastTransfer"
3 #include "artdaq/TransferPlugins/TransferInterface.hh"
5 #include "artdaq-core/Data/Fragment.hh"
6 #include "artdaq-core/Utilities/ExceptionHandler.hh"
8 #include "fhiclcpp/ParameterSet.h"
9 #include "cetlib_except/exception.h"
11 #include <boost/asio.hpp>
12 #include <boost/bind.hpp>
18 #include <type_traits>
21 #pragma GCC diagnostic push
22 #pragma GCC diagnostic ignored "-Wunused-parameter"
34 using byte_t = artdaq::Fragment::byte_t;
67 size_t receiveTimeout)
override;
103 bool isRunning()
override {
return socket_ !=
nullptr; }
106 void fill_staging_memory(
const artdaq::Fragment& frag);
108 template <
typename T>
109 void book_container_of_buffers(std::vector<T>& buffers,
110 const size_t fragment_size,
111 const size_t total_subfragments,
112 const size_t first_subfragment_num,
113 const size_t last_subfragment_num);
115 void get_fragment_quantities(
const boost::asio::mutable_buffer& buf,
size_t& payload_size,
size_t& fragment_size,
116 size_t& expected_subfragments);
118 void set_receive_buffer_size(
size_t recv_buff_size);
120 class subfragment_identifier
124 subfragment_identifier(
size_t sequenceID,
size_t fragmentID,
size_t subfragment_number) :
125 sequenceID_(sequenceID)
126 , fragmentID_(fragmentID)
127 , subfragment_number_(subfragment_number) { }
129 size_t sequenceID()
const {
return sequenceID_; }
130 size_t fragmentID()
const {
return fragmentID_; }
131 size_t subfragment_number()
const {
return subfragment_number_; }
136 size_t subfragment_number_;
139 std::unique_ptr<boost::asio::io_service> io_service_;
141 std::unique_ptr<boost::asio::ip::udp::endpoint> local_endpoint_;
142 std::unique_ptr<boost::asio::ip::udp::endpoint> multicast_endpoint_;
143 std::unique_ptr<boost::asio::ip::udp::endpoint> opposite_endpoint_;
145 std::unique_ptr<boost::asio::ip::udp::socket> socket_;
147 size_t subfragment_size_;
148 size_t subfragments_per_send_;
150 size_t pause_on_copy_usecs_;
151 Fragment fragment_buffer_;
153 std::vector<byte_t> staging_memory_;
155 std::vector<boost::asio::mutable_buffer> receive_buffers_;
161 , io_service_(std::make_unique<std::remove_reference<decltype(*io_service_)>::type>())
162 , local_endpoint_(nullptr)
163 , multicast_endpoint_(nullptr)
164 , opposite_endpoint_(std::make_unique<std::remove_reference<decltype(*opposite_endpoint_)>::type>())
166 , subfragment_size_(pset.get<size_t>(
"subfragment_size"))
167 , subfragments_per_send_(pset.get<size_t>(
"subfragments_per_send"))
168 , pause_on_copy_usecs_(pset.get<size_t>(
"pause_on_copy_usecs", 0))
172 auto port = pset.get<
unsigned short>(
"multicast_port");
173 auto multicast_address = boost::asio::ip::address::from_string(pset.get<std::string>(
"multicast_address"));
174 auto local_address = boost::asio::ip::address::from_string(pset.get<std::string>(
"local_address"));
176 TLOG(TLVL_DEBUG) <<
GetTraceName() <<
": multicast address is set to " << multicast_address ;
177 TLOG(TLVL_DEBUG) <<
GetTraceName() <<
": local address is set to " << local_address ;
181 local_endpoint_ = std::make_unique<std::remove_reference<decltype(*local_endpoint_)>::type>(local_address, 0);
182 multicast_endpoint_ = std::make_unique<std::remove_reference<decltype(*multicast_endpoint_)>::type>(multicast_address, port);
184 socket_ = std::make_unique<std::remove_reference<decltype(*socket_)>::type>(*io_service_,
185 multicast_endpoint_->protocol());
186 socket_->bind(*local_endpoint_);
193 local_endpoint_ = std::make_unique<std::remove_reference<decltype(*local_endpoint_)>::type>(local_address, port);
194 socket_ = std::make_unique<std::remove_reference<decltype(*socket_)>::type>(*io_service_,
195 local_endpoint_->protocol());
197 boost::system::error_code ec;
199 socket_->set_option(boost::asio::ip::udp::socket::reuse_address(
true), ec);
203 std::cerr <<
"boost::system::error_code with value " << ec <<
" was found in setting reuse_address option" << std::endl;
206 set_receive_buffer_size(pset.get<
size_t>(
"receive_buffer_size", 0));
208 socket_->bind(boost::asio::ip::udp::endpoint(multicast_address, port));
212 socket_->set_option(boost::asio::ip::multicast::join_group(multicast_address), ec);
216 std::cerr <<
"boost::system::error_code with value " << ec <<
" was found in attempt to join multicast group" << std::endl;
222 ExceptionHandler(ExceptionHandlerRethrow::yes,
"Problem setting up the socket in MulticastTransfer");
225 auto max_subfragments =
228 staging_memory_.resize(max_subfragments * (
sizeof(subfragment_identifier) + subfragment_size_));
235 TLOG(TLVL_DEBUG) <<
GetTraceName() <<
": max_subfragments is " << max_subfragments ;
236 TLOG(TLVL_DEBUG) <<
GetTraceName() <<
": Staging buffer size is " << staging_memory_.size() ;
239 #pragma GCC diagnostic push
240 #pragma GCC diagnostic ignored "-Wunused-variable"
243 size_t receiveTimeout)
247 if (fragment.dataSizeBytes() > 0)
249 throw cet::exception(
"MulticastTransfer") <<
"Error in MulticastTransfer::receiveFragmentFrom: " <<
250 "nonzero payload found in fragment passed as argument";
253 static bool print_warning =
true;
257 std::cerr <<
"Please note that MulticastTransfer::receiveFragmentFrom does not use its receiveTimeout argument" << std::endl;
258 print_warning =
false;
261 fragment.resizeBytes(max_fragment_size_words_ -
sizeof(artdaq::detail::RawFragmentHeader));
263 static auto current_sequenceID = std::numeric_limits<Fragment::sequence_id_t>::max();
264 static auto current_fragmentID = std::numeric_limits<Fragment::fragment_id_t>::max();
266 size_t fragment_size = 0;
267 size_t expected_subfragments = 0;
268 size_t current_subfragments = 0;
269 bool fragment_complete =
false;
270 bool last_fragment_truncated =
false;
274 auto bytes_received = socket_->receive_from(receive_buffers_, *opposite_endpoint_);
276 size_t bytes_processed = 0;
278 for (
auto& buf : receive_buffers_)
280 auto buf_size = boost::asio::buffer_size(buf);
281 auto size_t_ptr = boost::asio::buffer_cast<
const size_t*>(buf);
282 auto seqID = *size_t_ptr;
283 auto fragID = *(size_t_ptr + 1);
284 auto subfragID = *(size_t_ptr + 2);
286 if (seqID != current_sequenceID || fragID != current_fragmentID)
291 assert(bytes_processed == 0);
293 if (current_subfragments < expected_subfragments)
295 last_fragment_truncated =
true;
297 if (expected_subfragments != std::numeric_limits<size_t>::max())
299 std::cerr <<
"Warning: only received " << current_subfragments <<
" subfragments for fragment with seqID = " <<
300 current_sequenceID <<
", fragID = " << current_fragmentID <<
" (expected " << expected_subfragments <<
")\n"
305 std::cerr <<
"Warning: only received " << current_subfragments <<
306 " subfragments for fragment with seqID = " <<
307 current_sequenceID <<
", fragID = " << current_fragmentID <<
308 ", # of expected subfragments is unknown as fragment header was not received)\n"
313 current_subfragments = 0;
314 fragment_size = std::numeric_limits<size_t>::max();
315 expected_subfragments = std::numeric_limits<size_t>::max();
316 current_sequenceID = seqID;
317 current_fragmentID = fragID;
320 auto ptr_into_fragment = fragment.headerBeginBytes() + subfragID * subfragment_size_;
322 auto ptr_into_buffer = boost::asio::buffer_cast<
const byte_t*>(buf) +
sizeof(subfragment_identifier);
324 std::copy(ptr_into_buffer, ptr_into_buffer + buf_size -
sizeof(subfragment_identifier), ptr_into_fragment);
328 if (buf_size >=
sizeof(subfragment_identifier) +
sizeof(artdaq::detail::RawFragmentHeader))
330 auto payload_size = std::numeric_limits<size_t>::max();
331 get_fragment_quantities(buf, payload_size, fragment_size, expected_subfragments);
333 fragment.resizeBytes(payload_size);
337 throw cet::exception(
"MulticastTransfer") <<
"Buffer size is too small to completely contain an artdaq::Fragment header; " <<
338 "please increase the default size";
342 current_subfragments++;
344 if (current_subfragments == expected_subfragments)
346 fragment_complete =
true;
349 bytes_processed += buf_size;
351 if (bytes_processed >= bytes_received)
357 if (last_fragment_truncated)
366 assert(!fragment_complete);
367 TLOG(TLVL_WARNING) << GetTraceName() <<
": Got an incomplete fragment" ;
371 if (fragment_complete)
373 return source_rank();
380 #pragma GCC diagnostic pop
384 auto ret = receiveFragment(fragment_buffer_, receiveTimeout);
385 if (ret == source_rank())
387 header = *
reinterpret_cast<detail::RawFragmentHeader*
>(fragment_buffer_.headerAddress());
388 return source_rank();
395 if (fragment_buffer_.size() > detail::RawFragmentHeader::num_words()) {
396 auto dataSize = (fragment_buffer_.size() - detail::RawFragmentHeader::num_words()) *
sizeof(RawDataType);
397 memcpy(destination, fragment_buffer_.headerAddress() + detail::RawFragmentHeader::num_words(), dataSize);
398 return source_rank();
408 return copyFragment(f, 100000000);
413 size_t send_timeout_usec)
417 if (fragment.sizeBytes() > max_fragment_size_words_)
419 throw cet::exception(
"MulticastTransfer") <<
"Error in MulticastTransfer::copyFragmentTo: " <<
420 fragment.sizeBytes() <<
" byte fragment exceeds max_fragment_size of " << max_fragment_size_words_;
423 static size_t ncalls = 1;
424 auto num_subfragments =
static_cast<size_t>(std::ceil(fragment.sizeBytes() /
static_cast<float>(subfragment_size_)));
428 fill_staging_memory(fragment);
430 for (
size_t batch_index = 0; ; batch_index++)
432 auto first_subfragment = batch_index * subfragments_per_send_;
433 auto last_subfragment = (batch_index + 1) * subfragments_per_send_ >= num_subfragments ?
434 num_subfragments - 1 :
435 (batch_index + 1) * subfragments_per_send_ - 1;
437 std::vector<boost::asio::const_buffer> buffers;
439 book_container_of_buffers(buffers, fragment.sizeBytes(), num_subfragments, first_subfragment, last_subfragment);
441 socket_->send_to(buffers, *multicast_endpoint_);
443 usleep(pause_on_copy_usecs_);
445 if (last_subfragment == num_subfragments - 1)
450 return CopyStatus::kSuccess;
453 #pragma GCC diagnostic push
454 #pragma GCC diagnostic ignored "-Wsign-compare"
456 void artdaq::MulticastTransfer::fill_staging_memory(
const artdaq::Fragment& fragment)
458 auto num_subfragments =
static_cast<size_t>(std::ceil(fragment.sizeBytes() /
static_cast<float>(subfragment_size_)));
459 TLOG(TLVL_DEBUG) << GetTraceName() <<
": # of subfragments to use is " << num_subfragments ;
461 for (
auto i_s = 0; i_s < num_subfragments; ++i_s)
463 auto staging_memory_copyto = &staging_memory_.at(i_s * (
sizeof(subfragment_identifier) + subfragment_size_));
465 subfragment_identifier sfi(fragment.sequenceID(), fragment.fragmentID(), i_s);
467 std::copy(reinterpret_cast<byte_t*>(&sfi),
468 reinterpret_cast<byte_t*>(&sfi) +
sizeof(subfragment_identifier),
469 staging_memory_copyto);
471 auto low_ptr_into_fragment = fragment.headerBeginBytes() + subfragment_size_ * i_s;
473 auto high_ptr_into_fragment = (i_s == num_subfragments - 1) ?
474 fragment.dataEndBytes() :
475 fragment.headerBeginBytes() + subfragment_size_ * (i_s + 1);
477 std::copy(low_ptr_into_fragment,
478 high_ptr_into_fragment,
479 staging_memory_copyto +
sizeof(subfragment_identifier));
483 #pragma GCC diagnostic pop
490 template <
typename T>
491 void artdaq::MulticastTransfer::book_container_of_buffers(std::vector<T>& buffers,
492 const size_t fragment_size,
493 const size_t total_subfragments,
494 const size_t first_subfragment_num,
495 const size_t last_subfragment_num)
497 assert(staging_memory_.size() >= total_subfragments * (
sizeof(subfragment_identifier) + subfragment_size_));
498 assert(buffers.size() == 0);
499 assert(last_subfragment_num < total_subfragments);
501 for (
auto i_f = first_subfragment_num; i_f <= last_subfragment_num; ++i_f)
503 auto bytes_to_store = (i_f == total_subfragments - 1) ?
504 sizeof(subfragment_identifier) + (fragment_size - (total_subfragments - 1) * subfragment_size_) :
505 sizeof(subfragment_identifier) + subfragment_size_;
507 buffers.emplace_back(&staging_memory_.at(i_f * (
sizeof(subfragment_identifier) + subfragment_size_)),
513 #pragma GCC diagnostic push // Needed since profile builds will ignore the assert
514 #pragma GCC diagnostic ignored "-Wunused-variable"
516 void artdaq::MulticastTransfer::get_fragment_quantities(
const boost::asio::mutable_buffer& buf,
size_t& payload_size,
517 size_t& fragment_size,
518 size_t& expected_subfragments)
520 byte_t* buffer_ptr = boost::asio::buffer_cast<byte_t*>(buf);
522 auto subfragment_num = *(
reinterpret_cast<size_t*
>(buffer_ptr) + 2);
524 assert(subfragment_num == 0);
526 artdaq::detail::RawFragmentHeader* header =
527 reinterpret_cast<artdaq::detail::RawFragmentHeader*
>(buffer_ptr +
sizeof(subfragment_identifier));
529 fragment_size = header->word_count *
sizeof(artdaq::RawDataType);
531 auto metadata_size = header->metadata_word_count *
sizeof(artdaq::RawDataType);
532 payload_size = fragment_size - metadata_size - artdaq::detail::RawFragmentHeader::num_words() *
533 sizeof(artdaq::RawDataType);
535 assert(fragment_size ==
536 artdaq::detail::RawFragmentHeader::num_words() *
sizeof(artdaq::RawDataType) +
540 expected_subfragments =
static_cast<size_t>(std::ceil(fragment_size / static_cast<float>(subfragment_size_)));
542 #pragma GCC diagnostic pop
544 void artdaq::MulticastTransfer::set_receive_buffer_size(
size_t recv_buff_size)
546 if (recv_buff_size == 0)
return;
547 boost::asio::socket_base::receive_buffer_size actual_recv_buff_size;
548 socket_->get_option(actual_recv_buff_size);
550 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Receive buffer size is currently " << actual_recv_buff_size.value() <<
551 " bytes, will try to change it to " << recv_buff_size ;
553 boost::asio::socket_base::receive_buffer_size recv_buff_option(recv_buff_size);
555 boost::system::error_code ec;
556 socket_->set_option(recv_buff_option, ec);
560 std::cerr <<
"boost::system::error_code with value " << ec <<
561 " was found in attempt to change receive buffer" << std::endl;
564 socket_->get_option(actual_recv_buff_size);
565 TLOG(TLVL_DEBUG) << GetTraceName() <<
": After attempted change, receive buffer size is now " << actual_recv_buff_size.value() ;
568 #pragma GCC diagnostic pop
int receiveFragment(artdaq::Fragment &fragment, size_t receiveTimeout) override
Receive a Fragment using Multicast.
Value to be returned upon receive timeout.
Role role() const
Get the TransferInterface::Role of this TransferInterface.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
std::string GetTraceName() const
Constructs a name suitable for TRACE messages.
CopyStatus copyFragment(artdaq::Fragment &fragment, size_t send_timeout_usec) override
Copy a Fragment to the destination. Multicast is always unreliable.
MulticastTransfer is a TransferInterface implementation plugin that transfers data using Multicast...
This TransferInterface is a Receiver.
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
This TransferInterface is a Sender.
virtual ~MulticastTransfer()=default
Default destructor.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
MulticastTransfer(fhicl::ParameterSet const &ps, Role role)
MulticastTransfer Constructor.
CopyStatus moveFragment(artdaq::Fragment &&fragment) override
Move a Fragment to the destination. Multicast is always unreliable.
This interface defines the functions used to transfer data between artdaq applications.
artdaq::Fragment::byte_t byte_t
Copy Fragment::byte_t into local scope.
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.
const size_t max_fragment_size_words_
The maximum size of the transferred Fragment objects, in artdaq::Fragment::RawDataType words...