1 #include "artdaq/TransferPlugins/TransferInterface.hh"
3 #include "artdaq-core/Data/Fragment.hh"
4 #include "artdaq-core/Utilities/ExceptionHandler.hh"
6 #include "fhiclcpp/ParameterSet.h"
8 #include <boost/asio.hpp>
9 #include <boost/bind.hpp>
15 #include <type_traits>
18 #pragma GCC diagnostic push
19 #pragma GCC diagnostic ignored "-Wunused-parameter"
31 using byte_t = artdaq::Fragment::byte_t;
64 size_t receiveTimeout)
override;
73 size_t send_timeout_usec = std::numeric_limits<size_t>::max())
override;
82 size_t send_timeout_usec = std::numeric_limits<size_t>::max())
override;
86 void fill_staging_memory(
const artdaq::Fragment& frag);
89 void book_container_of_buffers(std::vector<T>& buffers,
90 const size_t fragment_size,
91 const size_t total_subfragments,
92 const size_t first_subfragment_num,
93 const size_t last_subfragment_num);
95 void get_fragment_quantities(
const boost::asio::mutable_buffer& buf,
size_t& payload_size,
size_t& fragment_size,
96 size_t& expected_subfragments);
98 void set_receive_buffer_size(
size_t recv_buff_size);
100 class subfragment_identifier
104 subfragment_identifier(
size_t sequenceID,
size_t fragmentID,
size_t subfragment_number) :
105 sequenceID_(sequenceID)
106 , fragmentID_(fragmentID)
107 , subfragment_number_(subfragment_number) { }
109 size_t sequenceID()
const {
return sequenceID_; }
110 size_t fragmentID()
const {
return fragmentID_; }
111 size_t subfragment_number()
const {
return subfragment_number_; }
116 size_t subfragment_number_;
119 std::unique_ptr<boost::asio::io_service> io_service_;
121 std::unique_ptr<boost::asio::ip::udp::endpoint> local_endpoint_;
122 std::unique_ptr<boost::asio::ip::udp::endpoint> multicast_endpoint_;
123 std::unique_ptr<boost::asio::ip::udp::endpoint> opposite_endpoint_;
125 std::unique_ptr<boost::asio::ip::udp::socket> socket_;
127 size_t subfragment_size_;
128 size_t subfragments_per_send_;
130 size_t pause_on_copy_usecs_;
132 std::vector<byte_t> staging_memory_;
134 std::vector<boost::asio::mutable_buffer> receive_buffers_;
140 , io_service_(std::make_unique<std::remove_reference<decltype(*io_service_)>::type>())
141 , local_endpoint_(nullptr)
142 , multicast_endpoint_(nullptr)
143 , opposite_endpoint_(std::make_unique<std::remove_reference<decltype(*opposite_endpoint_)>::type>())
145 , subfragment_size_(pset.get<size_t>(
"subfragment_size"))
146 , subfragments_per_send_(pset.get<size_t>(
"subfragments_per_send"))
147 , pause_on_copy_usecs_(pset.get<size_t>(
"pause_on_copy_usecs", 0))
151 auto port = pset.get<
unsigned short>(
"multicast_port");
152 auto multicast_address = boost::asio::ip::address::from_string(pset.get<std::string>(
"multicast_address"));
153 auto local_address = boost::asio::ip::address::from_string(pset.get<std::string>(
"local_address"));
155 TLOG_DEBUG(
uniqueLabel()) <<
"multicast address is set to " << multicast_address << TLOG_ENDL;
156 TLOG_DEBUG(
uniqueLabel()) <<
"local address is set to " << local_address << TLOG_ENDL;
160 local_endpoint_ = std::make_unique<std::remove_reference<decltype(*local_endpoint_)>::type>(local_address, 0);
161 multicast_endpoint_ = std::make_unique<std::remove_reference<decltype(*multicast_endpoint_)>::type>(multicast_address, port);
163 socket_ = std::make_unique<std::remove_reference<decltype(*socket_)>::type>(*io_service_,
164 multicast_endpoint_->protocol());
165 socket_->bind(*local_endpoint_);
172 local_endpoint_ = std::make_unique<std::remove_reference<decltype(*local_endpoint_)>::type>(local_address, port);
173 socket_ = std::make_unique<std::remove_reference<decltype(*socket_)>::type>(*io_service_,
174 local_endpoint_->protocol());
176 boost::system::error_code ec;
178 socket_->set_option(boost::asio::ip::udp::socket::reuse_address(
true), ec);
182 std::cerr <<
"boost::system::error_code with value " << ec <<
" was found in setting reuse_address option" << std::endl;
185 set_receive_buffer_size(pset.get<
size_t>(
"receive_buffer_size", 0));
187 socket_->bind(boost::asio::ip::udp::endpoint(multicast_address, port));
191 socket_->set_option(boost::asio::ip::multicast::join_group(multicast_address), ec);
195 std::cerr <<
"boost::system::error_code with value " << ec <<
" was found in attempt to join multicast group" << std::endl;
201 ExceptionHandler(ExceptionHandlerRethrow::yes,
"Problem setting up the socket in MulticastTransfer");
204 auto max_subfragments =
207 staging_memory_.resize(max_subfragments * (
sizeof(subfragment_identifier) + subfragment_size_));
214 TLOG_DEBUG(
uniqueLabel()) <<
"max_subfragments is " << max_subfragments << TLOG_ENDL;
215 TLOG_DEBUG(
uniqueLabel()) <<
"Staging buffer size is " << staging_memory_.size() << TLOG_ENDL;
218 #pragma GCC diagnostic push
219 #pragma GCC diagnostic ignored "-Wunused-variable"
222 size_t receiveTimeout)
226 if (fragment.dataSizeBytes() > 0)
228 throw cet::exception(
"MulticastTransfer") <<
"Error in MulticastTransfer::receiveFragmentFrom: " <<
229 "nonzero payload found in fragment passed as argument";
232 static bool print_warning =
true;
236 std::cerr <<
"Please note that MulticastTransfer::receiveFragmentFrom does not use its receiveTimeout argument" << std::endl;
237 print_warning =
false;
240 fragment.resizeBytes(max_fragment_size_words_ -
sizeof(artdaq::detail::RawFragmentHeader));
242 static auto current_sequenceID = std::numeric_limits<Fragment::sequence_id_t>::max();
243 static auto current_fragmentID = std::numeric_limits<Fragment::fragment_id_t>::max();
245 size_t fragment_size = 0;
246 size_t expected_subfragments = 0;
247 size_t current_subfragments = 0;
248 bool fragment_complete =
false;
249 bool last_fragment_truncated =
false;
253 auto bytes_received = socket_->receive_from(receive_buffers_, *opposite_endpoint_);
255 size_t bytes_processed = 0;
257 for (
auto& buf : receive_buffers_)
259 auto buf_size = boost::asio::buffer_size(buf);
260 auto size_t_ptr = boost::asio::buffer_cast<
const size_t*>(buf);
261 auto seqID = *size_t_ptr;
262 auto fragID = *(size_t_ptr + 1);
263 auto subfragID = *(size_t_ptr + 2);
265 if (seqID != current_sequenceID || fragID != current_fragmentID)
270 assert(bytes_processed == 0);
272 if (current_subfragments < expected_subfragments)
274 last_fragment_truncated =
true;
276 if (expected_subfragments != std::numeric_limits<size_t>::max())
278 std::cerr <<
"Warning: only received " << current_subfragments <<
" subfragments for fragment with seqID = " <<
279 current_sequenceID <<
", fragID = " << current_fragmentID <<
" (expected " << expected_subfragments <<
")\n"
284 std::cerr <<
"Warning: only received " << current_subfragments <<
285 " subfragments for fragment with seqID = " <<
286 current_sequenceID <<
", fragID = " << current_fragmentID <<
287 ", # of expected subfragments is unknown as fragment header was not received)\n"
292 current_subfragments = 0;
293 fragment_size = std::numeric_limits<size_t>::max();
294 expected_subfragments = std::numeric_limits<size_t>::max();
295 current_sequenceID = seqID;
296 current_fragmentID = fragID;
299 auto ptr_into_fragment = fragment.headerBeginBytes() + subfragID * subfragment_size_;
301 auto ptr_into_buffer = boost::asio::buffer_cast<
const byte_t*>(buf) +
sizeof(subfragment_identifier);
303 std::copy(ptr_into_buffer, ptr_into_buffer + buf_size -
sizeof(subfragment_identifier), ptr_into_fragment);
307 if (buf_size >=
sizeof(subfragment_identifier) +
sizeof(artdaq::detail::RawFragmentHeader))
309 auto payload_size = std::numeric_limits<size_t>::max();
310 get_fragment_quantities(buf, payload_size, fragment_size, expected_subfragments);
312 fragment.resizeBytes(payload_size);
316 throw cet::exception(
"MulticastTransfer") <<
"Buffer size is too small to completely contain an artdaq::Fragment header; " <<
317 "please increase the default size";
321 current_subfragments++;
323 if (current_subfragments == expected_subfragments)
325 fragment_complete =
true;
328 bytes_processed += buf_size;
330 if (bytes_processed >= bytes_received)
336 if (last_fragment_truncated)
345 assert( !fragment_complete );
346 TLOG_WARNING(uniqueLabel()) <<
"Got an incomplete fragment" << TLOG_ENDL;
350 if (fragment_complete)
352 return source_rank();
359 #pragma GCC diagnostic pop
365 return copyFragment(f, tmo);
370 size_t send_timeout_usec)
374 if (fragment.sizeBytes() > max_fragment_size_words_)
376 throw cet::exception(
"MulticastTransfer") <<
"Error in MulticastTransfer::copyFragmentTo: " <<
377 fragment.sizeBytes() <<
" byte fragment exceeds max_fragment_size of " << max_fragment_size_words_;
380 static size_t ncalls = 1;
381 auto num_subfragments =
static_cast<size_t>(std::ceil(fragment.sizeBytes() /
static_cast<float>(subfragment_size_)));
385 fill_staging_memory(fragment);
387 for (
size_t batch_index = 0; ; batch_index++)
389 auto first_subfragment = batch_index * subfragments_per_send_;
390 auto last_subfragment = (batch_index + 1) * subfragments_per_send_ >= num_subfragments ?
391 num_subfragments - 1 :
392 (batch_index + 1) * subfragments_per_send_ - 1;
394 std::vector<boost::asio::const_buffer> buffers;
396 book_container_of_buffers(buffers, fragment.sizeBytes(), num_subfragments, first_subfragment, last_subfragment);
398 socket_->send_to(buffers, *multicast_endpoint_);
400 usleep(pause_on_copy_usecs_);
402 if (last_subfragment == num_subfragments - 1)
407 return CopyStatus::kSuccess;
410 #pragma GCC diagnostic push
411 #pragma GCC diagnostic ignored "-Wsign-compare"
413 void artdaq::MulticastTransfer::fill_staging_memory(
const artdaq::Fragment& fragment)
415 auto num_subfragments =
static_cast<size_t>(std::ceil(fragment.sizeBytes() /
static_cast<float>(subfragment_size_)));
416 TLOG_DEBUG(uniqueLabel()) <<
"# of subfragments to use is " << num_subfragments << TLOG_ENDL;
418 for (
auto i_s = 0; i_s < num_subfragments; ++i_s)
420 auto staging_memory_copyto = &staging_memory_.at(i_s * (
sizeof(subfragment_identifier) + subfragment_size_));
422 subfragment_identifier sfi(fragment.sequenceID(), fragment.fragmentID(), i_s);
424 std::copy(reinterpret_cast<byte_t*>(&sfi),
425 reinterpret_cast<byte_t*>(&sfi) +
sizeof(subfragment_identifier),
426 staging_memory_copyto);
428 auto low_ptr_into_fragment = fragment.headerBeginBytes() + subfragment_size_ * i_s;
430 auto high_ptr_into_fragment = (i_s == num_subfragments - 1) ?
431 fragment.dataEndBytes() :
432 fragment.headerBeginBytes() + subfragment_size_ * (i_s + 1);
434 std::copy(low_ptr_into_fragment,
435 high_ptr_into_fragment,
436 staging_memory_copyto +
sizeof(subfragment_identifier));
440 #pragma GCC diagnostic pop
447 template <
typename T>
448 void artdaq::MulticastTransfer::book_container_of_buffers(std::vector<T>& buffers,
449 const size_t fragment_size,
450 const size_t total_subfragments,
451 const size_t first_subfragment_num,
452 const size_t last_subfragment_num)
454 assert(staging_memory_.size() >= total_subfragments * (
sizeof(subfragment_identifier) + subfragment_size_) );
455 assert(buffers.size() == 0);
456 assert(last_subfragment_num < total_subfragments);
458 for (
auto i_f = first_subfragment_num; i_f <= last_subfragment_num; ++i_f)
460 auto bytes_to_store = (i_f == total_subfragments - 1) ?
461 sizeof(subfragment_identifier) + (fragment_size - (total_subfragments - 1) * subfragment_size_) :
462 sizeof(subfragment_identifier) + subfragment_size_;
464 buffers.emplace_back(&staging_memory_.at(i_f * (
sizeof(subfragment_identifier) + subfragment_size_)),
470 #pragma GCC diagnostic push // Needed since profile builds will ignore the assert
471 #pragma GCC diagnostic ignored "-Wunused-variable"
473 void artdaq::MulticastTransfer::get_fragment_quantities(
const boost::asio::mutable_buffer& buf,
size_t& payload_size,
474 size_t& fragment_size,
475 size_t& expected_subfragments)
477 byte_t* buffer_ptr = boost::asio::buffer_cast<byte_t*>(buf);
479 auto subfragment_num = *(
reinterpret_cast<size_t*
>(buffer_ptr) + 2);
481 assert( subfragment_num == 0 );
483 artdaq::detail::RawFragmentHeader* header =
484 reinterpret_cast<artdaq::detail::RawFragmentHeader*
>(buffer_ptr +
sizeof(subfragment_identifier));
486 fragment_size = header->word_count *
sizeof(artdaq::RawDataType);
488 auto metadata_size = header->metadata_word_count *
sizeof(artdaq::RawDataType);
489 payload_size = fragment_size - metadata_size - artdaq::detail::RawFragmentHeader::num_words() *
490 sizeof(artdaq::RawDataType);
492 assert(fragment_size ==
493 artdaq::detail::RawFragmentHeader::num_words() *
sizeof(artdaq::RawDataType) +
497 expected_subfragments =
static_cast<size_t>(std::ceil(fragment_size / static_cast<float>(subfragment_size_)));
499 #pragma GCC diagnostic pop
501 void artdaq::MulticastTransfer::set_receive_buffer_size(
size_t recv_buff_size)
503 if (recv_buff_size == 0)
return;
504 boost::asio::socket_base::receive_buffer_size actual_recv_buff_size;
505 socket_->get_option(actual_recv_buff_size);
507 TLOG_DEBUG(uniqueLabel()) <<
"Receive buffer size is currently " << actual_recv_buff_size.value() <<
508 " bytes, will try to change it to " << recv_buff_size << TLOG_ENDL;
510 boost::asio::socket_base::receive_buffer_size recv_buff_option(recv_buff_size);
512 boost::system::error_code ec;
513 socket_->set_option(recv_buff_option, ec);
517 std::cerr <<
"boost::system::error_code with value " << ec <<
518 " was found in attempt to change receive buffer" << std::endl;
521 socket_->get_option(actual_recv_buff_size);
522 TLOG_DEBUG(uniqueLabel()) <<
"After attempted change, receive buffer size is now " << actual_recv_buff_size.value() << TLOG_ENDL;
526 #pragma GCC diagnostic pop
int receiveFragment(artdaq::Fragment &fragment, size_t receiveTimeout) override
Receive a Fragment using Multicast.
Role role() const
Get the TransferInterface::Role of this TransferInterface.
MulticastTransfer is a TransferInterface implementation plugin that transfers data using Multicast...
static const int RECV_TIMEOUT
Value to be returned upon receive timeout. Because receivers otherwise return rank, this is also the limit on the number of ranks that artdaq currently supports.
This TransferInterface is a Receiver.
CopyStatus copyFragment(artdaq::Fragment &fragment, size_t send_timeout_usec=std::numeric_limits< size_t >::max()) override
Copy a Fragment to the destination. Multicast is always unreliable.
This TransferInterface is a Sender.
virtual ~MulticastTransfer()=default
Default destructor.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
MulticastTransfer(fhicl::ParameterSet const &ps, Role role)
MulticastTransfer Constructor.
std::string uniqueLabel() const
Get the unique label of this TransferInterface instance.
This interface defines the functions used to transfer data between artdaq applications.
artdaq::Fragment::byte_t byte_t
Copy Fragment::byte_t into local scope.
CopyStatus moveFragment(artdaq::Fragment &&fragment, size_t send_timeout_usec=std::numeric_limits< size_t >::max()) override
Move a Fragment to the destination. Multicast is always unreliable.
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.
const size_t max_fragment_size_words_
The maximum size of the transferred Fragment objects, in artdaq::Fragment::RawDataType words...