1 #include "artdaq/DAQdata/Globals.hh"
2 #define TRACE_NAME (app_name + "_MulticastTransfer").c_str()
4 #include "artdaq/TransferPlugins/TransferInterface.hh"
6 #include "artdaq-core/Data/Fragment.hh"
7 #include "artdaq-core/Utilities/ExceptionHandler.hh"
9 #include "fhiclcpp/ParameterSet.h"
10 #include "cetlib_except/exception.h"
12 #include <boost/asio.hpp>
13 #include <boost/bind.hpp>
19 #include <type_traits>
22 #pragma GCC diagnostic push
23 #pragma GCC diagnostic ignored "-Wunused-parameter"
35 using byte_t = artdaq::Fragment::byte_t;
68 size_t receiveTimeout)
override;
105 bool isRunning()
override {
return socket_ !=
nullptr; }
114 void fill_staging_memory(
const artdaq::Fragment& frag);
116 template <
typename T>
117 void book_container_of_buffers(std::vector<T>& buffers,
118 const size_t fragment_size,
119 const size_t total_subfragments,
120 const size_t first_subfragment_num,
121 const size_t last_subfragment_num);
123 void get_fragment_quantities(
const boost::asio::mutable_buffer& buf,
size_t& payload_size,
size_t& fragment_size,
124 size_t& expected_subfragments);
126 void set_receive_buffer_size(
size_t recv_buff_size);
128 class subfragment_identifier
132 subfragment_identifier(
size_t sequenceID,
size_t fragmentID,
size_t subfragment_number) :
133 sequenceID_(sequenceID)
134 , fragmentID_(fragmentID)
135 , subfragment_number_(subfragment_number) { }
137 size_t sequenceID()
const {
return sequenceID_; }
138 size_t fragmentID()
const {
return fragmentID_; }
139 size_t subfragment_number()
const {
return subfragment_number_; }
144 size_t subfragment_number_;
147 std::unique_ptr<boost::asio::io_service> io_service_;
149 std::unique_ptr<boost::asio::ip::udp::endpoint> local_endpoint_;
150 std::unique_ptr<boost::asio::ip::udp::endpoint> multicast_endpoint_;
151 std::unique_ptr<boost::asio::ip::udp::endpoint> opposite_endpoint_;
153 std::unique_ptr<boost::asio::ip::udp::socket> socket_;
155 size_t subfragment_size_;
156 size_t subfragments_per_send_;
158 size_t pause_on_copy_usecs_;
159 Fragment fragment_buffer_;
161 std::vector<byte_t> staging_memory_;
163 std::vector<boost::asio::mutable_buffer> receive_buffers_;
169 , io_service_(std::make_unique<std::remove_reference<decltype(*io_service_)>::type>())
170 , local_endpoint_(nullptr)
171 , multicast_endpoint_(nullptr)
172 , opposite_endpoint_(std::make_unique<std::remove_reference<decltype(*opposite_endpoint_)>::type>())
174 , subfragment_size_(pset.get<size_t>(
"subfragment_size"))
175 , subfragments_per_send_(pset.get<size_t>(
"subfragments_per_send"))
176 , pause_on_copy_usecs_(pset.get<size_t>(
"pause_on_copy_usecs", 0))
180 portMan->UpdateConfiguration(pset);
181 auto port = portMan->GetMulticastTransferPort(
source_rank());
182 auto multicast_address = boost::asio::ip::address::from_string(portMan->GetMulticastTransferGroupAddress());
183 auto local_address = boost::asio::ip::address::from_string(pset.get<std::string>(
"local_address"));
185 TLOG(TLVL_DEBUG) << GetTraceName() <<
": multicast address is set to " << multicast_address ;
186 TLOG(TLVL_DEBUG) << GetTraceName() <<
": local address is set to " << local_address ;
190 local_endpoint_ = std::make_unique<std::remove_reference<decltype(*local_endpoint_)>::type>(local_address, 0);
191 multicast_endpoint_ = std::make_unique<std::remove_reference<decltype(*multicast_endpoint_)>::type>(multicast_address, port);
193 socket_ = std::make_unique<std::remove_reference<decltype(*socket_)>::type>(*io_service_,
194 multicast_endpoint_->protocol());
195 socket_->bind(*local_endpoint_);
202 local_endpoint_ = std::make_unique<std::remove_reference<decltype(*local_endpoint_)>::type>(local_address, port);
203 socket_ = std::make_unique<std::remove_reference<decltype(*socket_)>::type>(*io_service_,
204 local_endpoint_->protocol());
206 boost::system::error_code ec;
208 socket_->set_option(boost::asio::ip::udp::socket::reuse_address(
true), ec);
212 std::cerr <<
"boost::system::error_code with value " << ec <<
" was found in setting reuse_address option" << std::endl;
215 set_receive_buffer_size(pset.get<
size_t>(
"receive_buffer_size", 0));
217 socket_->bind(boost::asio::ip::udp::endpoint(multicast_address, port));
221 socket_->set_option(boost::asio::ip::multicast::join_group(multicast_address), ec);
225 std::cerr <<
"boost::system::error_code with value " << ec <<
" was found in attempt to join multicast group" << std::endl;
231 ExceptionHandler(ExceptionHandlerRethrow::yes,
"Problem setting up the socket in MulticastTransfer");
234 auto max_subfragments =
237 staging_memory_.resize(max_subfragments * (
sizeof(subfragment_identifier) + subfragment_size_));
244 TLOG(TLVL_DEBUG) << GetTraceName() <<
": max_subfragments is " << max_subfragments ;
245 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Staging buffer size is " << staging_memory_.size() ;
248 #pragma GCC diagnostic push
249 #pragma GCC diagnostic ignored "-Wunused-variable"
252 size_t receiveTimeout)
256 if (fragment.dataSizeBytes() > 0)
258 throw cet::exception(
"MulticastTransfer") <<
"Error in MulticastTransfer::receiveFragmentFrom: " <<
259 "nonzero payload found in fragment passed as argument";
262 static bool print_warning =
true;
266 std::cerr <<
"Please note that MulticastTransfer::receiveFragmentFrom does not use its receiveTimeout argument" << std::endl;
267 print_warning =
false;
270 fragment.resizeBytes(max_fragment_size_words_ -
sizeof(artdaq::detail::RawFragmentHeader));
272 static auto current_sequenceID = std::numeric_limits<Fragment::sequence_id_t>::max();
273 static auto current_fragmentID = std::numeric_limits<Fragment::fragment_id_t>::max();
275 size_t fragment_size = 0;
276 size_t expected_subfragments = 0;
277 size_t current_subfragments = 0;
278 bool fragment_complete =
false;
279 bool last_fragment_truncated =
false;
283 auto bytes_received = socket_->receive_from(receive_buffers_, *opposite_endpoint_);
285 size_t bytes_processed = 0;
287 for (
auto& buf : receive_buffers_)
289 auto buf_size = boost::asio::buffer_size(buf);
290 auto size_t_ptr = boost::asio::buffer_cast<
const size_t*>(buf);
291 auto seqID = *size_t_ptr;
292 auto fragID = *(size_t_ptr + 1);
293 auto subfragID = *(size_t_ptr + 2);
295 if (seqID != current_sequenceID || fragID != current_fragmentID)
300 assert(bytes_processed == 0);
302 if (current_subfragments < expected_subfragments)
304 last_fragment_truncated =
true;
306 if (expected_subfragments != std::numeric_limits<size_t>::max())
308 std::cerr <<
"Warning: only received " << current_subfragments <<
" subfragments for fragment with seqID = " <<
309 current_sequenceID <<
", fragID = " << current_fragmentID <<
" (expected " << expected_subfragments <<
")\n"
314 std::cerr <<
"Warning: only received " << current_subfragments <<
315 " subfragments for fragment with seqID = " <<
316 current_sequenceID <<
", fragID = " << current_fragmentID <<
317 ", # of expected subfragments is unknown as fragment header was not received)\n"
322 current_subfragments = 0;
323 fragment_size = std::numeric_limits<size_t>::max();
324 expected_subfragments = std::numeric_limits<size_t>::max();
325 current_sequenceID = seqID;
326 current_fragmentID = fragID;
329 auto ptr_into_fragment = fragment.headerBeginBytes() + subfragID * subfragment_size_;
331 auto ptr_into_buffer = boost::asio::buffer_cast<
const byte_t*>(buf) +
sizeof(subfragment_identifier);
333 std::copy(ptr_into_buffer, ptr_into_buffer + buf_size -
sizeof(subfragment_identifier), ptr_into_fragment);
337 if (buf_size >=
sizeof(subfragment_identifier) +
sizeof(artdaq::detail::RawFragmentHeader))
339 auto payload_size = std::numeric_limits<size_t>::max();
340 get_fragment_quantities(buf, payload_size, fragment_size, expected_subfragments);
342 fragment.resizeBytes(payload_size);
346 throw cet::exception(
"MulticastTransfer") <<
"Buffer size is too small to completely contain an artdaq::Fragment header; " <<
347 "please increase the default size";
351 current_subfragments++;
353 if (current_subfragments == expected_subfragments)
355 fragment_complete =
true;
358 bytes_processed += buf_size;
360 if (bytes_processed >= bytes_received)
366 if (last_fragment_truncated)
375 assert(!fragment_complete);
376 TLOG(TLVL_WARNING) << GetTraceName() <<
": Got an incomplete fragment" ;
380 if (fragment_complete)
382 return source_rank();
389 #pragma GCC diagnostic pop
393 auto ret = receiveFragment(fragment_buffer_, receiveTimeout);
394 if (ret == source_rank())
396 header = *
reinterpret_cast<detail::RawFragmentHeader*
>(fragment_buffer_.headerAddress());
397 return source_rank();
404 if (fragment_buffer_.size() > detail::RawFragmentHeader::num_words()) {
405 auto dataSize = (fragment_buffer_.size() - detail::RawFragmentHeader::num_words()) *
sizeof(RawDataType);
406 memcpy(destination, fragment_buffer_.headerAddress() + detail::RawFragmentHeader::num_words(), dataSize);
407 return source_rank();
417 return transfer_fragment_min_blocking_mode(f, 100000000);
422 size_t send_timeout_usec)
426 if (fragment.sizeBytes() > max_fragment_size_words_)
428 throw cet::exception(
"MulticastTransfer") <<
"Error in MulticastTransfer::copyFragmentTo: " <<
429 fragment.sizeBytes() <<
" byte fragment exceeds max_fragment_size of " << max_fragment_size_words_;
432 static size_t ncalls = 1;
433 auto num_subfragments =
static_cast<size_t>(std::ceil(fragment.sizeBytes() /
static_cast<float>(subfragment_size_)));
437 fill_staging_memory(fragment);
439 for (
size_t batch_index = 0; ; batch_index++)
441 auto first_subfragment = batch_index * subfragments_per_send_;
442 auto last_subfragment = (batch_index + 1) * subfragments_per_send_ >= num_subfragments ?
443 num_subfragments - 1 :
444 (batch_index + 1) * subfragments_per_send_ - 1;
446 std::vector<boost::asio::const_buffer> buffers;
448 book_container_of_buffers(buffers, fragment.sizeBytes(), num_subfragments, first_subfragment, last_subfragment);
450 socket_->send_to(buffers, *multicast_endpoint_);
452 usleep(pause_on_copy_usecs_);
454 if (last_subfragment == num_subfragments - 1)
459 return CopyStatus::kSuccess;
462 #pragma GCC diagnostic push
463 #pragma GCC diagnostic ignored "-Wsign-compare"
465 void artdaq::MulticastTransfer::fill_staging_memory(
const artdaq::Fragment& fragment)
467 auto num_subfragments =
static_cast<size_t>(std::ceil(fragment.sizeBytes() /
static_cast<float>(subfragment_size_)));
468 TLOG(TLVL_DEBUG) << GetTraceName() <<
": # of subfragments to use is " << num_subfragments ;
470 for (
auto i_s = 0; i_s < num_subfragments; ++i_s)
472 auto staging_memory_copyto = &staging_memory_.at(i_s * (
sizeof(subfragment_identifier) + subfragment_size_));
474 subfragment_identifier sfi(fragment.sequenceID(), fragment.fragmentID(), i_s);
476 std::copy(reinterpret_cast<byte_t*>(&sfi),
477 reinterpret_cast<byte_t*>(&sfi) +
sizeof(subfragment_identifier),
478 staging_memory_copyto);
480 auto low_ptr_into_fragment = fragment.headerBeginBytes() + subfragment_size_ * i_s;
482 auto high_ptr_into_fragment = (i_s == num_subfragments - 1) ?
483 fragment.dataEndBytes() :
484 fragment.headerBeginBytes() + subfragment_size_ * (i_s + 1);
486 std::copy(low_ptr_into_fragment,
487 high_ptr_into_fragment,
488 staging_memory_copyto +
sizeof(subfragment_identifier));
492 #pragma GCC diagnostic pop
499 template <
typename T>
500 void artdaq::MulticastTransfer::book_container_of_buffers(std::vector<T>& buffers,
501 const size_t fragment_size,
502 const size_t total_subfragments,
503 const size_t first_subfragment_num,
504 const size_t last_subfragment_num)
506 assert(staging_memory_.size() >= total_subfragments * (
sizeof(subfragment_identifier) + subfragment_size_));
507 assert(buffers.size() == 0);
508 assert(last_subfragment_num < total_subfragments);
510 for (
auto i_f = first_subfragment_num; i_f <= last_subfragment_num; ++i_f)
512 auto bytes_to_store = (i_f == total_subfragments - 1) ?
513 sizeof(subfragment_identifier) + (fragment_size - (total_subfragments - 1) * subfragment_size_) :
514 sizeof(subfragment_identifier) + subfragment_size_;
516 buffers.emplace_back(&staging_memory_.at(i_f * (
sizeof(subfragment_identifier) + subfragment_size_)),
522 #pragma GCC diagnostic push // Needed since profile builds will ignore the assert
523 #pragma GCC diagnostic ignored "-Wunused-variable"
525 void artdaq::MulticastTransfer::get_fragment_quantities(
const boost::asio::mutable_buffer& buf,
size_t& payload_size,
526 size_t& fragment_size,
527 size_t& expected_subfragments)
529 byte_t* buffer_ptr = boost::asio::buffer_cast<byte_t*>(buf);
531 auto subfragment_num = *(
reinterpret_cast<size_t*
>(buffer_ptr) + 2);
533 assert(subfragment_num == 0);
535 artdaq::detail::RawFragmentHeader* header =
536 reinterpret_cast<artdaq::detail::RawFragmentHeader*
>(buffer_ptr +
sizeof(subfragment_identifier));
538 fragment_size = header->word_count *
sizeof(artdaq::RawDataType);
540 auto metadata_size = header->metadata_word_count *
sizeof(artdaq::RawDataType);
541 payload_size = fragment_size - metadata_size - artdaq::detail::RawFragmentHeader::num_words() *
542 sizeof(artdaq::RawDataType);
544 assert(fragment_size ==
545 artdaq::detail::RawFragmentHeader::num_words() *
sizeof(artdaq::RawDataType) +
549 expected_subfragments =
static_cast<size_t>(std::ceil(fragment_size / static_cast<float>(subfragment_size_)));
551 #pragma GCC diagnostic pop
553 void artdaq::MulticastTransfer::set_receive_buffer_size(
size_t recv_buff_size)
555 if (recv_buff_size == 0)
return;
556 boost::asio::socket_base::receive_buffer_size actual_recv_buff_size;
557 socket_->get_option(actual_recv_buff_size);
559 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Receive buffer size is currently " << actual_recv_buff_size.value() <<
560 " bytes, will try to change it to " << recv_buff_size ;
562 boost::asio::socket_base::receive_buffer_size recv_buff_option(recv_buff_size);
564 boost::system::error_code ec;
565 socket_->set_option(recv_buff_option, ec);
569 std::cerr <<
"boost::system::error_code with value " << ec <<
570 " was found in attempt to change receive buffer" << std::endl;
573 socket_->get_option(actual_recv_buff_size);
574 TLOG(TLVL_DEBUG) << GetTraceName() <<
": After attempted change, receive buffer size is now " << actual_recv_buff_size.value() ;
577 #pragma GCC diagnostic pop
virtual int source_rank() const
Get the source rank for this TransferInterface instance.
int receiveFragment(artdaq::Fragment &fragment, size_t receiveTimeout) override
Receive a Fragment using Multicast.
void flush_buffers() override
Flush any in-flight data. This should be used by the receiver after the receive loop has ended...
Role role() const
Get the TransferInterface::Role of this TransferInterface.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
CopyStatus transfer_fragment_min_blocking_mode(artdaq::Fragment const &fragment, size_t send_timeout_usec) override
Copy a Fragment to the destination. Multicast is always unreliable.
MulticastTransfer is a TransferInterface implementation plugin that transfers data using Multicast...
This TransferInterface is a Receiver.
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
This TransferInterface is a Sender.
virtual ~MulticastTransfer()=default
Default destructor.
CopyStatus transfer_fragment_reliable_mode(artdaq::Fragment &&fragment) override
Move a Fragment to the destination. Multicast is always unreliable.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
MulticastTransfer(fhicl::ParameterSet const &ps, Role role)
MulticastTransfer Constructor.
This interface defines the functions used to transfer data between artdaq applications.
artdaq::Fragment::byte_t byte_t
Copy Fragment::byte_t into local scope.
Value to be returned upon receive timeout.
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.
const size_t max_fragment_size_words_
The maximum size of the transferred Fragment objects, in artdaq::Fragment::RawDataType words...