1 #include "artdaq/DAQdata/Globals.hh"
2 #define TRACE_NAME (app_name + "_MulticastTransfer").c_str()
4 #include "artdaq/TransferPlugins/TransferInterface.hh"
6 #include "artdaq-core/Data/Fragment.hh"
7 #include "artdaq-core/Utilities/ExceptionHandler.hh"
9 #include "cetlib_except/exception.h"
10 #include "fhiclcpp/ParameterSet.h"
12 #include <boost/asio.hpp>
13 #include <boost/bind.hpp>
19 #include <type_traits>
22 #pragma GCC diagnostic push
23 #pragma GCC diagnostic ignored "-Wunused-parameter"
32 using byte_t = artdaq::Fragment::byte_t;
65 size_t receiveTimeout)
override;
102 bool isRunning()
override {
return socket_ !=
nullptr; }
111 void fill_staging_memory(
const artdaq::Fragment& frag);
114 void book_container_of_buffers(std::vector<T>& buffers,
115 const size_t fragment_size,
116 const size_t total_subfragments,
117 const size_t first_subfragment_num,
118 const size_t last_subfragment_num);
120 void get_fragment_quantities(
const boost::asio::mutable_buffer& buf,
size_t& payload_size,
size_t& fragment_size,
121 size_t& expected_subfragments);
123 void set_receive_buffer_size(
size_t recv_buff_size);
125 class subfragment_identifier
128 subfragment_identifier(
size_t sequenceID,
size_t fragmentID,
size_t subfragment_number)
129 : sequenceID_(sequenceID)
130 , fragmentID_(fragmentID)
131 , subfragment_number_(subfragment_number) {}
133 size_t sequenceID()
const {
return sequenceID_; }
134 size_t fragmentID()
const {
return fragmentID_; }
135 size_t subfragment_number()
const {
return subfragment_number_; }
140 size_t subfragment_number_;
143 std::unique_ptr<boost::asio::io_service> io_service_;
145 std::unique_ptr<boost::asio::ip::udp::endpoint> local_endpoint_;
146 std::unique_ptr<boost::asio::ip::udp::endpoint> multicast_endpoint_;
147 std::unique_ptr<boost::asio::ip::udp::endpoint> opposite_endpoint_;
149 std::unique_ptr<boost::asio::ip::udp::socket> socket_;
151 size_t subfragment_size_;
152 size_t subfragments_per_send_;
154 size_t pause_on_copy_usecs_;
155 Fragment fragment_buffer_;
157 std::vector<byte_t> staging_memory_;
159 std::vector<boost::asio::mutable_buffer> receive_buffers_;
165 , io_service_(std::make_unique<std::remove_reference<decltype(*io_service_)>::type>())
166 , local_endpoint_(nullptr)
167 , multicast_endpoint_(nullptr)
168 , opposite_endpoint_(std::make_unique<std::remove_reference<decltype(*opposite_endpoint_)>::type>())
170 , subfragment_size_(pset.get<size_t>(
"subfragment_size"))
171 , subfragments_per_send_(pset.get<size_t>(
"subfragments_per_send"))
172 , pause_on_copy_usecs_(pset.get<size_t>(
"pause_on_copy_usecs", 0))
176 portMan->UpdateConfiguration(pset);
177 auto port = portMan->GetMulticastTransferPort(
source_rank());
178 auto multicast_address = boost::asio::ip::address::from_string(portMan->GetMulticastTransferGroupAddress());
179 auto local_address = boost::asio::ip::address::from_string(pset.get<std::string>(
"local_address"));
181 TLOG(TLVL_DEBUG) << GetTraceName() <<
": multicast address is set to " << multicast_address;
182 TLOG(TLVL_DEBUG) << GetTraceName() <<
": local address is set to " << local_address;
186 local_endpoint_ = std::make_unique<std::remove_reference<decltype(*local_endpoint_)>::type>(local_address, 0);
187 multicast_endpoint_ = std::make_unique<std::remove_reference<decltype(*multicast_endpoint_)>::type>(multicast_address, port);
189 socket_ = std::make_unique<std::remove_reference<decltype(*socket_)>::type>(*io_service_,
190 multicast_endpoint_->protocol());
191 socket_->bind(*local_endpoint_);
198 local_endpoint_ = std::make_unique<std::remove_reference<decltype(*local_endpoint_)>::type>(local_address, port);
199 socket_ = std::make_unique<std::remove_reference<decltype(*socket_)>::type>(*io_service_,
200 local_endpoint_->protocol());
202 boost::system::error_code ec;
204 socket_->set_option(boost::asio::ip::udp::socket::reuse_address(
true), ec);
208 std::cerr <<
"boost::system::error_code with value " << ec <<
" was found in setting reuse_address option" << std::endl;
211 set_receive_buffer_size(pset.get<
size_t>(
"receive_buffer_size", 0));
213 socket_->bind(boost::asio::ip::udp::endpoint(multicast_address, port));
217 socket_->set_option(boost::asio::ip::multicast::join_group(multicast_address), ec);
221 std::cerr <<
"boost::system::error_code with value " << ec <<
" was found in attempt to join multicast group" << std::endl;
227 ExceptionHandler(ExceptionHandlerRethrow::yes,
"Problem setting up the socket in MulticastTransfer");
230 auto max_subfragments =
233 staging_memory_.resize(max_subfragments * (
sizeof(subfragment_identifier) + subfragment_size_));
240 TLOG(TLVL_DEBUG) << GetTraceName() <<
": max_subfragments is " << max_subfragments;
241 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Staging buffer size is " << staging_memory_.size();
244 #pragma GCC diagnostic push
245 #pragma GCC diagnostic ignored "-Wunused-variable"
248 size_t receiveTimeout)
252 if (fragment.dataSizeBytes() > 0)
254 throw cet::exception(
"MulticastTransfer") <<
"Error in MulticastTransfer::receiveFragmentFrom: "
255 <<
"nonzero payload found in fragment passed as argument";
258 static bool print_warning =
true;
262 std::cerr <<
"Please note that MulticastTransfer::receiveFragmentFrom does not use its receiveTimeout argument" << std::endl;
263 print_warning =
false;
266 fragment.resizeBytes(max_fragment_size_words_ -
sizeof(artdaq::detail::RawFragmentHeader));
268 static auto current_sequenceID = std::numeric_limits<Fragment::sequence_id_t>::max();
269 static auto current_fragmentID = std::numeric_limits<Fragment::fragment_id_t>::max();
271 size_t fragment_size = 0;
272 size_t expected_subfragments = 0;
273 size_t current_subfragments = 0;
274 bool fragment_complete =
false;
275 bool last_fragment_truncated =
false;
279 auto bytes_received = socket_->receive_from(receive_buffers_, *opposite_endpoint_);
281 size_t bytes_processed = 0;
283 for (
auto& buf : receive_buffers_)
285 auto buf_size = boost::asio::buffer_size(buf);
286 auto size_t_ptr = boost::asio::buffer_cast<
const size_t*>(buf);
287 auto seqID = *size_t_ptr;
288 auto fragID = *(size_t_ptr + 1);
289 auto subfragID = *(size_t_ptr + 2);
291 if (seqID != current_sequenceID || fragID != current_fragmentID)
296 assert(bytes_processed == 0);
298 if (current_subfragments < expected_subfragments)
300 last_fragment_truncated =
true;
302 if (expected_subfragments != std::numeric_limits<size_t>::max())
304 std::cerr <<
"Warning: only received " << current_subfragments <<
" subfragments for fragment with seqID = " << current_sequenceID <<
", fragID = " << current_fragmentID <<
" (expected " << expected_subfragments <<
")\n"
309 std::cerr <<
"Warning: only received " << current_subfragments <<
" subfragments for fragment with seqID = " << current_sequenceID <<
", fragID = " << current_fragmentID <<
", # of expected subfragments is unknown as fragment header was not received)\n"
314 current_subfragments = 0;
315 fragment_size = std::numeric_limits<size_t>::max();
316 expected_subfragments = std::numeric_limits<size_t>::max();
317 current_sequenceID = seqID;
318 current_fragmentID = fragID;
321 auto ptr_into_fragment = fragment.headerBeginBytes() + subfragID * subfragment_size_;
323 auto ptr_into_buffer = boost::asio::buffer_cast<
const byte_t*>(buf) +
sizeof(subfragment_identifier);
325 std::copy(ptr_into_buffer, ptr_into_buffer + buf_size -
sizeof(subfragment_identifier), ptr_into_fragment);
329 if (buf_size >=
sizeof(subfragment_identifier) +
sizeof(artdaq::detail::RawFragmentHeader))
331 auto payload_size = std::numeric_limits<size_t>::max();
332 get_fragment_quantities(buf, payload_size, fragment_size, expected_subfragments);
334 fragment.resizeBytes(payload_size);
338 throw cet::exception(
"MulticastTransfer") <<
"Buffer size is too small to completely contain an artdaq::Fragment header; "
339 <<
"please increase the default size";
343 current_subfragments++;
345 if (current_subfragments == expected_subfragments)
347 fragment_complete =
true;
350 bytes_processed += buf_size;
352 if (bytes_processed >= bytes_received)
358 if (last_fragment_truncated)
367 assert(!fragment_complete);
368 TLOG(TLVL_WARNING) << GetTraceName() <<
": Got an incomplete fragment";
372 if (fragment_complete)
374 return source_rank();
381 #pragma GCC diagnostic pop
385 auto ret = receiveFragment(fragment_buffer_, receiveTimeout);
386 if (ret == source_rank())
388 header = *
reinterpret_cast<detail::RawFragmentHeader*
>(fragment_buffer_.headerAddress());
389 return source_rank();
396 if (fragment_buffer_.size() > detail::RawFragmentHeader::num_words())
398 auto dataSize = (fragment_buffer_.size() - detail::RawFragmentHeader::num_words()) *
sizeof(RawDataType);
399 memcpy(destination, fragment_buffer_.headerAddress() + detail::RawFragmentHeader::num_words(), dataSize);
400 return source_rank();
409 return transfer_fragment_min_blocking_mode(f, 100000000);
414 size_t send_timeout_usec)
418 if (fragment.sizeBytes() > max_fragment_size_words_)
420 throw cet::exception(
"MulticastTransfer") <<
"Error in MulticastTransfer::copyFragmentTo: " << fragment.sizeBytes() <<
" byte fragment exceeds max_fragment_size of " << max_fragment_size_words_;
423 static size_t ncalls = 1;
424 auto num_subfragments =
static_cast<size_t>(std::ceil(fragment.sizeBytes() /
static_cast<float>(subfragment_size_)));
428 fill_staging_memory(fragment);
430 for (
size_t batch_index = 0;; batch_index++)
432 auto first_subfragment = batch_index * subfragments_per_send_;
433 auto last_subfragment = (batch_index + 1) * subfragments_per_send_ >= num_subfragments ? num_subfragments - 1 : (batch_index + 1) * subfragments_per_send_ - 1;
435 std::vector<boost::asio::const_buffer> buffers;
437 book_container_of_buffers(buffers, fragment.sizeBytes(), num_subfragments, first_subfragment, last_subfragment);
439 socket_->send_to(buffers, *multicast_endpoint_);
441 usleep(pause_on_copy_usecs_);
443 if (last_subfragment == num_subfragments - 1)
448 return CopyStatus::kSuccess;
451 #pragma GCC diagnostic push
452 #pragma GCC diagnostic ignored "-Wsign-compare"
454 void artdaq::MulticastTransfer::fill_staging_memory(
const artdaq::Fragment& fragment)
456 auto num_subfragments =
static_cast<size_t>(std::ceil(fragment.sizeBytes() /
static_cast<float>(subfragment_size_)));
457 TLOG(TLVL_DEBUG) << GetTraceName() <<
": # of subfragments to use is " << num_subfragments;
459 for (
auto i_s = 0; i_s < num_subfragments; ++i_s)
461 auto staging_memory_copyto = &staging_memory_.at(i_s * (
sizeof(subfragment_identifier) + subfragment_size_));
463 subfragment_identifier sfi(fragment.sequenceID(), fragment.fragmentID(), i_s);
465 std::copy(reinterpret_cast<byte_t*>(&sfi),
466 reinterpret_cast<byte_t*>(&sfi) +
sizeof(subfragment_identifier),
467 staging_memory_copyto);
469 auto low_ptr_into_fragment = fragment.headerBeginBytes() + subfragment_size_ * i_s;
471 auto high_ptr_into_fragment = (i_s == num_subfragments - 1) ? fragment.dataEndBytes() : fragment.headerBeginBytes() + subfragment_size_ * (i_s + 1);
473 std::copy(low_ptr_into_fragment,
474 high_ptr_into_fragment,
475 staging_memory_copyto +
sizeof(subfragment_identifier));
479 #pragma GCC diagnostic pop
487 void artdaq::MulticastTransfer::book_container_of_buffers(std::vector<T>& buffers,
488 const size_t fragment_size,
489 const size_t total_subfragments,
490 const size_t first_subfragment_num,
491 const size_t last_subfragment_num)
493 assert(staging_memory_.size() >= total_subfragments * (
sizeof(subfragment_identifier) + subfragment_size_));
494 assert(buffers.size() == 0);
495 assert(last_subfragment_num < total_subfragments);
497 for (
auto i_f = first_subfragment_num; i_f <= last_subfragment_num; ++i_f)
499 auto bytes_to_store = (i_f == total_subfragments - 1) ?
sizeof(subfragment_identifier) + (fragment_size - (total_subfragments - 1) * subfragment_size_) :
sizeof(subfragment_identifier) + subfragment_size_;
501 buffers.emplace_back(&staging_memory_.at(i_f * (
sizeof(subfragment_identifier) + subfragment_size_)),
506 #pragma GCC diagnostic push // Needed since profile builds will ignore the assert
507 #pragma GCC diagnostic ignored "-Wunused-variable"
509 void artdaq::MulticastTransfer::get_fragment_quantities(
const boost::asio::mutable_buffer& buf,
size_t& payload_size,
510 size_t& fragment_size,
511 size_t& expected_subfragments)
513 byte_t* buffer_ptr = boost::asio::buffer_cast<byte_t*>(buf);
515 auto subfragment_num = *(
reinterpret_cast<size_t*
>(buffer_ptr) + 2);
517 assert(subfragment_num == 0);
519 artdaq::detail::RawFragmentHeader* header =
520 reinterpret_cast<artdaq::detail::RawFragmentHeader*
>(buffer_ptr +
sizeof(subfragment_identifier));
522 fragment_size = header->word_count *
sizeof(artdaq::RawDataType);
524 auto metadata_size = header->metadata_word_count *
sizeof(artdaq::RawDataType);
525 payload_size = fragment_size - metadata_size - artdaq::detail::RawFragmentHeader::num_words() *
sizeof(artdaq::RawDataType);
527 assert(fragment_size ==
528 artdaq::detail::RawFragmentHeader::num_words() *
sizeof(artdaq::RawDataType) +
532 expected_subfragments =
static_cast<size_t>(std::ceil(fragment_size / static_cast<float>(subfragment_size_)));
534 #pragma GCC diagnostic pop
536 void artdaq::MulticastTransfer::set_receive_buffer_size(
size_t recv_buff_size)
538 if (recv_buff_size == 0)
return;
539 boost::asio::socket_base::receive_buffer_size actual_recv_buff_size;
540 socket_->get_option(actual_recv_buff_size);
542 TLOG(TLVL_DEBUG) << GetTraceName() <<
": Receive buffer size is currently " << actual_recv_buff_size.value() <<
" bytes, will try to change it to " << recv_buff_size;
544 boost::asio::socket_base::receive_buffer_size recv_buff_option(recv_buff_size);
546 boost::system::error_code ec;
547 socket_->set_option(recv_buff_option, ec);
551 std::cerr <<
"boost::system::error_code with value " << ec <<
" was found in attempt to change receive buffer" << std::endl;
554 socket_->get_option(actual_recv_buff_size);
555 TLOG(TLVL_DEBUG) << GetTraceName() <<
": After attempted change, receive buffer size is now " << actual_recv_buff_size.value();
558 #pragma GCC diagnostic pop
virtual int source_rank() const
Get the source rank for this TransferInterface instance.
int receiveFragment(artdaq::Fragment &fragment, size_t receiveTimeout) override
Receive a Fragment using Multicast.
void flush_buffers() override
Flush any in-flight data. This should be used by the receiver after the receive loop has ended...
Role role() const
Get the TransferInterface::Role of this TransferInterface.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
CopyStatus transfer_fragment_min_blocking_mode(artdaq::Fragment const &fragment, size_t send_timeout_usec) override
Copy a Fragment to the destination. Multicast is always unreliable.
MulticastTransfer is a TransferInterface implementation plugin that transfers data using Multicast...
This TransferInterface is a Receiver.
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
This TransferInterface is a Sender.
virtual ~MulticastTransfer()=default
Default destructor.
CopyStatus transfer_fragment_reliable_mode(artdaq::Fragment &&fragment) override
Move a Fragment to the destination. Multicast is always unreliable.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
MulticastTransfer(fhicl::ParameterSet const &ps, Role role)
MulticastTransfer Constructor.
This interface defines the functions used to transfer data between artdaq applications.
artdaq::Fragment::byte_t byte_t
Copy Fragment::byte_t into local scope.
Value to be returned upon receive timeout.
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.
const size_t max_fragment_size_words_
The maximum size of the transferred Fragment objects, in artdaq::Fragment::RawDataType words...