1 #include "artdaq/DAQdata/Globals.hh"
2 #define TRACE_NAME (app_name + "_MulticastTransfer").c_str()
4 #include "artdaq/TransferPlugins/TransferInterface.hh"
6 #include "artdaq-core/Data/Fragment.hh"
7 #include "artdaq-core/Utilities/ExceptionHandler.hh"
9 #include "cetlib_except/exception.h"
10 #include "fhiclcpp/ParameterSet.h"
12 #include <boost/asio.hpp>
13 #include <boost/bind.hpp>
19 #include <type_traits>
22 #pragma GCC diagnostic push
23 #pragma GCC diagnostic ignored "-Wunused-parameter"
32 using byte_t = artdaq::Fragment::byte_t;
65 size_t receiveTimeout)
override;
102 bool isRunning()
override {
return socket_ !=
nullptr; }
116 void fill_staging_memory(
const artdaq::Fragment& frag);
119 void book_container_of_buffers(std::vector<T>& buffers,
120 size_t fragment_size,
121 size_t total_subfragments,
122 size_t first_subfragment_num,
123 size_t last_subfragment_num);
125 void get_fragment_quantities(
const boost::asio::mutable_buffer& buf,
size_t& payload_size,
size_t& fragment_size,
126 size_t& expected_subfragments);
128 void set_receive_buffer_size(
size_t recv_buff_size);
130 class subfragment_identifier
133 subfragment_identifier(
size_t sequenceID,
size_t fragmentID,
size_t subfragment_number)
134 : sequenceID_(sequenceID)
135 , fragmentID_(fragmentID)
136 , subfragment_number_(subfragment_number) {}
138 size_t sequenceID()
const {
return sequenceID_; }
139 size_t fragmentID()
const {
return fragmentID_; }
140 size_t subfragment_number()
const {
return subfragment_number_; }
145 size_t subfragment_number_;
148 std::unique_ptr<boost::asio::io_service> io_service_;
150 std::unique_ptr<boost::asio::ip::udp::endpoint> local_endpoint_;
151 std::unique_ptr<boost::asio::ip::udp::endpoint> multicast_endpoint_;
152 std::unique_ptr<boost::asio::ip::udp::endpoint> opposite_endpoint_;
154 std::unique_ptr<boost::asio::ip::udp::socket> socket_;
156 size_t subfragment_size_;
157 size_t subfragments_per_send_;
159 size_t pause_on_copy_usecs_;
160 Fragment fragment_buffer_;
162 std::vector<byte_t> staging_memory_;
164 std::vector<boost::asio::mutable_buffer> receive_buffers_;
170 , io_service_(std::make_unique<std::remove_reference<decltype(*io_service_)>::type>())
171 , local_endpoint_(nullptr)
172 , multicast_endpoint_(nullptr)
173 , opposite_endpoint_(std::make_unique<std::remove_reference<decltype(*opposite_endpoint_)>::type>())
175 , subfragment_size_(pset.get<size_t>(
"subfragment_size"))
176 , subfragments_per_send_(pset.get<size_t>(
"subfragments_per_send"))
177 , pause_on_copy_usecs_(pset.get<size_t>(
"pause_on_copy_usecs", 0))
181 portMan->UpdateConfiguration(pset);
182 auto port = portMan->GetMulticastTransferPort(
source_rank());
183 auto multicast_address = boost::asio::ip::address::from_string(portMan->GetMulticastTransferGroupAddress());
184 auto local_address = boost::asio::ip::address::from_string(pset.get<std::string>(
"local_address"));
186 TLOG(TLVL_DEBUG) << GetTraceName() <<
"multicast address is set to " << multicast_address;
187 TLOG(TLVL_DEBUG) << GetTraceName() <<
"local address is set to " << local_address;
191 local_endpoint_ = std::make_unique<std::remove_reference<decltype(*local_endpoint_)>::type>(local_address, 0);
192 multicast_endpoint_ = std::make_unique<std::remove_reference<decltype(*multicast_endpoint_)>::type>(multicast_address, port);
194 socket_ = std::make_unique<std::remove_reference<decltype(*socket_)>::type>(*io_service_,
195 multicast_endpoint_->protocol());
196 socket_->bind(*local_endpoint_);
203 local_endpoint_ = std::make_unique<std::remove_reference<decltype(*local_endpoint_)>::type>(local_address, port);
204 socket_ = std::make_unique<std::remove_reference<decltype(*socket_)>::type>(*io_service_,
205 local_endpoint_->protocol());
207 boost::system::error_code ec;
209 socket_->set_option(boost::asio::ip::udp::socket::reuse_address(
true), ec);
213 std::cerr <<
"boost::system::error_code with value " << ec <<
" was found in setting reuse_address option" << std::endl;
216 set_receive_buffer_size(pset.get<
size_t>(
"receive_buffer_size", 0));
218 socket_->bind(boost::asio::ip::udp::endpoint(multicast_address, port));
222 socket_->set_option(boost::asio::ip::multicast::join_group(multicast_address), ec);
226 std::cerr <<
"boost::system::error_code with value " << ec <<
" was found in attempt to join multicast group" << std::endl;
232 ExceptionHandler(ExceptionHandlerRethrow::yes,
"Problem setting up the socket in MulticastTransfer");
235 auto max_subfragments =
238 staging_memory_.resize(max_subfragments * (
sizeof(subfragment_identifier) + subfragment_size_));
245 TLOG(TLVL_DEBUG) << GetTraceName() <<
"max_subfragments is " << max_subfragments;
246 TLOG(TLVL_DEBUG) << GetTraceName() <<
"Staging buffer size is " << staging_memory_.size();
249 #pragma GCC diagnostic push
250 #pragma GCC diagnostic ignored "-Wunused-variable"
253 size_t receiveTimeout)
257 if (fragment.dataSizeBytes() > 0)
259 throw cet::exception(
"MulticastTransfer") <<
"Error in MulticastTransfer::receiveFragmentFrom: "
260 <<
"nonzero payload found in fragment passed as argument";
263 static bool print_warning =
true;
267 std::cerr <<
"Please note that MulticastTransfer::receiveFragmentFrom does not use its receiveTimeout argument" << std::endl;
268 print_warning =
false;
271 fragment.resizeBytes(max_fragment_size_words_ -
sizeof(artdaq::detail::RawFragmentHeader));
273 static auto current_sequenceID = std::numeric_limits<Fragment::sequence_id_t>::max();
274 static auto current_fragmentID = std::numeric_limits<Fragment::fragment_id_t>::max();
276 size_t fragment_size = 0;
277 size_t expected_subfragments = 0;
278 size_t current_subfragments = 0;
279 bool fragment_complete =
false;
280 bool last_fragment_truncated =
false;
284 auto bytes_received = socket_->receive_from(receive_buffers_, *opposite_endpoint_);
286 size_t bytes_processed = 0;
288 for (
auto& buf : receive_buffers_)
290 auto buf_size = boost::asio::buffer_size(buf);
291 auto size_t_ptr = boost::asio::buffer_cast<
const size_t*>(buf);
292 auto seqID = *size_t_ptr;
293 auto fragID = *(size_t_ptr + 1);
294 auto subfragID = *(size_t_ptr + 2);
296 if (seqID != current_sequenceID || fragID != current_fragmentID)
301 assert(bytes_processed == 0);
303 if (current_subfragments < expected_subfragments)
305 last_fragment_truncated =
true;
307 if (expected_subfragments != std::numeric_limits<size_t>::max())
309 std::cerr <<
"Warning: only received " << current_subfragments <<
" subfragments for fragment with seqID = " << current_sequenceID <<
", fragID = " << current_fragmentID <<
" (expected " << expected_subfragments <<
")\n"
314 std::cerr <<
"Warning: only received " << current_subfragments <<
" subfragments for fragment with seqID = " << current_sequenceID <<
", fragID = " << current_fragmentID <<
", # of expected subfragments is unknown as fragment header was not received)\n"
319 current_subfragments = 0;
320 fragment_size = std::numeric_limits<size_t>::max();
321 expected_subfragments = std::numeric_limits<size_t>::max();
322 current_sequenceID = seqID;
323 current_fragmentID = fragID;
326 auto ptr_into_fragment = fragment.headerBeginBytes() + subfragID * subfragment_size_;
328 auto ptr_into_buffer = boost::asio::buffer_cast<
const byte_t*>(buf) +
sizeof(subfragment_identifier);
330 std::copy(ptr_into_buffer, ptr_into_buffer + buf_size -
sizeof(subfragment_identifier), ptr_into_fragment);
334 if (buf_size >=
sizeof(subfragment_identifier) +
sizeof(artdaq::detail::RawFragmentHeader))
336 auto payload_size = std::numeric_limits<size_t>::max();
337 get_fragment_quantities(buf, payload_size, fragment_size, expected_subfragments);
339 fragment.resizeBytes(payload_size);
343 throw cet::exception(
"MulticastTransfer") <<
"Buffer size is too small to completely contain an artdaq::Fragment header; "
344 <<
"please increase the default size";
348 current_subfragments++;
350 if (current_subfragments == expected_subfragments)
352 fragment_complete =
true;
355 bytes_processed += buf_size;
357 if (bytes_processed >= bytes_received)
363 if (last_fragment_truncated)
372 assert(!fragment_complete);
373 TLOG(TLVL_WARNING) << GetTraceName() <<
"Got an incomplete fragment";
377 if (fragment_complete)
379 return source_rank();
386 #pragma GCC diagnostic pop
390 auto ret = receiveFragment(fragment_buffer_, receiveTimeout);
391 if (ret == source_rank())
393 header = *
reinterpret_cast<detail::RawFragmentHeader*
>(fragment_buffer_.headerAddress());
394 return source_rank();
401 if (fragment_buffer_.size() > detail::RawFragmentHeader::num_words())
403 auto dataSize = (fragment_buffer_.size() - detail::RawFragmentHeader::num_words()) *
sizeof(RawDataType);
404 memcpy(destination, fragment_buffer_.headerAddress() + detail::RawFragmentHeader::num_words(), dataSize);
405 return source_rank();
414 return transfer_fragment_min_blocking_mode(f, 100000000);
419 size_t send_timeout_usec)
423 if (fragment.sizeBytes() > max_fragment_size_words_)
425 throw cet::exception(
"MulticastTransfer") <<
"Error in MulticastTransfer::copyFragmentTo: " << fragment.sizeBytes() <<
" byte fragment exceeds max_fragment_size of " << max_fragment_size_words_;
428 static size_t ncalls = 1;
429 auto num_subfragments =
static_cast<size_t>(std::ceil(fragment.sizeBytes() /
static_cast<float>(subfragment_size_)));
433 fill_staging_memory(fragment);
435 for (
size_t batch_index = 0;; batch_index++)
437 auto first_subfragment = batch_index * subfragments_per_send_;
438 auto last_subfragment = (batch_index + 1) * subfragments_per_send_ >= num_subfragments ? num_subfragments - 1 : (batch_index + 1) * subfragments_per_send_ - 1;
440 std::vector<boost::asio::const_buffer> buffers;
442 book_container_of_buffers(buffers, fragment.sizeBytes(), num_subfragments, first_subfragment, last_subfragment);
444 socket_->send_to(buffers, *multicast_endpoint_);
446 usleep(pause_on_copy_usecs_);
448 if (last_subfragment == num_subfragments - 1)
453 return CopyStatus::kSuccess;
456 #pragma GCC diagnostic push
457 #pragma GCC diagnostic ignored "-Wsign-compare"
459 void artdaq::MulticastTransfer::fill_staging_memory(
const artdaq::Fragment& fragment)
461 auto num_subfragments =
static_cast<size_t>(std::ceil(fragment.sizeBytes() /
static_cast<float>(subfragment_size_)));
462 TLOG(TLVL_DEBUG) << GetTraceName() <<
"# of subfragments to use is " << num_subfragments;
464 for (
auto i_s = 0; i_s < num_subfragments; ++i_s)
466 auto staging_memory_copyto = &staging_memory_.at(i_s * (
sizeof(subfragment_identifier) + subfragment_size_));
468 subfragment_identifier sfi(fragment.sequenceID(), fragment.fragmentID(), i_s);
470 std::copy(reinterpret_cast<byte_t*>(&sfi),
471 reinterpret_cast<byte_t*>(&sfi) +
sizeof(subfragment_identifier),
472 staging_memory_copyto);
474 auto low_ptr_into_fragment = fragment.headerBeginBytes() + subfragment_size_ * i_s;
476 auto high_ptr_into_fragment = (i_s == num_subfragments - 1) ? fragment.dataEndBytes() : fragment.headerBeginBytes() + subfragment_size_ * (i_s + 1);
478 std::copy(low_ptr_into_fragment,
479 high_ptr_into_fragment,
480 staging_memory_copyto +
sizeof(subfragment_identifier));
484 #pragma GCC diagnostic pop
492 void artdaq::MulticastTransfer::book_container_of_buffers(std::vector<T>& buffers,
493 const size_t fragment_size,
494 const size_t total_subfragments,
495 const size_t first_subfragment_num,
496 const size_t last_subfragment_num)
498 assert(staging_memory_.size() >= total_subfragments * (
sizeof(subfragment_identifier) + subfragment_size_));
499 assert(buffers.empty());
500 assert(last_subfragment_num < total_subfragments);
502 for (
auto i_f = first_subfragment_num; i_f <= last_subfragment_num; ++i_f)
504 auto bytes_to_store = (i_f == total_subfragments - 1) ?
sizeof(subfragment_identifier) + (fragment_size - (total_subfragments - 1) * subfragment_size_) :
sizeof(subfragment_identifier) + subfragment_size_;
506 buffers.emplace_back(&staging_memory_.at(i_f * (
sizeof(subfragment_identifier) + subfragment_size_)),
511 #pragma GCC diagnostic push // Needed since profile builds will ignore the assert
512 #pragma GCC diagnostic ignored "-Wunused-variable"
514 void artdaq::MulticastTransfer::get_fragment_quantities(
const boost::asio::mutable_buffer& buf,
size_t& payload_size,
515 size_t& fragment_size,
516 size_t& expected_subfragments)
518 auto* buffer_ptr = boost::asio::buffer_cast<byte_t*>(buf);
520 auto subfragment_num = *(
reinterpret_cast<size_t*
>(buffer_ptr) + 2);
522 assert(subfragment_num == 0);
525 reinterpret_cast<artdaq::detail::RawFragmentHeader*
>(buffer_ptr +
sizeof(subfragment_identifier));
527 fragment_size = header->word_count *
sizeof(artdaq::RawDataType);
529 auto metadata_size = header->metadata_word_count *
sizeof(artdaq::RawDataType);
530 payload_size = fragment_size - metadata_size - artdaq::detail::RawFragmentHeader::num_words() *
sizeof(artdaq::RawDataType);
532 assert(fragment_size ==
533 artdaq::detail::RawFragmentHeader::num_words() *
sizeof(artdaq::RawDataType) +
537 expected_subfragments =
static_cast<size_t>(std::ceil(fragment_size / static_cast<float>(subfragment_size_)));
539 #pragma GCC diagnostic pop
541 void artdaq::MulticastTransfer::set_receive_buffer_size(
size_t recv_buff_size)
543 if (recv_buff_size == 0)
547 boost::asio::socket_base::receive_buffer_size actual_recv_buff_size;
548 socket_->get_option(actual_recv_buff_size);
550 TLOG(TLVL_DEBUG) << GetTraceName() <<
"Receive buffer size is currently " << actual_recv_buff_size.value() <<
" bytes, will try to change it to " << recv_buff_size;
552 boost::asio::socket_base::receive_buffer_size recv_buff_option(recv_buff_size);
554 boost::system::error_code ec;
555 socket_->set_option(recv_buff_option, ec);
559 std::cerr <<
"boost::system::error_code with value " << ec <<
" was found in attempt to change receive buffer" << std::endl;
562 socket_->get_option(actual_recv_buff_size);
563 TLOG(TLVL_DEBUG) << GetTraceName() <<
"After attempted change, receive buffer size is now " << actual_recv_buff_size.value();
566 #pragma GCC diagnostic pop
virtual int source_rank() const
Get the source rank for this TransferInterface instance.
int receiveFragment(artdaq::Fragment &fragment, size_t receiveTimeout) override
Receive a Fragment using Multicast.
void flush_buffers() override
Flush any in-flight data. This should be used by the receiver after the receive loop has ended...
Role role() const
Get the TransferInterface::Role of this TransferInterface.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
~MulticastTransfer() override=default
Default destructor.
CopyStatus transfer_fragment_min_blocking_mode(artdaq::Fragment const &fragment, size_t send_timeout_usec) override
Copy a Fragment to the destination. Multicast is always unreliable.
MulticastTransfer is a TransferInterface implementation plugin that transfers data using Multicast...
This TransferInterface is a Receiver.
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
This TransferInterface is a Sender.
CopyStatus transfer_fragment_reliable_mode(artdaq::Fragment &&fragment) override
Move a Fragment to the destination. Multicast is always unreliable.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
MulticastTransfer(fhicl::ParameterSet const &ps, Role role)
MulticastTransfer Constructor.
This interface defines the functions used to transfer data between artdaq applications.
artdaq::Fragment::byte_t byte_t
Copy Fragment::byte_t into local scope.
Value to be returned upon receive timeout.
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.
const size_t max_fragment_size_words_
The maximum size of the transferred Fragment objects, in artdaq::Fragment::RawDataType words...