artdaq  v3_09_00
Multicast_transfer.cc
1 #include "artdaq/DAQdata/Globals.hh"
2 #define TRACE_NAME (app_name + "_MulticastTransfer").c_str()
3 
4 #include "artdaq/TransferPlugins/TransferInterface.hh"
5 
6 #include "artdaq-core/Data/Fragment.hh"
7 #include "artdaq-core/Utilities/ExceptionHandler.hh"
8 
9 #include "cetlib_except/exception.h"
10 #include "fhiclcpp/ParameterSet.h"
11 
12 #include <boost/asio.hpp>
13 #include <boost/bind.hpp>
14 
15 #include <bitset>
16 #include <cassert>
17 #include <iostream>
18 #include <string>
19 #include <type_traits>
20 #include <vector>
21 
22 #pragma GCC diagnostic push
23 #pragma GCC diagnostic ignored "-Wunused-parameter"
24 
25 namespace artdaq {
30 {
31 public:
32  using byte_t = artdaq::Fragment::byte_t;
33 
37  ~MulticastTransfer() override = default;
38 
56  MulticastTransfer(fhicl::ParameterSet const& ps, Role role);
57 
64  int receiveFragment(artdaq::Fragment& fragment,
65  size_t receiveTimeout) override;
66 
73  int receiveFragmentHeader(detail::RawFragmentHeader& header, size_t receiveTimeout) override;
74 
81  int receiveFragmentData(RawDataType* destination, size_t wordCount) override;
82 
89  CopyStatus transfer_fragment_min_blocking_mode(artdaq::Fragment const& fragment, size_t send_timeout_usec) override;
90 
96  CopyStatus transfer_fragment_reliable_mode(artdaq::Fragment&& fragment) override;
97 
102  bool isRunning() override { return socket_ != nullptr; }
103 
108  void flush_buffers() override {}
109 
110 private:
111  MulticastTransfer(MulticastTransfer const&) = delete;
113  MulticastTransfer& operator=(MulticastTransfer const&) = delete;
114  MulticastTransfer& operator=(MulticastTransfer&&) = delete;
115 
116  void fill_staging_memory(const artdaq::Fragment& frag);
117 
118  template<typename T>
119  void book_container_of_buffers(std::vector<T>& buffers,
120  size_t fragment_size,
121  size_t total_subfragments,
122  size_t first_subfragment_num,
123  size_t last_subfragment_num);
124 
125  void get_fragment_quantities(const boost::asio::mutable_buffer& buf, size_t& payload_size, size_t& fragment_size,
126  size_t& expected_subfragments);
127 
128  void set_receive_buffer_size(size_t recv_buff_size);
129 
130  class subfragment_identifier
131  {
132  public:
133  subfragment_identifier(size_t sequenceID, size_t fragmentID, size_t subfragment_number)
134  : sequenceID_(sequenceID)
135  , fragmentID_(fragmentID)
136  , subfragment_number_(subfragment_number) {}
137 
138  size_t sequenceID() const { return sequenceID_; }
139  size_t fragmentID() const { return fragmentID_; }
140  size_t subfragment_number() const { return subfragment_number_; }
141 
142  private:
143  size_t sequenceID_;
144  size_t fragmentID_;
145  size_t subfragment_number_;
146  };
147 
148  std::unique_ptr<boost::asio::io_service> io_service_;
149 
150  std::unique_ptr<boost::asio::ip::udp::endpoint> local_endpoint_;
151  std::unique_ptr<boost::asio::ip::udp::endpoint> multicast_endpoint_;
152  std::unique_ptr<boost::asio::ip::udp::endpoint> opposite_endpoint_;
153 
154  std::unique_ptr<boost::asio::ip::udp::socket> socket_;
155 
156  size_t subfragment_size_;
157  size_t subfragments_per_send_;
158 
159  size_t pause_on_copy_usecs_;
160  Fragment fragment_buffer_;
161 
162  std::vector<byte_t> staging_memory_;
163 
164  std::vector<boost::asio::mutable_buffer> receive_buffers_;
165 };
166 } // namespace artdaq
167 
168 artdaq::MulticastTransfer::MulticastTransfer(fhicl::ParameterSet const& pset, Role role)
169  : TransferInterface(pset, role)
170  , io_service_(std::make_unique<std::remove_reference<decltype(*io_service_)>::type>())
171  , local_endpoint_(nullptr)
172  , multicast_endpoint_(nullptr)
173  , opposite_endpoint_(std::make_unique<std::remove_reference<decltype(*opposite_endpoint_)>::type>())
174  , socket_(nullptr)
175  , subfragment_size_(pset.get<size_t>("subfragment_size"))
176  , subfragments_per_send_(pset.get<size_t>("subfragments_per_send"))
177  , pause_on_copy_usecs_(pset.get<size_t>("pause_on_copy_usecs", 0))
178 {
179  try
180  {
181  portMan->UpdateConfiguration(pset);
182  auto port = portMan->GetMulticastTransferPort(source_rank());
183  auto multicast_address = boost::asio::ip::address::from_string(portMan->GetMulticastTransferGroupAddress());
184  auto local_address = boost::asio::ip::address::from_string(pset.get<std::string>("local_address"));
185 
186  TLOG(TLVL_DEBUG) << GetTraceName() << "multicast address is set to " << multicast_address;
187  TLOG(TLVL_DEBUG) << GetTraceName() << "local address is set to " << local_address;
188 
190  {
191  local_endpoint_ = std::make_unique<std::remove_reference<decltype(*local_endpoint_)>::type>(local_address, 0);
192  multicast_endpoint_ = std::make_unique<std::remove_reference<decltype(*multicast_endpoint_)>::type>(multicast_address, port);
193 
194  socket_ = std::make_unique<std::remove_reference<decltype(*socket_)>::type>(*io_service_,
195  multicast_endpoint_->protocol());
196  socket_->bind(*local_endpoint_);
197  }
198  else
199  { // TransferInterface::role() == Role::kReceive
200 
201  // Create the socket so that multiple may be bound to the same address.
202 
203  local_endpoint_ = std::make_unique<std::remove_reference<decltype(*local_endpoint_)>::type>(local_address, port);
204  socket_ = std::make_unique<std::remove_reference<decltype(*socket_)>::type>(*io_service_,
205  local_endpoint_->protocol());
206 
207  boost::system::error_code ec;
208 
209  socket_->set_option(boost::asio::ip::udp::socket::reuse_address(true), ec);
210 
211  if (ec.value() != 0)
212  {
213  std::cerr << "boost::system::error_code with value " << ec << " was found in setting reuse_address option" << std::endl;
214  }
215 
216  set_receive_buffer_size(pset.get<size_t>("receive_buffer_size", 0));
217 
218  socket_->bind(boost::asio::ip::udp::endpoint(multicast_address, port));
219 
220  // Join the multicast group.
221 
222  socket_->set_option(boost::asio::ip::multicast::join_group(multicast_address), ec);
223 
224  if (ec.value() != 0)
225  {
226  std::cerr << "boost::system::error_code with value " << ec << " was found in attempt to join multicast group" << std::endl;
227  }
228  }
229  }
230  catch (...)
231  {
232  ExceptionHandler(ExceptionHandlerRethrow::yes, "Problem setting up the socket in MulticastTransfer");
233  }
234 
235  auto max_subfragments =
236  static_cast<size_t>(std::ceil(max_fragment_size_words_ / static_cast<float>(subfragment_size_)));
237 
238  staging_memory_.resize(max_subfragments * (sizeof(subfragment_identifier) + subfragment_size_));
239 
241  {
242  book_container_of_buffers(receive_buffers_, max_fragment_size_words_, max_subfragments, 0, max_subfragments - 1);
243  }
244 
245  TLOG(TLVL_DEBUG) << GetTraceName() << "max_subfragments is " << max_subfragments;
246  TLOG(TLVL_DEBUG) << GetTraceName() << "Staging buffer size is " << staging_memory_.size();
247 }
248 
249 #pragma GCC diagnostic push
250 #pragma GCC diagnostic ignored "-Wunused-variable"
251 
252 int artdaq::MulticastTransfer::receiveFragment(artdaq::Fragment& fragment,
253  size_t receiveTimeout)
254 {
255  assert(TransferInterface::role() == Role::kReceive);
256 
257  if (fragment.dataSizeBytes() > 0)
258  {
259  throw cet::exception("MulticastTransfer") << "Error in MulticastTransfer::receiveFragmentFrom: " // NOLINT(cert-err60-cpp)
260  << "nonzero payload found in fragment passed as argument";
261  }
262 
263  static bool print_warning = true;
264 
265  if (print_warning)
266  {
267  std::cerr << "Please note that MulticastTransfer::receiveFragmentFrom does not use its receiveTimeout argument" << std::endl;
268  print_warning = false;
269  }
270 
271  fragment.resizeBytes(max_fragment_size_words_ - sizeof(artdaq::detail::RawFragmentHeader));
272 
273  static auto current_sequenceID = std::numeric_limits<Fragment::sequence_id_t>::max();
274  static auto current_fragmentID = std::numeric_limits<Fragment::fragment_id_t>::max();
275 
276  size_t fragment_size = 0;
277  size_t expected_subfragments = 0;
278  size_t current_subfragments = 0;
279  bool fragment_complete = false;
280  bool last_fragment_truncated = false;
281 
282  while (true)
283  {
284  auto bytes_received = socket_->receive_from(receive_buffers_, *opposite_endpoint_);
285 
286  size_t bytes_processed = 0;
287 
288  for (auto& buf : receive_buffers_)
289  {
290  auto buf_size = boost::asio::buffer_size(buf);
291  auto size_t_ptr = boost::asio::buffer_cast<const size_t*>(buf);
292  auto seqID = *size_t_ptr;
293  auto fragID = *(size_t_ptr + 1); // NOLINT(cppcoreguidelines-pro-bounds-pointer-arithmetic)
294  auto subfragID = *(size_t_ptr + 2); // NOLINT(cppcoreguidelines-pro-bounds-pointer-arithmetic)
295 
296  if (seqID != current_sequenceID || fragID != current_fragmentID)
297  {
298  // JCF, Jun-22-2016
299  // Code currently operates under the assumption that all subfragments from the call are from the same fragment
300 
301  assert(bytes_processed == 0);
302 
303  if (current_subfragments < expected_subfragments)
304  {
305  last_fragment_truncated = true;
306 
307  if (expected_subfragments != std::numeric_limits<size_t>::max())
308  {
309  std::cerr << "Warning: only received " << current_subfragments << " subfragments for fragment with seqID = " << current_sequenceID << ", fragID = " << current_fragmentID << " (expected " << expected_subfragments << ")\n"
310  << std::endl;
311  }
312  else
313  {
314  std::cerr << "Warning: only received " << current_subfragments << " subfragments for fragment with seqID = " << current_sequenceID << ", fragID = " << current_fragmentID << ", # of expected subfragments is unknown as fragment header was not received)\n"
315  << std::endl;
316  }
317  }
318 
319  current_subfragments = 0;
320  fragment_size = std::numeric_limits<size_t>::max();
321  expected_subfragments = std::numeric_limits<size_t>::max();
322  current_sequenceID = seqID;
323  current_fragmentID = fragID;
324  }
325 
326  auto ptr_into_fragment = fragment.headerBeginBytes() + subfragID * subfragment_size_; // NOLINT(cppcoreguidelines-pro-bounds-pointer-arithmetic)
327 
328  auto ptr_into_buffer = boost::asio::buffer_cast<const byte_t*>(buf) + sizeof(subfragment_identifier);
329 
330  std::copy(ptr_into_buffer, ptr_into_buffer + buf_size - sizeof(subfragment_identifier), ptr_into_fragment); // NOLINT(cppcoreguidelines-pro-bounds-pointer-arithmetic)
331 
332  if (subfragID == 0)
333  {
334  if (buf_size >= sizeof(subfragment_identifier) + sizeof(artdaq::detail::RawFragmentHeader))
335  {
336  auto payload_size = std::numeric_limits<size_t>::max();
337  get_fragment_quantities(buf, payload_size, fragment_size, expected_subfragments);
338 
339  fragment.resizeBytes(payload_size);
340  }
341  else
342  {
343  throw cet::exception("MulticastTransfer") << "Buffer size is too small to completely contain an artdaq::Fragment header; " // NOLINT(cert-err60-cpp)
344  << "please increase the default size";
345  }
346  }
347 
348  current_subfragments++;
349 
350  if (current_subfragments == expected_subfragments)
351  {
352  fragment_complete = true;
353  }
354 
355  bytes_processed += buf_size;
356 
357  if (bytes_processed >= bytes_received)
358  {
359  break;
360  }
361  }
362 
363  if (last_fragment_truncated)
364  {
365  // JCF, 7-7-2017
366 
367  // Don't yet have code to handle the scenario where the set of
368  // subfragments received in the last iteration of the loop was
369  // its own complete fragment, but we know the previous fragment
370  // to be incomplete
371 
372  assert(!fragment_complete);
373  TLOG(TLVL_WARNING) << GetTraceName() << "Got an incomplete fragment";
375  }
376 
377  if (fragment_complete)
378  {
379  return source_rank();
380  }
381  }
382 
384 }
385 
386 #pragma GCC diagnostic pop
387 
388 int artdaq::MulticastTransfer::receiveFragmentHeader(detail::RawFragmentHeader& header, size_t receiveTimeout)
389 {
390  auto ret = receiveFragment(fragment_buffer_, receiveTimeout);
391  if (ret == source_rank())
392  {
393  header = *reinterpret_cast<detail::RawFragmentHeader*>(fragment_buffer_.headerAddress()); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
394  return source_rank();
395  }
396  return ret;
397 }
398 
399 int artdaq::MulticastTransfer::receiveFragmentData(RawDataType* destination, size_t wordCount)
400 {
401  if (fragment_buffer_.size() > detail::RawFragmentHeader::num_words())
402  {
403  auto dataSize = (fragment_buffer_.size() - detail::RawFragmentHeader::num_words()) * sizeof(RawDataType);
404  memcpy(destination, fragment_buffer_.headerAddress() + detail::RawFragmentHeader::num_words(), dataSize); // NOLINT(cppcoreguidelines-pro-bounds-pointer-arithmetic)
405  return source_rank();
406  }
407  return RECV_TIMEOUT;
408 }
409 
410 // Reliable transport is undefined for multicast; just use copy
413 {
414  return transfer_fragment_min_blocking_mode(f, 100000000);
415 }
416 
419  size_t send_timeout_usec)
420 {
421  assert(TransferInterface::role() == Role::kSend);
422 
423  if (fragment.sizeBytes() > max_fragment_size_words_)
424  {
425  throw cet::exception("MulticastTransfer") << "Error in MulticastTransfer::copyFragmentTo: " << fragment.sizeBytes() << " byte fragment exceeds max_fragment_size of " << max_fragment_size_words_; // NOLINT(cert-err60-cpp)
426  }
427 
428  static size_t ncalls = 1;
429  auto num_subfragments = static_cast<size_t>(std::ceil(fragment.sizeBytes() / static_cast<float>(subfragment_size_)));
430 
431  ncalls++;
432 
433  fill_staging_memory(fragment);
434 
435  for (size_t batch_index = 0;; batch_index++)
436  {
437  auto first_subfragment = batch_index * subfragments_per_send_;
438  auto last_subfragment = (batch_index + 1) * subfragments_per_send_ >= num_subfragments ? num_subfragments - 1 : (batch_index + 1) * subfragments_per_send_ - 1;
439 
440  std::vector<boost::asio::const_buffer> buffers;
441 
442  book_container_of_buffers(buffers, fragment.sizeBytes(), num_subfragments, first_subfragment, last_subfragment);
443 
444  socket_->send_to(buffers, *multicast_endpoint_);
445 
446  usleep(pause_on_copy_usecs_);
447 
448  if (last_subfragment == num_subfragments - 1)
449  {
450  break;
451  }
452  }
453  return CopyStatus::kSuccess;
454 }
455 
456 #pragma GCC diagnostic push
457 #pragma GCC diagnostic ignored "-Wsign-compare"
458 
459 void artdaq::MulticastTransfer::fill_staging_memory(const artdaq::Fragment& fragment)
460 {
461  auto num_subfragments = static_cast<size_t>(std::ceil(fragment.sizeBytes() / static_cast<float>(subfragment_size_)));
462  TLOG(TLVL_DEBUG) << GetTraceName() << "# of subfragments to use is " << num_subfragments;
463 
464  for (auto i_s = 0; i_s < num_subfragments; ++i_s)
465  {
466  auto staging_memory_copyto = &staging_memory_.at(i_s * (sizeof(subfragment_identifier) + subfragment_size_));
467 
468  subfragment_identifier sfi(fragment.sequenceID(), fragment.fragmentID(), i_s);
469 
470  std::copy(reinterpret_cast<byte_t*>(&sfi), // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
471  reinterpret_cast<byte_t*>(&sfi) + sizeof(subfragment_identifier), // NOLINT(cppcoreguidelines-pro-bounds-pointer-arithmetic,cppcoreguidelines-pro-type-reinterpret-cast)
472  staging_memory_copyto);
473 
474  auto low_ptr_into_fragment = fragment.headerBeginBytes() + subfragment_size_ * i_s; // NOLINT(cppcoreguidelines-pro-bounds-pointer-arithmetic)
475 
476  auto high_ptr_into_fragment = (i_s == num_subfragments - 1) ? fragment.dataEndBytes() : fragment.headerBeginBytes() + subfragment_size_ * (i_s + 1); // NOLINT(cppcoreguidelines-pro-bounds-pointer-arithmetic)
477 
478  std::copy(low_ptr_into_fragment,
479  high_ptr_into_fragment,
480  staging_memory_copyto + sizeof(subfragment_identifier)); // NOLINT(cppcoreguidelines-pro-bounds-pointer-arithmetic)
481  }
482 }
483 
484 #pragma GCC diagnostic pop
485 
486 // Note that book_container_of_buffers includes, rather than excludes,
487 // "last_subfragment_num"; in this regard it's different than the way
488 // STL functions receive iterators. Note also that the lowest possible
489 // value for "first_subfragment_num" is 0, not 1.
490 
491 template<typename T>
492 void artdaq::MulticastTransfer::book_container_of_buffers(std::vector<T>& buffers,
493  const size_t fragment_size,
494  const size_t total_subfragments,
495  const size_t first_subfragment_num,
496  const size_t last_subfragment_num)
497 {
498  assert(staging_memory_.size() >= total_subfragments * (sizeof(subfragment_identifier) + subfragment_size_));
499  assert(buffers.empty());
500  assert(last_subfragment_num < total_subfragments);
501 
502  for (auto i_f = first_subfragment_num; i_f <= last_subfragment_num; ++i_f)
503  {
504  auto bytes_to_store = (i_f == total_subfragments - 1) ? sizeof(subfragment_identifier) + (fragment_size - (total_subfragments - 1) * subfragment_size_) : sizeof(subfragment_identifier) + subfragment_size_;
505 
506  buffers.emplace_back(&staging_memory_.at(i_f * (sizeof(subfragment_identifier) + subfragment_size_)),
507  bytes_to_store);
508  }
509 }
510 
511 #pragma GCC diagnostic push // Needed since profile builds will ignore the assert
512 #pragma GCC diagnostic ignored "-Wunused-variable"
513 
514 void artdaq::MulticastTransfer::get_fragment_quantities(const boost::asio::mutable_buffer& buf, size_t& payload_size,
515  size_t& fragment_size,
516  size_t& expected_subfragments)
517 {
518  auto* buffer_ptr = boost::asio::buffer_cast<byte_t*>(buf);
519 
520  auto subfragment_num = *(reinterpret_cast<size_t*>(buffer_ptr) + 2); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast,cppcoreguidelines-pro-bounds-pointer-arithmetic)
521 
522  assert(subfragment_num == 0);
523 
524  auto* header =
525  reinterpret_cast<artdaq::detail::RawFragmentHeader*>(buffer_ptr + sizeof(subfragment_identifier)); // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast,cppcoreguidelines-pro-bounds-pointer-arithmetic)
526 
527  fragment_size = header->word_count * sizeof(artdaq::RawDataType);
528 
529  auto metadata_size = header->metadata_word_count * sizeof(artdaq::RawDataType);
530  payload_size = fragment_size - metadata_size - artdaq::detail::RawFragmentHeader::num_words() * sizeof(artdaq::RawDataType);
531 
532  assert(fragment_size ==
533  artdaq::detail::RawFragmentHeader::num_words() * sizeof(artdaq::RawDataType) +
534  metadata_size +
535  payload_size);
536 
537  expected_subfragments = static_cast<size_t>(std::ceil(fragment_size / static_cast<float>(subfragment_size_)));
538 }
539 #pragma GCC diagnostic pop
540 
541 void artdaq::MulticastTransfer::set_receive_buffer_size(size_t recv_buff_size)
542 {
543  if (recv_buff_size == 0)
544  {
545  return;
546  }
547  boost::asio::socket_base::receive_buffer_size actual_recv_buff_size;
548  socket_->get_option(actual_recv_buff_size);
549 
550  TLOG(TLVL_DEBUG) << GetTraceName() << "Receive buffer size is currently " << actual_recv_buff_size.value() << " bytes, will try to change it to " << recv_buff_size;
551 
552  boost::asio::socket_base::receive_buffer_size recv_buff_option(recv_buff_size);
553 
554  boost::system::error_code ec;
555  socket_->set_option(recv_buff_option, ec);
556 
557  if (ec.value() != 0)
558  {
559  std::cerr << "boost::system::error_code with value " << ec << " was found in attempt to change receive buffer" << std::endl;
560  }
561 
562  socket_->get_option(actual_recv_buff_size);
563  TLOG(TLVL_DEBUG) << GetTraceName() << "After attempted change, receive buffer size is now " << actual_recv_buff_size.value();
564 }
565 
566 #pragma GCC diagnostic pop
567 
568 DEFINE_ARTDAQ_TRANSFER(artdaq::MulticastTransfer)
virtual int source_rank() const
Get the source rank for this TransferInterface instance.
int receiveFragment(artdaq::Fragment &fragment, size_t receiveTimeout) override
Receive a Fragment using Multicast.
void flush_buffers() override
Flush any in-flight data. This should be used by the receiver after the receive loop has ended...
Role role() const
Get the TransferInterface::Role of this TransferInterface.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
~MulticastTransfer() override=default
Default destructor.
CopyStatus transfer_fragment_min_blocking_mode(artdaq::Fragment const &fragment, size_t send_timeout_usec) override
Copy a Fragment to the destination. Multicast is always unreliable.
MulticastTransfer is a TransferInterface implementation plugin that transfers data using Multicast...
This TransferInterface is a Receiver.
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
This TransferInterface is a Sender.
CopyStatus transfer_fragment_reliable_mode(artdaq::Fragment &&fragment) override
Move a Fragment to the destination. Multicast is always unreliable.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
MulticastTransfer(fhicl::ParameterSet const &ps, Role role)
MulticastTransfer Constructor.
This interface defines the functions used to transfer data between artdaq applications.
artdaq::Fragment::byte_t byte_t
Copy Fragment::byte_t into local scope.
Value to be returned upon receive timeout.
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.
const size_t max_fragment_size_words_
The maximum size of the transferred Fragment objects, in artdaq::Fragment::RawDataType words...