artdaq  v3_04_00
Multicast_transfer.cc
1 #define TRACE_NAME (app_name + "_MulticastTransfer").c_str()
2 #include "artdaq/DAQdata/Globals.hh"
3 
4 #include "artdaq/TransferPlugins/TransferInterface.hh"
5 
6 #include "artdaq-core/Data/Fragment.hh"
7 #include "artdaq-core/Utilities/ExceptionHandler.hh"
8 
9 #include "fhiclcpp/ParameterSet.h"
10 #include "cetlib_except/exception.h"
11 
12 #include <boost/asio.hpp>
13 #include <boost/bind.hpp>
14 
15 #include <iostream>
16 #include <vector>
17 #include <cassert>
18 #include <string>
19 #include <type_traits>
20 #include <bitset>
21 
22 #pragma GCC diagnostic push
23 #pragma GCC diagnostic ignored "-Wunused-parameter"
24 
25 
26 namespace artdaq
27 {
32  {
33  public:
34 
35  using byte_t = artdaq::Fragment::byte_t;
36 
40  virtual ~MulticastTransfer() = default;
41 
59  MulticastTransfer(fhicl::ParameterSet const& ps, Role role);
60 
67  int receiveFragment(artdaq::Fragment& fragment,
68  size_t receiveTimeout) override;
69 
76  int receiveFragmentHeader(detail::RawFragmentHeader& header, size_t receiveTimeout) override;
77 
84  int receiveFragmentData(RawDataType* destination, size_t wordCount) override;
85 
92  CopyStatus transfer_fragment_min_blocking_mode(artdaq::Fragment const& fragment, size_t send_timeout_usec) override;
93 
99  CopyStatus transfer_fragment_reliable_mode(artdaq::Fragment&& fragment) override;
100 
105  bool isRunning() override { return socket_ != nullptr; }
106 
111  void flush_buffers() override {}
112  private:
113 
114  void fill_staging_memory(const artdaq::Fragment& frag);
115 
116  template <typename T>
117  void book_container_of_buffers(std::vector<T>& buffers,
118  const size_t fragment_size,
119  const size_t total_subfragments,
120  const size_t first_subfragment_num,
121  const size_t last_subfragment_num);
122 
123  void get_fragment_quantities(const boost::asio::mutable_buffer& buf, size_t& payload_size, size_t& fragment_size,
124  size_t& expected_subfragments);
125 
126  void set_receive_buffer_size(size_t recv_buff_size);
127 
128  class subfragment_identifier
129  {
130  public:
131 
132  subfragment_identifier(size_t sequenceID, size_t fragmentID, size_t subfragment_number) :
133  sequenceID_(sequenceID)
134  , fragmentID_(fragmentID)
135  , subfragment_number_(subfragment_number) { }
136 
137  size_t sequenceID() const { return sequenceID_; }
138  size_t fragmentID() const { return fragmentID_; }
139  size_t subfragment_number() const { return subfragment_number_; }
140 
141  private:
142  size_t sequenceID_;
143  size_t fragmentID_;
144  size_t subfragment_number_;
145  };
146 
147  std::unique_ptr<boost::asio::io_service> io_service_;
148 
149  std::unique_ptr<boost::asio::ip::udp::endpoint> local_endpoint_;
150  std::unique_ptr<boost::asio::ip::udp::endpoint> multicast_endpoint_;
151  std::unique_ptr<boost::asio::ip::udp::endpoint> opposite_endpoint_;
152 
153  std::unique_ptr<boost::asio::ip::udp::socket> socket_;
154 
155  size_t subfragment_size_;
156  size_t subfragments_per_send_;
157 
158  size_t pause_on_copy_usecs_;
159  Fragment fragment_buffer_;
160 
161  std::vector<byte_t> staging_memory_;
162 
163  std::vector<boost::asio::mutable_buffer> receive_buffers_;
164  };
165 }
166 
167 artdaq::MulticastTransfer::MulticastTransfer(fhicl::ParameterSet const& pset, Role role) :
168  TransferInterface(pset, role)
169  , io_service_(std::make_unique<std::remove_reference<decltype(*io_service_)>::type>())
170  , local_endpoint_(nullptr)
171  , multicast_endpoint_(nullptr)
172  , opposite_endpoint_(std::make_unique<std::remove_reference<decltype(*opposite_endpoint_)>::type>())
173  , socket_(nullptr)
174  , subfragment_size_(pset.get<size_t>("subfragment_size"))
175  , subfragments_per_send_(pset.get<size_t>("subfragments_per_send"))
176  , pause_on_copy_usecs_(pset.get<size_t>("pause_on_copy_usecs", 0))
177 {
178  try
179  {
180  portMan->UpdateConfiguration(pset);
181  auto port = portMan->GetMulticastTransferPort(source_rank());
182  auto multicast_address = boost::asio::ip::address::from_string(portMan->GetMulticastTransferGroupAddress());
183  auto local_address = boost::asio::ip::address::from_string(pset.get<std::string>("local_address"));
184 
185  TLOG(TLVL_DEBUG) << GetTraceName() << ": multicast address is set to " << multicast_address ;
186  TLOG(TLVL_DEBUG) << GetTraceName() << ": local address is set to " << local_address ;
187 
189  {
190  local_endpoint_ = std::make_unique<std::remove_reference<decltype(*local_endpoint_)>::type>(local_address, 0);
191  multicast_endpoint_ = std::make_unique<std::remove_reference<decltype(*multicast_endpoint_)>::type>(multicast_address, port);
192 
193  socket_ = std::make_unique<std::remove_reference<decltype(*socket_)>::type>(*io_service_,
194  multicast_endpoint_->protocol());
195  socket_->bind(*local_endpoint_);
196  }
197  else
198  { // TransferInterface::role() == Role::kReceive
199 
200  // Create the socket so that multiple may be bound to the same address.
201 
202  local_endpoint_ = std::make_unique<std::remove_reference<decltype(*local_endpoint_)>::type>(local_address, port);
203  socket_ = std::make_unique<std::remove_reference<decltype(*socket_)>::type>(*io_service_,
204  local_endpoint_->protocol());
205 
206  boost::system::error_code ec;
207 
208  socket_->set_option(boost::asio::ip::udp::socket::reuse_address(true), ec);
209 
210  if (ec != 0)
211  {
212  std::cerr << "boost::system::error_code with value " << ec << " was found in setting reuse_address option" << std::endl;
213  }
214 
215  set_receive_buffer_size(pset.get<size_t>("receive_buffer_size", 0));
216 
217  socket_->bind(boost::asio::ip::udp::endpoint(multicast_address, port));
218 
219  // Join the multicast group.
220 
221  socket_->set_option(boost::asio::ip::multicast::join_group(multicast_address), ec);
222 
223  if (ec != 0)
224  {
225  std::cerr << "boost::system::error_code with value " << ec << " was found in attempt to join multicast group" << std::endl;
226  }
227  }
228  }
229  catch (...)
230  {
231  ExceptionHandler(ExceptionHandlerRethrow::yes, "Problem setting up the socket in MulticastTransfer");
232  }
233 
234  auto max_subfragments =
235  static_cast<size_t>(std::ceil(max_fragment_size_words_ / static_cast<float>(subfragment_size_)));
236 
237  staging_memory_.resize(max_subfragments * (sizeof(subfragment_identifier) + subfragment_size_));
238 
240  {
241  book_container_of_buffers(receive_buffers_, max_fragment_size_words_, max_subfragments, 0, max_subfragments - 1);
242  }
243 
244  TLOG(TLVL_DEBUG) << GetTraceName() << ": max_subfragments is " << max_subfragments ;
245  TLOG(TLVL_DEBUG) << GetTraceName() << ": Staging buffer size is " << staging_memory_.size() ;
246 }
247 
248 #pragma GCC diagnostic push
249 #pragma GCC diagnostic ignored "-Wunused-variable"
250 
251 int artdaq::MulticastTransfer::receiveFragment(artdaq::Fragment& fragment,
252  size_t receiveTimeout)
253 {
254  assert(TransferInterface::role() == Role::kReceive);
255 
256  if (fragment.dataSizeBytes() > 0)
257  {
258  throw cet::exception("MulticastTransfer") << "Error in MulticastTransfer::receiveFragmentFrom: " <<
259  "nonzero payload found in fragment passed as argument";
260  }
261 
262  static bool print_warning = true;
263 
264  if (print_warning)
265  {
266  std::cerr << "Please note that MulticastTransfer::receiveFragmentFrom does not use its receiveTimeout argument" << std::endl;
267  print_warning = false;
268  }
269 
270  fragment.resizeBytes(max_fragment_size_words_ - sizeof(artdaq::detail::RawFragmentHeader));
271 
272  static auto current_sequenceID = std::numeric_limits<Fragment::sequence_id_t>::max();
273  static auto current_fragmentID = std::numeric_limits<Fragment::fragment_id_t>::max();
274 
275  size_t fragment_size = 0;
276  size_t expected_subfragments = 0;
277  size_t current_subfragments = 0;
278  bool fragment_complete = false;
279  bool last_fragment_truncated = false;
280 
281  while (true)
282  {
283  auto bytes_received = socket_->receive_from(receive_buffers_, *opposite_endpoint_);
284 
285  size_t bytes_processed = 0;
286 
287  for (auto& buf : receive_buffers_)
288  {
289  auto buf_size = boost::asio::buffer_size(buf);
290  auto size_t_ptr = boost::asio::buffer_cast<const size_t*>(buf);
291  auto seqID = *size_t_ptr;
292  auto fragID = *(size_t_ptr + 1);
293  auto subfragID = *(size_t_ptr + 2);
294 
295  if (seqID != current_sequenceID || fragID != current_fragmentID)
296  {
297  // JCF, Jun-22-2016
298  // Code currently operates under the assumption that all subfragments from the call are from the same fragment
299 
300  assert(bytes_processed == 0);
301 
302  if (current_subfragments < expected_subfragments)
303  {
304  last_fragment_truncated = true;
305 
306  if (expected_subfragments != std::numeric_limits<size_t>::max())
307  {
308  std::cerr << "Warning: only received " << current_subfragments << " subfragments for fragment with seqID = " <<
309  current_sequenceID << ", fragID = " << current_fragmentID << " (expected " << expected_subfragments << ")\n"
310  << std::endl;
311  }
312  else
313  {
314  std::cerr << "Warning: only received " << current_subfragments <<
315  " subfragments for fragment with seqID = " <<
316  current_sequenceID << ", fragID = " << current_fragmentID <<
317  ", # of expected subfragments is unknown as fragment header was not received)\n"
318  << std::endl;
319  }
320  }
321 
322  current_subfragments = 0;
323  fragment_size = std::numeric_limits<size_t>::max();
324  expected_subfragments = std::numeric_limits<size_t>::max();
325  current_sequenceID = seqID;
326  current_fragmentID = fragID;
327  }
328 
329  auto ptr_into_fragment = fragment.headerBeginBytes() + subfragID * subfragment_size_;
330 
331  auto ptr_into_buffer = boost::asio::buffer_cast<const byte_t*>(buf) + sizeof(subfragment_identifier);
332 
333  std::copy(ptr_into_buffer, ptr_into_buffer + buf_size - sizeof(subfragment_identifier), ptr_into_fragment);
334 
335  if (subfragID == 0)
336  {
337  if (buf_size >= sizeof(subfragment_identifier) + sizeof(artdaq::detail::RawFragmentHeader))
338  {
339  auto payload_size = std::numeric_limits<size_t>::max();
340  get_fragment_quantities(buf, payload_size, fragment_size, expected_subfragments);
341 
342  fragment.resizeBytes(payload_size);
343  }
344  else
345  {
346  throw cet::exception("MulticastTransfer") << "Buffer size is too small to completely contain an artdaq::Fragment header; " <<
347  "please increase the default size";
348  }
349  }
350 
351  current_subfragments++;
352 
353  if (current_subfragments == expected_subfragments)
354  {
355  fragment_complete = true;
356  }
357 
358  bytes_processed += buf_size;
359 
360  if (bytes_processed >= bytes_received)
361  {
362  break;
363  }
364  }
365 
366  if (last_fragment_truncated)
367  {
368  // JCF, 7-7-2017
369 
370  // Don't yet have code to handle the scenario where the set of
371  // subfragments received in the last iteration of the loop was
372  // its own complete fragment, but we know the previous fragment
373  // to be incomplete
374 
375  assert(!fragment_complete);
376  TLOG(TLVL_WARNING) << GetTraceName() << ": Got an incomplete fragment" ;
378  }
379 
380  if (fragment_complete)
381  {
382  return source_rank();
383  }
384  }
385 
387 }
388 
389 #pragma GCC diagnostic pop
390 
391 int artdaq::MulticastTransfer::receiveFragmentHeader(detail::RawFragmentHeader& header, size_t receiveTimeout)
392 {
393  auto ret = receiveFragment(fragment_buffer_, receiveTimeout);
394  if (ret == source_rank())
395  {
396  header = *reinterpret_cast<detail::RawFragmentHeader*>(fragment_buffer_.headerAddress());
397  return source_rank();
398  }
399  return ret;
400 }
401 
402 int artdaq::MulticastTransfer::receiveFragmentData(RawDataType* destination, size_t wordCount)
403 {
404  if (fragment_buffer_.size() > detail::RawFragmentHeader::num_words()) {
405  auto dataSize = (fragment_buffer_.size() - detail::RawFragmentHeader::num_words()) * sizeof(RawDataType);
406  memcpy(destination, fragment_buffer_.headerAddress() + detail::RawFragmentHeader::num_words(), dataSize);
407  return source_rank();
408  }
409  return RECV_TIMEOUT;
410 }
411 
412 
413 // Reliable transport is undefined for multicast; just use copy
416 {
417  return transfer_fragment_min_blocking_mode(f, 100000000);
418 }
419 
422  size_t send_timeout_usec)
423 {
424  assert(TransferInterface::role() == Role::kSend);
425 
426  if (fragment.sizeBytes() > max_fragment_size_words_)
427  {
428  throw cet::exception("MulticastTransfer") << "Error in MulticastTransfer::copyFragmentTo: " <<
429  fragment.sizeBytes() << " byte fragment exceeds max_fragment_size of " << max_fragment_size_words_;
430  }
431 
432  static size_t ncalls = 1;
433  auto num_subfragments = static_cast<size_t>(std::ceil(fragment.sizeBytes() / static_cast<float>(subfragment_size_)));
434 
435  ncalls++;
436 
437  fill_staging_memory(fragment);
438 
439  for (size_t batch_index = 0; ; batch_index++)
440  {
441  auto first_subfragment = batch_index * subfragments_per_send_;
442  auto last_subfragment = (batch_index + 1) * subfragments_per_send_ >= num_subfragments ?
443  num_subfragments - 1 :
444  (batch_index + 1) * subfragments_per_send_ - 1;
445 
446  std::vector<boost::asio::const_buffer> buffers;
447 
448  book_container_of_buffers(buffers, fragment.sizeBytes(), num_subfragments, first_subfragment, last_subfragment);
449 
450  socket_->send_to(buffers, *multicast_endpoint_);
451 
452  usleep(pause_on_copy_usecs_);
453 
454  if (last_subfragment == num_subfragments - 1)
455  {
456  break;
457  }
458  }
459  return CopyStatus::kSuccess;
460 }
461 
462 #pragma GCC diagnostic push
463 #pragma GCC diagnostic ignored "-Wsign-compare"
464 
465 void artdaq::MulticastTransfer::fill_staging_memory(const artdaq::Fragment& fragment)
466 {
467  auto num_subfragments = static_cast<size_t>(std::ceil(fragment.sizeBytes() / static_cast<float>(subfragment_size_)));
468  TLOG(TLVL_DEBUG) << GetTraceName() << ": # of subfragments to use is " << num_subfragments ;
469 
470  for (auto i_s = 0; i_s < num_subfragments; ++i_s)
471  {
472  auto staging_memory_copyto = &staging_memory_.at(i_s * (sizeof(subfragment_identifier) + subfragment_size_));
473 
474  subfragment_identifier sfi(fragment.sequenceID(), fragment.fragmentID(), i_s);
475 
476  std::copy(reinterpret_cast<byte_t*>(&sfi),
477  reinterpret_cast<byte_t*>(&sfi) + sizeof(subfragment_identifier),
478  staging_memory_copyto);
479 
480  auto low_ptr_into_fragment = fragment.headerBeginBytes() + subfragment_size_ * i_s;
481 
482  auto high_ptr_into_fragment = (i_s == num_subfragments - 1) ?
483  fragment.dataEndBytes() :
484  fragment.headerBeginBytes() + subfragment_size_ * (i_s + 1);
485 
486  std::copy(low_ptr_into_fragment,
487  high_ptr_into_fragment,
488  staging_memory_copyto + sizeof(subfragment_identifier));
489  }
490 }
491 
492 #pragma GCC diagnostic pop
493 
494 // Note that book_container_of_buffers includes, rather than excludes,
495 // "last_subfragment_num"; in this regard it's different than the way
496 // STL functions receive iterators. Note also that the lowest possible
497 // value for "first_subfragment_num" is 0, not 1.
498 
499 template <typename T>
500 void artdaq::MulticastTransfer::book_container_of_buffers(std::vector<T>& buffers,
501  const size_t fragment_size,
502  const size_t total_subfragments,
503  const size_t first_subfragment_num,
504  const size_t last_subfragment_num)
505 {
506  assert(staging_memory_.size() >= total_subfragments * (sizeof(subfragment_identifier) + subfragment_size_));
507  assert(buffers.size() == 0);
508  assert(last_subfragment_num < total_subfragments);
509 
510  for (auto i_f = first_subfragment_num; i_f <= last_subfragment_num; ++i_f)
511  {
512  auto bytes_to_store = (i_f == total_subfragments - 1) ?
513  sizeof(subfragment_identifier) + (fragment_size - (total_subfragments - 1) * subfragment_size_) :
514  sizeof(subfragment_identifier) + subfragment_size_;
515 
516  buffers.emplace_back(&staging_memory_.at(i_f * (sizeof(subfragment_identifier) + subfragment_size_)),
517  bytes_to_store);
518  }
519 }
520 
521 
522 #pragma GCC diagnostic push // Needed since profile builds will ignore the assert
523 #pragma GCC diagnostic ignored "-Wunused-variable"
524 
525 void artdaq::MulticastTransfer::get_fragment_quantities(const boost::asio::mutable_buffer& buf, size_t& payload_size,
526  size_t& fragment_size,
527  size_t& expected_subfragments)
528 {
529  byte_t* buffer_ptr = boost::asio::buffer_cast<byte_t*>(buf);
530 
531  auto subfragment_num = *(reinterpret_cast<size_t*>(buffer_ptr) + 2);
532 
533  assert(subfragment_num == 0);
534 
535  artdaq::detail::RawFragmentHeader* header =
536  reinterpret_cast<artdaq::detail::RawFragmentHeader*>(buffer_ptr + sizeof(subfragment_identifier));
537 
538  fragment_size = header->word_count * sizeof(artdaq::RawDataType);
539 
540  auto metadata_size = header->metadata_word_count * sizeof(artdaq::RawDataType);
541  payload_size = fragment_size - metadata_size - artdaq::detail::RawFragmentHeader::num_words() *
542  sizeof(artdaq::RawDataType);
543 
544  assert(fragment_size ==
545  artdaq::detail::RawFragmentHeader::num_words() * sizeof(artdaq::RawDataType) +
546  metadata_size +
547  payload_size);
548 
549  expected_subfragments = static_cast<size_t>(std::ceil(fragment_size / static_cast<float>(subfragment_size_)));
550 }
551 #pragma GCC diagnostic pop
552 
553 void artdaq::MulticastTransfer::set_receive_buffer_size(size_t recv_buff_size)
554 {
555  if (recv_buff_size == 0) return;
556  boost::asio::socket_base::receive_buffer_size actual_recv_buff_size;
557  socket_->get_option(actual_recv_buff_size);
558 
559  TLOG(TLVL_DEBUG) << GetTraceName() << ": Receive buffer size is currently " << actual_recv_buff_size.value() <<
560  " bytes, will try to change it to " << recv_buff_size ;
561 
562  boost::asio::socket_base::receive_buffer_size recv_buff_option(recv_buff_size);
563 
564  boost::system::error_code ec;
565  socket_->set_option(recv_buff_option, ec);
566 
567  if (ec != 0)
568  {
569  std::cerr << "boost::system::error_code with value " << ec <<
570  " was found in attempt to change receive buffer" << std::endl;
571  }
572 
573  socket_->get_option(actual_recv_buff_size);
574  TLOG(TLVL_DEBUG) << GetTraceName() << ": After attempted change, receive buffer size is now " << actual_recv_buff_size.value() ;
575 }
576 
577 #pragma GCC diagnostic pop
578 
579 DEFINE_ARTDAQ_TRANSFER(artdaq::MulticastTransfer)
virtual int source_rank() const
Get the source rank for this TransferInterface instance.
int receiveFragment(artdaq::Fragment &fragment, size_t receiveTimeout) override
Receive a Fragment using Multicast.
void flush_buffers() override
Flush any in-flight data. This should be used by the receiver after the receive loop has ended...
Role role() const
Get the TransferInterface::Role of this TransferInterface.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
CopyStatus transfer_fragment_min_blocking_mode(artdaq::Fragment const &fragment, size_t send_timeout_usec) override
Copy a Fragment to the destination. Multicast is always unreliable.
MulticastTransfer is a TransferInterface implementation plugin that transfers data using Multicast...
This TransferInterface is a Receiver.
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
This TransferInterface is a Sender.
virtual ~MulticastTransfer()=default
Default destructor.
CopyStatus transfer_fragment_reliable_mode(artdaq::Fragment &&fragment) override
Move a Fragment to the destination. Multicast is always unreliable.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
MulticastTransfer(fhicl::ParameterSet const &ps, Role role)
MulticastTransfer Constructor.
This interface defines the functions used to transfer data between artdaq applications.
artdaq::Fragment::byte_t byte_t
Copy Fragment::byte_t into local scope.
Value to be returned upon receive timeout.
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.
const size_t max_fragment_size_words_
The maximum size of the transferred Fragment objects, in artdaq::Fragment::RawDataType words...