artdaq  v3_01_00
Multicast_transfer.cc
1 #define TRACE_NAME "MulticastTransfer"
2 
3 #include "artdaq/TransferPlugins/TransferInterface.hh"
4 
5 #include "artdaq-core/Data/Fragment.hh"
6 #include "artdaq-core/Utilities/ExceptionHandler.hh"
7 
8 #include "fhiclcpp/ParameterSet.h"
9 #include "cetlib_except/exception.h"
10 
11 #include <boost/asio.hpp>
12 #include <boost/bind.hpp>
13 
14 #include <iostream>
15 #include <vector>
16 #include <cassert>
17 #include <string>
18 #include <type_traits>
19 #include <bitset>
20 
21 #pragma GCC diagnostic push
22 #pragma GCC diagnostic ignored "-Wunused-parameter"
23 
24 
25 namespace artdaq
26 {
31  {
32  public:
33 
34  using byte_t = artdaq::Fragment::byte_t;
35 
39  virtual ~MulticastTransfer() = default;
40 
58  MulticastTransfer(fhicl::ParameterSet const& ps, Role role);
59 
66  int receiveFragment(artdaq::Fragment& fragment,
67  size_t receiveTimeout) override;
68 
75  int receiveFragmentHeader(detail::RawFragmentHeader& header, size_t receiveTimeout) override;
76 
83  int receiveFragmentData(RawDataType* destination, size_t wordCount) override;
84 
90  CopyStatus copyFragment(artdaq::Fragment& fragment, size_t send_timeout_usec) override;
91 
97  CopyStatus moveFragment(artdaq::Fragment&& fragment) override;
98 
99  private:
100 
101  void fill_staging_memory(const artdaq::Fragment& frag);
102 
103  template <typename T>
104  void book_container_of_buffers(std::vector<T>& buffers,
105  const size_t fragment_size,
106  const size_t total_subfragments,
107  const size_t first_subfragment_num,
108  const size_t last_subfragment_num);
109 
110  void get_fragment_quantities(const boost::asio::mutable_buffer& buf, size_t& payload_size, size_t& fragment_size,
111  size_t& expected_subfragments);
112 
113  void set_receive_buffer_size(size_t recv_buff_size);
114 
115  class subfragment_identifier
116  {
117  public:
118 
119  subfragment_identifier(size_t sequenceID, size_t fragmentID, size_t subfragment_number) :
120  sequenceID_(sequenceID)
121  , fragmentID_(fragmentID)
122  , subfragment_number_(subfragment_number) { }
123 
124  size_t sequenceID() const { return sequenceID_; }
125  size_t fragmentID() const { return fragmentID_; }
126  size_t subfragment_number() const { return subfragment_number_; }
127 
128  private:
129  size_t sequenceID_;
130  size_t fragmentID_;
131  size_t subfragment_number_;
132  };
133 
134  std::unique_ptr<boost::asio::io_service> io_service_;
135 
136  std::unique_ptr<boost::asio::ip::udp::endpoint> local_endpoint_;
137  std::unique_ptr<boost::asio::ip::udp::endpoint> multicast_endpoint_;
138  std::unique_ptr<boost::asio::ip::udp::endpoint> opposite_endpoint_;
139 
140  std::unique_ptr<boost::asio::ip::udp::socket> socket_;
141 
142  size_t subfragment_size_;
143  size_t subfragments_per_send_;
144 
145  size_t pause_on_copy_usecs_;
146  Fragment fragment_buffer_;
147 
148  std::vector<byte_t> staging_memory_;
149 
150  std::vector<boost::asio::mutable_buffer> receive_buffers_;
151  };
152 }
153 
154 artdaq::MulticastTransfer::MulticastTransfer(fhicl::ParameterSet const& pset, Role role) :
155  TransferInterface(pset, role)
156  , io_service_(std::make_unique<std::remove_reference<decltype(*io_service_)>::type>())
157  , local_endpoint_(nullptr)
158  , multicast_endpoint_(nullptr)
159  , opposite_endpoint_(std::make_unique<std::remove_reference<decltype(*opposite_endpoint_)>::type>())
160  , socket_(nullptr)
161  , subfragment_size_(pset.get<size_t>("subfragment_size"))
162  , subfragments_per_send_(pset.get<size_t>("subfragments_per_send"))
163  , pause_on_copy_usecs_(pset.get<size_t>("pause_on_copy_usecs", 0))
164 {
165  try
166  {
167  auto port = pset.get<unsigned short>("multicast_port");
168  auto multicast_address = boost::asio::ip::address::from_string(pset.get<std::string>("multicast_address"));
169  auto local_address = boost::asio::ip::address::from_string(pset.get<std::string>("local_address"));
170 
171  TLOG(TLVL_DEBUG) << GetTraceName() << ": multicast address is set to " << multicast_address ;
172  TLOG(TLVL_DEBUG) << GetTraceName() << ": local address is set to " << local_address ;
173 
175  {
176  local_endpoint_ = std::make_unique<std::remove_reference<decltype(*local_endpoint_)>::type>(local_address, 0);
177  multicast_endpoint_ = std::make_unique<std::remove_reference<decltype(*multicast_endpoint_)>::type>(multicast_address, port);
178 
179  socket_ = std::make_unique<std::remove_reference<decltype(*socket_)>::type>(*io_service_,
180  multicast_endpoint_->protocol());
181  socket_->bind(*local_endpoint_);
182  }
183  else
184  { // TransferInterface::role() == Role::kReceive
185 
186  // Create the socket so that multiple may be bound to the same address.
187 
188  local_endpoint_ = std::make_unique<std::remove_reference<decltype(*local_endpoint_)>::type>(local_address, port);
189  socket_ = std::make_unique<std::remove_reference<decltype(*socket_)>::type>(*io_service_,
190  local_endpoint_->protocol());
191 
192  boost::system::error_code ec;
193 
194  socket_->set_option(boost::asio::ip::udp::socket::reuse_address(true), ec);
195 
196  if (ec != 0)
197  {
198  std::cerr << "boost::system::error_code with value " << ec << " was found in setting reuse_address option" << std::endl;
199  }
200 
201  set_receive_buffer_size(pset.get<size_t>("receive_buffer_size", 0));
202 
203  socket_->bind(boost::asio::ip::udp::endpoint(multicast_address, port));
204 
205  // Join the multicast group.
206 
207  socket_->set_option(boost::asio::ip::multicast::join_group(multicast_address), ec);
208 
209  if (ec != 0)
210  {
211  std::cerr << "boost::system::error_code with value " << ec << " was found in attempt to join multicast group" << std::endl;
212  }
213  }
214  }
215  catch (...)
216  {
217  ExceptionHandler(ExceptionHandlerRethrow::yes, "Problem setting up the socket in MulticastTransfer");
218  }
219 
220  auto max_subfragments =
221  static_cast<size_t>(std::ceil(max_fragment_size_words_ / static_cast<float>(subfragment_size_)));
222 
223  staging_memory_.resize(max_subfragments * (sizeof(subfragment_identifier) + subfragment_size_));
224 
226  {
227  book_container_of_buffers(receive_buffers_, max_fragment_size_words_, max_subfragments, 0, max_subfragments - 1);
228  }
229 
230  TLOG(TLVL_DEBUG) << GetTraceName() << ": max_subfragments is " << max_subfragments ;
231  TLOG(TLVL_DEBUG) << GetTraceName() << ": Staging buffer size is " << staging_memory_.size() ;
232 }
233 
234 #pragma GCC diagnostic push
235 #pragma GCC diagnostic ignored "-Wunused-variable"
236 
237 int artdaq::MulticastTransfer::receiveFragment(artdaq::Fragment& fragment,
238  size_t receiveTimeout)
239 {
240  assert(TransferInterface::role() == Role::kReceive);
241 
242  if (fragment.dataSizeBytes() > 0)
243  {
244  throw cet::exception("MulticastTransfer") << "Error in MulticastTransfer::receiveFragmentFrom: " <<
245  "nonzero payload found in fragment passed as argument";
246  }
247 
248  static bool print_warning = true;
249 
250  if (print_warning)
251  {
252  std::cerr << "Please note that MulticastTransfer::receiveFragmentFrom does not use its receiveTimeout argument" << std::endl;
253  print_warning = false;
254  }
255 
256  fragment.resizeBytes(max_fragment_size_words_ - sizeof(artdaq::detail::RawFragmentHeader));
257 
258  static auto current_sequenceID = std::numeric_limits<Fragment::sequence_id_t>::max();
259  static auto current_fragmentID = std::numeric_limits<Fragment::fragment_id_t>::max();
260 
261  size_t fragment_size = 0;
262  size_t expected_subfragments = 0;
263  size_t current_subfragments = 0;
264  bool fragment_complete = false;
265  bool last_fragment_truncated = false;
266 
267  while (true)
268  {
269  auto bytes_received = socket_->receive_from(receive_buffers_, *opposite_endpoint_);
270 
271  size_t bytes_processed = 0;
272 
273  for (auto& buf : receive_buffers_)
274  {
275  auto buf_size = boost::asio::buffer_size(buf);
276  auto size_t_ptr = boost::asio::buffer_cast<const size_t*>(buf);
277  auto seqID = *size_t_ptr;
278  auto fragID = *(size_t_ptr + 1);
279  auto subfragID = *(size_t_ptr + 2);
280 
281  if (seqID != current_sequenceID || fragID != current_fragmentID)
282  {
283  // JCF, Jun-22-2016
284  // Code currently operates under the assumption that all subfragments from the call are from the same fragment
285 
286  assert(bytes_processed == 0);
287 
288  if (current_subfragments < expected_subfragments)
289  {
290  last_fragment_truncated = true;
291 
292  if (expected_subfragments != std::numeric_limits<size_t>::max())
293  {
294  std::cerr << "Warning: only received " << current_subfragments << " subfragments for fragment with seqID = " <<
295  current_sequenceID << ", fragID = " << current_fragmentID << " (expected " << expected_subfragments << ")\n"
296  << std::endl;
297  }
298  else
299  {
300  std::cerr << "Warning: only received " << current_subfragments <<
301  " subfragments for fragment with seqID = " <<
302  current_sequenceID << ", fragID = " << current_fragmentID <<
303  ", # of expected subfragments is unknown as fragment header was not received)\n"
304  << std::endl;
305  }
306  }
307 
308  current_subfragments = 0;
309  fragment_size = std::numeric_limits<size_t>::max();
310  expected_subfragments = std::numeric_limits<size_t>::max();
311  current_sequenceID = seqID;
312  current_fragmentID = fragID;
313  }
314 
315  auto ptr_into_fragment = fragment.headerBeginBytes() + subfragID * subfragment_size_;
316 
317  auto ptr_into_buffer = boost::asio::buffer_cast<const byte_t*>(buf) + sizeof(subfragment_identifier);
318 
319  std::copy(ptr_into_buffer, ptr_into_buffer + buf_size - sizeof(subfragment_identifier), ptr_into_fragment);
320 
321  if (subfragID == 0)
322  {
323  if (buf_size >= sizeof(subfragment_identifier) + sizeof(artdaq::detail::RawFragmentHeader))
324  {
325  auto payload_size = std::numeric_limits<size_t>::max();
326  get_fragment_quantities(buf, payload_size, fragment_size, expected_subfragments);
327 
328  fragment.resizeBytes(payload_size);
329  }
330  else
331  {
332  throw cet::exception("MulticastTransfer") << "Buffer size is too small to completely contain an artdaq::Fragment header; " <<
333  "please increase the default size";
334  }
335  }
336 
337  current_subfragments++;
338 
339  if (current_subfragments == expected_subfragments)
340  {
341  fragment_complete = true;
342  }
343 
344  bytes_processed += buf_size;
345 
346  if (bytes_processed >= bytes_received)
347  {
348  break;
349  }
350  }
351 
352  if (last_fragment_truncated)
353  {
354  // JCF, 7-7-2017
355 
356  // Don't yet have code to handle the scenario where the set of
357  // subfragments received in the last iteration of the loop was
358  // its own complete fragment, but we know the previous fragment
359  // to be incomplete
360 
361  assert(!fragment_complete);
362  TLOG(TLVL_WARNING) << GetTraceName() << ": Got an incomplete fragment" ;
364  }
365 
366  if (fragment_complete)
367  {
368  return source_rank();
369  }
370  }
371 
373 }
374 
375 #pragma GCC diagnostic pop
376 
377 int artdaq::MulticastTransfer::receiveFragmentHeader(detail::RawFragmentHeader& header, size_t receiveTimeout)
378 {
379  auto ret = receiveFragment(fragment_buffer_, receiveTimeout);
380  if (ret == source_rank())
381  {
382  header = *reinterpret_cast<detail::RawFragmentHeader*>(fragment_buffer_.headerAddress());
383  return source_rank();
384  }
385  return ret;
386 }
387 
388 int artdaq::MulticastTransfer::receiveFragmentData(RawDataType* destination, size_t wordCount)
389 {
390  if (fragment_buffer_.size() > detail::RawFragmentHeader::num_words()) {
391  auto dataSize = (fragment_buffer_.size() - detail::RawFragmentHeader::num_words()) * sizeof(RawDataType);
392  memcpy(destination, fragment_buffer_.headerAddress() + detail::RawFragmentHeader::num_words(), dataSize);
393  return source_rank();
394  }
395  return RECV_TIMEOUT;
396 }
397 
398 
399 // Reliable transport is undefined for multicast; just use copy
402 {
403  return copyFragment(f, 100000000);
404 }
405 
407 artdaq::MulticastTransfer::copyFragment(artdaq::Fragment& fragment,
408  size_t send_timeout_usec)
409 {
410  assert(TransferInterface::role() == Role::kSend);
411 
412  if (fragment.sizeBytes() > max_fragment_size_words_)
413  {
414  throw cet::exception("MulticastTransfer") << "Error in MulticastTransfer::copyFragmentTo: " <<
415  fragment.sizeBytes() << " byte fragment exceeds max_fragment_size of " << max_fragment_size_words_;
416  }
417 
418  static size_t ncalls = 1;
419  auto num_subfragments = static_cast<size_t>(std::ceil(fragment.sizeBytes() / static_cast<float>(subfragment_size_)));
420 
421  ncalls++;
422 
423  fill_staging_memory(fragment);
424 
425  for (size_t batch_index = 0; ; batch_index++)
426  {
427  auto first_subfragment = batch_index * subfragments_per_send_;
428  auto last_subfragment = (batch_index + 1) * subfragments_per_send_ >= num_subfragments ?
429  num_subfragments - 1 :
430  (batch_index + 1) * subfragments_per_send_ - 1;
431 
432  std::vector<boost::asio::const_buffer> buffers;
433 
434  book_container_of_buffers(buffers, fragment.sizeBytes(), num_subfragments, first_subfragment, last_subfragment);
435 
436  socket_->send_to(buffers, *multicast_endpoint_);
437 
438  usleep(pause_on_copy_usecs_);
439 
440  if (last_subfragment == num_subfragments - 1)
441  {
442  break;
443  }
444  }
445  return CopyStatus::kSuccess;
446 }
447 
448 #pragma GCC diagnostic push
449 #pragma GCC diagnostic ignored "-Wsign-compare"
450 
451 void artdaq::MulticastTransfer::fill_staging_memory(const artdaq::Fragment& fragment)
452 {
453  auto num_subfragments = static_cast<size_t>(std::ceil(fragment.sizeBytes() / static_cast<float>(subfragment_size_)));
454  TLOG(TLVL_DEBUG) << GetTraceName() << ": # of subfragments to use is " << num_subfragments ;
455 
456  for (auto i_s = 0; i_s < num_subfragments; ++i_s)
457  {
458  auto staging_memory_copyto = &staging_memory_.at(i_s * (sizeof(subfragment_identifier) + subfragment_size_));
459 
460  subfragment_identifier sfi(fragment.sequenceID(), fragment.fragmentID(), i_s);
461 
462  std::copy(reinterpret_cast<byte_t*>(&sfi),
463  reinterpret_cast<byte_t*>(&sfi) + sizeof(subfragment_identifier),
464  staging_memory_copyto);
465 
466  auto low_ptr_into_fragment = fragment.headerBeginBytes() + subfragment_size_ * i_s;
467 
468  auto high_ptr_into_fragment = (i_s == num_subfragments - 1) ?
469  fragment.dataEndBytes() :
470  fragment.headerBeginBytes() + subfragment_size_ * (i_s + 1);
471 
472  std::copy(low_ptr_into_fragment,
473  high_ptr_into_fragment,
474  staging_memory_copyto + sizeof(subfragment_identifier));
475  }
476 }
477 
478 #pragma GCC diagnostic pop
479 
480 // Note that book_container_of_buffers includes, rather than excludes,
481 // "last_subfragment_num"; in this regard it's different than the way
482 // STL functions receive iterators. Note also that the lowest possible
483 // value for "first_subfragment_num" is 0, not 1.
484 
485 template <typename T>
486 void artdaq::MulticastTransfer::book_container_of_buffers(std::vector<T>& buffers,
487  const size_t fragment_size,
488  const size_t total_subfragments,
489  const size_t first_subfragment_num,
490  const size_t last_subfragment_num)
491 {
492  assert(staging_memory_.size() >= total_subfragments * (sizeof(subfragment_identifier) + subfragment_size_));
493  assert(buffers.size() == 0);
494  assert(last_subfragment_num < total_subfragments);
495 
496  for (auto i_f = first_subfragment_num; i_f <= last_subfragment_num; ++i_f)
497  {
498  auto bytes_to_store = (i_f == total_subfragments - 1) ?
499  sizeof(subfragment_identifier) + (fragment_size - (total_subfragments - 1) * subfragment_size_) :
500  sizeof(subfragment_identifier) + subfragment_size_;
501 
502  buffers.emplace_back(&staging_memory_.at(i_f * (sizeof(subfragment_identifier) + subfragment_size_)),
503  bytes_to_store);
504  }
505 }
506 
507 
508 #pragma GCC diagnostic push // Needed since profile builds will ignore the assert
509 #pragma GCC diagnostic ignored "-Wunused-variable"
510 
511 void artdaq::MulticastTransfer::get_fragment_quantities(const boost::asio::mutable_buffer& buf, size_t& payload_size,
512  size_t& fragment_size,
513  size_t& expected_subfragments)
514 {
515  byte_t* buffer_ptr = boost::asio::buffer_cast<byte_t*>(buf);
516 
517  auto subfragment_num = *(reinterpret_cast<size_t*>(buffer_ptr) + 2);
518 
519  assert(subfragment_num == 0);
520 
521  artdaq::detail::RawFragmentHeader* header =
522  reinterpret_cast<artdaq::detail::RawFragmentHeader*>(buffer_ptr + sizeof(subfragment_identifier));
523 
524  fragment_size = header->word_count * sizeof(artdaq::RawDataType);
525 
526  auto metadata_size = header->metadata_word_count * sizeof(artdaq::RawDataType);
527  payload_size = fragment_size - metadata_size - artdaq::detail::RawFragmentHeader::num_words() *
528  sizeof(artdaq::RawDataType);
529 
530  assert(fragment_size ==
531  artdaq::detail::RawFragmentHeader::num_words() * sizeof(artdaq::RawDataType) +
532  metadata_size +
533  payload_size);
534 
535  expected_subfragments = static_cast<size_t>(std::ceil(fragment_size / static_cast<float>(subfragment_size_)));
536 }
537 #pragma GCC diagnostic pop
538 
539 void artdaq::MulticastTransfer::set_receive_buffer_size(size_t recv_buff_size)
540 {
541  if (recv_buff_size == 0) return;
542  boost::asio::socket_base::receive_buffer_size actual_recv_buff_size;
543  socket_->get_option(actual_recv_buff_size);
544 
545  TLOG(TLVL_DEBUG) << GetTraceName() << ": Receive buffer size is currently " << actual_recv_buff_size.value() <<
546  " bytes, will try to change it to " << recv_buff_size ;
547 
548  boost::asio::socket_base::receive_buffer_size recv_buff_option(recv_buff_size);
549 
550  boost::system::error_code ec;
551  socket_->set_option(recv_buff_option, ec);
552 
553  if (ec != 0)
554  {
555  std::cerr << "boost::system::error_code with value " << ec <<
556  " was found in attempt to change receive buffer" << std::endl;
557  }
558 
559  socket_->get_option(actual_recv_buff_size);
560  TLOG(TLVL_DEBUG) << GetTraceName() << ": After attempted change, receive buffer size is now " << actual_recv_buff_size.value() ;
561 }
562 
563 #pragma GCC diagnostic pop
564 
565 DEFINE_ARTDAQ_TRANSFER(artdaq::MulticastTransfer)
int receiveFragment(artdaq::Fragment &fragment, size_t receiveTimeout) override
Receive a Fragment using Multicast.
Value to be returned upon receive timeout.
Role role() const
Get the TransferInterface::Role of this TransferInterface.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
std::string GetTraceName() const
Constructs a name suitable for TRACE messages.
CopyStatus copyFragment(artdaq::Fragment &fragment, size_t send_timeout_usec) override
Copy a Fragment to the destination. Multicast is always unreliable.
MulticastTransfer is a TransferInterface implementation plugin that transfers data using Multicast...
This TransferInterface is a Receiver.
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
This TransferInterface is a Sender.
virtual ~MulticastTransfer()=default
Default destructor.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
MulticastTransfer(fhicl::ParameterSet const &ps, Role role)
MulticastTransfer Constructor.
CopyStatus moveFragment(artdaq::Fragment &&fragment) override
Move a Fragment to the destination. Multicast is always unreliable.
This interface defines the functions used to transfer data between artdaq applications.
artdaq::Fragment::byte_t byte_t
Copy Fragment::byte_t into local scope.
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.
const size_t max_fragment_size_words_
The maximum size of the transferred Fragment objects, in artdaq::Fragment::RawDataType words...