artdaq  v3_02_00
Multicast_transfer.cc
1 #define TRACE_NAME "MulticastTransfer"
2 
3 #include "artdaq/TransferPlugins/TransferInterface.hh"
4 
5 #include "artdaq-core/Data/Fragment.hh"
6 #include "artdaq-core/Utilities/ExceptionHandler.hh"
7 
8 #include "fhiclcpp/ParameterSet.h"
9 #include "cetlib_except/exception.h"
10 
11 #include <boost/asio.hpp>
12 #include <boost/bind.hpp>
13 
14 #include <iostream>
15 #include <vector>
16 #include <cassert>
17 #include <string>
18 #include <type_traits>
19 #include <bitset>
20 
21 #pragma GCC diagnostic push
22 #pragma GCC diagnostic ignored "-Wunused-parameter"
23 
24 
25 namespace artdaq
26 {
31  {
32  public:
33 
34  using byte_t = artdaq::Fragment::byte_t;
35 
39  virtual ~MulticastTransfer() = default;
40 
58  MulticastTransfer(fhicl::ParameterSet const& ps, Role role);
59 
66  int receiveFragment(artdaq::Fragment& fragment,
67  size_t receiveTimeout) override;
68 
75  int receiveFragmentHeader(detail::RawFragmentHeader& header, size_t receiveTimeout) override;
76 
83  int receiveFragmentData(RawDataType* destination, size_t wordCount) override;
84 
90  CopyStatus copyFragment(artdaq::Fragment& fragment, size_t send_timeout_usec) override;
91 
97  CopyStatus moveFragment(artdaq::Fragment&& fragment) override;
98 
103  bool isRunning() override { return socket_ != nullptr; }
104  private:
105 
106  void fill_staging_memory(const artdaq::Fragment& frag);
107 
108  template <typename T>
109  void book_container_of_buffers(std::vector<T>& buffers,
110  const size_t fragment_size,
111  const size_t total_subfragments,
112  const size_t first_subfragment_num,
113  const size_t last_subfragment_num);
114 
115  void get_fragment_quantities(const boost::asio::mutable_buffer& buf, size_t& payload_size, size_t& fragment_size,
116  size_t& expected_subfragments);
117 
118  void set_receive_buffer_size(size_t recv_buff_size);
119 
120  class subfragment_identifier
121  {
122  public:
123 
124  subfragment_identifier(size_t sequenceID, size_t fragmentID, size_t subfragment_number) :
125  sequenceID_(sequenceID)
126  , fragmentID_(fragmentID)
127  , subfragment_number_(subfragment_number) { }
128 
129  size_t sequenceID() const { return sequenceID_; }
130  size_t fragmentID() const { return fragmentID_; }
131  size_t subfragment_number() const { return subfragment_number_; }
132 
133  private:
134  size_t sequenceID_;
135  size_t fragmentID_;
136  size_t subfragment_number_;
137  };
138 
139  std::unique_ptr<boost::asio::io_service> io_service_;
140 
141  std::unique_ptr<boost::asio::ip::udp::endpoint> local_endpoint_;
142  std::unique_ptr<boost::asio::ip::udp::endpoint> multicast_endpoint_;
143  std::unique_ptr<boost::asio::ip::udp::endpoint> opposite_endpoint_;
144 
145  std::unique_ptr<boost::asio::ip::udp::socket> socket_;
146 
147  size_t subfragment_size_;
148  size_t subfragments_per_send_;
149 
150  size_t pause_on_copy_usecs_;
151  Fragment fragment_buffer_;
152 
153  std::vector<byte_t> staging_memory_;
154 
155  std::vector<boost::asio::mutable_buffer> receive_buffers_;
156  };
157 }
158 
159 artdaq::MulticastTransfer::MulticastTransfer(fhicl::ParameterSet const& pset, Role role) :
160  TransferInterface(pset, role)
161  , io_service_(std::make_unique<std::remove_reference<decltype(*io_service_)>::type>())
162  , local_endpoint_(nullptr)
163  , multicast_endpoint_(nullptr)
164  , opposite_endpoint_(std::make_unique<std::remove_reference<decltype(*opposite_endpoint_)>::type>())
165  , socket_(nullptr)
166  , subfragment_size_(pset.get<size_t>("subfragment_size"))
167  , subfragments_per_send_(pset.get<size_t>("subfragments_per_send"))
168  , pause_on_copy_usecs_(pset.get<size_t>("pause_on_copy_usecs", 0))
169 {
170  try
171  {
172  auto port = pset.get<unsigned short>("multicast_port");
173  auto multicast_address = boost::asio::ip::address::from_string(pset.get<std::string>("multicast_address"));
174  auto local_address = boost::asio::ip::address::from_string(pset.get<std::string>("local_address"));
175 
176  TLOG(TLVL_DEBUG) << GetTraceName() << ": multicast address is set to " << multicast_address ;
177  TLOG(TLVL_DEBUG) << GetTraceName() << ": local address is set to " << local_address ;
178 
180  {
181  local_endpoint_ = std::make_unique<std::remove_reference<decltype(*local_endpoint_)>::type>(local_address, 0);
182  multicast_endpoint_ = std::make_unique<std::remove_reference<decltype(*multicast_endpoint_)>::type>(multicast_address, port);
183 
184  socket_ = std::make_unique<std::remove_reference<decltype(*socket_)>::type>(*io_service_,
185  multicast_endpoint_->protocol());
186  socket_->bind(*local_endpoint_);
187  }
188  else
189  { // TransferInterface::role() == Role::kReceive
190 
191  // Create the socket so that multiple may be bound to the same address.
192 
193  local_endpoint_ = std::make_unique<std::remove_reference<decltype(*local_endpoint_)>::type>(local_address, port);
194  socket_ = std::make_unique<std::remove_reference<decltype(*socket_)>::type>(*io_service_,
195  local_endpoint_->protocol());
196 
197  boost::system::error_code ec;
198 
199  socket_->set_option(boost::asio::ip::udp::socket::reuse_address(true), ec);
200 
201  if (ec != 0)
202  {
203  std::cerr << "boost::system::error_code with value " << ec << " was found in setting reuse_address option" << std::endl;
204  }
205 
206  set_receive_buffer_size(pset.get<size_t>("receive_buffer_size", 0));
207 
208  socket_->bind(boost::asio::ip::udp::endpoint(multicast_address, port));
209 
210  // Join the multicast group.
211 
212  socket_->set_option(boost::asio::ip::multicast::join_group(multicast_address), ec);
213 
214  if (ec != 0)
215  {
216  std::cerr << "boost::system::error_code with value " << ec << " was found in attempt to join multicast group" << std::endl;
217  }
218  }
219  }
220  catch (...)
221  {
222  ExceptionHandler(ExceptionHandlerRethrow::yes, "Problem setting up the socket in MulticastTransfer");
223  }
224 
225  auto max_subfragments =
226  static_cast<size_t>(std::ceil(max_fragment_size_words_ / static_cast<float>(subfragment_size_)));
227 
228  staging_memory_.resize(max_subfragments * (sizeof(subfragment_identifier) + subfragment_size_));
229 
231  {
232  book_container_of_buffers(receive_buffers_, max_fragment_size_words_, max_subfragments, 0, max_subfragments - 1);
233  }
234 
235  TLOG(TLVL_DEBUG) << GetTraceName() << ": max_subfragments is " << max_subfragments ;
236  TLOG(TLVL_DEBUG) << GetTraceName() << ": Staging buffer size is " << staging_memory_.size() ;
237 }
238 
239 #pragma GCC diagnostic push
240 #pragma GCC diagnostic ignored "-Wunused-variable"
241 
242 int artdaq::MulticastTransfer::receiveFragment(artdaq::Fragment& fragment,
243  size_t receiveTimeout)
244 {
245  assert(TransferInterface::role() == Role::kReceive);
246 
247  if (fragment.dataSizeBytes() > 0)
248  {
249  throw cet::exception("MulticastTransfer") << "Error in MulticastTransfer::receiveFragmentFrom: " <<
250  "nonzero payload found in fragment passed as argument";
251  }
252 
253  static bool print_warning = true;
254 
255  if (print_warning)
256  {
257  std::cerr << "Please note that MulticastTransfer::receiveFragmentFrom does not use its receiveTimeout argument" << std::endl;
258  print_warning = false;
259  }
260 
261  fragment.resizeBytes(max_fragment_size_words_ - sizeof(artdaq::detail::RawFragmentHeader));
262 
263  static auto current_sequenceID = std::numeric_limits<Fragment::sequence_id_t>::max();
264  static auto current_fragmentID = std::numeric_limits<Fragment::fragment_id_t>::max();
265 
266  size_t fragment_size = 0;
267  size_t expected_subfragments = 0;
268  size_t current_subfragments = 0;
269  bool fragment_complete = false;
270  bool last_fragment_truncated = false;
271 
272  while (true)
273  {
274  auto bytes_received = socket_->receive_from(receive_buffers_, *opposite_endpoint_);
275 
276  size_t bytes_processed = 0;
277 
278  for (auto& buf : receive_buffers_)
279  {
280  auto buf_size = boost::asio::buffer_size(buf);
281  auto size_t_ptr = boost::asio::buffer_cast<const size_t*>(buf);
282  auto seqID = *size_t_ptr;
283  auto fragID = *(size_t_ptr + 1);
284  auto subfragID = *(size_t_ptr + 2);
285 
286  if (seqID != current_sequenceID || fragID != current_fragmentID)
287  {
288  // JCF, Jun-22-2016
289  // Code currently operates under the assumption that all subfragments from the call are from the same fragment
290 
291  assert(bytes_processed == 0);
292 
293  if (current_subfragments < expected_subfragments)
294  {
295  last_fragment_truncated = true;
296 
297  if (expected_subfragments != std::numeric_limits<size_t>::max())
298  {
299  std::cerr << "Warning: only received " << current_subfragments << " subfragments for fragment with seqID = " <<
300  current_sequenceID << ", fragID = " << current_fragmentID << " (expected " << expected_subfragments << ")\n"
301  << std::endl;
302  }
303  else
304  {
305  std::cerr << "Warning: only received " << current_subfragments <<
306  " subfragments for fragment with seqID = " <<
307  current_sequenceID << ", fragID = " << current_fragmentID <<
308  ", # of expected subfragments is unknown as fragment header was not received)\n"
309  << std::endl;
310  }
311  }
312 
313  current_subfragments = 0;
314  fragment_size = std::numeric_limits<size_t>::max();
315  expected_subfragments = std::numeric_limits<size_t>::max();
316  current_sequenceID = seqID;
317  current_fragmentID = fragID;
318  }
319 
320  auto ptr_into_fragment = fragment.headerBeginBytes() + subfragID * subfragment_size_;
321 
322  auto ptr_into_buffer = boost::asio::buffer_cast<const byte_t*>(buf) + sizeof(subfragment_identifier);
323 
324  std::copy(ptr_into_buffer, ptr_into_buffer + buf_size - sizeof(subfragment_identifier), ptr_into_fragment);
325 
326  if (subfragID == 0)
327  {
328  if (buf_size >= sizeof(subfragment_identifier) + sizeof(artdaq::detail::RawFragmentHeader))
329  {
330  auto payload_size = std::numeric_limits<size_t>::max();
331  get_fragment_quantities(buf, payload_size, fragment_size, expected_subfragments);
332 
333  fragment.resizeBytes(payload_size);
334  }
335  else
336  {
337  throw cet::exception("MulticastTransfer") << "Buffer size is too small to completely contain an artdaq::Fragment header; " <<
338  "please increase the default size";
339  }
340  }
341 
342  current_subfragments++;
343 
344  if (current_subfragments == expected_subfragments)
345  {
346  fragment_complete = true;
347  }
348 
349  bytes_processed += buf_size;
350 
351  if (bytes_processed >= bytes_received)
352  {
353  break;
354  }
355  }
356 
357  if (last_fragment_truncated)
358  {
359  // JCF, 7-7-2017
360 
361  // Don't yet have code to handle the scenario where the set of
362  // subfragments received in the last iteration of the loop was
363  // its own complete fragment, but we know the previous fragment
364  // to be incomplete
365 
366  assert(!fragment_complete);
367  TLOG(TLVL_WARNING) << GetTraceName() << ": Got an incomplete fragment" ;
369  }
370 
371  if (fragment_complete)
372  {
373  return source_rank();
374  }
375  }
376 
378 }
379 
380 #pragma GCC diagnostic pop
381 
382 int artdaq::MulticastTransfer::receiveFragmentHeader(detail::RawFragmentHeader& header, size_t receiveTimeout)
383 {
384  auto ret = receiveFragment(fragment_buffer_, receiveTimeout);
385  if (ret == source_rank())
386  {
387  header = *reinterpret_cast<detail::RawFragmentHeader*>(fragment_buffer_.headerAddress());
388  return source_rank();
389  }
390  return ret;
391 }
392 
393 int artdaq::MulticastTransfer::receiveFragmentData(RawDataType* destination, size_t wordCount)
394 {
395  if (fragment_buffer_.size() > detail::RawFragmentHeader::num_words()) {
396  auto dataSize = (fragment_buffer_.size() - detail::RawFragmentHeader::num_words()) * sizeof(RawDataType);
397  memcpy(destination, fragment_buffer_.headerAddress() + detail::RawFragmentHeader::num_words(), dataSize);
398  return source_rank();
399  }
400  return RECV_TIMEOUT;
401 }
402 
403 
404 // Reliable transport is undefined for multicast; just use copy
407 {
408  return copyFragment(f, 100000000);
409 }
410 
412 artdaq::MulticastTransfer::copyFragment(artdaq::Fragment& fragment,
413  size_t send_timeout_usec)
414 {
415  assert(TransferInterface::role() == Role::kSend);
416 
417  if (fragment.sizeBytes() > max_fragment_size_words_)
418  {
419  throw cet::exception("MulticastTransfer") << "Error in MulticastTransfer::copyFragmentTo: " <<
420  fragment.sizeBytes() << " byte fragment exceeds max_fragment_size of " << max_fragment_size_words_;
421  }
422 
423  static size_t ncalls = 1;
424  auto num_subfragments = static_cast<size_t>(std::ceil(fragment.sizeBytes() / static_cast<float>(subfragment_size_)));
425 
426  ncalls++;
427 
428  fill_staging_memory(fragment);
429 
430  for (size_t batch_index = 0; ; batch_index++)
431  {
432  auto first_subfragment = batch_index * subfragments_per_send_;
433  auto last_subfragment = (batch_index + 1) * subfragments_per_send_ >= num_subfragments ?
434  num_subfragments - 1 :
435  (batch_index + 1) * subfragments_per_send_ - 1;
436 
437  std::vector<boost::asio::const_buffer> buffers;
438 
439  book_container_of_buffers(buffers, fragment.sizeBytes(), num_subfragments, first_subfragment, last_subfragment);
440 
441  socket_->send_to(buffers, *multicast_endpoint_);
442 
443  usleep(pause_on_copy_usecs_);
444 
445  if (last_subfragment == num_subfragments - 1)
446  {
447  break;
448  }
449  }
450  return CopyStatus::kSuccess;
451 }
452 
453 #pragma GCC diagnostic push
454 #pragma GCC diagnostic ignored "-Wsign-compare"
455 
456 void artdaq::MulticastTransfer::fill_staging_memory(const artdaq::Fragment& fragment)
457 {
458  auto num_subfragments = static_cast<size_t>(std::ceil(fragment.sizeBytes() / static_cast<float>(subfragment_size_)));
459  TLOG(TLVL_DEBUG) << GetTraceName() << ": # of subfragments to use is " << num_subfragments ;
460 
461  for (auto i_s = 0; i_s < num_subfragments; ++i_s)
462  {
463  auto staging_memory_copyto = &staging_memory_.at(i_s * (sizeof(subfragment_identifier) + subfragment_size_));
464 
465  subfragment_identifier sfi(fragment.sequenceID(), fragment.fragmentID(), i_s);
466 
467  std::copy(reinterpret_cast<byte_t*>(&sfi),
468  reinterpret_cast<byte_t*>(&sfi) + sizeof(subfragment_identifier),
469  staging_memory_copyto);
470 
471  auto low_ptr_into_fragment = fragment.headerBeginBytes() + subfragment_size_ * i_s;
472 
473  auto high_ptr_into_fragment = (i_s == num_subfragments - 1) ?
474  fragment.dataEndBytes() :
475  fragment.headerBeginBytes() + subfragment_size_ * (i_s + 1);
476 
477  std::copy(low_ptr_into_fragment,
478  high_ptr_into_fragment,
479  staging_memory_copyto + sizeof(subfragment_identifier));
480  }
481 }
482 
483 #pragma GCC diagnostic pop
484 
485 // Note that book_container_of_buffers includes, rather than excludes,
486 // "last_subfragment_num"; in this regard it's different than the way
487 // STL functions receive iterators. Note also that the lowest possible
488 // value for "first_subfragment_num" is 0, not 1.
489 
490 template <typename T>
491 void artdaq::MulticastTransfer::book_container_of_buffers(std::vector<T>& buffers,
492  const size_t fragment_size,
493  const size_t total_subfragments,
494  const size_t first_subfragment_num,
495  const size_t last_subfragment_num)
496 {
497  assert(staging_memory_.size() >= total_subfragments * (sizeof(subfragment_identifier) + subfragment_size_));
498  assert(buffers.size() == 0);
499  assert(last_subfragment_num < total_subfragments);
500 
501  for (auto i_f = first_subfragment_num; i_f <= last_subfragment_num; ++i_f)
502  {
503  auto bytes_to_store = (i_f == total_subfragments - 1) ?
504  sizeof(subfragment_identifier) + (fragment_size - (total_subfragments - 1) * subfragment_size_) :
505  sizeof(subfragment_identifier) + subfragment_size_;
506 
507  buffers.emplace_back(&staging_memory_.at(i_f * (sizeof(subfragment_identifier) + subfragment_size_)),
508  bytes_to_store);
509  }
510 }
511 
512 
513 #pragma GCC diagnostic push // Needed since profile builds will ignore the assert
514 #pragma GCC diagnostic ignored "-Wunused-variable"
515 
516 void artdaq::MulticastTransfer::get_fragment_quantities(const boost::asio::mutable_buffer& buf, size_t& payload_size,
517  size_t& fragment_size,
518  size_t& expected_subfragments)
519 {
520  byte_t* buffer_ptr = boost::asio::buffer_cast<byte_t*>(buf);
521 
522  auto subfragment_num = *(reinterpret_cast<size_t*>(buffer_ptr) + 2);
523 
524  assert(subfragment_num == 0);
525 
526  artdaq::detail::RawFragmentHeader* header =
527  reinterpret_cast<artdaq::detail::RawFragmentHeader*>(buffer_ptr + sizeof(subfragment_identifier));
528 
529  fragment_size = header->word_count * sizeof(artdaq::RawDataType);
530 
531  auto metadata_size = header->metadata_word_count * sizeof(artdaq::RawDataType);
532  payload_size = fragment_size - metadata_size - artdaq::detail::RawFragmentHeader::num_words() *
533  sizeof(artdaq::RawDataType);
534 
535  assert(fragment_size ==
536  artdaq::detail::RawFragmentHeader::num_words() * sizeof(artdaq::RawDataType) +
537  metadata_size +
538  payload_size);
539 
540  expected_subfragments = static_cast<size_t>(std::ceil(fragment_size / static_cast<float>(subfragment_size_)));
541 }
542 #pragma GCC diagnostic pop
543 
544 void artdaq::MulticastTransfer::set_receive_buffer_size(size_t recv_buff_size)
545 {
546  if (recv_buff_size == 0) return;
547  boost::asio::socket_base::receive_buffer_size actual_recv_buff_size;
548  socket_->get_option(actual_recv_buff_size);
549 
550  TLOG(TLVL_DEBUG) << GetTraceName() << ": Receive buffer size is currently " << actual_recv_buff_size.value() <<
551  " bytes, will try to change it to " << recv_buff_size ;
552 
553  boost::asio::socket_base::receive_buffer_size recv_buff_option(recv_buff_size);
554 
555  boost::system::error_code ec;
556  socket_->set_option(recv_buff_option, ec);
557 
558  if (ec != 0)
559  {
560  std::cerr << "boost::system::error_code with value " << ec <<
561  " was found in attempt to change receive buffer" << std::endl;
562  }
563 
564  socket_->get_option(actual_recv_buff_size);
565  TLOG(TLVL_DEBUG) << GetTraceName() << ": After attempted change, receive buffer size is now " << actual_recv_buff_size.value() ;
566 }
567 
568 #pragma GCC diagnostic pop
569 
570 DEFINE_ARTDAQ_TRANSFER(artdaq::MulticastTransfer)
int receiveFragment(artdaq::Fragment &fragment, size_t receiveTimeout) override
Receive a Fragment using Multicast.
Value to be returned upon receive timeout.
Role role() const
Get the TransferInterface::Role of this TransferInterface.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
std::string GetTraceName() const
Constructs a name suitable for TRACE messages.
CopyStatus copyFragment(artdaq::Fragment &fragment, size_t send_timeout_usec) override
Copy a Fragment to the destination. Multicast is always unreliable.
MulticastTransfer is a TransferInterface implementation plugin that transfers data using Multicast...
This TransferInterface is a Receiver.
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
This TransferInterface is a Sender.
virtual ~MulticastTransfer()=default
Default destructor.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
MulticastTransfer(fhicl::ParameterSet const &ps, Role role)
MulticastTransfer Constructor.
CopyStatus moveFragment(artdaq::Fragment &&fragment) override
Move a Fragment to the destination. Multicast is always unreliable.
This interface defines the functions used to transfer data between artdaq applications.
artdaq::Fragment::byte_t byte_t
Copy Fragment::byte_t into local scope.
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.
const size_t max_fragment_size_words_
The maximum size of the transferred Fragment objects, in artdaq::Fragment::RawDataType words...