artdaq  v3_07_01
Multicast_transfer.cc
1 #include "artdaq/DAQdata/Globals.hh"
2 #define TRACE_NAME (app_name + "_MulticastTransfer").c_str()
3 
4 #include "artdaq/TransferPlugins/TransferInterface.hh"
5 
6 #include "artdaq-core/Data/Fragment.hh"
7 #include "artdaq-core/Utilities/ExceptionHandler.hh"
8 
9 #include "cetlib_except/exception.h"
10 #include "fhiclcpp/ParameterSet.h"
11 
12 #include <boost/asio.hpp>
13 #include <boost/bind.hpp>
14 
15 #include <bitset>
16 #include <cassert>
17 #include <iostream>
18 #include <string>
19 #include <type_traits>
20 #include <vector>
21 
22 #pragma GCC diagnostic push
23 #pragma GCC diagnostic ignored "-Wunused-parameter"
24 
25 namespace artdaq {
30 {
31 public:
32  using byte_t = artdaq::Fragment::byte_t;
33 
37  virtual ~MulticastTransfer() = default;
38 
56  MulticastTransfer(fhicl::ParameterSet const& ps, Role role);
57 
64  int receiveFragment(artdaq::Fragment& fragment,
65  size_t receiveTimeout) override;
66 
73  int receiveFragmentHeader(detail::RawFragmentHeader& header, size_t receiveTimeout) override;
74 
81  int receiveFragmentData(RawDataType* destination, size_t wordCount) override;
82 
89  CopyStatus transfer_fragment_min_blocking_mode(artdaq::Fragment const& fragment, size_t send_timeout_usec) override;
90 
96  CopyStatus transfer_fragment_reliable_mode(artdaq::Fragment&& fragment) override;
97 
102  bool isRunning() override { return socket_ != nullptr; }
103 
108  void flush_buffers() override {}
109 
110 private:
111  void fill_staging_memory(const artdaq::Fragment& frag);
112 
113  template<typename T>
114  void book_container_of_buffers(std::vector<T>& buffers,
115  const size_t fragment_size,
116  const size_t total_subfragments,
117  const size_t first_subfragment_num,
118  const size_t last_subfragment_num);
119 
120  void get_fragment_quantities(const boost::asio::mutable_buffer& buf, size_t& payload_size, size_t& fragment_size,
121  size_t& expected_subfragments);
122 
123  void set_receive_buffer_size(size_t recv_buff_size);
124 
125  class subfragment_identifier
126  {
127  public:
128  subfragment_identifier(size_t sequenceID, size_t fragmentID, size_t subfragment_number)
129  : sequenceID_(sequenceID)
130  , fragmentID_(fragmentID)
131  , subfragment_number_(subfragment_number) {}
132 
133  size_t sequenceID() const { return sequenceID_; }
134  size_t fragmentID() const { return fragmentID_; }
135  size_t subfragment_number() const { return subfragment_number_; }
136 
137  private:
138  size_t sequenceID_;
139  size_t fragmentID_;
140  size_t subfragment_number_;
141  };
142 
143  std::unique_ptr<boost::asio::io_service> io_service_;
144 
145  std::unique_ptr<boost::asio::ip::udp::endpoint> local_endpoint_;
146  std::unique_ptr<boost::asio::ip::udp::endpoint> multicast_endpoint_;
147  std::unique_ptr<boost::asio::ip::udp::endpoint> opposite_endpoint_;
148 
149  std::unique_ptr<boost::asio::ip::udp::socket> socket_;
150 
151  size_t subfragment_size_;
152  size_t subfragments_per_send_;
153 
154  size_t pause_on_copy_usecs_;
155  Fragment fragment_buffer_;
156 
157  std::vector<byte_t> staging_memory_;
158 
159  std::vector<boost::asio::mutable_buffer> receive_buffers_;
160 };
161 } // namespace artdaq
162 
163 artdaq::MulticastTransfer::MulticastTransfer(fhicl::ParameterSet const& pset, Role role)
164  : TransferInterface(pset, role)
165  , io_service_(std::make_unique<std::remove_reference<decltype(*io_service_)>::type>())
166  , local_endpoint_(nullptr)
167  , multicast_endpoint_(nullptr)
168  , opposite_endpoint_(std::make_unique<std::remove_reference<decltype(*opposite_endpoint_)>::type>())
169  , socket_(nullptr)
170  , subfragment_size_(pset.get<size_t>("subfragment_size"))
171  , subfragments_per_send_(pset.get<size_t>("subfragments_per_send"))
172  , pause_on_copy_usecs_(pset.get<size_t>("pause_on_copy_usecs", 0))
173 {
174  try
175  {
176  portMan->UpdateConfiguration(pset);
177  auto port = portMan->GetMulticastTransferPort(source_rank());
178  auto multicast_address = boost::asio::ip::address::from_string(portMan->GetMulticastTransferGroupAddress());
179  auto local_address = boost::asio::ip::address::from_string(pset.get<std::string>("local_address"));
180 
181  TLOG(TLVL_DEBUG) << GetTraceName() << ": multicast address is set to " << multicast_address;
182  TLOG(TLVL_DEBUG) << GetTraceName() << ": local address is set to " << local_address;
183 
185  {
186  local_endpoint_ = std::make_unique<std::remove_reference<decltype(*local_endpoint_)>::type>(local_address, 0);
187  multicast_endpoint_ = std::make_unique<std::remove_reference<decltype(*multicast_endpoint_)>::type>(multicast_address, port);
188 
189  socket_ = std::make_unique<std::remove_reference<decltype(*socket_)>::type>(*io_service_,
190  multicast_endpoint_->protocol());
191  socket_->bind(*local_endpoint_);
192  }
193  else
194  { // TransferInterface::role() == Role::kReceive
195 
196  // Create the socket so that multiple may be bound to the same address.
197 
198  local_endpoint_ = std::make_unique<std::remove_reference<decltype(*local_endpoint_)>::type>(local_address, port);
199  socket_ = std::make_unique<std::remove_reference<decltype(*socket_)>::type>(*io_service_,
200  local_endpoint_->protocol());
201 
202  boost::system::error_code ec;
203 
204  socket_->set_option(boost::asio::ip::udp::socket::reuse_address(true), ec);
205 
206  if (ec.value() != 0)
207  {
208  std::cerr << "boost::system::error_code with value " << ec << " was found in setting reuse_address option" << std::endl;
209  }
210 
211  set_receive_buffer_size(pset.get<size_t>("receive_buffer_size", 0));
212 
213  socket_->bind(boost::asio::ip::udp::endpoint(multicast_address, port));
214 
215  // Join the multicast group.
216 
217  socket_->set_option(boost::asio::ip::multicast::join_group(multicast_address), ec);
218 
219  if (ec.value() != 0)
220  {
221  std::cerr << "boost::system::error_code with value " << ec << " was found in attempt to join multicast group" << std::endl;
222  }
223  }
224  }
225  catch (...)
226  {
227  ExceptionHandler(ExceptionHandlerRethrow::yes, "Problem setting up the socket in MulticastTransfer");
228  }
229 
230  auto max_subfragments =
231  static_cast<size_t>(std::ceil(max_fragment_size_words_ / static_cast<float>(subfragment_size_)));
232 
233  staging_memory_.resize(max_subfragments * (sizeof(subfragment_identifier) + subfragment_size_));
234 
236  {
237  book_container_of_buffers(receive_buffers_, max_fragment_size_words_, max_subfragments, 0, max_subfragments - 1);
238  }
239 
240  TLOG(TLVL_DEBUG) << GetTraceName() << ": max_subfragments is " << max_subfragments;
241  TLOG(TLVL_DEBUG) << GetTraceName() << ": Staging buffer size is " << staging_memory_.size();
242 }
243 
244 #pragma GCC diagnostic push
245 #pragma GCC diagnostic ignored "-Wunused-variable"
246 
247 int artdaq::MulticastTransfer::receiveFragment(artdaq::Fragment& fragment,
248  size_t receiveTimeout)
249 {
250  assert(TransferInterface::role() == Role::kReceive);
251 
252  if (fragment.dataSizeBytes() > 0)
253  {
254  throw cet::exception("MulticastTransfer") << "Error in MulticastTransfer::receiveFragmentFrom: "
255  << "nonzero payload found in fragment passed as argument";
256  }
257 
258  static bool print_warning = true;
259 
260  if (print_warning)
261  {
262  std::cerr << "Please note that MulticastTransfer::receiveFragmentFrom does not use its receiveTimeout argument" << std::endl;
263  print_warning = false;
264  }
265 
266  fragment.resizeBytes(max_fragment_size_words_ - sizeof(artdaq::detail::RawFragmentHeader));
267 
268  static auto current_sequenceID = std::numeric_limits<Fragment::sequence_id_t>::max();
269  static auto current_fragmentID = std::numeric_limits<Fragment::fragment_id_t>::max();
270 
271  size_t fragment_size = 0;
272  size_t expected_subfragments = 0;
273  size_t current_subfragments = 0;
274  bool fragment_complete = false;
275  bool last_fragment_truncated = false;
276 
277  while (true)
278  {
279  auto bytes_received = socket_->receive_from(receive_buffers_, *opposite_endpoint_);
280 
281  size_t bytes_processed = 0;
282 
283  for (auto& buf : receive_buffers_)
284  {
285  auto buf_size = boost::asio::buffer_size(buf);
286  auto size_t_ptr = boost::asio::buffer_cast<const size_t*>(buf);
287  auto seqID = *size_t_ptr;
288  auto fragID = *(size_t_ptr + 1);
289  auto subfragID = *(size_t_ptr + 2);
290 
291  if (seqID != current_sequenceID || fragID != current_fragmentID)
292  {
293  // JCF, Jun-22-2016
294  // Code currently operates under the assumption that all subfragments from the call are from the same fragment
295 
296  assert(bytes_processed == 0);
297 
298  if (current_subfragments < expected_subfragments)
299  {
300  last_fragment_truncated = true;
301 
302  if (expected_subfragments != std::numeric_limits<size_t>::max())
303  {
304  std::cerr << "Warning: only received " << current_subfragments << " subfragments for fragment with seqID = " << current_sequenceID << ", fragID = " << current_fragmentID << " (expected " << expected_subfragments << ")\n"
305  << std::endl;
306  }
307  else
308  {
309  std::cerr << "Warning: only received " << current_subfragments << " subfragments for fragment with seqID = " << current_sequenceID << ", fragID = " << current_fragmentID << ", # of expected subfragments is unknown as fragment header was not received)\n"
310  << std::endl;
311  }
312  }
313 
314  current_subfragments = 0;
315  fragment_size = std::numeric_limits<size_t>::max();
316  expected_subfragments = std::numeric_limits<size_t>::max();
317  current_sequenceID = seqID;
318  current_fragmentID = fragID;
319  }
320 
321  auto ptr_into_fragment = fragment.headerBeginBytes() + subfragID * subfragment_size_;
322 
323  auto ptr_into_buffer = boost::asio::buffer_cast<const byte_t*>(buf) + sizeof(subfragment_identifier);
324 
325  std::copy(ptr_into_buffer, ptr_into_buffer + buf_size - sizeof(subfragment_identifier), ptr_into_fragment);
326 
327  if (subfragID == 0)
328  {
329  if (buf_size >= sizeof(subfragment_identifier) + sizeof(artdaq::detail::RawFragmentHeader))
330  {
331  auto payload_size = std::numeric_limits<size_t>::max();
332  get_fragment_quantities(buf, payload_size, fragment_size, expected_subfragments);
333 
334  fragment.resizeBytes(payload_size);
335  }
336  else
337  {
338  throw cet::exception("MulticastTransfer") << "Buffer size is too small to completely contain an artdaq::Fragment header; "
339  << "please increase the default size";
340  }
341  }
342 
343  current_subfragments++;
344 
345  if (current_subfragments == expected_subfragments)
346  {
347  fragment_complete = true;
348  }
349 
350  bytes_processed += buf_size;
351 
352  if (bytes_processed >= bytes_received)
353  {
354  break;
355  }
356  }
357 
358  if (last_fragment_truncated)
359  {
360  // JCF, 7-7-2017
361 
362  // Don't yet have code to handle the scenario where the set of
363  // subfragments received in the last iteration of the loop was
364  // its own complete fragment, but we know the previous fragment
365  // to be incomplete
366 
367  assert(!fragment_complete);
368  TLOG(TLVL_WARNING) << GetTraceName() << ": Got an incomplete fragment";
370  }
371 
372  if (fragment_complete)
373  {
374  return source_rank();
375  }
376  }
377 
379 }
380 
381 #pragma GCC diagnostic pop
382 
383 int artdaq::MulticastTransfer::receiveFragmentHeader(detail::RawFragmentHeader& header, size_t receiveTimeout)
384 {
385  auto ret = receiveFragment(fragment_buffer_, receiveTimeout);
386  if (ret == source_rank())
387  {
388  header = *reinterpret_cast<detail::RawFragmentHeader*>(fragment_buffer_.headerAddress());
389  return source_rank();
390  }
391  return ret;
392 }
393 
394 int artdaq::MulticastTransfer::receiveFragmentData(RawDataType* destination, size_t wordCount)
395 {
396  if (fragment_buffer_.size() > detail::RawFragmentHeader::num_words())
397  {
398  auto dataSize = (fragment_buffer_.size() - detail::RawFragmentHeader::num_words()) * sizeof(RawDataType);
399  memcpy(destination, fragment_buffer_.headerAddress() + detail::RawFragmentHeader::num_words(), dataSize);
400  return source_rank();
401  }
402  return RECV_TIMEOUT;
403 }
404 
405 // Reliable transport is undefined for multicast; just use copy
408 {
409  return transfer_fragment_min_blocking_mode(f, 100000000);
410 }
411 
414  size_t send_timeout_usec)
415 {
416  assert(TransferInterface::role() == Role::kSend);
417 
418  if (fragment.sizeBytes() > max_fragment_size_words_)
419  {
420  throw cet::exception("MulticastTransfer") << "Error in MulticastTransfer::copyFragmentTo: " << fragment.sizeBytes() << " byte fragment exceeds max_fragment_size of " << max_fragment_size_words_;
421  }
422 
423  static size_t ncalls = 1;
424  auto num_subfragments = static_cast<size_t>(std::ceil(fragment.sizeBytes() / static_cast<float>(subfragment_size_)));
425 
426  ncalls++;
427 
428  fill_staging_memory(fragment);
429 
430  for (size_t batch_index = 0;; batch_index++)
431  {
432  auto first_subfragment = batch_index * subfragments_per_send_;
433  auto last_subfragment = (batch_index + 1) * subfragments_per_send_ >= num_subfragments ? num_subfragments - 1 : (batch_index + 1) * subfragments_per_send_ - 1;
434 
435  std::vector<boost::asio::const_buffer> buffers;
436 
437  book_container_of_buffers(buffers, fragment.sizeBytes(), num_subfragments, first_subfragment, last_subfragment);
438 
439  socket_->send_to(buffers, *multicast_endpoint_);
440 
441  usleep(pause_on_copy_usecs_);
442 
443  if (last_subfragment == num_subfragments - 1)
444  {
445  break;
446  }
447  }
448  return CopyStatus::kSuccess;
449 }
450 
451 #pragma GCC diagnostic push
452 #pragma GCC diagnostic ignored "-Wsign-compare"
453 
454 void artdaq::MulticastTransfer::fill_staging_memory(const artdaq::Fragment& fragment)
455 {
456  auto num_subfragments = static_cast<size_t>(std::ceil(fragment.sizeBytes() / static_cast<float>(subfragment_size_)));
457  TLOG(TLVL_DEBUG) << GetTraceName() << ": # of subfragments to use is " << num_subfragments;
458 
459  for (auto i_s = 0; i_s < num_subfragments; ++i_s)
460  {
461  auto staging_memory_copyto = &staging_memory_.at(i_s * (sizeof(subfragment_identifier) + subfragment_size_));
462 
463  subfragment_identifier sfi(fragment.sequenceID(), fragment.fragmentID(), i_s);
464 
465  std::copy(reinterpret_cast<byte_t*>(&sfi),
466  reinterpret_cast<byte_t*>(&sfi) + sizeof(subfragment_identifier),
467  staging_memory_copyto);
468 
469  auto low_ptr_into_fragment = fragment.headerBeginBytes() + subfragment_size_ * i_s;
470 
471  auto high_ptr_into_fragment = (i_s == num_subfragments - 1) ? fragment.dataEndBytes() : fragment.headerBeginBytes() + subfragment_size_ * (i_s + 1);
472 
473  std::copy(low_ptr_into_fragment,
474  high_ptr_into_fragment,
475  staging_memory_copyto + sizeof(subfragment_identifier));
476  }
477 }
478 
479 #pragma GCC diagnostic pop
480 
481 // Note that book_container_of_buffers includes, rather than excludes,
482 // "last_subfragment_num"; in this regard it's different than the way
483 // STL functions receive iterators. Note also that the lowest possible
484 // value for "first_subfragment_num" is 0, not 1.
485 
486 template<typename T>
487 void artdaq::MulticastTransfer::book_container_of_buffers(std::vector<T>& buffers,
488  const size_t fragment_size,
489  const size_t total_subfragments,
490  const size_t first_subfragment_num,
491  const size_t last_subfragment_num)
492 {
493  assert(staging_memory_.size() >= total_subfragments * (sizeof(subfragment_identifier) + subfragment_size_));
494  assert(buffers.size() == 0);
495  assert(last_subfragment_num < total_subfragments);
496 
497  for (auto i_f = first_subfragment_num; i_f <= last_subfragment_num; ++i_f)
498  {
499  auto bytes_to_store = (i_f == total_subfragments - 1) ? sizeof(subfragment_identifier) + (fragment_size - (total_subfragments - 1) * subfragment_size_) : sizeof(subfragment_identifier) + subfragment_size_;
500 
501  buffers.emplace_back(&staging_memory_.at(i_f * (sizeof(subfragment_identifier) + subfragment_size_)),
502  bytes_to_store);
503  }
504 }
505 
506 #pragma GCC diagnostic push // Needed since profile builds will ignore the assert
507 #pragma GCC diagnostic ignored "-Wunused-variable"
508 
509 void artdaq::MulticastTransfer::get_fragment_quantities(const boost::asio::mutable_buffer& buf, size_t& payload_size,
510  size_t& fragment_size,
511  size_t& expected_subfragments)
512 {
513  byte_t* buffer_ptr = boost::asio::buffer_cast<byte_t*>(buf);
514 
515  auto subfragment_num = *(reinterpret_cast<size_t*>(buffer_ptr) + 2);
516 
517  assert(subfragment_num == 0);
518 
519  artdaq::detail::RawFragmentHeader* header =
520  reinterpret_cast<artdaq::detail::RawFragmentHeader*>(buffer_ptr + sizeof(subfragment_identifier));
521 
522  fragment_size = header->word_count * sizeof(artdaq::RawDataType);
523 
524  auto metadata_size = header->metadata_word_count * sizeof(artdaq::RawDataType);
525  payload_size = fragment_size - metadata_size - artdaq::detail::RawFragmentHeader::num_words() * sizeof(artdaq::RawDataType);
526 
527  assert(fragment_size ==
528  artdaq::detail::RawFragmentHeader::num_words() * sizeof(artdaq::RawDataType) +
529  metadata_size +
530  payload_size);
531 
532  expected_subfragments = static_cast<size_t>(std::ceil(fragment_size / static_cast<float>(subfragment_size_)));
533 }
534 #pragma GCC diagnostic pop
535 
536 void artdaq::MulticastTransfer::set_receive_buffer_size(size_t recv_buff_size)
537 {
538  if (recv_buff_size == 0) return;
539  boost::asio::socket_base::receive_buffer_size actual_recv_buff_size;
540  socket_->get_option(actual_recv_buff_size);
541 
542  TLOG(TLVL_DEBUG) << GetTraceName() << ": Receive buffer size is currently " << actual_recv_buff_size.value() << " bytes, will try to change it to " << recv_buff_size;
543 
544  boost::asio::socket_base::receive_buffer_size recv_buff_option(recv_buff_size);
545 
546  boost::system::error_code ec;
547  socket_->set_option(recv_buff_option, ec);
548 
549  if (ec.value() != 0)
550  {
551  std::cerr << "boost::system::error_code with value " << ec << " was found in attempt to change receive buffer" << std::endl;
552  }
553 
554  socket_->get_option(actual_recv_buff_size);
555  TLOG(TLVL_DEBUG) << GetTraceName() << ": After attempted change, receive buffer size is now " << actual_recv_buff_size.value();
556 }
557 
558 #pragma GCC diagnostic pop
559 
560 DEFINE_ARTDAQ_TRANSFER(artdaq::MulticastTransfer)
virtual int source_rank() const
Get the source rank for this TransferInterface instance.
int receiveFragment(artdaq::Fragment &fragment, size_t receiveTimeout) override
Receive a Fragment using Multicast.
void flush_buffers() override
Flush any in-flight data. This should be used by the receiver after the receive loop has ended...
Role role() const
Get the TransferInterface::Role of this TransferInterface.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
CopyStatus transfer_fragment_min_blocking_mode(artdaq::Fragment const &fragment, size_t send_timeout_usec) override
Copy a Fragment to the destination. Multicast is always unreliable.
MulticastTransfer is a TransferInterface implementation plugin that transfers data using Multicast...
This TransferInterface is a Receiver.
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
This TransferInterface is a Sender.
virtual ~MulticastTransfer()=default
Default destructor.
CopyStatus transfer_fragment_reliable_mode(artdaq::Fragment &&fragment) override
Move a Fragment to the destination. Multicast is always unreliable.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
MulticastTransfer(fhicl::ParameterSet const &ps, Role role)
MulticastTransfer Constructor.
This interface defines the functions used to transfer data between artdaq applications.
artdaq::Fragment::byte_t byte_t
Copy Fragment::byte_t into local scope.
Value to be returned upon receive timeout.
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.
const size_t max_fragment_size_words_
The maximum size of the transferred Fragment objects, in artdaq::Fragment::RawDataType words...