artdaq  v2_03_00
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Pages
Multicast_transfer.cc
1 #include "artdaq/TransferPlugins/TransferInterface.hh"
2 
3 #include "artdaq-core/Data/Fragment.hh"
4 #include "artdaq-core/Utilities/ExceptionHandler.hh"
5 
6 #include "fhiclcpp/ParameterSet.h"
7 
8 #include <boost/asio.hpp>
9 #include "boost/bind.hpp"
10 
11 #include <iostream>
12 #include <vector>
13 #include <cassert>
14 #include <string>
15 #include <type_traits>
16 #include <bitset>
17 
18 #pragma GCC diagnostic push
19 #pragma GCC diagnostic ignored "-Wunused-parameter"
20 
21 
22 namespace artdaq
23 {
28  {
29  public:
30 
31  using byte_t = artdaq::Fragment::byte_t;
32 
36  virtual ~MulticastTransfer() = default;
37 
55  MulticastTransfer(fhicl::ParameterSet const& ps, Role role);
56 
63  int receiveFragment(artdaq::Fragment& fragment,
64  size_t receiveTimeout) override;
65 
72  CopyStatus copyFragment(artdaq::Fragment& fragment,
73  size_t send_timeout_usec = std::numeric_limits<size_t>::max()) override;
74 
81  CopyStatus moveFragment(artdaq::Fragment&& fragment,
82  size_t send_timeout_usec = std::numeric_limits<size_t>::max()) override;
83 
84  private:
85 
86  void fill_staging_memory(const artdaq::Fragment& frag);
87 
88  template <typename T>
89  void book_container_of_buffers(std::vector<T>& buffers,
90  const size_t fragment_size,
91  const size_t total_subfragments,
92  const size_t first_subfragment_num,
93  const size_t last_subfragment_num);
94 
95  void get_fragment_quantities(const boost::asio::mutable_buffer& buf, size_t& payload_size, size_t& fragment_size,
96  size_t& expected_subfragments);
97 
98  void set_receive_buffer_size(size_t recv_buff_size);
99 
100  class subfragment_identifier
101  {
102  public:
103 
104  subfragment_identifier(size_t sequenceID, size_t fragmentID, size_t subfragment_number) :
105  sequenceID_(sequenceID)
106  , fragmentID_(fragmentID)
107  , subfragment_number_(subfragment_number) { }
108 
109  size_t sequenceID() const { return sequenceID_; }
110  size_t fragmentID() const { return fragmentID_; }
111  size_t subfragment_number() const { return subfragment_number_; }
112 
113  private:
114  size_t sequenceID_;
115  size_t fragmentID_;
116  size_t subfragment_number_;
117  };
118 
119  std::unique_ptr<boost::asio::io_service> io_service_;
120 
121  std::unique_ptr<boost::asio::ip::udp::endpoint> local_endpoint_;
122  std::unique_ptr<boost::asio::ip::udp::endpoint> multicast_endpoint_;
123  std::unique_ptr<boost::asio::ip::udp::endpoint> opposite_endpoint_;
124 
125  std::unique_ptr<boost::asio::ip::udp::socket> socket_;
126 
127  size_t subfragment_size_;
128  size_t subfragments_per_send_;
129 
130  size_t pause_on_copy_usecs_;
131 
132  std::vector<byte_t> staging_memory_;
133 
134  std::vector<boost::asio::mutable_buffer> receive_buffers_;
135  };
136 }
137 
138 artdaq::MulticastTransfer::MulticastTransfer(fhicl::ParameterSet const& pset, Role role) :
139  TransferInterface(pset, role)
140  , io_service_(std::make_unique<std::remove_reference<decltype(*io_service_)>::type>())
141  , local_endpoint_(nullptr)
142  , multicast_endpoint_(nullptr)
143  , opposite_endpoint_(std::make_unique<std::remove_reference<decltype(*opposite_endpoint_)>::type>())
144  , socket_(nullptr)
145  , subfragment_size_(pset.get<size_t>("subfragment_size"))
146  , subfragments_per_send_(pset.get<size_t>("subfragments_per_send"))
147  , pause_on_copy_usecs_(pset.get<size_t>("pause_on_copy_usecs", 0))
148 {
149  try
150  {
151  auto port = pset.get<unsigned short>("multicast_port");
152  auto multicast_address = boost::asio::ip::address::from_string(pset.get<std::string>("multicast_address"));
153  auto local_address = boost::asio::ip::address::from_string(pset.get<std::string>("local_address"));
154 
155  TLOG_DEBUG(uniqueLabel()) << "multicast address is set to " << multicast_address << TLOG_ENDL;
156  TLOG_DEBUG(uniqueLabel()) << "local address is set to " << local_address << TLOG_ENDL;
157 
159  {
160  local_endpoint_ = std::make_unique<std::remove_reference<decltype(*local_endpoint_)>::type>(local_address, 0);
161  multicast_endpoint_ = std::make_unique<std::remove_reference<decltype(*multicast_endpoint_)>::type>(multicast_address, port);
162 
163  socket_ = std::make_unique<std::remove_reference<decltype(*socket_)>::type>(*io_service_,
164  multicast_endpoint_->protocol());
165  socket_->bind(*local_endpoint_);
166  }
167  else
168  { // TransferInterface::role() == Role::kReceive
169 
170  // Create the socket so that multiple may be bound to the same address.
171 
172  local_endpoint_ = std::make_unique<std::remove_reference<decltype(*local_endpoint_)>::type>(local_address, port);
173  socket_ = std::make_unique<std::remove_reference<decltype(*socket_)>::type>(*io_service_,
174  local_endpoint_->protocol());
175 
176  boost::system::error_code ec;
177 
178  socket_->set_option(boost::asio::ip::udp::socket::reuse_address(true), ec);
179 
180  if (ec != 0)
181  {
182  std::cerr << "boost::system::error_code with value " << ec << " was found in setting reuse_address option" << std::endl;
183  }
184 
185  set_receive_buffer_size(pset.get<size_t>("receive_buffer_size", 0));
186 
187  socket_->bind(boost::asio::ip::udp::endpoint(multicast_address, port));
188 
189  // Join the multicast group.
190 
191  socket_->set_option(boost::asio::ip::multicast::join_group(multicast_address), ec);
192 
193  if (ec != 0)
194  {
195  std::cerr << "boost::system::error_code with value " << ec << " was found in attempt to join multicast group" << std::endl;
196  }
197  }
198  }
199  catch (...)
200  {
201  ExceptionHandler(ExceptionHandlerRethrow::yes, "Problem setting up the socket in MulticastTransfer");
202  }
203 
204  auto max_subfragments =
205  static_cast<size_t>(std::ceil(max_fragment_size_words_ / static_cast<float>(subfragment_size_)));
206 
207  staging_memory_.resize(max_subfragments * (sizeof(subfragment_identifier) + subfragment_size_));
208 
210  {
211  book_container_of_buffers(receive_buffers_, max_fragment_size_words_, max_subfragments, 0, max_subfragments - 1);
212  }
213 
214  TLOG_DEBUG(uniqueLabel()) << "max_subfragments is " << max_subfragments << TLOG_ENDL;
215  TLOG_DEBUG(uniqueLabel()) << "Staging buffer size is " << staging_memory_.size() << TLOG_ENDL;
216 }
217 
218 #pragma GCC diagnostic push
219 #pragma GCC diagnostic ignored "-Wunused-variable"
220 
221 int artdaq::MulticastTransfer::receiveFragment(artdaq::Fragment& fragment,
222  size_t receiveTimeout)
223 {
224  assert(TransferInterface::role() == Role::kReceive);
225 
226  if (fragment.dataSizeBytes() > 0)
227  {
228  throw cet::exception("MulticastTransfer") << "Error in MulticastTransfer::receiveFragmentFrom: " <<
229  "nonzero payload found in fragment passed as argument";
230  }
231 
232  static bool print_warning = true;
233 
234  if (print_warning)
235  {
236  std::cerr << "Please note that MulticastTransfer::receiveFragmentFrom does not use its receiveTimeout argument" << std::endl;
237  print_warning = false;
238  }
239 
240  fragment.resizeBytes(max_fragment_size_words_ - sizeof(artdaq::detail::RawFragmentHeader));
241 
242  static auto current_sequenceID = std::numeric_limits<Fragment::sequence_id_t>::max();
243  static auto current_fragmentID = std::numeric_limits<Fragment::fragment_id_t>::max();
244 
245  size_t fragment_size = 0;
246  size_t expected_subfragments = 0;
247  size_t current_subfragments = 0;
248  bool fragment_complete = false;
249  bool last_fragment_truncated = false;
250 
251  while (true)
252  {
253  auto bytes_received = socket_->receive_from(receive_buffers_, *opposite_endpoint_);
254 
255  size_t bytes_processed = 0;
256 
257  for (auto& buf : receive_buffers_)
258  {
259  auto buf_size = boost::asio::buffer_size(buf);
260  auto size_t_ptr = boost::asio::buffer_cast<const size_t*>(buf);
261  auto seqID = *size_t_ptr;
262  auto fragID = *(size_t_ptr + 1);
263  auto subfragID = *(size_t_ptr + 2);
264 
265  if (seqID != current_sequenceID || fragID != current_fragmentID)
266  {
267  // JCF, Jun-22-2016
268  // Code currently operates under the assumption that all subfragments from the call are from the same fragment
269 
270  assert(bytes_processed == 0);
271 
272  if (current_subfragments < expected_subfragments)
273  {
274  last_fragment_truncated = true;
275 
276  if (expected_subfragments != std::numeric_limits<size_t>::max())
277  {
278  std::cerr << "Warning: only received " << current_subfragments << " subfragments for fragment with seqID = " <<
279  current_sequenceID << ", fragID = " << current_fragmentID << " (expected " << expected_subfragments << ")\n"
280  << std::endl;
281  }
282  else
283  {
284  std::cerr << "Warning: only received " << current_subfragments <<
285  " subfragments for fragment with seqID = " <<
286  current_sequenceID << ", fragID = " << current_fragmentID <<
287  ", # of expected subfragments is unknown as fragment header was not received)\n"
288  << std::endl;
289  }
290  }
291 
292  current_subfragments = 0;
293  fragment_size = std::numeric_limits<size_t>::max();
294  expected_subfragments = std::numeric_limits<size_t>::max();
295  current_sequenceID = seqID;
296  current_fragmentID = fragID;
297  }
298 
299  auto ptr_into_fragment = fragment.headerBeginBytes() + subfragID * subfragment_size_;
300 
301  auto ptr_into_buffer = boost::asio::buffer_cast<const byte_t*>(buf) + sizeof(subfragment_identifier);
302 
303  std::copy(ptr_into_buffer, ptr_into_buffer + buf_size - sizeof(subfragment_identifier), ptr_into_fragment);
304 
305  if (subfragID == 0)
306  {
307  if (buf_size >= sizeof(subfragment_identifier) + sizeof(artdaq::detail::RawFragmentHeader))
308  {
309  auto payload_size = std::numeric_limits<size_t>::max();
310  get_fragment_quantities(buf, payload_size, fragment_size, expected_subfragments);
311 
312  fragment.resizeBytes(payload_size);
313  }
314  else
315  {
316  throw cet::exception("MulticastTransfer") << "Buffer size is too small to completely contain an artdaq::Fragment header; " <<
317  "please increase the default size";
318  }
319  }
320 
321  current_subfragments++;
322 
323  if (current_subfragments == expected_subfragments)
324  {
325  fragment_complete = true;
326  }
327 
328  bytes_processed += buf_size;
329 
330  if (bytes_processed >= bytes_received)
331  {
332  break;
333  }
334  }
335 
336  if (last_fragment_truncated)
337  {
338  // JCF, 7-7-2017
339 
340  // Don't yet have code to handle the scenario where the set of
341  // subfragments received in the last iteration of the loop was
342  // its own complete fragment, but we know the previous fragment
343  // to be incomplete
344 
345  assert( !fragment_complete );
346  TLOG_WARNING(uniqueLabel()) << "Got an incomplete fragment" << TLOG_ENDL;
348  }
349 
350  if (fragment_complete)
351  {
352  return source_rank();
353  }
354  }
355 
357 }
358 
359 #pragma GCC diagnostic pop
360 
361 // Reliable transport is undefined for multicast; just use copy
363 artdaq::MulticastTransfer::moveFragment(artdaq::Fragment&& f, size_t tmo)
364 {
365  return copyFragment(f, tmo);
366 }
367 
369 artdaq::MulticastTransfer::copyFragment(artdaq::Fragment& fragment,
370  size_t send_timeout_usec)
371 {
372  assert(TransferInterface::role() == Role::kSend);
373 
374  if (fragment.sizeBytes() > max_fragment_size_words_)
375  {
376  throw cet::exception("MulticastTransfer") << "Error in MulticastTransfer::copyFragmentTo: " <<
377  fragment.sizeBytes() << " byte fragment exceeds max_fragment_size of " << max_fragment_size_words_;
378  }
379 
380  static size_t ncalls = 1;
381  auto num_subfragments = static_cast<size_t>(std::ceil(fragment.sizeBytes() / static_cast<float>(subfragment_size_)));
382 
383  ncalls++;
384 
385  fill_staging_memory(fragment);
386 
387  for (size_t batch_index = 0; ; batch_index++)
388  {
389  auto first_subfragment = batch_index * subfragments_per_send_;
390  auto last_subfragment = (batch_index + 1) * subfragments_per_send_ >= num_subfragments ?
391  num_subfragments - 1 :
392  (batch_index + 1) * subfragments_per_send_ - 1;
393 
394  std::vector<boost::asio::const_buffer> buffers;
395 
396  book_container_of_buffers(buffers, fragment.sizeBytes(), num_subfragments, first_subfragment, last_subfragment);
397 
398  socket_->send_to(buffers, *multicast_endpoint_);
399 
400  usleep(pause_on_copy_usecs_);
401 
402  if (last_subfragment == num_subfragments - 1)
403  {
404  break;
405  }
406  }
407  return CopyStatus::kSuccess;
408 }
409 
410 #pragma GCC diagnostic push
411 #pragma GCC diagnostic ignored "-Wsign-compare"
412 
413 void artdaq::MulticastTransfer::fill_staging_memory(const artdaq::Fragment& fragment)
414 {
415  auto num_subfragments = static_cast<size_t>(std::ceil(fragment.sizeBytes() / static_cast<float>(subfragment_size_)));
416  TLOG_DEBUG(uniqueLabel()) << "# of subfragments to use is " << num_subfragments << TLOG_ENDL;
417 
418  for (auto i_s = 0; i_s < num_subfragments; ++i_s)
419  {
420  auto staging_memory_copyto = &staging_memory_.at(i_s * (sizeof(subfragment_identifier) + subfragment_size_));
421 
422  subfragment_identifier sfi(fragment.sequenceID(), fragment.fragmentID(), i_s);
423 
424  std::copy(reinterpret_cast<byte_t*>(&sfi),
425  reinterpret_cast<byte_t*>(&sfi) + sizeof(subfragment_identifier),
426  staging_memory_copyto);
427 
428  auto low_ptr_into_fragment = fragment.headerBeginBytes() + subfragment_size_ * i_s;
429 
430  auto high_ptr_into_fragment = (i_s == num_subfragments - 1) ?
431  fragment.dataEndBytes() :
432  fragment.headerBeginBytes() + subfragment_size_ * (i_s + 1);
433 
434  std::copy(low_ptr_into_fragment,
435  high_ptr_into_fragment,
436  staging_memory_copyto + sizeof(subfragment_identifier));
437  }
438 }
439 
440 #pragma GCC diagnostic pop
441 
442 // Note that book_container_of_buffers includes, rather than excludes,
443 // "last_subfragment_num"; in this regard it's different than the way
444 // STL functions receive iterators. Note also that the lowest possible
445 // value for "first_subfragment_num" is 0, not 1.
446 
447 template <typename T>
448 void artdaq::MulticastTransfer::book_container_of_buffers(std::vector<T>& buffers,
449  const size_t fragment_size,
450  const size_t total_subfragments,
451  const size_t first_subfragment_num,
452  const size_t last_subfragment_num)
453 {
454  assert(staging_memory_.size() >= total_subfragments * (sizeof(subfragment_identifier) + subfragment_size_) );
455  assert(buffers.size() == 0);
456  assert(last_subfragment_num < total_subfragments);
457 
458  for (auto i_f = first_subfragment_num; i_f <= last_subfragment_num; ++i_f)
459  {
460  auto bytes_to_store = (i_f == total_subfragments - 1) ?
461  sizeof(subfragment_identifier) + (fragment_size - (total_subfragments - 1) * subfragment_size_) :
462  sizeof(subfragment_identifier) + subfragment_size_;
463 
464  buffers.emplace_back(&staging_memory_.at(i_f * (sizeof(subfragment_identifier) + subfragment_size_)),
465  bytes_to_store);
466  }
467 }
468 
469 
470 #pragma GCC diagnostic push // Needed since profile builds will ignore the assert
471 #pragma GCC diagnostic ignored "-Wunused-variable"
472 
473 void artdaq::MulticastTransfer::get_fragment_quantities(const boost::asio::mutable_buffer& buf, size_t& payload_size,
474  size_t& fragment_size,
475  size_t& expected_subfragments)
476 {
477  byte_t* buffer_ptr = boost::asio::buffer_cast<byte_t*>(buf);
478 
479  auto subfragment_num = *(reinterpret_cast<size_t*>(buffer_ptr) + 2);
480 
481  assert( subfragment_num == 0 );
482 
483  artdaq::detail::RawFragmentHeader* header =
484  reinterpret_cast<artdaq::detail::RawFragmentHeader*>(buffer_ptr + sizeof(subfragment_identifier));
485 
486  fragment_size = header->word_count * sizeof(artdaq::RawDataType);
487 
488  auto metadata_size = header->metadata_word_count * sizeof(artdaq::RawDataType);
489  payload_size = fragment_size - metadata_size - artdaq::detail::RawFragmentHeader::num_words() *
490  sizeof(artdaq::RawDataType);
491 
492  assert(fragment_size ==
493  artdaq::detail::RawFragmentHeader::num_words() * sizeof(artdaq::RawDataType) +
494  metadata_size +
495  payload_size);
496 
497  expected_subfragments = static_cast<size_t>(std::ceil(fragment_size / static_cast<float>(subfragment_size_)));
498 }
499 #pragma GCC diagnostic pop
500 
501 void artdaq::MulticastTransfer::set_receive_buffer_size(size_t recv_buff_size)
502 {
503  if (recv_buff_size == 0) return;
504  boost::asio::socket_base::receive_buffer_size actual_recv_buff_size;
505  socket_->get_option(actual_recv_buff_size);
506 
507  TLOG_DEBUG(uniqueLabel()) << "Receive buffer size is currently " << actual_recv_buff_size.value() <<
508  " bytes, will try to change it to " << recv_buff_size << TLOG_ENDL;
509 
510  boost::asio::socket_base::receive_buffer_size recv_buff_option(recv_buff_size);
511 
512  boost::system::error_code ec;
513  socket_->set_option(recv_buff_option, ec);
514 
515  if (ec != 0)
516  {
517  std::cerr << "boost::system::error_code with value " << ec <<
518  " was found in attempt to change receive buffer" << std::endl;
519  }
520 
521  socket_->get_option(actual_recv_buff_size);
522  TLOG_DEBUG(uniqueLabel()) << "After attempted change, receive buffer size is now " << actual_recv_buff_size.value() << TLOG_ENDL;
523 }
524 
525 
526 #pragma GCC diagnostic pop
527 
528 DEFINE_ARTDAQ_TRANSFER(artdaq::MulticastTransfer)
int receiveFragment(artdaq::Fragment &fragment, size_t receiveTimeout) override
Receive a Fragment using Multicast.
Role role() const
Get the TransferInterface::Role of this TransferInterface.
MulticastTransfer is a TransferInterface implementation plugin that transfers data using Multicast...
static const int RECV_TIMEOUT
Value to be returned upon receive timeout. Because receivers otherwise return rank, this is also the limit on the number of ranks that artdaq currently supports.
This TransferInterface is a Receiver.
CopyStatus copyFragment(artdaq::Fragment &fragment, size_t send_timeout_usec=std::numeric_limits< size_t >::max()) override
Copy a Fragment to the destination. Multicast is always unreliable.
This TransferInterface is a Sender.
virtual ~MulticastTransfer()=default
Default destructor.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
MulticastTransfer(fhicl::ParameterSet const &ps, Role role)
MulticastTransfer Constructor.
std::string uniqueLabel() const
Get the unique label of this TransferInterface instance.
This interface defines the functions used to transfer data between artdaq applications.
artdaq::Fragment::byte_t byte_t
Copy Fragment::byte_t into local scope.
CopyStatus moveFragment(artdaq::Fragment &&fragment, size_t send_timeout_usec=std::numeric_limits< size_t >::max()) override
Move a Fragment to the destination. Multicast is always unreliable.
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.
const size_t max_fragment_size_words_
The maximum size of the transferred Fragment objects, in artdaq::Fragment::RawDataType words...