artdaq  3.12.07
Bundle_transfer.cc
1 #include <memory>
2 
3 #include "artdaq/DAQdata/Globals.hh"
4 #define TRACE_NAME (app_name + "_BundleTransfer").c_str()
5 
6 #include "artdaq-core/Data/ContainerFragmentLoader.hh"
7 #include "artdaq/TransferPlugins/TCPSocketTransfer.hh"
8 #include "artdaq/TransferPlugins/TransferInterface.hh"
9 
10 #include <boost/thread.hpp>
11 
12 namespace artdaq {
20 {
21 public:
27  BundleTransfer(const fhicl::ParameterSet& pset, Role role);
28 
32  ~BundleTransfer() override;
33 
40  int receiveFragment(artdaq::Fragment& fragment,
41  size_t receiveTimeout) override
42  {
43  if (bundle_fragment_ == nullptr)
44  {
45  receive_bundle_fragment_(receiveTimeout);
46  if (current_rank_ < RECV_SUCCESS) return current_rank_;
47  }
48 
49  ContainerFragment cf(*bundle_fragment_);
50  TLOG(TLVL_DEBUG + 32) << "Retrieving Fragment " << (current_block_index_ + 1) << " of " << cf.block_count();
51  fragment.resizeBytes(cf.fragSize(current_block_index_) - sizeof(detail::RawFragmentHeader));
52  memcpy(fragment.headerAddress(), static_cast<const uint8_t*>(cf.dataBegin()) + cf.fragmentIndex(current_block_index_), cf.fragSize(current_block_index_));
53  current_block_index_++;
54  if (current_block_index_ >= cf.block_count()) // Index vs. count!
55  {
56  bundle_fragment_.reset(nullptr);
57  }
58  return current_rank_;
59  }
60 
67  int receiveFragmentHeader(detail::RawFragmentHeader& header, size_t receiveTimeout) override
68  {
69  if (bundle_fragment_ == nullptr)
70  {
71  receive_bundle_fragment_(receiveTimeout);
72  if (current_rank_ < RECV_SUCCESS) return current_rank_;
73  }
74  ContainerFragment cf(*bundle_fragment_);
75  TLOG(TLVL_DEBUG + 32) << "Retrieving Fragment Header " << (current_block_index_ + 1) << " of " << cf.block_count();
76  memcpy(&header, static_cast<const uint8_t*>(cf.dataBegin()) + cf.fragmentIndex(current_block_index_), sizeof(detail::RawFragmentHeader));
77  return current_rank_;
78  }
79 
86  int receiveFragmentData(RawDataType* destination, size_t /*wordCount*/) override
87  {
88  if (bundle_fragment_ == nullptr) // Should be impossible!
89  {
90  return RECV_TIMEOUT;
91  }
92  ContainerFragment cf(*bundle_fragment_);
93  TLOG(TLVL_DEBUG + 32) << "Retrieving Fragment Data " << (current_block_index_ + 1) << " of " << cf.block_count();
94  memcpy(destination, static_cast<const uint8_t*>(cf.dataBegin()) + cf.fragmentIndex(current_block_index_) + sizeof(detail::RawFragmentHeader), cf.fragSize(current_block_index_) - sizeof(detail::RawFragmentHeader));
95  current_block_index_++;
96  if (current_block_index_ >= cf.block_count()) // Index vs. count!
97  {
98  bundle_fragment_.reset(nullptr);
99  }
100  return current_rank_;
101  }
102 
109  CopyStatus transfer_fragment_min_blocking_mode(artdaq::Fragment const& fragment, size_t send_timeout_usec) override
110  {
111  last_send_call_reliable_ = false;
112  {
113  std::lock_guard<std::mutex> lk(fragment_mutex_);
114  if (bundle_fragment_ == nullptr)
115  {
116  bundle_fragment_.reset(new artdaq::Fragment(fragment.sequenceID() + 1, fragment.fragmentID()));
117  bundle_fragment_->setTimestamp(fragment.timestamp());
118  bundle_fragment_->reserve(max_hold_size_bytes_ / sizeof(artdaq::RawDataType));
119  send_fragment_started_ = std::chrono::steady_clock::now();
120 
121  container_fragment_.reset(new ContainerFragmentLoader(*bundle_fragment_));
122  container_fragment_->set_missing_data(false); // Buffer mode is never missing data, even if there IS no data.
123  }
124 
125  // Eww, we have to copy
126  artdaq::Fragment frag(fragment);
127 
128  container_fragment_->addFragment(frag, true);
129  }
130  return send_bundle_fragment_(send_timeout_usec);
131  }
132 
138  CopyStatus transfer_fragment_reliable_mode(artdaq::Fragment&& fragment) override
139  {
140  last_send_call_reliable_ = true;
141  {
142  std::lock_guard<std::mutex> lk(fragment_mutex_);
143  if (bundle_fragment_ == nullptr)
144  {
145  bundle_fragment_.reset(new artdaq::Fragment(fragment.sequenceID() + 1, fragment.fragmentID()));
146  bundle_fragment_->setTimestamp(fragment.timestamp());
147  bundle_fragment_->reserve(max_hold_size_bytes_ / sizeof(artdaq::RawDataType));
148  send_fragment_started_ = std::chrono::steady_clock::now();
149 
150  container_fragment_.reset(new ContainerFragmentLoader(*bundle_fragment_));
151  container_fragment_->set_missing_data(false); // Buffer mode is never missing data, even if there IS no data.
152  }
153 
154  container_fragment_->addFragment(fragment, true);
155  }
156  return send_bundle_fragment_(0);
157  }
158 
163  bool isRunning() override { return running_; }
164 
169  void flush_buffers() override { theTransfer_->flush_buffers(); }
170 
171 private:
172  BundleTransfer(BundleTransfer const&) = delete;
173  BundleTransfer(BundleTransfer&&) = delete;
174  BundleTransfer& operator=(BundleTransfer const&) = delete;
175  BundleTransfer& operator=(BundleTransfer&&) = delete;
176 
177 private:
178  std::unique_ptr<TransferInterface> theTransfer_;
179  size_t max_hold_size_bytes_;
180  int max_hold_time_us_;
181  FragmentPtr bundle_fragment_{nullptr};
182  std::unique_ptr<ContainerFragmentLoader> container_fragment_{nullptr};
183  size_t current_block_index_{0};
184  int current_rank_ = 0;
185 
186  std::chrono::steady_clock::time_point send_fragment_started_;
187  std::unique_ptr<boost::thread> send_timeout_thread_;
188  std::atomic<bool> send_timeout_thread_running_{false};
189  std::atomic<bool> last_send_call_reliable_{true};
190  std::atomic<bool> running_{true};
191  std::mutex fragment_mutex_;
192 
193  void start_timeout_thread_();
194  void send_timeout_thread_proc_();
195  CopyStatus send_bundle_fragment_(size_t send_timeout_usec, bool forceSend = false);
196  void receive_bundle_fragment_(size_t receiveTimeout);
197 };
198 } // namespace artdaq
199 
200 artdaq::BundleTransfer::BundleTransfer(const fhicl::ParameterSet& pset, Role role)
201  : TransferInterface(pset, role)
202  , max_hold_size_bytes_(pset.get<size_t>("max_hold_size_bytes", 0x1000000)) // 16 MB
203  , max_hold_time_us_(pset.get<int>("max_hold_time_us", 250000))
204 {
205  TLOG(TLVL_INFO) << GetTraceName() << "Begin BundleTransfer constructor";
206  TLOG(TLVL_INFO) << GetTraceName() << "Constructing TCPSocketTransfer";
207  theTransfer_ = std::make_unique<TCPSocketTransfer>(pset, role);
208 
209  if (role == Role::kSend)
210  {
211  start_timeout_thread_();
212  }
213 }
214 
216 {
217  if (role_ == Role::kSend)
218  {
219  send_timeout_thread_running_ = false;
220  if (send_timeout_thread_ && send_timeout_thread_->joinable())
221  {
222  send_timeout_thread_->join();
223  }
224  send_bundle_fragment_(1000000, true);
225  }
226  running_ = false;
227 }
228 
229 void artdaq::BundleTransfer::start_timeout_thread_()
230 {
231  if (send_timeout_thread_ && send_timeout_thread_->joinable())
232  {
233  send_timeout_thread_->join();
234  }
235  send_timeout_thread_running_ = true;
236  TLOG(TLVL_INFO) << GetTraceName() << "Starting Send Timeout Thread";
237 
238  try
239  {
240  send_timeout_thread_ = std::make_unique<boost::thread>(&BundleTransfer::send_timeout_thread_proc_, this);
241  char tname[16]; // Size 16 - see man page pthread_setname_np(3) and/or prctl(2)
242  snprintf(tname, sizeof(tname) - 1, "%d-SNDTMO", my_rank); // NOLINT
243  tname[sizeof(tname) - 1] = '\0'; // assure term. snprintf is not too evil :)
244  auto handle = send_timeout_thread_->native_handle();
245  pthread_setname_np(handle, tname);
246  }
247  catch (const boost::exception& e)
248  {
249  TLOG(TLVL_ERROR) << "Caught boost::exception starting Send Timeout thread: " << boost::diagnostic_information(e) << ", errno=" << errno;
250  std::cerr << "Caught boost::exception starting Send Timeout thread: " << boost::diagnostic_information(e) << ", errno=" << errno << std::endl;
251  exit(5);
252  }
253 }
254 
255 void artdaq::BundleTransfer::send_timeout_thread_proc_()
256 {
257  while (send_timeout_thread_running_)
258  {
259  if (send_bundle_fragment_(1000000) != CopyStatus::kSuccess)
260  {
261  usleep(5000);
262  }
263  }
264 }
265 
266 artdaq::TransferInterface::CopyStatus artdaq::BundleTransfer::send_bundle_fragment_(size_t send_timeout_usec, bool forceSend)
267 {
268  CopyStatus sts = CopyStatus::kErrorNotRequiringException;
269  std::lock_guard<std::mutex> lk(fragment_mutex_);
270 
271  if (bundle_fragment_ == nullptr)
272  {
273  return sts;
274  }
275 
276  bool send_fragment = forceSend;
277  if (std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() - send_fragment_started_).count() >= max_hold_time_us_)
278  {
279  send_fragment = true;
280  }
281 
282  if (bundle_fragment_->dataSizeBytes() >= max_hold_size_bytes_)
283  {
284  send_fragment = true;
285  }
286 
287  if (send_fragment)
288  {
289  if (last_send_call_reliable_)
290  {
291  sts = theTransfer_->transfer_fragment_reliable_mode(std::move(*bundle_fragment_.get()));
292  bundle_fragment_.reset(nullptr);
293  }
294  else
295  {
296  while (sts != CopyStatus::kSuccess && send_timeout_thread_running_)
297  {
298  sts = theTransfer_->transfer_fragment_min_blocking_mode(*bundle_fragment_.get(), send_timeout_usec);
299  }
300  bundle_fragment_.reset(nullptr);
301  }
302  return sts; // Status of actual transfer
303  }
304 
305  return CopyStatus::kSuccess; // Waiting on more data
306 }
307 
308 void artdaq::BundleTransfer::receive_bundle_fragment_(size_t receiveTimeout)
309 {
310  std::lock_guard<std::mutex> lk(fragment_mutex_);
311  bundle_fragment_.reset(new artdaq::Fragment(1));
312 
313  TLOG(TLVL_DEBUG + 34) << "Going to receive next bundle fragment";
314  current_rank_ = theTransfer_->receiveFragment(*bundle_fragment_, receiveTimeout);
315  TLOG(TLVL_DEBUG + 34) << "Done with receiveFragment, current_rank_ = " << current_rank_;
316 
317  if (current_rank_ < RECV_SUCCESS)
318  {
319  bundle_fragment_.reset(nullptr);
320  }
321  current_block_index_ = 0;
322 }
323 
324 DEFINE_ARTDAQ_TRANSFER(artdaq::BundleTransfer)
The BundleTransfer TransferInterface plugin sets up a Shmem_transfer plugin or TCPSocket_transfer plu...
Role role() const
Get the TransferInterface::Role of this TransferInterface.
void flush_buffers() override
Flush any in-flight data. This should be used by the receiver after the receive loop has ended...
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
BundleTransfer(const fhicl::ParameterSet &pset, Role role)
BundleTransfer Constructor.
This TransferInterface is a Sender.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
This interface defines the functions used to transfer data between artdaq applications.
CopyStatus transfer_fragment_reliable_mode(artdaq::Fragment &&fragment) override
Send a Fragment in reliable mode, using the underlying transfer plugin.
int receiveFragmentData(RawDataType *destination, size_t) override
Receive the body of a Fragment to the given destination pointer.
CopyStatus transfer_fragment_min_blocking_mode(artdaq::Fragment const &fragment, size_t send_timeout_usec) override
Send a Fragment in non-reliable mode, using the underlying transfer plugin.
For code clarity, things checking for successful receive should check retval &gt;= NO_RANK_INFO.
int receiveFragment(artdaq::Fragment &fragment, size_t receiveTimeout) override
Receive a Fragment, using the underlying transfer plugin.
Value to be returned upon receive timeout.
~BundleTransfer() override
BundleTransfer default Destructor.
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.