artdaq  v3_07_02
CommandableFragmentGenerator.cc
1 #include "artdaq/DAQdata/Globals.hh"
2 #define TRACE_NAME (app_name + "_CommandableFragmentGenerator").c_str() // include these 2 first -
3 
4 #include "artdaq/Generators/CommandableFragmentGenerator.hh"
5 
6 #include <boost/exception/all.hpp>
7 #include <boost/throw_exception.hpp>
8 
9 #include <iterator>
10 #include <limits>
11 
12 #include "canvas/Utilities/Exception.h"
13 #include "cetlib_except/exception.h"
14 #include "fhiclcpp/ParameterSet.h"
15 
16 #include "artdaq-core/Data/ContainerFragmentLoader.hh"
17 #include "artdaq-core/Data/Fragment.hh"
18 #include "artdaq-core/Utilities/ExceptionHandler.hh"
19 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh"
20 #include "artdaq-core/Utilities/TimeUtils.hh"
21 
22 #include <sys/poll.h>
23 #include <algorithm>
24 #include <fstream>
25 #include <iomanip>
26 #include <iostream>
27 #include <iterator>
29 
30 #define TLVL_GETNEXT 10
31 #define TLVL_GETNEXT_VERBOSE 20
32 #define TLVL_CHECKSTOP 11
33 #define TLVL_EVCOUNTERINC 12
34 #define TLVL_GETDATALOOP 13
35 #define TLVL_GETDATALOOP_DATABUFFWAIT 21
36 #define TLVL_GETDATALOOP_VERBOSE 20
37 #define TLVL_WAITFORBUFFERREADY 15
38 #define TLVL_GETBUFFERSTATS 16
39 #define TLVL_CHECKDATABUFFER 17
40 #define TLVL_GETMONITORINGDATA 18
41 #define TLVL_APPLYREQUESTS 9
42 #define TLVL_SENDEMPTYFRAGMENTS 19
43 #define TLVL_CHECKWINDOWS 14
44 
46  : mutex_()
47  , requestReceiver_(nullptr)
48  , bufferModeKeepLatest_(ps.get<bool>("buffer_mode_keep_latest", false))
49  , windowOffset_(ps.get<Fragment::timestamp_t>("request_window_offset", 0))
50  , windowWidth_(ps.get<Fragment::timestamp_t>("request_window_width", 0))
51  , staleTimeout_(ps.get<Fragment::timestamp_t>("stale_request_timeout", 0xFFFFFFFF))
52  , expectedType_(ps.get<Fragment::type_t>("expected_fragment_type", Fragment::type_t(Fragment::EmptyFragmentType)))
53  , uniqueWindows_(ps.get<bool>("request_windows_are_unique", true))
54  , missing_request_window_timeout_us_(ps.get<size_t>("missing_request_window_timeout_us", 5000000))
55  , window_close_timeout_us_(ps.get<size_t>("window_close_timeout_us", 2000000))
56  , useDataThread_(ps.get<bool>("separate_data_thread", false))
57  , circularDataBufferMode_(ps.get<bool>("circular_buffer_mode", false))
58  , sleep_on_no_data_us_(ps.get<size_t>("sleep_on_no_data_us", 0))
59  , data_thread_running_(false)
60  , maxDataBufferDepthFragments_(ps.get<int>("data_buffer_depth_fragments", 1000))
61  , maxDataBufferDepthBytes_(ps.get<size_t>("data_buffer_depth_mb", 1000) * 1024 * 1024)
62  , useMonitoringThread_(ps.get<bool>("separate_monitoring_thread", false))
63  , monitoringInterval_(ps.get<int64_t>("hardware_poll_interval_us", 0))
64  , lastMonitoringCall_()
65  , isHardwareOK_(true)
66  , run_number_(-1)
67  , subrun_number_(-1)
68  , timeout_(std::numeric_limits<uint64_t>::max())
69  , timestamp_(std::numeric_limits<uint64_t>::max())
70  , should_stop_(false)
71  , exception_(false)
72  , force_stop_(false)
73  , latest_exception_report_("none")
74  , ev_counter_(1)
75  , board_id_(-1)
76  , sleep_on_stop_us_(0)
77 {
78  board_id_ = ps.get<int>("board_id");
79  instance_name_for_metrics_ = "BoardReader." + boost::lexical_cast<std::string>(board_id_);
80 
81  auto fragment_ids = ps.get<std::vector<artdaq::Fragment::fragment_id_t>>("fragment_ids", std::vector<artdaq::Fragment::fragment_id_t>());
82 
83  TLOG(TLVL_TRACE) << "artdaq::CommandableFragmentGenerator::CommandableFragmentGenerator(ps)";
84  int fragment_id = ps.get<int>("fragment_id", -99);
85 
86  if (fragment_id != -99)
87  {
88  if (fragment_ids.size() != 0)
89  {
90  latest_exception_report_ = "Error in CommandableFragmentGenerator: can't both define \"fragment_id\" and \"fragment_ids\" in FHiCL document";
91  throw cet::exception(latest_exception_report_);
92  }
93  else
94  {
95  fragment_ids.emplace_back(fragment_id);
96  }
97  }
98 
99  for (auto& id : fragment_ids)
100  {
101  dataBuffers_[id].DataBufferDepthBytes = 0;
102  dataBuffers_[id].DataBufferDepthFragments = 0;
103  dataBuffers_[id].HighestRequestSeen = 0;
104  dataBuffers_[id].BufferFragmentKept = false;
105  dataBuffers_[id].DataBuffer.emplace_back(FragmentPtr(new Fragment()));
106  (*dataBuffers_[id].DataBuffer.begin())->setSystemType(Fragment::EmptyFragmentType);
107  }
108 
109  sleep_on_stop_us_ = ps.get<int>("sleep_on_stop_us", 0);
110 
111  std::string modeString = ps.get<std::string>("request_mode", "ignored");
112  if (modeString == "single" || modeString == "Single")
113  {
114  mode_ = RequestMode::Single;
115  }
116  else if (modeString.find("buffer") != std::string::npos || modeString.find("Buffer") != std::string::npos)
117  {
118  mode_ = RequestMode::Buffer;
119  }
120  else if (modeString == "window" || modeString == "Window")
121  {
122  mode_ = RequestMode::Window;
123  }
124  else if (modeString.find("ignore") != std::string::npos || modeString.find("Ignore") != std::string::npos)
125  {
126  mode_ = RequestMode::Ignored;
127  }
128  else if (modeString.find("sequence") != std::string::npos || modeString.find("Sequence") != std::string::npos)
129  {
130  mode_ = RequestMode::SequenceID;
131  }
132  TLOG(TLVL_DEBUG) << "Request mode is " << printMode_();
133 
134  if (mode_ != RequestMode::Ignored)
135  {
136  if (!useDataThread_)
137  {
138  latest_exception_report_ = "Error in CommandableFragmentGenerator: use_data_thread must be true when request_mode is not \"Ignored\"!";
139  throw cet::exception(latest_exception_report_);
140  }
141  requestReceiver_.reset(new RequestReceiver(ps));
142  }
143 }
144 
146 {
147  joinThreads();
148  requestReceiver_.reset(nullptr);
149 }
150 
152 {
153  should_stop_ = true;
154  force_stop_ = true;
155  TLOG(TLVL_DEBUG) << "Joining dataThread";
156  if (dataThread_.joinable()) dataThread_.join();
157  TLOG(TLVL_DEBUG) << "Joining monitoringThread";
158  if (monitoringThread_.joinable()) monitoringThread_.join();
159  TLOG(TLVL_DEBUG) << "joinThreads complete";
160 }
161 
163 {
164  bool result = true;
165 
166  if (check_stop()) usleep(sleep_on_stop_us_);
167  if (exception() || force_stop_) return false;
168 
169  if (!useMonitoringThread_ && monitoringInterval_ > 0)
170  {
171  TLOG(TLVL_GETNEXT) << "getNext: Checking whether to collect Monitoring Data";
172  auto now = std::chrono::steady_clock::now();
173 
174  if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
175  {
176  TLOG(TLVL_GETNEXT) << "getNext: Collecting Monitoring Data";
177  isHardwareOK_ = checkHWStatus_();
178  TLOG(TLVL_GETNEXT) << "getNext: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_;
179  lastMonitoringCall_ = now;
180  }
181  }
182 
183  try
184  {
185  std::lock_guard<std::mutex> lk(mutex_);
186  if (useDataThread_)
187  {
188  TLOG(TLVL_TRACE) << "getNext: Calling applyRequests";
189  result = applyRequests(output);
190  TLOG(TLVL_TRACE) << "getNext: Done with applyRequests result=" << std::boolalpha << result;
191  for (auto dataIter = output.begin(); dataIter != output.end(); ++dataIter)
192  {
193  TLOG(20) << "getNext: applyRequests() returned fragment with sequenceID = " << (*dataIter)->sequenceID()
194  << ", type = " << (*dataIter)->typeString() << ", id = " << std::to_string((*dataIter)->fragmentID())
195  << ", timestamp = " << (*dataIter)->timestamp() << ", and sizeBytes = " << (*dataIter)->sizeBytes();
196  }
197 
198  if (exception())
199  {
200  TLOG(TLVL_ERROR) << "Exception found in BoardReader with board ID " << board_id() << "; BoardReader will now return error status when queried";
201  throw cet::exception("CommandableFragmentGenerator") << "Exception found in BoardReader with board ID " << board_id() << "; BoardReader will now return error status when queried";
202  }
203  }
204  else
205  {
206  if (!isHardwareOK_)
207  {
208  TLOG(TLVL_ERROR) << "Stopping CFG because the hardware reports bad status!";
209  return false;
210  }
211  TLOG(TLVL_TRACE) << "getNext: Calling getNext_ w/ ev_counter()=" << ev_counter();
212  try
213  {
214  result = getNext_(output);
215  }
216  catch (...)
217  {
218  throw;
219  }
220  TLOG(TLVL_TRACE) << "getNext: Done with getNext_ - ev_counter() now " << ev_counter();
221  for (auto dataIter = output.begin(); dataIter != output.end(); ++dataIter)
222  {
223  TLOG(TLVL_GETNEXT_VERBOSE) << "getNext: getNext_() returned fragment with sequenceID = " << (*dataIter)->sequenceID()
224  << ", type = " << (*dataIter)->typeString() << ", id = " << std::to_string((*dataIter)->fragmentID())
225  << ", timestamp = " << (*dataIter)->timestamp() << ", and sizeBytes = " << (*dataIter)->sizeBytes();
226  }
227  }
228  }
229  catch (const cet::exception& e)
230  {
231  latest_exception_report_ = "cet::exception caught in getNext(): ";
232  latest_exception_report_.append(e.what());
233  TLOG(TLVL_ERROR) << "getNext: cet::exception caught: " << e;
234  set_exception(true);
235  return false;
236  }
237  catch (const boost::exception& e)
238  {
239  latest_exception_report_ = "boost::exception caught in getNext(): ";
240  latest_exception_report_.append(boost::diagnostic_information(e));
241  TLOG(TLVL_ERROR) << "getNext: boost::exception caught: " << boost::diagnostic_information(e);
242  set_exception(true);
243  return false;
244  }
245  catch (const std::exception& e)
246  {
247  latest_exception_report_ = "std::exception caught in getNext(): ";
248  latest_exception_report_.append(e.what());
249  TLOG(TLVL_ERROR) << "getNext: std::exception caught: " << e.what();
250  set_exception(true);
251  return false;
252  }
253  catch (...)
254  {
255  latest_exception_report_ = "Unknown exception caught in getNext().";
256  TLOG(TLVL_ERROR) << "getNext: unknown exception caught";
257  set_exception(true);
258  return false;
259  }
260 
261  if (!result)
262  {
263  TLOG(TLVL_DEBUG) << "getNext: Either getNext_ or applyRequests returned false, stopping";
264  }
265 
266  if (metricMan && !output.empty())
267  {
268  auto timestamp = output.front()->timestamp();
269 
270  if (output.size() > 1)
271  { // Only bother sorting if >1 entry
272  for (auto& outputfrag : output)
273  {
274  if (outputfrag->timestamp() > timestamp)
275  {
276  timestamp = outputfrag->timestamp();
277  }
278  }
279  }
280 
281  metricMan->sendMetric("Last Timestamp", timestamp, "Ticks", 1, MetricMode::LastPoint);
282  }
283 
284  return result;
285 }
286 
288 {
289  TLOG(TLVL_CHECKSTOP) << "CFG::check_stop: should_stop=" << should_stop() << ", useDataThread_=" << useDataThread_ << ", exception status =" << int(exception());
290 
291  if (!should_stop()) return false;
292  if (!useDataThread_ || mode_ == RequestMode::Ignored) return true;
293  if (force_stop_) return true;
294 
295  // check_stop returns true if the CFG should stop. We should wait for the RequestReceiver to stop before stopping.
296  TLOG(TLVL_DEBUG) << "should_stop is true, force_stop_ is false, requestReceiver_->isRunning() is " << std::boolalpha << requestReceiver_->isRunning();
297  return !requestReceiver_->isRunning();
298 }
299 
301 {
302  if (force || mode_ == RequestMode::Ignored || mode_ == RequestMode::SequenceID)
303  {
304  TLOG(TLVL_EVCOUNTERINC) << "ev_counter_inc: Incrementing ev_counter from " << ev_counter() << " by " << step;
305  return ev_counter_.fetch_add(step);
306  }
307  return ev_counter_.load();
308 } // returns the prev value
309 
310 void artdaq::CommandableFragmentGenerator::StartCmd(int run, uint64_t timeout, uint64_t timestamp)
311 {
312  TLOG(TLVL_TRACE) << "Start Command received.";
313  if (run < 0) throw cet::exception("CommandableFragmentGenerator") << "negative run number";
314 
315  timeout_ = timeout;
316  timestamp_ = timestamp;
317  ev_counter_.store(1);
318  {
319  std::unique_lock<std::mutex> lk(dataBuffersMutex_);
320  for (auto& id : dataBuffers_)
321  {
322  id.second.DataBufferDepthBytes = 0;
323  id.second.DataBufferDepthFragments = 0;
324  id.second.HighestRequestSeen = 0;
325  id.second.BufferFragmentKept = false;
326  id.second.DataBuffer.clear();
327  id.second.WindowsSent.clear();
328  }
329  }
330 
331  should_stop_.store(false);
332  force_stop_.store(false);
333  exception_.store(false);
334  run_number_ = run;
335  subrun_number_ = 1;
336  latest_exception_report_ = "none";
337 
338  start();
339 
340  std::unique_lock<std::mutex> lk(mutex_);
341  if (useDataThread_) startDataThread();
342  if (useMonitoringThread_) startMonitoringThread();
343  if (mode_ != RequestMode::Ignored)
344  {
345  requestReceiver_->SetRunNumber(static_cast<uint32_t>(run));
346  requestReceiver_->startRequestReception();
347  }
348  TLOG(TLVL_TRACE) << "Start Command complete.";
349 }
350 
351 void artdaq::CommandableFragmentGenerator::StopCmd(uint64_t timeout, uint64_t timestamp)
352 {
353  TLOG(TLVL_TRACE) << "Stop Command received.";
354 
355  timeout_ = timeout;
356  timestamp_ = timestamp;
357  if (requestReceiver_)
358  {
359  TLOG(TLVL_DEBUG) << "Stopping Request reception BEGIN";
360  requestReceiver_->stopRequestReception();
361  TLOG(TLVL_DEBUG) << "Stopping Request reception END";
362  }
363 
364  stopNoMutex();
365  should_stop_.store(true);
366  std::unique_lock<std::mutex> lk(mutex_);
367  stop();
368 
369  joinThreads();
370  TLOG(TLVL_TRACE) << "Stop Command complete.";
371 }
372 
373 void artdaq::CommandableFragmentGenerator::PauseCmd(uint64_t timeout, uint64_t timestamp)
374 {
375  TLOG(TLVL_TRACE) << "Pause Command received.";
376  timeout_ = timeout;
377  timestamp_ = timestamp;
378  //if (requestReceiver_->isRunning()) requestReceiver_->stopRequestReceiverThread();
379 
380  pauseNoMutex();
381  should_stop_.store(true);
382  std::unique_lock<std::mutex> lk(mutex_);
383 
384  pause();
385 }
386 
387 void artdaq::CommandableFragmentGenerator::ResumeCmd(uint64_t timeout, uint64_t timestamp)
388 {
389  TLOG(TLVL_TRACE) << "Resume Command received.";
390  timeout_ = timeout;
391  timestamp_ = timestamp;
392 
393  subrun_number_ += 1;
394  should_stop_ = false;
395 
396  {
397  std::unique_lock<std::mutex> lk(dataBuffersMutex_);
398  for (auto& id : dataBuffers_)
399  {
400  id.second.DataBufferDepthBytes = 0;
401  id.second.DataBufferDepthFragments = 0;
402  id.second.BufferFragmentKept = false;
403  id.second.DataBuffer.clear();
404  }
405  }
406  // no lock required: thread not started yet
407  resume();
408 
409  std::unique_lock<std::mutex> lk(mutex_);
410  //if (useDataThread_) startDataThread();
411  //if (useMonitoringThread_) startMonitoringThread();
412  //if (mode_ != RequestMode::Ignored && !requestReceiver_->isRunning()) requestReceiver_->startRequestReceiverThread();
413  TLOG(TLVL_TRACE) << "Resume Command complete.";
414 }
415 
416 std::string artdaq::CommandableFragmentGenerator::ReportCmd(std::string const& which)
417 {
418  TLOG(TLVL_TRACE) << "Report Command received.";
419  std::lock_guard<std::mutex> lk(mutex_);
420 
421  // 14-May-2015, KAB: please see the comments associated with the report()
422  // methods in the CommandableFragmentGenerator.hh file for more information
423  // on the use of those methods in this method.
424 
425  // check if the child class has something meaningful for this request
426  std::string childReport = reportSpecific(which);
427  if (childReport.length() > 0) { return childReport; }
428 
429  // handle the requests that we can take care of at this level
430  if (which == "latest_exception")
431  {
432  return latest_exception_report_;
433  }
434 
435  // check if the child class has provided a catch-all report function
436  childReport = report();
437  if (childReport.length() > 0) { return childReport; }
438 
439  // ELF: 5/31/2019: Let BoardReaderCore's report handle this...
440  /*
441  // if we haven't been able to come up with any report so far, say so
442  std::string tmpString = "The \"" + which + "\" command is not ";
443  tmpString.append("currently supported by the ");
444  tmpString.append(metricsReportingInstanceName());
445  tmpString.append(" fragment generator.");
446  */
447  TLOG(TLVL_TRACE) << "Report Command complete.";
448  return ""; //tmpString;
449 }
450 
451 // Default implemenetations of state functions
453 {
454 #pragma message "Using default implementation of CommandableFragmentGenerator::pauseNoMutex()"
455 }
456 
458 {
459 #pragma message "Using default implementation of CommandableFragmentGenerator::pause()"
460 }
461 
463 #pragma message "Using default implementation of CommandableFragmentGenerator::resume()"
464 }
465 
467 {
468 #pragma message "Using default implementation of CommandableFragmentGenerator::report()"
469  return "";
470 }
471 
473 {
474 #pragma message "Using default implementation of CommandableFragmentGenerator::reportSpecific(std::string)"
475  return "";
476 }
477 
479 {
480 #pragma message "Using default implementation of CommandableFragmentGenerator::checkHWStatus_()"
481  return true;
482 }
483 
484 bool artdaq::CommandableFragmentGenerator::metaCommand(std::string const&, std::string const&)
485 {
486 #pragma message "Using default implementation of CommandableFragmentGenerator::metaCommand(std::string, std::string)"
487  return true;
488 }
489 
491 {
492  if (dataThread_.joinable()) dataThread_.join();
493  TLOG(TLVL_INFO) << "Starting Data Receiver Thread";
494  try
495  {
496  dataThread_ = boost::thread(&CommandableFragmentGenerator::getDataLoop, this);
497  }
498  catch (const boost::exception& e)
499  {
500  TLOG(TLVL_ERROR) << "Caught boost::exception starting Data Receiver thread: " << boost::diagnostic_information(e) << ", errno=" << errno;
501  std::cerr << "Caught boost::exception starting Data Receiver thread: " << boost::diagnostic_information(e) << ", errno=" << errno << std::endl;
502  exit(5);
503  }
504 }
505 
507 {
508  if (monitoringThread_.joinable()) monitoringThread_.join();
509  TLOG(TLVL_INFO) << "Starting Hardware Monitoring Thread";
510  try
511  {
512  monitoringThread_ = boost::thread(&CommandableFragmentGenerator::getMonitoringDataLoop, this);
513  }
514  catch (const boost::exception& e)
515  {
516  TLOG(TLVL_ERROR) << "Caught boost::exception starting Hardware Monitoring thread: " << boost::diagnostic_information(e) << ", errno=" << errno;
517  std::cerr << "Caught boost::exception starting Hardware Monitoring thread: " << boost::diagnostic_information(e) << ", errno=" << errno << std::endl;
518  exit(5);
519  }
520 }
521 
523 {
524  switch (mode_)
525  {
526  case RequestMode::Single:
527  return "Single";
528  case RequestMode::Buffer:
529  return "Buffer";
530  case RequestMode::Window:
531  return "Window";
532  case RequestMode::Ignored:
533  return "Ignored";
534  case RequestMode::SequenceID:
535  return "SequenceID";
536  }
537 
538  return "ERROR";
539 }
540 
541 //
542 // The "useDataThread_" thread
543 //
545 {
546  data_thread_running_ = true;
547  while (!force_stop_)
548  {
549  if (!isHardwareOK_)
550  {
551  TLOG(TLVL_DEBUG) << "getDataLoop: isHardwareOK is " << isHardwareOK_ << ", aborting data thread";
552  data_thread_running_ = false;
553  return;
554  }
555 
556  TLOG(TLVL_GETDATALOOP) << "getDataLoop: calling getNext_";
557 
558  bool data = false;
559  auto startdata = std::chrono::steady_clock::now();
560  FragmentPtrs newData;
561 
562  try
563  {
564  data = getNext_(newData);
565  }
566  catch (...)
567  {
568  ExceptionHandler(ExceptionHandlerRethrow::no,
569  "Exception thrown by fragment generator in CommandableFragmentGenerator::getDataLoop; setting exception state to \"true\"");
570  set_exception(true);
571 
572  data_thread_running_ = false;
573  return;
574  }
575 
576  if (metricMan)
577  {
578  metricMan->sendMetric("Avg Data Acquisition Time", TimeUtils::GetElapsedTime(startdata), "s", 3, artdaq::MetricMode::Average);
579  }
580 
581  if (newData.size() == 0 && sleep_on_no_data_us_ > 0)
582  {
583  usleep(sleep_on_no_data_us_);
584  }
585 
586  auto dataIter = newData.begin();
587  while (dataIter != newData.end())
588  {
589  TLOG(TLVL_GETDATALOOP_VERBOSE) << "getDataLoop: getNext_() returned fragment with timestamp = " << (*dataIter)->timestamp() << ", and sizeBytes = " << (*dataIter)->sizeBytes();
590 
591  auto frag_id = (*dataIter)->fragmentID();
592  if (!dataBuffers_.count(frag_id))
593  {
594  throw cet::exception("DataBufferError") << "Error in CommandableFragmentGenerator: Recevied Fragment with fragment_id " << frag_id << ", but this ID was not declared in fragment_ids!";
595  }
596 
597  TLOG(TLVL_GETDATALOOP_DATABUFFWAIT) << "Waiting for data buffer ready";
598  if (!waitForDataBufferReady(frag_id)) return;
599  TLOG(TLVL_GETDATALOOP_DATABUFFWAIT) << "Done waiting for data buffer ready";
600 
601  TLOG(TLVL_GETDATALOOP) << "getDataLoop: processing data";
602  if (data && !force_stop_)
603  {
604  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
605  switch (mode_)
606  {
607  case RequestMode::Single:
608  dataBuffers_[frag_id].DataBuffer.clear();
609  dataBuffers_[frag_id].DataBufferDepthBytes = (*dataIter)->sizeBytes();
610  dataBuffers_[frag_id].DataBuffer.emplace_back(std::move(*dataIter));
611  dataIter = newData.erase(dataIter);
612  break;
613  case RequestMode::Buffer:
614  case RequestMode::Ignored:
615  case RequestMode::Window:
616  case RequestMode::SequenceID:
617  default:
618  //dataBuffer_.reserve(dataBuffer_.size() + newDataBuffer_.size());
619  dataBuffers_[frag_id].DataBufferDepthBytes += (*dataIter)->sizeBytes();
620  dataBuffers_[frag_id].DataBuffer.emplace_back(std::move(*dataIter));
621  dataIter = newData.erase(dataIter);
622  break;
623  }
624  getDataBufferStats(frag_id);
625  }
626  else
627  {
628  break;
629  }
630  }
631 
632  {
633  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
634  for (auto& id : dataBuffers_)
635  {
636  if (id.second.DataBuffer.size() > 0)
637  {
638  dataCondition_.notify_all();
639  break;
640  }
641  }
642  }
643  if (!data || force_stop_)
644  {
645  TLOG(TLVL_INFO) << "Data flow has stopped. Ending data collection thread";
646  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
647  data_thread_running_ = false;
648  if (requestReceiver_) requestReceiver_->ClearRequests();
649  newData.clear();
650  TLOG(TLVL_INFO) << "getDataLoop: Ending thread";
651  return;
652  }
653  }
654 }
655 
657 {
658  if (!dataBuffers_.count(id))
659  {
660  throw cet::exception("DataBufferError") << "Error in CommandableFragmentGenerator: Cannot wait for data buffer for ID " << id << " because it does not exist!";
661  }
662  auto startwait = std::chrono::steady_clock::now();
663  auto first = true;
664  auto lastwaittime = 0ULL;
665 
666  {
667  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
668  getDataBufferStats(id);
669  }
670 
671  while (dataBufferIsTooLarge(id))
672  {
673  if (!circularDataBufferMode_)
674  {
675  if (should_stop())
676  {
677  TLOG(TLVL_DEBUG) << "Run ended while waiting for buffer to shrink!";
678  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
679  getDataBufferStats(id);
680  dataCondition_.notify_all();
681  data_thread_running_ = false;
682  return false;
683  }
684  auto waittime = TimeUtils::GetElapsedTimeMilliseconds(startwait);
685 
686  if (first || (waittime != lastwaittime && waittime % 1000 == 0))
687  {
688  TLOG(TLVL_WARNING) << "Bad Omen: Data Buffer has exceeded its size limits. "
689  << "(seq_id=" << ev_counter() << ", frag_id=" << id
690  << ", frags=" << dataBuffers_[id].DataBufferDepthFragments << "/" << maxDataBufferDepthFragments_
691  << ", szB=" << dataBuffers_[id].DataBufferDepthBytes << "/" << maxDataBufferDepthBytes_ << ")";
692  TLOG(TLVL_TRACE) << "Bad Omen: Possible causes include requests not getting through or Ignored-mode BR issues";
693  first = false;
694  }
695  if (waittime % 5 && waittime != lastwaittime)
696  {
697  TLOG(TLVL_WAITFORBUFFERREADY) << "getDataLoop: Data Retreival paused for " << waittime << " ms waiting for data buffer to drain";
698  }
699  lastwaittime = waittime;
700  usleep(1000);
701  }
702  else
703  {
704  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
705  getDataBufferStats(id); // Re-check under lock
706  if (dataBufferIsTooLarge(id))
707  {
708  auto begin = dataBuffers_[id].DataBuffer.begin();
709  if (begin == dataBuffers_[id].DataBuffer.end())
710  {
711  TLOG(TLVL_WARNING) << "Data buffer is reported as too large, but doesn't contain any Fragments! Possible corrupt memory!";
712  continue;
713  }
714  if (*begin)
715  {
716  TLOG(TLVL_WAITFORBUFFERREADY) << "waitForDataBufferReady: Dropping Fragment with timestamp " << (*begin)->timestamp() << " from data buffer (Buffer over-size, circular data buffer mode)";
717  }
718  dataBuffers_[id].DataBufferDepthBytes -= (*begin)->sizeBytes();
719  dataBuffers_[id].DataBuffer.erase(begin);
720  dataBuffers_[id].BufferFragmentKept = false; // If any Fragments are removed from data buffer, then we know we don't have to ignore the first one anymore
721  getDataBufferStats(id);
722  }
723  }
724  }
725  return true;
726 }
727 
729 {
730  if (!dataBuffers_.count(id))
731  {
732  throw cet::exception("DataBufferError") << "Error in CommandableFragmentGenerator: Cannot check size of data buffer for ID " << id << " because it does not exist!";
733  }
734  return (maxDataBufferDepthFragments_ > 0 && dataBuffers_[id].DataBufferDepthFragments > maxDataBufferDepthFragments_) || (maxDataBufferDepthBytes_ > 0 && dataBuffers_[id].DataBufferDepthBytes > maxDataBufferDepthBytes_);
735 }
736 
738 {
739  if (!dataBuffers_.count(id))
740  {
741  throw cet::exception("DataBufferError") << "Error in CommandableFragmentGenerator: Cannot get stats of data buffer for ID " << id << " because it does not exist!";
742  }
744  dataBuffers_[id].DataBufferDepthFragments = dataBuffers_[id].DataBuffer.size();
745 
746  if (metricMan)
747  {
748  TLOG(TLVL_GETBUFFERSTATS) << "getDataBufferStats: Sending Metrics";
749  metricMan->sendMetric("Buffer Depth Fragments", dataBuffers_[id].DataBufferDepthFragments.load(), "fragments", 1, MetricMode::LastPoint);
750  metricMan->sendMetric("Buffer Depth Bytes", dataBuffers_[id].DataBufferDepthBytes.load(), "bytes", 1, MetricMode::LastPoint);
751 
752  auto bufferDepthFragmentsPercent = dataBuffers_[id].DataBufferDepthFragments.load() * 100 / static_cast<double>(maxDataBufferDepthFragments_);
753  auto bufferDepthBytesPercent = dataBuffers_[id].DataBufferDepthBytes.load() * 100 / static_cast<double>(maxDataBufferDepthBytes_);
754  metricMan->sendMetric("Fragment Generator Buffer Full %Fragments", bufferDepthFragmentsPercent, "%", 3, MetricMode::LastPoint);
755  metricMan->sendMetric("Fragment Generator Buffer Full %Bytes", bufferDepthBytesPercent, "%", 3, MetricMode::LastPoint);
756  metricMan->sendMetric("Fragment Generator Buffer Full %", bufferDepthFragmentsPercent > bufferDepthBytesPercent ? bufferDepthFragmentsPercent : bufferDepthBytesPercent, "%", 1, MetricMode::LastPoint);
757  }
758  TLOG(TLVL_GETBUFFERSTATS) << "getDataBufferStats: frags=" << dataBuffers_[id].DataBufferDepthFragments.load() << "/" << maxDataBufferDepthFragments_
759  << ", sz=" << dataBuffers_[id].DataBufferDepthBytes.load() << "/" << maxDataBufferDepthBytes_;
760 }
761 
763 {
764  if (!dataBuffers_.count(id))
765  {
766  throw cet::exception("DataBufferError") << "Error in CommandableFragmentGenerator: Cannot check data buffer for ID " << id << " because it does not exist!";
767  }
768 
769  if (dataBuffers_[id].DataBufferDepthFragments > 0 && mode_ != RequestMode::Single && mode_ != RequestMode::Ignored)
770  {
771  // Eliminate extra fragments
772  getDataBufferStats(id);
773  while (dataBufferIsTooLarge(id))
774  {
775  auto begin = dataBuffers_[id].DataBuffer.begin();
776  TLOG(TLVL_CHECKDATABUFFER) << "checkDataBuffer: Dropping Fragment with timestamp " << (*begin)->timestamp() << " from data buffer (Buffer over-size)";
777  dataBuffers_[id].DataBufferDepthBytes -= (*begin)->sizeBytes();
778  dataBuffers_[id].DataBuffer.erase(begin);
779  dataBuffers_[id].BufferFragmentKept = false; // If any Fragments are removed from data buffer, then we know we don't have to ignore the first one anymore
780  getDataBufferStats(id);
781  }
782  if (dataBuffers_[id].DataBuffer.size() > 0)
783  {
784  TLOG(TLVL_CHECKDATABUFFER) << "Determining if Fragments can be dropped from data buffer";
785  Fragment::timestamp_t last = dataBuffers_[id].DataBuffer.back()->timestamp();
786  Fragment::timestamp_t min = last > staleTimeout_ ? last - staleTimeout_ : 0;
787  for (auto it = dataBuffers_[id].DataBuffer.begin(); it != dataBuffers_[id].DataBuffer.end();)
788  {
789  if ((*it)->timestamp() < min)
790  {
791  TLOG(TLVL_CHECKDATABUFFER) << "checkDataBuffer: Dropping Fragment with timestamp " << (*it)->timestamp() << " from data buffer (timeout=" << staleTimeout_ << ", min=" << min << ")";
792  dataBuffers_[id].DataBufferDepthBytes -= (*it)->sizeBytes();
793  dataBuffers_[id].BufferFragmentKept = false; // If any Fragments are removed from data buffer, then we know we don't have to ignore the first one anymore
794  it = dataBuffers_[id].DataBuffer.erase(it);
795  }
796  else
797  {
798  break;
799  }
800  }
801  getDataBufferStats(id);
802  }
803  }
804 }
805 
807 {
808  while (!force_stop_)
809  {
810  if (should_stop() || monitoringInterval_ <= 0)
811  {
812  TLOG(TLVL_DEBUG) << "getMonitoringDataLoop: should_stop() is " << std::boolalpha << should_stop()
813  << " and monitoringInterval is " << monitoringInterval_ << ", returning";
814  return;
815  }
816  TLOG(TLVL_GETMONITORINGDATA) << "getMonitoringDataLoop: Determining whether to call checkHWStatus_";
817 
818  auto now = std::chrono::steady_clock::now();
819  if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
820  {
821  isHardwareOK_ = checkHWStatus_();
822  TLOG(TLVL_GETMONITORINGDATA) << "getMonitoringDataLoop: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_;
823  lastMonitoringCall_ = now;
824  }
825  usleep(monitoringInterval_ / 10);
826  }
827 }
828 
830 {
831  // dataBuffersMutex_ is held by calling function
832  // We just copy everything that's here into the output.
833  TLOG(TLVL_APPLYREQUESTS) << "Mode is Ignored; Copying data to output";
834  for (auto& id : dataBuffers_)
835  {
836  std::move(id.second.DataBuffer.begin(), id.second.DataBuffer.end(), std::inserter(frags, frags.end()));
837  id.second.DataBufferDepthBytes = 0;
838  id.second.DataBufferDepthFragments = 0;
839  id.second.BufferFragmentKept = false;
840  id.second.DataBuffer.clear();
841  }
842 }
843 
845 {
846  // We only care about the latest request received. Send empties for all others.
847  auto requests = requestReceiver_->GetRequests();
848  while (requests.size() > 1)
849  {
850  // std::map is ordered by key => Last sequence ID in the map is the one we care about
851  requestReceiver_->RemoveRequest(requests.begin()->first);
852  requests.erase(requests.begin());
853  }
854  sendEmptyFragments(frags, requests);
855 
856  // If no requests remain after sendEmptyFragments, return
857  if (requests.size() == 0 || !requests.count(ev_counter())) return;
858 
859  for (auto& id : dataBuffers_)
860  {
861  if (id.second.DataBuffer.size() > 0)
862  {
863  assert(id.second.DataBuffer.size() == 1);
864  TLOG(TLVL_APPLYREQUESTS) << "Mode is Single; Sending copy of last event";
865  for (auto& fragptr : id.second.DataBuffer)
866  {
867  // Return the latest data point
868  auto frag = fragptr.get();
869  auto newfrag = std::unique_ptr<artdaq::Fragment>(new Fragment(ev_counter(), frag->fragmentID()));
870  newfrag->resize(frag->size() - detail::RawFragmentHeader::num_words());
871  memcpy(newfrag->headerAddress(), frag->headerAddress(), frag->sizeBytes());
872  newfrag->setTimestamp(requests[ev_counter()]);
873  newfrag->setSequenceID(ev_counter());
874  frags.push_back(std::move(newfrag));
875  }
876  }
877  else
878  {
879  sendEmptyFragment(frags, ev_counter(), id.first, "No data for");
880  }
881  }
882  requestReceiver_->RemoveRequest(ev_counter());
883  ev_counter_inc(1, true);
884 }
885 
887 {
888  // We only care about the latest request received. Send empties for all others.
889  auto requests = requestReceiver_->GetRequests();
890  while (requests.size() > 1)
891  {
892  // std::map is ordered by key => Last sequence ID in the map is the one we care about
893  requestReceiver_->RemoveRequest(requests.begin()->first);
894  requests.erase(requests.begin());
895  }
896  sendEmptyFragments(frags, requests);
897 
898  // If no requests remain after sendEmptyFragments, return
899  if (requests.size() == 0 || !requests.count(ev_counter())) return;
900 
901  for (auto& id : dataBuffers_)
902  {
903  TLOG(TLVL_DEBUG) << "Creating ContainerFragment for Buffered Fragments";
904  frags.emplace_back(new artdaq::Fragment(ev_counter(), id.first));
905  frags.back()->setTimestamp(requests[ev_counter()]);
906  ContainerFragmentLoader cfl(*frags.back());
907  cfl.set_missing_data(false); // Buffer mode is never missing data, even if there IS no data.
908 
909  // If we kept a Fragment from the previous iteration, but more data has arrived, discard it
910  auto it = id.second.DataBuffer.begin();
911  if (id.second.BufferFragmentKept && id.second.DataBuffer.size() > 1)
912  {
913  id.second.DataBufferDepthBytes -= (*it)->sizeBytes();
914  it = id.second.DataBuffer.erase(it);
915  }
916 
917  // Buffer mode TFGs should simply copy out the whole dataBuffer_ into a ContainerFragment
918  while (it != id.second.DataBuffer.end())
919  {
920  TLOG(TLVL_APPLYREQUESTS) << "ApplyRequests: Adding Fragment with timestamp " << (*it)->timestamp() << " to Container with sequence ID " << ev_counter();
921  cfl.addFragment(*it);
922  if (bufferModeKeepLatest_ && id.second.DataBuffer.size() == 1)
923  {
924  id.second.BufferFragmentKept = true;
925  break;
926  }
927  id.second.DataBufferDepthBytes -= (*it)->sizeBytes();
928  it = id.second.DataBuffer.erase(it);
929  }
930  }
931  requestReceiver_->RemoveRequest(ev_counter());
932  ev_counter_inc(1, true);
933 }
934 
935 void artdaq::CommandableFragmentGenerator::applyRequestsWindowMode_CheckAndFillDataBuffer(artdaq::FragmentPtrs& frags, artdaq::Fragment::fragment_id_t id, artdaq::Fragment::sequence_id_t seq, artdaq::Fragment::timestamp_t ts)
936 {
937  TLOG(TLVL_APPLYREQUESTS) << "applyRequestsWindowMode_CheckAndFillDataBuffer: Checking that data exists for request window " << seq;
938  Fragment::timestamp_t min = ts > windowOffset_ ? ts - windowOffset_ : 0;
939  Fragment::timestamp_t max = min + windowWidth_;
940  TLOG(TLVL_APPLYREQUESTS) << "ApplyRequestsWindowsMode_CheckAndFillDataBuffer: min is " << min << ", max is " << max
941  << " and first/last points in buffer are " << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.front()->timestamp() : 0)
942  << "/" << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.back()->timestamp() : 0)
943  << " (sz=" << dataBuffers_[id].DataBuffer.size() << " [" << dataBuffers_[id].DataBufferDepthBytes.load()
944  << "/" << maxDataBufferDepthBytes_ << "])";
945  bool windowClosed = dataBuffers_[id].DataBuffer.size() > 0 && dataBuffers_[id].DataBuffer.back()->timestamp() >= max;
946  bool windowTimeout = !windowClosed && TimeUtils::GetElapsedTimeMicroseconds(requestReceiver_->GetRequestTime(seq)) > window_close_timeout_us_;
947  if (windowTimeout)
948  {
949  TLOG(TLVL_WARNING) << "applyRequestsWindowMode_CheckAndFillDataBuffer: A timeout occurred waiting for data to close the request window ({" << min << "-" << max
950  << "}, buffer={" << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.front()->timestamp() : 0) << "-"
951  << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.back()->timestamp() : 0)
952  << "} ). Time waiting: "
953  << TimeUtils::GetElapsedTimeMicroseconds(requestReceiver_->GetRequestTime(seq)) << " us "
954  << "(> " << window_close_timeout_us_ << " us).";
955  }
956  if (windowClosed || !data_thread_running_ || windowTimeout)
957  {
958  TLOG(TLVL_DEBUG) << "applyRequestsWindowMode_CheckAndFillDataBuffer: Creating ContainerFragment for Window-requested Fragments";
959  frags.emplace_back(new artdaq::Fragment(seq, id));
960  frags.back()->setTimestamp(ts);
961  ContainerFragmentLoader cfl(*frags.back());
962 
963  // In the spirit of NOvA's MegaPool: (RS = Request start (min), RE = Request End (max))
964  // --- | Buffer Start | --- | Buffer End | ---
965  //1. RS RE | | | |
966  //2. RS | | RE | |
967  //3. RS | | | | RE
968  //4. | | RS RE | |
969  //5. | | RS | | RE
970  //6. | | | | RS RE
971  //
972  // If RE (or RS) is after the end of the buffer, we wait for window_close_timeout_us_. If we're here, then that means that windowClosed is false, and the missing_data flag should be set.
973  // If RS (or RE) is before the start of the buffer, then missing_data should be set to true, as data is assumed to arrive in the buffer in timestamp order
974  // If the dataBuffer has size 0, then windowClosed will be false
975  if (!windowClosed || (dataBuffers_[id].DataBuffer.size() > 0 && dataBuffers_[id].DataBuffer.front()->timestamp() > min))
976  {
977  TLOG(TLVL_DEBUG) << "applyRequestsWindowMode_CheckAndFillDataBuffer: Request window starts before and/or ends after the current data buffer, setting ContainerFragment's missing_data flag!"
978  << " (requestWindowRange=[" << min << "," << max << "], "
979  << "buffer={" << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.front()->timestamp() : 0) << "-"
980  << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.back()->timestamp() : 0) << "}";
981  cfl.set_missing_data(true);
982  }
983 
984  // Do a little bit more work to decide which fragments to send for a given request
985  for (auto it = dataBuffers_[id].DataBuffer.begin(); it != dataBuffers_[id].DataBuffer.end();)
986  {
987  Fragment::timestamp_t fragT = (*it)->timestamp();
988  if (fragT < min || fragT > max || (fragT == max && windowWidth_ > 0))
989  {
990  ++it;
991  continue;
992  }
993 
994  TLOG(TLVL_APPLYREQUESTS) << "applyRequestsWindowMode_CheckAndFillDataBuffer: Adding Fragment with timestamp " << (*it)->timestamp() << " to Container";
995  cfl.addFragment(*it);
996 
997  if (uniqueWindows_)
998  {
999  dataBuffers_[id].DataBufferDepthBytes -= (*it)->sizeBytes();
1000  it = dataBuffers_[id].DataBuffer.erase(it);
1001  }
1002  else
1003  {
1004  ++it;
1005  }
1006  }
1007 
1008  dataBuffers_[id].WindowsSent[seq] = std::chrono::steady_clock::now();
1009  if (seq > dataBuffers_[id].HighestRequestSeen) dataBuffers_[id].HighestRequestSeen = seq;
1010  }
1011 }
1012 
1014 {
1015  TLOG(TLVL_APPLYREQUESTS) << "applyRequestsWindowMode BEGIN";
1016 
1017  auto requests = requestReceiver_->GetRequests();
1018 
1019  TLOG(TLVL_APPLYREQUESTS) << "applyRequestsWindowMode: Starting request processing";
1020  for (auto req = requests.begin(); req != requests.end();)
1021  {
1022  TLOG(TLVL_APPLYREQUESTS) << "applyRequestsWindowMode: processing request with sequence ID " << req->first << ", timestamp " << req->second;
1023 
1024  while (req->first < ev_counter() && requests.size() > 0)
1025  {
1026  TLOG(TLVL_APPLYREQUESTS) << "applyRequestsWindowMode: Clearing passed request for sequence ID " << req->first;
1027  requestReceiver_->RemoveRequest(req->first);
1028  req = requests.erase(req);
1029  }
1030  if (requests.size() == 0) break;
1031 
1032  for (auto& id : dataBuffers_)
1033  {
1034  if (!id.second.WindowsSent.count(req->first))
1035  {
1036  applyRequestsWindowMode_CheckAndFillDataBuffer(frags, id.first, req->first, req->second);
1037  }
1038  }
1039  checkSentWindows(req->first);
1040  ++req;
1041  }
1042 
1043  // Check sent windows for requests that can be removed
1044  for (auto& id : dataBuffers_)
1045  {
1046  std::set<artdaq::Fragment::sequence_id_t> seqs;
1047  for (auto& seq : id.second.WindowsSent)
1048  {
1049  seqs.insert(seq.first);
1050  }
1051  for (auto& seq : seqs)
1052  {
1053  checkSentWindows(seq);
1054  }
1055  }
1056 }
1057 
1059 {
1060  TLOG(TLVL_APPLYREQUESTS) << "applyRequestsSequenceIDMode BEGIN";
1061 
1062  auto requests = requestReceiver_->GetRequests();
1063 
1064  TLOG(TLVL_APPLYREQUESTS) << "applyRequestsSequenceIDMode: Starting request processing";
1065  for (auto req = requests.begin(); req != requests.end();)
1066  {
1067  TLOG(TLVL_APPLYREQUESTS) << "applyRequestsSequenceIDMode: Checking that data exists for request SequenceID " << req->first;
1068 
1069  for (auto& id : dataBuffers_)
1070  {
1071  if (!id.second.WindowsSent.count(req->first))
1072  {
1073  TLOG(29) << "Searching id " << id.first << " for Fragments with Sequence ID " << req->first;
1074  for (auto it = id.second.DataBuffer.begin(); it != id.second.DataBuffer.end();)
1075  {
1076  auto seq = (*it)->sequenceID();
1077  TLOG(29) << "applyRequestsSequenceIDMode: Fragment SeqID " << seq << ", request ID " << req->first;
1078  if (seq == req->first)
1079  {
1080  TLOG(29) << "applyRequestsSequenceIDMode: Adding Fragment to output";
1081  id.second.WindowsSent[req->first] = std::chrono::steady_clock::now();
1082  id.second.DataBufferDepthBytes -= (*it)->sizeBytes();
1083  frags.push_back(std::move(*it));
1084  it = id.second.DataBuffer.erase(it);
1085  }
1086  else
1087  {
1088  ++it;
1089  }
1090  }
1091  }
1092  if (req->first > id.second.HighestRequestSeen) id.second.HighestRequestSeen = req->first;
1093  }
1094  checkSentWindows(req->first);
1095  ++req;
1096  }
1097 
1098  // Check sent windows for requests that can be removed
1099  for (auto& id : dataBuffers_)
1100  {
1101  std::set<artdaq::Fragment::sequence_id_t> seqs;
1102  for (auto& seq : id.second.WindowsSent)
1103  {
1104  seqs.insert(seq.first);
1105  }
1106  for (auto& seq : seqs)
1107  {
1108  checkSentWindows(seq);
1109  }
1110  }
1111 }
1112 
1114 {
1115  if (check_stop() || exception())
1116  {
1117  return false;
1118  }
1119 
1120  // Wait for data, if in ignored mode, or a request otherwise
1121  if (mode_ == RequestMode::Ignored)
1122  {
1123  while (dataBufferFragmentCount_() == 0)
1124  {
1125  if (check_stop() || exception() || !isHardwareOK_) return false;
1126  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
1127  dataCondition_.wait_for(lock, std::chrono::milliseconds(10), [this]() { return dataBufferFragmentCount_() > 0; });
1128  }
1129  }
1130  else
1131  {
1132  if ((check_stop() && requestReceiver_->size() == 0) || exception()) return false;
1133 
1134  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
1135  dataCondition_.wait_for(lock, std::chrono::milliseconds(10));
1136 
1137  checkDataBuffers();
1138 
1139  // Wait up to 1000 ms for a request...
1140  auto counter = 0;
1141 
1142  while (requestReceiver_->size() == 0 && counter < 100)
1143  {
1144  if (check_stop() || exception()) return false;
1145 
1146  checkDataBuffers();
1147 
1148  requestReceiver_->WaitForRequests(10); // milliseconds
1149  counter++;
1150  }
1151  }
1152 
1153  {
1154  std::unique_lock<std::mutex> dlk(dataBuffersMutex_);
1155 
1156  switch (mode_)
1157  {
1158  case RequestMode::Single:
1159  applyRequestsSingleMode(frags);
1160  break;
1161  case RequestMode::Window:
1162  applyRequestsWindowMode(frags);
1163  break;
1164  case RequestMode::Buffer:
1165  applyRequestsBufferMode(frags);
1166  break;
1167  case RequestMode::SequenceID:
1168  applyRequestsSequenceIDMode(frags);
1169  break;
1170  case RequestMode::Ignored:
1171  default:
1172  applyRequestsIgnoredMode(frags);
1173  break;
1174  }
1175 
1176  if (!data_thread_running_ || force_stop_)
1177  {
1178  TLOG(TLVL_INFO) << "Data thread has stopped; Clearing data buffers";
1179  for (auto& id : dataBuffers_)
1180  {
1181  id.second.DataBufferDepthBytes = 0;
1182  id.second.DataBufferDepthFragments = 0;
1183  id.second.BufferFragmentKept = false;
1184  id.second.DataBuffer.clear();
1185  }
1186  }
1187 
1188  getDataBuffersStats();
1189  }
1190 
1191  if (frags.size() > 0)
1192  TLOG(TLVL_APPLYREQUESTS) << "Finished Processing requests, returning " << frags.size() << " fragments, current ev_counter is " << ev_counter();
1193  return true;
1194 }
1195 
1196 bool artdaq::CommandableFragmentGenerator::sendEmptyFragment(artdaq::FragmentPtrs& frags, size_t seqId, Fragment::fragment_id_t fragmentId, std::string desc)
1197 {
1198  TLOG(TLVL_WARNING) << desc << " sequence ID " << seqId << ", sending empty fragment";
1199  auto frag = new Fragment();
1200  frag->setSequenceID(seqId);
1201  frag->setFragmentID(fragmentId);
1202  frag->setSystemType(Fragment::EmptyFragmentType);
1203  frags.emplace_back(FragmentPtr(frag));
1204  return true;
1205 }
1206 
1207 void artdaq::CommandableFragmentGenerator::sendEmptyFragments(artdaq::FragmentPtrs& frags, std::map<Fragment::sequence_id_t, Fragment::timestamp_t>& requests)
1208 {
1209  if (requests.size() > 0)
1210  {
1211  TLOG(TLVL_SENDEMPTYFRAGMENTS) << "Sending Empty Fragments for Sequence IDs from " << ev_counter() << " up to but not including " << requests.begin()->first;
1212  while (requests.begin()->first > ev_counter())
1213  {
1214  for (auto& fid : dataBuffers_)
1215  {
1216  sendEmptyFragment(frags, ev_counter(), fid.first, "Missed request for");
1217  }
1218  ev_counter_inc(1, true);
1219  }
1220  }
1221 }
1222 
1223 void artdaq::CommandableFragmentGenerator::checkSentWindows(artdaq::Fragment::sequence_id_t seq)
1224 {
1225  TLOG(TLVL_CHECKWINDOWS) << "checkSentWindows: Checking if request " << seq << " can be removed from request list";
1226  bool seqComplete = true;
1227  bool seqTimeout = false;
1228  for (auto& id : dataBuffers_)
1229  {
1230  if (!id.second.WindowsSent.count(seq) || id.second.HighestRequestSeen < seq)
1231  {
1232  seqComplete = false;
1233  }
1234  if (id.second.WindowsSent.count(seq) && TimeUtils::GetElapsedTimeMicroseconds(id.second.WindowsSent[seq]) > missing_request_window_timeout_us_)
1235  {
1236  seqTimeout = true;
1237  }
1238  }
1239  if (seqComplete)
1240  {
1241  TLOG(TLVL_CHECKWINDOWS) << "checkSentWindows: Request " << seq << " is complete, removing from requestReceiver.";
1242  requestReceiver_->RemoveRequest(seq);
1243 
1244  if (ev_counter() == seq)
1245  {
1246  TLOG(TLVL_CHECKWINDOWS) << "checkSentWindows: Sequence ID matches ev_counter, incrementing ev_counter (" << ev_counter() << ")";
1247 
1248  for (auto& id : dataBuffers_)
1249  {
1250  id.second.WindowsSent.erase(seq);
1251  }
1252 
1253  ev_counter_inc(1, true);
1254  }
1255  }
1256  if (seqTimeout)
1257  {
1258  TLOG(TLVL_CHECKWINDOWS) << "checkSentWindows: Sent Window history indicates that requests between " << ev_counter() << " and " << seq << " have timed out.";
1259  while (ev_counter() <= seq)
1260  {
1261  if (ev_counter() < seq) TLOG(TLVL_WARNING) << "Missed request for sequence ID " << ev_counter() << "! Will not send any data for this sequence ID!";
1262  requestReceiver_->RemoveRequest(ev_counter());
1263 
1264  for (auto& id : dataBuffers_)
1265  {
1266  id.second.WindowsSent.erase(ev_counter());
1267  }
1268 
1269  ev_counter_inc(1, true);
1270  }
1271  }
1272 }
CommandableFragmentGenerator(const fhicl::ParameterSet &ps)
CommandableFragmentGenerator Constructor.
void applyRequestsSingleMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Single. Precondition: dataBufferMutex_ and reques...
virtual bool checkHWStatus_()
Check any relavent hardware status registers. Return false if an error condition exists that should h...
virtual ~CommandableFragmentGenerator()
CommandableFragmentGenerator Destructor.
void applyRequestsBufferMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Buffer. Precondition: dataBufferMutex_ and reques...
void getDataBufferStats(Fragment::fragment_id_t id)
Calculate the size of the dataBuffer and report appropriate metrics.
void getMonitoringDataLoop()
This function regularly calls checkHWStatus_(), and sets the isHardwareOK flag accordingly.
void startDataThread()
Function that launches the data thread (getDataLoop())
artdaq::Fragment::fragment_id_t fragment_id() const
Get the Fragment ID of this Fragment generator.
std::string ReportCmd(std::string const &which="")
Get a report about a user-specified run-time quantity.
virtual bool metaCommand(std::string const &command, std::string const &arg)
The meta-command is used for implementing user-specific commands in a CommandableFragmentGenerator.
void StopCmd(uint64_t timeout, uint64_t timestamp)
Stop the CommandableFragmentGenerator.
void applyRequestsWindowMode_CheckAndFillDataBuffer(artdaq::FragmentPtrs &frags, artdaq::Fragment::fragment_id_t id, artdaq::Fragment::sequence_id_t seq, artdaq::Fragment::timestamp_t ts)
void checkSentWindows(Fragment::sequence_id_t seq)
Check the windows_sent_ooo_ map for sequence IDs that may be removed.
bool sendEmptyFragment(FragmentPtrs &frags, size_t sequenceId, Fragment::fragment_id_t fragmentId, std::string desc)
Send an EmptyFragmentType Fragment.
void applyRequestsWindowMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Window. Precondition: dataBufferMutex_ and reques...
void StartCmd(int run, uint64_t timeout, uint64_t timestamp)
Start the CommandableFragmentGenerator.
virtual void pauseNoMutex()
On call to PauseCmd, pauseNoMutex() is called prior to PauseCmd acquiring the mutex ...
bool check_stop()
Routine used by applyRequests to make sure that all outstanding requests have been fulfilled before r...
void ResumeCmd(uint64_t timeout, uint64_t timestamp)
Resume the CommandableFragmentGenerator.
bool getNext(FragmentPtrs &output) overridefinal
getNext calls either applyRequests or getNext_ to get any data that is ready to be sent to the EventB...
size_t ev_counter_inc(size_t step=1, bool force=false)
Increment the event counter, if the current RequestMode allows it.
bool applyRequests(FragmentPtrs &frags)
See if any requests have been received, and add the corresponding data Fragment objects to the output...
void applyRequestsIgnoredMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Ignored. Precondition: dataBufferMutex_ and reque...
void PauseCmd(uint64_t timeout, uint64_t timestamp)
Pause the CommandableFragmentGenerator.
void getDataLoop()
When separate_data_thread is set to true, this loop repeatedly calls getNext_ and adds returned Fragm...
void sendEmptyFragments(FragmentPtrs &frags, std::map< Fragment::sequence_id_t, Fragment::timestamp_t > &requests)
This function is for Buffered and Single request modes, as they can only respond to one data request ...
Receive data requests and make them available to CommandableFragmentGenerator or other interested par...
void startMonitoringThread()
Function that launches the monitoring thread (getMonitoringDataLoop())
virtual void pause()
If a CommandableFragmentGenerator subclass is reading from hardware, the implementation of pause() sh...
virtual void resume()
The subrun number will be incremented before a call to resume.
void applyRequestsSequenceIDMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode SequenceID. Precondition: dataBufferMutex_ and re...
bool waitForDataBufferReady(Fragment::fragment_id_t id)
Wait for the data buffer to drain (dataBufferIsTooLarge returns false), periodically reporting status...
virtual std::string report()
Let&#39;s say that the contract with the report() functions is that they return a non-empty string if the...
std::string printMode_()
Return the string representation of the current RequestMode.
void checkDataBuffer(Fragment::fragment_id_t id)
Perform data buffer pruning operations for the given buffer. If the RequestMode is Single...
virtual std::string reportSpecific(std::string const &what)
Report the status of a specific quantity
bool dataBufferIsTooLarge(Fragment::fragment_id_t id)
Test the configured constraints on the data buffer.
void joinThreads()
Join any data-taking threads. Should be called when destructing CommandableFragmentGenerator.