artdaq  v3_06_02
CommandableFragmentGenerator.cc
1 #include "artdaq/DAQdata/Globals.hh"
2 #define TRACE_NAME (app_name + "_CommandableFragmentGenerator").c_str() // include these 2 first -
3 
4 #include "artdaq/Generators/CommandableFragmentGenerator.hh"
5 
6 #include <boost/exception/all.hpp>
7 #include <boost/throw_exception.hpp>
8 
9 #include <iterator>
10 #include <limits>
11 
12 #include "canvas/Utilities/Exception.h"
13 #include "cetlib_except/exception.h"
14 #include "fhiclcpp/ParameterSet.h"
15 
16 #include "artdaq-core/Data/ContainerFragmentLoader.hh"
17 #include "artdaq-core/Data/Fragment.hh"
18 #include "artdaq-core/Utilities/ExceptionHandler.hh"
19 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh"
20 #include "artdaq-core/Utilities/TimeUtils.hh"
21 
22 #include <sys/poll.h>
23 #include <algorithm>
24 #include <fstream>
25 #include <iomanip>
26 #include <iostream>
27 #include <iterator>
29 
30 #define TLVL_GETNEXT 10
31 #define TLVL_GETNEXT_VERBOSE 20
32 #define TLVL_CHECKSTOP 11
33 #define TLVL_EVCOUNTERINC 12
34 #define TLVL_GETDATALOOP 13
35 #define TLVL_GETDATALOOP_DATABUFFWAIT 21
36 #define TLVL_GETDATALOOP_VERBOSE 20
37 #define TLVL_WAITFORBUFFERREADY 15
38 #define TLVL_GETBUFFERSTATS 16
39 #define TLVL_CHECKDATABUFFER 17
40 #define TLVL_GETMONITORINGDATA 18
41 #define TLVL_APPLYREQUESTS 9
42 #define TLVL_SENDEMPTYFRAGMENTS 19
43 #define TLVL_CHECKWINDOWS 14
44 
46  : mutex_()
47  , requestReceiver_(nullptr)
48  , windowOffset_(ps.get<Fragment::timestamp_t>("request_window_offset", 0))
49  , windowWidth_(ps.get<Fragment::timestamp_t>("request_window_width", 0))
50  , staleTimeout_(ps.get<Fragment::timestamp_t>("stale_request_timeout", 0xFFFFFFFF))
51  , expectedType_(ps.get<Fragment::type_t>("expected_fragment_type", Fragment::type_t(Fragment::EmptyFragmentType)))
52  , uniqueWindows_(ps.get<bool>("request_windows_are_unique", true))
53  , missing_request_window_timeout_us_(ps.get<size_t>("missing_request_window_timeout_us", 5000000))
54  , window_close_timeout_us_(ps.get<size_t>("window_close_timeout_us", 2000000))
55  , useDataThread_(ps.get<bool>("separate_data_thread", false))
56  , circularDataBufferMode_(ps.get<bool>("circular_buffer_mode", false))
57  , sleep_on_no_data_us_(ps.get<size_t>("sleep_on_no_data_us", 0))
58  , data_thread_running_(false)
59  , maxDataBufferDepthFragments_(ps.get<int>("data_buffer_depth_fragments", 1000))
60  , maxDataBufferDepthBytes_(ps.get<size_t>("data_buffer_depth_mb", 1000) * 1024 * 1024)
61  , useMonitoringThread_(ps.get<bool>("separate_monitoring_thread", false))
62  , monitoringInterval_(ps.get<int64_t>("hardware_poll_interval_us", 0))
63  , lastMonitoringCall_()
64  , isHardwareOK_(true)
65  , run_number_(-1)
66  , subrun_number_(-1)
67  , timeout_(std::numeric_limits<uint64_t>::max())
68  , timestamp_(std::numeric_limits<uint64_t>::max())
69  , should_stop_(false)
70  , exception_(false)
71  , force_stop_(false)
72  , latest_exception_report_("none")
73  , ev_counter_(1)
74  , board_id_(-1)
75  , sleep_on_stop_us_(0)
76 {
77  board_id_ = ps.get<int>("board_id");
78  instance_name_for_metrics_ = "BoardReader." + boost::lexical_cast<std::string>(board_id_);
79 
80  auto fragment_ids = ps.get<std::vector<artdaq::Fragment::fragment_id_t>>("fragment_ids", std::vector<artdaq::Fragment::fragment_id_t>());
81 
82  TLOG(TLVL_TRACE) << "artdaq::CommandableFragmentGenerator::CommandableFragmentGenerator(ps)";
83  int fragment_id = ps.get<int>("fragment_id", -99);
84 
85  if (fragment_id != -99)
86  {
87  if (fragment_ids.size() != 0)
88  {
89  latest_exception_report_ = "Error in CommandableFragmentGenerator: can't both define \"fragment_id\" and \"fragment_ids\" in FHiCL document";
90  throw cet::exception(latest_exception_report_);
91  }
92  else
93  {
94  fragment_ids.emplace_back(fragment_id);
95  }
96  }
97 
98  for (auto& id : fragment_ids)
99  {
100  dataBuffers_[id].DataBufferDepthBytes = 0;
101  dataBuffers_[id].DataBufferDepthFragments = 0;
102  dataBuffers_[id].HighestRequestSeen = 0;
103  dataBuffers_[id].DataBuffer.emplace_back(FragmentPtr(new Fragment()));
104  (*dataBuffers_[id].DataBuffer.begin())->setSystemType(Fragment::EmptyFragmentType);
105  }
106 
107  sleep_on_stop_us_ = ps.get<int>("sleep_on_stop_us", 0);
108 
109  std::string modeString = ps.get<std::string>("request_mode", "ignored");
110  if (modeString == "single" || modeString == "Single")
111  {
112  mode_ = RequestMode::Single;
113  }
114  else if (modeString.find("buffer") != std::string::npos || modeString.find("Buffer") != std::string::npos)
115  {
116  mode_ = RequestMode::Buffer;
117  }
118  else if (modeString == "window" || modeString == "Window")
119  {
120  mode_ = RequestMode::Window;
121  }
122  else if (modeString.find("ignore") != std::string::npos || modeString.find("Ignore") != std::string::npos)
123  {
124  mode_ = RequestMode::Ignored;
125  }
126  TLOG(TLVL_DEBUG) << "Request mode is " << printMode_();
127 
128  if (mode_ != RequestMode::Ignored)
129  {
130  if (!useDataThread_)
131  {
132  latest_exception_report_ = "Error in CommandableFragmentGenerator: use_data_thread must be true when request_mode is not \"Ignored\"!";
133  throw cet::exception(latest_exception_report_);
134  }
135  requestReceiver_.reset(new RequestReceiver(ps));
136  }
137 }
138 
140 {
141  joinThreads();
142  requestReceiver_.reset(nullptr);
143 }
144 
146 {
147  should_stop_ = true;
148  force_stop_ = true;
149  TLOG(TLVL_DEBUG) << "Joining dataThread";
150  if (dataThread_.joinable()) dataThread_.join();
151  TLOG(TLVL_DEBUG) << "Joining monitoringThread";
152  if (monitoringThread_.joinable()) monitoringThread_.join();
153  TLOG(TLVL_DEBUG) << "joinThreads complete";
154 }
155 
157 {
158  bool result = true;
159 
160  if (check_stop()) usleep(sleep_on_stop_us_);
161  if (exception() || force_stop_) return false;
162 
163  if (!useMonitoringThread_ && monitoringInterval_ > 0)
164  {
165  TLOG(TLVL_GETNEXT) << "getNext: Checking whether to collect Monitoring Data";
166  auto now = std::chrono::steady_clock::now();
167 
168  if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
169  {
170  TLOG(TLVL_GETNEXT) << "getNext: Collecting Monitoring Data";
171  isHardwareOK_ = checkHWStatus_();
172  TLOG(TLVL_GETNEXT) << "getNext: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_;
173  lastMonitoringCall_ = now;
174  }
175  }
176 
177  try
178  {
179  std::lock_guard<std::mutex> lk(mutex_);
180  if (useDataThread_)
181  {
182  TLOG(TLVL_TRACE) << "getNext: Calling applyRequests";
183  result = applyRequests(output);
184  TLOG(TLVL_TRACE) << "getNext: Done with applyRequests result=" << std::boolalpha << result;
185  for (auto dataIter = output.begin(); dataIter != output.end(); ++dataIter)
186  {
187  TLOG(20) << "getNext: applyRequests() returned fragment with sequenceID = " << (*dataIter)->sequenceID()
188  << ", type = " << (*dataIter)->typeString() << ", id = " << std::to_string((*dataIter)->fragmentID())
189  << ", timestamp = " << (*dataIter)->timestamp() << ", and sizeBytes = " << (*dataIter)->sizeBytes();
190  }
191 
192  if (exception())
193  {
194  TLOG(TLVL_ERROR) << "Exception found in BoardReader with board ID " << board_id() << "; BoardReader will now return error status when queried";
195  throw cet::exception("CommandableFragmentGenerator") << "Exception found in BoardReader with board ID " << board_id() << "; BoardReader will now return error status when queried";
196  }
197  }
198  else
199  {
200  if (!isHardwareOK_)
201  {
202  TLOG(TLVL_ERROR) << "Stopping CFG because the hardware reports bad status!";
203  return false;
204  }
205  TLOG(TLVL_TRACE) << "getNext: Calling getNext_ w/ ev_counter()=" << ev_counter();
206  try
207  {
208  result = getNext_(output);
209  }
210  catch (...)
211  {
212  throw;
213  }
214  TLOG(TLVL_TRACE) << "getNext: Done with getNext_ - ev_counter() now " << ev_counter();
215  for (auto dataIter = output.begin(); dataIter != output.end(); ++dataIter)
216  {
217  TLOG(TLVL_GETNEXT_VERBOSE) << "getNext: getNext_() returned fragment with sequenceID = " << (*dataIter)->sequenceID()
218  << ", type = " << (*dataIter)->typeString() << ", id = " << std::to_string((*dataIter)->fragmentID())
219  << ", timestamp = " << (*dataIter)->timestamp() << ", and sizeBytes = " << (*dataIter)->sizeBytes();
220  }
221  }
222  }
223  catch (const cet::exception& e)
224  {
225  latest_exception_report_ = "cet::exception caught in getNext(): ";
226  latest_exception_report_.append(e.what());
227  TLOG(TLVL_ERROR) << "getNext: cet::exception caught: " << e;
228  set_exception(true);
229  return false;
230  }
231  catch (const boost::exception& e)
232  {
233  latest_exception_report_ = "boost::exception caught in getNext(): ";
234  latest_exception_report_.append(boost::diagnostic_information(e));
235  TLOG(TLVL_ERROR) << "getNext: boost::exception caught: " << boost::diagnostic_information(e);
236  set_exception(true);
237  return false;
238  }
239  catch (const std::exception& e)
240  {
241  latest_exception_report_ = "std::exception caught in getNext(): ";
242  latest_exception_report_.append(e.what());
243  TLOG(TLVL_ERROR) << "getNext: std::exception caught: " << e.what();
244  set_exception(true);
245  return false;
246  }
247  catch (...)
248  {
249  latest_exception_report_ = "Unknown exception caught in getNext().";
250  TLOG(TLVL_ERROR) << "getNext: unknown exception caught";
251  set_exception(true);
252  return false;
253  }
254 
255  if (!result)
256  {
257  TLOG(TLVL_DEBUG) << "getNext: Either getNext_ or applyRequests returned false, stopping";
258  }
259 
260  if (metricMan && !output.empty())
261  {
262  auto timestamp = output.front()->timestamp();
263 
264  if (output.size() > 1)
265  { // Only bother sorting if >1 entry
266  for (auto& outputfrag : output)
267  {
268  if (outputfrag->timestamp() > timestamp)
269  {
270  timestamp = outputfrag->timestamp();
271  }
272  }
273  }
274 
275  metricMan->sendMetric("Last Timestamp", timestamp, "Ticks", 1,
276  MetricMode::LastPoint, app_name);
277  }
278 
279  return result;
280 }
281 
283 {
284  TLOG(TLVL_CHECKSTOP) << "CFG::check_stop: should_stop=" << should_stop() << ", useDataThread_=" << useDataThread_ << ", exception status =" << int(exception());
285 
286  if (!should_stop()) return false;
287  if (!useDataThread_ || mode_ == RequestMode::Ignored) return true;
288  if (force_stop_) return true;
289 
290  // check_stop returns true if the CFG should stop. We should wait for the RequestReceiver to stop before stopping.
291  TLOG(TLVL_DEBUG) << "should_stop is true, force_stop_ is false, requestReceiver_->isRunning() is " << std::boolalpha << requestReceiver_->isRunning();
292  return !requestReceiver_->isRunning();
293 }
294 
296 {
297  if (force || mode_ == RequestMode::Ignored)
298  {
299  TLOG(TLVL_EVCOUNTERINC) << "ev_counter_inc: Incrementing ev_counter from " << ev_counter() << " by " << step;
300  return ev_counter_.fetch_add(step);
301  }
302  return ev_counter_.load();
303 } // returns the prev value
304 
305 void artdaq::CommandableFragmentGenerator::StartCmd(int run, uint64_t timeout, uint64_t timestamp)
306 {
307  TLOG(TLVL_TRACE) << "Start Command received.";
308  if (run < 0) throw cet::exception("CommandableFragmentGenerator") << "negative run number";
309 
310  timeout_ = timeout;
311  timestamp_ = timestamp;
312  ev_counter_.store(1);
313  {
314  std::unique_lock<std::mutex> lk(dataBuffersMutex_);
315  for (auto& id : dataBuffers_)
316  {
317  id.second.DataBufferDepthBytes = 0;
318  id.second.DataBufferDepthFragments = 0;
319  id.second.HighestRequestSeen = 0;
320  id.second.DataBuffer.clear();
321  id.second.WindowsSent.clear();
322  }
323  }
324 
325  should_stop_.store(false);
326  force_stop_.store(false);
327  exception_.store(false);
328  run_number_ = run;
329  subrun_number_ = 1;
330  latest_exception_report_ = "none";
331 
332  start();
333 
334  std::unique_lock<std::mutex> lk(mutex_);
335  if (useDataThread_) startDataThread();
336  if (useMonitoringThread_) startMonitoringThread();
337  if (mode_ != RequestMode::Ignored)
338  {
339  requestReceiver_->SetRunNumber(static_cast<uint32_t>(run));
340  requestReceiver_->startRequestReception();
341  }
342  TLOG(TLVL_TRACE) << "Start Command complete.";
343 }
344 
345 void artdaq::CommandableFragmentGenerator::StopCmd(uint64_t timeout, uint64_t timestamp)
346 {
347  TLOG(TLVL_TRACE) << "Stop Command received.";
348 
349  timeout_ = timeout;
350  timestamp_ = timestamp;
351  if (requestReceiver_)
352  {
353  TLOG(TLVL_DEBUG) << "Stopping Request reception BEGIN";
354  requestReceiver_->stopRequestReception();
355  TLOG(TLVL_DEBUG) << "Stopping Request reception END";
356  }
357 
358  stopNoMutex();
359  should_stop_.store(true);
360  std::unique_lock<std::mutex> lk(mutex_);
361  stop();
362 
363  joinThreads();
364  TLOG(TLVL_TRACE) << "Stop Command complete.";
365 }
366 
367 void artdaq::CommandableFragmentGenerator::PauseCmd(uint64_t timeout, uint64_t timestamp)
368 {
369  TLOG(TLVL_TRACE) << "Pause Command received.";
370  timeout_ = timeout;
371  timestamp_ = timestamp;
372  //if (requestReceiver_->isRunning()) requestReceiver_->stopRequestReceiverThread();
373 
374  pauseNoMutex();
375  should_stop_.store(true);
376  std::unique_lock<std::mutex> lk(mutex_);
377 
378  pause();
379 }
380 
381 void artdaq::CommandableFragmentGenerator::ResumeCmd(uint64_t timeout, uint64_t timestamp)
382 {
383  TLOG(TLVL_TRACE) << "Resume Command received.";
384  timeout_ = timeout;
385  timestamp_ = timestamp;
386 
387  subrun_number_ += 1;
388  should_stop_ = false;
389 
390  {
391  std::unique_lock<std::mutex> lk(dataBuffersMutex_);
392  for (auto& id : dataBuffers_)
393  {
394  id.second.DataBufferDepthBytes = 0;
395  id.second.DataBufferDepthFragments = 0;
396  id.second.DataBuffer.clear();
397  }
398  }
399  // no lock required: thread not started yet
400  resume();
401 
402  std::unique_lock<std::mutex> lk(mutex_);
403  //if (useDataThread_) startDataThread();
404  //if (useMonitoringThread_) startMonitoringThread();
405  //if (mode_ != RequestMode::Ignored && !requestReceiver_->isRunning()) requestReceiver_->startRequestReceiverThread();
406  TLOG(TLVL_TRACE) << "Resume Command complete.";
407 }
408 
409 std::string artdaq::CommandableFragmentGenerator::ReportCmd(std::string const& which)
410 {
411  TLOG(TLVL_TRACE) << "Report Command received.";
412  std::lock_guard<std::mutex> lk(mutex_);
413 
414  // 14-May-2015, KAB: please see the comments associated with the report()
415  // methods in the CommandableFragmentGenerator.hh file for more information
416  // on the use of those methods in this method.
417 
418  // check if the child class has something meaningful for this request
419  std::string childReport = reportSpecific(which);
420  if (childReport.length() > 0) { return childReport; }
421 
422  // handle the requests that we can take care of at this level
423  if (which == "latest_exception")
424  {
425  return latest_exception_report_;
426  }
427 
428  // check if the child class has provided a catch-all report function
429  childReport = report();
430  if (childReport.length() > 0) { return childReport; }
431 
432  // ELF: 5/31/2019: Let BoardReaderCore's report handle this...
433  /*
434  // if we haven't been able to come up with any report so far, say so
435  std::string tmpString = "The \"" + which + "\" command is not ";
436  tmpString.append("currently supported by the ");
437  tmpString.append(metricsReportingInstanceName());
438  tmpString.append(" fragment generator.");
439  */
440  TLOG(TLVL_TRACE) << "Report Command complete.";
441  return ""; //tmpString;
442 }
443 
444 // Default implemenetations of state functions
446 {
447 #pragma message "Using default implementation of CommandableFragmentGenerator::pauseNoMutex()"
448 }
449 
451 {
452 #pragma message "Using default implementation of CommandableFragmentGenerator::pause()"
453 }
454 
456 #pragma message "Using default implementation of CommandableFragmentGenerator::resume()"
457 }
458 
460 {
461 #pragma message "Using default implementation of CommandableFragmentGenerator::report()"
462  return "";
463 }
464 
466 {
467 #pragma message "Using default implementation of CommandableFragmentGenerator::reportSpecific(std::string)"
468  return "";
469 }
470 
472 {
473 #pragma message "Using default implementation of CommandableFragmentGenerator::checkHWStatus_()"
474  return true;
475 }
476 
477 bool artdaq::CommandableFragmentGenerator::metaCommand(std::string const&, std::string const&)
478 {
479 #pragma message "Using default implementation of CommandableFragmentGenerator::metaCommand(std::string, std::string)"
480  return true;
481 }
482 
484 {
485  if (dataThread_.joinable()) dataThread_.join();
486  TLOG(TLVL_INFO) << "Starting Data Receiver Thread";
487  try
488  {
489  dataThread_ = boost::thread(&CommandableFragmentGenerator::getDataLoop, this);
490  }
491  catch (const boost::exception& e)
492  {
493  TLOG(TLVL_ERROR) << "Caught boost::exception starting Data Receiver thread: " << boost::diagnostic_information(e) << ", errno=" << errno;
494  std::cerr << "Caught boost::exception starting Data Receiver thread: " << boost::diagnostic_information(e) << ", errno=" << errno << std::endl;
495  exit(5);
496  }
497 }
498 
500 {
501  if (monitoringThread_.joinable()) monitoringThread_.join();
502  TLOG(TLVL_INFO) << "Starting Hardware Monitoring Thread";
503  try
504  {
505  monitoringThread_ = boost::thread(&CommandableFragmentGenerator::getMonitoringDataLoop, this);
506  }
507  catch (const boost::exception& e)
508  {
509  TLOG(TLVL_ERROR) << "Caught boost::exception starting Hardware Monitoring thread: " << boost::diagnostic_information(e) << ", errno=" << errno;
510  std::cerr << "Caught boost::exception starting Hardware Monitoring thread: " << boost::diagnostic_information(e) << ", errno=" << errno << std::endl;
511  exit(5);
512  }
513 }
514 
516 {
517  switch (mode_)
518  {
519  case RequestMode::Single:
520  return "Single";
521  case RequestMode::Buffer:
522  return "Buffer";
523  case RequestMode::Window:
524  return "Window";
525  case RequestMode::Ignored:
526  return "Ignored";
527  }
528 
529  return "ERROR";
530 }
531 
532 //
533 // The "useDataThread_" thread
534 //
536 {
537  data_thread_running_ = true;
538  while (!force_stop_)
539  {
540  if (!isHardwareOK_)
541  {
542  TLOG(TLVL_DEBUG) << "getDataLoop: isHardwareOK is " << isHardwareOK_ << ", aborting data thread";
543  data_thread_running_ = false;
544  return;
545  }
546 
547  TLOG(TLVL_GETDATALOOP) << "getDataLoop: calling getNext_";
548 
549  bool data = false;
550  auto startdata = std::chrono::steady_clock::now();
551  FragmentPtrs newData;
552 
553  try
554  {
555  data = getNext_(newData);
556  }
557  catch (...)
558  {
559  ExceptionHandler(ExceptionHandlerRethrow::no,
560  "Exception thrown by fragment generator in CommandableFragmentGenerator::getDataLoop; setting exception state to \"true\"");
561  set_exception(true);
562 
563  data_thread_running_ = false;
564  return;
565  }
566 
567  if (metricMan)
568  {
569  metricMan->sendMetric("Avg Data Acquisition Time", TimeUtils::GetElapsedTime(startdata), "s", 3, artdaq::MetricMode::Average);
570  }
571 
572  if (newData.size() == 0 && sleep_on_no_data_us_ > 0)
573  {
574  usleep(sleep_on_no_data_us_);
575  }
576 
577  auto dataIter = newData.begin();
578  while (dataIter != newData.end())
579  {
580  TLOG(TLVL_GETDATALOOP_VERBOSE) << "getDataLoop: getNext_() returned fragment with timestamp = " << (*dataIter)->timestamp() << ", and sizeBytes = " << (*dataIter)->sizeBytes();
581 
582  auto frag_id = (*dataIter)->fragmentID();
583  if (!dataBuffers_.count(frag_id))
584  {
585  throw cet::exception("DataBufferError") << "Error in CommandableFragmentGenerator: Recevied Fragment with fragment_id " << frag_id << ", but this ID was not declared in fragment_ids!";
586  }
587 
588  TLOG(TLVL_GETDATALOOP_DATABUFFWAIT) << "Waiting for data buffer ready";
589  if (!waitForDataBufferReady(frag_id)) return;
590  TLOG(TLVL_GETDATALOOP_DATABUFFWAIT) << "Done waiting for data buffer ready";
591 
592  TLOG(TLVL_GETDATALOOP) << "getDataLoop: processing data";
593  if (data && !force_stop_)
594  {
595  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
596  switch (mode_)
597  {
598  case RequestMode::Single:
599  dataBuffers_[frag_id].DataBuffer.clear();
600  dataBuffers_[frag_id].DataBufferDepthBytes = (*dataIter)->sizeBytes();
601  dataBuffers_[frag_id].DataBuffer.emplace_back(std::move(*dataIter));
602  dataIter = newData.erase(dataIter);
603  break;
604  case RequestMode::Buffer:
605  case RequestMode::Ignored:
606  case RequestMode::Window:
607  default:
608  //dataBuffer_.reserve(dataBuffer_.size() + newDataBuffer_.size());
609  dataBuffers_[frag_id].DataBufferDepthBytes += (*dataIter)->sizeBytes();
610  dataBuffers_[frag_id].DataBuffer.emplace_back(std::move(*dataIter));
611  dataIter = newData.erase(dataIter);
612  break;
613  }
614  getDataBufferStats(frag_id);
615  }
616  else
617  {
618  break;
619  }
620  }
621 
622  {
623  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
624  for (auto& id : dataBuffers_)
625  {
626  if (id.second.DataBuffer.size() > 0)
627  {
628  dataCondition_.notify_all();
629  break;
630  }
631  }
632  }
633  if (!data || force_stop_)
634  {
635  TLOG(TLVL_INFO) << "Data flow has stopped. Ending data collection thread";
636  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
637  data_thread_running_ = false;
638  if (requestReceiver_) requestReceiver_->ClearRequests();
639  newData.clear();
640  TLOG(TLVL_INFO) << "getDataLoop: Ending thread";
641  return;
642  }
643  }
644 }
645 
647 {
648  if (!dataBuffers_.count(id))
649  {
650  throw cet::exception("DataBufferError") << "Error in CommandableFragmentGenerator: Cannot wait for data buffer for ID " << id << " because it does not exist!";
651  }
652  auto startwait = std::chrono::steady_clock::now();
653  auto first = true;
654  auto lastwaittime = 0ULL;
655 
656  {
657  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
658  getDataBufferStats(id);
659  }
660 
661  while (dataBufferIsTooLarge(id))
662  {
663  if (!circularDataBufferMode_)
664  {
665  if (should_stop())
666  {
667  TLOG(TLVL_DEBUG) << "Run ended while waiting for buffer to shrink!";
668  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
669  getDataBufferStats(id);
670  dataCondition_.notify_all();
671  data_thread_running_ = false;
672  return false;
673  }
674  auto waittime = TimeUtils::GetElapsedTimeMilliseconds(startwait);
675 
676  if (first || (waittime != lastwaittime && waittime % 1000 == 0))
677  {
678  TLOG(TLVL_WARNING) << "Bad Omen: Data Buffer has exceeded its size limits. "
679  << "(seq_id=" << ev_counter() << ", frag_id=" << id
680  << ", frags=" << dataBuffers_[id].DataBufferDepthFragments << "/" << maxDataBufferDepthFragments_
681  << ", szB=" << dataBuffers_[id].DataBufferDepthBytes << "/" << maxDataBufferDepthBytes_ << ")";
682  TLOG(TLVL_TRACE) << "Bad Omen: Possible causes include requests not getting through or Ignored-mode BR issues";
683  first = false;
684  }
685  if (waittime % 5 && waittime != lastwaittime)
686  {
687  TLOG(TLVL_WAITFORBUFFERREADY) << "getDataLoop: Data Retreival paused for " << waittime << " ms waiting for data buffer to drain";
688  }
689  lastwaittime = waittime;
690  usleep(1000);
691  }
692  else
693  {
694  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
695  getDataBufferStats(id); // Re-check under lock
696  if (dataBufferIsTooLarge(id))
697  {
698  auto begin = dataBuffers_[id].DataBuffer.begin();
699  if (begin == dataBuffers_[id].DataBuffer.end())
700  {
701  TLOG(TLVL_WARNING) << "Data buffer is reported as too large, but doesn't contain any Fragments! Possible corrupt memory!";
702  continue;
703  }
704  if (*begin)
705  {
706  TLOG(TLVL_WAITFORBUFFERREADY) << "waitForDataBufferReady: Dropping Fragment with timestamp " << (*begin)->timestamp() << " from data buffer (Buffer over-size, circular data buffer mode)";
707  }
708  dataBuffers_[id].DataBufferDepthBytes -= (*begin)->sizeBytes();
709  dataBuffers_[id].DataBuffer.erase(begin);
710  getDataBufferStats(id);
711  }
712  }
713  }
714  return true;
715 }
716 
718 {
719  if (!dataBuffers_.count(id))
720  {
721  throw cet::exception("DataBufferError") << "Error in CommandableFragmentGenerator: Cannot check size of data buffer for ID " << id << " because it does not exist!";
722  }
723  return (maxDataBufferDepthFragments_ > 0 && dataBuffers_[id].DataBufferDepthFragments > maxDataBufferDepthFragments_) || (maxDataBufferDepthBytes_ > 0 && dataBuffers_[id].DataBufferDepthBytes > maxDataBufferDepthBytes_);
724 }
725 
727 {
728  if (!dataBuffers_.count(id))
729  {
730  throw cet::exception("DataBufferError") << "Error in CommandableFragmentGenerator: Cannot get stats of data buffer for ID " << id << " because it does not exist!";
731  }
733  dataBuffers_[id].DataBufferDepthFragments = dataBuffers_[id].DataBuffer.size();
734 
735  if (metricMan)
736  {
737  TLOG(TLVL_GETBUFFERSTATS) << "getDataBufferStats: Sending Metrics";
738  metricMan->sendMetric("Buffer Depth Fragments", dataBuffers_[id].DataBufferDepthFragments.load(), "fragments", 1, MetricMode::LastPoint);
739  metricMan->sendMetric("Buffer Depth Bytes", dataBuffers_[id].DataBufferDepthBytes.load(), "bytes", 1, MetricMode::LastPoint);
740  }
741  TLOG(TLVL_GETBUFFERSTATS) << "getDataBufferStats: frags=" << dataBuffers_[id].DataBufferDepthFragments.load() << "/" << maxDataBufferDepthFragments_
742  << ", sz=" << dataBuffers_[id].DataBufferDepthBytes.load() << "/" << maxDataBufferDepthBytes_;
743 }
744 
746 {
747  if (!dataBuffers_.count(id))
748  {
749  throw cet::exception("DataBufferError") << "Error in CommandableFragmentGenerator: Cannot check data buffer for ID " << id << " because it does not exist!";
750  }
751 
752  if (dataBuffers_[id].DataBufferDepthFragments > 0 && mode_ != RequestMode::Single && mode_ != RequestMode::Ignored)
753  {
754  // Eliminate extra fragments
755  getDataBufferStats(id);
756  while (dataBufferIsTooLarge(id))
757  {
758  auto begin = dataBuffers_[id].DataBuffer.begin();
759  TLOG(TLVL_CHECKDATABUFFER) << "checkDataBuffer: Dropping Fragment with timestamp " << (*begin)->timestamp() << " from data buffer (Buffer over-size)";
760  dataBuffers_[id].DataBufferDepthBytes -= (*begin)->sizeBytes();
761  dataBuffers_[id].DataBuffer.erase(begin);
762  getDataBufferStats(id);
763  }
764  if (dataBuffers_[id].DataBuffer.size() > 0)
765  {
766  TLOG(TLVL_CHECKDATABUFFER) << "Determining if Fragments can be dropped from data buffer";
767  Fragment::timestamp_t last = dataBuffers_[id].DataBuffer.back()->timestamp();
768  Fragment::timestamp_t min = last > staleTimeout_ ? last - staleTimeout_ : 0;
769  for (auto it = dataBuffers_[id].DataBuffer.begin(); it != dataBuffers_[id].DataBuffer.end();)
770  {
771  if ((*it)->timestamp() < min)
772  {
773  TLOG(TLVL_CHECKDATABUFFER) << "checkDataBuffer: Dropping Fragment with timestamp " << (*it)->timestamp() << " from data buffer (timeout=" << staleTimeout_ << ", min=" << min << ")";
774  dataBuffers_[id].DataBufferDepthBytes -= (*it)->sizeBytes();
775  it = dataBuffers_[id].DataBuffer.erase(it);
776  }
777  else
778  {
779  break;
780  }
781  }
782  getDataBufferStats(id);
783  }
784  }
785 }
786 
788 {
789  while (!force_stop_)
790  {
791  if (should_stop() || monitoringInterval_ <= 0)
792  {
793  TLOG(TLVL_DEBUG) << "getMonitoringDataLoop: should_stop() is " << std::boolalpha << should_stop()
794  << " and monitoringInterval is " << monitoringInterval_ << ", returning";
795  return;
796  }
797  TLOG(TLVL_GETMONITORINGDATA) << "getMonitoringDataLoop: Determining whether to call checkHWStatus_";
798 
799  auto now = std::chrono::steady_clock::now();
800  if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
801  {
802  isHardwareOK_ = checkHWStatus_();
803  TLOG(TLVL_GETMONITORINGDATA) << "getMonitoringDataLoop: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_;
804  lastMonitoringCall_ = now;
805  }
806  usleep(monitoringInterval_ / 10);
807  }
808 }
809 
811 {
812  // dataBuffersMutex_ is held by calling function
813  // We just copy everything that's here into the output.
814  TLOG(TLVL_APPLYREQUESTS) << "Mode is Ignored; Copying data to output";
815  for (auto& id : dataBuffers_)
816  {
817  std::move(id.second.DataBuffer.begin(), id.second.DataBuffer.end(), std::inserter(frags, frags.end()));
818  id.second.DataBufferDepthBytes = 0;
819  id.second.DataBufferDepthFragments = 0;
820  id.second.DataBuffer.clear();
821  }
822 }
823 
825 {
826  // We only care about the latest request received. Send empties for all others.
827  auto requests = requestReceiver_->GetRequests();
828  while (requests.size() > 1)
829  {
830  // std::map is ordered by key => Last sequence ID in the map is the one we care about
831  requestReceiver_->RemoveRequest(requests.begin()->first);
832  requests.erase(requests.begin());
833  }
834  sendEmptyFragments(frags, requests);
835 
836  // If no requests remain after sendEmptyFragments, return
837  if (requests.size() == 0 || !requests.count(ev_counter())) return;
838 
839  for (auto& id : dataBuffers_)
840  {
841  if (id.second.DataBuffer.size() > 0)
842  {
843  assert(id.second.DataBuffer.size() == 1);
844  TLOG(TLVL_APPLYREQUESTS) << "Mode is Single; Sending copy of last event";
845  for (auto& fragptr : id.second.DataBuffer)
846  {
847  // Return the latest data point
848  auto frag = fragptr.get();
849  auto newfrag = std::unique_ptr<artdaq::Fragment>(new Fragment(ev_counter(), frag->fragmentID()));
850  newfrag->resize(frag->size() - detail::RawFragmentHeader::num_words());
851  memcpy(newfrag->headerAddress(), frag->headerAddress(), frag->sizeBytes());
852  newfrag->setTimestamp(requests[ev_counter()]);
853  newfrag->setSequenceID(ev_counter());
854  frags.push_back(std::move(newfrag));
855  }
856  }
857  else
858  {
859  sendEmptyFragment(frags, ev_counter(), id.first, "No data for");
860  }
861  }
862  requestReceiver_->RemoveRequest(ev_counter());
863  ev_counter_inc(1, true);
864 }
865 
867 {
868  // We only care about the latest request received. Send empties for all others.
869  auto requests = requestReceiver_->GetRequests();
870  while (requests.size() > 1)
871  {
872  // std::map is ordered by key => Last sequence ID in the map is the one we care about
873  requestReceiver_->RemoveRequest(requests.begin()->first);
874  requests.erase(requests.begin());
875  }
876  sendEmptyFragments(frags, requests);
877 
878  // If no requests remain after sendEmptyFragments, return
879  if (requests.size() == 0 || !requests.count(ev_counter())) return;
880 
881  for (auto& id : dataBuffers_)
882  {
883  TLOG(TLVL_DEBUG) << "Creating ContainerFragment for Buffered Fragments";
884  frags.emplace_back(new artdaq::Fragment(ev_counter(), id.first));
885  frags.back()->setTimestamp(requests[ev_counter()]);
886  ContainerFragmentLoader cfl(*frags.back());
887  cfl.set_missing_data(false); // Buffer mode is never missing data, even if there IS no data.
888 
889  // Buffer mode TFGs should simply copy out the whole dataBuffer_ into a ContainerFragment
890  // Window mode TFGs must do a little bit more work to decide which fragments to send for a given request
891  for (auto it = id.second.DataBuffer.begin(); it != id.second.DataBuffer.end();)
892  {
893  TLOG(TLVL_APPLYREQUESTS) << "ApplyRequests: Adding Fragment with timestamp " << (*it)->timestamp() << " to Container with sequence ID " << ev_counter();
894  cfl.addFragment(*it);
895  id.second.DataBufferDepthBytes -= (*it)->sizeBytes();
896  it = id.second.DataBuffer.erase(it);
897  }
898  }
899  requestReceiver_->RemoveRequest(ev_counter());
900  ev_counter_inc(1, true);
901 }
902 
903 void artdaq::CommandableFragmentGenerator::applyRequestsWindowMode_CheckAndFillDataBuffer(artdaq::FragmentPtrs& frags, artdaq::Fragment::fragment_id_t id, artdaq::Fragment::sequence_id_t seq, artdaq::Fragment::timestamp_t ts)
904 {
905  TLOG(TLVL_APPLYREQUESTS) << "applyRequestsWindowMode_CheckAndFillDataBuffer: Checking that data exists for request window " << seq;
906  Fragment::timestamp_t min = ts > windowOffset_ ? ts - windowOffset_ : 0;
907  Fragment::timestamp_t max = min + windowWidth_;
908  TLOG(TLVL_APPLYREQUESTS) << "ApplyRequestsWindowsMode_CheckAndFillDataBuffer: min is " << min << ", max is " << max
909  << " and first/last points in buffer are " << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.front()->timestamp() : 0)
910  << "/" << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.back()->timestamp() : 0)
911  << " (sz=" << dataBuffers_[id].DataBuffer.size() << " [" << dataBuffers_[id].DataBufferDepthBytes.load()
912  << "/" << maxDataBufferDepthBytes_ << "])";
913  bool windowClosed = dataBuffers_[id].DataBuffer.size() > 0 && dataBuffers_[id].DataBuffer.back()->timestamp() >= max;
914  bool windowTimeout = !windowClosed && TimeUtils::GetElapsedTimeMicroseconds(requestReceiver_->GetRequestTime(seq)) > window_close_timeout_us_;
915  if (windowTimeout)
916  {
917  TLOG(TLVL_WARNING) << "applyRequestsWindowMode_CheckAndFillDataBuffer: A timeout occurred waiting for data to close the request window ({" << min << "-" << max
918  << "}, buffer={" << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.front()->timestamp() : 0) << "-"
919  << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.back()->timestamp() : 0)
920  << "} ). Time waiting: "
921  << TimeUtils::GetElapsedTimeMicroseconds(requestReceiver_->GetRequestTime(seq)) << " us "
922  << "(> " << window_close_timeout_us_ << " us).";
923  }
924  if (windowClosed || !data_thread_running_ || windowTimeout)
925  {
926  TLOG(TLVL_DEBUG) << "applyRequestsWindowMode_CheckAndFillDataBuffer: Creating ContainerFragment for Window-requested Fragments";
927  frags.emplace_back(new artdaq::Fragment(seq, id));
928  frags.back()->setTimestamp(ts);
929  ContainerFragmentLoader cfl(*frags.back());
930 
931  // In the spirit of NOvA's MegaPool: (RS = Request start (min), RE = Request End (max))
932  // --- | Buffer Start | --- | Buffer End | ---
933  //1. RS RE | | | |
934  //2. RS | | RE | |
935  //3. RS | | | | RE
936  //4. | | RS RE | |
937  //5. | | RS | | RE
938  //6. | | | | RS RE
939  //
940  // If RE (or RS) is after the end of the buffer, we wait for window_close_timeout_us_. If we're here, then that means that windowClosed is false, and the missing_data flag should be set.
941  // If RS (or RE) is before the start of the buffer, then missing_data should be set to true, as data is assumed to arrive in the buffer in timestamp order
942  // If the dataBuffer has size 0, then windowClosed will be false
943  if (!windowClosed || (dataBuffers_[id].DataBuffer.size() > 0 && dataBuffers_[id].DataBuffer.front()->timestamp() > min))
944  {
945  TLOG(TLVL_DEBUG) << "applyRequestsWindowMode_CheckAndFillDataBuffer: Request window starts before and/or ends after the current data buffer, setting ContainerFragment's missing_data flag!"
946  << " (requestWindowRange=[" << min << "," << max << "], "
947  << "buffer={" << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.front()->timestamp() : 0) << "-"
948  << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.back()->timestamp() : 0) << "}";
949  cfl.set_missing_data(true);
950  }
951 
952  // Do a little bit more work to decide which fragments to send for a given request
953  for (auto it = dataBuffers_[id].DataBuffer.begin(); it != dataBuffers_[id].DataBuffer.end();)
954  {
955  Fragment::timestamp_t fragT = (*it)->timestamp();
956  if (fragT < min || fragT > max || (fragT == max && windowWidth_ > 0))
957  {
958  ++it;
959  continue;
960  }
961 
962  TLOG(TLVL_APPLYREQUESTS) << "applyRequestsWindowMode_CheckAndFillDataBuffer: Adding Fragment with timestamp " << (*it)->timestamp() << " to Container";
963  cfl.addFragment(*it);
964 
965  if (uniqueWindows_)
966  {
967  dataBuffers_[id].DataBufferDepthBytes -= (*it)->sizeBytes();
968  it = dataBuffers_[id].DataBuffer.erase(it);
969  }
970  else
971  {
972  ++it;
973  }
974  }
975 
976  dataBuffers_[id].WindowsSent[seq] = std::chrono::steady_clock::now();
977  if (seq > dataBuffers_[id].HighestRequestSeen) dataBuffers_[id].HighestRequestSeen = seq;
978  }
979 }
980 
982 {
983  TLOG(TLVL_APPLYREQUESTS) << "applyRequestsWindowMode BEGIN";
984 
985  auto requests = requestReceiver_->GetRequests();
986 
987  TLOG(TLVL_APPLYREQUESTS) << "applyRequestsWindowMode: Starting request processing";
988  for (auto req = requests.begin(); req != requests.end();)
989  {
990  TLOG(TLVL_APPLYREQUESTS) << "applyRequestsWindowMode: processing request with sequence ID " << req->first << ", timestamp " << req->second;
991 
992  while (req->first < ev_counter() && requests.size() > 0)
993  {
994  TLOG(TLVL_APPLYREQUESTS) << "applyRequestsWindowMode: Clearing passed request for sequence ID " << req->first;
995  requestReceiver_->RemoveRequest(req->first);
996  req = requests.erase(req);
997  }
998  if (requests.size() == 0) break;
999 
1000  for (auto& id : dataBuffers_)
1001  {
1002  if (!id.second.WindowsSent.count(req->first))
1003  {
1004  applyRequestsWindowMode_CheckAndFillDataBuffer(frags, id.first, req->first, req->second);
1005  }
1006  }
1007  checkSentWindows(req->first);
1008  ++req;
1009  }
1010 
1011  // Check sent windows for requests that can be removed
1012  for (auto& id : dataBuffers_)
1013  {
1014  std::set<artdaq::Fragment::sequence_id_t> seqs;
1015  for (auto& seq : id.second.WindowsSent)
1016  {
1017  seqs.insert(seq.first);
1018  }
1019  for (auto& seq : seqs)
1020  {
1021  checkSentWindows(seq);
1022  }
1023  }
1024 }
1025 
1027 {
1028  if (check_stop() || exception())
1029  {
1030  return false;
1031  }
1032 
1033  // Wait for data, if in ignored mode, or a request otherwise
1034  if (mode_ == RequestMode::Ignored)
1035  {
1036  while (dataBufferFragmentCount_() == 0)
1037  {
1038  if (check_stop() || exception() || !isHardwareOK_) return false;
1039  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
1040  dataCondition_.wait_for(lock, std::chrono::milliseconds(10), [this]() { return dataBufferFragmentCount_() > 0; });
1041  }
1042  }
1043  else
1044  {
1045  if ((check_stop() && requestReceiver_->size() == 0) || exception()) return false;
1046 
1047  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
1048  dataCondition_.wait_for(lock, std::chrono::milliseconds(10));
1049 
1050  checkDataBuffers();
1051 
1052  // Wait up to 1000 ms for a request...
1053  auto counter = 0;
1054 
1055  while (requestReceiver_->size() == 0 && counter < 100)
1056  {
1057  if (check_stop() || exception()) return false;
1058 
1059  checkDataBuffers();
1060 
1061  requestReceiver_->WaitForRequests(10); // milliseconds
1062  counter++;
1063  }
1064  }
1065 
1066  {
1067  std::unique_lock<std::mutex> dlk(dataBuffersMutex_);
1068 
1069  switch (mode_)
1070  {
1071  case RequestMode::Single:
1072  applyRequestsSingleMode(frags);
1073  break;
1074  case RequestMode::Window:
1075  applyRequestsWindowMode(frags);
1076  break;
1077  case RequestMode::Buffer:
1078  applyRequestsBufferMode(frags);
1079  break;
1080  case RequestMode::Ignored:
1081  default:
1082  applyRequestsIgnoredMode(frags);
1083  break;
1084  }
1085 
1086  if (!data_thread_running_ || force_stop_)
1087  {
1088  TLOG(TLVL_INFO) << "Data thread has stopped; Clearing data buffers";
1089  for (auto& id : dataBuffers_)
1090  {
1091  id.second.DataBufferDepthBytes = 0;
1092  id.second.DataBufferDepthFragments = 0;
1093  id.second.DataBuffer.clear();
1094  }
1095  }
1096 
1097  getDataBuffersStats();
1098  }
1099 
1100  if (frags.size() > 0)
1101  TLOG(TLVL_APPLYREQUESTS) << "Finished Processing requests, returning " << frags.size() << " fragments, current ev_counter is " << ev_counter();
1102  return true;
1103 }
1104 
1105 bool artdaq::CommandableFragmentGenerator::sendEmptyFragment(artdaq::FragmentPtrs& frags, size_t seqId, Fragment::fragment_id_t fragmentId, std::string desc)
1106 {
1107  TLOG(TLVL_WARNING) << desc << " sequence ID " << seqId << ", sending empty fragment";
1108  auto frag = new Fragment();
1109  frag->setSequenceID(seqId);
1110  frag->setFragmentID(fragmentId);
1111  frag->setSystemType(Fragment::EmptyFragmentType);
1112  frags.emplace_back(FragmentPtr(frag));
1113  return true;
1114 }
1115 
1116 void artdaq::CommandableFragmentGenerator::sendEmptyFragments(artdaq::FragmentPtrs& frags, std::map<Fragment::sequence_id_t, Fragment::timestamp_t>& requests)
1117 {
1118  if (requests.size() > 0)
1119  {
1120  TLOG(TLVL_SENDEMPTYFRAGMENTS) << "Sending Empty Fragments for Sequence IDs from " << ev_counter() << " up to but not including " << requests.begin()->first;
1121  while (requests.begin()->first > ev_counter())
1122  {
1123  for (auto& fid : dataBuffers_)
1124  {
1125  sendEmptyFragment(frags, ev_counter(), fid.first, "Missed request for");
1126  }
1127  ev_counter_inc(1, true);
1128  }
1129  }
1130 }
1131 
1132 void artdaq::CommandableFragmentGenerator::checkSentWindows(artdaq::Fragment::sequence_id_t seq)
1133 {
1134  bool seqComplete = true;
1135  bool seqTimeout = false;
1136  for (auto& id : dataBuffers_)
1137  {
1138  if (!id.second.WindowsSent.count(seq) || id.second.HighestRequestSeen < seq)
1139  {
1140  seqComplete = false;
1141  }
1142  if (id.second.WindowsSent.count(seq) && TimeUtils::GetElapsedTimeMicroseconds(id.second.WindowsSent[seq]) > missing_request_window_timeout_us_)
1143  {
1144  seqTimeout = true;
1145  }
1146  }
1147  if (seqComplete)
1148  {
1149  TLOG(TLVL_CHECKWINDOWS) << "checkSentWindows: Request " << seq << " is complete, removing from requestReceiver.";
1150  requestReceiver_->RemoveRequest(seq);
1151 
1152  if (ev_counter() == seq)
1153  {
1154  TLOG(TLVL_CHECKWINDOWS) << "checkSentWindows: Sequence ID matches ev_counter, incrementing ev_counter (" << ev_counter() << ")";
1155 
1156  for (auto& id : dataBuffers_)
1157  {
1158  id.second.WindowsSent.erase(seq);
1159  }
1160 
1161  ev_counter_inc(1, true);
1162  }
1163  }
1164  if (seqTimeout)
1165  {
1166  TLOG(TLVL_CHECKWINDOWS) << "checkSentWindows: Sent Window history indicates that requests between " << ev_counter() << " and " << seq << " have timed out.";
1167  while (ev_counter() <= seq)
1168  {
1169  if (ev_counter() < seq) TLOG(TLVL_WARNING) << "Missed request for sequence ID " << ev_counter() << "! Will not send any data for this sequence ID!";
1170  requestReceiver_->RemoveRequest(ev_counter());
1171 
1172  for (auto& id : dataBuffers_)
1173  {
1174  id.second.WindowsSent.erase(ev_counter());
1175  }
1176 
1177  ev_counter_inc(1, true);
1178  }
1179  }
1180 }
CommandableFragmentGenerator(const fhicl::ParameterSet &ps)
CommandableFragmentGenerator Constructor.
void applyRequestsSingleMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Single. Precondition: dataBufferMutex_ and reques...
virtual bool checkHWStatus_()
Check any relavent hardware status registers. Return false if an error condition exists that should h...
virtual ~CommandableFragmentGenerator()
CommandableFragmentGenerator Destructor.
void applyRequestsBufferMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Buffer. Precondition: dataBufferMutex_ and reques...
void getDataBufferStats(Fragment::fragment_id_t id)
Calculate the size of the dataBuffer and report appropriate metrics.
void getMonitoringDataLoop()
This function regularly calls checkHWStatus_(), and sets the isHardwareOK flag accordingly.
void startDataThread()
Function that launches the data thread (getDataLoop())
artdaq::Fragment::fragment_id_t fragment_id() const
Get the Fragment ID of this Fragment generator.
std::string ReportCmd(std::string const &which="")
Get a report about a user-specified run-time quantity.
virtual bool metaCommand(std::string const &command, std::string const &arg)
The meta-command is used for implementing user-specific commands in a CommandableFragmentGenerator.
void StopCmd(uint64_t timeout, uint64_t timestamp)
Stop the CommandableFragmentGenerator.
void applyRequestsWindowMode_CheckAndFillDataBuffer(artdaq::FragmentPtrs &frags, artdaq::Fragment::fragment_id_t id, artdaq::Fragment::sequence_id_t seq, artdaq::Fragment::timestamp_t ts)
Copy data from the relevant data buffer that matches the given timestamp.
void checkSentWindows(Fragment::sequence_id_t seq)
Check the windows_sent_ooo_ map for sequence IDs that may be removed.
bool sendEmptyFragment(FragmentPtrs &frags, size_t sequenceId, Fragment::fragment_id_t fragmentId, std::string desc)
Send an EmptyFragmentType Fragment.
void applyRequestsWindowMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Window. Precondition: dataBufferMutex_ and reques...
void StartCmd(int run, uint64_t timeout, uint64_t timestamp)
Start the CommandableFragmentGenerator.
virtual void pauseNoMutex()
On call to PauseCmd, pauseNoMutex() is called prior to PauseCmd acquiring the mutex ...
bool check_stop()
Routine used by applyRequests to make sure that all outstanding requests have been fulfilled before r...
void ResumeCmd(uint64_t timeout, uint64_t timestamp)
Resume the CommandableFragmentGenerator.
bool getNext(FragmentPtrs &output) overridefinal
getNext calls either applyRequests or getNext_ to get any data that is ready to be sent to the EventB...
size_t ev_counter_inc(size_t step=1, bool force=false)
Increment the event counter, if the current RequestMode allows it.
bool applyRequests(FragmentPtrs &frags)
See if any requests have been received, and add the corresponding data Fragment objects to the output...
void applyRequestsIgnoredMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Ignored. Precondition: dataBufferMutex_ and reque...
void PauseCmd(uint64_t timeout, uint64_t timestamp)
Pause the CommandableFragmentGenerator.
void getDataLoop()
When separate_data_thread is set to true, this loop repeatedly calls getNext_ and adds returned Fragm...
void sendEmptyFragments(FragmentPtrs &frags, std::map< Fragment::sequence_id_t, Fragment::timestamp_t > &requests)
This function is for Buffered and Single request modes, as they can only respond to one data request ...
Receive data requests and make them available to CommandableFragmentGenerator or other interested par...
void startMonitoringThread()
Function that launches the monitoring thread (getMonitoringDataLoop())
virtual void pause()
If a CommandableFragmentGenerator subclass is reading from hardware, the implementation of pause() sh...
virtual void resume()
The subrun number will be incremented before a call to resume.
bool waitForDataBufferReady(Fragment::fragment_id_t id)
Wait for the data buffer to drain (dataBufferIsTooLarge returns false), periodically reporting status...
virtual std::string report()
Let&#39;s say that the contract with the report() functions is that they return a non-empty string if the...
std::string printMode_()
Return the string representation of the current RequestMode.
void checkDataBuffer(Fragment::fragment_id_t id)
Perform data buffer pruning operations for the given buffer. If the RequestMode is Single...
virtual std::string reportSpecific(std::string const &what)
Report the status of a specific quantity
bool dataBufferIsTooLarge(Fragment::fragment_id_t id)
Test the configured constraints on the data buffer.
void joinThreads()
Join any data-taking threads. Should be called when destructing CommandableFragmentGenerator.