artdaq  v3_07_01
CommandableFragmentGenerator.cc
1 #include "artdaq/DAQdata/Globals.hh"
2 #define TRACE_NAME (app_name + "_CommandableFragmentGenerator").c_str() // include these 2 first -
3 
4 #include "artdaq/Generators/CommandableFragmentGenerator.hh"
5 
6 #include <boost/exception/all.hpp>
7 #include <boost/throw_exception.hpp>
8 
9 #include <iterator>
10 #include <limits>
11 
12 #include "canvas/Utilities/Exception.h"
13 #include "cetlib_except/exception.h"
14 #include "fhiclcpp/ParameterSet.h"
15 
16 #include "artdaq-core/Data/ContainerFragmentLoader.hh"
17 #include "artdaq-core/Data/Fragment.hh"
18 #include "artdaq-core/Utilities/ExceptionHandler.hh"
19 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh"
20 #include "artdaq-core/Utilities/TimeUtils.hh"
21 
22 #include <sys/poll.h>
23 #include <algorithm>
24 #include <fstream>
25 #include <iomanip>
26 #include <iostream>
27 #include <iterator>
29 
30 #define TLVL_GETNEXT 10
31 #define TLVL_GETNEXT_VERBOSE 20
32 #define TLVL_CHECKSTOP 11
33 #define TLVL_EVCOUNTERINC 12
34 #define TLVL_GETDATALOOP 13
35 #define TLVL_GETDATALOOP_DATABUFFWAIT 21
36 #define TLVL_GETDATALOOP_VERBOSE 20
37 #define TLVL_WAITFORBUFFERREADY 15
38 #define TLVL_GETBUFFERSTATS 16
39 #define TLVL_CHECKDATABUFFER 17
40 #define TLVL_GETMONITORINGDATA 18
41 #define TLVL_APPLYREQUESTS 9
42 #define TLVL_SENDEMPTYFRAGMENTS 19
43 #define TLVL_CHECKWINDOWS 14
44 
46  : mutex_()
47  , requestReceiver_(nullptr)
48  , bufferModeKeepLatest_(ps.get<bool>("buffer_mode_keep_latest", false))
49  , windowOffset_(ps.get<Fragment::timestamp_t>("request_window_offset", 0))
50  , windowWidth_(ps.get<Fragment::timestamp_t>("request_window_width", 0))
51  , staleTimeout_(ps.get<Fragment::timestamp_t>("stale_request_timeout", 0xFFFFFFFF))
52  , expectedType_(ps.get<Fragment::type_t>("expected_fragment_type", Fragment::type_t(Fragment::EmptyFragmentType)))
53  , uniqueWindows_(ps.get<bool>("request_windows_are_unique", true))
54  , missing_request_window_timeout_us_(ps.get<size_t>("missing_request_window_timeout_us", 5000000))
55  , window_close_timeout_us_(ps.get<size_t>("window_close_timeout_us", 2000000))
56  , useDataThread_(ps.get<bool>("separate_data_thread", false))
57  , circularDataBufferMode_(ps.get<bool>("circular_buffer_mode", false))
58  , sleep_on_no_data_us_(ps.get<size_t>("sleep_on_no_data_us", 0))
59  , data_thread_running_(false)
60  , maxDataBufferDepthFragments_(ps.get<int>("data_buffer_depth_fragments", 1000))
61  , maxDataBufferDepthBytes_(ps.get<size_t>("data_buffer_depth_mb", 1000) * 1024 * 1024)
62  , useMonitoringThread_(ps.get<bool>("separate_monitoring_thread", false))
63  , monitoringInterval_(ps.get<int64_t>("hardware_poll_interval_us", 0))
64  , lastMonitoringCall_()
65  , isHardwareOK_(true)
66  , run_number_(-1)
67  , subrun_number_(-1)
68  , timeout_(std::numeric_limits<uint64_t>::max())
69  , timestamp_(std::numeric_limits<uint64_t>::max())
70  , should_stop_(false)
71  , exception_(false)
72  , force_stop_(false)
73  , latest_exception_report_("none")
74  , ev_counter_(1)
75  , board_id_(-1)
76  , sleep_on_stop_us_(0)
77 {
78  board_id_ = ps.get<int>("board_id");
79  instance_name_for_metrics_ = "BoardReader." + boost::lexical_cast<std::string>(board_id_);
80 
81  auto fragment_ids = ps.get<std::vector<artdaq::Fragment::fragment_id_t>>("fragment_ids", std::vector<artdaq::Fragment::fragment_id_t>());
82 
83  TLOG(TLVL_TRACE) << "artdaq::CommandableFragmentGenerator::CommandableFragmentGenerator(ps)";
84  int fragment_id = ps.get<int>("fragment_id", -99);
85 
86  if (fragment_id != -99)
87  {
88  if (fragment_ids.size() != 0)
89  {
90  latest_exception_report_ = "Error in CommandableFragmentGenerator: can't both define \"fragment_id\" and \"fragment_ids\" in FHiCL document";
91  throw cet::exception(latest_exception_report_);
92  }
93  else
94  {
95  fragment_ids.emplace_back(fragment_id);
96  }
97  }
98 
99  for (auto& id : fragment_ids)
100  {
101  dataBuffers_[id].DataBufferDepthBytes = 0;
102  dataBuffers_[id].DataBufferDepthFragments = 0;
103  dataBuffers_[id].HighestRequestSeen = 0;
104  dataBuffers_[id].BufferFragmentKept = false;
105  dataBuffers_[id].DataBuffer.emplace_back(FragmentPtr(new Fragment()));
106  (*dataBuffers_[id].DataBuffer.begin())->setSystemType(Fragment::EmptyFragmentType);
107  }
108 
109  sleep_on_stop_us_ = ps.get<int>("sleep_on_stop_us", 0);
110 
111  std::string modeString = ps.get<std::string>("request_mode", "ignored");
112  if (modeString == "single" || modeString == "Single")
113  {
114  mode_ = RequestMode::Single;
115  }
116  else if (modeString.find("buffer") != std::string::npos || modeString.find("Buffer") != std::string::npos)
117  {
118  mode_ = RequestMode::Buffer;
119  }
120  else if (modeString == "window" || modeString == "Window")
121  {
122  mode_ = RequestMode::Window;
123  }
124  else if (modeString.find("ignore") != std::string::npos || modeString.find("Ignore") != std::string::npos)
125  {
126  mode_ = RequestMode::Ignored;
127  }
128  TLOG(TLVL_DEBUG) << "Request mode is " << printMode_();
129 
130  if (mode_ != RequestMode::Ignored)
131  {
132  if (!useDataThread_)
133  {
134  latest_exception_report_ = "Error in CommandableFragmentGenerator: use_data_thread must be true when request_mode is not \"Ignored\"!";
135  throw cet::exception(latest_exception_report_);
136  }
137  requestReceiver_.reset(new RequestReceiver(ps));
138  }
139 }
140 
142 {
143  joinThreads();
144  requestReceiver_.reset(nullptr);
145 }
146 
148 {
149  should_stop_ = true;
150  force_stop_ = true;
151  TLOG(TLVL_DEBUG) << "Joining dataThread";
152  if (dataThread_.joinable()) dataThread_.join();
153  TLOG(TLVL_DEBUG) << "Joining monitoringThread";
154  if (monitoringThread_.joinable()) monitoringThread_.join();
155  TLOG(TLVL_DEBUG) << "joinThreads complete";
156 }
157 
159 {
160  bool result = true;
161 
162  if (check_stop()) usleep(sleep_on_stop_us_);
163  if (exception() || force_stop_) return false;
164 
165  if (!useMonitoringThread_ && monitoringInterval_ > 0)
166  {
167  TLOG(TLVL_GETNEXT) << "getNext: Checking whether to collect Monitoring Data";
168  auto now = std::chrono::steady_clock::now();
169 
170  if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
171  {
172  TLOG(TLVL_GETNEXT) << "getNext: Collecting Monitoring Data";
173  isHardwareOK_ = checkHWStatus_();
174  TLOG(TLVL_GETNEXT) << "getNext: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_;
175  lastMonitoringCall_ = now;
176  }
177  }
178 
179  try
180  {
181  std::lock_guard<std::mutex> lk(mutex_);
182  if (useDataThread_)
183  {
184  TLOG(TLVL_TRACE) << "getNext: Calling applyRequests";
185  result = applyRequests(output);
186  TLOG(TLVL_TRACE) << "getNext: Done with applyRequests result=" << std::boolalpha << result;
187  for (auto dataIter = output.begin(); dataIter != output.end(); ++dataIter)
188  {
189  TLOG(20) << "getNext: applyRequests() returned fragment with sequenceID = " << (*dataIter)->sequenceID()
190  << ", type = " << (*dataIter)->typeString() << ", id = " << std::to_string((*dataIter)->fragmentID())
191  << ", timestamp = " << (*dataIter)->timestamp() << ", and sizeBytes = " << (*dataIter)->sizeBytes();
192  }
193 
194  if (exception())
195  {
196  TLOG(TLVL_ERROR) << "Exception found in BoardReader with board ID " << board_id() << "; BoardReader will now return error status when queried";
197  throw cet::exception("CommandableFragmentGenerator") << "Exception found in BoardReader with board ID " << board_id() << "; BoardReader will now return error status when queried";
198  }
199  }
200  else
201  {
202  if (!isHardwareOK_)
203  {
204  TLOG(TLVL_ERROR) << "Stopping CFG because the hardware reports bad status!";
205  return false;
206  }
207  TLOG(TLVL_TRACE) << "getNext: Calling getNext_ w/ ev_counter()=" << ev_counter();
208  try
209  {
210  result = getNext_(output);
211  }
212  catch (...)
213  {
214  throw;
215  }
216  TLOG(TLVL_TRACE) << "getNext: Done with getNext_ - ev_counter() now " << ev_counter();
217  for (auto dataIter = output.begin(); dataIter != output.end(); ++dataIter)
218  {
219  TLOG(TLVL_GETNEXT_VERBOSE) << "getNext: getNext_() returned fragment with sequenceID = " << (*dataIter)->sequenceID()
220  << ", type = " << (*dataIter)->typeString() << ", id = " << std::to_string((*dataIter)->fragmentID())
221  << ", timestamp = " << (*dataIter)->timestamp() << ", and sizeBytes = " << (*dataIter)->sizeBytes();
222  }
223  }
224  }
225  catch (const cet::exception& e)
226  {
227  latest_exception_report_ = "cet::exception caught in getNext(): ";
228  latest_exception_report_.append(e.what());
229  TLOG(TLVL_ERROR) << "getNext: cet::exception caught: " << e;
230  set_exception(true);
231  return false;
232  }
233  catch (const boost::exception& e)
234  {
235  latest_exception_report_ = "boost::exception caught in getNext(): ";
236  latest_exception_report_.append(boost::diagnostic_information(e));
237  TLOG(TLVL_ERROR) << "getNext: boost::exception caught: " << boost::diagnostic_information(e);
238  set_exception(true);
239  return false;
240  }
241  catch (const std::exception& e)
242  {
243  latest_exception_report_ = "std::exception caught in getNext(): ";
244  latest_exception_report_.append(e.what());
245  TLOG(TLVL_ERROR) << "getNext: std::exception caught: " << e.what();
246  set_exception(true);
247  return false;
248  }
249  catch (...)
250  {
251  latest_exception_report_ = "Unknown exception caught in getNext().";
252  TLOG(TLVL_ERROR) << "getNext: unknown exception caught";
253  set_exception(true);
254  return false;
255  }
256 
257  if (!result)
258  {
259  TLOG(TLVL_DEBUG) << "getNext: Either getNext_ or applyRequests returned false, stopping";
260  }
261 
262  if (metricMan && !output.empty())
263  {
264  auto timestamp = output.front()->timestamp();
265 
266  if (output.size() > 1)
267  { // Only bother sorting if >1 entry
268  for (auto& outputfrag : output)
269  {
270  if (outputfrag->timestamp() > timestamp)
271  {
272  timestamp = outputfrag->timestamp();
273  }
274  }
275  }
276 
277  metricMan->sendMetric("Last Timestamp", timestamp, "Ticks", 1, MetricMode::LastPoint);
278  }
279 
280  return result;
281 }
282 
284 {
285  TLOG(TLVL_CHECKSTOP) << "CFG::check_stop: should_stop=" << should_stop() << ", useDataThread_=" << useDataThread_ << ", exception status =" << int(exception());
286 
287  if (!should_stop()) return false;
288  if (!useDataThread_ || mode_ == RequestMode::Ignored) return true;
289  if (force_stop_) return true;
290 
291  // check_stop returns true if the CFG should stop. We should wait for the RequestReceiver to stop before stopping.
292  TLOG(TLVL_DEBUG) << "should_stop is true, force_stop_ is false, requestReceiver_->isRunning() is " << std::boolalpha << requestReceiver_->isRunning();
293  return !requestReceiver_->isRunning();
294 }
295 
297 {
298  if (force || mode_ == RequestMode::Ignored)
299  {
300  TLOG(TLVL_EVCOUNTERINC) << "ev_counter_inc: Incrementing ev_counter from " << ev_counter() << " by " << step;
301  return ev_counter_.fetch_add(step);
302  }
303  return ev_counter_.load();
304 } // returns the prev value
305 
306 void artdaq::CommandableFragmentGenerator::StartCmd(int run, uint64_t timeout, uint64_t timestamp)
307 {
308  TLOG(TLVL_TRACE) << "Start Command received.";
309  if (run < 0) throw cet::exception("CommandableFragmentGenerator") << "negative run number";
310 
311  timeout_ = timeout;
312  timestamp_ = timestamp;
313  ev_counter_.store(1);
314  {
315  std::unique_lock<std::mutex> lk(dataBuffersMutex_);
316  for (auto& id : dataBuffers_)
317  {
318  id.second.DataBufferDepthBytes = 0;
319  id.second.DataBufferDepthFragments = 0;
320  id.second.HighestRequestSeen = 0;
321  id.second.BufferFragmentKept = false;
322  id.second.DataBuffer.clear();
323  id.second.WindowsSent.clear();
324  }
325  }
326 
327  should_stop_.store(false);
328  force_stop_.store(false);
329  exception_.store(false);
330  run_number_ = run;
331  subrun_number_ = 1;
332  latest_exception_report_ = "none";
333 
334  start();
335 
336  std::unique_lock<std::mutex> lk(mutex_);
337  if (useDataThread_) startDataThread();
338  if (useMonitoringThread_) startMonitoringThread();
339  if (mode_ != RequestMode::Ignored)
340  {
341  requestReceiver_->SetRunNumber(static_cast<uint32_t>(run));
342  requestReceiver_->startRequestReception();
343  }
344  TLOG(TLVL_TRACE) << "Start Command complete.";
345 }
346 
347 void artdaq::CommandableFragmentGenerator::StopCmd(uint64_t timeout, uint64_t timestamp)
348 {
349  TLOG(TLVL_TRACE) << "Stop Command received.";
350 
351  timeout_ = timeout;
352  timestamp_ = timestamp;
353  if (requestReceiver_)
354  {
355  TLOG(TLVL_DEBUG) << "Stopping Request reception BEGIN";
356  requestReceiver_->stopRequestReception();
357  TLOG(TLVL_DEBUG) << "Stopping Request reception END";
358  }
359 
360  stopNoMutex();
361  should_stop_.store(true);
362  std::unique_lock<std::mutex> lk(mutex_);
363  stop();
364 
365  joinThreads();
366  TLOG(TLVL_TRACE) << "Stop Command complete.";
367 }
368 
369 void artdaq::CommandableFragmentGenerator::PauseCmd(uint64_t timeout, uint64_t timestamp)
370 {
371  TLOG(TLVL_TRACE) << "Pause Command received.";
372  timeout_ = timeout;
373  timestamp_ = timestamp;
374  //if (requestReceiver_->isRunning()) requestReceiver_->stopRequestReceiverThread();
375 
376  pauseNoMutex();
377  should_stop_.store(true);
378  std::unique_lock<std::mutex> lk(mutex_);
379 
380  pause();
381 }
382 
383 void artdaq::CommandableFragmentGenerator::ResumeCmd(uint64_t timeout, uint64_t timestamp)
384 {
385  TLOG(TLVL_TRACE) << "Resume Command received.";
386  timeout_ = timeout;
387  timestamp_ = timestamp;
388 
389  subrun_number_ += 1;
390  should_stop_ = false;
391 
392  {
393  std::unique_lock<std::mutex> lk(dataBuffersMutex_);
394  for (auto& id : dataBuffers_)
395  {
396  id.second.DataBufferDepthBytes = 0;
397  id.second.DataBufferDepthFragments = 0;
398  id.second.BufferFragmentKept = false;
399  id.second.DataBuffer.clear();
400  }
401  }
402  // no lock required: thread not started yet
403  resume();
404 
405  std::unique_lock<std::mutex> lk(mutex_);
406  //if (useDataThread_) startDataThread();
407  //if (useMonitoringThread_) startMonitoringThread();
408  //if (mode_ != RequestMode::Ignored && !requestReceiver_->isRunning()) requestReceiver_->startRequestReceiverThread();
409  TLOG(TLVL_TRACE) << "Resume Command complete.";
410 }
411 
412 std::string artdaq::CommandableFragmentGenerator::ReportCmd(std::string const& which)
413 {
414  TLOG(TLVL_TRACE) << "Report Command received.";
415  std::lock_guard<std::mutex> lk(mutex_);
416 
417  // 14-May-2015, KAB: please see the comments associated with the report()
418  // methods in the CommandableFragmentGenerator.hh file for more information
419  // on the use of those methods in this method.
420 
421  // check if the child class has something meaningful for this request
422  std::string childReport = reportSpecific(which);
423  if (childReport.length() > 0) { return childReport; }
424 
425  // handle the requests that we can take care of at this level
426  if (which == "latest_exception")
427  {
428  return latest_exception_report_;
429  }
430 
431  // check if the child class has provided a catch-all report function
432  childReport = report();
433  if (childReport.length() > 0) { return childReport; }
434 
435  // ELF: 5/31/2019: Let BoardReaderCore's report handle this...
436  /*
437  // if we haven't been able to come up with any report so far, say so
438  std::string tmpString = "The \"" + which + "\" command is not ";
439  tmpString.append("currently supported by the ");
440  tmpString.append(metricsReportingInstanceName());
441  tmpString.append(" fragment generator.");
442  */
443  TLOG(TLVL_TRACE) << "Report Command complete.";
444  return ""; //tmpString;
445 }
446 
447 // Default implemenetations of state functions
449 {
450 #pragma message "Using default implementation of CommandableFragmentGenerator::pauseNoMutex()"
451 }
452 
454 {
455 #pragma message "Using default implementation of CommandableFragmentGenerator::pause()"
456 }
457 
459 #pragma message "Using default implementation of CommandableFragmentGenerator::resume()"
460 }
461 
463 {
464 #pragma message "Using default implementation of CommandableFragmentGenerator::report()"
465  return "";
466 }
467 
469 {
470 #pragma message "Using default implementation of CommandableFragmentGenerator::reportSpecific(std::string)"
471  return "";
472 }
473 
475 {
476 #pragma message "Using default implementation of CommandableFragmentGenerator::checkHWStatus_()"
477  return true;
478 }
479 
480 bool artdaq::CommandableFragmentGenerator::metaCommand(std::string const&, std::string const&)
481 {
482 #pragma message "Using default implementation of CommandableFragmentGenerator::metaCommand(std::string, std::string)"
483  return true;
484 }
485 
487 {
488  if (dataThread_.joinable()) dataThread_.join();
489  TLOG(TLVL_INFO) << "Starting Data Receiver Thread";
490  try
491  {
492  dataThread_ = boost::thread(&CommandableFragmentGenerator::getDataLoop, this);
493  }
494  catch (const boost::exception& e)
495  {
496  TLOG(TLVL_ERROR) << "Caught boost::exception starting Data Receiver thread: " << boost::diagnostic_information(e) << ", errno=" << errno;
497  std::cerr << "Caught boost::exception starting Data Receiver thread: " << boost::diagnostic_information(e) << ", errno=" << errno << std::endl;
498  exit(5);
499  }
500 }
501 
503 {
504  if (monitoringThread_.joinable()) monitoringThread_.join();
505  TLOG(TLVL_INFO) << "Starting Hardware Monitoring Thread";
506  try
507  {
508  monitoringThread_ = boost::thread(&CommandableFragmentGenerator::getMonitoringDataLoop, this);
509  }
510  catch (const boost::exception& e)
511  {
512  TLOG(TLVL_ERROR) << "Caught boost::exception starting Hardware Monitoring thread: " << boost::diagnostic_information(e) << ", errno=" << errno;
513  std::cerr << "Caught boost::exception starting Hardware Monitoring thread: " << boost::diagnostic_information(e) << ", errno=" << errno << std::endl;
514  exit(5);
515  }
516 }
517 
519 {
520  switch (mode_)
521  {
522  case RequestMode::Single:
523  return "Single";
524  case RequestMode::Buffer:
525  return "Buffer";
526  case RequestMode::Window:
527  return "Window";
528  case RequestMode::Ignored:
529  return "Ignored";
530  }
531 
532  return "ERROR";
533 }
534 
535 //
536 // The "useDataThread_" thread
537 //
539 {
540  data_thread_running_ = true;
541  while (!force_stop_)
542  {
543  if (!isHardwareOK_)
544  {
545  TLOG(TLVL_DEBUG) << "getDataLoop: isHardwareOK is " << isHardwareOK_ << ", aborting data thread";
546  data_thread_running_ = false;
547  return;
548  }
549 
550  TLOG(TLVL_GETDATALOOP) << "getDataLoop: calling getNext_";
551 
552  bool data = false;
553  auto startdata = std::chrono::steady_clock::now();
554  FragmentPtrs newData;
555 
556  try
557  {
558  data = getNext_(newData);
559  }
560  catch (...)
561  {
562  ExceptionHandler(ExceptionHandlerRethrow::no,
563  "Exception thrown by fragment generator in CommandableFragmentGenerator::getDataLoop; setting exception state to \"true\"");
564  set_exception(true);
565 
566  data_thread_running_ = false;
567  return;
568  }
569 
570  if (metricMan)
571  {
572  metricMan->sendMetric("Avg Data Acquisition Time", TimeUtils::GetElapsedTime(startdata), "s", 3, artdaq::MetricMode::Average);
573  }
574 
575  if (newData.size() == 0 && sleep_on_no_data_us_ > 0)
576  {
577  usleep(sleep_on_no_data_us_);
578  }
579 
580  auto dataIter = newData.begin();
581  while (dataIter != newData.end())
582  {
583  TLOG(TLVL_GETDATALOOP_VERBOSE) << "getDataLoop: getNext_() returned fragment with timestamp = " << (*dataIter)->timestamp() << ", and sizeBytes = " << (*dataIter)->sizeBytes();
584 
585  auto frag_id = (*dataIter)->fragmentID();
586  if (!dataBuffers_.count(frag_id))
587  {
588  throw cet::exception("DataBufferError") << "Error in CommandableFragmentGenerator: Recevied Fragment with fragment_id " << frag_id << ", but this ID was not declared in fragment_ids!";
589  }
590 
591  TLOG(TLVL_GETDATALOOP_DATABUFFWAIT) << "Waiting for data buffer ready";
592  if (!waitForDataBufferReady(frag_id)) return;
593  TLOG(TLVL_GETDATALOOP_DATABUFFWAIT) << "Done waiting for data buffer ready";
594 
595  TLOG(TLVL_GETDATALOOP) << "getDataLoop: processing data";
596  if (data && !force_stop_)
597  {
598  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
599  switch (mode_)
600  {
601  case RequestMode::Single:
602  dataBuffers_[frag_id].DataBuffer.clear();
603  dataBuffers_[frag_id].DataBufferDepthBytes = (*dataIter)->sizeBytes();
604  dataBuffers_[frag_id].DataBuffer.emplace_back(std::move(*dataIter));
605  dataIter = newData.erase(dataIter);
606  break;
607  case RequestMode::Buffer:
608  case RequestMode::Ignored:
609  case RequestMode::Window:
610  default:
611  //dataBuffer_.reserve(dataBuffer_.size() + newDataBuffer_.size());
612  dataBuffers_[frag_id].DataBufferDepthBytes += (*dataIter)->sizeBytes();
613  dataBuffers_[frag_id].DataBuffer.emplace_back(std::move(*dataIter));
614  dataIter = newData.erase(dataIter);
615  break;
616  }
617  getDataBufferStats(frag_id);
618  }
619  else
620  {
621  break;
622  }
623  }
624 
625  {
626  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
627  for (auto& id : dataBuffers_)
628  {
629  if (id.second.DataBuffer.size() > 0)
630  {
631  dataCondition_.notify_all();
632  break;
633  }
634  }
635  }
636  if (!data || force_stop_)
637  {
638  TLOG(TLVL_INFO) << "Data flow has stopped. Ending data collection thread";
639  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
640  data_thread_running_ = false;
641  if (requestReceiver_) requestReceiver_->ClearRequests();
642  newData.clear();
643  TLOG(TLVL_INFO) << "getDataLoop: Ending thread";
644  return;
645  }
646  }
647 }
648 
650 {
651  if (!dataBuffers_.count(id))
652  {
653  throw cet::exception("DataBufferError") << "Error in CommandableFragmentGenerator: Cannot wait for data buffer for ID " << id << " because it does not exist!";
654  }
655  auto startwait = std::chrono::steady_clock::now();
656  auto first = true;
657  auto lastwaittime = 0ULL;
658 
659  {
660  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
661  getDataBufferStats(id);
662  }
663 
664  while (dataBufferIsTooLarge(id))
665  {
666  if (!circularDataBufferMode_)
667  {
668  if (should_stop())
669  {
670  TLOG(TLVL_DEBUG) << "Run ended while waiting for buffer to shrink!";
671  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
672  getDataBufferStats(id);
673  dataCondition_.notify_all();
674  data_thread_running_ = false;
675  return false;
676  }
677  auto waittime = TimeUtils::GetElapsedTimeMilliseconds(startwait);
678 
679  if (first || (waittime != lastwaittime && waittime % 1000 == 0))
680  {
681  TLOG(TLVL_WARNING) << "Bad Omen: Data Buffer has exceeded its size limits. "
682  << "(seq_id=" << ev_counter() << ", frag_id=" << id
683  << ", frags=" << dataBuffers_[id].DataBufferDepthFragments << "/" << maxDataBufferDepthFragments_
684  << ", szB=" << dataBuffers_[id].DataBufferDepthBytes << "/" << maxDataBufferDepthBytes_ << ")";
685  TLOG(TLVL_TRACE) << "Bad Omen: Possible causes include requests not getting through or Ignored-mode BR issues";
686  first = false;
687  }
688  if (waittime % 5 && waittime != lastwaittime)
689  {
690  TLOG(TLVL_WAITFORBUFFERREADY) << "getDataLoop: Data Retreival paused for " << waittime << " ms waiting for data buffer to drain";
691  }
692  lastwaittime = waittime;
693  usleep(1000);
694  }
695  else
696  {
697  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
698  getDataBufferStats(id); // Re-check under lock
699  if (dataBufferIsTooLarge(id))
700  {
701  auto begin = dataBuffers_[id].DataBuffer.begin();
702  if (begin == dataBuffers_[id].DataBuffer.end())
703  {
704  TLOG(TLVL_WARNING) << "Data buffer is reported as too large, but doesn't contain any Fragments! Possible corrupt memory!";
705  continue;
706  }
707  if (*begin)
708  {
709  TLOG(TLVL_WAITFORBUFFERREADY) << "waitForDataBufferReady: Dropping Fragment with timestamp " << (*begin)->timestamp() << " from data buffer (Buffer over-size, circular data buffer mode)";
710  }
711  dataBuffers_[id].DataBufferDepthBytes -= (*begin)->sizeBytes();
712  dataBuffers_[id].DataBuffer.erase(begin);
713  dataBuffers_[id].BufferFragmentKept = false; // If any Fragments are removed from data buffer, then we know we don't have to ignore the first one anymore
714  getDataBufferStats(id);
715  }
716  }
717  }
718  return true;
719 }
720 
722 {
723  if (!dataBuffers_.count(id))
724  {
725  throw cet::exception("DataBufferError") << "Error in CommandableFragmentGenerator: Cannot check size of data buffer for ID " << id << " because it does not exist!";
726  }
727  return (maxDataBufferDepthFragments_ > 0 && dataBuffers_[id].DataBufferDepthFragments > maxDataBufferDepthFragments_) || (maxDataBufferDepthBytes_ > 0 && dataBuffers_[id].DataBufferDepthBytes > maxDataBufferDepthBytes_);
728 }
729 
731 {
732  if (!dataBuffers_.count(id))
733  {
734  throw cet::exception("DataBufferError") << "Error in CommandableFragmentGenerator: Cannot get stats of data buffer for ID " << id << " because it does not exist!";
735  }
737  dataBuffers_[id].DataBufferDepthFragments = dataBuffers_[id].DataBuffer.size();
738 
739  if (metricMan)
740  {
741  TLOG(TLVL_GETBUFFERSTATS) << "getDataBufferStats: Sending Metrics";
742  metricMan->sendMetric("Buffer Depth Fragments", dataBuffers_[id].DataBufferDepthFragments.load(), "fragments", 1, MetricMode::LastPoint);
743  metricMan->sendMetric("Buffer Depth Bytes", dataBuffers_[id].DataBufferDepthBytes.load(), "bytes", 1, MetricMode::LastPoint);
744 
745  auto bufferDepthFragmentsPercent = dataBuffers_[id].DataBufferDepthFragments.load() * 100 / static_cast<double>(maxDataBufferDepthFragments_);
746  auto bufferDepthBytesPercent = dataBuffers_[id].DataBufferDepthBytes.load() * 100 / static_cast<double>(maxDataBufferDepthBytes_);
747  metricMan->sendMetric("Fragment Generator Buffer Full %Fragments", bufferDepthFragmentsPercent, "%", 3, MetricMode::LastPoint);
748  metricMan->sendMetric("Fragment Generator Buffer Full %Bytes", bufferDepthBytesPercent, "%", 3, MetricMode::LastPoint);
749  metricMan->sendMetric("Fragment Generator Buffer Full %", bufferDepthFragmentsPercent > bufferDepthBytesPercent ? bufferDepthFragmentsPercent : bufferDepthBytesPercent, "%", 1, MetricMode::LastPoint);
750  }
751  TLOG(TLVL_GETBUFFERSTATS) << "getDataBufferStats: frags=" << dataBuffers_[id].DataBufferDepthFragments.load() << "/" << maxDataBufferDepthFragments_
752  << ", sz=" << dataBuffers_[id].DataBufferDepthBytes.load() << "/" << maxDataBufferDepthBytes_;
753 }
754 
756 {
757  if (!dataBuffers_.count(id))
758  {
759  throw cet::exception("DataBufferError") << "Error in CommandableFragmentGenerator: Cannot check data buffer for ID " << id << " because it does not exist!";
760  }
761 
762  if (dataBuffers_[id].DataBufferDepthFragments > 0 && mode_ != RequestMode::Single && mode_ != RequestMode::Ignored)
763  {
764  // Eliminate extra fragments
765  getDataBufferStats(id);
766  while (dataBufferIsTooLarge(id))
767  {
768  auto begin = dataBuffers_[id].DataBuffer.begin();
769  TLOG(TLVL_CHECKDATABUFFER) << "checkDataBuffer: Dropping Fragment with timestamp " << (*begin)->timestamp() << " from data buffer (Buffer over-size)";
770  dataBuffers_[id].DataBufferDepthBytes -= (*begin)->sizeBytes();
771  dataBuffers_[id].DataBuffer.erase(begin);
772  dataBuffers_[id].BufferFragmentKept = false; // If any Fragments are removed from data buffer, then we know we don't have to ignore the first one anymore
773  getDataBufferStats(id);
774  }
775  if (dataBuffers_[id].DataBuffer.size() > 0)
776  {
777  TLOG(TLVL_CHECKDATABUFFER) << "Determining if Fragments can be dropped from data buffer";
778  Fragment::timestamp_t last = dataBuffers_[id].DataBuffer.back()->timestamp();
779  Fragment::timestamp_t min = last > staleTimeout_ ? last - staleTimeout_ : 0;
780  for (auto it = dataBuffers_[id].DataBuffer.begin(); it != dataBuffers_[id].DataBuffer.end();)
781  {
782  if ((*it)->timestamp() < min)
783  {
784  TLOG(TLVL_CHECKDATABUFFER) << "checkDataBuffer: Dropping Fragment with timestamp " << (*it)->timestamp() << " from data buffer (timeout=" << staleTimeout_ << ", min=" << min << ")";
785  dataBuffers_[id].DataBufferDepthBytes -= (*it)->sizeBytes();
786  dataBuffers_[id].BufferFragmentKept = false; // If any Fragments are removed from data buffer, then we know we don't have to ignore the first one anymore
787  it = dataBuffers_[id].DataBuffer.erase(it);
788  }
789  else
790  {
791  break;
792  }
793  }
794  getDataBufferStats(id);
795  }
796  }
797 }
798 
800 {
801  while (!force_stop_)
802  {
803  if (should_stop() || monitoringInterval_ <= 0)
804  {
805  TLOG(TLVL_DEBUG) << "getMonitoringDataLoop: should_stop() is " << std::boolalpha << should_stop()
806  << " and monitoringInterval is " << monitoringInterval_ << ", returning";
807  return;
808  }
809  TLOG(TLVL_GETMONITORINGDATA) << "getMonitoringDataLoop: Determining whether to call checkHWStatus_";
810 
811  auto now = std::chrono::steady_clock::now();
812  if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
813  {
814  isHardwareOK_ = checkHWStatus_();
815  TLOG(TLVL_GETMONITORINGDATA) << "getMonitoringDataLoop: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_;
816  lastMonitoringCall_ = now;
817  }
818  usleep(monitoringInterval_ / 10);
819  }
820 }
821 
823 {
824  // dataBuffersMutex_ is held by calling function
825  // We just copy everything that's here into the output.
826  TLOG(TLVL_APPLYREQUESTS) << "Mode is Ignored; Copying data to output";
827  for (auto& id : dataBuffers_)
828  {
829  std::move(id.second.DataBuffer.begin(), id.second.DataBuffer.end(), std::inserter(frags, frags.end()));
830  id.second.DataBufferDepthBytes = 0;
831  id.second.DataBufferDepthFragments = 0;
832  id.second.BufferFragmentKept = false;
833  id.second.DataBuffer.clear();
834  }
835 }
836 
838 {
839  // We only care about the latest request received. Send empties for all others.
840  auto requests = requestReceiver_->GetRequests();
841  while (requests.size() > 1)
842  {
843  // std::map is ordered by key => Last sequence ID in the map is the one we care about
844  requestReceiver_->RemoveRequest(requests.begin()->first);
845  requests.erase(requests.begin());
846  }
847  sendEmptyFragments(frags, requests);
848 
849  // If no requests remain after sendEmptyFragments, return
850  if (requests.size() == 0 || !requests.count(ev_counter())) return;
851 
852  for (auto& id : dataBuffers_)
853  {
854  if (id.second.DataBuffer.size() > 0)
855  {
856  assert(id.second.DataBuffer.size() == 1);
857  TLOG(TLVL_APPLYREQUESTS) << "Mode is Single; Sending copy of last event";
858  for (auto& fragptr : id.second.DataBuffer)
859  {
860  // Return the latest data point
861  auto frag = fragptr.get();
862  auto newfrag = std::unique_ptr<artdaq::Fragment>(new Fragment(ev_counter(), frag->fragmentID()));
863  newfrag->resize(frag->size() - detail::RawFragmentHeader::num_words());
864  memcpy(newfrag->headerAddress(), frag->headerAddress(), frag->sizeBytes());
865  newfrag->setTimestamp(requests[ev_counter()]);
866  newfrag->setSequenceID(ev_counter());
867  frags.push_back(std::move(newfrag));
868  }
869  }
870  else
871  {
872  sendEmptyFragment(frags, ev_counter(), id.first, "No data for");
873  }
874  }
875  requestReceiver_->RemoveRequest(ev_counter());
876  ev_counter_inc(1, true);
877 }
878 
880 {
881  // We only care about the latest request received. Send empties for all others.
882  auto requests = requestReceiver_->GetRequests();
883  while (requests.size() > 1)
884  {
885  // std::map is ordered by key => Last sequence ID in the map is the one we care about
886  requestReceiver_->RemoveRequest(requests.begin()->first);
887  requests.erase(requests.begin());
888  }
889  sendEmptyFragments(frags, requests);
890 
891  // If no requests remain after sendEmptyFragments, return
892  if (requests.size() == 0 || !requests.count(ev_counter())) return;
893 
894  for (auto& id : dataBuffers_)
895  {
896  TLOG(TLVL_DEBUG) << "Creating ContainerFragment for Buffered Fragments";
897  frags.emplace_back(new artdaq::Fragment(ev_counter(), id.first));
898  frags.back()->setTimestamp(requests[ev_counter()]);
899  ContainerFragmentLoader cfl(*frags.back());
900  cfl.set_missing_data(false); // Buffer mode is never missing data, even if there IS no data.
901 
902  // If we kept a Fragment from the previous iteration, but more data has arrived, discard it
903  auto it = id.second.DataBuffer.begin();
904  if (id.second.BufferFragmentKept && id.second.DataBuffer.size() > 1)
905  {
906  id.second.DataBufferDepthBytes -= (*it)->sizeBytes();
907  it = id.second.DataBuffer.erase(it);
908  }
909 
910  // Buffer mode TFGs should simply copy out the whole dataBuffer_ into a ContainerFragment
911  while (it != id.second.DataBuffer.end())
912  {
913  TLOG(TLVL_APPLYREQUESTS) << "ApplyRequests: Adding Fragment with timestamp " << (*it)->timestamp() << " to Container with sequence ID " << ev_counter();
914  cfl.addFragment(*it);
915  if (bufferModeKeepLatest_ && id.second.DataBuffer.size() == 1)
916  {
917  id.second.BufferFragmentKept = true;
918  break;
919  }
920  id.second.DataBufferDepthBytes -= (*it)->sizeBytes();
921  it = id.second.DataBuffer.erase(it);
922  }
923  }
924  requestReceiver_->RemoveRequest(ev_counter());
925  ev_counter_inc(1, true);
926 }
927 
928 void artdaq::CommandableFragmentGenerator::applyRequestsWindowMode_CheckAndFillDataBuffer(artdaq::FragmentPtrs& frags, artdaq::Fragment::fragment_id_t id, artdaq::Fragment::sequence_id_t seq, artdaq::Fragment::timestamp_t ts)
929 {
930  TLOG(TLVL_APPLYREQUESTS) << "applyRequestsWindowMode_CheckAndFillDataBuffer: Checking that data exists for request window " << seq;
931  Fragment::timestamp_t min = ts > windowOffset_ ? ts - windowOffset_ : 0;
932  Fragment::timestamp_t max = min + windowWidth_;
933  TLOG(TLVL_APPLYREQUESTS) << "ApplyRequestsWindowsMode_CheckAndFillDataBuffer: min is " << min << ", max is " << max
934  << " and first/last points in buffer are " << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.front()->timestamp() : 0)
935  << "/" << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.back()->timestamp() : 0)
936  << " (sz=" << dataBuffers_[id].DataBuffer.size() << " [" << dataBuffers_[id].DataBufferDepthBytes.load()
937  << "/" << maxDataBufferDepthBytes_ << "])";
938  bool windowClosed = dataBuffers_[id].DataBuffer.size() > 0 && dataBuffers_[id].DataBuffer.back()->timestamp() >= max;
939  bool windowTimeout = !windowClosed && TimeUtils::GetElapsedTimeMicroseconds(requestReceiver_->GetRequestTime(seq)) > window_close_timeout_us_;
940  if (windowTimeout)
941  {
942  TLOG(TLVL_WARNING) << "applyRequestsWindowMode_CheckAndFillDataBuffer: A timeout occurred waiting for data to close the request window ({" << min << "-" << max
943  << "}, buffer={" << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.front()->timestamp() : 0) << "-"
944  << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.back()->timestamp() : 0)
945  << "} ). Time waiting: "
946  << TimeUtils::GetElapsedTimeMicroseconds(requestReceiver_->GetRequestTime(seq)) << " us "
947  << "(> " << window_close_timeout_us_ << " us).";
948  }
949  if (windowClosed || !data_thread_running_ || windowTimeout)
950  {
951  TLOG(TLVL_DEBUG) << "applyRequestsWindowMode_CheckAndFillDataBuffer: Creating ContainerFragment for Window-requested Fragments";
952  frags.emplace_back(new artdaq::Fragment(seq, id));
953  frags.back()->setTimestamp(ts);
954  ContainerFragmentLoader cfl(*frags.back());
955 
956  // In the spirit of NOvA's MegaPool: (RS = Request start (min), RE = Request End (max))
957  // --- | Buffer Start | --- | Buffer End | ---
958  //1. RS RE | | | |
959  //2. RS | | RE | |
960  //3. RS | | | | RE
961  //4. | | RS RE | |
962  //5. | | RS | | RE
963  //6. | | | | RS RE
964  //
965  // If RE (or RS) is after the end of the buffer, we wait for window_close_timeout_us_. If we're here, then that means that windowClosed is false, and the missing_data flag should be set.
966  // If RS (or RE) is before the start of the buffer, then missing_data should be set to true, as data is assumed to arrive in the buffer in timestamp order
967  // If the dataBuffer has size 0, then windowClosed will be false
968  if (!windowClosed || (dataBuffers_[id].DataBuffer.size() > 0 && dataBuffers_[id].DataBuffer.front()->timestamp() > min))
969  {
970  TLOG(TLVL_DEBUG) << "applyRequestsWindowMode_CheckAndFillDataBuffer: Request window starts before and/or ends after the current data buffer, setting ContainerFragment's missing_data flag!"
971  << " (requestWindowRange=[" << min << "," << max << "], "
972  << "buffer={" << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.front()->timestamp() : 0) << "-"
973  << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.back()->timestamp() : 0) << "}";
974  cfl.set_missing_data(true);
975  }
976 
977  // Do a little bit more work to decide which fragments to send for a given request
978  for (auto it = dataBuffers_[id].DataBuffer.begin(); it != dataBuffers_[id].DataBuffer.end();)
979  {
980  Fragment::timestamp_t fragT = (*it)->timestamp();
981  if (fragT < min || fragT > max || (fragT == max && windowWidth_ > 0))
982  {
983  ++it;
984  continue;
985  }
986 
987  TLOG(TLVL_APPLYREQUESTS) << "applyRequestsWindowMode_CheckAndFillDataBuffer: Adding Fragment with timestamp " << (*it)->timestamp() << " to Container";
988  cfl.addFragment(*it);
989 
990  if (uniqueWindows_)
991  {
992  dataBuffers_[id].DataBufferDepthBytes -= (*it)->sizeBytes();
993  it = dataBuffers_[id].DataBuffer.erase(it);
994  }
995  else
996  {
997  ++it;
998  }
999  }
1000 
1001  dataBuffers_[id].WindowsSent[seq] = std::chrono::steady_clock::now();
1002  if (seq > dataBuffers_[id].HighestRequestSeen) dataBuffers_[id].HighestRequestSeen = seq;
1003  }
1004 }
1005 
1007 {
1008  TLOG(TLVL_APPLYREQUESTS) << "applyRequestsWindowMode BEGIN";
1009 
1010  auto requests = requestReceiver_->GetRequests();
1011 
1012  TLOG(TLVL_APPLYREQUESTS) << "applyRequestsWindowMode: Starting request processing";
1013  for (auto req = requests.begin(); req != requests.end();)
1014  {
1015  TLOG(TLVL_APPLYREQUESTS) << "applyRequestsWindowMode: processing request with sequence ID " << req->first << ", timestamp " << req->second;
1016 
1017  while (req->first < ev_counter() && requests.size() > 0)
1018  {
1019  TLOG(TLVL_APPLYREQUESTS) << "applyRequestsWindowMode: Clearing passed request for sequence ID " << req->first;
1020  requestReceiver_->RemoveRequest(req->first);
1021  req = requests.erase(req);
1022  }
1023  if (requests.size() == 0) break;
1024 
1025  for (auto& id : dataBuffers_)
1026  {
1027  if (!id.second.WindowsSent.count(req->first))
1028  {
1029  applyRequestsWindowMode_CheckAndFillDataBuffer(frags, id.first, req->first, req->second);
1030  }
1031  }
1032  checkSentWindows(req->first);
1033  ++req;
1034  }
1035 
1036  // Check sent windows for requests that can be removed
1037  for (auto& id : dataBuffers_)
1038  {
1039  std::set<artdaq::Fragment::sequence_id_t> seqs;
1040  for (auto& seq : id.second.WindowsSent)
1041  {
1042  seqs.insert(seq.first);
1043  }
1044  for (auto& seq : seqs)
1045  {
1046  checkSentWindows(seq);
1047  }
1048  }
1049 }
1050 
1052 {
1053  if (check_stop() || exception())
1054  {
1055  return false;
1056  }
1057 
1058  // Wait for data, if in ignored mode, or a request otherwise
1059  if (mode_ == RequestMode::Ignored)
1060  {
1061  while (dataBufferFragmentCount_() == 0)
1062  {
1063  if (check_stop() || exception() || !isHardwareOK_) return false;
1064  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
1065  dataCondition_.wait_for(lock, std::chrono::milliseconds(10), [this]() { return dataBufferFragmentCount_() > 0; });
1066  }
1067  }
1068  else
1069  {
1070  if ((check_stop() && requestReceiver_->size() == 0) || exception()) return false;
1071 
1072  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
1073  dataCondition_.wait_for(lock, std::chrono::milliseconds(10));
1074 
1075  checkDataBuffers();
1076 
1077  // Wait up to 1000 ms for a request...
1078  auto counter = 0;
1079 
1080  while (requestReceiver_->size() == 0 && counter < 100)
1081  {
1082  if (check_stop() || exception()) return false;
1083 
1084  checkDataBuffers();
1085 
1086  requestReceiver_->WaitForRequests(10); // milliseconds
1087  counter++;
1088  }
1089  }
1090 
1091  {
1092  std::unique_lock<std::mutex> dlk(dataBuffersMutex_);
1093 
1094  switch (mode_)
1095  {
1096  case RequestMode::Single:
1097  applyRequestsSingleMode(frags);
1098  break;
1099  case RequestMode::Window:
1100  applyRequestsWindowMode(frags);
1101  break;
1102  case RequestMode::Buffer:
1103  applyRequestsBufferMode(frags);
1104  break;
1105  case RequestMode::Ignored:
1106  default:
1107  applyRequestsIgnoredMode(frags);
1108  break;
1109  }
1110 
1111  if (!data_thread_running_ || force_stop_)
1112  {
1113  TLOG(TLVL_INFO) << "Data thread has stopped; Clearing data buffers";
1114  for (auto& id : dataBuffers_)
1115  {
1116  id.second.DataBufferDepthBytes = 0;
1117  id.second.DataBufferDepthFragments = 0;
1118  id.second.BufferFragmentKept = false;
1119  id.second.DataBuffer.clear();
1120  }
1121  }
1122 
1123  getDataBuffersStats();
1124  }
1125 
1126  if (frags.size() > 0)
1127  TLOG(TLVL_APPLYREQUESTS) << "Finished Processing requests, returning " << frags.size() << " fragments, current ev_counter is " << ev_counter();
1128  return true;
1129 }
1130 
1131 bool artdaq::CommandableFragmentGenerator::sendEmptyFragment(artdaq::FragmentPtrs& frags, size_t seqId, Fragment::fragment_id_t fragmentId, std::string desc)
1132 {
1133  TLOG(TLVL_WARNING) << desc << " sequence ID " << seqId << ", sending empty fragment";
1134  auto frag = new Fragment();
1135  frag->setSequenceID(seqId);
1136  frag->setFragmentID(fragmentId);
1137  frag->setSystemType(Fragment::EmptyFragmentType);
1138  frags.emplace_back(FragmentPtr(frag));
1139  return true;
1140 }
1141 
1142 void artdaq::CommandableFragmentGenerator::sendEmptyFragments(artdaq::FragmentPtrs& frags, std::map<Fragment::sequence_id_t, Fragment::timestamp_t>& requests)
1143 {
1144  if (requests.size() > 0)
1145  {
1146  TLOG(TLVL_SENDEMPTYFRAGMENTS) << "Sending Empty Fragments for Sequence IDs from " << ev_counter() << " up to but not including " << requests.begin()->first;
1147  while (requests.begin()->first > ev_counter())
1148  {
1149  for (auto& fid : dataBuffers_)
1150  {
1151  sendEmptyFragment(frags, ev_counter(), fid.first, "Missed request for");
1152  }
1153  ev_counter_inc(1, true);
1154  }
1155  }
1156 }
1157 
1158 void artdaq::CommandableFragmentGenerator::checkSentWindows(artdaq::Fragment::sequence_id_t seq)
1159 {
1160  bool seqComplete = true;
1161  bool seqTimeout = false;
1162  for (auto& id : dataBuffers_)
1163  {
1164  if (!id.second.WindowsSent.count(seq) || id.second.HighestRequestSeen < seq)
1165  {
1166  seqComplete = false;
1167  }
1168  if (id.second.WindowsSent.count(seq) && TimeUtils::GetElapsedTimeMicroseconds(id.second.WindowsSent[seq]) > missing_request_window_timeout_us_)
1169  {
1170  seqTimeout = true;
1171  }
1172  }
1173  if (seqComplete)
1174  {
1175  TLOG(TLVL_CHECKWINDOWS) << "checkSentWindows: Request " << seq << " is complete, removing from requestReceiver.";
1176  requestReceiver_->RemoveRequest(seq);
1177 
1178  if (ev_counter() == seq)
1179  {
1180  TLOG(TLVL_CHECKWINDOWS) << "checkSentWindows: Sequence ID matches ev_counter, incrementing ev_counter (" << ev_counter() << ")";
1181 
1182  for (auto& id : dataBuffers_)
1183  {
1184  id.second.WindowsSent.erase(seq);
1185  }
1186 
1187  ev_counter_inc(1, true);
1188  }
1189  }
1190  if (seqTimeout)
1191  {
1192  TLOG(TLVL_CHECKWINDOWS) << "checkSentWindows: Sent Window history indicates that requests between " << ev_counter() << " and " << seq << " have timed out.";
1193  while (ev_counter() <= seq)
1194  {
1195  if (ev_counter() < seq) TLOG(TLVL_WARNING) << "Missed request for sequence ID " << ev_counter() << "! Will not send any data for this sequence ID!";
1196  requestReceiver_->RemoveRequest(ev_counter());
1197 
1198  for (auto& id : dataBuffers_)
1199  {
1200  id.second.WindowsSent.erase(ev_counter());
1201  }
1202 
1203  ev_counter_inc(1, true);
1204  }
1205  }
1206 }
CommandableFragmentGenerator(const fhicl::ParameterSet &ps)
CommandableFragmentGenerator Constructor.
void applyRequestsSingleMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Single. Precondition: dataBufferMutex_ and reques...
virtual bool checkHWStatus_()
Check any relavent hardware status registers. Return false if an error condition exists that should h...
virtual ~CommandableFragmentGenerator()
CommandableFragmentGenerator Destructor.
void applyRequestsBufferMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Buffer. Precondition: dataBufferMutex_ and reques...
void getDataBufferStats(Fragment::fragment_id_t id)
Calculate the size of the dataBuffer and report appropriate metrics.
void getMonitoringDataLoop()
This function regularly calls checkHWStatus_(), and sets the isHardwareOK flag accordingly.
void startDataThread()
Function that launches the data thread (getDataLoop())
artdaq::Fragment::fragment_id_t fragment_id() const
Get the Fragment ID of this Fragment generator.
std::string ReportCmd(std::string const &which="")
Get a report about a user-specified run-time quantity.
virtual bool metaCommand(std::string const &command, std::string const &arg)
The meta-command is used for implementing user-specific commands in a CommandableFragmentGenerator.
void StopCmd(uint64_t timeout, uint64_t timestamp)
Stop the CommandableFragmentGenerator.
void applyRequestsWindowMode_CheckAndFillDataBuffer(artdaq::FragmentPtrs &frags, artdaq::Fragment::fragment_id_t id, artdaq::Fragment::sequence_id_t seq, artdaq::Fragment::timestamp_t ts)
Copy data from the relevant data buffer that matches the given timestamp.
void checkSentWindows(Fragment::sequence_id_t seq)
Check the windows_sent_ooo_ map for sequence IDs that may be removed.
bool sendEmptyFragment(FragmentPtrs &frags, size_t sequenceId, Fragment::fragment_id_t fragmentId, std::string desc)
Send an EmptyFragmentType Fragment.
void applyRequestsWindowMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Window. Precondition: dataBufferMutex_ and reques...
void StartCmd(int run, uint64_t timeout, uint64_t timestamp)
Start the CommandableFragmentGenerator.
virtual void pauseNoMutex()
On call to PauseCmd, pauseNoMutex() is called prior to PauseCmd acquiring the mutex ...
bool check_stop()
Routine used by applyRequests to make sure that all outstanding requests have been fulfilled before r...
void ResumeCmd(uint64_t timeout, uint64_t timestamp)
Resume the CommandableFragmentGenerator.
bool getNext(FragmentPtrs &output) overridefinal
getNext calls either applyRequests or getNext_ to get any data that is ready to be sent to the EventB...
size_t ev_counter_inc(size_t step=1, bool force=false)
Increment the event counter, if the current RequestMode allows it.
bool applyRequests(FragmentPtrs &frags)
See if any requests have been received, and add the corresponding data Fragment objects to the output...
void applyRequestsIgnoredMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Ignored. Precondition: dataBufferMutex_ and reque...
void PauseCmd(uint64_t timeout, uint64_t timestamp)
Pause the CommandableFragmentGenerator.
void getDataLoop()
When separate_data_thread is set to true, this loop repeatedly calls getNext_ and adds returned Fragm...
void sendEmptyFragments(FragmentPtrs &frags, std::map< Fragment::sequence_id_t, Fragment::timestamp_t > &requests)
This function is for Buffered and Single request modes, as they can only respond to one data request ...
Receive data requests and make them available to CommandableFragmentGenerator or other interested par...
void startMonitoringThread()
Function that launches the monitoring thread (getMonitoringDataLoop())
virtual void pause()
If a CommandableFragmentGenerator subclass is reading from hardware, the implementation of pause() sh...
virtual void resume()
The subrun number will be incremented before a call to resume.
bool waitForDataBufferReady(Fragment::fragment_id_t id)
Wait for the data buffer to drain (dataBufferIsTooLarge returns false), periodically reporting status...
virtual std::string report()
Let&#39;s say that the contract with the report() functions is that they return a non-empty string if the...
std::string printMode_()
Return the string representation of the current RequestMode.
void checkDataBuffer(Fragment::fragment_id_t id)
Perform data buffer pruning operations for the given buffer. If the RequestMode is Single...
virtual std::string reportSpecific(std::string const &what)
Report the status of a specific quantity
bool dataBufferIsTooLarge(Fragment::fragment_id_t id)
Test the configured constraints on the data buffer.
void joinThreads()
Join any data-taking threads. Should be called when destructing CommandableFragmentGenerator.