artdaq  v3_06_00
CommandableFragmentGenerator.cc
1 #include "artdaq/DAQdata/Globals.hh"
2 #define TRACE_NAME (app_name + "_CommandableFragmentGenerator").c_str() // include these 2 first -
3 
4 #include "artdaq/Generators/CommandableFragmentGenerator.hh"
5 
6 #include <boost/exception/all.hpp>
7 #include <boost/throw_exception.hpp>
8 
9 #include <limits>
10 #include <iterator>
11 
12 #include "canvas/Utilities/Exception.h"
13 #include "cetlib_except/exception.h"
14 #include "fhiclcpp/ParameterSet.h"
15 
16 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh"
17 #include "artdaq-core/Data/Fragment.hh"
18 #include "artdaq-core/Data/ContainerFragmentLoader.hh"
19 #include "artdaq-core/Utilities/ExceptionHandler.hh"
20 #include "artdaq-core/Utilities/TimeUtils.hh"
21 
22 #include <fstream>
23 #include <iomanip>
24 #include <iterator>
25 #include <iostream>
26 #include <iomanip>
27 #include <algorithm>
28 #include <sys/poll.h>
30 
31 #define TLVL_GETNEXT 10
32 #define TLVL_GETNEXT_VERBOSE 20
33 #define TLVL_CHECKSTOP 11
34 #define TLVL_EVCOUNTERINC 12
35 #define TLVL_GETDATALOOP 13
36 #define TLVL_GETDATALOOP_DATABUFFWAIT 21
37 #define TLVL_GETDATALOOP_VERBOSE 20
38 #define TLVL_WAITFORBUFFERREADY 15
39 #define TLVL_GETBUFFERSTATS 16
40 #define TLVL_CHECKDATABUFFER 17
41 #define TLVL_GETMONITORINGDATA 18
42 #define TLVL_APPLYREQUESTS 9
43 #define TLVL_SENDEMPTYFRAGMENTS 19
44 #define TLVL_CHECKWINDOWS 14
45 
47  : mutex_()
48  , requestReceiver_(nullptr)
49  , windowOffset_(ps.get<Fragment::timestamp_t>("request_window_offset", 0))
50  , windowWidth_(ps.get<Fragment::timestamp_t>("request_window_width", 0))
51  , staleTimeout_(ps.get<Fragment::timestamp_t>("stale_request_timeout", 0xFFFFFFFF))
52  , expectedType_(ps.get<Fragment::type_t>("expected_fragment_type", Fragment::type_t(Fragment::EmptyFragmentType)))
53  , uniqueWindows_(ps.get<bool>("request_windows_are_unique", true))
54  , missing_request_window_timeout_us_(ps.get<size_t>("missing_request_window_timeout_us", 5000000))
55  , window_close_timeout_us_(ps.get<size_t>("window_close_timeout_us", 2000000))
56  , useDataThread_(ps.get<bool>("separate_data_thread", false))
57  , circularDataBufferMode_(ps.get<bool>("circular_buffer_mode", false))
58  , sleep_on_no_data_us_(ps.get<size_t>("sleep_on_no_data_us", 0))
59  , data_thread_running_(false)
60  , maxDataBufferDepthFragments_(ps.get<int>("data_buffer_depth_fragments", 1000))
61  , maxDataBufferDepthBytes_(ps.get<size_t>("data_buffer_depth_mb", 1000) * 1024 * 1024)
62  , useMonitoringThread_(ps.get<bool>("separate_monitoring_thread", false))
63  , monitoringInterval_(ps.get<int64_t>("hardware_poll_interval_us", 0))
64  , lastMonitoringCall_()
65  , isHardwareOK_(true)
66  , run_number_(-1)
67  , subrun_number_(-1)
68  , timeout_(std::numeric_limits<uint64_t>::max())
69  , timestamp_(std::numeric_limits<uint64_t>::max())
70  , should_stop_(false)
71  , exception_(false)
72  , force_stop_(false)
73  , latest_exception_report_("none")
74  , ev_counter_(1)
75  , board_id_(-1)
76  , sleep_on_stop_us_(0)
77 {
78  board_id_ = ps.get<int>("board_id");
79  instance_name_for_metrics_ = "BoardReader." + boost::lexical_cast<std::string>(board_id_);
80 
81  auto fragment_ids = ps.get<std::vector<artdaq::Fragment::fragment_id_t>>("fragment_ids", std::vector<artdaq::Fragment::fragment_id_t>());
82 
83  TLOG(TLVL_TRACE) << "artdaq::CommandableFragmentGenerator::CommandableFragmentGenerator(ps)";
84  int fragment_id = ps.get<int>("fragment_id", -99);
85 
86  if (fragment_id != -99)
87  {
88  if (fragment_ids.size() != 0)
89  {
90  latest_exception_report_ = "Error in CommandableFragmentGenerator: can't both define \"fragment_id\" and \"fragment_ids\" in FHiCL document";
91  throw cet::exception(latest_exception_report_);
92  }
93  else
94  {
95  fragment_ids.emplace_back(fragment_id);
96  }
97  }
98 
99  for (auto& id : fragment_ids)
100  {
101  dataBuffers_[id].DataBufferDepthBytes = 0;
102  dataBuffers_[id].DataBufferDepthFragments = 0;
103  dataBuffers_[id].HighestRequestSeen = 0;
104  dataBuffers_[id].DataBuffer.emplace_back(FragmentPtr(new Fragment()));
105  (*dataBuffers_[id].DataBuffer.begin())->setSystemType(Fragment::EmptyFragmentType);
106  }
107 
108  sleep_on_stop_us_ = ps.get<int>("sleep_on_stop_us", 0);
109 
110  std::string modeString = ps.get<std::string>("request_mode", "ignored");
111  if (modeString == "single" || modeString == "Single")
112  {
113  mode_ = RequestMode::Single;
114  }
115  else if (modeString.find("buffer") != std::string::npos || modeString.find("Buffer") != std::string::npos)
116  {
117  mode_ = RequestMode::Buffer;
118  }
119  else if (modeString == "window" || modeString == "Window")
120  {
121  mode_ = RequestMode::Window;
122  }
123  else if (modeString.find("ignore") != std::string::npos || modeString.find("Ignore") != std::string::npos)
124  {
125  mode_ = RequestMode::Ignored;
126  }
127  TLOG(TLVL_DEBUG) << "Request mode is " << printMode_();
128 
129  if (mode_ != RequestMode::Ignored)
130  {
131  if (!useDataThread_)
132  {
133  latest_exception_report_ = "Error in CommandableFragmentGenerator: use_data_thread must be true when request_mode is not \"Ignored\"!";
134  throw cet::exception(latest_exception_report_);
135  }
136  requestReceiver_.reset(new RequestReceiver(ps));
137  }
138 }
139 
141 {
142  joinThreads();
143  requestReceiver_.reset(nullptr);
144 }
145 
147 {
148  should_stop_ = true;
149  force_stop_ = true;
150  TLOG(TLVL_DEBUG) << "Joining dataThread";
151  if (dataThread_.joinable()) dataThread_.join();
152  TLOG(TLVL_DEBUG) << "Joining monitoringThread";
153  if (monitoringThread_.joinable()) monitoringThread_.join();
154  TLOG(TLVL_DEBUG) << "joinThreads complete";
155 }
156 
158 {
159  bool result = true;
160 
161  if (check_stop()) usleep(sleep_on_stop_us_);
162  if (exception() || force_stop_) return false;
163 
164  if (!useMonitoringThread_ && monitoringInterval_ > 0)
165  {
166  TLOG(TLVL_GETNEXT) << "getNext: Checking whether to collect Monitoring Data";
167  auto now = std::chrono::steady_clock::now();
168 
169  if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
170  {
171  TLOG(TLVL_GETNEXT) << "getNext: Collecting Monitoring Data";
172  isHardwareOK_ = checkHWStatus_();
173  TLOG(TLVL_GETNEXT) << "getNext: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_;
174  lastMonitoringCall_ = now;
175  }
176  }
177 
178  try
179  {
180  std::lock_guard<std::mutex> lk(mutex_);
181  if (useDataThread_)
182  {
183  TLOG(TLVL_TRACE) << "getNext: Calling applyRequests";
184  result = applyRequests(output);
185  TLOG(TLVL_TRACE) << "getNext: Done with applyRequests result=" << std::boolalpha << result;
186  for (auto dataIter = output.begin(); dataIter != output.end(); ++dataIter)
187  {
188  TLOG(20) << "getNext: applyRequests() returned fragment with sequenceID = " << (*dataIter)->sequenceID()
189  << ", type = " << (*dataIter)->typeString() << ", id = " << std::to_string((*dataIter)->fragmentID())
190  << ", timestamp = " << (*dataIter)->timestamp() << ", and sizeBytes = " << (*dataIter)->sizeBytes();
191  }
192 
193  if (exception())
194  {
195  TLOG(TLVL_ERROR) << "Exception found in BoardReader with board ID " << board_id() << "; BoardReader will now return error status when queried";
196  throw cet::exception("CommandableFragmentGenerator") << "Exception found in BoardReader with board ID " << board_id() << "; BoardReader will now return error status when queried";
197  }
198  }
199  else
200  {
201  if (!isHardwareOK_)
202  {
203  TLOG(TLVL_ERROR) << "Stopping CFG because the hardware reports bad status!";
204  return false;
205  }
206  TLOG(TLVL_TRACE) << "getNext: Calling getNext_ w/ ev_counter()=" << ev_counter();
207  try
208  {
209  result = getNext_(output);
210  }
211  catch (...)
212  {
213  throw;
214  }
215  TLOG(TLVL_TRACE) << "getNext: Done with getNext_ - ev_counter() now " << ev_counter();
216  for (auto dataIter = output.begin(); dataIter != output.end(); ++dataIter)
217  {
218  TLOG(TLVL_GETNEXT_VERBOSE) << "getNext: getNext_() returned fragment with sequenceID = " << (*dataIter)->sequenceID()
219  << ", type = " << (*dataIter)->typeString() << ", id = " << std::to_string((*dataIter)->fragmentID())
220  << ", timestamp = " << (*dataIter)->timestamp() << ", and sizeBytes = " << (*dataIter)->sizeBytes();
221  }
222  }
223  }
224  catch (const cet::exception& e)
225  {
226  latest_exception_report_ = "cet::exception caught in getNext(): ";
227  latest_exception_report_.append(e.what());
228  TLOG(TLVL_ERROR) << "getNext: cet::exception caught: " << e;
229  set_exception(true);
230  return false;
231  }
232  catch (const boost::exception& e)
233  {
234  latest_exception_report_ = "boost::exception caught in getNext(): ";
235  latest_exception_report_.append(boost::diagnostic_information(e));
236  TLOG(TLVL_ERROR) << "getNext: boost::exception caught: " << boost::diagnostic_information(e);
237  set_exception(true);
238  return false;
239  }
240  catch (const std::exception& e)
241  {
242  latest_exception_report_ = "std::exception caught in getNext(): ";
243  latest_exception_report_.append(e.what());
244  TLOG(TLVL_ERROR) << "getNext: std::exception caught: " << e.what();
245  set_exception(true);
246  return false;
247  }
248  catch (...)
249  {
250  latest_exception_report_ = "Unknown exception caught in getNext().";
251  TLOG(TLVL_ERROR) << "getNext: unknown exception caught";
252  set_exception(true);
253  return false;
254  }
255 
256  if (!result)
257  {
258  TLOG(TLVL_DEBUG) << "getNext: Either getNext_ or applyRequests returned false, stopping";
259  }
260 
261  if (metricMan && !output.empty())
262  {
263  auto timestamp = output.front()->timestamp();
264 
265  if (output.size() > 1)
266  { // Only bother sorting if >1 entry
267  for (auto& outputfrag : output)
268  {
269  if (outputfrag->timestamp() > timestamp)
270  {
271  timestamp = outputfrag->timestamp();
272  }
273  }
274  }
275 
276  metricMan->sendMetric("Last Timestamp", timestamp, "Ticks", 1,
277  MetricMode::LastPoint, app_name);
278  }
279 
280  return result;
281 }
282 
284 {
285  TLOG(TLVL_CHECKSTOP) << "CFG::check_stop: should_stop=" << should_stop() << ", useDataThread_=" << useDataThread_ << ", exception status =" << int(exception());
286 
287  if (!should_stop()) return false;
288  if (!useDataThread_ || mode_ == RequestMode::Ignored) return true;
289  if (force_stop_) return true;
290 
291  // check_stop returns true if the CFG should stop. We should wait for the RequestReceiver to stop before stopping.
292  TLOG(TLVL_DEBUG) << "should_stop is true, force_stop_ is false, requestReceiver_->isRunning() is " << std::boolalpha << requestReceiver_->isRunning();
293  return !requestReceiver_->isRunning();
294 }
295 
297 {
298  if (force || mode_ == RequestMode::Ignored)
299  {
300  TLOG(TLVL_EVCOUNTERINC) << "ev_counter_inc: Incrementing ev_counter from " << ev_counter() << " by " << step;
301  return ev_counter_.fetch_add(step);
302  }
303  return ev_counter_.load();
304 } // returns the prev value
305 
306 void artdaq::CommandableFragmentGenerator::StartCmd(int run, uint64_t timeout, uint64_t timestamp)
307 {
308  TLOG(TLVL_TRACE) << "Start Command received.";
309  if (run < 0) throw cet::exception("CommandableFragmentGenerator") << "negative run number";
310 
311  timeout_ = timeout;
312  timestamp_ = timestamp;
313  ev_counter_.store(1);
314  {
315  std::unique_lock<std::mutex> lk(dataBuffersMutex_);
316  for (auto& id : dataBuffers_)
317  {
318  id.second.DataBufferDepthBytes = 0;
319  id.second.DataBufferDepthFragments = 0;
320  id.second.HighestRequestSeen = 0;
321  id.second.DataBuffer.clear();
322  id.second.WindowsSent.clear();
323  }
324  }
325 
326  should_stop_.store(false);
327  force_stop_.store(false);
328  exception_.store(false);
329  run_number_ = run;
330  subrun_number_ = 1;
331  latest_exception_report_ = "none";
332 
333  start();
334 
335  std::unique_lock<std::mutex> lk(mutex_);
336  if (useDataThread_) startDataThread();
337  if (useMonitoringThread_) startMonitoringThread();
338  if (mode_ != RequestMode::Ignored)
339  {
340  requestReceiver_->SetRunNumber(static_cast<uint32_t>(run));
341  requestReceiver_->startRequestReception();
342  }
343  TLOG(TLVL_TRACE) << "Start Command complete.";
344 }
345 
346 void artdaq::CommandableFragmentGenerator::StopCmd(uint64_t timeout, uint64_t timestamp)
347 {
348  TLOG(TLVL_TRACE) << "Stop Command received.";
349 
350  timeout_ = timeout;
351  timestamp_ = timestamp;
352  if (requestReceiver_) {
353  TLOG(TLVL_DEBUG) << "Stopping Request reception BEGIN";
354  requestReceiver_->stopRequestReception();
355  TLOG(TLVL_DEBUG) << "Stopping Request reception END";
356  }
357 
358  stopNoMutex();
359  should_stop_.store(true);
360  std::unique_lock<std::mutex> lk(mutex_);
361  stop();
362 
363  joinThreads();
364  TLOG(TLVL_TRACE) << "Stop Command complete.";
365 }
366 
367 void artdaq::CommandableFragmentGenerator::PauseCmd(uint64_t timeout, uint64_t timestamp)
368 {
369  TLOG(TLVL_TRACE) << "Pause Command received.";
370  timeout_ = timeout;
371  timestamp_ = timestamp;
372  //if (requestReceiver_->isRunning()) requestReceiver_->stopRequestReceiverThread();
373 
374  pauseNoMutex();
375  should_stop_.store(true);
376  std::unique_lock<std::mutex> lk(mutex_);
377 
378  pause();
379 }
380 
381 void artdaq::CommandableFragmentGenerator::ResumeCmd(uint64_t timeout, uint64_t timestamp)
382 {
383  TLOG(TLVL_TRACE) << "Resume Command received.";
384  timeout_ = timeout;
385  timestamp_ = timestamp;
386 
387  subrun_number_ += 1;
388  should_stop_ = false;
389 
390  {
391  std::unique_lock<std::mutex> lk(dataBuffersMutex_);
392  for (auto& id : dataBuffers_)
393  {
394  id.second.DataBufferDepthBytes = 0;
395  id.second.DataBufferDepthFragments = 0;
396  id.second.DataBuffer.clear();
397  }
398  }
399  // no lock required: thread not started yet
400  resume();
401 
402  std::unique_lock<std::mutex> lk(mutex_);
403  //if (useDataThread_) startDataThread();
404  //if (useMonitoringThread_) startMonitoringThread();
405  //if (mode_ != RequestMode::Ignored && !requestReceiver_->isRunning()) requestReceiver_->startRequestReceiverThread();
406  TLOG(TLVL_TRACE) << "Resume Command complete.";
407 }
408 
409 std::string artdaq::CommandableFragmentGenerator::ReportCmd(std::string const& which)
410 {
411  TLOG(TLVL_TRACE) << "Report Command received.";
412  std::lock_guard<std::mutex> lk(mutex_);
413 
414  // 14-May-2015, KAB: please see the comments associated with the report()
415  // methods in the CommandableFragmentGenerator.hh file for more information
416  // on the use of those methods in this method.
417 
418  // check if the child class has something meaningful for this request
419  std::string childReport = reportSpecific(which);
420  if (childReport.length() > 0) { return childReport; }
421 
422  // handle the requests that we can take care of at this level
423  if (which == "latest_exception")
424  {
425  return latest_exception_report_;
426  }
427 
428  // check if the child class has provided a catch-all report function
429  childReport = report();
430  if (childReport.length() > 0) { return childReport; }
431 
432  // if we haven't been able to come up with any report so far, say so
433  std::string tmpString = "The \"" + which + "\" command is not ";
434  tmpString.append("currently supported by the ");
435  tmpString.append(metricsReportingInstanceName());
436  tmpString.append(" fragment generator.");
437  TLOG(TLVL_TRACE) << "Report Command complete.";
438  return tmpString;
439 }
440 
441 // Default implemenetations of state functions
443 {
444 #pragma message "Using default implementation of CommandableFragmentGenerator::pauseNoMutex()"
445 }
446 
448 {
449 #pragma message "Using default implementation of CommandableFragmentGenerator::pause()"
450 }
451 
453 {
454 #pragma message "Using default implementation of CommandableFragmentGenerator::resume()"
455 }
456 
458 {
459 #pragma message "Using default implementation of CommandableFragmentGenerator::report()"
460  return "";
461 }
462 
464 {
465 #pragma message "Using default implementation of CommandableFragmentGenerator::reportSpecific(std::string)"
466  return "";
467 }
468 
470 {
471 #pragma message "Using default implementation of CommandableFragmentGenerator::checkHWStatus_()"
472  return true;
473 }
474 
475 bool artdaq::CommandableFragmentGenerator::metaCommand(std::string const&, std::string const&)
476 {
477 #pragma message "Using default implementation of CommandableFragmentGenerator::metaCommand(std::string, std::string)"
478  return true;
479 }
480 
482 {
483  if (dataThread_.joinable()) dataThread_.join();
484  TLOG(TLVL_INFO) << "Starting Data Receiver Thread";
485  try {
486  dataThread_ = boost::thread(&CommandableFragmentGenerator::getDataLoop, this);
487  }
488  catch (const boost::exception& e)
489  {
490  TLOG(TLVL_ERROR) << "Caught boost::exception starting Data Receiver thread: " << boost::diagnostic_information(e) << ", errno=" << errno;
491  std::cerr << "Caught boost::exception starting Data Receiver thread: " << boost::diagnostic_information(e) << ", errno=" << errno << std::endl;
492  exit(5);
493  }
494 }
495 
497 {
498  if (monitoringThread_.joinable()) monitoringThread_.join();
499  TLOG(TLVL_INFO) << "Starting Hardware Monitoring Thread";
500  try {
501  monitoringThread_ = boost::thread(&CommandableFragmentGenerator::getMonitoringDataLoop, this);
502  }
503  catch (const boost::exception& e)
504  {
505  TLOG(TLVL_ERROR) << "Caught boost::exception starting Hardware Monitoring thread: " << boost::diagnostic_information(e) << ", errno=" << errno;
506  std::cerr << "Caught boost::exception starting Hardware Monitoring thread: " << boost::diagnostic_information(e) << ", errno=" << errno << std::endl;
507  exit(5);
508  }
509 }
510 
512 {
513  switch (mode_)
514  {
515  case RequestMode::Single:
516  return "Single";
517  case RequestMode::Buffer:
518  return "Buffer";
519  case RequestMode::Window:
520  return "Window";
521  case RequestMode::Ignored:
522  return "Ignored";
523  }
524 
525  return "ERROR";
526 }
527 
528 
529 //
530 // The "useDataThread_" thread
531 //
533 {
534  data_thread_running_ = true;
535  while (!force_stop_)
536  {
537  if (!isHardwareOK_)
538  {
539  TLOG(TLVL_DEBUG) << "getDataLoop: isHardwareOK is " << isHardwareOK_ << ", aborting data thread";
540  data_thread_running_ = false;
541  return;
542  }
543 
544  TLOG(TLVL_GETDATALOOP) << "getDataLoop: calling getNext_";
545 
546  bool data = false;
547  auto startdata = std::chrono::steady_clock::now();
548  FragmentPtrs newData;
549 
550  try
551  {
552  data = getNext_(newData);
553  }
554  catch (...)
555  {
556  ExceptionHandler(ExceptionHandlerRethrow::no,
557  "Exception thrown by fragment generator in CommandableFragmentGenerator::getDataLoop; setting exception state to \"true\"");
558  set_exception(true);
559 
560  data_thread_running_ = false;
561  return;
562  }
563 
564  if (metricMan)
565  {
566  metricMan->sendMetric("Avg Data Acquisition Time", TimeUtils::GetElapsedTime(startdata), "s", 3, artdaq::MetricMode::Average);
567  }
568 
569  if (newData.size() == 0 && sleep_on_no_data_us_ > 0)
570  {
571  usleep(sleep_on_no_data_us_);
572  }
573 
574  auto dataIter = newData.begin();
575  while (dataIter != newData.end())
576  {
577  TLOG(TLVL_GETDATALOOP_VERBOSE) << "getDataLoop: getNext_() returned fragment with timestamp = " << (*dataIter)->timestamp() << ", and sizeBytes = " << (*dataIter)->sizeBytes();
578 
579  auto frag_id = (*dataIter)->fragmentID();
580  if (!dataBuffers_.count(frag_id)) {
581  throw cet::exception("DataBufferError") << "Error in CommandableFragmentGenerator: Recevied Fragment with fragment_id " << frag_id << ", but this ID was not declared in fragment_ids!";
582  }
583 
584  TLOG(TLVL_GETDATALOOP_DATABUFFWAIT) << "Waiting for data buffer ready";
585  if (!waitForDataBufferReady(frag_id)) return;
586  TLOG(TLVL_GETDATALOOP_DATABUFFWAIT) << "Done waiting for data buffer ready";
587 
588  TLOG(TLVL_GETDATALOOP) << "getDataLoop: processing data";
589  if (data && !force_stop_)
590  {
591  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
592  switch (mode_)
593  {
594  case RequestMode::Single:
595  dataBuffers_[frag_id].DataBuffer.clear();
596  dataBuffers_[frag_id].DataBufferDepthBytes = (*dataIter)->sizeBytes();
597  dataBuffers_[frag_id].DataBuffer.emplace_back(std::move(*dataIter));
598  dataIter = newData.erase(dataIter);
599  break;
600  case RequestMode::Buffer:
601  case RequestMode::Ignored:
602  case RequestMode::Window:
603  default:
604  //dataBuffer_.reserve(dataBuffer_.size() + newDataBuffer_.size());
605  dataBuffers_[frag_id].DataBufferDepthBytes += (*dataIter)->sizeBytes();
606  dataBuffers_[frag_id].DataBuffer.emplace_back(std::move(*dataIter));
607  dataIter = newData.erase(dataIter);
608  break;
609  }
610  getDataBufferStats(frag_id);
611  }
612  else {
613  break;
614  }
615  }
616 
617  {
618  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
619  for (auto& id : dataBuffers_) {
620  if (id.second.DataBuffer.size() > 0)
621  {
622  dataCondition_.notify_all();
623  break;
624  }
625  }
626  }
627  if (!data || force_stop_)
628  {
629  TLOG(TLVL_INFO) << "Data flow has stopped. Ending data collection thread";
630  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
631  data_thread_running_ = false;
632  if (requestReceiver_) requestReceiver_->ClearRequests();
633  newData.clear();
634  TLOG(TLVL_INFO) << "getDataLoop: Ending thread";
635  return;
636  }
637  }
638 }
639 
641 {
642  if (!dataBuffers_.count(id)) {
643  throw cet::exception("DataBufferError") << "Error in CommandableFragmentGenerator: Cannot wait for data buffer for ID " << id << " because it does not exist!";
644  }
645  auto startwait = std::chrono::steady_clock::now();
646  auto first = true;
647  auto lastwaittime = 0ULL;
648 
649  {
650  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
651  getDataBufferStats(id);
652  }
653 
654  while (dataBufferIsTooLarge(id))
655  {
656  if (!circularDataBufferMode_)
657  {
658  if (should_stop())
659  {
660  TLOG(TLVL_DEBUG) << "Run ended while waiting for buffer to shrink!";
661  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
662  getDataBufferStats(id);
663  dataCondition_.notify_all();
664  data_thread_running_ = false;
665  return false;
666  }
667  auto waittime = TimeUtils::GetElapsedTimeMilliseconds(startwait);
668 
669  if (first || (waittime != lastwaittime && waittime % 1000 == 0))
670  {
671  TLOG(TLVL_WARNING) << "Bad Omen: Data Buffer has exceeded its size limits. "
672  << "(seq_id=" << ev_counter() << ", frag_id=" << id
673  << ", frags=" << dataBuffers_[id].DataBufferDepthFragments << "/" << maxDataBufferDepthFragments_
674  << ", szB=" << dataBuffers_[id].DataBufferDepthBytes << "/" << maxDataBufferDepthBytes_ << ")";
675  TLOG(TLVL_TRACE) << "Bad Omen: Possible causes include requests not getting through or Ignored-mode BR issues";
676  first = false;
677  }
678  if (waittime % 5 && waittime != lastwaittime)
679  {
680  TLOG(TLVL_WAITFORBUFFERREADY) << "getDataLoop: Data Retreival paused for " << waittime << " ms waiting for data buffer to drain";
681  }
682  lastwaittime = waittime;
683  usleep(1000);
684  }
685  else
686  {
687  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
688  getDataBufferStats(id); // Re-check under lock
689  if (dataBufferIsTooLarge(id))
690  {
691  auto begin = dataBuffers_[id].DataBuffer.begin();
692  if (begin == dataBuffers_[id].DataBuffer.end())
693  {
694  TLOG(TLVL_WARNING) << "Data buffer is reported as too large, but doesn't contain any Fragments! Possible corrupt memory!";
695  continue;
696  }
697  if (*begin)
698  {
699  TLOG(TLVL_WAITFORBUFFERREADY) << "waitForDataBufferReady: Dropping Fragment with timestamp " << (*begin)->timestamp() << " from data buffer (Buffer over-size, circular data buffer mode)";
700  }
701  dataBuffers_[id].DataBufferDepthBytes -= (*begin)->sizeBytes();
702  dataBuffers_[id].DataBuffer.erase(begin);
703  getDataBufferStats(id);
704  }
705 
706  }
707  }
708  return true;
709 }
710 
712 {
713  if (!dataBuffers_.count(id)) {
714  throw cet::exception("DataBufferError") << "Error in CommandableFragmentGenerator: Cannot check size of data buffer for ID " << id << " because it does not exist!";
715  }
716  return (maxDataBufferDepthFragments_ > 0 && dataBuffers_[id].DataBufferDepthFragments > maxDataBufferDepthFragments_) || (maxDataBufferDepthBytes_ > 0 && dataBuffers_[id].DataBufferDepthBytes > maxDataBufferDepthBytes_);
717 }
718 
720 {
721  if (!dataBuffers_.count(id)) {
722  throw cet::exception("DataBufferError") << "Error in CommandableFragmentGenerator: Cannot get stats of data buffer for ID " << id << " because it does not exist!";
723  }
725  dataBuffers_[id].DataBufferDepthFragments = dataBuffers_[id].DataBuffer.size();
726 
727  if (metricMan)
728  {
729  TLOG(TLVL_GETBUFFERSTATS) << "getDataBufferStats: Sending Metrics";
730  metricMan->sendMetric("Buffer Depth Fragments", dataBuffers_[id].DataBufferDepthFragments.load(), "fragments", 1, MetricMode::LastPoint);
731  metricMan->sendMetric("Buffer Depth Bytes", dataBuffers_[id].DataBufferDepthBytes.load(), "bytes", 1, MetricMode::LastPoint);
732  }
733  TLOG(TLVL_GETBUFFERSTATS) << "getDataBufferStats: frags=" << dataBuffers_[id].DataBufferDepthFragments.load() << "/" << maxDataBufferDepthFragments_
734  << ", sz=" << dataBuffers_[id].DataBufferDepthBytes.load() << "/" << maxDataBufferDepthBytes_;
735 }
736 
738 {
739  if (!dataBuffers_.count(id)) {
740  throw cet::exception("DataBufferError") << "Error in CommandableFragmentGenerator: Cannot check data buffer for ID " << id << " because it does not exist!";
741  }
742 
743  if (dataBuffers_[id].DataBufferDepthFragments > 0 && mode_ != RequestMode::Single && mode_ != RequestMode::Ignored)
744  {
745  // Eliminate extra fragments
746  getDataBufferStats(id);
747  while (dataBufferIsTooLarge(id))
748  {
749  auto begin = dataBuffers_[id].DataBuffer.begin();
750  TLOG(TLVL_CHECKDATABUFFER) << "checkDataBuffer: Dropping Fragment with timestamp " << (*begin)->timestamp() << " from data buffer (Buffer over-size)";
751  dataBuffers_[id].DataBufferDepthBytes -= (*begin)->sizeBytes();
752  dataBuffers_[id].DataBuffer.erase(begin);
753  getDataBufferStats(id);
754  }
755  if (dataBuffers_[id].DataBuffer.size() > 0)
756  {
757  TLOG(TLVL_CHECKDATABUFFER) << "Determining if Fragments can be dropped from data buffer";
758  Fragment::timestamp_t last = dataBuffers_[id].DataBuffer.back()->timestamp();
759  Fragment::timestamp_t min = last > staleTimeout_ ? last - staleTimeout_ : 0;
760  for (auto it = dataBuffers_[id].DataBuffer.begin(); it != dataBuffers_[id].DataBuffer.end();)
761  {
762  if ((*it)->timestamp() < min)
763  {
764  TLOG(TLVL_CHECKDATABUFFER) << "checkDataBuffer: Dropping Fragment with timestamp " << (*it)->timestamp() << " from data buffer (timeout=" << staleTimeout_ << ", min=" << min << ")";
765  dataBuffers_[id].DataBufferDepthBytes -= (*it)->sizeBytes();
766  it = dataBuffers_[id].DataBuffer.erase(it);
767  }
768  else
769  {
770  break;
771  }
772  }
773  getDataBufferStats(id);
774  }
775  }
776 }
777 
779 {
780  while (!force_stop_)
781  {
782  if (should_stop() || monitoringInterval_ <= 0)
783  {
784  TLOG(TLVL_DEBUG) << "getMonitoringDataLoop: should_stop() is " << std::boolalpha << should_stop()
785  << " and monitoringInterval is " << monitoringInterval_ << ", returning";
786  return;
787  }
788  TLOG(TLVL_GETMONITORINGDATA) << "getMonitoringDataLoop: Determining whether to call checkHWStatus_";
789 
790  auto now = std::chrono::steady_clock::now();
791  if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
792  {
793  isHardwareOK_ = checkHWStatus_();
794  TLOG(TLVL_GETMONITORINGDATA) << "getMonitoringDataLoop: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_;
795  lastMonitoringCall_ = now;
796  }
797  usleep(monitoringInterval_ / 10);
798  }
799 }
800 
802 {
803  // dataBuffersMutex_ is held by calling function
804  // We just copy everything that's here into the output.
805  TLOG(TLVL_APPLYREQUESTS) << "Mode is Ignored; Copying data to output";
806  for (auto& id : dataBuffers_) {
807  std::move(id.second.DataBuffer.begin(), id.second.DataBuffer.end(), std::inserter(frags, frags.end()));
808  id.second.DataBufferDepthBytes = 0;
809  id.second.DataBufferDepthFragments = 0;
810  id.second.DataBuffer.clear();
811  }
812 }
813 
815 {
816  // We only care about the latest request received. Send empties for all others.
817  auto requests = requestReceiver_->GetRequests();
818  while (requests.size() > 1)
819  {
820  // std::map is ordered by key => Last sequence ID in the map is the one we care about
821  requestReceiver_->RemoveRequest(requests.begin()->first);
822  requests.erase(requests.begin());
823  }
824  sendEmptyFragments(frags, requests);
825 
826  // If no requests remain after sendEmptyFragments, return
827  if (requests.size() == 0 || !requests.count(ev_counter())) return;
828 
829  for (auto& id : dataBuffers_) {
830  if (id.second.DataBuffer.size() > 0)
831  {
832  assert(id.second.DataBuffer.size() == 1);
833  TLOG(TLVL_APPLYREQUESTS) << "Mode is Single; Sending copy of last event";
834  for (auto& fragptr : id.second.DataBuffer)
835  {
836  // Return the latest data point
837  auto frag = fragptr.get();
838  auto newfrag = std::unique_ptr<artdaq::Fragment>(new Fragment(ev_counter(), frag->fragmentID()));
839  newfrag->resize(frag->size() - detail::RawFragmentHeader::num_words());
840  memcpy(newfrag->headerAddress(), frag->headerAddress(), frag->sizeBytes());
841  newfrag->setTimestamp(requests[ev_counter()]);
842  newfrag->setSequenceID(ev_counter());
843  frags.push_back(std::move(newfrag));
844  }
845  }
846  else
847  {
848  sendEmptyFragment(frags, ev_counter(), id.first, "No data for");
849  }
850  }
851  requestReceiver_->RemoveRequest(ev_counter());
852  ev_counter_inc(1, true);
853 }
854 
856 {
857  // We only care about the latest request received. Send empties for all others.
858  auto requests = requestReceiver_->GetRequests();
859  while (requests.size() > 1)
860  {
861  // std::map is ordered by key => Last sequence ID in the map is the one we care about
862  requestReceiver_->RemoveRequest(requests.begin()->first);
863  requests.erase(requests.begin());
864  }
865  sendEmptyFragments(frags, requests);
866 
867  // If no requests remain after sendEmptyFragments, return
868  if (requests.size() == 0 || !requests.count(ev_counter())) return;
869 
870  for (auto& id : dataBuffers_) {
871 
872  TLOG(TLVL_DEBUG) << "Creating ContainerFragment for Buffered Fragments";
873  frags.emplace_back(new artdaq::Fragment(ev_counter(), id.first));
874  frags.back()->setTimestamp(requests[ev_counter()]);
875  ContainerFragmentLoader cfl(*frags.back());
876  cfl.set_missing_data(false); // Buffer mode is never missing data, even if there IS no data.
877 
878  // Buffer mode TFGs should simply copy out the whole dataBuffer_ into a ContainerFragment
879  // Window mode TFGs must do a little bit more work to decide which fragments to send for a given request
880  for (auto it = id.second.DataBuffer.begin(); it != id.second.DataBuffer.end();)
881  {
882  TLOG(TLVL_APPLYREQUESTS) << "ApplyRequests: Adding Fragment with timestamp " << (*it)->timestamp() << " to Container with sequence ID " << ev_counter();
883  cfl.addFragment(*it);
884  id.second.DataBufferDepthBytes -= (*it)->sizeBytes();
885  it = id.second.DataBuffer.erase(it);
886  }
887  }
888  requestReceiver_->RemoveRequest(ev_counter());
889  ev_counter_inc(1, true);
890 }
891 
892 
893 void artdaq::CommandableFragmentGenerator::applyRequestsWindowMode_CheckAndFillDataBuffer(artdaq::FragmentPtrs& frags, artdaq::Fragment::fragment_id_t id, artdaq::Fragment::sequence_id_t seq, artdaq::Fragment::timestamp_t ts)
894 {
895  TLOG(TLVL_APPLYREQUESTS) << "applyRequestsWindowMode_CheckAndFillDataBuffer: Checking that data exists for request window " << seq;
896  Fragment::timestamp_t min = ts > windowOffset_ ? ts - windowOffset_ : 0;
897  Fragment::timestamp_t max = min + windowWidth_;
898  TLOG(TLVL_APPLYREQUESTS) << "ApplyRequestsWindowsMode_CheckAndFillDataBuffer: min is " << min << ", max is " << max
899  << " and first/last points in buffer are " << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.front()->timestamp() : 0)
900  << "/" << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.back()->timestamp() : 0)
901  << " (sz=" << dataBuffers_[id].DataBuffer.size() << " [" << dataBuffers_[id].DataBufferDepthBytes.load()
902  << "/" << maxDataBufferDepthBytes_ << "])";
903  bool windowClosed = dataBuffers_[id].DataBuffer.size() > 0 && dataBuffers_[id].DataBuffer.back()->timestamp() >= max;
904  bool windowTimeout = !windowClosed && TimeUtils::GetElapsedTimeMicroseconds(requestReceiver_->GetRequestTime(seq)) > window_close_timeout_us_;
905  if (windowTimeout)
906  {
907  TLOG(TLVL_WARNING) << "applyRequestsWindowMode_CheckAndFillDataBuffer: A timeout occurred waiting for data to close the request window ({" << min << "-" << max
908  << "}, buffer={" << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.front()->timestamp() : 0) << "-"
909  << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.back()->timestamp() : 0)
910  << "} ). Time waiting: "
911  << TimeUtils::GetElapsedTimeMicroseconds(requestReceiver_->GetRequestTime(seq)) << " us "
912  << "(> " << window_close_timeout_us_ << " us).";
913  }
914  if (windowClosed || !data_thread_running_ || windowTimeout)
915  {
916  TLOG(TLVL_DEBUG) << "applyRequestsWindowMode_CheckAndFillDataBuffer: Creating ContainerFragment for Window-requested Fragments";
917  frags.emplace_back(new artdaq::Fragment(seq, id));
918  frags.back()->setTimestamp(ts);
919  ContainerFragmentLoader cfl(*frags.back());
920 
921  // In the spirit of NOvA's MegaPool: (RS = Request start (min), RE = Request End (max))
922  // --- | Buffer Start | --- | Buffer End | ---
923  //1. RS RE | | | |
924  //2. RS | | RE | |
925  //3. RS | | | | RE
926  //4. | | RS RE | |
927  //5. | | RS | | RE
928  //6. | | | | RS RE
929  //
930  // If RE (or RS) is after the end of the buffer, we wait for window_close_timeout_us_. If we're here, then that means that windowClosed is false, and the missing_data flag should be set.
931  // If RS (or RE) is before the start of the buffer, then missing_data should be set to true, as data is assumed to arrive in the buffer in timestamp order
932  // If the dataBuffer has size 0, then windowClosed will be false
933  if (!windowClosed || (dataBuffers_[id].DataBuffer.size() > 0 && dataBuffers_[id].DataBuffer.front()->timestamp() > min))
934  {
935  TLOG(TLVL_DEBUG) << "applyRequestsWindowMode_CheckAndFillDataBuffer: Request window starts before and/or ends after the current data buffer, setting ContainerFragment's missing_data flag!"
936  << " (requestWindowRange=[" << min << "," << max << "], "
937  << "buffer={" << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.front()->timestamp() : 0) << "-"
938  << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.back()->timestamp() : 0) << "}";
939  cfl.set_missing_data(true);
940  }
941 
942  // Do a little bit more work to decide which fragments to send for a given request
943  for (auto it = dataBuffers_[id].DataBuffer.begin(); it != dataBuffers_[id].DataBuffer.end();)
944  {
945  Fragment::timestamp_t fragT = (*it)->timestamp();
946  if (fragT < min || fragT > max || (fragT == max && windowWidth_ > 0))
947  {
948  ++it;
949  continue;
950  }
951 
952  TLOG(TLVL_APPLYREQUESTS) << "applyRequestsWindowMode_CheckAndFillDataBuffer: Adding Fragment with timestamp " << (*it)->timestamp() << " to Container";
953  cfl.addFragment(*it);
954 
955  if (uniqueWindows_)
956  {
957  dataBuffers_[id].DataBufferDepthBytes -= (*it)->sizeBytes();
958  it = dataBuffers_[id].DataBuffer.erase(it);
959  }
960  else
961  {
962  ++it;
963  }
964  }
965 
966  dataBuffers_[id].WindowsSent[seq] = std::chrono::steady_clock::now();
967  if (seq > dataBuffers_[id].HighestRequestSeen) dataBuffers_[id].HighestRequestSeen = seq;
968  }
969 }
970 
972 {
973  TLOG(TLVL_APPLYREQUESTS) << "applyRequestsWindowMode BEGIN";
974 
975  auto requests = requestReceiver_->GetRequests();
976 
977  TLOG(TLVL_APPLYREQUESTS) << "applyRequestsWindowMode: Starting request processing";
978  for (auto req = requests.begin(); req != requests.end();)
979  {
980  TLOG(TLVL_APPLYREQUESTS) << "applyRequestsWindowMode: processing request with sequence ID " << req->first << ", timestamp " << req->second;
981 
982 
983  while (req->first < ev_counter() && requests.size() > 0)
984  {
985  TLOG(TLVL_APPLYREQUESTS) << "applyRequestsWindowMode: Clearing passed request for sequence ID " << req->first;
986  requestReceiver_->RemoveRequest(req->first);
987  req = requests.erase(req);
988  }
989  if (requests.size() == 0) break;
990 
991  for (auto& id : dataBuffers_)
992  {
993  if (!id.second.WindowsSent.count(req->first)) {
994  applyRequestsWindowMode_CheckAndFillDataBuffer(frags, id.first, req->first, req->second);
995  }
996  }
997  checkSentWindows(req->first);
998  ++req;
999  }
1000 
1001  // Check sent windows for requests that can be removed
1002  for (auto& id : dataBuffers_) {
1003  std::set<artdaq::Fragment::sequence_id_t> seqs;
1004  for (auto& seq : id.second.WindowsSent) {
1005  seqs.insert(seq.first);
1006  }
1007  for (auto& seq : seqs)
1008  {
1009  checkSentWindows(seq);
1010  }
1011  }
1012 }
1013 
1015 {
1016  if (check_stop() || exception())
1017  {
1018  return false;
1019  }
1020 
1021  // Wait for data, if in ignored mode, or a request otherwise
1022  if (mode_ == RequestMode::Ignored)
1023  {
1024  while (dataBufferFragmentCount_() == 0)
1025  {
1026  if (check_stop() || exception() || !isHardwareOK_) return false;
1027  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
1028  dataCondition_.wait_for(lock, std::chrono::milliseconds(10), [this]() { return dataBufferFragmentCount_() > 0; });
1029  }
1030  }
1031  else
1032  {
1033  if ((check_stop() && requestReceiver_->size() == 0) || exception()) return false;
1034 
1035  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
1036  dataCondition_.wait_for(lock, std::chrono::milliseconds(10));
1037 
1038  checkDataBuffers();
1039 
1040  // Wait up to 1000 ms for a request...
1041  auto counter = 0;
1042 
1043  while (requestReceiver_->size() == 0 && counter < 100)
1044  {
1045  if (check_stop() || exception()) return false;
1046 
1047  checkDataBuffers();
1048 
1049  requestReceiver_->WaitForRequests(10); // milliseconds
1050  counter++;
1051  }
1052  }
1053 
1054  {
1055  std::unique_lock<std::mutex> dlk(dataBuffersMutex_);
1056 
1057  switch (mode_)
1058  {
1059  case RequestMode::Single:
1060  applyRequestsSingleMode(frags);
1061  break;
1062  case RequestMode::Window:
1063  applyRequestsWindowMode(frags);
1064  break;
1065  case RequestMode::Buffer:
1066  applyRequestsBufferMode(frags);
1067  break;
1068  case RequestMode::Ignored:
1069  default:
1070  applyRequestsIgnoredMode(frags);
1071  break;
1072  }
1073 
1074  if (!data_thread_running_ || force_stop_)
1075  {
1076  TLOG(TLVL_INFO) << "Data thread has stopped; Clearing data buffers";
1077  for (auto& id : dataBuffers_) {
1078  id.second.DataBufferDepthBytes = 0;
1079  id.second.DataBufferDepthFragments = 0;
1080  id.second.DataBuffer.clear();
1081  }
1082  }
1083 
1084  getDataBuffersStats();
1085  }
1086 
1087  if (frags.size() > 0)
1088  TLOG(TLVL_APPLYREQUESTS) << "Finished Processing requests, returning " << frags.size() << " fragments, current ev_counter is " << ev_counter();
1089  return true;
1090 }
1091 
1092 bool artdaq::CommandableFragmentGenerator::sendEmptyFragment(artdaq::FragmentPtrs& frags, size_t seqId, Fragment::fragment_id_t fragmentId, std::string desc)
1093 {
1094  TLOG(TLVL_WARNING) << desc << " sequence ID " << seqId << ", sending empty fragment";
1095  auto frag = new Fragment();
1096  frag->setSequenceID(seqId);
1097  frag->setFragmentID(fragmentId);
1098  frag->setSystemType(Fragment::EmptyFragmentType);
1099  frags.emplace_back(FragmentPtr(frag));
1100  return true;
1101 }
1102 
1103 void artdaq::CommandableFragmentGenerator::sendEmptyFragments(artdaq::FragmentPtrs& frags, std::map<Fragment::sequence_id_t, Fragment::timestamp_t>& requests)
1104 {
1105  if (requests.size() > 0)
1106  {
1107  TLOG(TLVL_SENDEMPTYFRAGMENTS) << "Sending Empty Fragments for Sequence IDs from " << ev_counter() << " up to but not including " << requests.begin()->first;
1108  while (requests.begin()->first > ev_counter())
1109  {
1110  for (auto& fid : dataBuffers_)
1111  {
1112  sendEmptyFragment(frags, ev_counter(), fid.first, "Missed request for");
1113  }
1114  ev_counter_inc(1, true);
1115  }
1116  }
1117 }
1118 
1119 void artdaq::CommandableFragmentGenerator::checkSentWindows(artdaq::Fragment::sequence_id_t seq)
1120 {
1121  bool seqComplete = true;
1122  bool seqTimeout = false;
1123  for (auto& id : dataBuffers_) {
1124  if (!id.second.WindowsSent.count(seq) || id.second.HighestRequestSeen < seq) {
1125  seqComplete = false;
1126  }
1127  if (id.second.WindowsSent.count(seq) && TimeUtils::GetElapsedTimeMicroseconds(id.second.WindowsSent[seq]) > missing_request_window_timeout_us_) {
1128  seqTimeout = true;
1129  }
1130  }
1131  if (seqComplete) {
1132  TLOG(TLVL_CHECKWINDOWS) << "checkSentWindows: Request " << seq << " is complete, removing from requestReceiver.";
1133  requestReceiver_->RemoveRequest(seq);
1134 
1135  if (ev_counter() == seq) {
1136  TLOG(TLVL_CHECKWINDOWS) << "checkSentWindows: Sequence ID matches ev_counter, incrementing ev_counter (" << ev_counter() << ")";
1137 
1138  for (auto& id : dataBuffers_) {
1139  id.second.WindowsSent.erase(seq);
1140  }
1141 
1142  ev_counter_inc(1, true);
1143  }
1144  }
1145  if (seqTimeout) {
1146  TLOG(TLVL_CHECKWINDOWS) << "checkSentWindows: Sent Window history indicates that requests between " << ev_counter() << " and " << seq << " have timed out.";
1147  while (ev_counter() <= seq)
1148  {
1149  if (ev_counter() < seq) TLOG(TLVL_WARNING) << "Missed request for sequence ID " << ev_counter() << "! Will not send any data for this sequence ID!";
1150  requestReceiver_->RemoveRequest(ev_counter());
1151 
1152  for (auto& id : dataBuffers_) {
1153  id.second.WindowsSent.erase(ev_counter());
1154  }
1155 
1156  ev_counter_inc(1, true);
1157  }
1158  }
1159 }
1160 
CommandableFragmentGenerator(const fhicl::ParameterSet &ps)
CommandableFragmentGenerator Constructor.
void applyRequestsSingleMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Single. Precondition: dataBufferMutex_ and reques...
virtual bool checkHWStatus_()
Check any relavent hardware status registers. Return false if an error condition exists that should h...
virtual ~CommandableFragmentGenerator()
CommandableFragmentGenerator Destructor.
void applyRequestsBufferMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Buffer. Precondition: dataBufferMutex_ and reques...
void getDataBufferStats(Fragment::fragment_id_t id)
Calculate the size of the dataBuffer and report appropriate metrics.
void getMonitoringDataLoop()
This function regularly calls checkHWStatus_(), and sets the isHardwareOK flag accordingly.
void startDataThread()
Function that launches the data thread (getDataLoop())
std::string ReportCmd(std::string const &which="")
Get a report about a user-specified run-time quantity.
virtual bool metaCommand(std::string const &command, std::string const &arg)
The meta-command is used for implementing user-specific commands in a CommandableFragmentGenerator.
void StopCmd(uint64_t timeout, uint64_t timestamp)
Stop the CommandableFragmentGenerator.
void checkSentWindows(Fragment::sequence_id_t seq)
Check the windows_sent_ooo_ map for sequence IDs that may be removed.
bool sendEmptyFragment(FragmentPtrs &frags, size_t sequenceId, Fragment::fragment_id_t fragmentId, std::string desc)
Send an EmptyFragmentType Fragment.
void applyRequestsWindowMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Window. Precondition: dataBufferMutex_ and reques...
void StartCmd(int run, uint64_t timeout, uint64_t timestamp)
Start the CommandableFragmentGenerator.
virtual void pauseNoMutex()
On call to PauseCmd, pauseNoMutex() is called prior to PauseCmd acquiring the mutex ...
bool check_stop()
Routine used by applyRequests to make sure that all outstanding requests have been fulfilled before r...
void ResumeCmd(uint64_t timeout, uint64_t timestamp)
Resume the CommandableFragmentGenerator.
bool getNext(FragmentPtrs &output) overridefinal
getNext calls either applyRequests or getNext_ to get any data that is ready to be sent to the EventB...
size_t ev_counter_inc(size_t step=1, bool force=false)
Increment the event counter, if the current RequestMode allows it.
bool applyRequests(FragmentPtrs &frags)
See if any requests have been received, and add the corresponding data Fragment objects to the output...
void applyRequestsIgnoredMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Ignored. Precondition: dataBufferMutex_ and reque...
void PauseCmd(uint64_t timeout, uint64_t timestamp)
Pause the CommandableFragmentGenerator.
void getDataLoop()
When separate_data_thread is set to true, this loop repeatedly calls getNext_ and adds returned Fragm...
void sendEmptyFragments(FragmentPtrs &frags, std::map< Fragment::sequence_id_t, Fragment::timestamp_t > &requests)
This function is for Buffered and Single request modes, as they can only respond to one data request ...
Receive data requests and make them available to CommandableFragmentGenerator or other interested par...
void startMonitoringThread()
Function that launches the monitoring thread (getMonitoringDataLoop())
virtual void pause()
If a CommandableFragmentGenerator subclass is reading from hardware, the implementation of pause() sh...
virtual void resume()
The subrun number will be incremented before a call to resume.
bool waitForDataBufferReady(Fragment::fragment_id_t id)
Wait for the data buffer to drain (dataBufferIsTooLarge returns false), periodically reporting status...
virtual std::string report()
Let&#39;s say that the contract with the report() functions is that they return a non-empty string if the...
std::string printMode_()
Return the string representation of the current RequestMode.
void checkDataBuffer(Fragment::fragment_id_t id)
Perform data buffer pruning operations. If the RequestMode is Single, removes all but the latest Frag...
virtual std::string reportSpecific(std::string const &what)
Report the status of a specific quantity
bool dataBufferIsTooLarge(Fragment::fragment_id_t id)
Test the configured constraints on the data buffer.
void joinThreads()
Join any data-taking threads. Should be called when destructing CommandableFragmentGenerator.