artdaq  v3_08_00
CommandableFragmentGenerator.cc
1 #include "artdaq/DAQdata/Globals.hh"
2 #define TRACE_NAME (app_name + "_CommandableFragmentGenerator").c_str() // include these 2 first -
3 
4 #include "artdaq/Generators/CommandableFragmentGenerator.hh"
5 
6 #include <boost/exception/all.hpp>
7 #include <boost/throw_exception.hpp>
8 
9 #include <iterator>
10 #include <limits>
11 
12 #include "canvas/Utilities/Exception.h"
13 #include "cetlib_except/exception.h"
14 #include "fhiclcpp/ParameterSet.h"
15 
16 #include "artdaq-core/Data/ContainerFragmentLoader.hh"
17 #include "artdaq-core/Data/Fragment.hh"
18 #include "artdaq-core/Utilities/ExceptionHandler.hh"
19 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh"
20 #include "artdaq-core/Utilities/TimeUtils.hh"
21 
22 #include <sys/poll.h>
23 #include <algorithm>
24 #include <fstream>
25 #include <iomanip>
26 #include <iostream>
27 #include <iterator>
29 
30 #define TLVL_GETNEXT 10
31 #define TLVL_GETNEXT_VERBOSE 20
32 #define TLVL_CHECKSTOP 11
33 #define TLVL_EVCOUNTERINC 12
34 #define TLVL_GETDATALOOP 13
35 #define TLVL_GETDATALOOP_DATABUFFWAIT 21
36 #define TLVL_GETDATALOOP_VERBOSE 20
37 #define TLVL_WAITFORBUFFERREADY 15
38 #define TLVL_GETBUFFERSTATS 16
39 #define TLVL_CHECKDATABUFFER 17
40 #define TLVL_GETMONITORINGDATA 18
41 #define TLVL_APPLYREQUESTS 9
42 #define TLVL_SENDEMPTYFRAGMENTS 19
43 #define TLVL_CHECKWINDOWS 14
44 
46  : mutex_()
47  , requestReceiver_(nullptr)
48  , bufferModeKeepLatest_(ps.get<bool>("buffer_mode_keep_latest", false))
49  , windowOffset_(ps.get<Fragment::timestamp_t>("request_window_offset", 0))
50  , windowWidth_(ps.get<Fragment::timestamp_t>("request_window_width", 0))
51  , staleTimeout_(ps.get<Fragment::timestamp_t>("stale_request_timeout", 0xFFFFFFFF))
52  , expectedType_(ps.get<Fragment::type_t>("expected_fragment_type", Fragment::type_t(Fragment::EmptyFragmentType)))
53  , uniqueWindows_(ps.get<bool>("request_windows_are_unique", true))
54  , missing_request_window_timeout_us_(ps.get<size_t>("missing_request_window_timeout_us", 5000000))
55  , window_close_timeout_us_(ps.get<size_t>("window_close_timeout_us", 2000000))
56  , useDataThread_(ps.get<bool>("separate_data_thread", false))
57  , circularDataBufferMode_(ps.get<bool>("circular_buffer_mode", false))
58  , sleep_on_no_data_us_(ps.get<size_t>("sleep_on_no_data_us", 0))
59  , data_thread_running_(false)
60  , maxDataBufferDepthFragments_(ps.get<int>("data_buffer_depth_fragments", 1000))
61  , maxDataBufferDepthBytes_(ps.get<size_t>("data_buffer_depth_mb", 1000) * 1024 * 1024)
62  , useMonitoringThread_(ps.get<bool>("separate_monitoring_thread", false))
63  , monitoringInterval_(ps.get<int64_t>("hardware_poll_interval_us", 0))
64  , lastMonitoringCall_()
65  , isHardwareOK_(true)
66  , run_number_(-1)
67  , subrun_number_(-1)
68  , timeout_(std::numeric_limits<uint64_t>::max())
69  , timestamp_(std::numeric_limits<uint64_t>::max())
70  , should_stop_(false)
71  , exception_(false)
72  , force_stop_(false)
73  , latest_exception_report_("none")
74  , ev_counter_(1)
75  , board_id_(-1)
76  , sleep_on_stop_us_(0)
77 {
78  board_id_ = ps.get<int>("board_id");
79  instance_name_for_metrics_ = "BoardReader." + boost::lexical_cast<std::string>(board_id_);
80 
81  auto fragment_ids = ps.get<std::vector<artdaq::Fragment::fragment_id_t>>("fragment_ids", std::vector<artdaq::Fragment::fragment_id_t>());
82 
83  TLOG(TLVL_TRACE) << "artdaq::CommandableFragmentGenerator::CommandableFragmentGenerator(ps)";
84  int fragment_id = ps.get<int>("fragment_id", -99);
85 
86  if (fragment_id != -99)
87  {
88  if (fragment_ids.size() != 0)
89  {
90  latest_exception_report_ = "Error in CommandableFragmentGenerator: can't both define \"fragment_id\" and \"fragment_ids\" in FHiCL document";
91  TLOG(TLVL_ERROR) << latest_exception_report_;
92  throw cet::exception(latest_exception_report_);
93  }
94  else
95  {
96  fragment_ids.emplace_back(fragment_id);
97  }
98  }
99 
100  for (auto& id : fragment_ids)
101  {
102  dataBuffers_[id].DataBufferDepthBytes = 0;
103  dataBuffers_[id].DataBufferDepthFragments = 0;
104  dataBuffers_[id].HighestRequestSeen = 0;
105  dataBuffers_[id].BufferFragmentKept = false;
106  dataBuffers_[id].DataBuffer.emplace_back(FragmentPtr(new Fragment()));
107  (*dataBuffers_[id].DataBuffer.begin())->setSystemType(Fragment::EmptyFragmentType);
108  }
109 
110  sleep_on_stop_us_ = ps.get<int>("sleep_on_stop_us", 0);
111 
112  std::string modeString = ps.get<std::string>("request_mode", "ignored");
113  if (modeString == "single" || modeString == "Single")
114  {
115  mode_ = RequestMode::Single;
116  }
117  else if (modeString.find("buffer") != std::string::npos || modeString.find("Buffer") != std::string::npos)
118  {
119  mode_ = RequestMode::Buffer;
120  }
121  else if (modeString == "window" || modeString == "Window")
122  {
123  mode_ = RequestMode::Window;
124  }
125  else if (modeString.find("ignore") != std::string::npos || modeString.find("Ignore") != std::string::npos)
126  {
127  mode_ = RequestMode::Ignored;
128  }
129  else if (modeString.find("sequence") != std::string::npos || modeString.find("Sequence") != std::string::npos)
130  {
131  mode_ = RequestMode::SequenceID;
132  }
133  TLOG(TLVL_DEBUG) << "Request mode is " << printMode_();
134 
135  if (mode_ != RequestMode::Ignored)
136  {
137  if (!useDataThread_)
138  {
139  latest_exception_report_ = "Error in CommandableFragmentGenerator: use_data_thread must be true when request_mode is not \"Ignored\"!";
140  TLOG(TLVL_ERROR) << latest_exception_report_;
141  throw cet::exception(latest_exception_report_);
142  }
143  requestReceiver_.reset(new RequestReceiver(ps));
144  }
145 }
146 
148 {
149  joinThreads();
150  requestReceiver_.reset(nullptr);
151 }
152 
154 {
155  should_stop_ = true;
156  force_stop_ = true;
157  TLOG(TLVL_DEBUG) << "Joining dataThread";
158  if (dataThread_.joinable()) dataThread_.join();
159  TLOG(TLVL_DEBUG) << "Joining monitoringThread";
160  if (monitoringThread_.joinable()) monitoringThread_.join();
161  TLOG(TLVL_DEBUG) << "joinThreads complete";
162 }
163 
165 {
166  bool result = true;
167 
168  if (check_stop()) usleep(sleep_on_stop_us_);
169  if (exception() || force_stop_) return false;
170 
171  if (!useMonitoringThread_ && monitoringInterval_ > 0)
172  {
173  TLOG(TLVL_GETNEXT) << "getNext: Checking whether to collect Monitoring Data";
174  auto now = std::chrono::steady_clock::now();
175 
176  if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
177  {
178  TLOG(TLVL_GETNEXT) << "getNext: Collecting Monitoring Data";
179  isHardwareOK_ = checkHWStatus_();
180  TLOG(TLVL_GETNEXT) << "getNext: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_;
181  lastMonitoringCall_ = now;
182  }
183  }
184 
185  try
186  {
187  std::lock_guard<std::mutex> lk(mutex_);
188  if (useDataThread_)
189  {
190  TLOG(TLVL_TRACE) << "getNext: Calling applyRequests";
191  result = applyRequests(output);
192  TLOG(TLVL_TRACE) << "getNext: Done with applyRequests result=" << std::boolalpha << result;
193  for (auto dataIter = output.begin(); dataIter != output.end(); ++dataIter)
194  {
195  TLOG(20) << "getNext: applyRequests() returned fragment with sequenceID = " << (*dataIter)->sequenceID()
196  << ", type = " << (*dataIter)->typeString() << ", id = " << std::to_string((*dataIter)->fragmentID())
197  << ", timestamp = " << (*dataIter)->timestamp() << ", and sizeBytes = " << (*dataIter)->sizeBytes();
198  }
199 
200  if (exception())
201  {
202  TLOG(TLVL_ERROR) << "Exception found in BoardReader with board ID " << board_id() << "; BoardReader will now return error status when queried";
203  throw cet::exception("CommandableFragmentGenerator") << "Exception found in BoardReader with board ID " << board_id() << "; BoardReader will now return error status when queried";
204  }
205  }
206  else
207  {
208  if (!isHardwareOK_)
209  {
210  TLOG(TLVL_ERROR) << "Stopping CFG because the hardware reports bad status!";
211  return false;
212  }
213  TLOG(TLVL_TRACE) << "getNext: Calling getNext_ w/ ev_counter()=" << ev_counter();
214  try
215  {
216  result = getNext_(output);
217  }
218  catch (...)
219  {
220  throw;
221  }
222  TLOG(TLVL_TRACE) << "getNext: Done with getNext_ - ev_counter() now " << ev_counter();
223  for (auto dataIter = output.begin(); dataIter != output.end(); ++dataIter)
224  {
225  TLOG(TLVL_GETNEXT_VERBOSE) << "getNext: getNext_() returned fragment with sequenceID = " << (*dataIter)->sequenceID()
226  << ", type = " << (*dataIter)->typeString() << ", id = " << std::to_string((*dataIter)->fragmentID())
227  << ", timestamp = " << (*dataIter)->timestamp() << ", and sizeBytes = " << (*dataIter)->sizeBytes();
228  }
229  }
230  }
231  catch (const cet::exception& e)
232  {
233  latest_exception_report_ = "cet::exception caught in getNext(): ";
234  latest_exception_report_.append(e.what());
235  TLOG(TLVL_ERROR) << "getNext: cet::exception caught: " << e;
236  set_exception(true);
237  return false;
238  }
239  catch (const boost::exception& e)
240  {
241  latest_exception_report_ = "boost::exception caught in getNext(): ";
242  latest_exception_report_.append(boost::diagnostic_information(e));
243  TLOG(TLVL_ERROR) << "getNext: boost::exception caught: " << boost::diagnostic_information(e);
244  set_exception(true);
245  return false;
246  }
247  catch (const std::exception& e)
248  {
249  latest_exception_report_ = "std::exception caught in getNext(): ";
250  latest_exception_report_.append(e.what());
251  TLOG(TLVL_ERROR) << "getNext: std::exception caught: " << e.what();
252  set_exception(true);
253  return false;
254  }
255  catch (...)
256  {
257  latest_exception_report_ = "Unknown exception caught in getNext().";
258  TLOG(TLVL_ERROR) << "getNext: unknown exception caught";
259  set_exception(true);
260  return false;
261  }
262 
263  if (!result)
264  {
265  TLOG(TLVL_DEBUG) << "getNext: Either getNext_ or applyRequests returned false, stopping";
266  }
267 
268  if (metricMan && !output.empty())
269  {
270  auto timestamp = output.front()->timestamp();
271 
272  if (output.size() > 1)
273  { // Only bother sorting if >1 entry
274  for (auto& outputfrag : output)
275  {
276  if (outputfrag->timestamp() > timestamp)
277  {
278  timestamp = outputfrag->timestamp();
279  }
280  }
281  }
282 
283  metricMan->sendMetric("Last Timestamp", timestamp, "Ticks", 1, MetricMode::LastPoint);
284  }
285 
286  return result;
287 }
288 
290 {
291  TLOG(TLVL_CHECKSTOP) << "CFG::check_stop: should_stop=" << should_stop() << ", useDataThread_=" << useDataThread_ << ", exception status =" << int(exception());
292 
293  if (!should_stop()) return false;
294  if (!useDataThread_ || mode_ == RequestMode::Ignored) return true;
295  if (force_stop_) return true;
296 
297  // check_stop returns true if the CFG should stop. We should wait for the RequestReceiver to stop before stopping.
298  TLOG(TLVL_DEBUG) << "should_stop is true, force_stop_ is false, requestReceiver_->isRunning() is " << std::boolalpha << requestReceiver_->isRunning();
299  return !requestReceiver_->isRunning();
300 }
301 
303 {
304  if (force || mode_ == RequestMode::Ignored || mode_ == RequestMode::SequenceID)
305  {
306  TLOG(TLVL_EVCOUNTERINC) << "ev_counter_inc: Incrementing ev_counter from " << ev_counter() << " by " << step;
307  return ev_counter_.fetch_add(step);
308  }
309  return ev_counter_.load();
310 } // returns the prev value
311 
312 void artdaq::CommandableFragmentGenerator::StartCmd(int run, uint64_t timeout, uint64_t timestamp)
313 {
314  TLOG(TLVL_TRACE) << "Start Command received.";
315  if (run < 0)
316  {
317  TLOG(TLVL_ERROR) << "negative run number";
318  throw cet::exception("CommandableFragmentGenerator") << "negative run number";
319  }
320 
321  timeout_ = timeout;
322  timestamp_ = timestamp;
323  ev_counter_.store(1);
324  {
325  std::unique_lock<std::mutex> lk(dataBuffersMutex_);
326  for (auto& id : dataBuffers_)
327  {
328  id.second.DataBufferDepthBytes = 0;
329  id.second.DataBufferDepthFragments = 0;
330  id.second.HighestRequestSeen = 0;
331  id.second.BufferFragmentKept = false;
332  id.second.DataBuffer.clear();
333  id.second.WindowsSent.clear();
334  }
335  }
336 
337  should_stop_.store(false);
338  force_stop_.store(false);
339  exception_.store(false);
340  run_number_ = run;
341  subrun_number_ = 1;
342  latest_exception_report_ = "none";
343 
344  start();
345 
346  std::unique_lock<std::mutex> lk(mutex_);
347  if (useDataThread_) startDataThread();
348  if (useMonitoringThread_) startMonitoringThread();
349  if (mode_ != RequestMode::Ignored)
350  {
351  requestReceiver_->SetRunNumber(static_cast<uint32_t>(run));
352  requestReceiver_->startRequestReception();
353  }
354  TLOG(TLVL_TRACE) << "Start Command complete.";
355 }
356 
357 void artdaq::CommandableFragmentGenerator::StopCmd(uint64_t timeout, uint64_t timestamp)
358 {
359  TLOG(TLVL_TRACE) << "Stop Command received.";
360 
361  timeout_ = timeout;
362  timestamp_ = timestamp;
363  if (requestReceiver_)
364  {
365  TLOG(TLVL_DEBUG) << "Stopping Request reception BEGIN";
366  requestReceiver_->stopRequestReception();
367  TLOG(TLVL_DEBUG) << "Stopping Request reception END";
368  }
369 
370  stopNoMutex();
371  should_stop_.store(true);
372  std::unique_lock<std::mutex> lk(mutex_);
373  stop();
374 
375  joinThreads();
376  TLOG(TLVL_TRACE) << "Stop Command complete.";
377 }
378 
379 void artdaq::CommandableFragmentGenerator::PauseCmd(uint64_t timeout, uint64_t timestamp)
380 {
381  TLOG(TLVL_TRACE) << "Pause Command received.";
382  timeout_ = timeout;
383  timestamp_ = timestamp;
384  //if (requestReceiver_->isRunning()) requestReceiver_->stopRequestReceiverThread();
385 
386  pauseNoMutex();
387  should_stop_.store(true);
388  std::unique_lock<std::mutex> lk(mutex_);
389 
390  pause();
391 }
392 
393 void artdaq::CommandableFragmentGenerator::ResumeCmd(uint64_t timeout, uint64_t timestamp)
394 {
395  TLOG(TLVL_TRACE) << "Resume Command received.";
396  timeout_ = timeout;
397  timestamp_ = timestamp;
398 
399  subrun_number_ += 1;
400  should_stop_ = false;
401 
402  {
403  std::unique_lock<std::mutex> lk(dataBuffersMutex_);
404  for (auto& id : dataBuffers_)
405  {
406  id.second.DataBufferDepthBytes = 0;
407  id.second.DataBufferDepthFragments = 0;
408  id.second.BufferFragmentKept = false;
409  id.second.DataBuffer.clear();
410  }
411  }
412  // no lock required: thread not started yet
413  resume();
414 
415  std::unique_lock<std::mutex> lk(mutex_);
416  //if (useDataThread_) startDataThread();
417  //if (useMonitoringThread_) startMonitoringThread();
418  //if (mode_ != RequestMode::Ignored && !requestReceiver_->isRunning()) requestReceiver_->startRequestReceiverThread();
419  TLOG(TLVL_TRACE) << "Resume Command complete.";
420 }
421 
422 std::string artdaq::CommandableFragmentGenerator::ReportCmd(std::string const& which)
423 {
424  TLOG(TLVL_TRACE) << "Report Command received.";
425  std::lock_guard<std::mutex> lk(mutex_);
426 
427  // 14-May-2015, KAB: please see the comments associated with the report()
428  // methods in the CommandableFragmentGenerator.hh file for more information
429  // on the use of those methods in this method.
430 
431  // check if the child class has something meaningful for this request
432  std::string childReport = reportSpecific(which);
433  if (childReport.length() > 0) { return childReport; }
434 
435  // handle the requests that we can take care of at this level
436  if (which == "latest_exception")
437  {
438  return latest_exception_report_;
439  }
440 
441  // check if the child class has provided a catch-all report function
442  childReport = report();
443  if (childReport.length() > 0) { return childReport; }
444 
445  // ELF: 5/31/2019: Let BoardReaderCore's report handle this...
446  /*
447  // if we haven't been able to come up with any report so far, say so
448  std::string tmpString = "The \"" + which + "\" command is not ";
449  tmpString.append("currently supported by the ");
450  tmpString.append(metricsReportingInstanceName());
451  tmpString.append(" fragment generator.");
452  */
453  TLOG(TLVL_TRACE) << "Report Command complete.";
454  return ""; //tmpString;
455 }
456 
457 // Default implemenetations of state functions
459 {
460 #pragma message "Using default implementation of CommandableFragmentGenerator::pauseNoMutex()"
461 }
462 
464 {
465 #pragma message "Using default implementation of CommandableFragmentGenerator::pause()"
466 }
467 
469 #pragma message "Using default implementation of CommandableFragmentGenerator::resume()"
470 }
471 
473 {
474 #pragma message "Using default implementation of CommandableFragmentGenerator::report()"
475  return "";
476 }
477 
479 {
480 #pragma message "Using default implementation of CommandableFragmentGenerator::reportSpecific(std::string)"
481  return "";
482 }
483 
485 {
486 #pragma message "Using default implementation of CommandableFragmentGenerator::checkHWStatus_()"
487  return true;
488 }
489 
490 bool artdaq::CommandableFragmentGenerator::metaCommand(std::string const&, std::string const&)
491 {
492 #pragma message "Using default implementation of CommandableFragmentGenerator::metaCommand(std::string, std::string)"
493  return true;
494 }
495 
497 {
498  if (dataThread_.joinable()) dataThread_.join();
499  TLOG(TLVL_INFO) << "Starting Data Receiver Thread";
500  try
501  {
502  dataThread_ = boost::thread(&CommandableFragmentGenerator::getDataLoop, this);
503  }
504  catch (const boost::exception& e)
505  {
506  TLOG(TLVL_ERROR) << "Caught boost::exception starting Data Receiver thread: " << boost::diagnostic_information(e) << ", errno=" << errno;
507  std::cerr << "Caught boost::exception starting Data Receiver thread: " << boost::diagnostic_information(e) << ", errno=" << errno << std::endl;
508  exit(5);
509  }
510 }
511 
513 {
514  if (monitoringThread_.joinable()) monitoringThread_.join();
515  TLOG(TLVL_INFO) << "Starting Hardware Monitoring Thread";
516  try
517  {
518  monitoringThread_ = boost::thread(&CommandableFragmentGenerator::getMonitoringDataLoop, this);
519  }
520  catch (const boost::exception& e)
521  {
522  TLOG(TLVL_ERROR) << "Caught boost::exception starting Hardware Monitoring thread: " << boost::diagnostic_information(e) << ", errno=" << errno;
523  std::cerr << "Caught boost::exception starting Hardware Monitoring thread: " << boost::diagnostic_information(e) << ", errno=" << errno << std::endl;
524  exit(5);
525  }
526 }
527 
529 {
530  switch (mode_)
531  {
532  case RequestMode::Single:
533  return "Single";
534  case RequestMode::Buffer:
535  return "Buffer";
536  case RequestMode::Window:
537  return "Window";
538  case RequestMode::Ignored:
539  return "Ignored";
540  case RequestMode::SequenceID:
541  return "SequenceID";
542  }
543 
544  return "ERROR";
545 }
546 
547 //
548 // The "useDataThread_" thread
549 //
551 {
552  data_thread_running_ = true;
553  while (!force_stop_)
554  {
555  if (!isHardwareOK_)
556  {
557  TLOG(TLVL_DEBUG) << "getDataLoop: isHardwareOK is " << isHardwareOK_ << ", aborting data thread";
558  data_thread_running_ = false;
559  return;
560  }
561 
562  TLOG(TLVL_GETDATALOOP) << "getDataLoop: calling getNext_";
563 
564  bool data = false;
565  auto startdata = std::chrono::steady_clock::now();
566  FragmentPtrs newData;
567 
568  try
569  {
570  data = getNext_(newData);
571  }
572  catch (...)
573  {
574  ExceptionHandler(ExceptionHandlerRethrow::no,
575  "Exception thrown by fragment generator in CommandableFragmentGenerator::getDataLoop; setting exception state to \"true\"");
576  set_exception(true);
577 
578  data_thread_running_ = false;
579  return;
580  }
581 
582  if (metricMan)
583  {
584  metricMan->sendMetric("Avg Data Acquisition Time", TimeUtils::GetElapsedTime(startdata), "s", 3, artdaq::MetricMode::Average);
585  }
586 
587  if (newData.size() == 0 && sleep_on_no_data_us_ > 0)
588  {
589  usleep(sleep_on_no_data_us_);
590  }
591 
592  auto dataIter = newData.begin();
593  while (dataIter != newData.end())
594  {
595  TLOG(TLVL_GETDATALOOP_VERBOSE) << "getDataLoop: getNext_() returned fragment with timestamp = " << (*dataIter)->timestamp() << ", and sizeBytes = " << (*dataIter)->sizeBytes();
596 
597  auto frag_id = (*dataIter)->fragmentID();
598  if (!dataBuffers_.count(frag_id))
599  {
600  TLOG(TLVL_ERROR) << "DataBufferError: "
601  << "Error in CommandableFragmentGenerator: Recevied Fragment with fragment_id " << frag_id << ", but this ID was not declared in fragment_ids!";
602  throw cet::exception("DataBufferError") << "Error in CommandableFragmentGenerator: Recevied Fragment with fragment_id " << frag_id << ", but this ID was not declared in fragment_ids!";
603  }
604 
605  TLOG(TLVL_GETDATALOOP_DATABUFFWAIT) << "Waiting for data buffer ready";
606  if (!waitForDataBufferReady(frag_id)) return;
607  TLOG(TLVL_GETDATALOOP_DATABUFFWAIT) << "Done waiting for data buffer ready";
608 
609  TLOG(TLVL_GETDATALOOP) << "getDataLoop: processing data";
610  if (data && !force_stop_)
611  {
612  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
613  switch (mode_)
614  {
615  case RequestMode::Single:
616  dataBuffers_[frag_id].DataBuffer.clear();
617  dataBuffers_[frag_id].DataBufferDepthBytes = (*dataIter)->sizeBytes();
618  dataBuffers_[frag_id].DataBuffer.emplace_back(std::move(*dataIter));
619  dataIter = newData.erase(dataIter);
620  break;
621  case RequestMode::Buffer:
622  case RequestMode::Ignored:
623  case RequestMode::Window:
624  case RequestMode::SequenceID:
625  default:
626  //dataBuffer_.reserve(dataBuffer_.size() + newDataBuffer_.size());
627  dataBuffers_[frag_id].DataBufferDepthBytes += (*dataIter)->sizeBytes();
628  dataBuffers_[frag_id].DataBuffer.emplace_back(std::move(*dataIter));
629  dataIter = newData.erase(dataIter);
630  break;
631  }
632  getDataBufferStats(frag_id);
633  }
634  else
635  {
636  break;
637  }
638  }
639 
640  {
641  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
642  for (auto& id : dataBuffers_)
643  {
644  if (id.second.DataBuffer.size() > 0)
645  {
646  dataCondition_.notify_all();
647  break;
648  }
649  }
650  }
651  if (!data || force_stop_)
652  {
653  TLOG(TLVL_INFO) << "Data flow has stopped. Ending data collection thread";
654  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
655  data_thread_running_ = false;
656  if (requestReceiver_) requestReceiver_->ClearRequests();
657  newData.clear();
658  TLOG(TLVL_INFO) << "getDataLoop: Ending thread";
659  return;
660  }
661  }
662 }
663 
665 {
666  if (!dataBuffers_.count(id))
667  {
668  TLOG(TLVL_ERROR) << "DataBufferError: "
669  << "Error in CommandableFragmentGenerator: Cannot wait for data buffer for ID " << id << " because it does not exist!";
670  throw cet::exception("DataBufferError") << "Error in CommandableFragmentGenerator: Cannot wait for data buffer for ID " << id << " because it does not exist!";
671  }
672  auto startwait = std::chrono::steady_clock::now();
673  auto first = true;
674  auto lastwaittime = 0ULL;
675 
676  {
677  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
678  getDataBufferStats(id);
679  }
680 
681  while (dataBufferIsTooLarge(id))
682  {
683  if (!circularDataBufferMode_)
684  {
685  if (should_stop())
686  {
687  TLOG(TLVL_DEBUG) << "Run ended while waiting for buffer to shrink!";
688  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
689  getDataBufferStats(id);
690  dataCondition_.notify_all();
691  data_thread_running_ = false;
692  return false;
693  }
694  auto waittime = TimeUtils::GetElapsedTimeMilliseconds(startwait);
695 
696  if (first || (waittime != lastwaittime && waittime % 1000 == 0))
697  {
698  TLOG(TLVL_WARNING) << "Bad Omen: Data Buffer has exceeded its size limits. "
699  << "(seq_id=" << ev_counter() << ", frag_id=" << id
700  << ", frags=" << dataBuffers_[id].DataBufferDepthFragments << "/" << maxDataBufferDepthFragments_
701  << ", szB=" << dataBuffers_[id].DataBufferDepthBytes << "/" << maxDataBufferDepthBytes_ << ")"
702  << ", timestamps=" << dataBuffers_[id].DataBuffer.front()->timestamp() << "-" << dataBuffers_[id].DataBuffer.back()->timestamp();
703  TLOG(TLVL_TRACE) << "Bad Omen: Possible causes include requests not getting through or Ignored-mode BR issues";
704  first = false;
705  }
706  if (waittime % 5 && waittime != lastwaittime)
707  {
708  TLOG(TLVL_WAITFORBUFFERREADY) << "getDataLoop: Data Retreival paused for " << waittime << " ms waiting for data buffer to drain";
709  }
710  lastwaittime = waittime;
711  usleep(1000);
712  }
713  else
714  {
715  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
716  getDataBufferStats(id); // Re-check under lock
717  if (dataBufferIsTooLarge(id))
718  {
719  auto begin = dataBuffers_[id].DataBuffer.begin();
720  if (begin == dataBuffers_[id].DataBuffer.end())
721  {
722  TLOG(TLVL_WARNING) << "Data buffer is reported as too large, but doesn't contain any Fragments! Possible corrupt memory!";
723  continue;
724  }
725  if (*begin)
726  {
727  TLOG(TLVL_WAITFORBUFFERREADY) << "waitForDataBufferReady: Dropping Fragment with timestamp " << (*begin)->timestamp() << " from data buffer (Buffer over-size, circular data buffer mode)";
728  }
729  dataBuffers_[id].DataBufferDepthBytes -= (*begin)->sizeBytes();
730  dataBuffers_[id].DataBuffer.erase(begin);
731  dataBuffers_[id].BufferFragmentKept = false; // If any Fragments are removed from data buffer, then we know we don't have to ignore the first one anymore
732  getDataBufferStats(id);
733  }
734  }
735  }
736  return true;
737 }
738 
740 {
741  if (!dataBuffers_.count(id))
742  {
743  TLOG(TLVL_ERROR) << "DataBufferError: "
744  << "Error in CommandableFragmentGenerator: Cannot check size of data buffer for ID " << id << " because it does not exist!";
745  throw cet::exception("DataBufferError") << "Error in CommandableFragmentGenerator: Cannot check size of data buffer for ID " << id << " because it does not exist!";
746  }
747  return (maxDataBufferDepthFragments_ > 0 && dataBuffers_[id].DataBufferDepthFragments > maxDataBufferDepthFragments_) || (maxDataBufferDepthBytes_ > 0 && dataBuffers_[id].DataBufferDepthBytes > maxDataBufferDepthBytes_);
748 }
749 
751 {
752  if (!dataBuffers_.count(id))
753  {
754  TLOG(TLVL_ERROR) << "DataBufferError: "
755  << "Error in CommandableFragmentGenerator: Cannot get stats of data buffer for ID " << id << " because it does not exist!";
756  throw cet::exception("DataBufferError") << "Error in CommandableFragmentGenerator: Cannot get stats of data buffer for ID " << id << " because it does not exist!";
757  }
759  dataBuffers_[id].DataBufferDepthFragments = dataBuffers_[id].DataBuffer.size();
760 
761  if (metricMan)
762  {
763  TLOG(TLVL_GETBUFFERSTATS) << "getDataBufferStats: Sending Metrics";
764  metricMan->sendMetric("Buffer Depth Fragments", dataBuffers_[id].DataBufferDepthFragments.load(), "fragments", 1, MetricMode::LastPoint);
765  metricMan->sendMetric("Buffer Depth Bytes", dataBuffers_[id].DataBufferDepthBytes.load(), "bytes", 1, MetricMode::LastPoint);
766 
767  auto bufferDepthFragmentsPercent = dataBuffers_[id].DataBufferDepthFragments.load() * 100 / static_cast<double>(maxDataBufferDepthFragments_);
768  auto bufferDepthBytesPercent = dataBuffers_[id].DataBufferDepthBytes.load() * 100 / static_cast<double>(maxDataBufferDepthBytes_);
769  metricMan->sendMetric("Fragment Generator Buffer Full %Fragments", bufferDepthFragmentsPercent, "%", 3, MetricMode::LastPoint);
770  metricMan->sendMetric("Fragment Generator Buffer Full %Bytes", bufferDepthBytesPercent, "%", 3, MetricMode::LastPoint);
771  metricMan->sendMetric("Fragment Generator Buffer Full %", bufferDepthFragmentsPercent > bufferDepthBytesPercent ? bufferDepthFragmentsPercent : bufferDepthBytesPercent, "%", 1, MetricMode::LastPoint);
772  }
773  TLOG(TLVL_GETBUFFERSTATS) << "getDataBufferStats: frags=" << dataBuffers_[id].DataBufferDepthFragments.load() << "/" << maxDataBufferDepthFragments_
774  << ", sz=" << dataBuffers_[id].DataBufferDepthBytes.load() << "/" << maxDataBufferDepthBytes_;
775 }
776 
778 {
779  if (!dataBuffers_.count(id))
780  {
781  TLOG(TLVL_ERROR) << "DataBufferError: "
782  << "Error in CommandableFragmentGenerator: Cannot check data buffer for ID " << id << " because it does not exist!";
783  throw cet::exception("DataBufferError") << "Error in CommandableFragmentGenerator: Cannot check data buffer for ID " << id << " because it does not exist!";
784  }
785 
786  if (dataBuffers_[id].DataBufferDepthFragments > 0 && mode_ != RequestMode::Single && mode_ != RequestMode::Ignored)
787  {
788  // Eliminate extra fragments
789  getDataBufferStats(id);
790  while (dataBufferIsTooLarge(id))
791  {
792  auto begin = dataBuffers_[id].DataBuffer.begin();
793  TLOG(TLVL_CHECKDATABUFFER) << "checkDataBuffer: Dropping Fragment with timestamp " << (*begin)->timestamp() << " from data buffer (Buffer over-size)";
794  dataBuffers_[id].DataBufferDepthBytes -= (*begin)->sizeBytes();
795  dataBuffers_[id].DataBuffer.erase(begin);
796  dataBuffers_[id].BufferFragmentKept = false; // If any Fragments are removed from data buffer, then we know we don't have to ignore the first one anymore
797  getDataBufferStats(id);
798  }
799  if (dataBuffers_[id].DataBuffer.size() > 0)
800  {
801  TLOG(TLVL_CHECKDATABUFFER) << "Determining if Fragments can be dropped from data buffer";
802  Fragment::timestamp_t last = dataBuffers_[id].DataBuffer.back()->timestamp();
803  Fragment::timestamp_t min = last > staleTimeout_ ? last - staleTimeout_ : 0;
804  for (auto it = dataBuffers_[id].DataBuffer.begin(); it != dataBuffers_[id].DataBuffer.end();)
805  {
806  if ((*it)->timestamp() < min)
807  {
808  TLOG(TLVL_CHECKDATABUFFER) << "checkDataBuffer: Dropping Fragment with timestamp " << (*it)->timestamp() << " from data buffer (timeout=" << staleTimeout_ << ", min=" << min << ")";
809  dataBuffers_[id].DataBufferDepthBytes -= (*it)->sizeBytes();
810  dataBuffers_[id].BufferFragmentKept = false; // If any Fragments are removed from data buffer, then we know we don't have to ignore the first one anymore
811  it = dataBuffers_[id].DataBuffer.erase(it);
812  }
813  else
814  {
815  break;
816  }
817  }
818  getDataBufferStats(id);
819  }
820  }
821 }
822 
824 {
825  while (!force_stop_)
826  {
827  if (should_stop() || monitoringInterval_ <= 0)
828  {
829  TLOG(TLVL_DEBUG) << "getMonitoringDataLoop: should_stop() is " << std::boolalpha << should_stop()
830  << " and monitoringInterval is " << monitoringInterval_ << ", returning";
831  return;
832  }
833  TLOG(TLVL_GETMONITORINGDATA) << "getMonitoringDataLoop: Determining whether to call checkHWStatus_";
834 
835  auto now = std::chrono::steady_clock::now();
836  if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
837  {
838  isHardwareOK_ = checkHWStatus_();
839  TLOG(TLVL_GETMONITORINGDATA) << "getMonitoringDataLoop: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_;
840  lastMonitoringCall_ = now;
841  }
842  usleep(monitoringInterval_ / 10);
843  }
844 }
845 
847 {
848  // dataBuffersMutex_ is held by calling function
849  // We just copy everything that's here into the output.
850  TLOG(TLVL_APPLYREQUESTS) << "Mode is Ignored; Copying data to output";
851  for (auto& id : dataBuffers_)
852  {
853  std::move(id.second.DataBuffer.begin(), id.second.DataBuffer.end(), std::inserter(frags, frags.end()));
854  id.second.DataBufferDepthBytes = 0;
855  id.second.DataBufferDepthFragments = 0;
856  id.second.BufferFragmentKept = false;
857  id.second.DataBuffer.clear();
858  }
859 }
860 
862 {
863  // We only care about the latest request received. Send empties for all others.
864  auto requests = requestReceiver_->GetRequests();
865  while (requests.size() > 1)
866  {
867  // std::map is ordered by key => Last sequence ID in the map is the one we care about
868  requestReceiver_->RemoveRequest(requests.begin()->first);
869  requests.erase(requests.begin());
870  }
871  sendEmptyFragments(frags, requests);
872 
873  // If no requests remain after sendEmptyFragments, return
874  if (requests.size() == 0 || !requests.count(ev_counter())) return;
875 
876  for (auto& id : dataBuffers_)
877  {
878  if (id.second.DataBuffer.size() > 0)
879  {
880  assert(id.second.DataBuffer.size() == 1);
881  TLOG(TLVL_APPLYREQUESTS) << "Mode is Single; Sending copy of last event";
882  for (auto& fragptr : id.second.DataBuffer)
883  {
884  // Return the latest data point
885  auto frag = fragptr.get();
886  auto newfrag = std::unique_ptr<artdaq::Fragment>(new Fragment(ev_counter(), frag->fragmentID()));
887  newfrag->resize(frag->size() - detail::RawFragmentHeader::num_words());
888  memcpy(newfrag->headerAddress(), frag->headerAddress(), frag->sizeBytes());
889  newfrag->setTimestamp(requests[ev_counter()]);
890  newfrag->setSequenceID(ev_counter());
891  frags.push_back(std::move(newfrag));
892  }
893  }
894  else
895  {
896  sendEmptyFragment(frags, ev_counter(), id.first, "No data for");
897  }
898  }
899  requestReceiver_->RemoveRequest(ev_counter());
900  ev_counter_inc(1, true);
901 }
902 
904 {
905  // We only care about the latest request received. Send empties for all others.
906  auto requests = requestReceiver_->GetRequests();
907  while (requests.size() > 1)
908  {
909  // std::map is ordered by key => Last sequence ID in the map is the one we care about
910  requestReceiver_->RemoveRequest(requests.begin()->first);
911  requests.erase(requests.begin());
912  }
913  sendEmptyFragments(frags, requests);
914 
915  // If no requests remain after sendEmptyFragments, return
916  if (requests.size() == 0 || !requests.count(ev_counter())) return;
917 
918  for (auto& id : dataBuffers_)
919  {
920  TLOG(TLVL_DEBUG) << "Creating ContainerFragment for Buffered Fragments";
921  frags.emplace_back(new artdaq::Fragment(ev_counter(), id.first));
922  frags.back()->setTimestamp(requests[ev_counter()]);
923  ContainerFragmentLoader cfl(*frags.back());
924  cfl.set_missing_data(false); // Buffer mode is never missing data, even if there IS no data.
925 
926  // If we kept a Fragment from the previous iteration, but more data has arrived, discard it
927  auto it = id.second.DataBuffer.begin();
928  if (id.second.BufferFragmentKept && id.second.DataBuffer.size() > 1)
929  {
930  id.second.DataBufferDepthBytes -= (*it)->sizeBytes();
931  it = id.second.DataBuffer.erase(it);
932  }
933 
934  // Buffer mode TFGs should simply copy out the whole dataBuffer_ into a ContainerFragment
935  while (it != id.second.DataBuffer.end())
936  {
937  TLOG(TLVL_APPLYREQUESTS) << "ApplyRequests: Adding Fragment with timestamp " << (*it)->timestamp() << " to Container with sequence ID " << ev_counter();
938  cfl.addFragment(*it);
939  if (bufferModeKeepLatest_ && id.second.DataBuffer.size() == 1)
940  {
941  id.second.BufferFragmentKept = true;
942  break;
943  }
944  id.second.DataBufferDepthBytes -= (*it)->sizeBytes();
945  it = id.second.DataBuffer.erase(it);
946  }
947  }
948  requestReceiver_->RemoveRequest(ev_counter());
949  ev_counter_inc(1, true);
950 }
951 
952 void artdaq::CommandableFragmentGenerator::applyRequestsWindowMode_CheckAndFillDataBuffer(artdaq::FragmentPtrs& frags, artdaq::Fragment::fragment_id_t id, artdaq::Fragment::sequence_id_t seq, artdaq::Fragment::timestamp_t ts)
953 {
954  TLOG(TLVL_APPLYREQUESTS) << "applyRequestsWindowMode_CheckAndFillDataBuffer: Checking that data exists for request window " << seq;
955  Fragment::timestamp_t min = ts > windowOffset_ ? ts - windowOffset_ : 0;
956  Fragment::timestamp_t max = min + windowWidth_;
957  TLOG(TLVL_APPLYREQUESTS) << "ApplyRequestsWindowsMode_CheckAndFillDataBuffer: min is " << min << ", max is " << max
958  << " and first/last points in buffer are " << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.front()->timestamp() : 0)
959  << "/" << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.back()->timestamp() : 0)
960  << " (sz=" << dataBuffers_[id].DataBuffer.size() << " [" << dataBuffers_[id].DataBufferDepthBytes.load()
961  << "/" << maxDataBufferDepthBytes_ << "])";
962  bool windowClosed = dataBuffers_[id].DataBuffer.size() > 0 && dataBuffers_[id].DataBuffer.back()->timestamp() >= max;
963  bool windowTimeout = !windowClosed && TimeUtils::GetElapsedTimeMicroseconds(requestReceiver_->GetRequestTime(seq)) > window_close_timeout_us_;
964  if (windowTimeout)
965  {
966  TLOG(TLVL_WARNING) << "applyRequestsWindowMode_CheckAndFillDataBuffer: A timeout occurred waiting for data to close the request window ({" << min << "-" << max
967  << "}, buffer={" << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.front()->timestamp() : 0) << "-"
968  << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.back()->timestamp() : 0)
969  << "} ). Time waiting: "
970  << TimeUtils::GetElapsedTimeMicroseconds(requestReceiver_->GetRequestTime(seq)) << " us "
971  << "(> " << window_close_timeout_us_ << " us).";
972  }
973  if (windowClosed || !data_thread_running_ || windowTimeout)
974  {
975  TLOG(TLVL_DEBUG) << "applyRequestsWindowMode_CheckAndFillDataBuffer: Creating ContainerFragment for Window-requested Fragments";
976  frags.emplace_back(new artdaq::Fragment(seq, id));
977  frags.back()->setTimestamp(ts);
978  ContainerFragmentLoader cfl(*frags.back());
979 
980  // In the spirit of NOvA's MegaPool: (RS = Request start (min), RE = Request End (max))
981  // --- | Buffer Start | --- | Buffer End | ---
982  //1. RS RE | | | |
983  //2. RS | | RE | |
984  //3. RS | | | | RE
985  //4. | | RS RE | |
986  //5. | | RS | | RE
987  //6. | | | | RS RE
988  //
989  // If RE (or RS) is after the end of the buffer, we wait for window_close_timeout_us_. If we're here, then that means that windowClosed is false, and the missing_data flag should be set.
990  // If RS (or RE) is before the start of the buffer, then missing_data should be set to true, as data is assumed to arrive in the buffer in timestamp order
991  // If the dataBuffer has size 0, then windowClosed will be false
992  if (!windowClosed || (dataBuffers_[id].DataBuffer.size() > 0 && dataBuffers_[id].DataBuffer.front()->timestamp() > min))
993  {
994  TLOG(TLVL_DEBUG) << "applyRequestsWindowMode_CheckAndFillDataBuffer: Request window starts before and/or ends after the current data buffer, setting ContainerFragment's missing_data flag!"
995  << " (requestWindowRange=[" << min << "," << max << "], "
996  << "buffer={" << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.front()->timestamp() : 0) << "-"
997  << (dataBuffers_[id].DataBuffer.size() > 0 ? dataBuffers_[id].DataBuffer.back()->timestamp() : 0) << "}";
998  cfl.set_missing_data(true);
999  }
1000 
1001  // Do a little bit more work to decide which fragments to send for a given request
1002  for (auto it = dataBuffers_[id].DataBuffer.begin(); it != dataBuffers_[id].DataBuffer.end();)
1003  {
1004  Fragment::timestamp_t fragT = (*it)->timestamp();
1005  if (fragT < min || fragT > max || (fragT == max && windowWidth_ > 0))
1006  {
1007  ++it;
1008  continue;
1009  }
1010 
1011  TLOG(TLVL_APPLYREQUESTS) << "applyRequestsWindowMode_CheckAndFillDataBuffer: Adding Fragment with timestamp " << (*it)->timestamp() << " to Container";
1012  cfl.addFragment(*it);
1013 
1014  if (uniqueWindows_)
1015  {
1016  dataBuffers_[id].DataBufferDepthBytes -= (*it)->sizeBytes();
1017  it = dataBuffers_[id].DataBuffer.erase(it);
1018  }
1019  else
1020  {
1021  ++it;
1022  }
1023  }
1024 
1025  dataBuffers_[id].WindowsSent[seq] = std::chrono::steady_clock::now();
1026  if (seq > dataBuffers_[id].HighestRequestSeen) dataBuffers_[id].HighestRequestSeen = seq;
1027  }
1028 }
1029 
1031 {
1032  TLOG(TLVL_APPLYREQUESTS) << "applyRequestsWindowMode BEGIN";
1033 
1034  auto requests = requestReceiver_->GetRequests();
1035 
1036  TLOG(TLVL_APPLYREQUESTS) << "applyRequestsWindowMode: Starting request processing";
1037  for (auto req = requests.begin(); req != requests.end();)
1038  {
1039  TLOG(TLVL_APPLYREQUESTS) << "applyRequestsWindowMode: processing request with sequence ID " << req->first << ", timestamp " << req->second;
1040 
1041  while (req->first < ev_counter() && requests.size() > 0)
1042  {
1043  TLOG(TLVL_APPLYREQUESTS) << "applyRequestsWindowMode: Clearing passed request for sequence ID " << req->first;
1044  requestReceiver_->RemoveRequest(req->first);
1045  req = requests.erase(req);
1046  }
1047  if (requests.size() == 0) break;
1048 
1049  for (auto& id : dataBuffers_)
1050  {
1051  if (!id.second.WindowsSent.count(req->first))
1052  {
1053  applyRequestsWindowMode_CheckAndFillDataBuffer(frags, id.first, req->first, req->second);
1054  }
1055  }
1056  checkSentWindows(req->first);
1057  ++req;
1058  }
1059 
1060  // Check sent windows for requests that can be removed
1061  for (auto& id : dataBuffers_)
1062  {
1063  std::set<artdaq::Fragment::sequence_id_t> seqs;
1064  for (auto& seq : id.second.WindowsSent)
1065  {
1066  seqs.insert(seq.first);
1067  }
1068  for (auto& seq : seqs)
1069  {
1070  checkSentWindows(seq);
1071  }
1072  }
1073 }
1074 
1076 {
1077  TLOG(TLVL_APPLYREQUESTS) << "applyRequestsSequenceIDMode BEGIN";
1078 
1079  auto requests = requestReceiver_->GetRequests();
1080 
1081  TLOG(TLVL_APPLYREQUESTS) << "applyRequestsSequenceIDMode: Starting request processing";
1082  for (auto req = requests.begin(); req != requests.end();)
1083  {
1084  TLOG(TLVL_APPLYREQUESTS) << "applyRequestsSequenceIDMode: Checking that data exists for request SequenceID " << req->first;
1085 
1086  for (auto& id : dataBuffers_)
1087  {
1088  if (!id.second.WindowsSent.count(req->first))
1089  {
1090  TLOG(29) << "Searching id " << id.first << " for Fragments with Sequence ID " << req->first;
1091  for (auto it = id.second.DataBuffer.begin(); it != id.second.DataBuffer.end();)
1092  {
1093  auto seq = (*it)->sequenceID();
1094  TLOG(29) << "applyRequestsSequenceIDMode: Fragment SeqID " << seq << ", request ID " << req->first;
1095  if (seq == req->first)
1096  {
1097  TLOG(29) << "applyRequestsSequenceIDMode: Adding Fragment to output";
1098  id.second.WindowsSent[req->first] = std::chrono::steady_clock::now();
1099  id.second.DataBufferDepthBytes -= (*it)->sizeBytes();
1100  frags.push_back(std::move(*it));
1101  it = id.second.DataBuffer.erase(it);
1102  }
1103  else
1104  {
1105  ++it;
1106  }
1107  }
1108  }
1109  if (req->first > id.second.HighestRequestSeen) id.second.HighestRequestSeen = req->first;
1110  }
1111  checkSentWindows(req->first);
1112  ++req;
1113  }
1114 
1115  // Check sent windows for requests that can be removed
1116  for (auto& id : dataBuffers_)
1117  {
1118  std::set<artdaq::Fragment::sequence_id_t> seqs;
1119  for (auto& seq : id.second.WindowsSent)
1120  {
1121  seqs.insert(seq.first);
1122  }
1123  for (auto& seq : seqs)
1124  {
1125  checkSentWindows(seq);
1126  }
1127  }
1128 }
1129 
1131 {
1132  if (check_stop() || exception())
1133  {
1134  return false;
1135  }
1136 
1137  // Wait for data, if in ignored mode, or a request otherwise
1138  if (mode_ == RequestMode::Ignored)
1139  {
1140  while (dataBufferFragmentCount_() == 0)
1141  {
1142  if (check_stop() || exception() || !isHardwareOK_) return false;
1143  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
1144  dataCondition_.wait_for(lock, std::chrono::milliseconds(10), [this]() { return dataBufferFragmentCount_() > 0; });
1145  }
1146  }
1147  else
1148  {
1149  if ((check_stop() && requestReceiver_->size() == 0) || exception()) return false;
1150 
1151  std::unique_lock<std::mutex> lock(dataBuffersMutex_);
1152  dataCondition_.wait_for(lock, std::chrono::milliseconds(10));
1153 
1154  checkDataBuffers();
1155 
1156  // Wait up to 1000 ms for a request...
1157  auto counter = 0;
1158 
1159  while (requestReceiver_->size() == 0 && counter < 100)
1160  {
1161  if (check_stop() || exception()) return false;
1162 
1163  checkDataBuffers();
1164 
1165  requestReceiver_->WaitForRequests(10); // milliseconds
1166  counter++;
1167  }
1168  }
1169 
1170  {
1171  std::unique_lock<std::mutex> dlk(dataBuffersMutex_);
1172 
1173  switch (mode_)
1174  {
1175  case RequestMode::Single:
1176  applyRequestsSingleMode(frags);
1177  break;
1178  case RequestMode::Window:
1179  applyRequestsWindowMode(frags);
1180  break;
1181  case RequestMode::Buffer:
1182  applyRequestsBufferMode(frags);
1183  break;
1184  case RequestMode::SequenceID:
1185  applyRequestsSequenceIDMode(frags);
1186  break;
1187  case RequestMode::Ignored:
1188  default:
1189  applyRequestsIgnoredMode(frags);
1190  break;
1191  }
1192 
1193  if (!data_thread_running_ || force_stop_)
1194  {
1195  TLOG(TLVL_INFO) << "Data thread has stopped; Clearing data buffers";
1196  for (auto& id : dataBuffers_)
1197  {
1198  id.second.DataBufferDepthBytes = 0;
1199  id.second.DataBufferDepthFragments = 0;
1200  id.second.BufferFragmentKept = false;
1201  id.second.DataBuffer.clear();
1202  }
1203  }
1204 
1205  getDataBuffersStats();
1206  }
1207 
1208  if (frags.size() > 0)
1209  TLOG(TLVL_APPLYREQUESTS) << "Finished Processing requests, returning " << frags.size() << " fragments, current ev_counter is " << ev_counter();
1210  return true;
1211 }
1212 
1213 bool artdaq::CommandableFragmentGenerator::sendEmptyFragment(artdaq::FragmentPtrs& frags, size_t seqId, Fragment::fragment_id_t fragmentId, std::string desc)
1214 {
1215  TLOG(TLVL_WARNING) << desc << " sequence ID " << seqId << ", sending empty fragment";
1216  auto frag = new Fragment();
1217  frag->setSequenceID(seqId);
1218  frag->setFragmentID(fragmentId);
1219  frag->setSystemType(Fragment::EmptyFragmentType);
1220  frags.emplace_back(FragmentPtr(frag));
1221  return true;
1222 }
1223 
1224 void artdaq::CommandableFragmentGenerator::sendEmptyFragments(artdaq::FragmentPtrs& frags, std::map<Fragment::sequence_id_t, Fragment::timestamp_t>& requests)
1225 {
1226  if (requests.size() > 0)
1227  {
1228  TLOG(TLVL_SENDEMPTYFRAGMENTS) << "Sending Empty Fragments for Sequence IDs from " << ev_counter() << " up to but not including " << requests.begin()->first;
1229  while (requests.begin()->first > ev_counter())
1230  {
1231  for (auto& fid : dataBuffers_)
1232  {
1233  sendEmptyFragment(frags, ev_counter(), fid.first, "Missed request for");
1234  }
1235  ev_counter_inc(1, true);
1236  }
1237  }
1238 }
1239 
1240 void artdaq::CommandableFragmentGenerator::checkSentWindows(artdaq::Fragment::sequence_id_t seq)
1241 {
1242  TLOG(TLVL_CHECKWINDOWS) << "checkSentWindows: Checking if request " << seq << " can be removed from request list";
1243  bool seqComplete = true;
1244  bool seqTimeout = false;
1245  for (auto& id : dataBuffers_)
1246  {
1247  if (!id.second.WindowsSent.count(seq) || id.second.HighestRequestSeen < seq)
1248  {
1249  seqComplete = false;
1250  }
1251  if (id.second.WindowsSent.count(seq) && TimeUtils::GetElapsedTimeMicroseconds(id.second.WindowsSent[seq]) > missing_request_window_timeout_us_)
1252  {
1253  seqTimeout = true;
1254  }
1255  }
1256  if (seqComplete)
1257  {
1258  TLOG(TLVL_CHECKWINDOWS) << "checkSentWindows: Request " << seq << " is complete, removing from requestReceiver.";
1259  requestReceiver_->RemoveRequest(seq);
1260 
1261  if (ev_counter() == seq)
1262  {
1263  TLOG(TLVL_CHECKWINDOWS) << "checkSentWindows: Sequence ID matches ev_counter, incrementing ev_counter (" << ev_counter() << ")";
1264 
1265  for (auto& id : dataBuffers_)
1266  {
1267  id.second.WindowsSent.erase(seq);
1268  }
1269 
1270  ev_counter_inc(1, true);
1271  }
1272  }
1273  if (seqTimeout)
1274  {
1275  TLOG(TLVL_CHECKWINDOWS) << "checkSentWindows: Sent Window history indicates that requests between " << ev_counter() << " and " << seq << " have timed out.";
1276  while (ev_counter() <= seq)
1277  {
1278  if (ev_counter() < seq) TLOG(TLVL_WARNING) << "Missed request for sequence ID " << ev_counter() << "! Will not send any data for this sequence ID!";
1279  requestReceiver_->RemoveRequest(ev_counter());
1280 
1281  for (auto& id : dataBuffers_)
1282  {
1283  id.second.WindowsSent.erase(ev_counter());
1284  }
1285 
1286  ev_counter_inc(1, true);
1287  }
1288  }
1289 }
CommandableFragmentGenerator(const fhicl::ParameterSet &ps)
CommandableFragmentGenerator Constructor.
void applyRequestsSingleMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Single. Precondition: dataBufferMutex_ and reques...
virtual bool checkHWStatus_()
Check any relavent hardware status registers. Return false if an error condition exists that should h...
virtual ~CommandableFragmentGenerator()
CommandableFragmentGenerator Destructor.
void applyRequestsBufferMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Buffer. Precondition: dataBufferMutex_ and reques...
void getDataBufferStats(Fragment::fragment_id_t id)
Calculate the size of the dataBuffer and report appropriate metrics.
void getMonitoringDataLoop()
This function regularly calls checkHWStatus_(), and sets the isHardwareOK flag accordingly.
void startDataThread()
Function that launches the data thread (getDataLoop())
artdaq::Fragment::fragment_id_t fragment_id() const
Get the Fragment ID of this Fragment generator.
std::string ReportCmd(std::string const &which="")
Get a report about a user-specified run-time quantity.
virtual bool metaCommand(std::string const &command, std::string const &arg)
The meta-command is used for implementing user-specific commands in a CommandableFragmentGenerator.
void StopCmd(uint64_t timeout, uint64_t timestamp)
Stop the CommandableFragmentGenerator.
void applyRequestsWindowMode_CheckAndFillDataBuffer(artdaq::FragmentPtrs &frags, artdaq::Fragment::fragment_id_t id, artdaq::Fragment::sequence_id_t seq, artdaq::Fragment::timestamp_t ts)
void checkSentWindows(Fragment::sequence_id_t seq)
Check the windows_sent_ooo_ map for sequence IDs that may be removed.
bool sendEmptyFragment(FragmentPtrs &frags, size_t sequenceId, Fragment::fragment_id_t fragmentId, std::string desc)
Send an EmptyFragmentType Fragment.
void applyRequestsWindowMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Window. Precondition: dataBufferMutex_ and reques...
void StartCmd(int run, uint64_t timeout, uint64_t timestamp)
Start the CommandableFragmentGenerator.
virtual void pauseNoMutex()
On call to PauseCmd, pauseNoMutex() is called prior to PauseCmd acquiring the mutex ...
bool check_stop()
Routine used by applyRequests to make sure that all outstanding requests have been fulfilled before r...
void ResumeCmd(uint64_t timeout, uint64_t timestamp)
Resume the CommandableFragmentGenerator.
bool getNext(FragmentPtrs &output) overridefinal
getNext calls either applyRequests or getNext_ to get any data that is ready to be sent to the EventB...
size_t ev_counter_inc(size_t step=1, bool force=false)
Increment the event counter, if the current RequestMode allows it.
bool applyRequests(FragmentPtrs &frags)
See if any requests have been received, and add the corresponding data Fragment objects to the output...
void applyRequestsIgnoredMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Ignored. Precondition: dataBufferMutex_ and reque...
void PauseCmd(uint64_t timeout, uint64_t timestamp)
Pause the CommandableFragmentGenerator.
void getDataLoop()
When separate_data_thread is set to true, this loop repeatedly calls getNext_ and adds returned Fragm...
void sendEmptyFragments(FragmentPtrs &frags, std::map< Fragment::sequence_id_t, Fragment::timestamp_t > &requests)
This function is for Buffered and Single request modes, as they can only respond to one data request ...
Receive data requests and make them available to CommandableFragmentGenerator or other interested par...
void startMonitoringThread()
Function that launches the monitoring thread (getMonitoringDataLoop())
virtual void pause()
If a CommandableFragmentGenerator subclass is reading from hardware, the implementation of pause() sh...
virtual void resume()
The subrun number will be incremented before a call to resume.
void applyRequestsSequenceIDMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode SequenceID. Precondition: dataBufferMutex_ and re...
bool waitForDataBufferReady(Fragment::fragment_id_t id)
Wait for the data buffer to drain (dataBufferIsTooLarge returns false), periodically reporting status...
virtual std::string report()
Let&#39;s say that the contract with the report() functions is that they return a non-empty string if the...
std::string printMode_()
Return the string representation of the current RequestMode.
void checkDataBuffer(Fragment::fragment_id_t id)
Perform data buffer pruning operations for the given buffer. If the RequestMode is Single...
virtual std::string reportSpecific(std::string const &what)
Report the status of a specific quantity
bool dataBufferIsTooLarge(Fragment::fragment_id_t id)
Test the configured constraints on the data buffer.
void joinThreads()
Join any data-taking threads. Should be called when destructing CommandableFragmentGenerator.