artdaq  v3_09_01
CommandableFragmentGenerator.cc
1 #include "artdaq/DAQdata/Globals.hh"
2 #define TRACE_NAME (app_name + "_CommandableFragmentGenerator").c_str() // include these 2 first -
3 
4 #include "artdaq/Generators/CommandableFragmentGenerator.hh"
5 
6 #include <boost/exception/all.hpp>
7 #include <boost/throw_exception.hpp>
8 
9 #include <iterator>
10 #include <limits>
11 #include <thread>
12 
13 #include "canvas/Utilities/Exception.h"
14 #include "cetlib_except/exception.h"
15 #include "fhiclcpp/ParameterSet.h"
16 
17 #include "artdaq-core/Data/ContainerFragmentLoader.hh"
18 #include "artdaq-core/Data/Fragment.hh"
19 #include "artdaq-core/Utilities/ExceptionHandler.hh"
20 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh"
21 #include "artdaq-core/Utilities/TimeUtils.hh"
22 
23 #include <sys/poll.h>
24 #include <algorithm>
25 #include <fstream>
26 #include <iomanip>
27 #include <iostream>
28 #include <iterator>
29 #include <memory>
31 
32 #define TLVL_GETNEXT 10
33 #define TLVL_GETNEXT_VERBOSE 20
34 #define TLVL_CHECKSTOP 11
35 #define TLVL_EVCOUNTERINC 12
36 #define TLVL_GETDATALOOP 13
37 #define TLVL_GETDATALOOP_DATABUFFWAIT 21
38 #define TLVL_GETDATALOOP_VERBOSE 20
39 #define TLVL_WAITFORBUFFERREADY 15
40 #define TLVL_GETBUFFERSTATS 16
41 #define TLVL_CHECKDATABUFFER 17
42 #define TLVL_GETMONITORINGDATA 18
43 #define TLVL_APPLYREQUESTS 9
44 #define TLVL_SENDEMPTYFRAGMENTS 19
45 #define TLVL_CHECKWINDOWS 14
46 #define TLVL_EMPTYFRAGMENT 22
47 
49  : mutex_()
50  , useMonitoringThread_(ps.get<bool>("separate_monitoring_thread", false))
51  , monitoringInterval_(ps.get<int64_t>("hardware_poll_interval_us", 0))
52  , isHardwareOK_(true)
53  , run_number_(-1)
54  , subrun_number_(-1)
55  , timeout_(std::numeric_limits<uint64_t>::max())
56  , timestamp_(std::numeric_limits<uint64_t>::max())
57  , should_stop_(false)
58  , exception_(false)
59  , latest_exception_report_("none")
60  , ev_counter_(1)
61  , board_id_(-1)
62  , sleep_on_stop_us_(0)
63 {
64  board_id_ = ps.get<int>("board_id");
65  instance_name_for_metrics_ = "BoardReader." + boost::lexical_cast<std::string>(board_id_);
66 
67  auto fragment_ids = ps.get<std::vector<artdaq::Fragment::fragment_id_t>>("fragment_ids", std::vector<artdaq::Fragment::fragment_id_t>());
68 
69  TLOG(TLVL_TRACE) << "artdaq::CommandableFragmentGenerator::CommandableFragmentGenerator(ps)";
70  int fragment_id = ps.get<int>("fragment_id", -99);
71 
72  if (fragment_id != -99)
73  {
74  if (!fragment_ids.empty())
75  {
76  latest_exception_report_ = R"(Error in CommandableFragmentGenerator: can't both define "fragment_id" and "fragment_ids" in FHiCL document)";
77  TLOG(TLVL_ERROR) << latest_exception_report_;
78  throw cet::exception(latest_exception_report_);
79  }
80 
81  fragment_ids.emplace_back(fragment_id);
82  }
83 
84  for (auto& id : fragment_ids)
85  {
86  expectedTypes_[id] = artdaq::Fragment::EmptyFragmentType;
87  }
88 
89  sleep_on_stop_us_ = ps.get<int>("sleep_on_stop_us", 0);
90 }
91 
93 {
94  joinThreads();
95 }
96 
98 {
99  should_stop_ = true;
100  TLOG(TLVL_DEBUG) << "Joining monitoringThread";
101  try
102  {
103  if (monitoringThread_.joinable())
104  {
105  monitoringThread_.join();
106  }
107  }
108  catch (...)
109  {
110  // IGNORED
111  }
112  TLOG(TLVL_DEBUG) << "joinThreads complete";
113 }
114 
116 {
117  bool result = true;
118 
119  if (check_stop()) usleep(sleep_on_stop_us_);
120  if (exception() || should_stop_) return false;
121 
122  if (!useMonitoringThread_ && monitoringInterval_ > 0)
123  {
124  TLOG(TLVL_GETNEXT) << "getNext: Checking whether to collect Monitoring Data";
125  auto now = std::chrono::steady_clock::now();
126 
127  if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
128  {
129  TLOG(TLVL_GETNEXT) << "getNext: Collecting Monitoring Data";
130  isHardwareOK_ = checkHWStatus_();
131  TLOG(TLVL_GETNEXT) << "getNext: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_;
132  lastMonitoringCall_ = now;
133  }
134  }
135 
136  try
137  {
138  std::lock_guard<std::mutex> lk(mutex_);
139  if (!isHardwareOK_)
140  {
141  TLOG(TLVL_ERROR) << "Stopping CFG because the hardware reports bad status!";
142  return false;
143  }
144  TLOG(TLVL_TRACE) << "getNext: Calling getNext_ w/ ev_counter()=" << ev_counter();
145  try
146  {
147  result = getNext_(output);
148  }
149  catch (...)
150  {
151  throw;
152  }
153  TLOG(TLVL_TRACE) << "getNext: Done with getNext_ - ev_counter() now " << ev_counter();
154  for (auto& dataIter : output)
155  {
156  TLOG(TLVL_GETNEXT_VERBOSE) << "getNext: getNext_() returned fragment with sequenceID = " << dataIter->sequenceID()
157  << ", type = " << dataIter->typeString() << ", id = " << std::to_string(dataIter->fragmentID())
158  << ", timestamp = " << dataIter->timestamp() << ", and sizeBytes = " << dataIter->sizeBytes();
159 
160  auto fragId = dataIter->fragmentID();
161  auto type = dataIter->type();
162 
163  // ELF, 2020 July 16: System Fragments are excluded from these checks
164  if (Fragment::isSystemFragmentType(type)) {
165  continue;
166  }
167 
168  if (!expectedTypes_.count(fragId))
169  {
170  TLOG(TLVL_ERROR) << "Received Fragment with Fragment ID " << fragId << ", which is not in the declared list of Fragment IDs! Aborting!";
171  return false;
172  }
173  if (expectedTypes_[fragId] == Fragment::EmptyFragmentType)
174  expectedTypes_[fragId] = type;
175  else if (expectedTypes_[fragId] != type)
176  {
177  TLOG(TLVL_WARNING) << "Received Fragment with Fragment ID " << fragId << " and type " << dataIter->typeString() << "(" << type << "), which does not match expected type for this ID (" << expectedTypes_[fragId] << ")";
178  }
179  }
180  }
181  catch (const cet::exception& e)
182  {
183  latest_exception_report_ = "cet::exception caught in getNext(): ";
184  latest_exception_report_.append(e.what());
185  TLOG(TLVL_ERROR) << "getNext: cet::exception caught: " << e;
186  set_exception(true);
187  return false;
188  }
189  catch (const boost::exception& e)
190  {
191  latest_exception_report_ = "boost::exception caught in getNext(): ";
192  latest_exception_report_.append(boost::diagnostic_information(e));
193  TLOG(TLVL_ERROR) << "getNext: boost::exception caught: " << boost::diagnostic_information(e);
194  set_exception(true);
195  return false;
196  }
197  catch (const std::exception& e)
198  {
199  latest_exception_report_ = "std::exception caught in getNext(): ";
200  latest_exception_report_.append(e.what());
201  TLOG(TLVL_ERROR) << "getNext: std::exception caught: " << e.what();
202  set_exception(true);
203  return false;
204  }
205  catch (...)
206  {
207  latest_exception_report_ = "Unknown exception caught in getNext().";
208  TLOG(TLVL_ERROR) << "getNext: unknown exception caught";
209  set_exception(true);
210  return false;
211  }
212 
213  if (!result)
214  {
215  TLOG(TLVL_DEBUG) << "getNext: Either getNext_ or applyRequests returned false, stopping";
216  }
217 
218  if (metricMan && !output.empty())
219  {
220  auto timestamp = output.front()->timestamp();
221 
222  if (output.size() > 1)
223  { // Only bother sorting if >1 entry
224  for (auto& outputfrag : output)
225  {
226  if (outputfrag->timestamp() > timestamp)
227  {
228  timestamp = outputfrag->timestamp();
229  }
230  }
231  }
232 
233  metricMan->sendMetric("Last Timestamp", timestamp, "Ticks", 1, MetricMode::LastPoint);
234  }
235 
236  return result;
237 }
238 
240 {
241  TLOG(TLVL_CHECKSTOP) << "CFG::check_stop: should_stop=" << should_stop() << ", exception status =" << int(exception());
242 
243  if (!should_stop()) return false;
244 
245  return true;
246 }
247 
249 {
250  TLOG(TLVL_EVCOUNTERINC) << "ev_counter_inc: Incrementing ev_counter from " << ev_counter() << " by " << step;
251  return ev_counter_.fetch_add(step);
252 } // returns the prev value
253 
254 void artdaq::CommandableFragmentGenerator::StartCmd(int run, uint64_t timeout, uint64_t timestamp)
255 {
256  TLOG(TLVL_TRACE) << "Start Command received.";
257  if (run < 0)
258  {
259  TLOG(TLVL_ERROR) << "negative run number";
260  throw cet::exception("CommandableFragmentGenerator") << "negative run number"; // NOLINT(cert-err60-cpp)
261  }
262 
263  timeout_ = timeout;
264  timestamp_ = timestamp;
265  ev_counter_.store(1);
266 
267  should_stop_.store(false);
268  exception_.store(false);
269  run_number_ = run;
270  subrun_number_ = 1;
271  latest_exception_report_ = "none";
272 
273  start();
274 
275  std::unique_lock<std::mutex> lk(mutex_);
276  if (useMonitoringThread_) startMonitoringThread();
277  TLOG(TLVL_TRACE) << "Start Command complete.";
278 }
279 
280 void artdaq::CommandableFragmentGenerator::StopCmd(uint64_t timeout, uint64_t timestamp)
281 {
282  TLOG(TLVL_TRACE) << "Stop Command received.";
283 
284  timeout_ = timeout;
285  timestamp_ = timestamp;
286 
287  stopNoMutex();
288  should_stop_.store(true);
289  std::unique_lock<std::mutex> lk(mutex_);
290  stop();
291 
292  joinThreads();
293  TLOG(TLVL_TRACE) << "Stop Command complete.";
294 }
295 
296 void artdaq::CommandableFragmentGenerator::PauseCmd(uint64_t timeout, uint64_t timestamp)
297 {
298  TLOG(TLVL_TRACE) << "Pause Command received.";
299  timeout_ = timeout;
300  timestamp_ = timestamp;
301 
302  pauseNoMutex();
303  should_stop_.store(true);
304  std::unique_lock<std::mutex> lk(mutex_);
305 
306  pause();
307 }
308 
309 void artdaq::CommandableFragmentGenerator::ResumeCmd(uint64_t timeout, uint64_t timestamp)
310 {
311  TLOG(TLVL_TRACE) << "Resume Command received.";
312  timeout_ = timeout;
313  timestamp_ = timestamp;
314 
315  subrun_number_ += 1;
316  should_stop_ = false;
317 
318  // no lock required: thread not started yet
319  resume();
320 
321  std::unique_lock<std::mutex> lk(mutex_);
322  //if (useDataThread_) startDataThread();
323  //if (useMonitoringThread_) startMonitoringThread();
324  TLOG(TLVL_TRACE) << "Resume Command complete.";
325 }
326 
327 std::string artdaq::CommandableFragmentGenerator::ReportCmd(std::string const& which)
328 {
329  TLOG(TLVL_TRACE) << "Report Command received.";
330  std::lock_guard<std::mutex> lk(mutex_);
331 
332  // 14-May-2015, KAB: please see the comments associated with the report()
333  // methods in the CommandableFragmentGenerator.hh file for more information
334  // on the use of those methods in this method.
335 
336  // check if the child class has something meaningful for this request
337  std::string childReport = reportSpecific(which);
338  if (childReport.length() > 0) { return childReport; }
339 
340  // handle the requests that we can take care of at this level
341  if (which == "latest_exception")
342  {
343  return latest_exception_report_;
344  }
345 
346  // check if the child class has provided a catch-all report function
347  childReport = report();
348  if (childReport.length() > 0) { return childReport; }
349 
350  // ELF: 5/31/2019: Let BoardReaderCore's report handle this...
351  /*
352  // if we haven't been able to come up with any report so far, say so
353  std::string tmpString = "The \"" + which + "\" command is not ";
354  tmpString.append("currently supported by the ");
355  tmpString.append(metricsReportingInstanceName());
356  tmpString.append(" fragment generator.");
357  */
358  TLOG(TLVL_TRACE) << "Report Command complete.";
359  return ""; //tmpString;
360 }
361 
362 // Default implemenetations of state functions
364 {
365 #pragma message "Using default implementation of CommandableFragmentGenerator::pauseNoMutex()"
366 }
367 
369 {
370 #pragma message "Using default implementation of CommandableFragmentGenerator::pause()"
371 }
372 
374 #pragma message "Using default implementation of CommandableFragmentGenerator::resume()"
375 }
376 
378 {
379 #pragma message "Using default implementation of CommandableFragmentGenerator::report()"
380  return "";
381 }
382 
383 std::string artdaq::CommandableFragmentGenerator::reportSpecific(std::string const& /*unused*/)
384 {
385 #pragma message "Using default implementation of CommandableFragmentGenerator::reportSpecific(std::string)"
386  return "";
387 }
388 
390 {
391 #pragma message "Using default implementation of CommandableFragmentGenerator::checkHWStatus_()"
392  return true;
393 }
394 
395 bool artdaq::CommandableFragmentGenerator::metaCommand(std::string const& /*unused*/, std::string const& /*unused*/)
396 {
397 #pragma message "Using default implementation of CommandableFragmentGenerator::metaCommand(std::string, std::string)"
398  return true;
399 }
400 
402 {
403  if (monitoringThread_.joinable())
404  {
405  monitoringThread_.join();
406  }
407  TLOG(TLVL_INFO) << "Starting Hardware Monitoring Thread";
408  try
409  {
410  monitoringThread_ = boost::thread(&CommandableFragmentGenerator::getMonitoringDataLoop, this);
411  }
412  catch (const boost::exception& e)
413  {
414  TLOG(TLVL_ERROR) << "Caught boost::exception starting Hardware Monitoring thread: " << boost::diagnostic_information(e) << ", errno=" << errno;
415  throw cet::exception("ThreadError") << "Caught boost::exception starting Hardware Monitoring thread: " << boost::diagnostic_information(e) << ", errno=" << errno << std::endl;
416  }
417 }
418 
420 {
421  while (!should_stop())
422  {
423  if (should_stop() || monitoringInterval_ <= 0)
424  {
425  TLOG(TLVL_DEBUG) << "getMonitoringDataLoop: should_stop() is " << std::boolalpha << should_stop()
426  << " and monitoringInterval is " << monitoringInterval_ << ", returning";
427  return;
428  }
429  TLOG(TLVL_GETMONITORINGDATA) << "getMonitoringDataLoop: Determining whether to call checkHWStatus_";
430 
431  auto now = std::chrono::steady_clock::now();
432  if (TimeUtils::GetElapsedTimeMicroseconds(lastMonitoringCall_, now) >= static_cast<size_t>(monitoringInterval_))
433  {
434  isHardwareOK_ = checkHWStatus_();
435  TLOG(TLVL_GETMONITORINGDATA) << "getMonitoringDataLoop: isHardwareOK_ is now " << std::boolalpha << isHardwareOK_;
436  lastMonitoringCall_ = now;
437  }
438  usleep(monitoringInterval_ / 10);
439  }
440 }
CommandableFragmentGenerator(const fhicl::ParameterSet &ps)
CommandableFragmentGenerator Constructor.
virtual bool checkHWStatus_()
Check any relavent hardware status registers. Return false if an error condition exists that should h...
virtual ~CommandableFragmentGenerator()
CommandableFragmentGenerator Destructor.
void getMonitoringDataLoop()
This function regularly calls checkHWStatus_(), and sets the isHardwareOK flag accordingly.
artdaq::Fragment::fragment_id_t fragment_id() const
Get the Fragment ID of this Fragment generator.
std::string ReportCmd(std::string const &which="")
Get a report about a user-specified run-time quantity.
virtual bool metaCommand(std::string const &command, std::string const &arg)
The meta-command is used for implementing user-specific commands in a CommandableFragmentGenerator.
void StopCmd(uint64_t timeout, uint64_t timestamp)
Stop the CommandableFragmentGenerator.
void StartCmd(int run, uint64_t timeout, uint64_t timestamp)
Start the CommandableFragmentGenerator.
virtual void pauseNoMutex()
On call to PauseCmd, pauseNoMutex() is called prior to PauseCmd acquiring the mutex ...
bool check_stop()
Routine used by applyRequests to make sure that all outstanding requests have been fulfilled before r...
void ResumeCmd(uint64_t timeout, uint64_t timestamp)
Resume the CommandableFragmentGenerator.
bool getNext(FragmentPtrs &output) overridefinal
getNext calls either applyRequests or getNext_ to get any data that is ready to be sent to the EventB...
void PauseCmd(uint64_t timeout, uint64_t timestamp)
Pause the CommandableFragmentGenerator.
void startMonitoringThread()
Function that launches the monitoring thread (getMonitoringDataLoop())
virtual void pause()
If a CommandableFragmentGenerator subclass is reading from hardware, the implementation of pause() sh...
virtual void resume()
The subrun number will be incremented before a call to resume.
virtual std::string report()
Let&#39;s say that the contract with the report() functions is that they return a non-empty string if the...
size_t ev_counter_inc(size_t step=1)
Increment the event counter.
virtual std::string reportSpecific(std::string const &what)
Report the status of a specific quantity
void joinThreads()
Join any data-taking threads. Should be called when destructing CommandableFragmentGenerator.