artdaq  v3_02_00
CommandableFragmentGenerator.hh
1 #ifndef artdaq_Application_CommandableFragmentGenerator_hh
2 #define artdaq_Application_CommandableFragmentGenerator_hh
3 
4 // Socket Includes
5 #include <arpa/inet.h>
6 #include <netinet/in.h>
7 #include <sys/types.h>
8 #include <sys/socket.h>
9 #include <unistd.h>
10 
11 #include <atomic>
12 #include <condition_variable>
13 #include <mutex>
14 #include <queue>
15 #include <chrono>
16 #include <array>
17 #include <list>
18 
19 #include "fhiclcpp/fwd.h"
20 #include "fhiclcpp/ParameterSet.h"
21 
22 #include "artdaq/DAQdata/Globals.hh"
23 #include "artdaq-core/Data/Fragment.hh"
24 #include "artdaq-core/Generators/FragmentGenerator.hh"
25 #include "artdaq-utilities/Plugins/MetricManager.hh"
26 #include "artdaq/DAQrate/RequestReceiver.hh"
27 
28 namespace artdaq
29 {
33  enum class RequestMode
34  {
35  Single,
36  Buffer,
37  Window,
38  Ignored
39  };
40 
83  class CommandableFragmentGenerator : public FragmentGenerator
84  {
85  public:
86 
87  struct Config
88  {
89  fhicl::Atom<std::string> generator_type { fhicl::Name{"generator" }, fhicl::Comment{"Name of the CommandableFragmentGenerator plugin to load"} };
90  fhicl::Atom<Fragment::timestamp_t> request_window_offset { fhicl::Name{"request_window_offset" }, fhicl::Comment{"Request messages contain a timestamp. For Window request mode, start the window this far before the timestamp in the request"}, 0 };
91  fhicl::Atom<Fragment::timestamp_t> request_window_width { fhicl::Name{"request_window_width" }, fhicl::Comment{"For Window request mode, the window will be timestamp - offset to timestamp - offset + width"}, 0 };
92  fhicl::Atom<Fragment::timestamp_t> stale_request_timeout { fhicl::Name{"stale_request_timeout" }, fhicl::Comment{"How long should request messages be retained"}, 0xFFFFFFFF };
93  fhicl::Atom<Fragment::type_t> expected_fragment_type { fhicl::Name{"expected_fragment_type" }, fhicl::Comment{"The type of Fragments this CFG will be generating. \"Empty\" will auto-detect type based on Fragments generated."}, Fragment::type_t(Fragment::EmptyFragmentType) };
94  fhicl::Atom<bool> request_windows_are_unique { fhicl::Name{"request_windows_are_unique" }, fhicl::Comment{"Whether Fragments should be removed from the buffer when matched to a request window"}, true };
95  fhicl::Atom<size_t> missing_request_window_timeout_us { fhicl::Name{"missing_request_window_timeout_us"}, fhicl::Comment{"How long to wait for a missing request in Window mode (measured from the last time data was sent)"}, 1000000 };
96  fhicl::Atom<size_t> window_close_timeout_us { fhicl::Name{"window_close_timeout_us" }, fhicl::Comment{"How long to wait for the end of the data buffer to pass the end of a request window (measured from the last time data was sent)"}, 2000000 };
97  fhicl::Atom<bool> separate_data_thread { fhicl::Name{"separate_data_thread" }, fhicl::Comment{"Whether data collection should proceed on its own thread. Required for all data request processing"}, false };
98  fhicl::Atom<size_t> sleep_on_no_data_us { fhicl::Name{"sleep_on_no_data_us" }, fhicl::Comment{"How long to sleep after calling getNext_ if no data is returned"}, 0 };
99  fhicl::Atom<int> data_buffer_depth_fragments { fhicl::Name{"data_buffer_depth_fragments" }, fhicl::Comment{"How many Fragments to store in the buffer"}, 1000 };
100  fhicl::Atom<size_t> data_buffer_depth_mb { fhicl::Name{"data_buffer_depth_mb" }, fhicl::Comment{"The maximum size of the data buffer in MB"}, 1000 };
101  fhicl::Atom<bool> separate_monitoring_thread { fhicl::Name{"separate_monitoring_thread" }, fhicl::Comment{"Whether a thread that calls the checkHWStatus_ method should be created"}, false };
102  fhicl::Atom<int64_t> hardware_poll_interval_us { fhicl::Name{"hardware_poll_interval_us" }, fhicl::Comment{"If a separate monitoring thread is used, how often should it call checkHWStatus_"}, 0 };
103  fhicl::Atom<int> board_id { fhicl::Name{"board_id" }, fhicl::Comment{"The identification number for this CommandableFragmentGenerator"} };
104  fhicl::Sequence<Fragment::fragment_id_t> fragment_ids { fhicl::Name("fragment_ids" ), fhicl::Comment("A list of Fragment IDs created by this CommandableFragmentGenerator") };
105  fhicl::Atom<int> fragment_id { fhicl::Name{"fragment_id" }, fhicl::Comment{"The Fragment ID created by this CommandableFragmentGenerator"}, -99 };
106  fhicl::Atom<int> sleep_on_stop_us { fhicl::Name{"sleep_on_stop_us" }, fhicl::Comment{"How long to sleep before returning when stop transition is called"}, 0 };
107  fhicl::Atom<std::string> request_mode { fhicl::Name{"request_mode" }, fhicl::Comment{"The mode by which the CommandableFragmentGenerator will process reqeusts"}, "ignored" };
108  fhicl::TableFragment<artdaq::RequestReceiver::Config> receiverConfig;
109  };
110 #if MESSAGEFACILITY_HEX_VERSION >= 0x20103
111  using Parameters = fhicl::WrappedTable<Config>;
112 #endif
113 
119 
154  explicit CommandableFragmentGenerator(const fhicl::ParameterSet& ps);
155 
162 
163 
170  void joinThreads();
171 
177  bool getNext(FragmentPtrs& output) override final;
178 
179 
185  void applyRequestsIgnoredMode(artdaq::FragmentPtrs& frags);
186 
192  void applyRequestsSingleMode(artdaq::FragmentPtrs& frags);
193 
199  void applyRequestsBufferMode(artdaq::FragmentPtrs& frags);
200 
206  void applyRequestsWindowMode(artdaq::FragmentPtrs& frags);
207 
213  bool applyRequests(FragmentPtrs& output);
214 
222  bool sendEmptyFragment(FragmentPtrs& frags, size_t sequenceId, std::string desc);
223 
230  void sendEmptyFragments(FragmentPtrs& frags, std::map<Fragment::sequence_id_t, Fragment::timestamp_t>& requests);
231 
235  void startDataThread();
236 
240  void startMonitoringThread();
241 
246  void getDataLoop();
247 
252  bool waitForDataBufferReady();
253 
258  bool dataBufferIsTooLarge();
259 
263  void getDataBufferStats();
264 
270  void checkDataBuffer();
271 
275  void getMonitoringDataLoop();
276 
281  std::vector<Fragment::fragment_id_t> fragmentIDs() override
282  {
283  return fragment_ids_;
284  }
285 
286  //
287  // State-machine related interface below.
288  //
289 
304  void StartCmd(int run, uint64_t timeout, uint64_t timestamp);
305 
315  void StopCmd(uint64_t timeout, uint64_t timestamp);
316 
326  void PauseCmd(uint64_t timeout, uint64_t timestamp);
327 
336  void ResumeCmd(uint64_t timeout, uint64_t timestamp);
337 
348  std::string ReportCmd(std::string const& which = "");
349 
354  virtual std::string metricsReportingInstanceName() const
355  {
356  return instance_name_for_metrics_;
357  }
358 
359  // The following functions are not yet implemented, and their
360  // signatures may be subject to change.
361 
362  // John F., 12/6/13 -- do we want Reset and Shutdown commands?
363  // Kurt B., 15-Feb-2014. For the moment, I suspect that we don't
364  // want a Shutdown command. FragmentGenerator instances are
365  // Constructed at Initialization time, and they are destructed
366  // at Shutdown time. So, any shutdown operations that need to be
367  // done should be put in the FragmentGenerator child class
368  // destructors. If we find that want shutdown (or initialization)
369  // operations that are different from destruction (construction),
370  // then we'll have to add InitCmd and ShutdownCmd methods.
371 
372  // virtual void ResetCmd() final {}
373  // virtual void ShutdownCmd() final {}
374 
379  bool exception() const { return exception_.load(); }
380 
381 
388  virtual bool metaCommand(std::string const& command, std::string const& arg);
389 
390  protected:
391 
392  // John F., 12/6/13 -- need to figure out which of these getter
393  // functions should be promoted to "public"
394 
395  // John F., 1/21/15 -- after more than a year, there hasn't been a
396  // single complaint that a CommandableFragmentGenerator-derived
397  // class hasn't allowed its users to access these quantities, so
398  // they're probably fine as is
399 
404  int run_number() const { return run_number_; }
409  int subrun_number() const { return subrun_number_; }
414  uint64_t timeout() const { return timeout_; }
419  uint64_t timestamp() const { return timestamp_; }
420 
425  bool should_stop() const { return should_stop_.load(); }
426 
431  bool check_stop();
432 
437  int board_id() const { return board_id_; }
438 
444  int fragment_id() const;
445 
450  size_t ev_counter() const { return ev_counter_.load(); }
451 
458  size_t ev_counter_inc(size_t step = 1, bool force = false); // returns the prev value
459 
464  void set_exception(bool exception) { exception_.store(exception); }
465 
470  void metricsReportingInstanceName(std::string const& name)
471  {
472  instance_name_for_metrics_ = name;
473  }
474 
479  std::string printMode_();
480 
481  // John F., 12/10/13
482  // Is there a better way to handle mutex_ than leaving it a protected variable?
483 
484  // John F., 1/21/15
485  // Translation above is "should mutex_ be a private variable,
486  // accessible via a getter function". Probably, but at this point
487  // it's not worth breaking code by implementing this.
488 
489  std::mutex mutex_;
490 
491  private:
492  // FHiCL-configurable variables. Note that the C++ variable names
493  // are the FHiCL variable names with a "_" appended
494 
495  //Socket parameters
496  std::unique_ptr<RequestReceiver> requestReceiver_;
497 
498  RequestMode mode_;
499  Fragment::timestamp_t windowOffset_;
500  Fragment::timestamp_t windowWidth_;
501  Fragment::timestamp_t staleTimeout_;
502  Fragment::type_t expectedType_;
503  size_t maxFragmentCount_;
504  bool uniqueWindows_;
505  bool missing_request_;
506  std::chrono::steady_clock::time_point missing_request_time_;
507  std::chrono::steady_clock::time_point last_window_send_time_;
508  bool last_window_send_time_set_;
509  std::set<Fragment::sequence_id_t> windows_sent_ooo_;
510  size_t missing_request_window_timeout_us_;
511  size_t window_close_timeout_us_;
512 
513  bool useDataThread_;
514  size_t sleep_on_no_data_us_;
515  std::atomic<bool> data_thread_running_;
516  boost::thread dataThread_;
517 
518  std::condition_variable dataCondition_;
519  std::atomic<int> dataBufferDepthFragments_;
520  std::atomic<size_t> dataBufferDepthBytes_;
521  int maxDataBufferDepthFragments_;
522  size_t maxDataBufferDepthBytes_;
523 
524  bool useMonitoringThread_;
525  boost::thread monitoringThread_;
526  int64_t monitoringInterval_; // Microseconds
527  std::chrono::steady_clock::time_point lastMonitoringCall_;
528  bool isHardwareOK_;
529 
530  FragmentPtrs dataBuffer_;
531  FragmentPtrs newDataBuffer_;
532  std::mutex dataBufferMutex_;
533 
534  std::vector<artdaq::Fragment::fragment_id_t> fragment_ids_;
535 
536  // In order to support the state-machine related behavior, all
537  // CommandableFragmentGenerators must be able to remember a run number and a
538  // subrun number.
539  int run_number_, subrun_number_;
540 
541  // JCF, 8/28/14
542 
543  // Provide a user-adjustable timeout for the start transition
544  uint64_t timeout_;
545 
546  // JCF, 8/21/14
547 
548  // In response to a need to synchronize various components using
549  // different fragment generators in an experiment, keep a record
550  // of a timestamp (see Redmine Issue #6783 for more)
551 
552  uint64_t timestamp_;
553 
554  std::atomic<bool> should_stop_, exception_, force_stop_;
555  std::string latest_exception_report_;
556  std::atomic<size_t> ev_counter_;
557 
558  int board_id_;
559  std::string instance_name_for_metrics_;
560 
561  // Depending on what sleep_on_stop_us_ is set to, this gives the
562  // stopping thread the chance to gather the required lock
563 
564  int sleep_on_stop_us_;
565 
566  protected:
567 
568  // Obtain the next group of Fragments, if any are available. Return
569  // false if no more data are available, if we are 'stopped', or if
570  // we are not running in state-machine mode. Note that getNext_()
571  // must return n of each fragmentID declared by fragmentIDs_().
572  virtual bool getNext_(FragmentPtrs& output) = 0;
573 
574 
575  // Check any relavent hardware status registers. Return false if
576  // an error condition exists that should halt data-taking.
577  // This function should probably make MetricManager calls.
578  virtual bool checkHWStatus_();
579 
580  //
581  // State-machine related implementor interface below.
582  //
583 
584  // If a CommandableFragmentGenerator subclass is reading from a
585  // file, and start() is called, any run-, subrun-, and
586  // event-numbers in the data read from the file must be
587  // over-written by the specified run number, etc. After a call to
588  // StartCmd(), and until a call to StopCmd(), getNext_() is
589  // expected to return true as long as datataking is intended.
590  virtual void start() = 0;
591 
592  // On call to StopCmd, stopNoMutex() is called prior to StopCmd
593  // acquiring the mutex
594  virtual void stopNoMutex() = 0;
595 
596  // If a CommandableFragmentGenerator subclass is reading from a file, calling
597  // stop() should arrange that the next call to getNext_() returns
598  // false, rather than allowing getNext_() to read to the end of the
599  // file.
600  virtual void stop() = 0;
601 
602  // On call to PauseCmd, pauseNoMutex() is called prior to PauseCmd
603  // acquiring the mutex
604  virtual void pauseNoMutex();
605 
606  // If a CommandableFragmentGenerator subclass is reading from hardware, the
607  // implementation of pause() should tell the hardware to stop
608  // sending data.
609  virtual void pause();
610 
611  // The subrun number will be incremented *before* a call to
612  // resume. Subclasses are responsible for assuring that, after a
613  // call to resume, that getNext_() will return Fragments marked with
614  // the correct subrun number (and run number).
615  virtual void resume();
616 
617  // Let's say that the contract with the report() functions is that they
618  // return a non-empty string if they have something useful to report,
619  // but if they don't know how to handle a given request, they simply
620  // return an empty string and the ReportCmd() takes care of saying
621  // "the xyz command is not currently supported".
622  // For backward compatibility, we keep the report function that takes
623  // no arguments and add one that takes a "which" argument. In the
624  // ReportCmd function, we'll call the more specific one first.
625  virtual std::string report();
626 
627  virtual std::string reportSpecific(std::string const&);
628 
629  };
630 }
631 
632 #endif /* artdaq_Application_CommandableFragmentGenerator_hh */
int fragment_id() const
Get the current Fragment ID, if there is only one.
void applyRequestsSingleMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Single. Precondition: dataBufferMutex_ and reques...
int subrun_number() const
Get the current Subrun number.
virtual ~CommandableFragmentGenerator()
CommandableFragmentGenerator Destructor.
RequestMode
The RequestMode enumeration contains the possible ways which CommandableFragmentGenerator responds to...
void applyRequestsBufferMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Buffer. Precondition: dataBufferMutex_ and reques...
bool exception() const
Get the current value of the exception flag.
void metricsReportingInstanceName(std::string const &name)
Sets the name for metrics reporting.
std::mutex mutex_
Mutex used to ensure that multiple transition commands do not run at the same time.
bool sendEmptyFragment(FragmentPtrs &frags, size_t sequenceId, std::string desc)
Send an EmptyFragmentType Fragment.
void getMonitoringDataLoop()
This function regularly calls checkHWStatus_(), and sets the isHardwareOK flag accordingly.
void startDataThread()
Function that launches the data thread (getDataLoop())
std::string ReportCmd(std::string const &which="")
Get a report about a user-specified run-time quantity.
virtual bool metaCommand(std::string const &command, std::string const &arg)
The meta-command is used for implementing user-specific commands in a CommandableFragmentGenerator.
bool dataBufferIsTooLarge()
Test the configured constraints on the data buffer.
void StopCmd(uint64_t timeout, uint64_t timestamp)
Stop the CommandableFragmentGenerator.
uint64_t timeout() const
Timeout of last command.
void applyRequestsWindowMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Window. Precondition: dataBufferMutex_ and reques...
void StartCmd(int run, uint64_t timeout, uint64_t timestamp)
Start the CommandableFragmentGenerator.
bool check_stop()
Routine used by applyRequests to make sure that all outstanding requests have been fulfilled before r...
void ResumeCmd(uint64_t timeout, uint64_t timestamp)
Resume the CommandableFragmentGenerator.
CommandableFragmentGenerator()
CommandableFragmentGenerator default constructor.
bool getNext(FragmentPtrs &output) overridefinal
getNext calls either applyRequests or getNext_ to get any data that is ready to be sent to the EventB...
bool waitForDataBufferReady()
Wait for the data buffer to drain (dataBufferIsTooLarge returns false), periodically reporting status...
size_t ev_counter_inc(size_t step=1, bool force=false)
Increment the event counter, if the current RequestMode allows it.
std::vector< Fragment::fragment_id_t > fragmentIDs() override
Get the list of Fragment IDs handled by this CommandableFragmentGenerator.
void applyRequestsIgnoredMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Ignored. Precondition: dataBufferMutex_ and reque...
void PauseCmd(uint64_t timeout, uint64_t timestamp)
Pause the CommandableFragmentGenerator.
void getDataLoop()
When separate_data_thread is set to true, this loop repeatedly calls getNext_ and adds returned Fragm...
virtual std::string metricsReportingInstanceName() const
Get the name used when reporting metrics.
void sendEmptyFragments(FragmentPtrs &frags, std::map< Fragment::sequence_id_t, Fragment::timestamp_t > &requests)
This function is for Buffered and Single request modes, as they can only respond to one data request ...
bool should_stop() const
Get the current value of the should_stop flag.
CommandableFragmentGenerator is a FragmentGenerator-derived abstract class that defines the interface...
void startMonitoringThread()
Function that launches the monitoring thread (getMonitoringDataLoop())
size_t ev_counter() const
Get the current value of the event counter.
void checkDataBuffer()
Perform data buffer pruning operations. If the RequestMode is Single, removes all but the latest Frag...
int board_id() const
Gets the current board_id.
std::string printMode_()
Return the string representation of the current RequestMode.
void set_exception(bool exception)
Control the exception flag.
void getDataBufferStats()
Calculate the size of the dataBuffer and report appropriate metrics.
uint64_t timestamp() const
Timestamp of last command.
bool applyRequests(FragmentPtrs &output)
See if any requests have been received, and add the corresponding data Fragment objects to the output...
int run_number() const
Get the current Run number.
void joinThreads()
Join any data-taking threads. Should be called when destructing CommandableFragmentGenerator.