artdaq  v3_02_01
CommandableFragmentGenerator.hh
1 #ifndef artdaq_Application_CommandableFragmentGenerator_hh
2 #define artdaq_Application_CommandableFragmentGenerator_hh
3 
4 // Socket Includes
5 #include <arpa/inet.h>
6 #include <netinet/in.h>
7 #include <sys/types.h>
8 #include <sys/socket.h>
9 #include <unistd.h>
10 
11 #include <atomic>
12 #include <condition_variable>
13 #include <mutex>
14 #include <queue>
15 #include <chrono>
16 #include <array>
17 #include <list>
18 
19 #include "fhiclcpp/fwd.h"
20 #include "fhiclcpp/ParameterSet.h"
21 
22 #include "artdaq/DAQdata/Globals.hh"
23 #include "artdaq-core/Data/Fragment.hh"
24 #include "artdaq-core/Generators/FragmentGenerator.hh"
25 #include "artdaq-utilities/Plugins/MetricManager.hh"
26 #include "artdaq/DAQrate/RequestReceiver.hh"
27 
28 namespace artdaq
29 {
33  enum class RequestMode
34  {
35  Single,
36  Buffer,
37  Window,
38  Ignored
39  };
40 
83  class CommandableFragmentGenerator : public FragmentGenerator
84  {
85  public:
89  struct Config
90  {
92  fhicl::Atom<std::string> generator_type { fhicl::Name{"generator" }, fhicl::Comment{"Name of the CommandableFragmentGenerator plugin to load"} };
94  fhicl::Atom<Fragment::timestamp_t> request_window_offset { fhicl::Name{"request_window_offset" }, fhicl::Comment{"Request messages contain a timestamp. For Window request mode, start the window this far before the timestamp in the request"}, 0 };
96  fhicl::Atom<Fragment::timestamp_t> request_window_width { fhicl::Name{"request_window_width" }, fhicl::Comment{"For Window request mode, the window will be timestamp - offset to timestamp - offset + width"}, 0 };
98  fhicl::Atom<Fragment::timestamp_t> stale_request_timeout { fhicl::Name{"stale_request_timeout" }, fhicl::Comment{"How long should request messages be retained"}, 0xFFFFFFFF };
100  fhicl::Atom<Fragment::type_t> expected_fragment_type { fhicl::Name{"expected_fragment_type" }, fhicl::Comment{"The type of Fragments this CFG will be generating. \"Empty\" will auto-detect type based on Fragments generated."}, Fragment::type_t(Fragment::EmptyFragmentType) };
102  fhicl::Atom<bool> request_windows_are_unique { fhicl::Name{"request_windows_are_unique" }, fhicl::Comment{"Whether Fragments should be removed from the buffer when matched to a request window"}, true };
104  fhicl::Atom<size_t> missing_request_window_timeout_us { fhicl::Name{"missing_request_window_timeout_us"}, fhicl::Comment{"How long to track missing requests in the \"out - of - order Windows\" list"}, 5000000 };
106  fhicl::Atom<size_t> window_close_timeout_us { fhicl::Name{"window_close_timeout_us" }, fhicl::Comment{"How long to wait for the end of the data buffer to pass the end of a request window (measured from the time the request was received)"}, 2000000 };
108  fhicl::Atom<bool> separate_data_thread { fhicl::Name{"separate_data_thread" }, fhicl::Comment{"Whether data collection should proceed on its own thread. Required for all data request processing"}, false };
110  fhicl::Atom<size_t> sleep_on_no_data_us { fhicl::Name{"sleep_on_no_data_us" }, fhicl::Comment{"How long to sleep after calling getNext_ if no data is returned"}, 0 };
112  fhicl::Atom<int> data_buffer_depth_fragments { fhicl::Name{"data_buffer_depth_fragments" }, fhicl::Comment{"How many Fragments to store in the buffer"}, 1000 };
114  fhicl::Atom<size_t> data_buffer_depth_mb { fhicl::Name{"data_buffer_depth_mb" }, fhicl::Comment{"The maximum size of the data buffer in MB"}, 1000 };
116  fhicl::Atom<bool> separate_monitoring_thread { fhicl::Name{"separate_monitoring_thread" }, fhicl::Comment{"Whether a thread that calls the checkHWStatus_ method should be created"}, false };
118  fhicl::Atom<int64_t> hardware_poll_interval_us { fhicl::Name{"hardware_poll_interval_us" }, fhicl::Comment{"If a separate monitoring thread is used, how often should it call checkHWStatus_"}, 0 };
120  fhicl::Atom<int> board_id { fhicl::Name{"board_id" }, fhicl::Comment{"The identification number for this CommandableFragmentGenerator"} };
123  fhicl::Sequence<Fragment::fragment_id_t> fragment_ids { fhicl::Name("fragment_ids" ), fhicl::Comment("A list of Fragment IDs created by this CommandableFragmentGenerator") };
126  fhicl::Atom<int> fragment_id { fhicl::Name{"fragment_id" }, fhicl::Comment{"The Fragment ID created by this CommandableFragmentGenerator"}, -99 };
128  fhicl::Atom<int> sleep_on_stop_us { fhicl::Name{"sleep_on_stop_us" }, fhicl::Comment{"How long to sleep before returning when stop transition is called"}, 0 };
136  fhicl::Atom<std::string> request_mode { fhicl::Name{"request_mode" }, fhicl::Comment{"The mode by which the CommandableFragmentGenerator will process reqeusts"}, "ignored" };
137  fhicl::TableFragment<artdaq::RequestReceiver::Config> receiverConfig;
138  };
139 #if MESSAGEFACILITY_HEX_VERSION >= 0x20103
140  using Parameters = fhicl::WrappedTable<Config>;
141 #endif
142 
148 
153  explicit CommandableFragmentGenerator(const fhicl::ParameterSet& ps);
154 
161 
162 
169  void joinThreads();
170 
176  bool getNext(FragmentPtrs& output) override final;
177 
178 
184  void applyRequestsIgnoredMode(artdaq::FragmentPtrs& frags);
185 
191  void applyRequestsSingleMode(artdaq::FragmentPtrs& frags);
192 
198  void applyRequestsBufferMode(artdaq::FragmentPtrs& frags);
199 
205  void applyRequestsWindowMode(artdaq::FragmentPtrs& frags);
206 
212  bool applyRequests(FragmentPtrs& output);
213 
221  bool sendEmptyFragment(FragmentPtrs& frags, size_t sequenceId, std::string desc);
222 
229  void sendEmptyFragments(FragmentPtrs& frags, std::map<Fragment::sequence_id_t, Fragment::timestamp_t>& requests);
230 
235  void checkOutOfOrderWindows(Fragment::sequence_id_t seq);
236 
241  std::map<Fragment::sequence_id_t, std::chrono::steady_clock::time_point> getOutOfOrderWindowList() const
242  {
243  return windows_sent_ooo_;
244  }
245 
249  void startDataThread();
250 
254  void startMonitoringThread();
255 
260  void getDataLoop();
261 
266  bool waitForDataBufferReady();
267 
272  bool dataBufferIsTooLarge();
273 
277  void getDataBufferStats();
278 
284  void checkDataBuffer();
285 
289  void getMonitoringDataLoop();
290 
295  std::vector<Fragment::fragment_id_t> fragmentIDs() override
296  {
297  return fragment_ids_;
298  }
299 
304  size_t ev_counter() const { return ev_counter_.load(); }
305 
310  RequestMode request_mode() const { return mode_; }
311 
312  //
313  // State-machine related interface below.
314  //
315 
330  void StartCmd(int run, uint64_t timeout, uint64_t timestamp);
331 
341  void StopCmd(uint64_t timeout, uint64_t timestamp);
342 
352  void PauseCmd(uint64_t timeout, uint64_t timestamp);
353 
362  void ResumeCmd(uint64_t timeout, uint64_t timestamp);
363 
374  std::string ReportCmd(std::string const& which = "");
375 
380  virtual std::string metricsReportingInstanceName() const
381  {
382  return instance_name_for_metrics_;
383  }
384 
385  // The following functions are not yet implemented, and their
386  // signatures may be subject to change.
387 
388  // John F., 12/6/13 -- do we want Reset and Shutdown commands?
389  // Kurt B., 15-Feb-2014. For the moment, I suspect that we don't
390  // want a Shutdown command. FragmentGenerator instances are
391  // Constructed at Initialization time, and they are destructed
392  // at Shutdown time. So, any shutdown operations that need to be
393  // done should be put in the FragmentGenerator child class
394  // destructors. If we find that want shutdown (or initialization)
395  // operations that are different from destruction (construction),
396  // then we'll have to add InitCmd and ShutdownCmd methods.
397 
398  // virtual void ResetCmd() final {}
399  // virtual void ShutdownCmd() final {}
400 
405  bool exception() const { return exception_.load(); }
406 
407 
414  virtual bool metaCommand(std::string const& command, std::string const& arg);
415 
416  protected:
417 
418  // John F., 12/6/13 -- need to figure out which of these getter
419  // functions should be promoted to "public"
420 
421  // John F., 1/21/15 -- after more than a year, there hasn't been a
422  // single complaint that a CommandableFragmentGenerator-derived
423  // class hasn't allowed its users to access these quantities, so
424  // they're probably fine as is
425 
430  int run_number() const { return run_number_; }
435  int subrun_number() const { return subrun_number_; }
440  uint64_t timeout() const { return timeout_; }
445  uint64_t timestamp() const { return timestamp_; }
446 
451  bool should_stop() const { return should_stop_.load(); }
452 
457  bool check_stop();
458 
463  int board_id() const { return board_id_; }
464 
470  int fragment_id() const;
471 
478  size_t ev_counter_inc(size_t step = 1, bool force = false); // returns the prev value
479 
484  void set_exception(bool exception) { exception_.store(exception); }
485 
490  void metricsReportingInstanceName(std::string const& name)
491  {
492  instance_name_for_metrics_ = name;
493  }
494 
499  std::string printMode_();
500 
501  // John F., 12/10/13
502  // Is there a better way to handle mutex_ than leaving it a protected variable?
503 
504  // John F., 1/21/15
505  // Translation above is "should mutex_ be a private variable,
506  // accessible via a getter function". Probably, but at this point
507  // it's not worth breaking code by implementing this.
508 
509  std::mutex mutex_;
510 
511  private:
512  // FHiCL-configurable variables. Note that the C++ variable names
513  // are the FHiCL variable names with a "_" appended
514 
515  //Socket parameters
516  std::unique_ptr<RequestReceiver> requestReceiver_;
517 
518  RequestMode mode_;
519  Fragment::timestamp_t windowOffset_;
520  Fragment::timestamp_t windowWidth_;
521  Fragment::timestamp_t staleTimeout_;
522  Fragment::type_t expectedType_;
523  size_t maxFragmentCount_;
524  bool uniqueWindows_;
525  std::map<Fragment::sequence_id_t, std::chrono::steady_clock::time_point> windows_sent_ooo_;
526  size_t missing_request_window_timeout_us_;
527  size_t window_close_timeout_us_;
528 
529  bool useDataThread_;
530  size_t sleep_on_no_data_us_;
531  std::atomic<bool> data_thread_running_;
532  boost::thread dataThread_;
533 
534  std::condition_variable dataCondition_;
535  std::atomic<int> dataBufferDepthFragments_;
536  std::atomic<size_t> dataBufferDepthBytes_;
537  int maxDataBufferDepthFragments_;
538  size_t maxDataBufferDepthBytes_;
539 
540  bool useMonitoringThread_;
541  boost::thread monitoringThread_;
542  int64_t monitoringInterval_; // Microseconds
543  std::chrono::steady_clock::time_point lastMonitoringCall_;
544  bool isHardwareOK_;
545 
546  FragmentPtrs dataBuffer_;
547  FragmentPtrs newDataBuffer_;
548  std::mutex dataBufferMutex_;
549 
550  std::vector<artdaq::Fragment::fragment_id_t> fragment_ids_;
551 
552  // In order to support the state-machine related behavior, all
553  // CommandableFragmentGenerators must be able to remember a run number and a
554  // subrun number.
555  int run_number_, subrun_number_;
556 
557  // JCF, 8/28/14
558 
559  // Provide a user-adjustable timeout for the start transition
560  uint64_t timeout_;
561 
562  // JCF, 8/21/14
563 
564  // In response to a need to synchronize various components using
565  // different fragment generators in an experiment, keep a record
566  // of a timestamp (see Redmine Issue #6783 for more)
567 
568  uint64_t timestamp_;
569 
570  std::atomic<bool> should_stop_, exception_, force_stop_;
571  std::string latest_exception_report_;
572  std::atomic<size_t> ev_counter_;
573 
574  int board_id_;
575  std::string instance_name_for_metrics_;
576 
577  // Depending on what sleep_on_stop_us_ is set to, this gives the
578  // stopping thread the chance to gather the required lock
579 
580  int sleep_on_stop_us_;
581 
582  protected:
583 
591  virtual bool getNext_(FragmentPtrs& output) = 0;
592 
593 
600  virtual bool checkHWStatus_();
601 
602  //
603  // State-machine related implementor interface below.
604  //
605 
616  virtual void start() = 0;
617 
624  virtual void stopNoMutex() = 0;
625 
634  virtual void stop() = 0;
635 
640  virtual void pauseNoMutex();
641 
647  virtual void pause();
648 
652  virtual void resume();
653 
665  virtual std::string report();
666 
672  virtual std::string reportSpecific(std::string const& what);
673 
674  };
675 }
676 
677 #endif /* artdaq_Application_CommandableFragmentGenerator_hh */
fhicl::Atom< Fragment::type_t > expected_fragment_type
&quot;expected_fragment_type&quot; (Default: 231, EmptyFragmentType) : The type of Fragments this CFG will be g...
int fragment_id() const
Get the current Fragment ID, if there is only one.
fhicl::Atom< int > board_id
&quot;board_id&quot; (REQUIRED) : The identification number for this CommandableFragmentGenerator ...
void applyRequestsSingleMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Single. Precondition: dataBufferMutex_ and reques...
int subrun_number() const
Get the current Subrun number.
virtual void start()=0
If a CommandableFragmentGenerator subclass is reading from a file, and start() is called...
fhicl::Atom< int64_t > hardware_poll_interval_us
&quot;hardware_poll_interval_us&quot; (Default: 0) : If a separate monitoring thread is used, how often should it call checkHWStatus_
virtual bool checkHWStatus_()
Check any relavent hardware status registers. Return false if an error condition exists that should h...
virtual ~CommandableFragmentGenerator()
CommandableFragmentGenerator Destructor.
RequestMode
The RequestMode enumeration contains the possible ways which CommandableFragmentGenerator responds to...
void applyRequestsBufferMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Buffer. Precondition: dataBufferMutex_ and reques...
bool exception() const
Get the current value of the exception flag.
void metricsReportingInstanceName(std::string const &name)
Sets the name for metrics reporting.
std::mutex mutex_
Mutex used to ensure that multiple transition commands do not run at the same time.
bool sendEmptyFragment(FragmentPtrs &frags, size_t sequenceId, std::string desc)
Send an EmptyFragmentType Fragment.
void getMonitoringDataLoop()
This function regularly calls checkHWStatus_(), and sets the isHardwareOK flag accordingly.
void startDataThread()
Function that launches the data thread (getDataLoop())
fhicl::TableFragment< artdaq::RequestReceiver::Config > receiverConfig
Configuration for the Request Receiver. See artdaq::RequestReceiver::Config.
std::string ReportCmd(std::string const &which="")
Get a report about a user-specified run-time quantity.
virtual bool metaCommand(std::string const &command, std::string const &arg)
The meta-command is used for implementing user-specific commands in a CommandableFragmentGenerator.
fhicl::Atom< Fragment::timestamp_t > stale_request_timeout
&quot;stale_request_timeout&quot; (Default: -1) : How long should request messages be retained ...
bool dataBufferIsTooLarge()
Test the configured constraints on the data buffer.
void StopCmd(uint64_t timeout, uint64_t timestamp)
Stop the CommandableFragmentGenerator.
uint64_t timeout() const
Timeout of last command.
void applyRequestsWindowMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Window. Precondition: dataBufferMutex_ and reques...
void StartCmd(int run, uint64_t timeout, uint64_t timestamp)
Start the CommandableFragmentGenerator.
virtual void pauseNoMutex()
On call to PauseCmd, pauseNoMutex() is called prior to PauseCmd acquiring the mutex ...
bool check_stop()
Routine used by applyRequests to make sure that all outstanding requests have been fulfilled before r...
void ResumeCmd(uint64_t timeout, uint64_t timestamp)
Resume the CommandableFragmentGenerator.
CommandableFragmentGenerator()
CommandableFragmentGenerator default constructor.
bool getNext(FragmentPtrs &output) overridefinal
getNext calls either applyRequests or getNext_ to get any data that is ready to be sent to the EventB...
bool waitForDataBufferReady()
Wait for the data buffer to drain (dataBufferIsTooLarge returns false), periodically reporting status...
size_t ev_counter_inc(size_t step=1, bool force=false)
Increment the event counter, if the current RequestMode allows it.
fhicl::Atom< int > data_buffer_depth_fragments
&quot;data_buffer_depth_fragments&quot; (Default: 1000) : How many Fragments to store in the buffer ...
fhicl::Atom< size_t > data_buffer_depth_mb
&quot;data_buffer_depth_mb&quot; (Default: 1000) : The maximum size of the data buffer in MB ...
std::vector< Fragment::fragment_id_t > fragmentIDs() override
Get the list of Fragment IDs handled by this CommandableFragmentGenerator.
fhicl::Atom< int > sleep_on_stop_us
&quot;sleep_on_stop_us&quot; (Default: 0) : How long to sleep before returning when stop transition is called ...
void applyRequestsIgnoredMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Ignored. Precondition: dataBufferMutex_ and reque...
fhicl::Atom< size_t > missing_request_window_timeout_us
&quot;missing_request_window_timeout_us&quot; (Default: 5000000) : How long to track missing requests in the &quot;o...
void PauseCmd(uint64_t timeout, uint64_t timestamp)
Pause the CommandableFragmentGenerator.
void getDataLoop()
When separate_data_thread is set to true, this loop repeatedly calls getNext_ and adds returned Fragm...
fhicl::Sequence< Fragment::fragment_id_t > fragment_ids
virtual std::string metricsReportingInstanceName() const
Get the name used when reporting metrics.
void sendEmptyFragments(FragmentPtrs &frags, std::map< Fragment::sequence_id_t, Fragment::timestamp_t > &requests)
This function is for Buffered and Single request modes, as they can only respond to one data request ...
bool should_stop() const
Get the current value of the should_stop flag.
CommandableFragmentGenerator is a FragmentGenerator-derived abstract class that defines the interface...
virtual bool getNext_(FragmentPtrs &output)=0
Obtain the next group of Fragments, if any are available. Return false if readout cannot continue...
fhicl::Atom< std::string > request_mode
&quot;request_mode&quot; (Deafult: Ignored) : The mode by which the CommandableFragmentGenerator will process r...
void startMonitoringThread()
Function that launches the monitoring thread (getMonitoringDataLoop())
fhicl::Atom< Fragment::timestamp_t > request_window_offset
&quot;request_window_offset&quot; (Default: 0) : Request messages contain a timestamp.For Window request mode...
virtual void pause()
If a CommandableFragmentGenerator subclass is reading from hardware, the implementation of pause() sh...
virtual void resume()
The subrun number will be incremented before a call to resume.
size_t ev_counter() const
Get the current value of the event counter.
void checkDataBuffer()
Perform data buffer pruning operations. If the RequestMode is Single, removes all but the latest Frag...
fhicl::Atom< bool > request_windows_are_unique
&quot;request_windows_are_unique&quot; (Default: true) : Whether Fragments should be removed from the buffer wh...
std::map< Fragment::sequence_id_t, std::chrono::steady_clock::time_point > getOutOfOrderWindowList() const
Access the windows_sent_ooo_ map.
fhicl::Atom< size_t > window_close_timeout_us
&quot;window_close_timeout_us&quot; (Default: 2000000) : How long to wait for the end of the data buffer to pas...
fhicl::Atom< size_t > sleep_on_no_data_us
&quot;sleep_on_no_data_us&quot; (Default: 0 (no sleep)) : How long to sleep after calling getNext_ if no data i...
fhicl::Atom< Fragment::timestamp_t > request_window_width
&quot;request_window_width&quot; (Default: 0) : For Window request mode, the window will be timestamp - offset ...
virtual std::string report()
Let&#39;s say that the contract with the report() functions is that they return a non-empty string if the...
virtual void stopNoMutex()=0
On call to StopCmd, stopNoMutex() is called prior to StopCmd acquiring the mutex
fhicl::Atom< bool > separate_data_thread
&quot;separate_data_thread&quot; (Default: false) : Whether data collection should proceed on its own thread...
int board_id() const
Gets the current board_id.
std::string printMode_()
Return the string representation of the current RequestMode.
void set_exception(bool exception)
Control the exception flag.
void getDataBufferStats()
Calculate the size of the dataBuffer and report appropriate metrics.
RequestMode request_mode() const
Get the current request mode of the CommandableFragmentGenerator
uint64_t timestamp() const
Timestamp of last command.
Configuration of the CommandableFragmentGenerator. May be used for parameter validation ...
fhicl::Atom< std::string > generator_type
&quot;generator&quot; (REQUIRED) Name of the CommandableFragmentGenerator plugin to load
void checkOutOfOrderWindows(Fragment::sequence_id_t seq)
Check the windows_sent_ooo_ map for sequence IDs that may be removed.
virtual void stop()=0
If a CommandableFragmentGenerator subclass is reading from a file, calling stop() should arrange that...
virtual std::string reportSpecific(std::string const &what)
Report the status of a specific quantity
bool applyRequests(FragmentPtrs &output)
See if any requests have been received, and add the corresponding data Fragment objects to the output...
int run_number() const
Get the current Run number.
fhicl::Atom< bool > separate_monitoring_thread
&quot;separate_monitoring_thread&quot; (Default: false) : Whether a thread that calls the checkHWStatus_ method...
void joinThreads()
Join any data-taking threads. Should be called when destructing CommandableFragmentGenerator.