artdaq  v3_05_00
CommandableFragmentGenerator.hh
1 #ifndef artdaq_Application_CommandableFragmentGenerator_hh
2 #define artdaq_Application_CommandableFragmentGenerator_hh
3 
4 // Socket Includes
5 #include <arpa/inet.h>
6 #include <netinet/in.h>
7 #include <sys/types.h>
8 #include <sys/socket.h>
9 #include <unistd.h>
10 
11 #include <atomic>
12 #include <condition_variable>
13 #include <mutex>
14 #include <queue>
15 #include <chrono>
16 #include <array>
17 #include <list>
18 
19 #include "fhiclcpp/fwd.h"
20 #include "fhiclcpp/ParameterSet.h"
21 
22 #include "artdaq/DAQdata/Globals.hh"
23 #include "artdaq-core/Data/Fragment.hh"
24 #include "artdaq-core/Generators/FragmentGenerator.hh"
25 #include "artdaq-utilities/Plugins/MetricManager.hh"
26 #include "artdaq/DAQrate/RequestReceiver.hh"
27 
28 namespace artdaq
29 {
33  enum class RequestMode
34  {
35  Single,
36  Buffer,
37  Window,
38  Ignored
39  };
40 
83  class CommandableFragmentGenerator : public FragmentGenerator
84  {
85  public:
89  struct Config
90  {
92  fhicl::Atom<std::string> generator_type { fhicl::Name{"generator" }, fhicl::Comment{"Name of the CommandableFragmentGenerator plugin to load"} };
94  fhicl::Atom<Fragment::timestamp_t> request_window_offset { fhicl::Name{"request_window_offset" }, fhicl::Comment{"Request messages contain a timestamp. For Window request mode, start the window this far before the timestamp in the request"}, 0 };
96  fhicl::Atom<Fragment::timestamp_t> request_window_width { fhicl::Name{"request_window_width" }, fhicl::Comment{"For Window request mode, the window will be timestamp - offset to timestamp - offset + width"}, 0 };
98  fhicl::Atom<Fragment::timestamp_t> stale_request_timeout { fhicl::Name{"stale_request_timeout" }, fhicl::Comment{"Fragments stored in the fragment generator which are older than the newest stored fragment by at least stale_request_timeout units of request timestamp ticks will get discarded"}, 0xFFFFFFFF };
100  fhicl::Atom<Fragment::type_t> expected_fragment_type { fhicl::Name{"expected_fragment_type" }, fhicl::Comment{"The type of Fragments this CFG will be generating. \"Empty\" will auto-detect type based on Fragments generated."}, Fragment::type_t(Fragment::EmptyFragmentType) };
102  fhicl::Atom<bool> request_windows_are_unique { fhicl::Name{"request_windows_are_unique" }, fhicl::Comment{"Whether Fragments should be removed from the buffer when matched to a request window"}, true };
104  fhicl::Atom<size_t> missing_request_window_timeout_us { fhicl::Name{"missing_request_window_timeout_us"}, fhicl::Comment{"How long to track missing requests in the \"out - of - order Windows\" list"}, 5000000 };
106  fhicl::Atom<size_t> window_close_timeout_us { fhicl::Name{"window_close_timeout_us" }, fhicl::Comment{"How long to wait for the end of the data buffer to pass the end of a request window (measured from the time the request was received)"}, 2000000 };
108  fhicl::Atom<bool> separate_data_thread { fhicl::Name{"separate_data_thread" }, fhicl::Comment{"Whether data collection should proceed on its own thread. Required for all data request processing"}, false };
110  fhicl::Atom<bool> circular_buffer_mode { fhicl::Name{"circular_buffer_mode" }, fhicl::Comment{"Whether the data buffer should be treated as a circular buffer on the input side (i.e. old fragments are automatically discarded when the buffer is full to always call getNext_)."}, false };
112  fhicl::Atom<size_t> sleep_on_no_data_us { fhicl::Name{"sleep_on_no_data_us" }, fhicl::Comment{"How long to sleep after calling getNext_ if no data is returned"}, 0 };
114  fhicl::Atom<int> data_buffer_depth_fragments { fhicl::Name{"data_buffer_depth_fragments" }, fhicl::Comment{"The max fragments which can be stored before dropping occurs"}, 1000 };
116  fhicl::Atom<size_t> data_buffer_depth_mb { fhicl::Name{"data_buffer_depth_mb" }, fhicl::Comment{"The max cumulative size in megabytes of the fragments which can be stored before dropping occurs"}, 1000 };
118  fhicl::Atom<bool> separate_monitoring_thread { fhicl::Name{"separate_monitoring_thread" }, fhicl::Comment{"Whether a thread that calls the checkHWStatus_ method should be created"}, false };
120  fhicl::Atom<int64_t> hardware_poll_interval_us { fhicl::Name{"hardware_poll_interval_us" }, fhicl::Comment{"If a separate monitoring thread is used, how often should it call checkHWStatus_"}, 0 };
122  fhicl::Atom<int> board_id { fhicl::Name{"board_id" }, fhicl::Comment{"The identification number for this CommandableFragmentGenerator"} };
125  fhicl::Sequence<Fragment::fragment_id_t> fragment_ids { fhicl::Name("fragment_ids" ), fhicl::Comment("A list of Fragment IDs created by this CommandableFragmentGenerator") };
128  fhicl::Atom<int> fragment_id { fhicl::Name{"fragment_id" }, fhicl::Comment{"The Fragment ID created by this CommandableFragmentGenerator"}, -99 };
130  fhicl::Atom<int> sleep_on_stop_us { fhicl::Name{"sleep_on_stop_us" }, fhicl::Comment{"How long to sleep before returning when stop transition is called"}, 0 };
138  fhicl::Atom<std::string> request_mode { fhicl::Name{"request_mode" }, fhicl::Comment{"The mode by which the CommandableFragmentGenerator will process reqeusts"}, "ignored" };
139  fhicl::TableFragment<artdaq::RequestReceiver::Config> receiverConfig;
140  };
142  using Parameters = fhicl::WrappedTable<Config>;
143 
150 
155  explicit CommandableFragmentGenerator(const fhicl::ParameterSet& ps);
156 
163 
164 
171  void joinThreads();
172 
178  bool getNext(FragmentPtrs& output) override final;
179 
180 
186  void applyRequestsIgnoredMode(artdaq::FragmentPtrs& frags);
187 
193  void applyRequestsSingleMode(artdaq::FragmentPtrs& frags);
194 
200  void applyRequestsBufferMode(artdaq::FragmentPtrs& frags);
201 
207  void applyRequestsWindowMode(artdaq::FragmentPtrs& frags);
208 
214  bool applyRequests(FragmentPtrs& frags);
215 
223  bool sendEmptyFragment(FragmentPtrs& frags, size_t sequenceId, std::string desc);
224 
231  void sendEmptyFragments(FragmentPtrs& frags, std::map<Fragment::sequence_id_t, Fragment::timestamp_t>& requests);
232 
237  void checkOutOfOrderWindows(Fragment::sequence_id_t seq);
238 
243  std::map<Fragment::sequence_id_t, std::chrono::steady_clock::time_point> getOutOfOrderWindowList() const
244  {
245  return windows_sent_ooo_;
246  }
247 
251  void startDataThread();
252 
256  void startMonitoringThread();
257 
262  void getDataLoop();
263 
268  bool waitForDataBufferReady();
269 
274  bool dataBufferIsTooLarge();
275 
279  void getDataBufferStats();
280 
286  void checkDataBuffer();
287 
291  void getMonitoringDataLoop();
292 
297  std::vector<Fragment::fragment_id_t> fragmentIDs() override
298  {
299  return fragment_ids_;
300  }
301 
306  size_t ev_counter() const { return ev_counter_.load(); }
307 
312  RequestMode request_mode() const { return mode_; }
313 
314  //
315  // State-machine related interface below.
316  //
317 
332  void StartCmd(int run, uint64_t timeout, uint64_t timestamp);
333 
343  void StopCmd(uint64_t timeout, uint64_t timestamp);
344 
354  void PauseCmd(uint64_t timeout, uint64_t timestamp);
355 
364  void ResumeCmd(uint64_t timeout, uint64_t timestamp);
365 
376  std::string ReportCmd(std::string const& which = "");
377 
382  virtual std::string metricsReportingInstanceName() const
383  {
384  return instance_name_for_metrics_;
385  }
386 
387  // The following functions are not yet implemented, and their
388  // signatures may be subject to change.
389 
390  // John F., 12/6/13 -- do we want Reset and Shutdown commands?
391  // Kurt B., 15-Feb-2014. For the moment, I suspect that we don't
392  // want a Shutdown command. FragmentGenerator instances are
393  // Constructed at Initialization time, and they are destructed
394  // at Shutdown time. So, any shutdown operations that need to be
395  // done should be put in the FragmentGenerator child class
396  // destructors. If we find that want shutdown (or initialization)
397  // operations that are different from destruction (construction),
398  // then we'll have to add InitCmd and ShutdownCmd methods.
399 
400  // virtual void ResetCmd() final {}
401  // virtual void ShutdownCmd() final {}
402 
407  bool exception() const { return exception_.load(); }
408 
409 
416  virtual bool metaCommand(std::string const& command, std::string const& arg);
417 
418  protected:
419 
420  // John F., 12/6/13 -- need to figure out which of these getter
421  // functions should be promoted to "public"
422 
423  // John F., 1/21/15 -- after more than a year, there hasn't been a
424  // single complaint that a CommandableFragmentGenerator-derived
425  // class hasn't allowed its users to access these quantities, so
426  // they're probably fine as is
427 
432  int run_number() const { return run_number_; }
437  int subrun_number() const { return subrun_number_; }
442  uint64_t timeout() const { return timeout_; }
447  uint64_t timestamp() const { return timestamp_; }
448 
453  bool should_stop() const { return should_stop_.load(); }
454 
459  bool check_stop();
460 
465  int board_id() const { return board_id_; }
466 
472  int fragment_id() const;
473 
480  size_t ev_counter_inc(size_t step = 1, bool force = false); // returns the prev value
481 
486  void set_exception(bool exception) { exception_.store(exception); }
487 
492  void metricsReportingInstanceName(std::string const& name)
493  {
494  instance_name_for_metrics_ = name;
495  }
496 
501  std::string printMode_();
502 
503  // John F., 12/10/13
504  // Is there a better way to handle mutex_ than leaving it a protected variable?
505 
506  // John F., 1/21/15
507  // Translation above is "should mutex_ be a private variable,
508  // accessible via a getter function". Probably, but at this point
509  // it's not worth breaking code by implementing this.
510 
511  std::mutex mutex_;
512 
513  private:
514  // FHiCL-configurable variables. Note that the C++ variable names
515  // are the FHiCL variable names with a "_" appended
516 
517  //Socket parameters
518  std::unique_ptr<RequestReceiver> requestReceiver_;
519 
520  RequestMode mode_;
521  Fragment::timestamp_t windowOffset_;
522  Fragment::timestamp_t windowWidth_;
523  Fragment::timestamp_t staleTimeout_;
524  Fragment::type_t expectedType_;
525  size_t maxFragmentCount_;
526  bool uniqueWindows_;
527  std::map<Fragment::sequence_id_t, std::chrono::steady_clock::time_point> windows_sent_ooo_;
528  size_t missing_request_window_timeout_us_;
529  size_t window_close_timeout_us_;
530 
531  bool useDataThread_;
532  bool circularDataBufferMode_;
533  size_t sleep_on_no_data_us_;
534  std::atomic<bool> data_thread_running_;
535  boost::thread dataThread_;
536 
537  std::condition_variable dataCondition_;
538  std::atomic<int> dataBufferDepthFragments_;
539  std::atomic<size_t> dataBufferDepthBytes_;
540  int maxDataBufferDepthFragments_;
541  size_t maxDataBufferDepthBytes_;
542 
543  bool useMonitoringThread_;
544  boost::thread monitoringThread_;
545  int64_t monitoringInterval_; // Microseconds
546  std::chrono::steady_clock::time_point lastMonitoringCall_;
547  std::atomic<bool> isHardwareOK_;
548 
549  FragmentPtrs dataBuffer_;
550  FragmentPtrs newDataBuffer_;
551  std::mutex dataBufferMutex_;
552 
553  std::vector<artdaq::Fragment::fragment_id_t> fragment_ids_;
554 
555  // In order to support the state-machine related behavior, all
556  // CommandableFragmentGenerators must be able to remember a run number and a
557  // subrun number.
558  int run_number_, subrun_number_;
559 
560  // JCF, 8/28/14
561 
562  // Provide a user-adjustable timeout for the start transition
563  uint64_t timeout_;
564 
565  // JCF, 8/21/14
566 
567  // In response to a need to synchronize various components using
568  // different fragment generators in an experiment, keep a record
569  // of a timestamp (see Redmine Issue #6783 for more)
570 
571  uint64_t timestamp_;
572 
573  std::atomic<bool> should_stop_, exception_, force_stop_;
574  std::string latest_exception_report_;
575  std::atomic<size_t> ev_counter_;
576 
577  int board_id_;
578  std::string instance_name_for_metrics_;
579 
580  // Depending on what sleep_on_stop_us_ is set to, this gives the
581  // stopping thread the chance to gather the required lock
582 
583  int sleep_on_stop_us_;
584 
585  protected:
586 
594  virtual bool getNext_(FragmentPtrs& output) = 0;
595 
596 
603  virtual bool checkHWStatus_();
604 
605  //
606  // State-machine related implementor interface below.
607  //
608 
619  virtual void start() = 0;
620 
627  virtual void stopNoMutex() = 0;
628 
637  virtual void stop() = 0;
638 
643  virtual void pauseNoMutex();
644 
650  virtual void pause();
651 
655  virtual void resume();
656 
668  virtual std::string report();
669 
675  virtual std::string reportSpecific(std::string const& what);
676 
677  };
678 }
679 
680 #endif /* artdaq_Application_CommandableFragmentGenerator_hh */
fhicl::Atom< Fragment::type_t > expected_fragment_type
&quot;expected_fragment_type&quot; (Default: 231, EmptyFragmentType) : The type of Fragments this CFG will be g...
int fragment_id() const
Get the current Fragment ID, if there is only one.
fhicl::Atom< int > board_id
&quot;board_id&quot; (REQUIRED) : The identification number for this CommandableFragmentGenerator ...
void applyRequestsSingleMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Single. Precondition: dataBufferMutex_ and reques...
int subrun_number() const
Get the current Subrun number.
virtual void start()=0
If a CommandableFragmentGenerator subclass is reading from a file, and start() is called...
fhicl::Atom< int64_t > hardware_poll_interval_us
&quot;hardware_poll_interval_us&quot; (Default: 0) : If a separate monitoring thread is used, how often should it call checkHWStatus_
virtual bool checkHWStatus_()
Check any relavent hardware status registers. Return false if an error condition exists that should h...
virtual ~CommandableFragmentGenerator()
CommandableFragmentGenerator Destructor.
RequestMode
The RequestMode enumeration contains the possible ways which CommandableFragmentGenerator responds to...
void applyRequestsBufferMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Buffer. Precondition: dataBufferMutex_ and reques...
bool exception() const
Get the current value of the exception flag.
void metricsReportingInstanceName(std::string const &name)
Sets the name for metrics reporting.
std::mutex mutex_
Mutex used to ensure that multiple transition commands do not run at the same time.
bool sendEmptyFragment(FragmentPtrs &frags, size_t sequenceId, std::string desc)
Send an EmptyFragmentType Fragment.
void getMonitoringDataLoop()
This function regularly calls checkHWStatus_(), and sets the isHardwareOK flag accordingly.
void startDataThread()
Function that launches the data thread (getDataLoop())
fhicl::TableFragment< artdaq::RequestReceiver::Config > receiverConfig
Configuration for the Request Receiver. See artdaq::RequestReceiver::Config.
std::string ReportCmd(std::string const &which="")
Get a report about a user-specified run-time quantity.
virtual bool metaCommand(std::string const &command, std::string const &arg)
The meta-command is used for implementing user-specific commands in a CommandableFragmentGenerator.
fhicl::Atom< Fragment::timestamp_t > stale_request_timeout
&quot;stale_request_timeout&quot; (Default: -1) : Fragments stored in the fragment generator which are older th...
bool dataBufferIsTooLarge()
Test the configured constraints on the data buffer.
void StopCmd(uint64_t timeout, uint64_t timestamp)
Stop the CommandableFragmentGenerator.
uint64_t timeout() const
Timeout of last command.
void applyRequestsWindowMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Window. Precondition: dataBufferMutex_ and reques...
void StartCmd(int run, uint64_t timeout, uint64_t timestamp)
Start the CommandableFragmentGenerator.
virtual void pauseNoMutex()
On call to PauseCmd, pauseNoMutex() is called prior to PauseCmd acquiring the mutex ...
bool check_stop()
Routine used by applyRequests to make sure that all outstanding requests have been fulfilled before r...
void ResumeCmd(uint64_t timeout, uint64_t timestamp)
Resume the CommandableFragmentGenerator.
CommandableFragmentGenerator()
CommandableFragmentGenerator default constructor.
bool getNext(FragmentPtrs &output) overridefinal
getNext calls either applyRequests or getNext_ to get any data that is ready to be sent to the EventB...
bool waitForDataBufferReady()
Wait for the data buffer to drain (dataBufferIsTooLarge returns false), periodically reporting status...
size_t ev_counter_inc(size_t step=1, bool force=false)
Increment the event counter, if the current RequestMode allows it.
fhicl::Atom< int > data_buffer_depth_fragments
&quot;data_buffer_depth_fragments&quot; (Default: 1000) : The max fragments which can be stored before dropping...
fhicl::Atom< size_t > data_buffer_depth_mb
&quot;data_buffer_depth_mb&quot; (Default: 1000) : The max cumulative size in megabytes of the fragments which ...
bool applyRequests(FragmentPtrs &frags)
See if any requests have been received, and add the corresponding data Fragment objects to the output...
std::vector< Fragment::fragment_id_t > fragmentIDs() override
Get the list of Fragment IDs handled by this CommandableFragmentGenerator.
fhicl::Atom< int > sleep_on_stop_us
&quot;sleep_on_stop_us&quot; (Default: 0) : How long to sleep before returning when stop transition is called ...
void applyRequestsIgnoredMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Ignored. Precondition: dataBufferMutex_ and reque...
fhicl::Atom< size_t > missing_request_window_timeout_us
&quot;missing_request_window_timeout_us&quot; (Default: 5000000) : How long to track missing requests in the &quot;o...
void PauseCmd(uint64_t timeout, uint64_t timestamp)
Pause the CommandableFragmentGenerator.
void getDataLoop()
When separate_data_thread is set to true, this loop repeatedly calls getNext_ and adds returned Fragm...
fhicl::Sequence< Fragment::fragment_id_t > fragment_ids
virtual std::string metricsReportingInstanceName() const
Get the name used when reporting metrics.
void sendEmptyFragments(FragmentPtrs &frags, std::map< Fragment::sequence_id_t, Fragment::timestamp_t > &requests)
This function is for Buffered and Single request modes, as they can only respond to one data request ...
bool should_stop() const
Get the current value of the should_stop flag.
CommandableFragmentGenerator is a FragmentGenerator-derived abstract class that defines the interface...
fhicl::WrappedTable< Config > Parameters
Used for ParameterSet validation (if desired)
virtual bool getNext_(FragmentPtrs &output)=0
Obtain the next group of Fragments, if any are available. Return false if readout cannot continue...
fhicl::Atom< std::string > request_mode
&quot;request_mode&quot; (Deafult: Ignored) : The mode by which the CommandableFragmentGenerator will process r...
void startMonitoringThread()
Function that launches the monitoring thread (getMonitoringDataLoop())
fhicl::Atom< Fragment::timestamp_t > request_window_offset
&quot;request_window_offset&quot; (Default: 0) : Request messages contain a timestamp. For Window request mode...
virtual void pause()
If a CommandableFragmentGenerator subclass is reading from hardware, the implementation of pause() sh...
virtual void resume()
The subrun number will be incremented before a call to resume.
size_t ev_counter() const
Get the current value of the event counter.
void checkDataBuffer()
Perform data buffer pruning operations. If the RequestMode is Single, removes all but the latest Frag...
fhicl::Atom< bool > request_windows_are_unique
&quot;request_windows_are_unique&quot; (Default: true) : Whether Fragments should be removed from the buffer wh...
std::map< Fragment::sequence_id_t, std::chrono::steady_clock::time_point > getOutOfOrderWindowList() const
Access the windows_sent_ooo_ map.
fhicl::Atom< size_t > window_close_timeout_us
&quot;window_close_timeout_us&quot; (Default: 2000000) : How long to wait for the end of the data buffer to pas...
fhicl::Atom< size_t > sleep_on_no_data_us
&quot;sleep_on_no_data_us&quot; (Default: 0 (no sleep)) : How long to sleep after calling getNext_ if no data i...
fhicl::Atom< Fragment::timestamp_t > request_window_width
&quot;request_window_width&quot; (Default: 0) : For Window request mode, the window will be timestamp - offset ...
virtual std::string report()
Let&#39;s say that the contract with the report() functions is that they return a non-empty string if the...
virtual void stopNoMutex()=0
On call to StopCmd, stopNoMutex() is called prior to StopCmd acquiring the mutex
fhicl::Atom< bool > separate_data_thread
&quot;separate_data_thread&quot; (Default: false) : Whether data collection should proceed on its own thread...
int board_id() const
Gets the current board_id.
std::string printMode_()
Return the string representation of the current RequestMode.
void set_exception(bool exception)
Control the exception flag.
void getDataBufferStats()
Calculate the size of the dataBuffer and report appropriate metrics.
RequestMode request_mode() const
Get the current request mode of the CommandableFragmentGenerator
uint64_t timestamp() const
Timestamp of last command.
fhicl::Atom< bool > circular_buffer_mode
&quot;circular_buffer_mode&quot; (Default: false) : Whether the data buffer should be treated as a circular buf...
Configuration of the CommandableFragmentGenerator. May be used for parameter validation ...
fhicl::Atom< std::string > generator_type
&quot;generator&quot; (REQUIRED) Name of the CommandableFragmentGenerator plugin to load
void checkOutOfOrderWindows(Fragment::sequence_id_t seq)
Check the windows_sent_ooo_ map for sequence IDs that may be removed.
virtual void stop()=0
If a CommandableFragmentGenerator subclass is reading from a file, calling stop() should arrange that...
virtual std::string reportSpecific(std::string const &what)
Report the status of a specific quantity
int run_number() const
Get the current Run number.
fhicl::Atom< bool > separate_monitoring_thread
&quot;separate_monitoring_thread&quot; (Default: false) : Whether a thread that calls the checkHWStatus_ method...
void joinThreads()
Join any data-taking threads. Should be called when destructing CommandableFragmentGenerator.