artdaq  v3_03_00
CommandableFragmentGenerator.hh
1 #ifndef artdaq_Application_CommandableFragmentGenerator_hh
2 #define artdaq_Application_CommandableFragmentGenerator_hh
3 
4 // Socket Includes
5 #include <arpa/inet.h>
6 #include <netinet/in.h>
7 #include <sys/types.h>
8 #include <sys/socket.h>
9 #include <unistd.h>
10 
11 #include <atomic>
12 #include <condition_variable>
13 #include <mutex>
14 #include <queue>
15 #include <chrono>
16 #include <array>
17 #include <list>
18 
19 #include "fhiclcpp/fwd.h"
20 #include "fhiclcpp/ParameterSet.h"
21 
22 #include "artdaq/DAQdata/Globals.hh"
23 #include "artdaq-core/Data/Fragment.hh"
24 #include "artdaq-core/Generators/FragmentGenerator.hh"
25 #include "artdaq-utilities/Plugins/MetricManager.hh"
26 #include "artdaq/DAQrate/RequestReceiver.hh"
27 
28 namespace artdaq
29 {
33  enum class RequestMode
34  {
35  Single,
36  Buffer,
37  Window,
38  Ignored
39  };
40 
83  class CommandableFragmentGenerator : public FragmentGenerator
84  {
85  public:
89  struct Config
90  {
92  fhicl::Atom<std::string> generator_type { fhicl::Name{"generator" }, fhicl::Comment{"Name of the CommandableFragmentGenerator plugin to load"} };
94  fhicl::Atom<Fragment::timestamp_t> request_window_offset { fhicl::Name{"request_window_offset" }, fhicl::Comment{"Request messages contain a timestamp. For Window request mode, start the window this far before the timestamp in the request"}, 0 };
96  fhicl::Atom<Fragment::timestamp_t> request_window_width { fhicl::Name{"request_window_width" }, fhicl::Comment{"For Window request mode, the window will be timestamp - offset to timestamp - offset + width"}, 0 };
98  fhicl::Atom<Fragment::timestamp_t> stale_request_timeout { fhicl::Name{"stale_request_timeout" }, fhicl::Comment{"How long should request messages be retained"}, 0xFFFFFFFF };
100  fhicl::Atom<Fragment::type_t> expected_fragment_type { fhicl::Name{"expected_fragment_type" }, fhicl::Comment{"The type of Fragments this CFG will be generating. \"Empty\" will auto-detect type based on Fragments generated."}, Fragment::type_t(Fragment::EmptyFragmentType) };
102  fhicl::Atom<bool> request_windows_are_unique { fhicl::Name{"request_windows_are_unique" }, fhicl::Comment{"Whether Fragments should be removed from the buffer when matched to a request window"}, true };
104  fhicl::Atom<size_t> missing_request_window_timeout_us { fhicl::Name{"missing_request_window_timeout_us"}, fhicl::Comment{"How long to track missing requests in the \"out - of - order Windows\" list"}, 5000000 };
106  fhicl::Atom<size_t> window_close_timeout_us { fhicl::Name{"window_close_timeout_us" }, fhicl::Comment{"How long to wait for the end of the data buffer to pass the end of a request window (measured from the time the request was received)"}, 2000000 };
108  fhicl::Atom<bool> separate_data_thread { fhicl::Name{"separate_data_thread" }, fhicl::Comment{"Whether data collection should proceed on its own thread. Required for all data request processing"}, false };
110  fhicl::Atom<bool> circular_buffer_mode { fhicl::Name{"circular_buffer_mode" }, fhicl::Comment{"Whether the data buffer should be treated as a circular buffer on the input side (i.e. old fragments are automatically discarded when the buffer is full to always call getNext_)."}, false };
112  fhicl::Atom<size_t> sleep_on_no_data_us { fhicl::Name{"sleep_on_no_data_us" }, fhicl::Comment{"How long to sleep after calling getNext_ if no data is returned"}, 0 };
114  fhicl::Atom<int> data_buffer_depth_fragments { fhicl::Name{"data_buffer_depth_fragments" }, fhicl::Comment{"How many Fragments to store in the buffer"}, 1000 };
116  fhicl::Atom<size_t> data_buffer_depth_mb { fhicl::Name{"data_buffer_depth_mb" }, fhicl::Comment{"The maximum size of the data buffer in MB"}, 1000 };
118  fhicl::Atom<bool> separate_monitoring_thread { fhicl::Name{"separate_monitoring_thread" }, fhicl::Comment{"Whether a thread that calls the checkHWStatus_ method should be created"}, false };
120  fhicl::Atom<int64_t> hardware_poll_interval_us { fhicl::Name{"hardware_poll_interval_us" }, fhicl::Comment{"If a separate monitoring thread is used, how often should it call checkHWStatus_"}, 0 };
122  fhicl::Atom<int> board_id { fhicl::Name{"board_id" }, fhicl::Comment{"The identification number for this CommandableFragmentGenerator"} };
125  fhicl::Sequence<Fragment::fragment_id_t> fragment_ids { fhicl::Name("fragment_ids" ), fhicl::Comment("A list of Fragment IDs created by this CommandableFragmentGenerator") };
128  fhicl::Atom<int> fragment_id { fhicl::Name{"fragment_id" }, fhicl::Comment{"The Fragment ID created by this CommandableFragmentGenerator"}, -99 };
130  fhicl::Atom<int> sleep_on_stop_us { fhicl::Name{"sleep_on_stop_us" }, fhicl::Comment{"How long to sleep before returning when stop transition is called"}, 0 };
138  fhicl::Atom<std::string> request_mode { fhicl::Name{"request_mode" }, fhicl::Comment{"The mode by which the CommandableFragmentGenerator will process reqeusts"}, "ignored" };
139  fhicl::TableFragment<artdaq::RequestReceiver::Config> receiverConfig;
140  };
141  using Parameters = fhicl::WrappedTable<Config>;
142 
149 
154  explicit CommandableFragmentGenerator(const fhicl::ParameterSet& ps);
155 
162 
163 
170  void joinThreads();
171 
177  bool getNext(FragmentPtrs& output) override final;
178 
179 
185  void applyRequestsIgnoredMode(artdaq::FragmentPtrs& frags);
186 
192  void applyRequestsSingleMode(artdaq::FragmentPtrs& frags);
193 
199  void applyRequestsBufferMode(artdaq::FragmentPtrs& frags);
200 
206  void applyRequestsWindowMode(artdaq::FragmentPtrs& frags);
207 
213  bool applyRequests(FragmentPtrs& output);
214 
222  bool sendEmptyFragment(FragmentPtrs& frags, size_t sequenceId, std::string desc);
223 
230  void sendEmptyFragments(FragmentPtrs& frags, std::map<Fragment::sequence_id_t, Fragment::timestamp_t>& requests);
231 
236  void checkOutOfOrderWindows(Fragment::sequence_id_t seq);
237 
242  std::map<Fragment::sequence_id_t, std::chrono::steady_clock::time_point> getOutOfOrderWindowList() const
243  {
244  return windows_sent_ooo_;
245  }
246 
250  void startDataThread();
251 
255  void startMonitoringThread();
256 
261  void getDataLoop();
262 
267  bool waitForDataBufferReady();
268 
273  bool dataBufferIsTooLarge();
274 
278  void getDataBufferStats();
279 
285  void checkDataBuffer();
286 
290  void getMonitoringDataLoop();
291 
296  std::vector<Fragment::fragment_id_t> fragmentIDs() override
297  {
298  return fragment_ids_;
299  }
300 
305  size_t ev_counter() const { return ev_counter_.load(); }
306 
311  RequestMode request_mode() const { return mode_; }
312 
313  //
314  // State-machine related interface below.
315  //
316 
331  void StartCmd(int run, uint64_t timeout, uint64_t timestamp);
332 
342  void StopCmd(uint64_t timeout, uint64_t timestamp);
343 
353  void PauseCmd(uint64_t timeout, uint64_t timestamp);
354 
363  void ResumeCmd(uint64_t timeout, uint64_t timestamp);
364 
375  std::string ReportCmd(std::string const& which = "");
376 
381  virtual std::string metricsReportingInstanceName() const
382  {
383  return instance_name_for_metrics_;
384  }
385 
386  // The following functions are not yet implemented, and their
387  // signatures may be subject to change.
388 
389  // John F., 12/6/13 -- do we want Reset and Shutdown commands?
390  // Kurt B., 15-Feb-2014. For the moment, I suspect that we don't
391  // want a Shutdown command. FragmentGenerator instances are
392  // Constructed at Initialization time, and they are destructed
393  // at Shutdown time. So, any shutdown operations that need to be
394  // done should be put in the FragmentGenerator child class
395  // destructors. If we find that want shutdown (or initialization)
396  // operations that are different from destruction (construction),
397  // then we'll have to add InitCmd and ShutdownCmd methods.
398 
399  // virtual void ResetCmd() final {}
400  // virtual void ShutdownCmd() final {}
401 
406  bool exception() const { return exception_.load(); }
407 
408 
415  virtual bool metaCommand(std::string const& command, std::string const& arg);
416 
417  protected:
418 
419  // John F., 12/6/13 -- need to figure out which of these getter
420  // functions should be promoted to "public"
421 
422  // John F., 1/21/15 -- after more than a year, there hasn't been a
423  // single complaint that a CommandableFragmentGenerator-derived
424  // class hasn't allowed its users to access these quantities, so
425  // they're probably fine as is
426 
431  int run_number() const { return run_number_; }
436  int subrun_number() const { return subrun_number_; }
441  uint64_t timeout() const { return timeout_; }
446  uint64_t timestamp() const { return timestamp_; }
447 
452  bool should_stop() const { return should_stop_.load(); }
453 
458  bool check_stop();
459 
464  int board_id() const { return board_id_; }
465 
471  int fragment_id() const;
472 
479  size_t ev_counter_inc(size_t step = 1, bool force = false); // returns the prev value
480 
485  void set_exception(bool exception) { exception_.store(exception); }
486 
491  void metricsReportingInstanceName(std::string const& name)
492  {
493  instance_name_for_metrics_ = name;
494  }
495 
500  std::string printMode_();
501 
502  // John F., 12/10/13
503  // Is there a better way to handle mutex_ than leaving it a protected variable?
504 
505  // John F., 1/21/15
506  // Translation above is "should mutex_ be a private variable,
507  // accessible via a getter function". Probably, but at this point
508  // it's not worth breaking code by implementing this.
509 
510  std::mutex mutex_;
511 
512  private:
513  // FHiCL-configurable variables. Note that the C++ variable names
514  // are the FHiCL variable names with a "_" appended
515 
516  //Socket parameters
517  std::unique_ptr<RequestReceiver> requestReceiver_;
518 
519  RequestMode mode_;
520  Fragment::timestamp_t windowOffset_;
521  Fragment::timestamp_t windowWidth_;
522  Fragment::timestamp_t staleTimeout_;
523  Fragment::type_t expectedType_;
524  size_t maxFragmentCount_;
525  bool uniqueWindows_;
526  std::map<Fragment::sequence_id_t, std::chrono::steady_clock::time_point> windows_sent_ooo_;
527  size_t missing_request_window_timeout_us_;
528  size_t window_close_timeout_us_;
529 
530  bool useDataThread_;
531  bool circularDataBufferMode_;
532  size_t sleep_on_no_data_us_;
533  std::atomic<bool> data_thread_running_;
534  boost::thread dataThread_;
535 
536  std::condition_variable dataCondition_;
537  std::atomic<int> dataBufferDepthFragments_;
538  std::atomic<size_t> dataBufferDepthBytes_;
539  int maxDataBufferDepthFragments_;
540  size_t maxDataBufferDepthBytes_;
541 
542  bool useMonitoringThread_;
543  boost::thread monitoringThread_;
544  int64_t monitoringInterval_; // Microseconds
545  std::chrono::steady_clock::time_point lastMonitoringCall_;
546  std::atomic<bool> isHardwareOK_;
547 
548  FragmentPtrs dataBuffer_;
549  FragmentPtrs newDataBuffer_;
550  std::mutex dataBufferMutex_;
551 
552  std::vector<artdaq::Fragment::fragment_id_t> fragment_ids_;
553 
554  // In order to support the state-machine related behavior, all
555  // CommandableFragmentGenerators must be able to remember a run number and a
556  // subrun number.
557  int run_number_, subrun_number_;
558 
559  // JCF, 8/28/14
560 
561  // Provide a user-adjustable timeout for the start transition
562  uint64_t timeout_;
563 
564  // JCF, 8/21/14
565 
566  // In response to a need to synchronize various components using
567  // different fragment generators in an experiment, keep a record
568  // of a timestamp (see Redmine Issue #6783 for more)
569 
570  uint64_t timestamp_;
571 
572  std::atomic<bool> should_stop_, exception_, force_stop_;
573  std::string latest_exception_report_;
574  std::atomic<size_t> ev_counter_;
575 
576  int board_id_;
577  std::string instance_name_for_metrics_;
578 
579  // Depending on what sleep_on_stop_us_ is set to, this gives the
580  // stopping thread the chance to gather the required lock
581 
582  int sleep_on_stop_us_;
583 
584  protected:
585 
593  virtual bool getNext_(FragmentPtrs& output) = 0;
594 
595 
602  virtual bool checkHWStatus_();
603 
604  //
605  // State-machine related implementor interface below.
606  //
607 
618  virtual void start() = 0;
619 
626  virtual void stopNoMutex() = 0;
627 
636  virtual void stop() = 0;
637 
642  virtual void pauseNoMutex();
643 
649  virtual void pause();
650 
654  virtual void resume();
655 
667  virtual std::string report();
668 
674  virtual std::string reportSpecific(std::string const& what);
675 
676  };
677 }
678 
679 #endif /* artdaq_Application_CommandableFragmentGenerator_hh */
fhicl::Atom< Fragment::type_t > expected_fragment_type
&quot;expected_fragment_type&quot; (Default: 231, EmptyFragmentType) : The type of Fragments this CFG will be g...
int fragment_id() const
Get the current Fragment ID, if there is only one.
fhicl::Atom< int > board_id
&quot;board_id&quot; (REQUIRED) : The identification number for this CommandableFragmentGenerator ...
void applyRequestsSingleMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Single. Precondition: dataBufferMutex_ and reques...
int subrun_number() const
Get the current Subrun number.
virtual void start()=0
If a CommandableFragmentGenerator subclass is reading from a file, and start() is called...
fhicl::Atom< int64_t > hardware_poll_interval_us
&quot;hardware_poll_interval_us&quot; (Default: 0) : If a separate monitoring thread is used, how often should it call checkHWStatus_
virtual bool checkHWStatus_()
Check any relavent hardware status registers. Return false if an error condition exists that should h...
virtual ~CommandableFragmentGenerator()
CommandableFragmentGenerator Destructor.
RequestMode
The RequestMode enumeration contains the possible ways which CommandableFragmentGenerator responds to...
void applyRequestsBufferMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Buffer. Precondition: dataBufferMutex_ and reques...
bool exception() const
Get the current value of the exception flag.
void metricsReportingInstanceName(std::string const &name)
Sets the name for metrics reporting.
std::mutex mutex_
Mutex used to ensure that multiple transition commands do not run at the same time.
bool sendEmptyFragment(FragmentPtrs &frags, size_t sequenceId, std::string desc)
Send an EmptyFragmentType Fragment.
void getMonitoringDataLoop()
This function regularly calls checkHWStatus_(), and sets the isHardwareOK flag accordingly.
void startDataThread()
Function that launches the data thread (getDataLoop())
fhicl::TableFragment< artdaq::RequestReceiver::Config > receiverConfig
Configuration for the Request Receiver. See artdaq::RequestReceiver::Config.
std::string ReportCmd(std::string const &which="")
Get a report about a user-specified run-time quantity.
virtual bool metaCommand(std::string const &command, std::string const &arg)
The meta-command is used for implementing user-specific commands in a CommandableFragmentGenerator.
fhicl::Atom< Fragment::timestamp_t > stale_request_timeout
&quot;stale_request_timeout&quot; (Default: -1) : How long should request messages be retained ...
bool dataBufferIsTooLarge()
Test the configured constraints on the data buffer.
void StopCmd(uint64_t timeout, uint64_t timestamp)
Stop the CommandableFragmentGenerator.
uint64_t timeout() const
Timeout of last command.
void applyRequestsWindowMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Window. Precondition: dataBufferMutex_ and reques...
void StartCmd(int run, uint64_t timeout, uint64_t timestamp)
Start the CommandableFragmentGenerator.
virtual void pauseNoMutex()
On call to PauseCmd, pauseNoMutex() is called prior to PauseCmd acquiring the mutex ...
bool check_stop()
Routine used by applyRequests to make sure that all outstanding requests have been fulfilled before r...
void ResumeCmd(uint64_t timeout, uint64_t timestamp)
Resume the CommandableFragmentGenerator.
CommandableFragmentGenerator()
CommandableFragmentGenerator default constructor.
bool getNext(FragmentPtrs &output) overridefinal
getNext calls either applyRequests or getNext_ to get any data that is ready to be sent to the EventB...
bool waitForDataBufferReady()
Wait for the data buffer to drain (dataBufferIsTooLarge returns false), periodically reporting status...
size_t ev_counter_inc(size_t step=1, bool force=false)
Increment the event counter, if the current RequestMode allows it.
fhicl::Atom< int > data_buffer_depth_fragments
&quot;data_buffer_depth_fragments&quot; (Default: 1000) : How many Fragments to store in the buffer ...
fhicl::Atom< size_t > data_buffer_depth_mb
&quot;data_buffer_depth_mb&quot; (Default: 1000) : The maximum size of the data buffer in MB ...
std::vector< Fragment::fragment_id_t > fragmentIDs() override
Get the list of Fragment IDs handled by this CommandableFragmentGenerator.
fhicl::Atom< int > sleep_on_stop_us
&quot;sleep_on_stop_us&quot; (Default: 0) : How long to sleep before returning when stop transition is called ...
void applyRequestsIgnoredMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Ignored. Precondition: dataBufferMutex_ and reque...
fhicl::Atom< size_t > missing_request_window_timeout_us
&quot;missing_request_window_timeout_us&quot; (Default: 5000000) : How long to track missing requests in the &quot;o...
void PauseCmd(uint64_t timeout, uint64_t timestamp)
Pause the CommandableFragmentGenerator.
void getDataLoop()
When separate_data_thread is set to true, this loop repeatedly calls getNext_ and adds returned Fragm...
fhicl::Sequence< Fragment::fragment_id_t > fragment_ids
virtual std::string metricsReportingInstanceName() const
Get the name used when reporting metrics.
void sendEmptyFragments(FragmentPtrs &frags, std::map< Fragment::sequence_id_t, Fragment::timestamp_t > &requests)
This function is for Buffered and Single request modes, as they can only respond to one data request ...
bool should_stop() const
Get the current value of the should_stop flag.
CommandableFragmentGenerator is a FragmentGenerator-derived abstract class that defines the interface...
virtual bool getNext_(FragmentPtrs &output)=0
Obtain the next group of Fragments, if any are available. Return false if readout cannot continue...
fhicl::Atom< std::string > request_mode
&quot;request_mode&quot; (Deafult: Ignored) : The mode by which the CommandableFragmentGenerator will process r...
void startMonitoringThread()
Function that launches the monitoring thread (getMonitoringDataLoop())
fhicl::Atom< Fragment::timestamp_t > request_window_offset
&quot;request_window_offset&quot; (Default: 0) : Request messages contain a timestamp.For Window request mode...
virtual void pause()
If a CommandableFragmentGenerator subclass is reading from hardware, the implementation of pause() sh...
virtual void resume()
The subrun number will be incremented before a call to resume.
size_t ev_counter() const
Get the current value of the event counter.
void checkDataBuffer()
Perform data buffer pruning operations. If the RequestMode is Single, removes all but the latest Frag...
fhicl::Atom< bool > request_windows_are_unique
&quot;request_windows_are_unique&quot; (Default: true) : Whether Fragments should be removed from the buffer wh...
std::map< Fragment::sequence_id_t, std::chrono::steady_clock::time_point > getOutOfOrderWindowList() const
Access the windows_sent_ooo_ map.
fhicl::Atom< size_t > window_close_timeout_us
&quot;window_close_timeout_us&quot; (Default: 2000000) : How long to wait for the end of the data buffer to pas...
fhicl::Atom< size_t > sleep_on_no_data_us
&quot;sleep_on_no_data_us&quot; (Default: 0 (no sleep)) : How long to sleep after calling getNext_ if no data i...
fhicl::Atom< Fragment::timestamp_t > request_window_width
&quot;request_window_width&quot; (Default: 0) : For Window request mode, the window will be timestamp - offset ...
virtual std::string report()
Let&#39;s say that the contract with the report() functions is that they return a non-empty string if the...
virtual void stopNoMutex()=0
On call to StopCmd, stopNoMutex() is called prior to StopCmd acquiring the mutex
fhicl::Atom< bool > separate_data_thread
&quot;separate_data_thread&quot; (Default: false) : Whether data collection should proceed on its own thread...
int board_id() const
Gets the current board_id.
std::string printMode_()
Return the string representation of the current RequestMode.
void set_exception(bool exception)
Control the exception flag.
void getDataBufferStats()
Calculate the size of the dataBuffer and report appropriate metrics.
RequestMode request_mode() const
Get the current request mode of the CommandableFragmentGenerator
uint64_t timestamp() const
Timestamp of last command.
fhicl::Atom< bool > circular_buffer_mode
&quot;circular_buffer_mode&quot; (Default: false) : Whether the data buffer should be treated as a circular buf...
Configuration of the CommandableFragmentGenerator. May be used for parameter validation ...
fhicl::Atom< std::string > generator_type
&quot;generator&quot; (REQUIRED) Name of the CommandableFragmentGenerator plugin to load
void checkOutOfOrderWindows(Fragment::sequence_id_t seq)
Check the windows_sent_ooo_ map for sequence IDs that may be removed.
virtual void stop()=0
If a CommandableFragmentGenerator subclass is reading from a file, calling stop() should arrange that...
virtual std::string reportSpecific(std::string const &what)
Report the status of a specific quantity
bool applyRequests(FragmentPtrs &output)
See if any requests have been received, and add the corresponding data Fragment objects to the output...
int run_number() const
Get the current Run number.
fhicl::Atom< bool > separate_monitoring_thread
&quot;separate_monitoring_thread&quot; (Default: false) : Whether a thread that calls the checkHWStatus_ method...
void joinThreads()
Join any data-taking threads. Should be called when destructing CommandableFragmentGenerator.