artdaq  v3_09_01
FragmentBuffer.hh
1 #ifndef artdaq_Application_FragmentBuffer_hh
2 #define artdaq_Application_FragmentBuffer_hh
3 
4 // Socket Includes
5 #include <arpa/inet.h>
6 #include <netinet/in.h>
7 #include <sys/socket.h>
8 #include <sys/types.h>
9 #include <unistd.h>
10 
11 #include <array>
12 #include <atomic>
13 #include <chrono>
14 #include <condition_variable>
15 #include <list>
16 #include <mutex>
17 #include <queue>
18 
19 #include "fhiclcpp/ParameterSet.h"
20 #include "fhiclcpp/fwd.h"
21 
22 #include "artdaq-core/Data/Fragment.hh"
23 #include "artdaq-utilities/Plugins/MetricManager.hh"
24 #include "artdaq/DAQdata/Globals.hh"
25 #include "artdaq/DAQrate/RequestBuffer.hh"
26 
27 namespace artdaq {
31 enum class RequestMode
32 {
33  Single,
34  Buffer,
35  Window,
36  SequenceID,
37  Ignored
38 };
39 
83 {
84 public:
88  struct Config
89  {
91  fhicl::Atom<std::string> generator_type{fhicl::Name{"generator"}, fhicl::Comment{"Name of the FragmentBuffer plugin to load"}};
93  fhicl::Atom<Fragment::timestamp_t> request_window_offset{fhicl::Name{"request_window_offset"}, fhicl::Comment{"Request messages contain a timestamp. For Window request mode, start the window this far before the timestamp in the request"}, 0};
95  fhicl::Atom<Fragment::timestamp_t> request_window_width{fhicl::Name{"request_window_width"}, fhicl::Comment{"For Window request mode, the window will be timestamp - offset to timestamp - offset + width"}, 0};
97  fhicl::Atom<Fragment::timestamp_t> stale_request_timeout{fhicl::Name{"stale_request_timeout"}, fhicl::Comment{"Fragments stored in the fragment generator which are older than the newest stored fragment by at least stale_request_timeout units of request timestamp ticks will get discarded"}, 0xFFFFFFFF};
99  fhicl::Atom<bool> buffer_mode_keep_latest{fhicl::Name{"buffer_mode_keep_latest"}, fhicl::Comment{"Keep the latest Fragment when running in Buffer mode, so that each response has at least one Fragment (Fragment will be discarded if new data arrives before next request)"}, false};
101  fhicl::Atom<Fragment::type_t> expected_fragment_type{fhicl::Name{"expected_fragment_type"}, fhicl::Comment{"The type of Fragments this CFG will be generating. \"Empty\" will auto-detect type based on Fragments generated."}, Fragment::type_t(Fragment::EmptyFragmentType)};
103  fhicl::Atom<bool> request_windows_are_unique{fhicl::Name{"request_windows_are_unique"}, fhicl::Comment{"Whether Fragments should be removed from the buffer when matched to a request window"}, true};
105  fhicl::Atom<size_t> missing_request_window_timeout_us{fhicl::Name{"missing_request_window_timeout_us"}, fhicl::Comment{"How long to track missing requests in the \"out - of - order Windows\" list"}, 5000000};
107  fhicl::Atom<size_t> window_close_timeout_us{fhicl::Name{"window_close_timeout_us"}, fhicl::Comment{"How long to wait for the end of the data buffer to pass the end of a request window (measured from the time the request was received)"}, 2000000};
109  fhicl::Atom<bool> separate_data_thread{fhicl::Name{"separate_data_thread"}, fhicl::Comment{"Whether data collection should proceed on its own thread. Required for all data request processing"}, false};
111  fhicl::Atom<bool> circular_buffer_mode{fhicl::Name{"circular_buffer_mode"}, fhicl::Comment{"Whether the data buffer should be treated as a circular buffer on the input side (i.e. old fragments are automatically discarded when the buffer is full to always call getNext_)."}, false};
113  fhicl::Atom<size_t> sleep_on_no_data_us{fhicl::Name{"sleep_on_no_data_us"}, fhicl::Comment{"How long to sleep after calling getNext_ if no data is returned"}, 0};
115  fhicl::Atom<int> data_buffer_depth_fragments{fhicl::Name{"data_buffer_depth_fragments"}, fhicl::Comment{"The max fragments which can be stored before dropping occurs"}, 1000};
117  fhicl::Atom<size_t> data_buffer_depth_mb{fhicl::Name{"data_buffer_depth_mb"}, fhicl::Comment{"The max cumulative size in megabytes of the fragments which can be stored before dropping occurs"}, 1000};
119  fhicl::Atom<bool> separate_monitoring_thread{fhicl::Name{"separate_monitoring_thread"}, fhicl::Comment{"Whether a thread that calls the checkHWStatus_ method should be created"}, false};
121  fhicl::Atom<int64_t> hardware_poll_interval_us{fhicl::Name{"hardware_poll_interval_us"}, fhicl::Comment{"If a separate monitoring thread is used, how often should it call checkHWStatus_"}, 0};
123  fhicl::Atom<int> board_id{fhicl::Name{"board_id"}, fhicl::Comment{"The identification number for this FragmentBuffer"}};
126  fhicl::Sequence<Fragment::fragment_id_t> fragment_ids{fhicl::Name("fragment_ids"), fhicl::Comment("A list of Fragment IDs created by this FragmentBuffer")};
129  fhicl::Atom<int> fragment_id{fhicl::Name{"fragment_id"}, fhicl::Comment{"The Fragment ID created by this FragmentBuffer"}, -99};
131  fhicl::Atom<int> sleep_on_stop_us{fhicl::Name{"sleep_on_stop_us"}, fhicl::Comment{"How long to sleep before returning when stop transition is called"}, 0};
140  fhicl::Atom<std::string> request_mode{fhicl::Name{"request_mode"}, fhicl::Comment{"The mode by which the FragmentBuffer will process reqeusts"}, "ignored"};
141  };
143  using Parameters = fhicl::WrappedTable<Config>;
144 
149  explicit FragmentBuffer(const fhicl::ParameterSet& ps);
150 
156  virtual ~FragmentBuffer();
157 
158  void AddFragmentsToBuffer(FragmentPtrs frags);
159  void Stop() { should_stop_ = true; }
160  void Reset(bool stop);
161 
167  void applyRequestsIgnoredMode(artdaq::FragmentPtrs& frags);
168 
174  void applyRequestsSingleMode(artdaq::FragmentPtrs& frags);
175 
181  void applyRequestsBufferMode(artdaq::FragmentPtrs& frags);
182 
188  void applyRequestsWindowMode(artdaq::FragmentPtrs& frags);
189 
195  void applyRequestsSequenceIDMode(artdaq::FragmentPtrs& frags);
196 
203  void applyRequestsWindowMode_CheckAndFillDataBuffer(artdaq::FragmentPtrs& frags, artdaq::Fragment::fragment_id_t id, artdaq::Fragment::sequence_id_t seq, artdaq::Fragment::timestamp_t ts);
204 
210  bool applyRequests(FragmentPtrs& frags);
211 
220  bool sendEmptyFragment(FragmentPtrs& frags, size_t sequenceId, Fragment::fragment_id_t fragmentId, std::string desc);
221 
228  void sendEmptyFragments(FragmentPtrs& frags, std::map<Fragment::sequence_id_t, Fragment::timestamp_t>& requests);
229 
234  void checkSentWindows(Fragment::sequence_id_t seq);
235 
241  bool waitForDataBufferReady(Fragment::fragment_id_t id);
242 
248  bool dataBufferIsTooLarge(Fragment::fragment_id_t id);
249 
254  void getDataBufferStats(Fragment::fragment_id_t id);
255 
260  {
261  for (auto& id : dataBuffers_) getDataBufferStats(id.first);
262  }
263 
270  void checkDataBuffer(Fragment::fragment_id_t id);
271 
276  {
277  for (auto& id : dataBuffers_) checkDataBuffer(id.first);
278  }
279 
287  std::map<Fragment::sequence_id_t, std::chrono::steady_clock::time_point> GetSentWindowList(Fragment::fragment_id_t id)
288  {
289  if (!dataBuffers_.count(id))
290  {
291  throw cet::exception("DataBufferError") << "Error in FragmentBuffer: Cannot get Sent Windows for ID " << id << " because it does not exist!";
292  }
293 
294  return dataBuffers_[id]->WindowsSent;
295  }
296 
301  std::vector<Fragment::fragment_id_t> fragmentIDs()
302  {
303  std::vector<Fragment::fragment_id_t> output;
304 
305  for (auto& id : dataBuffers_)
306  {
307  output.push_back(id.first);
308  }
309 
310  return output;
311  }
312 
317  RequestMode request_mode() const { return mode_; }
318 
319  // The following functions are not yet implemented, and their
320  // signatures may be subject to change.
321 
322  // John F., 12/6/13 -- do we want Reset and Shutdown commands?
323  // Kurt B., 15-Feb-2014. For the moment, I suspect that we don't
324  // want a Shutdown command. FragmentGenerator instances are
325  // Constructed at Initialization time, and they are destructed
326  // at Shutdown time. So, any shutdown operations that need to be
327  // done should be put in the FragmentGenerator child class
328  // destructors. If we find that want shutdown (or initialization)
329  // operations that are different from destruction (construction),
330  // then we'll have to add InitCmd and ShutdownCmd methods.
331 
332  // virtual void ResetCmd() final {}
333  // virtual void ShutdownCmd() final {}
334 
339  void SetRequestBuffer(std::shared_ptr<RequestBuffer> buffer) { requestBuffer_ = buffer; }
340 
345  artdaq::Fragment::sequence_id_t GetNextSequenceID() const { return next_sequence_id_; }
346 
347 protected:
348  // John F., 12/6/13 -- need to figure out which of these getter
349  // functions should be promoted to "public"
350 
351  // John F., 1/21/15 -- after more than a year, there hasn't been a
352  // single complaint that a FragmentBuffer-derived
353  // class hasn't allowed its users to access these quantities, so
354  // they're probably fine as is
355 
361  artdaq::Fragment::fragment_id_t fragment_id() const
362  {
363  if (dataBuffers_.size() > 1) throw cet::exception("FragmentID") << "fragment_id() was called, indicating that Fragment Generator was expecting one and only one Fragment ID, but " << dataBuffers_.size() << " were declared!";
364  return (*dataBuffers_.begin()).first;
365  }
366 
371  bool check_stop();
372 
377  std::string printMode_();
378 
383  size_t dataBufferFragmentCount_();
384 
385 private:
386  // FHiCL-configurable variables. Note that the C++ variable names
387  // are the FHiCL variable names with a "_" appended
388 
389  //Socket parameters
390  Fragment::sequence_id_t next_sequence_id_;
391  std::shared_ptr<RequestBuffer> requestBuffer_;
392 
393  RequestMode mode_;
394  bool bufferModeKeepLatest_;
395  Fragment::timestamp_t windowOffset_;
396  Fragment::timestamp_t windowWidth_;
397  Fragment::timestamp_t staleTimeout_;
398  Fragment::type_t expectedType_;
399  bool uniqueWindows_;
400  size_t missing_request_window_timeout_us_;
401  size_t window_close_timeout_us_;
402 
403  bool circularDataBufferMode_;
404 
405  std::mutex dataConditionMutex_;
406  std::condition_variable dataCondition_;
407  int maxDataBufferDepthFragments_;
408  size_t maxDataBufferDepthBytes_;
409 
410  struct DataBuffer
411  {
412  std::atomic<int> DataBufferDepthFragments;
413  std::atomic<size_t> DataBufferDepthBytes;
414  std::map<Fragment::sequence_id_t, std::chrono::steady_clock::time_point> WindowsSent;
415  bool BufferFragmentKept;
416  Fragment::sequence_id_t HighestRequestSeen;
417  FragmentPtrs DataBuffer;
418  std::mutex DataBufferMutex;
419  };
420 
421  std::mutex systemFragmentMutex_;
422  FragmentPtrs systemFragments_;
423  std::atomic<size_t> systemFragmentCount_;
424 
425  std::unordered_map<artdaq::Fragment::fragment_id_t, std::shared_ptr<DataBuffer>> dataBuffers_;
426 
427  std::atomic<bool> should_stop_;
428 };
429 } // namespace artdaq
430 
431 #endif /* artdaq_Application_FragmentBuffer_hh */
void checkDataBuffers()
Perform data buffer pruning operations for all buffers.
void getDataBuffersStats()
Calculate the size of all dataBuffers and report appropriate metrics.
RequestMode
The RequestMode enumeration contains the possible ways which FragmentBuffer responds to data requests...
fhicl::Atom< size_t > data_buffer_depth_mb
&quot;data_buffer_depth_mb&quot; (Default: 1000) : The max cumulative size in megabytes of the fragments which ...
size_t dataBufferFragmentCount_()
Get the total number of Fragments in all data buffers.
std::string printMode_()
Return the string representation of the current RequestMode.
bool applyRequests(FragmentPtrs &frags)
See if any requests have been received, and add the corresponding data Fragment objects to the output...
void SetRequestBuffer(std::shared_ptr< RequestBuffer > buffer)
Set the pointer to the RequestBuffer used to retrieve requests.
FragmentBuffer(const fhicl::ParameterSet &ps)
FragmentBuffer Constructor.
bool waitForDataBufferReady(Fragment::fragment_id_t id)
Wait for the data buffer to drain (dataBufferIsTooLarge returns false), periodically reporting status...
std::vector< Fragment::fragment_id_t > fragmentIDs()
Get the list of Fragment IDs handled by this FragmentBuffer.
fhicl::Atom< Fragment::timestamp_t > request_window_offset
&quot;request_window_offset&quot; (Default: 0) : Request messages contain a timestamp. For Window request mode...
fhicl::Atom< Fragment::timestamp_t > stale_request_timeout
&quot;stale_request_timeout&quot; (Default: -1) : Fragments stored in the fragment generator which are older th...
fhicl::Atom< std::string > generator_type
&quot;generator&quot; (REQUIRED) Name of the FragmentBuffer plugin to load
fhicl::Atom< int > sleep_on_stop_us
&quot;sleep_on_stop_us&quot; (Default: 0) : How long to sleep before returning when stop transition is called ...
fhicl::Atom< size_t > sleep_on_no_data_us
&quot;sleep_on_no_data_us&quot; (Default: 0 (no sleep)) : How long to sleep after calling getNext_ if no data i...
fhicl::Atom< bool > request_windows_are_unique
&quot;request_windows_are_unique&quot; (Default: true) : Whether Fragments should be removed from the buffer wh...
fhicl::Atom< bool > separate_data_thread
&quot;separate_data_thread&quot; (Default: false) : Whether data collection should proceed on its own thread...
std::map< Fragment::sequence_id_t, std::chrono::steady_clock::time_point > GetSentWindowList(Fragment::fragment_id_t id)
Get the map of Window-mode requests fulfilled by this Fragment Geneerator for the given Fragment ID...
fhicl::Atom< Fragment::timestamp_t > request_window_width
&quot;request_window_width&quot; (Default: 0) : For Window request mode, the window will be timestamp - offset ...
void applyRequestsIgnoredMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Ignored. Precondition: dataBufferMutex_ and reque...
fhicl::Atom< bool > separate_monitoring_thread
&quot;separate_monitoring_thread&quot; (Default: false) : Whether a thread that calls the checkHWStatus_ method...
fhicl::Atom< size_t > window_close_timeout_us
&quot;window_close_timeout_us&quot; (Default: 2000000) : How long to wait for the end of the data buffer to pas...
fhicl::Atom< size_t > missing_request_window_timeout_us
&quot;missing_request_window_timeout_us&quot; (Default: 5000000) : How long to track missing requests in the &quot;o...
void applyRequestsWindowMode_CheckAndFillDataBuffer(artdaq::FragmentPtrs &frags, artdaq::Fragment::fragment_id_t id, artdaq::Fragment::sequence_id_t seq, artdaq::Fragment::timestamp_t ts)
bool check_stop()
Routine used by applyRequests to make sure that all outstanding requests have been fulfilled before r...
fhicl::Atom< Fragment::type_t > expected_fragment_type
&quot;expected_fragment_type&quot; (Default: 231, EmptyFragmentType) : The type of Fragments this CFG will be g...
artdaq::Fragment::sequence_id_t GetNextSequenceID() const
Get the next sequence ID expected by this FragmentBuffer. This is used to track sent windows and miss...
FragmentBuffer is a FragmentGenerator-derived abstract class that defines the interface for a Fragmen...
fhicl::Atom< int > data_buffer_depth_fragments
&quot;data_buffer_depth_fragments&quot; (Default: 1000) : The max fragments which can be stored before dropping...
void checkSentWindows(Fragment::sequence_id_t seq)
Check the windows_sent_ooo_ map for sequence IDs that may be removed.
fhicl::Atom< int64_t > hardware_poll_interval_us
&quot;hardware_poll_interval_us&quot; (Default: 0) : If a separate monitoring thread is used, how often should it call checkHWStatus_
Configuration of the FragmentBuffer. May be used for parameter validation
artdaq::Fragment::fragment_id_t fragment_id() const
Get the Fragment ID of this Fragment generator.
void checkDataBuffer(Fragment::fragment_id_t id)
Perform data buffer pruning operations for the given buffer. If the RequestMode is Single...
bool dataBufferIsTooLarge(Fragment::fragment_id_t id)
Test the configured constraints on the data buffer.
fhicl::Atom< std::string > request_mode
&quot;request_mode&quot; (Deafult: Ignored) : The mode by which the FragmentBuffer will process reqeusts Ignore...
bool sendEmptyFragment(FragmentPtrs &frags, size_t sequenceId, Fragment::fragment_id_t fragmentId, std::string desc)
Send an EmptyFragmentType Fragment.
void applyRequestsBufferMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Buffer. Precondition: dataBufferMutex_ and reques...
void applyRequestsWindowMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Window. Precondition: dataBufferMutex_ and reques...
fhicl::WrappedTable< Config > Parameters
Used for ParameterSet validation (if desired)
virtual ~FragmentBuffer()
FragmentBuffer Destructor.
void sendEmptyFragments(FragmentPtrs &frags, std::map< Fragment::sequence_id_t, Fragment::timestamp_t > &requests)
This function is for Buffered and Single request modes, as they can only respond to one data request ...
fhicl::Atom< bool > buffer_mode_keep_latest
&quot;buffer_mode_keep_latest&quot; (Default: false): Keep the latest Fragment when running in Buffer mode...
fhicl::Atom< int > board_id
&quot;board_id&quot; (REQUIRED) : The identification number for this FragmentBuffer
fhicl::Sequence< Fragment::fragment_id_t > fragment_ids
RequestMode request_mode() const
Get the current request mode of the FragmentBuffer
void applyRequestsSequenceIDMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode SequenceID. Precondition: dataBufferMutex_ and re...
fhicl::Atom< bool > circular_buffer_mode
&quot;circular_buffer_mode&quot; (Default: false) : Whether the data buffer should be treated as a circular buf...
void applyRequestsSingleMode(artdaq::FragmentPtrs &frags)
Create fragments using data buffer for request mode Single. Precondition: dataBufferMutex_ and reques...
void getDataBufferStats(Fragment::fragment_id_t id)
Calculate the size of the dataBuffer and report appropriate metrics.