artdaq  v3_01_00
SharedMemoryEventManager.hh
1 #ifndef ARTDAQ_DAQRATE_SHAREDMEMORYEVENTMANAGER_HH
2 #define ARTDAQ_DAQRATE_SHAREDMEMORYEVENTMANAGER_HH
3 
4 #include "artdaq/DAQdata/Globals.hh" // Before trace.h gets included in ConcurrentQueue (from GlobalQueue)
5 #include "artdaq-core/Core/SharedMemoryManager.hh"
6 #include "artdaq-core/Data/RawEvent.hh"
7 #include "artdaq/DAQrate/RequestSender.hh"
8 #include <set>
9 #include <deque>
10 #include <fstream>
11 #include <iomanip>
12 #include "fhiclcpp/fwd.h"
13 #include "artdaq/Application/StatisticsHelper.hh"
14 #define ART_SUPPORTS_DUPLICATE_EVENTS 0
15 
16 namespace artdaq {
17 
22  {
23  public:
28  art_config_file(fhicl::ParameterSet ps/*, uint32_t shm_key, uint32_t broadcast_key*/) : file_name_(std::tmpnam(nullptr))
29  {
30  std::ofstream of(file_name_, std::ofstream::trunc);
31  of << ps.to_string();
32 
33  //if (ps.has_key("services.NetMonTransportServiceInterface"))
34  //{
35  // of << " services.NetMonTransportServiceInterface.shared_memory_key: 0x" << std::hex << shm_key;
36  // of << " services.NetMonTransportServiceInterface.broadcast_shared_memory_key: 0x" << std::hex << broadcast_key;
37  // of << " services.NetMonTransportServiceInterface.rank: " << std::dec << my_rank;
38  //}
39  if (!ps.has_key("services.message"))
40  {
41  of << " services.message: { " << generateMessageFacilityConfiguration("art") << "} ";
42  }
43  //of << " source.shared_memory_key: 0x" << std::hex << shm_key;
44  //of << " source.broadcast_shared_memory_key: 0x" << std::hex << broadcast_key;
45  //of << " source.rank: " << std::dec << my_rank;
46  of.close();
47  }
48  ~art_config_file() { remove(file_name_.c_str()); }
53  std::string getFileName() const { return file_name_; }
54  private:
55  std::string file_name_;
56  };
57 
61  class SharedMemoryEventManager : public SharedMemoryManager
62  {
63  public:
64  typedef RawEvent::run_id_t run_id_t;
65  typedef RawEvent::subrun_id_t subrun_id_t;
66  typedef Fragment::sequence_id_t sequence_id_t;
67  typedef std::map<sequence_id_t, RawEvent_ptr> EventMap;
68 
101  SharedMemoryEventManager(fhicl::ParameterSet pset, fhicl::ParameterSet art_pset);
105  virtual ~SharedMemoryEventManager();
106 
107  private:
114  bool AddFragment(detail::RawFragmentHeader frag, void* dataPtr);
115 
116  public:
124  bool AddFragment(FragmentPtr frag, size_t timeout_usec, FragmentPtr& outfrag);
125 
132  RawDataType* WriteFragmentHeader(detail::RawFragmentHeader frag, bool dropIfNoBuffersAvailable = false);
133 
138  void DoneWritingFragment(detail::RawFragmentHeader frag);
139 
144  size_t GetIncompleteEventCount() { return active_buffers_.size(); }
145 
150  size_t GetPendingEventCount() { return pending_buffers_.size(); }
151 
156  size_t GetLockedBufferCount() { return GetBuffersOwnedByManager().size(); }
157 
162  size_t GetArtEventCount() { return subrun_event_count_; }
163 
170  size_t GetFragmentCount(Fragment::sequence_id_t seqID, Fragment::type_t type = Fragment::InvalidFragmentType);
171 
178  size_t GetFragmentCountInBuffer(int buffer, Fragment::type_t type = Fragment::InvalidFragmentType);
179 
183  void RunArt(std::shared_ptr<art_config_file> config_file, pid_t& pid_out);
187  void StartArt();
188 
194  pid_t StartArtProcess(fhicl::ParameterSet pset);
195 
200  void ShutdownArtProcesses(std::set<pid_t> pids);
201 
208  void ReconfigureArt(fhicl::ParameterSet art_pset, run_id_t newRun = 0, int n_art_processes = -1);
209 
219  bool endOfData();
220 
225  void startRun(run_id_t runID);
226 
230  void startSubrun();
231 
236  run_id_t runID() const { return run_id_; }
237 
242  subrun_id_t subrunID() const { return subrun_id_; }
243 
248  bool endRun();
249 
254  bool endSubrun();
255 
259  void sendMetrics();
260 
265  void setRequestMode(detail::RequestMessageMode mode) { if (requests_) requests_->SetRequestMode(mode); }
266 
271  void setOverwrite(bool overwrite) { overwrite_mode_ = overwrite; }
272 
276  void SetInitFragment(FragmentPtr frag);
277 
282  uint32_t GetBroadcastKey() { return broadcasts_.GetKey(); }
283 
288  RawDataType* GetDroppedDataAddress() { return dropped_data_->dataBegin(); }
289 
290  private:
291  size_t num_art_processes_;
292  size_t const num_fragments_per_event_;
293  size_t const queue_size_;
294  run_id_t run_id_;
295  subrun_id_t subrun_id_;
296 
297  std::set<int> active_buffers_;
298  std::set<int> pending_buffers_;
299  std::unordered_map<Fragment::sequence_id_t, size_t> released_incomplete_events_;
300 
301  bool update_run_ids_;
302  bool overwrite_mode_;
303  bool send_init_fragments_;
304  bool running_;
305 
306  std::unordered_map<int, std::atomic<int>> buffer_writes_pending_;
307  std::unordered_map<int, std::mutex> buffer_mutexes_;
308  static std::mutex sequence_id_mutex_;
309 
310  int incomplete_event_report_interval_ms_;
311  std::chrono::steady_clock::time_point last_incomplete_event_report_time_;
312  int broadcast_timeout_ms_;
313 
314  std::atomic<int> run_event_count_;
315  std::atomic<int> run_incomplete_event_count_;
316  std::atomic<int> subrun_event_count_;
317  std::atomic<int> subrun_incomplete_event_count_;
318 
319  std::set<pid_t> art_processes_;
320  std::atomic<bool> restart_art_;
321  fhicl::ParameterSet current_art_pset_;
322  std::shared_ptr<art_config_file> current_art_config_file_;
323  double minimum_art_lifetime_s_;
324  size_t art_event_processing_time_us_;
325 
326  std::unique_ptr<RequestSender> requests_;
327  fhicl::ParameterSet data_pset_;
328 
329  FragmentPtr init_fragment_;
330  FragmentPtr dropped_data_;
331 
332  bool broadcastFragment_(FragmentPtr frag, FragmentPtr& outFrag);
333 
334  detail::RawEventHeader* getEventHeader_(int buffer);
335 
336  int getBufferForSequenceID_(Fragment::sequence_id_t seqID, bool create_new, Fragment::timestamp_t timestamp = Fragment::InvalidTimestamp);
337  bool hasFragments_(int buffer);
338  void complete_buffer_(int buffer);
339  bool bufferComparator(int bufA, int bufB);
340  void check_pending_buffers_(std::unique_lock<std::mutex> const& lock = std::unique_lock<std::mutex>(sequence_id_mutex_));
341 
342  void send_init_frag_();
343  SharedMemoryManager broadcasts_;
344  };
345 }
346 
347 #endif //ARTDAQ_DAQRATE_SHAREDMEMORYEVENTMANAGER_HH
art_config_file wraps a temporary file used to configure art
void RunArt(std::shared_ptr< art_config_file > config_file, pid_t &pid_out)
Run an art instance, recording the return codes and restarting it until the end flag is raised...
size_t GetLockedBufferCount()
Returns the number of buffers currently owned by this manager.
virtual ~SharedMemoryEventManager()
SharedMemoryEventManager Destructor.
The SharedMemoryEventManager is a SharedMemoryManger which tracks events as they are built...
Fragment::sequence_id_t sequence_id_t
Copy Fragment::sequence_id_t into local scope.
void ReconfigureArt(fhicl::ParameterSet art_pset, run_id_t newRun=0, int n_art_processes=-1)
Restart all art processes, using the given fhicl code to configure the new art processes.
pid_t StartArtProcess(fhicl::ParameterSet pset)
Start one art process.
RawDataType * WriteFragmentHeader(detail::RawFragmentHeader frag, bool dropIfNoBuffersAvailable=false)
Get a pointer to a reserved memory area for the given Fragment header.
void setRequestMode(detail::RequestMessageMode mode)
Set the RequestMessageMode for all outgoing data requests.
size_t GetArtEventCount()
Returns the number of events sent to art this subrun.
RawEvent::run_id_t run_id_t
Copy RawEvent::run_id_t into local scope.
RawDataType * GetDroppedDataAddress()
Gets the address of the &quot;dropped data&quot; fragment. Used for testing.
size_t GetFragmentCount(Fragment::sequence_id_t seqID, Fragment::type_t type=Fragment::InvalidFragmentType)
Get the count of Fragments of a given type in an event.
size_t GetPendingEventCount()
Returns the number of events which are complete but waiting on lower sequenced events to finish...
void StartArt()
Start all the art processes.
run_id_t runID() const
Get the current Run number.
void ShutdownArtProcesses(std::set< pid_t > pids)
Shutdown a set of art processes.
void SetInitFragment(FragmentPtr frag)
Set the stored Init fragment, if one has not yet been set already.
subrun_id_t subrunID() const
Get the current subrun number.
std::map< sequence_id_t, RawEvent_ptr > EventMap
An EventMap is a map of RawEvent_ptr objects, keyed by sequence ID.
RequestMessageMode
Mode used to indicate current run conditions to the request receiver.
void sendMetrics()
Send metrics to the MetricManager, if one has been instantiated in the application.
void startSubrun()
Start a new Subrun, incrementing the subrun number.
size_t GetIncompleteEventCount()
Returns the number of buffers which contain data but are not yet complete.
SharedMemoryEventManager(fhicl::ParameterSet pset, fhicl::ParameterSet art_pset)
SharedMemoryEventManager Constructor.
bool endSubrun()
Send an EndOfSubRunFragment to the art thread.
bool endRun()
Send an EndOfRunFragment to the art thread.
void setOverwrite(bool overwrite)
Set the overwrite flag (non-reliable data transfer) for the Shared Memory.
std::string getFileName() const
Get the path of the temporary file.
void DoneWritingFragment(detail::RawFragmentHeader frag)
Used to indicate that the given Fragment is now completely in the buffer. Will check for buffer compl...
uint32_t GetBroadcastKey()
Gets the shared memory key of the broadcast SharedMemoryManager.
bool endOfData()
Indicate that the end of input has been reached to the art processes.
RawEvent::subrun_id_t subrun_id_t
Copy RawEvent::subrun_id_t into local scope.
art_config_file(fhicl::ParameterSet ps)
art_config_file Constructor
void startRun(run_id_t runID)
Start a Run.
size_t GetFragmentCountInBuffer(int buffer, Fragment::type_t type=Fragment::InvalidFragmentType)
Get the count of Fragments of a given type in a buffer.