artdaq  v3_00_03
SharedMemoryEventManager.hh
1 #ifndef ARTDAQ_DAQRATE_SHAREDMEMORYEVENTMANAGER_HH
2 #define ARTDAQ_DAQRATE_SHAREDMEMORYEVENTMANAGER_HH
3 
4 #include "artdaq/DAQdata/Globals.hh" // Before trace.h gets included in ConcurrentQueue (from GlobalQueue)
5 #include "artdaq-core/Core/SharedMemoryManager.hh"
6 #include "artdaq-core/Data/RawEvent.hh"
7 #include "artdaq/DAQrate/RequestSender.hh"
8 #include <set>
9 #include <deque>
10 #include <fstream>
11 #include <iomanip>
12 #include "fhiclcpp/fwd.h"
13 #include "artdaq/Application/StatisticsHelper.hh"
14 #define ART_SUPPORTS_DUPLICATE_EVENTS 0
15 
16 namespace artdaq {
17 
22  {
23  public:
28  art_config_file(fhicl::ParameterSet ps/*, uint32_t shm_key, uint32_t broadcast_key*/) : file_name_(std::tmpnam(nullptr))
29  {
30  std::ofstream of(file_name_, std::ofstream::trunc);
31  of << ps.to_string();
32 
33  //if (ps.has_key("services.NetMonTransportServiceInterface"))
34  //{
35  // of << " services.NetMonTransportServiceInterface.shared_memory_key: 0x" << std::hex << shm_key;
36  // of << " services.NetMonTransportServiceInterface.broadcast_shared_memory_key: 0x" << std::hex << broadcast_key;
37  // of << " services.NetMonTransportServiceInterface.rank: " << std::dec << my_rank;
38  //}
39  if (!ps.has_key("services.message"))
40  {
41  of << " services.message: { " << generateMessageFacilityConfiguration("art") << "} ";
42  }
43  //of << " source.shared_memory_key: 0x" << std::hex << shm_key;
44  //of << " source.broadcast_shared_memory_key: 0x" << std::hex << broadcast_key;
45  //of << " source.rank: " << std::dec << my_rank;
46  of.close();
47  }
48  ~art_config_file() { remove(file_name_.c_str()); }
53  std::string getFileName() const { return file_name_; }
54  private:
55  std::string file_name_;
56  };
57 
61  class SharedMemoryEventManager : public SharedMemoryManager
62  {
63  public:
64  typedef RawEvent::run_id_t run_id_t;
65  typedef RawEvent::subrun_id_t subrun_id_t;
66  typedef Fragment::sequence_id_t sequence_id_t;
67  typedef std::map<sequence_id_t, RawEvent_ptr> EventMap;
68 
98  SharedMemoryEventManager(fhicl::ParameterSet pset, fhicl::ParameterSet art_pset);
102  virtual ~SharedMemoryEventManager();
103 
104  private:
111  bool AddFragment(detail::RawFragmentHeader frag, void* dataPtr);
112 
113  public:
121  bool AddFragment(FragmentPtr frag, size_t timeout_usec, FragmentPtr& outfrag);
122 
129  RawDataType* WriteFragmentHeader(detail::RawFragmentHeader frag, bool dropIfNoBuffersAvailable = false);
130 
135  void DoneWritingFragment(detail::RawFragmentHeader frag);
136 
141  size_t GetIncompleteEventCount() { return active_buffers_.size(); }
142 
147  size_t GetPendingEventCount() { return pending_buffers_.size(); }
148 
153  size_t GetLockedBufferCount() { return GetBuffersOwnedByManager().size(); }
154 
159  size_t GetArtEventCount() { return subrun_event_count_; }
160 
167  size_t GetFragmentCount(Fragment::sequence_id_t seqID, Fragment::type_t type = Fragment::InvalidFragmentType);
168 
175  size_t GetFragmentCountInBuffer(int buffer, Fragment::type_t type = Fragment::InvalidFragmentType);
176 
180  void RunArt(std::shared_ptr<art_config_file> config_file, pid_t& pid_out);
184  void StartArt();
185 
191  pid_t StartArtProcess(fhicl::ParameterSet pset);
192 
197  void ShutdownArtProcesses(std::set<pid_t> pids);
198 
205  void ReconfigureArt(fhicl::ParameterSet art_pset, run_id_t newRun = 0, int n_art_processes = -1);
206 
216  bool endOfData();
217 
222  void startRun(run_id_t runID);
223 
227  void startSubrun();
228 
233  run_id_t runID() const { return run_id_; }
234 
239  subrun_id_t subrunID() const { return subrun_id_; }
240 
245  bool endRun();
246 
251  bool endSubrun();
252 
256  void sendMetrics();
257 
263 
268  void setOverwrite(bool overwrite) { overwrite_mode_ = overwrite; }
269 
273  void SetInitFragment(FragmentPtr frag);
274 
279  uint32_t GetBroadcastKey() { return broadcasts_.GetKey(); }
280 
285  RawDataType* GetDroppedDataAddress() { return dropped_data_->dataBegin(); }
286 
287  private:
288  size_t num_art_processes_;
289  size_t const num_fragments_per_event_;
290  size_t const queue_size_;
291  run_id_t run_id_;
292  subrun_id_t subrun_id_;
293 
294  std::set<int> active_buffers_;
295  std::set<int> pending_buffers_;
296  std::unordered_map<Fragment::sequence_id_t, size_t> released_incomplete_events_;
297 
298  bool update_run_ids_;
299  bool overwrite_mode_;
300  bool send_init_fragments_;
301 
302  std::unordered_map<int, std::atomic<int>> buffer_writes_pending_;
303  std::unordered_map<int, std::mutex> buffer_mutexes_;
304  static std::mutex sequence_id_mutex_;
305 
306  int incomplete_event_report_interval_ms_;
307  std::chrono::steady_clock::time_point last_incomplete_event_report_time_;
308  int broadcast_timeout_ms_;
309  int broadcast_count_;
310  int subrun_event_count_;
311 
312  std::set<pid_t> art_processes_;
313  std::atomic<bool> restart_art_;
314  fhicl::ParameterSet current_art_pset_;
315  std::shared_ptr<art_config_file> current_art_config_file_;
316 
317  RequestSender requests_;
318 
319  FragmentPtr init_fragment_;
320  FragmentPtr dropped_data_;
321 
322  bool broadcastFragment_(FragmentPtr frag, FragmentPtr& outFrag);
323 
324  detail::RawEventHeader* getEventHeader_(int buffer);
325 
326  int getBufferForSequenceID_(Fragment::sequence_id_t seqID, bool create_new, Fragment::timestamp_t timestamp = Fragment::InvalidTimestamp);
327  bool hasFragments_(int buffer);
328  void complete_buffer_(int buffer);
329  bool bufferComparator(int bufA, int bufB);
330  void check_pending_buffers_(std::unique_lock<std::mutex> const& lock = std::unique_lock<std::mutex>(sequence_id_mutex_));
331 
332  void send_init_frag_();
333  SharedMemoryManager broadcasts_;
334  };
335 }
336 
337 #endif //ARTDAQ_DAQRATE_SHAREDMEMORYEVENTMANAGER_HH
art_config_file wraps a temporary file used to configure art
void RunArt(std::shared_ptr< art_config_file > config_file, pid_t &pid_out)
Run an art instance, recording the return codes and restarting it until the end flag is raised...
size_t GetLockedBufferCount()
Returns the number of buffers currently owned by this manager.
virtual ~SharedMemoryEventManager()
SharedMemoryEventManager Destructor.
The SharedMemoryEventManager is a SharedMemoryManger which tracks events as they are built...
Fragment::sequence_id_t sequence_id_t
Copy Fragment::sequence_id_t into local scope.
void ReconfigureArt(fhicl::ParameterSet art_pset, run_id_t newRun=0, int n_art_processes=-1)
Restart all art processes, using the given fhicl code to configure the new art processes.
The RequestSender contains methods used to send data requests and Routing tokens. ...
pid_t StartArtProcess(fhicl::ParameterSet pset)
Start one art process.
RawDataType * WriteFragmentHeader(detail::RawFragmentHeader frag, bool dropIfNoBuffersAvailable=false)
Get a pointer to a reserved memory area for the given Fragment header.
void setRequestMode(detail::RequestMessageMode mode)
Set the RequestMessageMode for all outgoing data requests.
size_t GetArtEventCount()
Returns the number of events sent to art this subrun.
RawEvent::run_id_t run_id_t
Copy RawEvent::run_id_t into local scope.
RawDataType * GetDroppedDataAddress()
Gets the address of the &quot;dropped data&quot; fragment. Used for testing.
size_t GetFragmentCount(Fragment::sequence_id_t seqID, Fragment::type_t type=Fragment::InvalidFragmentType)
Get the count of Fragments of a given type in an event.
size_t GetPendingEventCount()
Returns the number of events which are complete but waiting on lower sequenced events to finish...
void StartArt()
Start all the art processes.
run_id_t runID() const
Get the current Run number.
void ShutdownArtProcesses(std::set< pid_t > pids)
Shutdown a set of art processes.
void SetInitFragment(FragmentPtr frag)
Set the stored Init fragment, if one has not yet been set already.
subrun_id_t subrunID() const
Get the current subrun number.
void SetRequestMode(detail::RequestMessageMode mode)
Set the mode for RequestMessages. Used to indicate when RequestSender should enter &quot;EndOfRun&quot; mode...
std::map< sequence_id_t, RawEvent_ptr > EventMap
An EventMap is a map of RawEvent_ptr objects, keyed by sequence ID.
RequestMessageMode
Mode used to indicate current run conditions to the request receiver.
void sendMetrics()
Send metrics to the MetricManager, if one has been instantiated in the application.
void startSubrun()
Start a new Subrun, incrementing the subrun number.
size_t GetIncompleteEventCount()
Returns the number of buffers which contain data but are not yet complete.
SharedMemoryEventManager(fhicl::ParameterSet pset, fhicl::ParameterSet art_pset)
SharedMemoryEventManager Constructor.
bool endSubrun()
Send an EndOfSubRunFragment to the art thread.
bool endRun()
Send an EndOfRunFragment to the art thread.
void setOverwrite(bool overwrite)
Set the overwrite flag (non-reliable data transfer) for the Shared Memory.
std::string getFileName() const
Get the path of the temporary file.
void DoneWritingFragment(detail::RawFragmentHeader frag)
Used to indicate that the given Fragment is now completely in the buffer. Will check for buffer compl...
uint32_t GetBroadcastKey()
Gets the shared memory key of the broadcast SharedMemoryManager.
bool endOfData()
Indicate that the end of input has been reached to the art processes.
RawEvent::subrun_id_t subrun_id_t
Copy RawEvent::subrun_id_t into local scope.
art_config_file(fhicl::ParameterSet ps)
art_config_file Constructor
void startRun(run_id_t runID)
Start a Run.
size_t GetFragmentCountInBuffer(int buffer, Fragment::type_t type=Fragment::InvalidFragmentType)
Get the count of Fragments of a given type in a buffer.