1 #ifndef ARTDAQ_DAQRATE_SHAREDMEMORYEVENTMANAGER_HH
2 #define ARTDAQ_DAQRATE_SHAREDMEMORYEVENTMANAGER_HH
4 #include "artdaq/DAQdata/Globals.hh"
5 #include "artdaq-core/Core/SharedMemoryManager.hh"
6 #include "artdaq-core/Data/RawEvent.hh"
7 #include "artdaq/DAQrate/RequestSender.hh"
13 #include "fhiclcpp/fwd.h"
14 #include "artdaq/Application/StatisticsHelper.hh"
15 #include "artdaq/DAQrate/detail/ArtConfig.hh"
16 #define ART_SUPPORTS_DUPLICATE_EVENTS 0
31 : dir_name_(
"/tmp/partition_" + std::to_string(
Globals::GetPartitionNumber()))
32 , file_name_(dir_name_ +
"/artConfig_" + std::to_string(my_rank) +
"_" + std::to_string(artdaq::TimeUtils::gettimeofday_us()) +
".fcl")
34 mkdir(dir_name_.c_str(), 0777);
36 std::ofstream of(file_name_, std::ofstream::trunc);
45 if (!ps.has_key(
"services.message"))
47 of <<
" services.message: { " << generateMessageFacilityConfiguration(
"art") <<
"} ";
56 remove(file_name_.c_str());
57 rmdir(dir_name_.c_str());
65 std::string dir_name_;
66 std::string file_name_;
78 typedef std::map<sequence_id_t, RawEvent_ptr>
EventMap;
87 fhicl::Atom<size_t>
max_event_size_bytes{ fhicl::Name{
"max_event_size_bytes"}, fhicl::Comment{
"Maximum event size (all Fragments), in bytes"} };
89 fhicl::Atom<size_t>
stale_buffer_timeout_usec{ fhicl::Name{
"stale_buffer_timeout_usec"}, fhicl::Comment{
"Maximum amount of time elapsed before a buffer is marked as abandoned. Time is reset each time an operation is performed on the buffer."}, 5000000 };
91 fhicl::Atom<bool>
overwrite_mode{ fhicl::Name{
"overwrite_mode"}, fhicl::Comment{
"Whether buffers are allowed to be overwritten when safe (state == Full or Reading)"},
false };
93 fhicl::Atom<bool>
restart_crashed_art_processes{ fhicl::Name{
"restart_crashed_art_processes"}, fhicl::Comment{
"Whether to automatically restart art processes that fail for any reason"},
true };
95 fhicl::Atom<uint32_t>
shared_memory_key{ fhicl::Name{
"shared_memory_key"}, fhicl::Comment{
"Key to use for shared memory access"}, 0xBEE70000 + getpid() };
97 fhicl::Atom<size_t>
buffer_count{ fhicl::Name{
"buffer_count"}, fhicl::Comment{
"Number of events in the Shared Memory (incomplete + pending art)"} };
100 fhicl::Atom<size_t>
max_fragment_size_bytes{ fhicl::Name{
"max_fragment_size_bytes"}, fhicl::Comment{
" Maximum Fragment size, in bytes"} };
102 fhicl::Atom<size_t>
event_queue_wait_time{ fhicl::Name{
"event_queue_wait_time"}, fhicl::Comment{
"Amount of time (in seconds) an event can exist in shared memory before being released to art. Used as input to default parameter of \"stale_buffer_timeout_usec\"."}, 5 };
104 fhicl::Atom<bool>
broadcast_mode{ fhicl::Name{
"broadcast_mode"}, fhicl::Comment{
"When true, buffers are not marked Empty when read, but return to Full state. Buffers are overwritten in order received."},
false };
106 fhicl::Atom<size_t>
art_analyzer_count{ fhicl::Name{
"art_analyzer_count"}, fhicl::Comment{
"Number of art procceses to start"}, 1 };
108 fhicl::Atom<size_t>
expected_fragments_per_event{ fhicl::Name{
"expected_fragments_per_event"}, fhicl::Comment{
"Number of Fragments to expect per event"} };
110 fhicl::Atom<int>
maximum_oversize_fragment_count{ fhicl::Name{
"maximum_oversize_fragment_count"}, fhicl::Comment{
"Maximum number of over-size Fragments to drop before throwing an exception. Default is 1, which means to throw an exception if any over-size Fragments are dropped. Set to 0 to disable."},1 };
112 fhicl::Atom<bool>
update_run_ids_on_new_fragment{ fhicl::Name{
"update_run_ids_on_new_fragment"}, fhicl::Comment{
"Whether the run and subrun ID of an event should be updated whenever a Fragment is added."},
true };
114 fhicl::Atom<bool>
use_sequence_id_for_event_number{ fhicl::Name{
"use_sequence_id_for_event_number"}, fhicl::Comment{
"Whether to use the artdaq Sequence ID (true) or the Timestamp (false) for art Event numbers"},
true };
116 fhicl::Atom<bool>
send_init_fragments{ fhicl::Name{
"send_init_fragments"}, fhicl::Comment{
"Whether Init Fragments are expected to be sent to art. If true, a Warning message is printed when an Init Fragment is requested but none are available."},
true };
118 fhicl::Atom<int>
incomplete_event_report_interval_ms{ fhicl::Name{
"incomplete_event_report_interval_ms"}, fhicl::Comment{
"Interval at which an incomplete event report should be written"}, -1 };
121 fhicl::Atom<int>
fragment_broadcast_timeout_ms{ fhicl::Name{
"fragment_broadcast_timeout_ms"}, fhicl::Comment{
"Amount of time broadcast fragments should live in the broadcast shared memory segment"}, 3000 };
123 fhicl::Atom<double>
minimum_art_lifetime_s{ fhicl::Name{
"minimum_art_lifetime_s"}, fhicl::Comment{
"Amount of time that an art process should run to not be considered \"DOA\""}, 2.0 };
126 fhicl::Atom<size_t>
expected_art_event_processing_time_us{ fhicl::Name{
"expected_art_event_processing_time_us"}, fhicl::Comment{
"During shutdown, SMEM will wait for this amount of time while it is checking that the art threads are done reading buffers."}, 100000 };
130 fhicl::Atom<size_t>
broadcast_buffer_count{ fhicl::Name{
"broadcast_buffer_count"}, fhicl::Comment{
"Buffers in the broadcast shared memory segment"}, 10 };
132 fhicl::Atom<size_t>
broadcast_buffer_size{ fhicl::Name{
"broadcast_buffer_size"}, fhicl::Comment{
"Size of the buffers in the broadcast shared memory segment"}, 0x100000 };
134 fhicl::Atom<bool>
use_art{ fhicl::Name{
"use_art"}, fhicl::Comment{
"Whether to start and manage art threads (Sets art_analyzer count to 0 and overwrite_mode to true when false)"},
true };
136 fhicl::Atom<bool>
manual_art{ fhicl::Name{
"manual_art"}, fhicl::Comment{
"Prints the startup command line for the art process so that the user may (for example) run it in GDB or valgrind"},
false };
140 #if MESSAGEFACILITY_HEX_VERSION >= 0x20103
141 using Parameters = fhicl::WrappedTable<Config>;
162 bool AddFragment(detail::RawFragmentHeader frag,
void* dataPtr);
172 bool AddFragment(FragmentPtr frag,
size_t timeout_usec, FragmentPtr& outfrag);
180 RawDataType*
WriteFragmentHeader(detail::RawFragmentHeader frag,
bool dropIfNoBuffersAvailable =
false);
218 size_t GetFragmentCount(Fragment::sequence_id_t seqID, Fragment::type_t type = Fragment::InvalidFragmentType);
231 void RunArt(std::shared_ptr<art_config_file> config_file, pid_t& pid_out);
345 size_t num_art_processes_;
346 size_t const num_fragments_per_event_;
347 size_t const queue_size_;
353 std::set<int> active_buffers_;
354 std::set<int> pending_buffers_;
355 std::unordered_map<Fragment::sequence_id_t, size_t> released_incomplete_events_;
357 bool update_run_ids_;
358 bool use_sequence_id_for_event_number_;
359 bool overwrite_mode_;
360 bool send_init_fragments_;
363 std::unordered_map<int, std::atomic<int>> buffer_writes_pending_;
364 std::unordered_map<int, std::mutex> buffer_mutexes_;
365 static std::mutex sequence_id_mutex_;
367 int incomplete_event_report_interval_ms_;
368 std::chrono::steady_clock::time_point last_incomplete_event_report_time_;
369 std::chrono::steady_clock::time_point last_shmem_buffer_metric_update_;
370 int broadcast_timeout_ms_;
372 std::atomic<int> run_event_count_;
373 std::atomic<int> run_incomplete_event_count_;
374 std::atomic<int> subrun_event_count_;
375 std::atomic<int> subrun_incomplete_event_count_;
376 std::atomic<int> oversize_fragment_count_;
377 int maximum_oversize_fragment_count_;
379 std::set<pid_t> art_processes_;
380 std::atomic<bool> restart_art_;
381 bool always_restart_art_;
383 fhicl::ParameterSet current_art_pset_;
384 std::shared_ptr<art_config_file> current_art_config_file_;
385 double minimum_art_lifetime_s_;
386 size_t art_event_processing_time_us_;
388 std::unique_ptr<RequestSender> requests_;
389 fhicl::ParameterSet data_pset_;
391 FragmentPtr init_fragment_;
392 FragmentPtr dropped_data_;
394 bool broadcastFragment_(FragmentPtr frag, FragmentPtr& outFrag);
396 detail::RawEventHeader* getEventHeader_(
int buffer);
398 int getBufferForSequenceID_(Fragment::sequence_id_t seqID,
bool create_new, Fragment::timestamp_t timestamp = Fragment::InvalidTimestamp);
399 bool hasFragments_(
int buffer);
400 void complete_buffer_(
int buffer);
401 bool bufferComparator(
int bufA,
int bufB);
402 void check_pending_buffers_(std::unique_lock<std::mutex>
const& lock = std::unique_lock<std::mutex>(sequence_id_mutex_));
404 void send_init_frag_();
405 SharedMemoryManager broadcasts_;
409 #endif //ARTDAQ_DAQRATE_SHAREDMEMORYEVENTMANAGER_HH
fhicl::Atom< size_t > event_queue_wait_time
"event_queue_wait_time" (Default: 5) : Amount of time(in seconds) an event can exist in shared memory...
fhicl::Atom< bool > overwrite_mode
"overwite_mode" (Default: false): Whether new data is allowed to overwrite buffers in the "Full" stat...
art_config_file wraps a temporary file used to configure art
void RunArt(std::shared_ptr< art_config_file > config_file, pid_t &pid_out)
Run an art instance, recording the return codes and restarting it until the end flag is raised...
size_t GetLockedBufferCount()
Returns the number of buffers currently owned by this manager.
virtual ~SharedMemoryEventManager()
SharedMemoryEventManager Destructor.
The SharedMemoryEventManager is a SharedMemoryManger which tracks events as they are built...
Fragment::sequence_id_t sequence_id_t
Copy Fragment::sequence_id_t into local scope.
fhicl::Atom< size_t > broadcast_buffer_size
"broadcast_buffer_size" (Default: 0x100000): Size of the buffers in the broadcast shared memory segme...
fhicl::Atom< bool > restart_crashed_art_processes
"restart_crashed_art_processes" (Default: true) : Whether to automatically restart art processes that...
void ReconfigureArt(fhicl::ParameterSet art_pset, run_id_t newRun=0, int n_art_processes=-1)
Restart all art processes, using the given fhicl code to configure the new art processes.
pid_t StartArtProcess(fhicl::ParameterSet pset)
Start one art process.
fhicl::Atom< double > minimum_art_lifetime_s
"minimum_art_lifetime_s" (Default: 2 seconds): Amount of time that an art process should run to not b...
fhicl::Atom< int > maximum_oversize_fragment_count
"maximum_oversize_fragment_count" (Default: 1): Maximum number of over-size Fragments to drop before ...
fhicl::Atom< bool > update_run_ids_on_new_fragment
"update_run_ids_on_new_fragment" (Default: true) : Whether the run and subrun ID of an event should b...
RawDataType * WriteFragmentHeader(detail::RawFragmentHeader frag, bool dropIfNoBuffersAvailable=false)
Get a pointer to a reserved memory area for the given Fragment header.
void setRequestMode(detail::RequestMessageMode mode)
Set the RequestMessageMode for all outgoing data requests.
size_t GetArtEventCount()
Returns the number of events sent to art this subrun.
fhicl::Atom< size_t > art_analyzer_count
"art_analyzer_count" (Default: 1) : Number of art procceses to start
Configuration of the SharedMemoryEventManager. May be used for parameter validation ...
fhicl::Atom< bool > use_sequence_id_for_event_number
"use_sequence_id_for_event_number" (Default: true): Whether to use the artdaq Sequence ID (true) or t...
fhicl::Atom< uint32_t > broadcast_shared_memory_key
"broadcast_shared_memory_key" (Default: 0xCEE7000 + PID): Key to use for broadcast shared memory acce...
RawEvent::run_id_t run_id_t
Copy RawEvent::run_id_t into local scope.
RawDataType * GetDroppedDataAddress()
Gets the address of the "dropped data" fragment. Used for testing.
size_t GetFragmentCount(Fragment::sequence_id_t seqID, Fragment::type_t type=Fragment::InvalidFragmentType)
Get the count of Fragments of a given type in an event.
size_t GetPendingEventCount()
Returns the number of events which are complete but waiting on lower sequenced events to finish...
void StartArt()
Start all the art processes.
run_id_t runID() const
Get the current Run number.
fhicl::Atom< size_t > max_fragment_size_bytes
void ShutdownArtProcesses(std::set< pid_t > pids)
Shutdown a set of art processes.
fhicl::Atom< size_t > buffer_count
"buffer_count" REQUIRED: Number of events in the Shared Memory(incomplete + pending art) ...
void SetInitFragment(FragmentPtr frag)
Set the stored Init fragment, if one has not yet been set already.
subrun_id_t subrunID() const
Get the current subrun number.
fhicl::TableFragment< artdaq::RequestSender::Config > requestSenderConfig
Configuration of the RequestSender. See artdaq::RequestSender::Config.
fhicl::Atom< size_t > expected_art_event_processing_time_us
fhicl::Atom< uint32_t > shared_memory_key
"shared_memory_key" (Default 0xBEE70000 + PID) : Key to use for shared memory access ...
void rolloverSubrun(sequence_id_t boundary)
Rollover the subrun after the specified event.
std::map< sequence_id_t, RawEvent_ptr > EventMap
An EventMap is a map of RawEvent_ptr objects, keyed by sequence ID.
RequestMessageMode
Mode used to indicate current run conditions to the request receiver.
void sendMetrics()
Send metrics to the MetricManager, if one has been instantiated in the application.
fhicl::Atom< size_t > stale_buffer_timeout_usec
"stale_buffer_timeout_usec" (Default: event_queue_wait_time * 1, 000, 000) : Maximum amount of time e...
void startSubrun()
Start a new Subrun, incrementing the subrun number.
fhicl::Atom< size_t > expected_fragments_per_event
"expected_fragments_per_event" (REQUIRED) : Number of Fragments to expect per event ...
size_t GetIncompleteEventCount()
Returns the number of buffers which contain data but are not yet complete.
fhicl::Atom< bool > send_init_fragments
"send_init_fragments" (Default: true): Whether Init Fragments are expected to be sent to art...
SharedMemoryEventManager(fhicl::ParameterSet pset, fhicl::ParameterSet art_pset)
SharedMemoryEventManager Constructor.
bool endSubrun()
Send an EndOfSubRunFragment to the art thread.
bool endRun()
Send an EndOfRunFragment to the art thread.
fhicl::Atom< size_t > max_event_size_bytes
void setOverwrite(bool overwrite)
Set the overwrite flag (non-reliable data transfer) for the Shared Memory.
fhicl::Atom< int > incomplete_event_report_interval_ms
"incomplete_event_report_interval_ms" (Default: -1): Interval at which an incomplete event report sho...
std::string getFileName() const
Get the path of the temporary file.
fhicl::Atom< bool > manual_art
"manual_art" (Default: false): Prints the startup command line for the art process so that the user m...
fhicl::Atom< bool > broadcast_mode
"broadcast_mode" (Default: false) : When true, buffers are not marked Empty when read, but return to Full state.Buffers are overwritten in order received.
The artdaq::Globals class contains several variables which are useful across the entire artdaq system...
fhicl::Atom< size_t > broadcast_buffer_count
"broadcast_buffer_count" (Default: 10): Buffers in the broadcast shared memory segment ...
void DoneWritingFragment(detail::RawFragmentHeader frag)
Used to indicate that the given Fragment is now completely in the buffer. Will check for buffer compl...
uint32_t GetBroadcastKey()
Gets the shared memory key of the broadcast SharedMemoryManager.
bool endOfData()
Indicate that the end of input has been reached to the art processes.
fhicl::Atom< bool > use_art
"use_art" (Default: true): Whether to start and manage art threads (Sets art_analyzer count to 0 and ...
RawEvent::subrun_id_t subrun_id_t
Copy RawEvent::subrun_id_t into local scope.
fhicl::Atom< int > fragment_broadcast_timeout_ms
art_config_file(fhicl::ParameterSet ps)
art_config_file Constructor
void startRun(run_id_t runID)
Start a Run.
size_t GetFragmentCountInBuffer(int buffer, Fragment::type_t type=Fragment::InvalidFragmentType)
Get the count of Fragments of a given type in a buffer.