1 #ifndef ARTDAQ_DAQRATE_SHAREDMEMORYEVENTMANAGER_HH
2 #define ARTDAQ_DAQRATE_SHAREDMEMORYEVENTMANAGER_HH
4 #include "TRACE/tracemf.h"
5 #include "artdaq-core/Data/Fragment.hh"
7 #include "artdaq-core/Core/SharedMemoryManager.hh"
8 #include "artdaq-core/Data/RawEvent.hh"
9 #include "artdaq-core/Utilities/configureMessageFacility.hh"
10 #include "artdaq/DAQrate/StatisticsHelper.hh"
11 #include "artdaq/DAQrate/detail/RequestSender.hh"
12 #include "artdaq/DAQrate/detail/TokenSender.hh"
14 #include "fhiclcpp/types/Atom.h"
15 #include "fhiclcpp/types/Comment.h"
16 #include "fhiclcpp/types/ConfigurationTable.h"
17 #include "fhiclcpp/types/OptionalTable.h"
18 #include "fhiclcpp/types/TableFragment.h"
20 #define ART_SUPPORTS_DUPLICATE_EVENTS 0
42 explicit art_config_file(fhicl::ParameterSet ps, uint32_t shm_key = 0, uint32_t broadcast_key = 0)
43 : dir_name_(
"/tmp/partition_" + std::to_string(GetPartitionNumber()))
44 , file_name_(dir_name_ +
"/artConfig_" + std::to_string(my_rank) +
"_" + std::to_string(artdaq::TimeUtils::gettimeofday_us()) +
".fcl")
46 mkdir(dir_name_.c_str(), 0777);
48 std::ofstream of(file_name_, std::ofstream::trunc);
52 dir_name_ =
"/tmp/partition_" + std::to_string(GetPartitionNumber()) +
"_" + std::to_string(getuid());
53 mkdir(dir_name_.c_str(), 0777);
54 file_name_ = dir_name_ +
"/artConfig_" + std::to_string(my_rank) +
"_" + std::to_string(artdaq::TimeUtils::gettimeofday_us()) +
".fcl";
56 of.open(file_name_, std::ofstream::trunc);
59 TLOG(TLVL_ERROR,
"ArtConfigFile") <<
"Failed to open configuration file after two attemps! ABORTING!";
65 if (!ps.has_key(
"services") || !ps.has_key(
"services.message"))
67 of <<
" services.message: { " << generateMessageFacilityConfiguration(mf::GetApplicationName().c_str(),
true,
false,
"-art") <<
"} ";
70 TLOG(TLVL_INFO,
"ArtConfigFile") <<
"Inserting Shared memory keys (0x" << std::hex << shm_key <<
", 0x" << std::hex << broadcast_key <<
") into source config";
71 if (shm_key > 0) of <<
" source.shared_memory_key: 0x" << std::hex << shm_key;
72 if (broadcast_key > 0) of <<
" source.broadcast_shared_memory_key: 0x" << std::hex << broadcast_key;
79 remove(file_name_.c_str());
80 rmdir(dir_name_.c_str());
94 std::string dir_name_;
95 std::string file_name_;
110 typedef std::map<sequence_id_t, RawEvent_ptr>
EventMap;
119 fhicl::Atom<size_t>
max_event_size_bytes{fhicl::Name{
"max_event_size_bytes"}, fhicl::Comment{
"Maximum event size (all Fragments), in bytes"}};
121 fhicl::Atom<size_t>
stale_buffer_timeout_usec{fhicl::Name{
"stale_buffer_timeout_usec"}, fhicl::Comment{
"Maximum amount of time elapsed before a buffer is marked as abandoned. Time is reset each time an operation is performed on the buffer."}, 5000000};
123 fhicl::Atom<bool>
overwrite_mode{fhicl::Name{
"overwrite_mode"}, fhicl::Comment{
"Whether buffers are allowed to be overwritten when safe (state == Full or Reading)"},
false};
125 fhicl::Atom<bool>
restart_crashed_art_processes{fhicl::Name{
"restart_crashed_art_processes"}, fhicl::Comment{
"Whether to automatically restart art processes that fail for any reason"},
true};
127 fhicl::Atom<uint32_t>
shared_memory_key{fhicl::Name{
"shared_memory_key"}, fhicl::Comment{
"Key to use for shared memory access"}, 0xBEE70000 + getpid()};
129 fhicl::Atom<size_t>
buffer_count{fhicl::Name{
"buffer_count"}, fhicl::Comment{
"Number of events in the Shared Memory (incomplete + pending art)"}};
132 fhicl::Atom<size_t>
max_fragment_size_bytes{fhicl::Name{
"max_fragment_size_bytes"}, fhicl::Comment{
" Maximum Fragment size, in bytes"}};
134 fhicl::Atom<size_t>
event_queue_wait_time{fhicl::Name{
"event_queue_wait_time"}, fhicl::Comment{
"Amount of time (in seconds) an event can exist in shared memory before being released to art. Used as input to default parameter of \"stale_buffer_timeout_usec\"."}, 5};
136 fhicl::Atom<bool>
broadcast_mode{fhicl::Name{
"broadcast_mode"}, fhicl::Comment{
"When true, buffers are not marked Empty when read, but return to Full state. Buffers are overwritten in order received."},
false};
138 fhicl::Atom<size_t>
art_analyzer_count{fhicl::Name{
"art_analyzer_count"}, fhicl::Comment{
"Number of art procceses to start"}, 1};
140 fhicl::Atom<size_t>
expected_fragments_per_event{fhicl::Name{
"expected_fragments_per_event"}, fhicl::Comment{
"Number of Fragments to expect per event"}};
142 fhicl::Atom<int>
maximum_oversize_fragment_count{fhicl::Name{
"maximum_oversize_fragment_count"}, fhicl::Comment{
"Maximum number of over-size Fragments to drop before throwing an exception. Default is 1, which means to throw an exception if any over-size Fragments are dropped. Set to 0 to disable."}, 1};
144 fhicl::Atom<bool>
update_run_ids_on_new_fragment{fhicl::Name{
"update_run_ids_on_new_fragment"}, fhicl::Comment{
"Whether the run and subrun ID of an event should be updated whenever a Fragment is added."},
true};
146 fhicl::Atom<bool>
use_sequence_id_for_event_number{fhicl::Name{
"use_sequence_id_for_event_number"}, fhicl::Comment{
"Whether to use the artdaq Sequence ID (true) or the Timestamp (false) for art Event numbers"},
true};
148 fhicl::Atom<size_t>
max_subrun_lookup_table_size{fhicl::Name{
"max_subrun_lookup_table_size"}, fhicl::Comment{
"The maximum number of entries to store in the sequence ID-SubRun ID lookup table"}, 100};
150 fhicl::Atom<size_t>
max_event_list_length{fhicl::Name{
"max_event_list_length"}, fhicl::Comment{
" The maximum number of entries to store in the released events list"}, 100};
152 fhicl::Atom<bool>
send_init_fragments{fhicl::Name{
"send_init_fragments"}, fhicl::Comment{
"Whether Init Fragments are expected to be sent to art. If true, a Warning message is printed when an Init Fragment is requested but none are available."},
true};
154 fhicl::Atom<int>
open_event_report_interval_ms{fhicl::Name{
"open_event_report_interval_ms"}, fhicl::Comment{
"Interval at which an open event report should be written"}, -1};
157 fhicl::Atom<int>
fragment_broadcast_timeout_ms{fhicl::Name{
"fragment_broadcast_timeout_ms"}, fhicl::Comment{
"Amount of time broadcast fragments should live in the broadcast shared memory segment"}, 3000};
159 fhicl::Atom<std::string>
art_command_line{fhicl::Name{
"art_command_line"}, fhicl::Comment{
"Command line used to start analysis processes. Supports two special sequences: #CONFIG_FILE# will be replaced with the fhicl config file. #PROCESS_INDEX# will be replaced by the index of the art process."},
"art -c #CONFIG_FILE#"};
161 fhicl::Atom<size_t>
art_index_offset{fhicl::Name{
"art_index_offset"}, fhicl::Comment{
"Offset to add to art process index when replacing #PROCESS_INDEX#"}, 0};
163 fhicl::Atom<double>
minimum_art_lifetime_s{fhicl::Name{
"minimum_art_lifetime_s"}, fhicl::Comment{
"Amount of time that an art process should run to not be considered \"DOA\""}, 2.0};
166 fhicl::Atom<size_t>
expected_art_event_processing_time_us{fhicl::Name{
"expected_art_event_processing_time_us"}, fhicl::Comment{
"During shutdown, SMEM will wait for this amount of time while it is checking that the art threads are done reading buffers."}, 100000};
170 fhicl::Atom<size_t>
broadcast_buffer_count{fhicl::Name{
"broadcast_buffer_count"}, fhicl::Comment{
"Buffers in the broadcast shared memory segment"}, 10};
172 fhicl::Atom<size_t>
broadcast_buffer_size{fhicl::Name{
"broadcast_buffer_size"}, fhicl::Comment{
"Size of the buffers in the broadcast shared memory segment"}, 0x100000};
174 fhicl::Atom<bool>
use_art{fhicl::Name{
"use_art"}, fhicl::Comment{
"Whether to start and manage art threads (Sets art_analyzer count to 0 and overwrite_mode to true when false)"},
true};
176 fhicl::Atom<bool>
manual_art{fhicl::Name{
"manual_art"}, fhicl::Comment{
"Prints the startup command line for the art process so that the user may (for example) run it in GDB or valgrind"},
false};
180 fhicl::OptionalTable<artdaq::TokenSender::Config>
tokenSenderConfig{fhicl::Name{
"routing_token_config"}, fhicl::Comment{
"Configuration for the Routing TokenSender"}};
203 bool AddFragment(detail::RawFragmentHeader frag,
void* dataPtr);
213 bool AddFragment(FragmentPtr frag,
size_t timeout_usec, FragmentPtr& outfrag);
221 RawDataType*
WriteFragmentHeader(detail::RawFragmentHeader frag,
bool dropIfNoBuffersAvailable =
false);
259 size_t GetFragmentCount(Fragment::sequence_id_t seqID, Fragment::type_t type = Fragment::InvalidFragmentType);
269 void UpdateFragmentHeader(
int buffer, detail::RawFragmentHeader hdr);
274 void RunArt(
const std::shared_ptr<art_config_file>& config_file,
size_t process_index,
const std::shared_ptr<std::atomic<pid_t>>& pid_out);
354 if (requests_) requests_->SetRequestMode(mode);
381 for (
auto it = dropped_data_.begin(); it != dropped_data_.end(); ++it)
383 if (it->first == frag)
385 return it->second->dataBegin();
426 size_t get_art_process_count_()
428 std::unique_lock<std::mutex> lk(art_process_mutex_);
429 return art_processes_.size();
432 std::string buildStatisticsString_()
const;
435 size_t num_art_processes_;
436 size_t const num_fragments_per_event_;
437 size_t const queue_size_;
440 std::map<sequence_id_t, subrun_id_t> subrun_event_map_;
441 size_t max_subrun_event_map_length_;
442 static std::mutex subrun_event_map_mutex_;
444 std::set<int> active_buffers_;
445 std::set<int> pending_buffers_;
446 std::unordered_map<Fragment::sequence_id_t, size_t> released_incomplete_events_;
447 std::set<Fragment::sequence_id_t> released_events_;
448 size_t max_event_list_length_;
450 bool update_run_ids_;
451 bool use_sequence_id_for_event_number_;
452 bool overwrite_mode_;
453 size_t init_fragment_count_;
454 std::atomic<bool> running_;
456 std::unordered_map<int, std::atomic<int>> buffer_writes_pending_;
457 std::unordered_map<int, std::mutex> buffer_mutexes_;
458 static std::mutex sequence_id_mutex_;
460 int open_event_report_interval_ms_;
461 std::chrono::steady_clock::time_point last_open_event_report_time_;
462 std::chrono::steady_clock::time_point last_backpressure_report_time_;
463 std::chrono::steady_clock::time_point last_fragment_header_write_time_;
464 std::vector<std::chrono::steady_clock::time_point> event_timing_;
466 StatisticsHelper statsHelper_;
468 int broadcast_timeout_ms_;
470 std::atomic<int> run_event_count_;
471 std::atomic<int> run_incomplete_event_count_;
472 std::atomic<int> subrun_event_count_;
473 std::atomic<int> subrun_incomplete_event_count_;
474 std::atomic<int> oversize_fragment_count_;
475 int maximum_oversize_fragment_count_;
477 mutable std::mutex art_process_mutex_;
478 std::set<pid_t> art_processes_;
479 std::atomic<bool> restart_art_;
480 bool always_restart_art_;
481 std::atomic<bool> manual_art_;
482 fhicl::ParameterSet current_art_pset_;
483 std::shared_ptr<art_config_file> current_art_config_file_;
484 std::string art_cmdline_;
485 size_t art_process_index_offset_;
486 double minimum_art_lifetime_s_;
487 size_t art_event_processing_time_us_;
489 std::unique_ptr<RequestSender> requests_;
490 std::unique_ptr<TokenSender> tokens_;
491 fhicl::ParameterSet data_pset_;
493 FragmentPtrs init_fragments_;
494 std::set<Fragment::fragment_id_t> received_init_frags_;
495 std::list<std::pair<detail::RawFragmentHeader, FragmentPtr>> dropped_data_;
497 bool broadcastFragments_(FragmentPtrs& frags);
499 detail::RawEventHeader* getEventHeader_(
int buffer);
501 int getBufferForSequenceID_(Fragment::sequence_id_t seqID,
bool create_new, Fragment::timestamp_t timestamp = Fragment::InvalidTimestamp);
502 bool hasFragments_(
int buffer);
503 void complete_buffer_(
int buffer);
504 bool bufferComparator(
int bufA,
int bufB);
505 void check_pending_buffers_(std::unique_lock<std::mutex>
const& lock);
506 std::vector<char*> parse_art_command_line_(
const std::shared_ptr<art_config_file>& config_file,
size_t process_index);
508 void send_init_frags_();
509 SharedMemoryManager broadcasts_;
513 #endif // ARTDAQ_DAQRATE_SHAREDMEMORYEVENTMANAGER_HH
fhicl::Atom< size_t > event_queue_wait_time
"event_queue_wait_time" (Default: 5) : Amount of time(in seconds) an event can exist in shared memory...
fhicl::Atom< bool > overwrite_mode
"overwite_mode" (Default: false): Whether new data is allowed to overwrite buffers in the "Full" stat...
art_config_file wraps a temporary file used to configure art
void AddInitFragment(FragmentPtr &frag)
Set the stored Init fragment, if one has not yet been set already.
void ShutdownArtProcesses(std::set< pid_t > &pids)
Shutdown a set of art processes.
size_t GetLockedBufferCount()
Returns the number of buffers currently owned by this manager.
virtual ~SharedMemoryEventManager()
SharedMemoryEventManager Destructor.
The SharedMemoryEventManager is a SharedMemoryManger which tracks events as they are built...
Fragment::sequence_id_t sequence_id_t
Copy Fragment::sequence_id_t into local scope.
fhicl::Atom< size_t > broadcast_buffer_size
"broadcast_buffer_size" (Default: 0x100000): Size of the buffers in the broadcast shared memory segme...
subrun_id_t GetCurrentSubrun()
Get the current subrun number (Gets the last defined subrun)
fhicl::Atom< bool > restart_crashed_art_processes
"restart_crashed_art_processes" (Default: true) : Whether to automatically restart art processes that...
void ReconfigureArt(fhicl::ParameterSet art_pset, run_id_t newRun=0, int n_art_processes=-1)
Restart all art processes, using the given fhicl code to configure the new art processes.
fhicl::Atom< double > minimum_art_lifetime_s
"minimum_art_lifetime_s" (Default: 2 seconds): Amount of time that an art process should run to not b...
fhicl::Atom< int > maximum_oversize_fragment_count
"maximum_oversize_fragment_count" (Default: 1): Maximum number of over-size Fragments to drop before ...
fhicl::Atom< bool > update_run_ids_on_new_fragment
"update_run_ids_on_new_fragment" (Default: true) : Whether the run and subrun ID of an event should b...
RawDataType * WriteFragmentHeader(detail::RawFragmentHeader frag, bool dropIfNoBuffersAvailable=false)
Get a pointer to a reserved memory area for the given Fragment header.
void setRequestMode(detail::RequestMessageMode mode)
Set the RequestMessageMode for all outgoing data requests.
size_t GetArtEventCount()
Returns the number of events sent to art this run.
fhicl::Atom< std::string > art_command_line
"art_command_line" (Default: "art -c \#CONFIG_FILE\#"): Command line used to start analysis processes...
fhicl::Atom< size_t > art_analyzer_count
"art_analyzer_count" (Default: 1) : Number of art procceses to start
Configuration of the SharedMemoryEventManager. May be used for parameter validation ...
fhicl::Atom< int > open_event_report_interval_ms
"open_event_report_interval_ms" (Default: -1): Interval at which an open event report should be writt...
fhicl::Atom< bool > use_sequence_id_for_event_number
"use_sequence_id_for_event_number" (Default: true): Whether to use the artdaq Sequence ID (true) or t...
fhicl::Atom< uint32_t > broadcast_shared_memory_key
"broadcast_shared_memory_key" (Default: 0xCEE7000 + PID): Key to use for broadcast shared memory acce...
RawEvent::run_id_t run_id_t
Copy RawEvent::run_id_t into local scope.
fhicl::Atom< size_t > art_index_offset
"art_index_offset" (Default: 0): Offset to add to art process index when replacing #PROCESS_INDEX# ...
fhicl::WrappedTable< Config > Parameters
Used for ParameterSet validation (if desired)
size_t GetFragmentCount(Fragment::sequence_id_t seqID, Fragment::type_t type=Fragment::InvalidFragmentType)
Get the count of Fragments of a given type in an event.
void UpdateArtConfiguration(fhicl::ParameterSet art_pset)
Updates the internally-stored copy of the art configuration.
pid_t StartArtProcess(fhicl::ParameterSet pset, size_t process_index)
Start one art process.
fhicl::Atom< size_t > max_subrun_lookup_table_size
"max_subrun_lookup_table_size" (Default: 100): The maximum number of entries to store in the sequence...
size_t GetPendingEventCount()
Returns the number of events which are complete but waiting on lower sequenced events to finish...
void StartArt()
Start all the art processes.
run_id_t runID() const
Get the current Run number.
fhicl::Atom< size_t > max_fragment_size_bytes
fhicl::Atom< size_t > buffer_count
"buffer_count" REQUIRED: Number of events in the Shared Memory(incomplete + pending art) ...
subrun_id_t GetSubrunForSequenceID(Fragment::sequence_id_t seqID)
Get the subrun number that the given Sequence ID would be assigned to.
fhicl::TableFragment< artdaq::RequestSender::Config > requestSenderConfig
Configuration of the RequestSender. See artdaq::RequestSender::Config.
SharedMemoryEventManager(const fhicl::ParameterSet &pset, fhicl::ParameterSet art_pset)
SharedMemoryEventManager Constructor.
fhicl::Atom< size_t > expected_art_event_processing_time_us
fhicl::Atom< uint32_t > shared_memory_key
"shared_memory_key" (Default 0xBEE70000 + PID) : Key to use for shared memory access ...
RawDataType * GetDroppedDataAddress(detail::RawFragmentHeader frag)
Gets the address of the "dropped data" fragment. Used for testing.
std::map< sequence_id_t, RawEvent_ptr > EventMap
An EventMap is a map of RawEvent_ptr objects, keyed by sequence ID.
RequestMessageMode
Mode used to indicate current run conditions to the request receiver.
fhicl::Atom< size_t > max_event_list_length
"max_event_list_length" (Default: 100): The maximum number of entries to store in the released events...
size_t GetOpenEventCount()
Returns the number of buffers which contain data but are not yet complete.
void RunArt(const std::shared_ptr< art_config_file > &config_file, size_t process_index, const std::shared_ptr< std::atomic< pid_t >> &pid_out)
Run an art instance, recording the return codes and restarting it until the end flag is raised...
void rolloverSubrun()
Add a subrun transition immediately after the highest currently define sequence ID.
void sendMetrics()
Send metrics to the MetricManager, if one has been instantiated in the application.
fhicl::Atom< size_t > stale_buffer_timeout_usec
"stale_buffer_timeout_usec" (Default: event_queue_wait_time * 1, 000, 000) : Maximum amount of time e...
art_config_file(fhicl::ParameterSet ps, uint32_t shm_key=0, uint32_t broadcast_key=0)
art_config_file Constructor
fhicl::Atom< size_t > expected_fragments_per_event
"expected_fragments_per_event" (REQUIRED) : Number of Fragments to expect per event ...
fhicl::Atom< bool > send_init_fragments
"send_init_fragments" (Default: true): Whether Init Fragments are expected to be sent to art...
static const std::string FRAGMENTS_RECEIVED_STAT_KEY
Key for Fragments Received MonitoredQuantity.
bool endRun()
Send an EndOfRunFragment to the art thread.
fhicl::Atom< size_t > max_event_size_bytes
fhicl::OptionalTable< artdaq::TokenSender::Config > tokenSenderConfig
Configuration of the TokenSender. See artdaq::TokenSender::Config.
void setOverwrite(bool overwrite)
Set the overwrite flag (non-reliable data transfer) for the Shared Memory.
std::string getFileName() const
Get the path of the temporary file.
fhicl::Atom< bool > manual_art
"manual_art" (Default: false): Prints the startup command line for the art process so that the user m...
fhicl::Atom< bool > broadcast_mode
"broadcast_mode" (Default: false) : When true, buffers are not marked Empty when read, but return to Full state.Buffers are overwritten in order received.
fhicl::Atom< size_t > broadcast_buffer_count
"broadcast_buffer_count" (Default: 10): Buffers in the broadcast shared memory segment ...
void DoneWritingFragment(detail::RawFragmentHeader frag)
Used to indicate that the given Fragment is now completely in the buffer. Will check for buffer compl...
static const std::string EVENTS_RELEASED_STAT_KEY
Key for the Events Released MonitoredQuantity.
uint32_t GetBroadcastKey()
Gets the shared memory key of the broadcast SharedMemoryManager.
bool endOfData()
Indicate that the end of input has been reached to the art processes.
fhicl::Atom< bool > use_art
"use_art" (Default: true): Whether to start and manage art threads (Sets art_analyzer count to 0 and ...
RawEvent::subrun_id_t subrun_id_t
Copy RawEvent::subrun_id_t into local scope.
fhicl::Atom< int > fragment_broadcast_timeout_ms
void startRun(run_id_t runID)
Start a Run.
size_t GetFragmentCountInBuffer(int buffer, Fragment::type_t type=Fragment::InvalidFragmentType)
Get the count of Fragments of a given type in a buffer.
void CheckPendingBuffers()
Check for buffers which are ready to be marked incomplete and released to art and issue tokens for an...