2 #define TRACE_NAME (app_name + "_SharedMemoryEventManager").c_str()
4 #include "artdaq/DAQrate/SharedMemoryEventManager.hh"
6 #include "artdaq-core/Core/StatisticsCollection.hh"
7 #include "artdaq-core/Utilities/TraceLock.hh"
10 #define TLVL_BUFLCK 41
12 std::mutex artdaq::SharedMemoryEventManager::sequence_id_mutex_;
13 std::mutex artdaq::SharedMemoryEventManager::subrun_event_map_mutex_;
16 : SharedMemoryManager(pset.get<uint32_t>(
"shared_memory_key", 0xBEE70000 + getpid()),
17 pset.get<size_t>(
"buffer_count"),
18 pset.has_key(
"max_event_size_bytes") ? pset.get<size_t>(
"max_event_size_bytes") : pset.get<size_t>(
"expected_fragments_per_event") * pset.get<size_t>(
"max_fragment_size_bytes"),
19 pset.get<size_t>(
"stale_buffer_timeout_usec", pset.get<size_t>(
"event_queue_wait_time", 5) * 1000000),
20 !pset.get<bool>(
"broadcast_mode", false))
21 , num_art_processes_(pset.get<size_t>(
"art_analyzer_count", 1))
22 , num_fragments_per_event_(pset.get<size_t>(
"expected_fragments_per_event"))
23 , queue_size_(pset.get<size_t>(
"buffer_count"))
25 , max_subrun_event_map_length_(pset.get<size_t>(
"max_subrun_lookup_table_size", 100))
26 , update_run_ids_(pset.get<bool>(
"update_run_ids_on_new_fragment", true))
27 , use_sequence_id_for_event_number_(pset.get<bool>(
"use_sequence_id_for_event_number", true))
28 , overwrite_mode_(!pset.get<bool>(
"use_art", true) || pset.get<bool>(
"overwrite_mode", false) || pset.get<bool>(
"broadcast_mode", false))
29 , send_init_fragments_(pset.get<bool>(
"send_init_fragments", true))
31 , buffer_writes_pending_()
32 , incomplete_event_report_interval_ms_(pset.get<int>(
"incomplete_event_report_interval_ms", -1))
33 , last_incomplete_event_report_time_(std::chrono::steady_clock::now())
34 , last_shmem_buffer_metric_update_(std::chrono::steady_clock::now())
35 , last_backpressure_report_time_(std::chrono::steady_clock::now())
36 , last_fragment_header_write_time_(std::chrono::steady_clock::now())
38 , broadcast_timeout_ms_(pset.get<int>(
"fragment_broadcast_timeout_ms", 3000))
40 , run_incomplete_event_count_(0)
41 , subrun_event_count_(0)
42 , subrun_incomplete_event_count_(0)
43 , oversize_fragment_count_(0)
44 , maximum_oversize_fragment_count_(pset.get<int>(
"maximum_oversize_fragment_count", 1))
47 , always_restart_art_(pset.get<bool>(
"restart_crashed_art_processes", true))
48 , manual_art_(pset.get<bool>(
"manual_art", false))
49 , current_art_pset_(art_pset)
50 , minimum_art_lifetime_s_(pset.get<double>(
"minimum_art_lifetime_s", 2.0))
51 , art_event_processing_time_us_(pset.get<size_t>(
"expected_art_event_processing_time_us", 1000000))
55 , broadcasts_(pset.get<uint32_t>(
"broadcast_shared_memory_key", 0xCEE70000 + getpid()),
56 pset.get<size_t>(
"broadcast_buffer_count", 10),
57 pset.get<size_t>(
"broadcast_buffer_size", 0x100000),
58 pset.get<int>(
"expected_art_event_processing_time_us", 100000) * pset.get<size_t>(
"buffer_count"), false)
60 subrun_event_map_[0] = 1;
61 SetMinWriteSize(
sizeof(detail::RawEventHeader) +
sizeof(detail::RawFragmentHeader));
62 broadcasts_.SetMinWriteSize(
sizeof(detail::RawEventHeader) +
sizeof(detail::RawFragmentHeader));
64 if (pset.get<
bool>(
"use_art",
true) ==
false)
66 TLOG(TLVL_INFO) <<
"BEGIN SharedMemoryEventManager CONSTRUCTOR with use_art:false";
67 num_art_processes_ = 0;
71 TLOG(TLVL_INFO) <<
"BEGIN SharedMemoryEventManager CONSTRUCTOR with use_art:true";
72 TLOG(TLVL_TRACE) <<
"art_pset is " << art_pset.to_string();
74 current_art_config_file_ = std::make_shared<art_config_file>(art_pset );
76 if (overwrite_mode_ && num_art_processes_ > 0)
78 TLOG(TLVL_WARNING) <<
"Art is configured to run, but overwrite mode is enabled! Check your configuration if this in unintentional!";
80 else if (overwrite_mode_)
82 TLOG(TLVL_INFO) <<
"Overwrite Mode enabled, no configured art processes at startup";
85 for (
size_t ii = 0; ii < size(); ++ii)
87 buffer_writes_pending_[ii] = 0;
90 if (!IsValid())
throw cet::exception(app_name +
"_SharedMemoryEventManager") <<
"Unable to attach to Shared Memory!";
92 TLOG(TLVL_TRACE) <<
"Setting Writer rank to " << my_rank;
94 TLOG(TLVL_DEBUG) <<
"Writer Rank is " << GetRank();
96 TLOG(TLVL_TRACE) <<
"END CONSTRUCTOR";
101 TLOG(TLVL_TRACE) <<
"DESTRUCTOR";
102 if (running_) endOfData();
103 TLOG(TLVL_TRACE) <<
"Destructor END";
106 bool artdaq::SharedMemoryEventManager::AddFragment(detail::RawFragmentHeader frag,
void* dataPtr)
108 TLOG(TLVL_TRACE) <<
"AddFragment(Header, ptr) BEGIN frag.word_count=" << frag.word_count
109 <<
", sequence_id=" << frag.sequence_id;
110 auto buffer = getBufferForSequenceID_(frag.sequence_id,
true, frag.timestamp);
111 TLOG(TLVL_TRACE) <<
"Using buffer " << buffer <<
" for seqid=" << frag.sequence_id;
112 if (buffer == -1)
return false;
115 TLOG(TLVL_ERROR) <<
"Dropping event because data taking has already passed this event number: " << frag.sequence_id;
119 auto hdr = getEventHeader_(buffer);
122 hdr->run_id = run_id_;
124 hdr->subrun_id = GetSubrunForSequenceID(frag.sequence_id);
126 TLOG(TLVL_TRACE) <<
"AddFragment before Write calls";
127 Write(buffer, dataPtr, frag.word_count *
sizeof(RawDataType));
129 TLOG(TLVL_TRACE) <<
"Checking for complete event";
130 auto fragmentCount = GetFragmentCount(frag.sequence_id);
131 hdr->is_complete = fragmentCount == num_fragments_per_event_ && buffer_writes_pending_[buffer] == 0;
132 TLOG(TLVL_TRACE) <<
"hdr->is_complete=" << std::boolalpha << hdr->is_complete
133 <<
", fragmentCount=" << fragmentCount
134 <<
", num_fragments_per_event=" << num_fragments_per_event_
135 <<
", buffer_writes_pending_[buffer]=" << buffer_writes_pending_[buffer];
137 complete_buffer_(buffer);
138 if (requests_) requests_->SendRequest(
true);
140 TLOG(TLVL_TRACE) <<
"AddFragment END";
144 bool artdaq::SharedMemoryEventManager::AddFragment(FragmentPtr frag,
size_t timeout_usec, FragmentPtr& outfrag)
146 TLOG(TLVL_TRACE) <<
"AddFragment(FragmentPtr) BEGIN";
147 auto hdr = *
reinterpret_cast<detail::RawFragmentHeader*
>(frag->headerAddress());
148 auto data = frag->headerAddress();
149 auto start = std::chrono::steady_clock::now();
151 while (!sts && TimeUtils::GetElapsedTimeMicroseconds(start) < timeout_usec)
153 sts = AddFragment(hdr, data);
154 if (!sts) usleep(1000);
158 outfrag = std::move(frag);
160 TLOG(TLVL_TRACE) <<
"AddFragment(FragmentPtr) RETURN " << std::boolalpha << sts;
166 TLOG(14) <<
"WriteFragmentHeader BEGIN";
167 auto buffer = getBufferForSequenceID_(frag.sequence_id,
true, frag.timestamp);
171 if (buffer == -1 && !dropIfNoBuffersAvailable)
173 std::unique_lock<std::mutex> bp_lk(sequence_id_mutex_);
174 if (TimeUtils::GetElapsedTime(last_backpressure_report_time_) > 1.0)
176 TLOG(TLVL_WARNING) << app_name <<
": Back-pressure condition: All Shared Memory buffers have been full for " << TimeUtils::GetElapsedTime(last_fragment_header_write_time_) <<
" s!";
177 last_backpressure_report_time_ = std::chrono::steady_clock::now();
183 TLOG(TLVL_ERROR) <<
"Dropping fragment with sequence id " << frag.sequence_id <<
" and fragment id " << frag.fragment_id <<
" because data taking has already passed this event.";
187 TLOG(TLVL_ERROR) <<
"Dropping fragment with sequence id " << frag.sequence_id <<
" and fragment id " << frag.fragment_id <<
" because there is no room in the queue and reliable mode is off.";
189 dropped_data_[frag.fragment_id].reset(
new Fragment(frag.word_count - frag.num_words()));
191 TLOG(6) <<
"Dropping fragment with sequence id " << frag.sequence_id <<
" and fragment id " << frag.fragment_id <<
" into " << (
void*)dropped_data_[frag.fragment_id]->dataBegin() <<
" sz=" << dropped_data_[frag.fragment_id]->dataSizeBytes();
192 return dropped_data_[frag.fragment_id]->dataBegin();
195 last_backpressure_report_time_ = std::chrono::steady_clock::now();
196 last_fragment_header_write_time_ = std::chrono::steady_clock::now();
198 buffer_writes_pending_[buffer]++;
202 metricMan->sendMetric(
"Input Fragment Rate", 1,
"Fragments/s", 1, MetricMode::Rate);
205 TLOG(TLVL_BUFLCK) <<
"WriteFragmentHeader: obtaining buffer_mutexes lock for buffer " << buffer;
207 std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
209 TLOG(TLVL_BUFLCK) <<
"WriteFragmentHeader: obtained buffer_mutexes lock for buffer " << buffer;
212 auto hdrpos =
reinterpret_cast<RawDataType*
>(GetWritePos(buffer));
213 Write(buffer, &frag, frag.num_words() *
sizeof(RawDataType));
215 auto pos =
reinterpret_cast<RawDataType*
>(GetWritePos(buffer));
216 if (frag.word_count - frag.num_words() > 0)
218 auto sts = IncrementWritePos(buffer, (frag.word_count - frag.num_words()) *
sizeof(RawDataType));
222 reinterpret_cast<detail::RawFragmentHeader*
>(hdrpos)->word_count = frag.num_words();
223 reinterpret_cast<detail::RawFragmentHeader*
>(hdrpos)->type = Fragment::InvalidFragmentType;
224 TLOG(TLVL_ERROR) <<
"Dropping over-size fragment with sequence id " << frag.sequence_id <<
" and fragment id " << frag.fragment_id <<
" because there is no room in the current buffer for this Fragment! (Keeping header)";
225 dropped_data_[frag.fragment_id].reset(
new Fragment(frag.word_count - frag.num_words()));
227 oversize_fragment_count_++;
229 if (maximum_oversize_fragment_count_ > 0 && oversize_fragment_count_ >= maximum_oversize_fragment_count_)
231 throw cet::exception(
"Too many over-size Fragments received! Please adjust max_event_size_bytes or max_fragment_size_bytes!");
234 TLOG(6) <<
"Dropping over-size fragment with sequence id " << frag.sequence_id <<
" and fragment id " << frag.fragment_id <<
" into " << (
void*)dropped_data_[frag.fragment_id]->dataBegin();
235 return dropped_data_[frag.fragment_id]->dataBegin();
238 TLOG(14) <<
"WriteFragmentHeader END";
244 TLOG(TLVL_TRACE) <<
"DoneWritingFragment BEGIN";
245 auto buffer = getBufferForSequenceID_(frag.sequence_id,
false, frag.timestamp);
246 if (buffer == -1) Detach(
true,
"SharedMemoryEventManager",
"getBufferForSequenceID_ returned -1 when it REALLY shouldn't have! Check program logic!");
247 if (buffer == -2) {
return; }
250 TLOG(TLVL_BUFLCK) <<
"DoneWritingFragment: obtaining buffer_mutexes lock for buffer " << buffer;
252 std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
254 TLOG(TLVL_BUFLCK) <<
"DoneWritingFragment: obtained buffer_mutexes lock for buffer " << buffer;
258 TLOG(TLVL_DEBUG) <<
"DoneWritingFragment: Received Fragment with sequence ID " << frag.sequence_id <<
" and fragment id " << frag.fragment_id <<
" (type " << (int)frag.type <<
")";
259 auto hdr = getEventHeader_(buffer);
262 hdr->run_id = run_id_;
264 hdr->subrun_id = GetSubrunForSequenceID(frag.sequence_id);
266 TLOG(TLVL_TRACE) <<
"DoneWritingFragment: Updating buffer touch time";
269 buffer_writes_pending_[buffer]--;
270 if (buffer_writes_pending_[buffer] != 0)
272 TLOG(TLVL_TRACE) <<
"Done writing fragment, but there's another writer. Not doing bookkeeping steps.";
275 TLOG(TLVL_TRACE) <<
"Done writing fragment, and no other writer. Doing bookkeeping steps.";
276 auto frag_count = GetFragmentCount(frag.sequence_id);
277 hdr->is_complete = frag_count == num_fragments_per_event_;
278 TLOG(TLVL_TRACE) <<
"DoneWritingFragment: Received Fragment with sequence ID " << frag.sequence_id <<
" and fragment id " << frag.fragment_id <<
", count/expected = " << frag_count <<
"/" << num_fragments_per_event_;
279 #if ART_SUPPORTS_DUPLICATE_EVENTS
280 if (!hdr->is_complete && released_incomplete_events_.count(frag.sequence_id))
282 hdr->is_complete = frag_count == released_incomplete_events_[frag.sequence_id] && buffer_writes_pending_[buffer] == 0;
287 complete_buffer_(buffer);
288 if (requests_) requests_->SendRequest(
true);
289 TLOG(TLVL_TRACE) <<
"DoneWritingFragment END";
294 return GetFragmentCountInBuffer(getBufferForSequenceID_(seqID,
false), type);
299 if (buffer == -1)
return 0;
300 ResetReadPos(buffer);
301 IncrementReadPos(buffer,
sizeof(detail::RawEventHeader));
305 while (MoreDataInBuffer(buffer))
307 auto fragHdr =
reinterpret_cast<artdaq::detail::RawFragmentHeader*
>(GetReadPos(buffer));
308 IncrementReadPos(buffer, fragHdr->word_count *
sizeof(RawDataType));
309 if (type != Fragment::InvalidFragmentType && fragHdr->type != type)
continue;
310 TLOG(TLVL_TRACE) <<
"Adding Fragment with size=" << fragHdr->word_count <<
" to Fragment count";
321 auto start_time = std::chrono::steady_clock::now();
323 TLOG(TLVL_INFO) <<
"Starting art process with config file " << config_file->getFileName();
329 char* filename =
new char[config_file->getFileName().length() + 1];
330 strcpy(filename, config_file->getFileName().c_str());
332 std::vector<char*> args{(
char*)
"art", (
char*)
"-c", filename, NULL};
340 std::string envVarKey =
"ARTDAQ_PARTITION_NUMBER";
341 std::string envVarValue = std::to_string(GetPartitionNumber());
342 if (setenv(envVarKey.c_str(), envVarValue.c_str(), 1) != 0)
344 TLOG(TLVL_ERROR) <<
"Error setting environment variable \"" << envVarKey
345 <<
"\" in the environment of a child art process. "
346 <<
"This may result in incorrect TCP port number "
347 <<
"assignments or other issues, and data may "
348 <<
"not flow through the system correctly.";
350 envVarKey =
"ARTDAQ_APPLICATION_NAME";
351 envVarValue = app_name;
352 if (setenv(envVarKey.c_str(), envVarValue.c_str(), 1) != 0)
354 TLOG(TLVL_DEBUG) <<
"Error setting environment variable \"" << envVarKey
355 <<
"\" in the environment of a child art process. ";
358 execvp(
"art", &args[0]);
367 std::cout <<
"Please run the following command in a separate terminal:" << std::endl
368 <<
"art -c " << config_file->getFileName() << std::endl
369 <<
"Then, in a third terminal, execute: \"ps aux|grep [a]rt -c " << config_file->getFileName() <<
"\" and note the PID of the art process." << std::endl
370 <<
"Finally, return to this window and enter the pid: " << std::endl;
375 TLOG(TLVL_INFO) <<
"PID of new art process is " << pid;
377 std::unique_lock<std::mutex> lk(art_process_mutex_);
378 art_processes_.insert(pid);
381 auto sts = waitid(P_PID, pid, &status, WEXITED);
382 TLOG(TLVL_INFO) <<
"Removing PID " << pid <<
" from process list";
384 std::unique_lock<std::mutex> lk(art_process_mutex_);
385 art_processes_.erase(pid);
389 TLOG(TLVL_WARNING) <<
"Error occurred in waitid for art process " << pid <<
": " << errno <<
" (" << strerror(errno) <<
").";
391 else if (status.si_code == CLD_EXITED && status.si_status == 0)
393 TLOG(TLVL_INFO) <<
"art process " << pid <<
" exited normally, " << (restart_art_ ?
"restarting" :
"not restarting");
397 auto art_lifetime = TimeUtils::GetElapsedTime(start_time);
398 if (art_lifetime < minimum_art_lifetime_s_) restart_art_ =
false;
400 auto exit_type =
"exited with status code";
401 switch (status.si_code)
405 exit_type =
"was killed with signal";
412 TLOG((restart_art_ ? TLVL_WARNING : TLVL_ERROR))
413 <<
"art process " << pid <<
" " << exit_type <<
" " << status.si_status
414 << (status.si_code == CLD_DUMPED ?
" (core dumped)" :
"")
415 <<
" after running for " << std::setprecision(2) << std::fixed << art_lifetime <<
" seconds, "
416 << (restart_art_ ?
"restarting" :
"not restarting");
418 }
while (restart_art_);
423 restart_art_ = always_restart_art_;
424 if (num_art_processes_ == 0)
return;
425 for (
size_t ii = 0; ii < num_art_processes_; ++ii)
427 StartArtProcess(current_art_pset_);
433 static std::mutex start_art_mutex;
434 std::unique_lock<std::mutex> lk(start_art_mutex);
436 restart_art_ = always_restart_art_;
437 auto initialCount = GetAttachedCount();
438 auto startTime = std::chrono::steady_clock::now();
440 if (pset != current_art_pset_ || !current_art_config_file_)
442 current_art_pset_ = pset;
443 current_art_config_file_ = std::make_shared<art_config_file>(pset );
445 std::shared_ptr<std::atomic<pid_t>> pid(
new std::atomic<pid_t>(-1));
446 boost::thread thread([&] { RunArt(current_art_config_file_, pid); });
449 auto currentCount = GetAttachedCount() - initialCount;
450 while ((currentCount < 1 || *pid <= 0) && (TimeUtils::GetElapsedTime(startTime) < 5 || manual_art_))
453 currentCount = GetAttachedCount() - initialCount;
455 if ((currentCount < 1 || *pid <= 0) && manual_art_)
457 TLOG(TLVL_WARNING) <<
"Manually-started art process has not connected to shared memory or has bad PID: connected:" << currentCount <<
", PID:" << pid;
460 else if (currentCount < 1 || *pid <= 0)
462 TLOG(TLVL_WARNING) <<
"art process has not started after 5s. Check art configuration!"
463 <<
" (pid=" << *pid <<
", attachedCount=" << currentCount <<
")";
468 TLOG(TLVL_INFO) << std::setw(4) << std::fixed <<
"art initialization took "
469 << TimeUtils::GetElapsedTime(startTime) <<
" seconds.";
477 restart_art_ =
false;
481 auto check_pids = [&](
bool print) {
482 std::unique_lock<std::mutex> lk(art_process_mutex_);
483 for (
auto pid = pids.begin(); pid != pids.end();)
489 TLOG(TLVL_WARNING) <<
"Removing an invalid PID (" << *pid
490 <<
") from the shutdown list.";
491 pid = pids.erase(pid);
493 else if (kill(*pid, 0) < 0)
495 pid = pids.erase(pid);
499 if (print) std::cout << *pid <<
" ";
504 auto count_pids = [&]() {
505 std::unique_lock<std::mutex> lk(art_process_mutex_);
509 if (count_pids() == 0)
511 TLOG(14) <<
"All art processes already exited, nothing to do.";
519 TLOG(TLVL_TRACE) <<
"Gently informing art processes that it is time to shut down";
520 std::unique_lock<std::mutex> lk(art_process_mutex_);
521 for (
auto pid : pids)
523 TLOG(TLVL_TRACE) <<
"Sending SIGQUIT to pid " << pid;
528 int graceful_wait_ms = 5000;
529 int int_wait_ms = 1000;
531 TLOG(TLVL_TRACE) <<
"Waiting up to " << graceful_wait_ms <<
" ms for all art processes to exit gracefully";
532 for (
int ii = 0; ii < graceful_wait_ms; ++ii)
537 if (count_pids() == 0)
539 TLOG(TLVL_TRACE) <<
"All art processes exited after " << ii <<
" ms.";
545 TLOG(TLVL_TRACE) <<
"Insisting that the art processes shut down";
546 std::unique_lock<std::mutex> lk(art_process_mutex_);
547 for (
auto pid : pids)
553 TLOG(TLVL_TRACE) <<
"Waiting up to " << int_wait_ms <<
" ms for all art processes to exit";
554 for (
int ii = graceful_wait_ms; ii < graceful_wait_ms + int_wait_ms; ++ii)
560 if (count_pids() == 0)
562 TLOG(TLVL_TRACE) <<
"All art processes exited after " << ii <<
" ms.";
567 TLOG(TLVL_TRACE) <<
"Killing remaning art processes with extreme prejudice";
568 while (count_pids() > 0)
571 std::unique_lock<std::mutex> lk(art_process_mutex_);
572 kill(*pids.begin(), SIGKILL);
580 std::cout <<
"Please shut down all art processes, then hit return/enter" << std::endl;
581 while (count_pids() > 0)
583 std::cout <<
"The following PIDs are running: ";
585 std::cout << std::endl;
594 TLOG(TLVL_DEBUG) <<
"ReconfigureArt BEGIN";
595 if (restart_art_ || !always_restart_art_)
599 for (
size_t ii = 0; ii < broadcasts_.size(); ++ii)
601 broadcasts_.MarkBufferEmpty(ii,
true);
603 if (newRun == 0) newRun = run_id_ + 1;
605 if (art_pset != current_art_pset_ || !current_art_config_file_)
607 current_art_pset_ = art_pset;
608 current_art_config_file_ = std::make_shared<art_config_file>(art_pset );
611 if (n_art_processes != -1)
613 TLOG(TLVL_INFO) <<
"Setting number of art processes to " << n_art_processes;
614 num_art_processes_ = n_art_processes;
617 TLOG(TLVL_DEBUG) <<
"ReconfigureArt END";
623 init_fragment_.reset(
nullptr);
624 TLOG(TLVL_DEBUG) <<
"SharedMemoryEventManager::endOfData";
625 restart_art_ =
false;
627 size_t initialStoreSize = GetIncompleteEventCount();
628 TLOG(TLVL_DEBUG) <<
"endOfData: Flushing " << initialStoreSize
629 <<
" stale events from the SharedMemoryEventManager.";
630 int counter = initialStoreSize;
631 while (active_buffers_.size() > 0 && counter > 0)
633 complete_buffer_(*active_buffers_.begin());
636 TLOG(TLVL_DEBUG) <<
"endOfData: Done flushing, there are now " << GetIncompleteEventCount()
637 <<
" stale events in the SharedMemoryEventManager.";
639 TLOG(TLVL_DEBUG) <<
"Waiting for " << (ReadReadyCount() + (size() - WriteReadyCount(overwrite_mode_))) <<
" outstanding buffers...";
640 auto start = std::chrono::steady_clock::now();
641 auto lastReadCount = ReadReadyCount() + (size() - WriteReadyCount(overwrite_mode_));
642 auto end_of_data_wait_us = art_event_processing_time_us_ * (lastReadCount > 0 ? lastReadCount : 1);
644 auto outstanding_buffer_wait_time = art_event_processing_time_us_ > 100000 ? 100000 : art_event_processing_time_us_;
647 while (lastReadCount > 0 && (end_of_data_wait_us == 0 || TimeUtils::GetElapsedTimeMicroseconds(start) < end_of_data_wait_us) && get_art_process_count_() > 0)
649 auto temp = ReadReadyCount() + (size() - WriteReadyCount(overwrite_mode_));
650 if (temp != lastReadCount)
652 TLOG(TLVL_TRACE) <<
"Waiting for " << temp <<
" outstanding buffers...";
653 lastReadCount = temp;
654 start = std::chrono::steady_clock::now();
656 if (lastReadCount > 0)
658 TRACE(19,
"About to sleep %lu us - lastReadCount=%lu size=%lu end_of_data_wait_us=%lu", outstanding_buffer_wait_time, lastReadCount, size(), end_of_data_wait_us);
659 usleep(outstanding_buffer_wait_time);
663 TLOG(TLVL_DEBUG) <<
"endOfData: After wait for outstanding buffers. Still outstanding: " << lastReadCount <<
", time waited: "
664 << TimeUtils::GetElapsedTime(start) <<
" s / " << (end_of_data_wait_us / 1000000.0) <<
" s, art process count: " << get_art_process_count_();
666 TLOG(TLVL_DEBUG) <<
"endOfData: Broadcasting EndOfData Fragment";
667 FragmentPtr outFrag = Fragment::eodFrag(GetBufferCount());
668 bool success = broadcastFragment_(std::move(outFrag), outFrag);
671 TLOG(TLVL_DEBUG) <<
"endOfData: Clearing buffers to make room for EndOfData Fragment";
672 for (
size_t ii = 0; ii < broadcasts_.size(); ++ii)
674 broadcasts_.MarkBufferEmpty(ii,
true);
676 broadcastFragment_(std::move(outFrag), outFrag);
678 auto endOfDataProcessingStart = std::chrono::steady_clock::now();
680 if (get_art_process_count_() > 0)
682 TLOG(TLVL_DEBUG) <<
"Allowing " << get_art_process_count_() <<
" art processes the chance to end gracefully";
683 if (end_of_data_wait_us == 0)
685 TLOG(TLVL_DEBUG) <<
"Expected art event processing time not specified. Waiting up to 100s for art to end gracefully.";
686 end_of_data_wait_us = 100 * 1000000;
689 auto sleep_count = (end_of_data_wait_us / 10000) + 1;
690 for (
size_t ii = 0; ii < sleep_count; ++ii)
693 if (get_art_process_count_() == 0)
break;
697 while (get_art_process_count_() > 0)
699 TLOG(TLVL_DEBUG) <<
"There are " << get_art_process_count_() <<
" art processes remaining. Proceeding to shutdown.";
701 ShutdownArtProcesses(art_processes_);
703 TLOG(TLVL_DEBUG) <<
"It took " << TimeUtils::GetElapsedTime(endOfDataProcessingStart) <<
" s for all art processes to close after sending EndOfData Fragment";
705 ResetAttachedCount();
707 TLOG(TLVL_DEBUG) <<
"endOfData: Clearing buffers";
708 for (
size_t ii = 0; ii < size(); ++ii)
710 MarkBufferEmpty(ii,
true);
718 released_incomplete_events_.clear();
720 TLOG(TLVL_DEBUG) <<
"endOfData: Shutting down RequestSender";
721 requests_.reset(
nullptr);
723 TLOG(TLVL_DEBUG) <<
"endOfData END";
724 TLOG(TLVL_INFO) <<
"EndOfData Complete. There were " << GetLastSeenBufferID() <<
" buffers processed.";
731 init_fragment_.reset(
nullptr);
732 TLOG(TLVL_TRACE) <<
"startRun: Clearing broadcast buffers";
733 for (
size_t ii = 0; ii < broadcasts_.size(); ++ii)
735 broadcasts_.MarkBufferEmpty(ii,
true);
740 std::unique_lock<std::mutex> lk(subrun_event_map_mutex_);
741 subrun_event_map_.clear();
742 subrun_event_map_[0] = 1;
744 run_event_count_ = 0;
745 run_incomplete_event_count_ = 0;
749 requests_->SetRunNumber(static_cast<uint32_t>(run_id_));
750 requests_->SendRoutingToken(queue_size_, run_id_);
752 TLOG(TLVL_DEBUG) <<
"Starting run " << run_id_
753 <<
", max queue size = "
756 << GetLockedBufferCount();
759 metricMan->sendMetric(
"Run Number", static_cast<unsigned long>(run_id_),
"Run", 1, MetricMode::LastPoint);
765 TLOG(TLVL_INFO) <<
"Ending run " << run_id_;
766 FragmentPtr endOfRunFrag(
new Fragment(static_cast<size_t>(ceil(
sizeof(my_rank) /
767 static_cast<double>(
sizeof(Fragment::value_type))))));
769 TLOG(TLVL_DEBUG) <<
"Broadcasting EndOfRun Fragment";
770 endOfRunFrag->setSystemType(Fragment::EndOfRunFragmentType);
771 *endOfRunFrag->dataBegin() = my_rank;
772 broadcastFragment_(std::move(endOfRunFrag), endOfRunFrag);
774 TLOG(TLVL_INFO) <<
"Run " << run_id_ <<
" has ended. There were " << run_event_count_ <<
" events in this run.";
775 run_event_count_ = 0;
776 run_incomplete_event_count_ = 0;
777 oversize_fragment_count_ = 0;
779 std::unique_lock<std::mutex> lk(subrun_event_map_mutex_);
780 subrun_event_map_.clear();
781 subrun_event_map_[0] = 1;
789 if (boundary == 0 || boundary == Fragment::InvalidSequenceID)
return;
791 std::unique_lock<std::mutex> lk(subrun_event_map_mutex_);
793 TLOG(TLVL_INFO) <<
"Will roll over to subrun " << subrun <<
" when I reach Sequence ID " << boundary;
794 subrun_event_map_[boundary] = subrun;
795 while (subrun_event_map_.size() > max_subrun_event_map_length_)
797 subrun_event_map_.erase(subrun_event_map_.begin());
803 Fragment::sequence_id_t seqID = 0;
806 std::unique_lock<std::mutex> lk(subrun_event_map_mutex_);
807 for (
auto& it : subrun_event_map_)
809 if (it.first >= seqID) seqID = it.first + 1;
810 if (it.second >= subrun) subrun = it.second + 1;
813 rolloverSubrun(seqID, subrun);
820 metricMan->sendMetric(
"Incomplete Event Count", GetIncompleteEventCount(),
"events", 1, MetricMode::LastPoint);
821 metricMan->sendMetric(
"Pending Event Count", GetPendingEventCount(),
"events", 1, MetricMode::LastPoint);
824 if (incomplete_event_report_interval_ms_ > 0 && GetLockedBufferCount())
826 if (TimeUtils::GetElapsedTimeMilliseconds(last_incomplete_event_report_time_) < static_cast<size_t>(incomplete_event_report_interval_ms_))
829 last_incomplete_event_report_time_ = std::chrono::steady_clock::now();
830 std::ostringstream oss;
831 oss <<
"Incomplete Events (" << num_fragments_per_event_ <<
"): ";
832 for (
auto& ev : active_buffers_)
834 auto hdr = getEventHeader_(ev);
835 oss << hdr->sequence_id <<
" (" << GetFragmentCount(hdr->sequence_id) <<
"), ";
837 TLOG(TLVL_DEBUG) << oss.str();
841 bool artdaq::SharedMemoryEventManager::broadcastFragment_(FragmentPtr frag, FragmentPtr& outFrag)
843 TLOG(TLVL_DEBUG) <<
"Broadcasting Fragment with seqID=" << frag->sequenceID() <<
", type " << detail::RawFragmentHeader::SystemTypeToString(frag->type()) <<
", size=" << frag->sizeBytes() <<
"B.";
844 auto buffer = broadcasts_.GetBufferForWriting(
false);
845 TLOG(TLVL_DEBUG) <<
"broadcastFragment_: after getting buffer 1st buffer=" << buffer;
846 auto start_time = std::chrono::steady_clock::now();
847 while (buffer == -1 && TimeUtils::GetElapsedTimeMilliseconds(start_time) < static_cast<size_t>(broadcast_timeout_ms_))
850 buffer = broadcasts_.GetBufferForWriting(
false);
852 TLOG(TLVL_DEBUG) <<
"broadcastFragment_: after getting buffer w/timeout, buffer=" << buffer <<
", elapsed time=" << TimeUtils::GetElapsedTime(start_time) <<
" s.";
855 TLOG(TLVL_ERROR) <<
"Broadcast of fragment type " << frag->typeString() <<
" failed due to timeout waiting for buffer!";
860 TLOG(TLVL_DEBUG) <<
"broadcastFragment_: Filling in RawEventHeader";
861 auto hdr =
reinterpret_cast<detail::RawEventHeader*
>(broadcasts_.GetBufferStart(buffer));
862 hdr->run_id = run_id_;
863 hdr->subrun_id = GetSubrunForSequenceID(frag->sequenceID());
864 hdr->sequence_id = frag->sequenceID();
865 hdr->is_complete =
true;
866 broadcasts_.IncrementWritePos(buffer,
sizeof(detail::RawEventHeader));
868 TLOG(TLVL_DEBUG) <<
"broadcastFragment_ before Write calls";
869 broadcasts_.Write(buffer, frag->headerAddress(), frag->size() *
sizeof(RawDataType));
871 TLOG(TLVL_DEBUG) <<
"broadcastFragment_ Marking buffer full";
872 broadcasts_.MarkBufferFull(buffer, -1);
874 TLOG(TLVL_DEBUG) <<
"broadcastFragment_ Complete";
878 artdaq::detail::RawEventHeader* artdaq::SharedMemoryEventManager::getEventHeader_(
int buffer)
880 return reinterpret_cast<detail::RawEventHeader*
>(GetBufferStart(buffer));
885 std::unique_lock<std::mutex> lk(subrun_event_map_mutex_);
887 TLOG(TLVL_TRACE) <<
"GetSubrunForSequenceID BEGIN map size = " << subrun_event_map_.size();
888 auto it = subrun_event_map_.begin();
891 while (it->first <= seqID && it != subrun_event_map_.end())
893 TLOG(TLVL_TRACE) <<
"Map has sequence ID " << it->first <<
", subrun " << it->second <<
" (looking for <= " << seqID <<
")";
898 TLOG(TLVL_DEBUG) <<
"GetSubrunForSequenceID returning subrun " << subrun <<
" for sequence ID " << seqID;
902 int artdaq::SharedMemoryEventManager::getBufferForSequenceID_(Fragment::sequence_id_t seqID,
bool create_new, Fragment::timestamp_t timestamp)
904 TLOG(14) <<
"getBufferForSequenceID " << seqID <<
" BEGIN";
905 std::unique_lock<std::mutex> lk(sequence_id_mutex_);
907 TLOG(14) <<
"getBufferForSequenceID obtained sequence_id_mutex for seqid=" << seqID;
909 auto buffers = GetBuffersOwnedByManager();
910 for (
auto& buf : buffers)
912 auto hdr = getEventHeader_(buf);
913 if (hdr->sequence_id == seqID)
915 TLOG(14) <<
"getBufferForSequenceID " << seqID <<
" returning " << buf;
920 #if !ART_SUPPORTS_DUPLICATE_EVENTS
921 if (released_incomplete_events_.count(seqID))
923 TLOG(TLVL_ERROR) <<
"Event " << seqID <<
" has already been marked \"Incomplete\" and sent to art!";
928 if (!create_new)
return -1;
930 check_pending_buffers_(lk);
931 int new_buffer = GetBufferForWriting(
false);
933 if (new_buffer == -1)
935 new_buffer = GetBufferForWriting(overwrite_mode_);
938 if (new_buffer == -1)
return -1;
939 TLOG(TLVL_BUFLCK) <<
"getBufferForSequenceID_: obtaining buffer_mutexes lock for buffer " << new_buffer;
940 std::unique_lock<std::mutex> buffer_lk(buffer_mutexes_[new_buffer]);
941 TLOG(TLVL_BUFLCK) <<
"getBufferForSequenceID_: obtained buffer_mutexes lock for buffer " << new_buffer;
943 auto hdr = getEventHeader_(new_buffer);
944 hdr->is_complete =
false;
945 hdr->run_id = run_id_;
946 hdr->subrun_id = GetSubrunForSequenceID(seqID);
947 hdr->event_id = use_sequence_id_for_event_number_ ?
static_cast<uint32_t
>(seqID) : static_cast<uint32_t>(timestamp);
948 hdr->sequence_id = seqID;
949 buffer_writes_pending_[new_buffer] = 0;
950 IncrementWritePos(new_buffer,
sizeof(detail::RawEventHeader));
951 SetMFIteration(
"Sequence ID " + std::to_string(seqID));
953 TLOG(TLVL_BUFFER) <<
"getBufferForSequenceID placing " << new_buffer <<
" to active.";
954 active_buffers_.insert(new_buffer);
955 TLOG(TLVL_BUFFER) <<
"Buffer occupancy now (total,full,reading,empty,pending,active)=("
957 << ReadReadyCount() <<
","
958 << WriteReadyCount(
true) - WriteReadyCount(
false) - ReadReadyCount() <<
","
959 << WriteReadyCount(
false) <<
","
960 << pending_buffers_.size() <<
","
961 << active_buffers_.size() <<
")";
965 if (timestamp != Fragment::InvalidTimestamp)
967 requests_->AddRequest(seqID, timestamp);
973 requests_->SendRequest();
976 TLOG(14) <<
"getBufferForSequenceID " << seqID <<
" returning newly initialized buffer " << new_buffer;
980 bool artdaq::SharedMemoryEventManager::hasFragments_(
int buffer)
982 if (buffer == -1)
return true;
983 if (!CheckBuffer(buffer, BufferSemaphoreFlags::Writing))
987 ResetReadPos(buffer);
988 IncrementReadPos(buffer,
sizeof(detail::RawEventHeader));
989 return MoreDataInBuffer(buffer);
992 void artdaq::SharedMemoryEventManager::complete_buffer_(
int buffer)
994 auto hdr = getEventHeader_(buffer);
995 if (hdr->is_complete)
997 TLOG(TLVL_DEBUG) <<
"complete_buffer_: This fragment completes event " << hdr->sequence_id <<
".";
1000 TLOG(TLVL_BUFFER) <<
"complete_buffer_ moving " << buffer <<
" from active to pending.";
1002 TLOG(TLVL_BUFLCK) <<
"complete_buffer_: obtaining sequence_id_mutex lock for seqid=" << hdr->sequence_id;
1003 std::unique_lock<std::mutex> lk(sequence_id_mutex_);
1004 TLOG(TLVL_BUFLCK) <<
"complete_buffer_: obtained sequence_id_mutex lock for seqid=" << hdr->sequence_id;
1005 active_buffers_.erase(buffer);
1006 pending_buffers_.insert(buffer);
1008 TLOG(TLVL_BUFFER) <<
"Buffer occupancy now (total,full,reading,empty,pending,active)=("
1010 << ReadReadyCount() <<
","
1011 << WriteReadyCount(
true) - WriteReadyCount(
false) - ReadReadyCount() <<
","
1012 << WriteReadyCount(
false) <<
","
1013 << pending_buffers_.size() <<
","
1014 << active_buffers_.size() <<
")";
1018 requests_->RemoveRequest(hdr->sequence_id);
1021 CheckPendingBuffers();
1024 bool artdaq::SharedMemoryEventManager::bufferComparator(
int bufA,
int bufB)
1026 return getEventHeader_(bufA)->sequence_id < getEventHeader_(bufB)->sequence_id;
1031 TLOG(TLVL_BUFLCK) <<
"CheckPendingBuffers: Obtaining sequence_id_mutex_";
1032 std::unique_lock<std::mutex> lk(sequence_id_mutex_);
1033 TLOG(TLVL_BUFLCK) <<
"CheckPendingBuffers: Obtained sequence_id_mutex_";
1034 check_pending_buffers_(lk);
1037 void artdaq::SharedMemoryEventManager::check_pending_buffers_(std::unique_lock<std::mutex>
const& lock)
1039 TLOG(TLVL_TRACE) <<
"check_pending_buffers_ BEGIN Locked=" << std::boolalpha << lock.owns_lock();
1041 auto buffers = GetBuffersOwnedByManager();
1042 for (
auto buf : buffers)
1044 if (ResetBuffer(buf) && !pending_buffers_.count(buf))
1046 TLOG(15) <<
"check_pending_buffers_ Incomplete buffer detected, buf=" << buf <<
" active_bufers_.count(buf)=" << active_buffers_.count(buf) <<
" buffer_writes_pending_[buf]=" << buffer_writes_pending_[buf].load();
1047 auto hdr = getEventHeader_(buf);
1048 if (active_buffers_.count(buf) && (buffer_writes_pending_[buf].load() == 0 || !running_))
1052 requests_->RemoveRequest(hdr->sequence_id);
1054 TLOG(TLVL_BUFFER) <<
"check_pending_buffers_ moving buffer " << buf <<
" from active to pending";
1055 active_buffers_.erase(buf);
1056 pending_buffers_.insert(buf);
1057 TLOG(TLVL_BUFFER) <<
"Buffer occupancy now (total,full,reading,empty,pending,active)=("
1059 << ReadReadyCount() <<
","
1060 << WriteReadyCount(
true) - WriteReadyCount(
false) - ReadReadyCount() <<
","
1061 << WriteReadyCount(
false) <<
","
1062 << pending_buffers_.size() <<
","
1063 << active_buffers_.size() <<
")";
1065 run_incomplete_event_count_++;
1066 if (metricMan) metricMan->sendMetric(
"Incomplete Event Rate", 1,
"events/s", 3, MetricMode::Rate);
1067 if (!released_incomplete_events_.count(hdr->sequence_id))
1069 released_incomplete_events_[hdr->sequence_id] = num_fragments_per_event_ - GetFragmentCountInBuffer(buf);
1073 released_incomplete_events_[hdr->sequence_id] -= GetFragmentCountInBuffer(buf);
1075 TLOG(TLVL_WARNING) <<
"Active event " << hdr->sequence_id <<
" is stale. Scheduling release of incomplete event (missing " << released_incomplete_events_[hdr->sequence_id] <<
" Fragments) to art.";
1080 std::list<int> sorted_buffers(pending_buffers_.begin(), pending_buffers_.end());
1081 sorted_buffers.sort([
this](
int a,
int b) {
return bufferComparator(a, b); });
1084 double eventSize = 0;
1085 for (
auto buf : sorted_buffers)
1087 auto hdr = getEventHeader_(buf);
1089 TLOG(TLVL_DEBUG) <<
"Releasing event " << std::to_string(hdr->sequence_id) <<
" in buffer " << buf <<
" to art, "
1090 <<
"event_size=" << BufferDataSize(buf) <<
", buffer_size=" << BufferSize();
1092 TLOG(TLVL_BUFFER) <<
"check_pending_buffers_ removing buffer " << buf <<
" moving from pending to full";
1093 MarkBufferFull(buf);
1096 eventSize += BufferDataSize(buf);
1097 pending_buffers_.erase(buf);
1098 TLOG(TLVL_BUFFER) <<
"Buffer occupancy now (total,full,reading,empty,pending,active)=("
1100 << ReadReadyCount() <<
","
1101 << WriteReadyCount(
true) - WriteReadyCount(
false) - ReadReadyCount() <<
","
1102 << WriteReadyCount(
false) <<
","
1103 << pending_buffers_.size() <<
","
1104 << active_buffers_.size() <<
")";
1109 TLOG(TLVL_TRACE) <<
"Sent tokens: " << requests_->GetSentTokenCount() <<
", Event count: " << run_event_count_;
1110 auto outstanding_tokens = requests_->GetSentTokenCount() - run_event_count_;
1111 auto available_buffers = WriteReadyCount(overwrite_mode_);
1113 TLOG(TLVL_TRACE) <<
"check_pending_buffers_: outstanding_tokens: " << outstanding_tokens <<
", available_buffers: " << available_buffers
1114 <<
", tokens_to_send: " << available_buffers - outstanding_tokens;
1116 if (available_buffers > outstanding_tokens)
1118 auto tokens_to_send = available_buffers - outstanding_tokens;
1120 while (tokens_to_send > 0)
1122 TLOG(35) <<
"check_pending_buffers_: Sending a Routing Token";
1123 requests_->SendRoutingToken(1, run_id_);
1129 metric_data_.event_count += counter;
1130 metric_data_.event_size += eventSize;
1132 if (metricMan && TimeUtils::GetElapsedTimeMilliseconds(last_shmem_buffer_metric_update_) > 500)
1134 TLOG(TLVL_TRACE) <<
"check_pending_buffers_: Sending Metrics";
1135 metricMan->sendMetric(
"Event Rate", metric_data_.event_count,
"Events/s", 1, MetricMode::Rate);
1136 if (metric_data_.event_count > 0) metricMan->sendMetric(
"Average Event Size", metric_data_.event_size / metric_data_.event_count,
"Bytes", 1, MetricMode::Average);
1137 metric_data_ = MetricData();
1139 metricMan->sendMetric(
"Events Released to art this run", run_event_count_,
"Events", 1, MetricMode::LastPoint);
1140 metricMan->sendMetric(
"Incomplete Events Released to art this run", run_incomplete_event_count_,
"Events", 1, MetricMode::LastPoint);
1141 if (requests_) metricMan->sendMetric(
"Tokens sent", requests_->GetSentTokenCount(),
"Tokens", 2, MetricMode::LastPoint);
1143 auto bufferReport = GetBufferReport();
1144 int full = std::count_if(bufferReport.begin(), bufferReport.end(), [](std::pair<int, BufferSemaphoreFlags> p) {
return p.second == BufferSemaphoreFlags::Full; });
1145 int empty = std::count_if(bufferReport.begin(), bufferReport.end(), [](std::pair<int, BufferSemaphoreFlags> p) {
return p.second == BufferSemaphoreFlags::Empty; });
1146 int writing = std::count_if(bufferReport.begin(), bufferReport.end(), [](std::pair<int, BufferSemaphoreFlags> p) {
return p.second == BufferSemaphoreFlags::Writing; });
1147 int reading = std::count_if(bufferReport.begin(), bufferReport.end(), [](std::pair<int, BufferSemaphoreFlags> p) {
return p.second == BufferSemaphoreFlags::Reading; });
1148 auto total = size();
1149 TLOG(TLVL_DEBUG) <<
"Buffer usage: full=" << full <<
", empty=" << empty <<
", writing=" << writing <<
", reading=" << reading <<
", total=" << total;
1151 metricMan->sendMetric(
"Shared Memory Full Buffers", full,
"buffers", 2, MetricMode::LastPoint);
1152 metricMan->sendMetric(
"Shared Memory Available Buffers", empty,
"buffers", 2, MetricMode::LastPoint);
1153 metricMan->sendMetric(
"Shared Memory Pending Buffers", writing,
"buffers", 2, MetricMode::LastPoint);
1154 metricMan->sendMetric(
"Shared Memory Reading Buffers", reading,
"buffers", 2, MetricMode::LastPoint);
1157 metricMan->sendMetric(
"Shared Memory Full %", full * 100 / static_cast<double>(total),
"%", 2, MetricMode::LastPoint);
1158 metricMan->sendMetric(
"Shared Memory Available %", empty * 100 / static_cast<double>(total),
"%", 2, MetricMode::LastPoint);
1161 last_shmem_buffer_metric_update_ = std::chrono::steady_clock::now();
1163 TLOG(TLVL_TRACE) <<
"check_pending_buffers_ END";
1166 void artdaq::SharedMemoryEventManager::send_init_frag_()
1168 if (init_fragment_ !=
nullptr)
1170 TLOG(TLVL_INFO) <<
"Broadcasting init fragment to all art subprocesses...";
1173 std::string fileName =
"receiveInitMessage_" + std::to_string(my_rank) +
".bin";
1174 std::fstream ostream(fileName.c_str(), std::ios::out | std::ios::binary);
1175 ostream.write(reinterpret_cast<char*>(init_fragment_->dataBeginBytes()), init_fragment_->dataSizeBytes());
1179 broadcastFragment_(std::move(init_fragment_), init_fragment_);
1180 TLOG(TLVL_TRACE) <<
"Init Fragment sent";
1182 else if (send_init_fragments_)
1184 TLOG(TLVL_WARNING) <<
"Cannot send init fragment because I haven't yet received one!";
1190 if (!init_fragment_ || init_fragment_ ==
nullptr)
1192 init_fragment_.swap(frag);
1199 TLOG(TLVL_DEBUG) <<
"UpdateArtConfiguration BEGIN";
1200 if (art_pset != current_art_pset_ || !current_art_config_file_)
1202 current_art_pset_ = art_pset;
1203 current_art_config_file_ = std::make_shared<art_config_file>(art_pset );
1205 TLOG(TLVL_DEBUG) <<
"UpdateArtConfiguration END";
1208 #if MESSAGEFACILITY_HEX_VERSION >= 0x20103
void RunArt(std::shared_ptr< art_config_file > config_file, std::shared_ptr< std::atomic< pid_t >> pid_out)
Run an art instance, recording the return codes and restarting it until the end flag is raised...
void ShutdownArtProcesses(std::set< pid_t > &pids)
Shutdown a set of art processes.
virtual ~SharedMemoryEventManager()
SharedMemoryEventManager Destructor.
The SharedMemoryEventManager is a SharedMemoryManger which tracks events as they are built...
Fragment::sequence_id_t sequence_id_t
Copy Fragment::sequence_id_t into local scope.
void ReconfigureArt(fhicl::ParameterSet art_pset, run_id_t newRun=0, int n_art_processes=-1)
Restart all art processes, using the given fhicl code to configure the new art processes.
The RequestSender contains methods used to send data requests and Routing tokens. ...
pid_t StartArtProcess(fhicl::ParameterSet pset)
Start one art process.
RawDataType * WriteFragmentHeader(detail::RawFragmentHeader frag, bool dropIfNoBuffersAvailable=false)
Get a pointer to a reserved memory area for the given Fragment header.
RawEvent::run_id_t run_id_t
Copy RawEvent::run_id_t into local scope.
size_t GetFragmentCount(Fragment::sequence_id_t seqID, Fragment::type_t type=Fragment::InvalidFragmentType)
Get the count of Fragments of a given type in an event.
void UpdateArtConfiguration(fhicl::ParameterSet art_pset)
Updates the internally-stored copy of the art configuration.
void StartArt()
Start all the art processes.
void SetInitFragment(FragmentPtr frag)
Set the stored Init fragment, if one has not yet been set already.
subrun_id_t GetSubrunForSequenceID(Fragment::sequence_id_t seqID)
Get the subrun number that the given Sequence ID would be assigned to.
void rolloverSubrun()
Add a subrun transition immediately after the highest currently define sequence ID.
void sendMetrics()
Send metrics to the MetricManager, if one has been instantiated in the application.
SharedMemoryEventManager(fhicl::ParameterSet pset, fhicl::ParameterSet art_pset)
SharedMemoryEventManager Constructor.
bool endRun()
Send an EndOfRunFragment to the art thread.
void DoneWritingFragment(detail::RawFragmentHeader frag)
Used to indicate that the given Fragment is now completely in the buffer. Will check for buffer compl...
bool endOfData()
Indicate that the end of input has been reached to the art processes.
RawEvent::subrun_id_t subrun_id_t
Copy RawEvent::subrun_id_t into local scope.
void startRun(run_id_t runID)
Start a Run.
size_t GetFragmentCountInBuffer(int buffer, Fragment::type_t type=Fragment::InvalidFragmentType)
Get the count of Fragments of a given type in a buffer.
void CheckPendingBuffers()
Check for buffers which are ready to be marked incomplete and released to art and issue tokens for an...