2 #define TRACE_NAME (app_name + "_SharedMemoryEventManager").c_str()
4 #include "artdaq/DAQrate/SharedMemoryEventManager.hh"
5 #include "artdaq-core/Core/StatisticsCollection.hh"
6 #include "artdaq-core/Utilities/TraceLock.hh"
10 #define TLVL_BUFLCK 41
12 std::mutex artdaq::SharedMemoryEventManager::sequence_id_mutex_;
15 : SharedMemoryManager(pset.get<uint32_t>(
"shared_memory_key", 0xBEE70000 + getpid()),
16 pset.get<size_t>(
"buffer_count"),
17 pset.has_key(
"max_event_size_bytes") ? pset.get<size_t>(
"max_event_size_bytes") : pset.get<size_t>(
"expected_fragments_per_event") * pset.get<size_t>(
"max_fragment_size_bytes"),
18 pset.get<size_t>(
"stale_buffer_timeout_usec", pset.get<size_t>(
"event_queue_wait_time", 5) * 1000000),
19 !pset.get<bool>(
"broadcast_mode", false))
20 , num_art_processes_(pset.get<size_t>(
"art_analyzer_count", 1))
21 , num_fragments_per_event_(pset.get<size_t>(
"expected_fragments_per_event"))
22 , queue_size_(pset.get<size_t>(
"buffer_count"))
25 , subrun_rollover_event_(Fragment::InvalidSequenceID)
26 , last_released_event_(0)
27 , update_run_ids_(pset.get<bool>(
"update_run_ids_on_new_fragment", true))
28 , use_sequence_id_for_event_number_(pset.get<bool>(
"use_sequence_id_for_event_number", true))
29 , overwrite_mode_(!pset.get<bool>(
"use_art", true) || pset.get<bool>(
"overwrite_mode", false) || pset.get<bool>(
"broadcast_mode", false))
30 , send_init_fragments_(pset.get<bool>(
"send_init_fragments", true))
32 , buffer_writes_pending_()
33 , incomplete_event_report_interval_ms_(pset.get<int>(
"incomplete_event_report_interval_ms", -1))
34 , last_incomplete_event_report_time_(std::chrono::steady_clock::now())
35 , last_shmem_buffer_metric_update_(std::chrono::steady_clock::now())
37 , broadcast_timeout_ms_(pset.get<int>(
"fragment_broadcast_timeout_ms", 3000))
39 , run_incomplete_event_count_(0)
40 , subrun_event_count_(0)
41 , subrun_incomplete_event_count_(0)
42 , oversize_fragment_count_(0)
43 , maximum_oversize_fragment_count_(pset.get<int>(
"maximum_oversize_fragment_count", 1))
46 , always_restart_art_(pset.get<bool>(
"restart_crashed_art_processes", true))
47 , manual_art_(pset.get<bool>(
"manual_art", false))
48 , current_art_pset_(art_pset)
49 , minimum_art_lifetime_s_(pset.get<double>(
"minimum_art_lifetime_s", 2.0))
50 , art_event_processing_time_us_(pset.get<size_t>(
"expected_art_event_processing_time_us", 100000))
54 , broadcasts_(pset.get<uint32_t>(
"broadcast_shared_memory_key", 0xCEE70000 + getpid()),
55 pset.get<size_t>(
"broadcast_buffer_count", 10),
56 pset.get<size_t>(
"broadcast_buffer_size", 0x100000),
57 pset.get<int>(
"expected_art_event_processing_time_us", 100000) * pset.get<size_t>(
"buffer_count"), false)
59 SetMinWriteSize(
sizeof(detail::RawEventHeader) +
sizeof(detail::RawFragmentHeader));
60 broadcasts_.SetMinWriteSize(
sizeof(detail::RawEventHeader) +
sizeof(detail::RawFragmentHeader));
62 if (pset.get<
bool>(
"use_art",
true) ==
false)
64 TLOG(TLVL_INFO) <<
"BEGIN SharedMemoryEventManager CONSTRUCTOR with use_art:false";
65 num_art_processes_ = 0;
69 TLOG(TLVL_INFO) <<
"BEGIN SharedMemoryEventManager CONSTRUCTOR with use_art:true";
70 TLOG(TLVL_TRACE) <<
"art_pset is " << art_pset.to_string();
72 current_art_config_file_ = std::make_shared<art_config_file>(art_pset);
74 if (overwrite_mode_ && num_art_processes_ > 0)
76 TLOG(TLVL_WARNING) <<
"Art is configured to run, but overwrite mode is enabled! Check your configuration if this in unintentional!";
78 else if (overwrite_mode_)
80 TLOG(TLVL_INFO) <<
"Overwrite Mode enabled, no configured art processes at startup";
83 for (
size_t ii = 0; ii < size(); ++ii)
85 buffer_writes_pending_[ii] = 0;
88 if (!IsValid())
throw cet::exception(app_name +
"_SharedMemoryEventManager") <<
"Unable to attach to Shared Memory!";
90 TLOG(TLVL_TRACE) <<
"Setting Writer rank to " << my_rank;
92 TLOG(TLVL_DEBUG) <<
"Writer Rank is " << GetRank();
95 TLOG(TLVL_TRACE) <<
"END CONSTRUCTOR";
100 TLOG(TLVL_TRACE) <<
"DESTRUCTOR";
101 if (running_) endOfData();
102 TLOG(TLVL_TRACE) <<
"Destructor END";
105 bool artdaq::SharedMemoryEventManager::AddFragment(detail::RawFragmentHeader frag,
void* dataPtr)
107 TLOG(TLVL_TRACE) <<
"AddFragment(Header, ptr) BEGIN frag.word_count=" << frag.word_count
108 <<
", sequence_id=" << frag.sequence_id;
109 auto buffer = getBufferForSequenceID_(frag.sequence_id,
true, frag.timestamp);
110 TLOG(TLVL_TRACE) <<
"Using buffer " << buffer <<
" for seqid=" << frag.sequence_id;
111 if (buffer == -1)
return false;
114 TLOG(TLVL_ERROR) <<
"Dropping event because data taking has already passed this event number: " << frag.sequence_id;
118 auto hdr = getEventHeader_(buffer);
121 hdr->run_id = run_id_;
122 hdr->subrun_id = subrun_id_;
125 TLOG(TLVL_TRACE) <<
"AddFragment before Write calls";
126 Write(buffer, dataPtr, frag.word_count *
sizeof(RawDataType));
128 TLOG(TLVL_TRACE) <<
"Checking for complete event";
129 auto fragmentCount = GetFragmentCount(frag.sequence_id);
130 hdr->is_complete = fragmentCount == num_fragments_per_event_ && buffer_writes_pending_[buffer] == 0;
131 TLOG(TLVL_TRACE) <<
"hdr->is_complete=" << std::boolalpha << hdr->is_complete
132 <<
", fragmentCount=" << fragmentCount
133 <<
", num_fragments_per_event=" << num_fragments_per_event_
134 <<
", buffer_writes_pending_[buffer]=" << buffer_writes_pending_[buffer];
136 complete_buffer_(buffer);
137 if (requests_) requests_->SendRequest(
true);
139 TLOG(TLVL_TRACE) <<
"AddFragment END";
143 bool artdaq::SharedMemoryEventManager::AddFragment(FragmentPtr frag,
size_t timeout_usec, FragmentPtr& outfrag)
145 TLOG(TLVL_TRACE) <<
"AddFragment(FragmentPtr) BEGIN";
146 auto hdr = *
reinterpret_cast<detail::RawFragmentHeader*
>(frag->headerAddress());
147 auto data = frag->headerAddress();
148 auto start = std::chrono::steady_clock::now();
150 while (!sts && TimeUtils::GetElapsedTimeMicroseconds(start) < timeout_usec)
152 sts = AddFragment(hdr, data);
153 if (!sts) usleep(1000);
157 outfrag = std::move(frag);
159 TLOG(TLVL_TRACE) <<
"AddFragment(FragmentPtr) RETURN " << std::boolalpha << sts;
165 TLOG(14) <<
"WriteFragmentHeader BEGIN";
166 auto buffer = getBufferForSequenceID_(frag.sequence_id,
true, frag.timestamp);
170 if (buffer == -1 && !dropIfNoBuffersAvailable)
return nullptr;
173 TLOG(TLVL_ERROR) <<
"Dropping fragment with sequence id " << frag.sequence_id <<
" and fragment id " << frag.fragment_id <<
" because data taking has already passed this event.";
177 TLOG(TLVL_ERROR) <<
"Dropping fragment with sequence id " << frag.sequence_id <<
" and fragment id " << frag.fragment_id <<
" because there is no room in the queue and reliable mode is off.";
179 dropped_data_[frag.fragment_id].reset(
new Fragment(frag.word_count - frag.num_words()));
181 TLOG(6) <<
"Dropping fragment with sequence id " << frag.sequence_id <<
" and fragment id " << frag.fragment_id <<
" into " << (
void*)dropped_data_[frag.fragment_id]->dataBegin() <<
" sz=" << dropped_data_[frag.fragment_id]->dataSizeBytes();
182 return dropped_data_[frag.fragment_id]->dataBegin();
186 buffer_writes_pending_[buffer]++;
190 metricMan->sendMetric(
"Input Fragment Rate", 1,
"Fragments/s", 1, MetricMode::Rate);
193 TLOG(TLVL_BUFLCK) <<
"WriteFragmentHeader: obtaining buffer_mutexes lock for buffer " << buffer;
195 std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
197 TLOG(TLVL_BUFLCK) <<
"WriteFragmentHeader: obtained buffer_mutexes lock for buffer " << buffer;
200 auto hdrpos =
reinterpret_cast<RawDataType*
>(GetWritePos(buffer));
201 Write(buffer, &frag, frag.num_words() *
sizeof(RawDataType));
203 auto pos =
reinterpret_cast<RawDataType*
>(GetWritePos(buffer));
204 if (frag.word_count - frag.num_words() > 0)
206 auto sts = IncrementWritePos(buffer, (frag.word_count - frag.num_words()) *
sizeof(RawDataType));
210 reinterpret_cast<detail::RawFragmentHeader*
>(hdrpos)->word_count = frag.num_words();
211 reinterpret_cast<detail::RawFragmentHeader*
>(hdrpos)->type = Fragment::InvalidFragmentType;
212 TLOG(TLVL_ERROR) <<
"Dropping over-size fragment with sequence id " << frag.sequence_id <<
" and fragment id " << frag.fragment_id <<
" because there is no room in the current buffer for this Fragment! (Keeping header)";
213 dropped_data_[frag.fragment_id].reset(
new Fragment(frag.word_count - frag.num_words()));
215 oversize_fragment_count_++;
217 if (maximum_oversize_fragment_count_ > 0 && oversize_fragment_count_ >= maximum_oversize_fragment_count_)
219 throw cet::exception(
"Too many over-size Fragments received! Please adjust max_event_size_bytes or max_fragment_size_bytes!");
222 TLOG(6) <<
"Dropping over-size fragment with sequence id " << frag.sequence_id <<
" and fragment id " << frag.fragment_id <<
" into " << (
void*)dropped_data_[frag.fragment_id]->dataBegin();
223 return dropped_data_[frag.fragment_id]->dataBegin();
226 TLOG(14) <<
"WriteFragmentHeader END";
233 TLOG(TLVL_TRACE) <<
"DoneWritingFragment BEGIN";
234 auto buffer = getBufferForSequenceID_(frag.sequence_id,
false, frag.timestamp);
235 if (buffer == -1) Detach(
true,
"SharedMemoryEventManager",
"getBufferForSequenceID_ returned -1 when it REALLY shouldn't have! Check program logic!");
236 if (buffer == -2) {
return; }
238 TLOG(TLVL_BUFLCK) <<
"DoneWritingFragment: obtaining buffer_mutexes lock for buffer " << buffer;
240 std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
242 TLOG(TLVL_BUFLCK) <<
"DoneWritingFragment: obtained buffer_mutexes lock for buffer " << buffer;
246 TLOG(TLVL_DEBUG) <<
"DoneWritingFragment: Received Fragment with sequence ID " << frag.sequence_id <<
" and fragment id " << frag.fragment_id <<
" (type " << (int)frag.type <<
")";
247 auto hdr = getEventHeader_(buffer);
250 hdr->run_id = run_id_;
251 hdr->subrun_id = subrun_id_;
254 buffer_writes_pending_[buffer]--;
255 if (buffer_writes_pending_[buffer] != 0)
257 TLOG(TLVL_TRACE) <<
"Done writing fragment, but there's another writer. Not doing bookkeeping steps.";
260 TLOG(TLVL_TRACE) <<
"Done writing fragment, and no other writer. Doing bookkeeping steps.";
261 auto frag_count = GetFragmentCount(frag.sequence_id);
262 hdr->is_complete = frag_count == num_fragments_per_event_;
263 TLOG(TLVL_TRACE) <<
"DoneWritingFragment: Received Fragment with sequence ID " << frag.sequence_id <<
" and fragment id " << frag.fragment_id <<
", count/expected = " << frag_count <<
"/" << num_fragments_per_event_;
264 #if ART_SUPPORTS_DUPLICATE_EVENTS
265 if (!hdr->is_complete && released_incomplete_events_.count(frag.sequence_id))
267 hdr->is_complete = frag_count == released_incomplete_events_[frag.sequence_id] && buffer_writes_pending_[buffer] == 0;
271 complete_buffer_(buffer);
272 if (requests_) requests_->SendRequest(
true);
273 TLOG(TLVL_TRACE) <<
"DoneWritingFragment END";
278 return GetFragmentCountInBuffer(getBufferForSequenceID_(seqID,
false), type);
283 if (buffer == -1)
return 0;
284 ResetReadPos(buffer);
285 IncrementReadPos(buffer,
sizeof(detail::RawEventHeader));
289 while (MoreDataInBuffer(buffer))
291 auto fragHdr =
reinterpret_cast<artdaq::detail::RawFragmentHeader*
>(GetReadPos(buffer));
292 IncrementReadPos(buffer, fragHdr->word_count *
sizeof(RawDataType));
293 if (type != Fragment::InvalidFragmentType && fragHdr->type != type)
continue;
294 TLOG(TLVL_TRACE) <<
"Adding Fragment with size=" << fragHdr->word_count <<
" to Fragment count";
305 auto start_time = std::chrono::steady_clock::now();
307 TLOG(TLVL_INFO) <<
"Starting art process with config file " << config_file->getFileName();
313 char* filename =
new char[config_file->getFileName().length() + 1];
314 strcpy(filename, config_file->getFileName().c_str());
316 std::vector<char*> args{ (
char*)
"art", (
char*)
"-c", filename, NULL };
324 std::string envVarKey =
"ARTDAQ_PARTITION_NUMBER";
325 std::string envVarValue = std::to_string(GetPartitionNumber());
326 if (setenv(envVarKey.c_str(), envVarValue.c_str(), 1) != 0)
328 TLOG(TLVL_ERROR) <<
"Error setting environment variable \"" << envVarKey
329 <<
"\" in the environment of a child art process. "
330 <<
"This may result in incorrect TCP port number "
331 <<
"assignments or other issues, and data may "
332 <<
"not flow through the system correctly.";
335 execvp(
"art", &args[0]);
344 std::cout <<
"Please run the following command in a separate terminal:" << std::endl
345 <<
"art -c " << config_file->getFileName() << std::endl
346 <<
"Then, in a third terminal, execute: \"ps aux|grep [a]rt -c " << config_file->getFileName() <<
"\" and note the PID of the art process." << std::endl
347 <<
"Finally, return to this window and enter the pid: " << std::endl;
352 TLOG(TLVL_INFO) <<
"PID of new art process is " << pid;
354 std::unique_lock<std::mutex> lk(art_process_mutex_);
355 art_processes_.insert(pid);
358 auto sts = waitid(P_PID, pid, &status, WEXITED);
359 TLOG(TLVL_INFO) <<
"Removing PID " << pid <<
" from process list";
361 std::unique_lock<std::mutex> lk(art_process_mutex_);
362 art_processes_.erase(pid);
366 TLOG(TLVL_WARNING) <<
"Error occurred in waitid for art process " << pid <<
": " << errno <<
" (" << strerror(errno) <<
").";
368 else if (status.si_code == CLD_EXITED && status.si_status == 0)
370 TLOG(TLVL_INFO) <<
"art process " << pid <<
" exited normally, " << (restart_art_ ?
"restarting" :
"not restarting");
374 auto art_lifetime = TimeUtils::GetElapsedTime(start_time);
375 if (art_lifetime < minimum_art_lifetime_s_) restart_art_ =
false;
377 auto exit_type =
"exited with status code";
378 switch (status.si_code)
382 exit_type =
"was killed with signal";
389 TLOG((restart_art_ ? TLVL_WARNING : TLVL_ERROR))
390 <<
"art process " << pid <<
" " << exit_type <<
" " << status.si_status
391 << (status.si_code == CLD_DUMPED ?
" (core dumped)" :
"")
392 <<
" after running for " << std::setprecision(2) << std::fixed << art_lifetime <<
" seconds, "
393 << (restart_art_ ?
"restarting" :
"not restarting");
395 }
while (restart_art_);
400 restart_art_ = always_restart_art_;
401 if (num_art_processes_ == 0)
return;
402 for (
size_t ii = 0; ii < num_art_processes_; ++ii)
404 StartArtProcess(current_art_pset_);
410 static std::mutex start_art_mutex;
411 std::unique_lock<std::mutex> lk(start_art_mutex);
413 restart_art_ = always_restart_art_;
414 auto initialCount = GetAttachedCount();
415 auto startTime = std::chrono::steady_clock::now();
417 if (pset != current_art_pset_ || !current_art_config_file_)
419 current_art_pset_ = pset;
420 current_art_config_file_ = std::make_shared<art_config_file>(pset);
422 std::shared_ptr<std::atomic<pid_t>> pid(
new std::atomic<pid_t>(-1));
423 boost::thread thread([&] { RunArt(current_art_config_file_, pid); });
426 auto currentCount = GetAttachedCount() - initialCount;
427 while ((currentCount < 1 || *pid <= 0) && (TimeUtils::GetElapsedTime(startTime) < 5 || manual_art_))
430 currentCount = GetAttachedCount() - initialCount;
432 if ((currentCount < 1 || *pid <= 0) && manual_art_)
434 TLOG(TLVL_WARNING) <<
"Manually-started art process has not connected to shared memory or has bad PID: connected:" << currentCount <<
", PID:" << pid;
437 else if (currentCount < 1 || *pid <= 0)
439 TLOG(TLVL_WARNING) <<
"art process has not started after 5s. Check art configuration!"
440 <<
" (pid=" << *pid <<
", attachedCount=" << currentCount <<
")";
445 TLOG(TLVL_INFO) << std::setw(4) << std::fixed <<
"art initialization took "
446 << TimeUtils::GetElapsedTime(startTime) <<
" seconds.";
455 restart_art_ =
false;
459 auto check_pids = [&](
bool print) {
461 for (
auto pid = pids.begin(); pid != pids.end();)
465 std::unique_lock<std::mutex> lk(art_process_mutex_);
468 TLOG(TLVL_WARNING) <<
"Removing an invalid PID (" << *pid
469 <<
") from the shutdown list.";
470 pid = pids.erase(pid);
472 else if (kill(*pid, 0) < 0)
474 pid = pids.erase(pid);
478 if (print) std::cout << *pid <<
" ";
484 if (pids.size() == 0)
486 TLOG(14) <<
"All art processes already exited, nothing to do.";
493 TLOG(TLVL_TRACE) <<
"Gently informing art processes that it is time to shut down";
494 for (
auto pid : pids)
496 TLOG(TLVL_TRACE) <<
"Sending SIGQUIT to pid " << pid;
500 int graceful_wait_ms = 5000;
501 int int_wait_ms = 1000;
503 TLOG(TLVL_TRACE) <<
"Waiting up to " << graceful_wait_ms <<
" ms for all art processes to exit gracefully";
504 for (
int ii = 0; ii < graceful_wait_ms; ++ii)
509 if (pids.size() == 0)
511 TLOG(TLVL_TRACE) <<
"All art processes exited after " << ii <<
" ms.";
516 TLOG(TLVL_TRACE) <<
"Insisting that the art processes shut down";
517 for (
auto pid : pids)
522 TLOG(TLVL_TRACE) <<
"Waiting up to " << int_wait_ms <<
" ms for all art processes to exit";
523 for (
int ii = graceful_wait_ms; ii < graceful_wait_ms + int_wait_ms; ++ii)
529 if (pids.size() == 0)
531 TLOG(TLVL_TRACE) <<
"All art processes exited after " << ii <<
" ms.";
536 TLOG(TLVL_TRACE) <<
"Killing remaning art processes with extreme prejudice";
537 while (pids.size() > 0)
539 kill(*pids.begin(), SIGKILL);
547 std::cout <<
"Please shut down all art processes, then hit return/enter" << std::endl;
548 while (pids.size() > 0)
550 std::cout <<
"The following PIDs are running: ";
552 std::cout << std::endl;
561 TLOG(TLVL_DEBUG) <<
"ReconfigureArt BEGIN";
562 if (restart_art_ || !always_restart_art_)
566 for (
size_t ii = 0; ii < broadcasts_.size(); ++ii)
568 broadcasts_.MarkBufferEmpty(ii,
true);
570 if (newRun == 0) newRun = run_id_ + 1;
572 if (art_pset != current_art_pset_ || !current_art_config_file_)
574 current_art_pset_ = art_pset;
575 current_art_config_file_ = std::make_shared<art_config_file>(art_pset);
578 if (n_art_processes != -1)
580 TLOG(TLVL_INFO) <<
"Setting number of art processes to " << n_art_processes;
581 num_art_processes_ = n_art_processes;
584 TLOG(TLVL_DEBUG) <<
"ReconfigureArt END";
590 init_fragment_.reset(
nullptr);
591 TLOG(TLVL_DEBUG) <<
"SharedMemoryEventManager::endOfData";
592 restart_art_ =
false;
594 size_t initialStoreSize = GetIncompleteEventCount();
595 TLOG(TLVL_DEBUG) <<
"endOfData: Flushing " << initialStoreSize
596 <<
" stale events from the SharedMemoryEventManager.";
597 int counter = initialStoreSize;
598 while (active_buffers_.size() > 0 && counter > 0)
600 complete_buffer_(*active_buffers_.begin());
603 TLOG(TLVL_DEBUG) <<
"endOfData: Done flushing, there are now " << GetIncompleteEventCount()
604 <<
" stale events in the SharedMemoryEventManager.";
607 TLOG(TLVL_DEBUG) <<
"Waiting for " << (ReadReadyCount() + (size() - WriteReadyCount(overwrite_mode_))) <<
" outstanding buffers...";
608 auto start = std::chrono::steady_clock::now();
609 auto lastReadCount = ReadReadyCount() + (size() - WriteReadyCount(overwrite_mode_));
610 auto end_of_data_wait_us = art_event_processing_time_us_ * lastReadCount;
612 auto outstanding_buffer_wait_time = art_event_processing_time_us_ > 100000 ? 100000 : art_event_processing_time_us_;
615 while (lastReadCount > 0 && (end_of_data_wait_us == 0 || TimeUtils::GetElapsedTimeMicroseconds(start) < end_of_data_wait_us) && get_art_process_count_() > 0)
617 auto temp = ReadReadyCount() + (size() - WriteReadyCount(overwrite_mode_));
618 if (temp != lastReadCount)
620 TLOG(TLVL_TRACE) <<
"Waiting for " << temp <<
" outstanding buffers...";
621 lastReadCount = temp;
622 start = std::chrono::steady_clock::now();
624 if (lastReadCount > 0) {
625 TRACE(19,
"About to sleep %lu us - lastReadCount=%lu size=%lu end_of_data_wait_us=%lu",outstanding_buffer_wait_time,lastReadCount,size(),end_of_data_wait_us );
626 usleep(outstanding_buffer_wait_time);
630 TLOG(TLVL_DEBUG) <<
"endOfData: After wait for outstanding buffers. Still outstanding: " << lastReadCount <<
", time waited: "
631 << TimeUtils::GetElapsedTime(start) <<
" s / " << (end_of_data_wait_us / 1000000.0) <<
" s, art process count: " << get_art_process_count_();
633 TLOG(TLVL_DEBUG) <<
"endOfData: Broadcasting EndOfData Fragment";
634 FragmentPtr outFrag = Fragment::eodFrag(GetBufferCount());
635 bool success = broadcastFragment_(std::move(outFrag), outFrag);
638 TLOG(TLVL_DEBUG) <<
"endOfData: Clearing buffers to make room for EndOfData Fragment";
639 for (
size_t ii = 0; ii < broadcasts_.size(); ++ii)
641 broadcasts_.MarkBufferEmpty(ii,
true);
643 broadcastFragment_(std::move(outFrag), outFrag);
645 auto endOfDataProcessingStart = std::chrono::steady_clock::now();
647 if (get_art_process_count_() > 0)
649 TLOG(TLVL_DEBUG) <<
"Allowing " << get_art_process_count_() <<
" art processes the chance to end gracefully";
650 if (end_of_data_wait_us == 0)
652 TLOG(TLVL_DEBUG) <<
"Expected art event processing time not specified. Waiting up to 100s for art to end gracefully.";
653 end_of_data_wait_us = 100 * 1000000;
656 auto sleep_count = (end_of_data_wait_us / 10000) + 1;
657 for (
size_t ii = 0; ii < sleep_count; ++ii)
660 if (get_art_process_count_() == 0)
break;
664 while (get_art_process_count_() > 0)
666 TLOG(TLVL_DEBUG) <<
"There are " << get_art_process_count_() <<
" art processes remaining. Proceeding to shutdown.";
668 ShutdownArtProcesses(art_processes_);
670 TLOG(TLVL_DEBUG) <<
"It took " << TimeUtils::GetElapsedTime(endOfDataProcessingStart) <<
" s for all art processes to close after sending EndOfData Fragment";
672 ResetAttachedCount();
674 TLOG(TLVL_DEBUG) <<
"endOfData: Clearing buffers";
675 for (
size_t ii = 0; ii < size(); ++ii)
677 MarkBufferEmpty(ii,
true);
685 released_incomplete_events_.clear();
687 TLOG(TLVL_DEBUG) <<
"endOfData: Shutting down RequestReceiver";
688 requests_.reset(
nullptr);
690 TLOG(TLVL_DEBUG) <<
"endOfData END";
691 TLOG(TLVL_INFO) <<
"EndOfData Complete. There were " << GetLastSeenBufferID() <<
" buffers processed.";
698 init_fragment_.reset(
nullptr);
699 TLOG(TLVL_TRACE) <<
"startRun: Clearing broadcast buffers";
700 for (
size_t ii = 0; ii < broadcasts_.size(); ++ii)
702 broadcasts_.MarkBufferEmpty(ii,
true);
707 subrun_rollover_event_ = Fragment::InvalidSequenceID;
708 last_released_event_ = 0;
712 requests_->SendRoutingToken(queue_size_);
714 TLOG(TLVL_DEBUG) <<
"Starting run " << run_id_
715 <<
", max queue size = "
718 << GetLockedBufferCount();
721 double runSubrun = run_id_ + ((double)subrun_id_ / 10000);
722 metricMan->sendMetric(
"Run Number", runSubrun,
"Run:Subrun", 1, MetricMode::LastPoint);
729 subrun_rollover_event_ = Fragment::InvalidSequenceID;
732 double runSubrun = run_id_ + ((double)subrun_id_ / 10000);
733 metricMan->sendMetric(
"Run Number", runSubrun,
"Run:Subrun", 1, MetricMode::LastPoint);
739 TLOG(TLVL_INFO) <<
"Ending run " << run_id_;
740 FragmentPtr endOfRunFrag(
new
741 Fragment(static_cast<size_t>
742 (ceil(
sizeof(my_rank) /
743 static_cast<double>(
sizeof(Fragment::value_type))))));
745 TLOG(TLVL_DEBUG) <<
"Broadcasting EndOfRun Fragment";
746 endOfRunFrag->setSystemType(Fragment::EndOfRunFragmentType);
747 *endOfRunFrag->dataBegin() = my_rank;
748 broadcastFragment_(std::move(endOfRunFrag), endOfRunFrag);
750 TLOG(TLVL_INFO) <<
"Run " << run_id_ <<
" has ended. There were " << run_event_count_ <<
" events in this run.";
751 run_event_count_ = 0;
752 run_incomplete_event_count_ = 0;
753 oversize_fragment_count_ = 0;
759 TLOG(TLVL_INFO) <<
"Ending subrun " << subrun_id_;
760 std::unique_ptr<artdaq::Fragment>
762 Fragment(static_cast<size_t>
763 (ceil(
sizeof(my_rank) /
764 static_cast<double>(
sizeof(Fragment::value_type))))));
766 TLOG(TLVL_DEBUG) <<
"Broadcasting EndOfSubrun Fragment";
767 endOfSubrunFrag->setSequenceID(subrun_rollover_event_);
768 endOfSubrunFrag->setSystemType(Fragment::EndOfSubrunFragmentType);
769 *endOfSubrunFrag->dataBegin() = my_rank;
771 broadcastFragment_(std::move(endOfSubrunFrag), endOfSubrunFrag);
773 TLOG(TLVL_INFO) <<
"Subrun " << subrun_id_ <<
" in run " << run_id_ <<
" has ended. There were " << subrun_event_count_ <<
" events in this subrun.";
774 subrun_event_count_ = 0;
775 subrun_incomplete_event_count_ = 0;
783 if (boundary == 0 || boundary == Fragment::InvalidSequenceID)
return;
785 if (boundary < last_released_event_)
787 auto logLevel = TLVL_ERROR;
788 bool processAnyway =
false;
789 if (last_released_event_ - boundary < 100)
791 logLevel = TLVL_WARNING;
792 processAnyway =
true;
794 TLOG(logLevel) <<
"Subrun rollover requested for event that is in the past. (last_released_event="
795 << last_released_event_ <<
",requested_rollover_boundary=" << boundary <<
").";
796 if (!processAnyway)
return;
798 TLOG(TLVL_INFO) <<
"Will roll over when I reach Sequence ID " << boundary;
807 if (boundary == last_released_event_ + 1) {
808 TLOG(TLVL_INFO) <<
"rolloverSubrun: Last released event had sequence id " << last_released_event_ << \
809 ", boundary is sequence id " << boundary <<
", so will start a new subrun here";
812 subrun_rollover_event_ = std::numeric_limits<sequence_id_t>::max();
815 subrun_rollover_event_ = boundary;
823 metricMan->sendMetric(
"Incomplete Event Count", GetIncompleteEventCount(),
"events", 1, MetricMode::LastPoint);
824 metricMan->sendMetric(
"Pending Event Count", GetPendingEventCount(),
"events", 1, MetricMode::LastPoint);
827 if (incomplete_event_report_interval_ms_ > 0 && GetLockedBufferCount())
829 if (TimeUtils::GetElapsedTimeMilliseconds(last_incomplete_event_report_time_) < static_cast<size_t>(incomplete_event_report_interval_ms_))
832 last_incomplete_event_report_time_ = std::chrono::steady_clock::now();
833 std::ostringstream oss;
834 oss <<
"Incomplete Events (" << num_fragments_per_event_ <<
"): ";
835 for (
auto& ev : active_buffers_)
837 auto hdr = getEventHeader_(ev);
838 oss << hdr->sequence_id <<
" (" << GetFragmentCount(hdr->sequence_id) <<
"), ";
840 TLOG(TLVL_DEBUG) << oss.str();
844 bool artdaq::SharedMemoryEventManager::broadcastFragment_(FragmentPtr frag, FragmentPtr& outFrag)
846 TLOG(TLVL_DEBUG) <<
"Broadcasting Fragment with seqID=" << frag->sequenceID() <<
", type " << detail::RawFragmentHeader::SystemTypeToString(frag->type()) <<
", size=" << frag->sizeBytes() <<
"B.";
847 auto buffer = broadcasts_.GetBufferForWriting(
false);
848 TLOG(TLVL_DEBUG) <<
"broadcastFragment_: after getting buffer 1st buffer=" << buffer;
849 auto start_time = std::chrono::steady_clock::now();
850 while (buffer == -1 && TimeUtils::GetElapsedTimeMilliseconds(start_time) < static_cast<size_t>(broadcast_timeout_ms_))
853 buffer = broadcasts_.GetBufferForWriting(
false);
855 TLOG(TLVL_DEBUG) <<
"broadcastFragment_: after getting buffer w/timeout, buffer=" << buffer <<
", elapsed time=" << TimeUtils::GetElapsedTime(start_time) <<
" s.";
858 TLOG(TLVL_ERROR) <<
"Broadcast of fragment type " << frag->typeString() <<
" failed due to timeout waiting for buffer!";
863 TLOG(TLVL_DEBUG) <<
"broadcastFragment_: Filling in RawEventHeader";
864 auto hdr =
reinterpret_cast<detail::RawEventHeader*
>(broadcasts_.GetBufferStart(buffer));
865 hdr->run_id = run_id_;
866 hdr->subrun_id = subrun_id_;
867 hdr->sequence_id = frag->sequenceID();
868 hdr->is_complete =
true;
869 broadcasts_.IncrementWritePos(buffer,
sizeof(detail::RawEventHeader));
871 TLOG(TLVL_DEBUG) <<
"broadcastFragment_ before Write calls";
872 broadcasts_.Write(buffer, frag->headerAddress(), frag->size() *
sizeof(RawDataType));
874 TLOG(TLVL_DEBUG) <<
"broadcastFragment_ Marking buffer full";
875 broadcasts_.MarkBufferFull(buffer, -1);
877 TLOG(TLVL_DEBUG) <<
"broadcastFragment_ Complete";
881 artdaq::detail::RawEventHeader* artdaq::SharedMemoryEventManager::getEventHeader_(
int buffer)
883 return reinterpret_cast<detail::RawEventHeader*
>(GetBufferStart(buffer));
886 int artdaq::SharedMemoryEventManager::getBufferForSequenceID_(Fragment::sequence_id_t seqID,
bool create_new, Fragment::timestamp_t timestamp)
888 TLOG(14) <<
"getBufferForSequenceID " << seqID <<
" BEGIN";
889 std::unique_lock<std::mutex> lk(sequence_id_mutex_);
891 TLOG(14) <<
"getBufferForSequenceID obtained sequence_id_mutex for seqid=" << seqID;
893 auto buffers = GetBuffersOwnedByManager();
894 for (
auto& buf : buffers)
896 auto hdr = getEventHeader_(buf);
897 if (hdr->sequence_id == seqID)
899 TLOG(14) <<
"getBufferForSequenceID " << seqID <<
" returning " << buf;
904 #if !ART_SUPPORTS_DUPLICATE_EVENTS
905 if (released_incomplete_events_.count(seqID))
907 TLOG(TLVL_ERROR) <<
"Event " << seqID <<
" has already been marked \"Incomplete\" and sent to art!";
912 if (!create_new)
return -1;
914 check_pending_buffers_(lk);
915 int new_buffer = GetBufferForWriting(
false);
917 if (new_buffer == -1)
919 new_buffer = GetBufferForWriting(overwrite_mode_);
922 if (new_buffer == -1)
return -1;
923 TLOG(TLVL_BUFLCK) <<
"getBufferForSequenceID_: obtaining buffer_mutexes lock for buffer " << new_buffer;
924 std::unique_lock<std::mutex> buffer_lk(buffer_mutexes_[new_buffer]);
925 TLOG(TLVL_BUFLCK) <<
"getBufferForSequenceID_: obtained buffer_mutexes lock for buffer " << new_buffer;
927 auto hdr = getEventHeader_(new_buffer);
928 hdr->is_complete =
false;
929 hdr->run_id = run_id_;
930 hdr->subrun_id = subrun_id_;
931 hdr->event_id = use_sequence_id_for_event_number_ ?
static_cast<uint32_t
>(seqID) : static_cast<uint32_t>(timestamp);
932 hdr->sequence_id = seqID;
933 buffer_writes_pending_[new_buffer] = 0;
934 IncrementWritePos(new_buffer,
sizeof(detail::RawEventHeader));
935 SetMFIteration(
"Sequence ID " + std::to_string(seqID));
937 TLOG(TLVL_BUFFER) <<
"getBufferForSequenceID placing " << new_buffer <<
" to active.";
938 active_buffers_.insert(new_buffer);
939 TLOG(TLVL_BUFFER) <<
"Buffer occupancy now (total,full,reading,empty,pending,active)=("
941 << ReadReadyCount() <<
","
942 << WriteReadyCount(
true) - WriteReadyCount(
false) - ReadReadyCount() <<
","
943 << WriteReadyCount(
false) <<
","
944 << pending_buffers_.size() <<
","
945 << active_buffers_.size() <<
")";
949 if (timestamp != Fragment::InvalidTimestamp)
951 requests_->AddRequest(seqID, timestamp);
957 requests_->SendRequest();
960 TLOG(14) <<
"getBufferForSequenceID " << seqID <<
" returning newly initialized buffer " << new_buffer;
964 bool artdaq::SharedMemoryEventManager::hasFragments_(
int buffer)
966 if (buffer == -1)
return true;
967 if (!CheckBuffer(buffer, BufferSemaphoreFlags::Writing))
971 ResetReadPos(buffer);
972 IncrementReadPos(buffer,
sizeof(detail::RawEventHeader));
973 return MoreDataInBuffer(buffer);
976 void artdaq::SharedMemoryEventManager::complete_buffer_(
int buffer)
978 auto hdr = getEventHeader_(buffer);
979 if (hdr->is_complete)
981 TLOG(TLVL_DEBUG) <<
"complete_buffer_: This fragment completes event " << hdr->sequence_id <<
".";
984 TLOG(TLVL_BUFFER) <<
"complete_buffer_ moving " << buffer <<
" from active to pending.";
986 TLOG(TLVL_BUFLCK) <<
"complete_buffer_: obtaining sequence_id_mutex lock for seqid=" << hdr->sequence_id;
987 std::unique_lock<std::mutex> lk(sequence_id_mutex_);
988 TLOG(TLVL_BUFLCK) <<
"complete_buffer_: obtained sequence_id_mutex lock for seqid=" << hdr->sequence_id;
989 active_buffers_.erase(buffer);
990 pending_buffers_.insert(buffer);
992 TLOG(TLVL_BUFFER) <<
"Buffer occupancy now (total,full,reading,empty,pending,active)=("
994 << ReadReadyCount() <<
","
995 << WriteReadyCount(
true) - WriteReadyCount(
false) - ReadReadyCount() <<
","
996 << WriteReadyCount(
false) <<
","
997 << pending_buffers_.size() <<
","
998 << active_buffers_.size() <<
")";
1002 requests_->RemoveRequest(hdr->sequence_id);
1005 CheckPendingBuffers();
1008 bool artdaq::SharedMemoryEventManager::bufferComparator(
int bufA,
int bufB)
1010 return getEventHeader_(bufA)->sequence_id < getEventHeader_(bufB)->sequence_id;
1015 TLOG(TLVL_BUFLCK) <<
"CheckPendingBuffers: Obtaining sequence_id_mutex_";
1016 std::unique_lock<std::mutex> lk(sequence_id_mutex_);
1017 TLOG(TLVL_BUFLCK) <<
"CheckPendingBuffers: Obtained sequence_id_mutex_";
1018 check_pending_buffers_(lk);
1021 void artdaq::SharedMemoryEventManager::check_pending_buffers_(std::unique_lock<std::mutex>
const& lock)
1023 TLOG(TLVL_TRACE) <<
"check_pending_buffers_ BEGIN Locked=" << std::boolalpha << lock.owns_lock();
1025 auto buffers = GetBuffersOwnedByManager();
1026 for (
auto buf : buffers)
1028 if (ResetBuffer(buf) && !pending_buffers_.count(buf))
1030 TLOG(15) <<
"check_pending_buffers_ Incomplete buffer detected, buf=" << buf <<
" active_bufers_.count(buf)=" << active_buffers_.count(buf) <<
" buffer_writes_pending_[buf]=" << buffer_writes_pending_[buf].load();
1031 auto hdr = getEventHeader_(buf);
1032 if (active_buffers_.count(buf) && (buffer_writes_pending_[buf].load() == 0 || !running_))
1036 requests_->RemoveRequest(hdr->sequence_id);
1038 TLOG(TLVL_BUFFER) <<
"check_pending_buffers_ moving buffer " << buf <<
" from active to pending";
1039 active_buffers_.erase(buf);
1040 pending_buffers_.insert(buf);
1041 TLOG(TLVL_BUFFER) <<
"Buffer occupancy now (total,full,reading,empty,pending,active)=("
1043 << ReadReadyCount() <<
","
1044 << WriteReadyCount(
true) - WriteReadyCount(
false) - ReadReadyCount() <<
","
1045 << WriteReadyCount(
false) <<
","
1046 << pending_buffers_.size() <<
","
1047 << active_buffers_.size() <<
")";
1049 subrun_incomplete_event_count_++;
1050 run_incomplete_event_count_++;
1051 if (metricMan) metricMan->sendMetric(
"Incomplete Event Rate", 1,
"events/s", 3, MetricMode::Rate);
1052 if (!released_incomplete_events_.count(hdr->sequence_id))
1054 released_incomplete_events_[hdr->sequence_id] = num_fragments_per_event_ - GetFragmentCountInBuffer(buf);
1058 released_incomplete_events_[hdr->sequence_id] -= GetFragmentCountInBuffer(buf);
1060 TLOG(TLVL_WARNING) <<
"Active event " << hdr->sequence_id <<
" is stale. Scheduling release of incomplete event (missing " << released_incomplete_events_[hdr->sequence_id] <<
" Fragments) to art.";
1066 Fragment::sequence_id_t lowestSeqId = Fragment::InvalidSequenceID;
1069 if (ReadyForWrite(
false))
1071 for (
auto buf : active_buffers_)
1073 auto hdr = getEventHeader_(buf);
1074 TLOG(TLVL_TRACE) <<
"Buffer: " << buf <<
", SeqID: " << hdr->sequence_id <<
", ACTIVE";
1075 if (hdr->sequence_id < lowestSeqId)
1077 lowestSeqId = hdr->sequence_id;
1080 TLOG(TLVL_TRACE) <<
"Lowest SeqID held: " << lowestSeqId;
1083 std::list<int> sorted_buffers(pending_buffers_.begin(), pending_buffers_.end());
1084 sorted_buffers.sort([
this](
int a,
int b) {
return bufferComparator(a, b); });
1087 double eventSize = 0;
1088 for (
auto buf : sorted_buffers)
1090 auto hdr = getEventHeader_(buf);
1091 if (hdr->sequence_id > lowestSeqId)
break;
1093 if (hdr->sequence_id >= subrun_rollover_event_)
1095 TLOG(TLVL_INFO) <<
"Subrun rollover reached at event " << hdr->sequence_id <<
" (boundary=" << subrun_rollover_event_ <<
"), last released event is " << last_released_event_ <<
".";
1099 if (hdr->sequence_id > last_released_event_) last_released_event_ = hdr->sequence_id;
1101 TLOG(TLVL_DEBUG) <<
"Releasing event " << std::to_string(hdr->sequence_id) <<
" in buffer " << buf <<
" to art, "
1102 <<
"event_size=" << BufferDataSize(buf) <<
", buffer_size=" << BufferSize();
1104 TLOG(TLVL_BUFFER) <<
"check_pending_buffers_ removing buffer " << buf <<
" moving from pending to full";
1105 MarkBufferFull(buf);
1106 subrun_event_count_++;
1109 eventSize += BufferDataSize(buf);
1110 pending_buffers_.erase(buf);
1111 TLOG(TLVL_BUFFER) <<
"Buffer occupancy now (total,full,reading,empty,pending,active)=("
1113 << ReadReadyCount() <<
","
1114 << WriteReadyCount(
true) - WriteReadyCount(
false) - ReadReadyCount() <<
","
1115 << WriteReadyCount(
false) <<
","
1116 << pending_buffers_.size() <<
","
1117 << active_buffers_.size() <<
")";
1122 auto outstanding_tokens = requests_->GetSentTokenCount() - run_event_count_;
1123 auto available_buffers = WriteReadyCount(overwrite_mode_);
1125 TLOG(TLVL_TRACE) <<
"check_pending_buffers_: outstanding_tokens: " << outstanding_tokens <<
", available_buffers: " << available_buffers
1126 <<
", tokens_to_send: " << available_buffers - outstanding_tokens;
1128 if (available_buffers > outstanding_tokens)
1130 auto tokens_to_send = available_buffers - outstanding_tokens;
1132 while (tokens_to_send > 0)
1134 TLOG(35) <<
"check_pending_buffers_: Sending a Routing Token";
1135 requests_->SendRoutingToken(1);
1141 metric_data_.event_count += counter;
1142 metric_data_.event_size += eventSize;
1144 if (metricMan && TimeUtils::GetElapsedTimeMilliseconds(last_shmem_buffer_metric_update_) > 500)
1146 TLOG(TLVL_TRACE) <<
"check_pending_buffers_: Sending Metrics";
1147 metricMan->sendMetric(
"Event Rate", metric_data_.event_count,
"Events/s", 1, MetricMode::Rate);
1148 if (metric_data_.event_count > 0) metricMan->sendMetric(
"Average Event Size", metric_data_.event_size / metric_data_.event_count,
"Bytes", 1, MetricMode::Average);
1149 metric_data_ = MetricData();
1151 metricMan->sendMetric(
"Events Released to art this run", run_event_count_,
"Events", 1, MetricMode::LastPoint);
1152 metricMan->sendMetric(
"Incomplete Events Released to art this run", run_incomplete_event_count_,
"Events", 1, MetricMode::LastPoint);
1153 metricMan->sendMetric(
"Events Released to art this subrun", subrun_event_count_,
"Events", 2, MetricMode::LastPoint);
1154 metricMan->sendMetric(
"Incomplete Events Released to art this subrun", subrun_incomplete_event_count_,
"Events", 2, MetricMode::LastPoint);
1155 if(requests_) metricMan->sendMetric(
"Tokens sent", requests_->GetSentTokenCount(),
"Tokens", 2, MetricMode::LastPoint);
1157 auto bufferReport = GetBufferReport();
1158 int full = std::count_if(bufferReport.begin(), bufferReport.end(), [](std::pair<int, BufferSemaphoreFlags> p) {
return p.second == BufferSemaphoreFlags::Full; });
1159 int empty = std::count_if(bufferReport.begin(), bufferReport.end(), [](std::pair<int, BufferSemaphoreFlags> p) {
return p.second == BufferSemaphoreFlags::Empty; });
1160 int writing = std::count_if(bufferReport.begin(), bufferReport.end(), [](std::pair<int, BufferSemaphoreFlags> p) {
return p.second == BufferSemaphoreFlags::Writing; });
1161 int reading = std::count_if(bufferReport.begin(), bufferReport.end(), [](std::pair<int, BufferSemaphoreFlags> p) {
return p.second == BufferSemaphoreFlags::Reading; });
1162 auto total = size();
1163 TLOG(TLVL_DEBUG) <<
"Buffer usage: full=" << full <<
", empty=" << empty <<
", writing=" << writing <<
", reading=" << reading <<
", total=" << total;
1165 metricMan->sendMetric(
"Shared Memory Full Buffers", full,
"buffers", 2, MetricMode::LastPoint);
1166 metricMan->sendMetric(
"Shared Memory Available Buffers", empty,
"buffers", 2, MetricMode::LastPoint);
1167 metricMan->sendMetric(
"Shared Memory Pending Buffers", writing,
"buffers", 2, MetricMode::LastPoint);
1168 metricMan->sendMetric(
"Shared Memory Reading Buffers", reading,
"buffers", 2, MetricMode::LastPoint);
1171 metricMan->sendMetric(
"Shared Memory Full %", full * 100 / static_cast<double>(total),
"%", 2, MetricMode::LastPoint);
1172 metricMan->sendMetric(
"Shared Memory Available %", empty * 100 / static_cast<double>(total),
"%", 2, MetricMode::LastPoint);
1175 last_shmem_buffer_metric_update_ = std::chrono::steady_clock::now();
1177 TLOG(TLVL_TRACE) <<
"check_pending_buffers_ END";
1180 void artdaq::SharedMemoryEventManager::send_init_frag_()
1182 if (init_fragment_ !=
nullptr)
1184 TLOG(TLVL_INFO) <<
"Broadcasting init fragment to all art subprocesses...";
1187 std::string fileName =
"receiveInitMessage_" + std::to_string(my_rank) +
".bin";
1188 std::fstream ostream(fileName.c_str(), std::ios::out | std::ios::binary);
1189 ostream.write(reinterpret_cast<char*>(init_fragment_->dataBeginBytes()), init_fragment_->dataSizeBytes());
1193 broadcastFragment_(std::move(init_fragment_), init_fragment_);
1194 TLOG(TLVL_TRACE) <<
"Init Fragment sent";
1196 else if (send_init_fragments_)
1198 TLOG(TLVL_WARNING) <<
"Cannot send init fragment because I haven't yet received one!";
1204 if (!init_fragment_ || init_fragment_ ==
nullptr)
1206 init_fragment_.swap(frag);
1213 TLOG(TLVL_DEBUG) <<
"UpdateArtConfiguration BEGIN";
1214 if (art_pset != current_art_pset_ || !current_art_config_file_)
1216 current_art_pset_ = art_pset;
1217 current_art_config_file_ = std::make_shared<art_config_file>(art_pset);
1219 TLOG(TLVL_DEBUG) <<
"UpdateArtConfiguration END";
1222 #if MESSAGEFACILITY_HEX_VERSION >= 0x20103
void RunArt(std::shared_ptr< art_config_file > config_file, std::shared_ptr< std::atomic< pid_t >> pid_out)
Run an art instance, recording the return codes and restarting it until the end flag is raised...
void ShutdownArtProcesses(std::set< pid_t > &pids)
Shutdown a set of art processes.
virtual ~SharedMemoryEventManager()
SharedMemoryEventManager Destructor.
The SharedMemoryEventManager is a SharedMemoryManger which tracks events as they are built...
Fragment::sequence_id_t sequence_id_t
Copy Fragment::sequence_id_t into local scope.
void ReconfigureArt(fhicl::ParameterSet art_pset, run_id_t newRun=0, int n_art_processes=-1)
Restart all art processes, using the given fhicl code to configure the new art processes.
The RequestSender contains methods used to send data requests and Routing tokens. ...
pid_t StartArtProcess(fhicl::ParameterSet pset)
Start one art process.
RawDataType * WriteFragmentHeader(detail::RawFragmentHeader frag, bool dropIfNoBuffersAvailable=false)
Get a pointer to a reserved memory area for the given Fragment header.
RawEvent::run_id_t run_id_t
Copy RawEvent::run_id_t into local scope.
size_t GetFragmentCount(Fragment::sequence_id_t seqID, Fragment::type_t type=Fragment::InvalidFragmentType)
Get the count of Fragments of a given type in an event.
void UpdateArtConfiguration(fhicl::ParameterSet art_pset)
Updates the internally-stored copy of the art configuration.
void StartArt()
Start all the art processes.
void SetInitFragment(FragmentPtr frag)
Set the stored Init fragment, if one has not yet been set already.
void rolloverSubrun(sequence_id_t boundary)
Rollover the subrun after the specified event.
void sendMetrics()
Send metrics to the MetricManager, if one has been instantiated in the application.
void startSubrun()
Start a new Subrun, incrementing the subrun number.
SharedMemoryEventManager(fhicl::ParameterSet pset, fhicl::ParameterSet art_pset)
SharedMemoryEventManager Constructor.
bool endSubrun()
Send an EndOfSubRunFragment to the art thread.
bool endRun()
Send an EndOfRunFragment to the art thread.
void DoneWritingFragment(detail::RawFragmentHeader frag)
Used to indicate that the given Fragment is now completely in the buffer. Will check for buffer compl...
bool endOfData()
Indicate that the end of input has been reached to the art processes.
void startRun(run_id_t runID)
Start a Run.
size_t GetFragmentCountInBuffer(int buffer, Fragment::type_t type=Fragment::InvalidFragmentType)
Get the count of Fragments of a given type in a buffer.
void CheckPendingBuffers()
Check for buffers which are ready to be marked incomplete and released to art and issue tokens for an...