2 #define TRACE_NAME (app_name + "_SharedMemoryEventManager").c_str()
4 #include "artdaq/DAQrate/SharedMemoryEventManager.hh"
5 #include "artdaq-core/Core/StatisticsCollection.hh"
6 #include "artdaq-core/Utilities/TraceLock.hh"
10 #define TLVL_BUFLCK 41
12 std::mutex artdaq::SharedMemoryEventManager::sequence_id_mutex_;
15 : SharedMemoryManager(pset.get<uint32_t>(
"shared_memory_key", 0xBEE70000 + getpid()),
16 pset.get<size_t>(
"buffer_count"),
17 pset.has_key(
"max_event_size_bytes") ? pset.get<size_t>(
"max_event_size_bytes") : pset.get<size_t>(
"expected_fragments_per_event") * pset.get<size_t>(
"max_fragment_size_bytes"),
18 pset.get<size_t>(
"stale_buffer_timeout_usec", pset.get<size_t>(
"event_queue_wait_time", 5) * 1000000),
19 !pset.get<bool>(
"broadcast_mode", false))
20 , num_art_processes_(pset.get<size_t>(
"art_analyzer_count", 1))
21 , num_fragments_per_event_(pset.get<size_t>(
"expected_fragments_per_event"))
22 , queue_size_(pset.get<size_t>(
"buffer_count"))
25 , subrun_rollover_event_(Fragment::InvalidSequenceID)
26 , last_released_event_(0)
27 , update_run_ids_(pset.get<bool>(
"update_run_ids_on_new_fragment", true))
28 , use_sequence_id_for_event_number_(pset.get<bool>(
"use_sequence_id_for_event_number", true))
29 , overwrite_mode_(!pset.get<bool>(
"use_art", true) || pset.get<bool>(
"overwrite_mode", false) || pset.get<bool>(
"broadcast_mode", false))
30 , send_init_fragments_(pset.get<bool>(
"send_init_fragments", true))
32 , buffer_writes_pending_()
33 , incomplete_event_report_interval_ms_(pset.get<int>(
"incomplete_event_report_interval_ms", -1))
34 , last_incomplete_event_report_time_(std::chrono::steady_clock::now())
35 , last_shmem_buffer_metric_update_(std::chrono::steady_clock::now())
37 , broadcast_timeout_ms_(pset.get<int>(
"fragment_broadcast_timeout_ms", 3000))
39 , run_incomplete_event_count_(0)
40 , subrun_event_count_(0)
41 , subrun_incomplete_event_count_(0)
42 , oversize_fragment_count_(0)
43 , maximum_oversize_fragment_count_(pset.get<int>(
"maximum_oversize_fragment_count", 1))
46 , always_restart_art_(pset.get<bool>(
"restart_crashed_art_processes", true))
47 , manual_art_(pset.get<bool>(
"manual_art", false))
48 , current_art_pset_(art_pset)
49 , minimum_art_lifetime_s_(pset.get<double>(
"minimum_art_lifetime_s", 2.0))
50 , art_event_processing_time_us_(pset.get<size_t>(
"expected_art_event_processing_time_us", 100000))
54 , broadcasts_(pset.get<uint32_t>(
"broadcast_shared_memory_key", 0xCEE70000 + getpid()),
55 pset.get<size_t>(
"broadcast_buffer_count", 10),
56 pset.get<size_t>(
"broadcast_buffer_size", 0x100000),
57 pset.get<int>(
"expected_art_event_processing_time_us", 100000) * pset.get<size_t>(
"buffer_count"), false)
59 SetMinWriteSize(
sizeof(detail::RawEventHeader) +
sizeof(detail::RawFragmentHeader));
60 broadcasts_.SetMinWriteSize(
sizeof(detail::RawEventHeader) +
sizeof(detail::RawFragmentHeader));
62 if (pset.get<
bool>(
"use_art",
true) ==
false)
64 TLOG(TLVL_INFO) <<
"BEGIN SharedMemoryEventManager CONSTRUCTOR with use_art:false";
65 num_art_processes_ = 0;
69 TLOG(TLVL_INFO) <<
"BEGIN SharedMemoryEventManager CONSTRUCTOR with use_art:true";
70 TLOG(TLVL_TRACE) <<
"art_pset is " << art_pset.to_string();
72 current_art_config_file_ = std::make_shared<art_config_file>(art_pset);
74 if (overwrite_mode_ && num_art_processes_ > 0)
76 TLOG(TLVL_WARNING) <<
"Art is configured to run, but overwrite mode is enabled! Check your configuration if this in unintentional!";
78 else if (overwrite_mode_)
80 TLOG(TLVL_INFO) <<
"Overwrite Mode enabled, no configured art processes at startup";
83 for (
size_t ii = 0; ii < size(); ++ii)
85 buffer_writes_pending_[ii] = 0;
88 if (!IsValid())
throw cet::exception(app_name +
"_SharedMemoryEventManager") <<
"Unable to attach to Shared Memory!";
90 TLOG(TLVL_TRACE) <<
"Setting Writer rank to " << my_rank;
92 TLOG(TLVL_DEBUG) <<
"Writer Rank is " << GetRank();
95 TLOG(TLVL_TRACE) <<
"END CONSTRUCTOR";
100 TLOG(TLVL_TRACE) <<
"DESTRUCTOR";
101 if (running_) endOfData();
102 TLOG(TLVL_TRACE) <<
"Destructor END";
105 bool artdaq::SharedMemoryEventManager::AddFragment(detail::RawFragmentHeader frag,
void* dataPtr)
107 TLOG(TLVL_TRACE) <<
"AddFragment(Header, ptr) BEGIN frag.word_count=" << frag.word_count
108 <<
", sequence_id=" << frag.sequence_id;
109 auto buffer = getBufferForSequenceID_(frag.sequence_id,
true, frag.timestamp);
110 TLOG(TLVL_TRACE) <<
"Using buffer " << buffer <<
" for seqid=" << frag.sequence_id;
111 if (buffer == -1)
return false;
114 TLOG(TLVL_ERROR) <<
"Dropping event because data taking has already passed this event number: " << frag.sequence_id;
118 auto hdr = getEventHeader_(buffer);
121 hdr->run_id = run_id_;
122 hdr->subrun_id = subrun_id_;
125 TLOG(TLVL_TRACE) <<
"AddFragment before Write calls";
126 Write(buffer, dataPtr, frag.word_count *
sizeof(RawDataType));
128 TLOG(TLVL_TRACE) <<
"Checking for complete event";
129 auto fragmentCount = GetFragmentCount(frag.sequence_id);
130 hdr->is_complete = fragmentCount == num_fragments_per_event_ && buffer_writes_pending_[buffer] == 0;
131 TLOG(TLVL_TRACE) <<
"hdr->is_complete=" << std::boolalpha << hdr->is_complete
132 <<
", fragmentCount=" << fragmentCount
133 <<
", num_fragments_per_event=" << num_fragments_per_event_
134 <<
", buffer_writes_pending_[buffer]=" << buffer_writes_pending_[buffer];
136 complete_buffer_(buffer);
137 if (requests_) requests_->SendRequest(
true);
139 TLOG(TLVL_TRACE) <<
"AddFragment END";
143 bool artdaq::SharedMemoryEventManager::AddFragment(FragmentPtr frag,
size_t timeout_usec, FragmentPtr& outfrag)
145 TLOG(TLVL_TRACE) <<
"AddFragment(FragmentPtr) BEGIN";
146 auto hdr = *
reinterpret_cast<detail::RawFragmentHeader*
>(frag->headerAddress());
147 auto data = frag->headerAddress();
148 auto start = std::chrono::steady_clock::now();
150 while (!sts && TimeUtils::GetElapsedTimeMicroseconds(start) < timeout_usec)
152 sts = AddFragment(hdr, data);
153 if (!sts) usleep(1000);
157 outfrag = std::move(frag);
159 TLOG(TLVL_TRACE) <<
"AddFragment(FragmentPtr) RETURN " << std::boolalpha << sts;
165 TLOG(14) <<
"WriteFragmentHeader BEGIN";
166 auto buffer = getBufferForSequenceID_(frag.sequence_id,
true, frag.timestamp);
170 if (buffer == -1 && !dropIfNoBuffersAvailable)
return nullptr;
173 TLOG(TLVL_ERROR) <<
"Dropping fragment with sequence id " << frag.sequence_id <<
" and fragment id " << frag.fragment_id <<
" because data taking has already passed this event.";
177 TLOG(TLVL_ERROR) <<
"Dropping fragment with sequence id " << frag.sequence_id <<
" and fragment id " << frag.fragment_id <<
" because there is no room in the queue and reliable mode is off.";
179 dropped_data_[frag.fragment_id].reset(
new Fragment(frag.word_count - frag.num_words()));
181 TLOG(6) <<
"Dropping fragment with sequence id " << frag.sequence_id <<
" and fragment id " << frag.fragment_id <<
" into " << (
void*)dropped_data_[frag.fragment_id]->dataBegin() <<
" sz=" << dropped_data_[frag.fragment_id]->dataSizeBytes();
182 return dropped_data_[frag.fragment_id]->dataBegin();
186 buffer_writes_pending_[buffer]++;
190 metricMan->sendMetric(
"Input Fragment Rate", 1,
"Fragments/s", 1, MetricMode::Rate);
193 TLOG(TLVL_BUFLCK) <<
"WriteFragmentHeader: obtaining buffer_mutexes lock for buffer " << buffer;
195 std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
197 TLOG(TLVL_BUFLCK) <<
"WriteFragmentHeader: obtained buffer_mutexes lock for buffer " << buffer;
200 auto hdrpos =
reinterpret_cast<RawDataType*
>(GetWritePos(buffer));
201 Write(buffer, &frag, frag.num_words() *
sizeof(RawDataType));
203 auto pos =
reinterpret_cast<RawDataType*
>(GetWritePos(buffer));
204 if (frag.word_count - frag.num_words() > 0)
206 auto sts = IncrementWritePos(buffer, (frag.word_count - frag.num_words()) *
sizeof(RawDataType));
210 reinterpret_cast<detail::RawFragmentHeader*
>(hdrpos)->word_count = frag.num_words();
211 reinterpret_cast<detail::RawFragmentHeader*
>(hdrpos)->type = Fragment::InvalidFragmentType;
212 TLOG(TLVL_ERROR) <<
"Dropping over-size fragment with sequence id " << frag.sequence_id <<
" and fragment id " << frag.fragment_id <<
" because there is no room in the current buffer for this Fragment! (Keeping header)";
213 dropped_data_[frag.fragment_id].reset(
new Fragment(frag.word_count - frag.num_words()));
215 oversize_fragment_count_++;
217 if (maximum_oversize_fragment_count_ > 0 && oversize_fragment_count_ >= maximum_oversize_fragment_count_)
219 throw cet::exception(
"Too many over-size Fragments received! Please adjust max_event_size_bytes or max_fragment_size_bytes!");
222 TLOG(6) <<
"Dropping over-size fragment with sequence id " << frag.sequence_id <<
" and fragment id " << frag.fragment_id <<
" into " << (
void*)dropped_data_[frag.fragment_id]->dataBegin();
223 return dropped_data_[frag.fragment_id]->dataBegin();
226 TLOG(14) <<
"WriteFragmentHeader END";
233 TLOG(TLVL_TRACE) <<
"DoneWritingFragment BEGIN";
234 auto buffer = getBufferForSequenceID_(frag.sequence_id,
false, frag.timestamp);
235 if (buffer == -1) Detach(
true,
"SharedMemoryEventManager",
"getBufferForSequenceID_ returned -1 when it REALLY shouldn't have! Check program logic!");
236 if (buffer == -2) {
return; }
239 TLOG(TLVL_BUFLCK) <<
"DoneWritingFragment: obtaining buffer_mutexes lock for buffer " << buffer;
241 std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
243 TLOG(TLVL_BUFLCK) <<
"DoneWritingFragment: obtained buffer_mutexes lock for buffer " << buffer;
247 TLOG(TLVL_DEBUG) <<
"DoneWritingFragment: Received Fragment with sequence ID " << frag.sequence_id <<
" and fragment id " << frag.fragment_id <<
" (type " << (int)frag.type <<
")";
248 auto hdr = getEventHeader_(buffer);
251 hdr->run_id = run_id_;
252 hdr->subrun_id = subrun_id_;
255 buffer_writes_pending_[buffer]--;
256 if (buffer_writes_pending_[buffer] != 0)
258 TLOG(TLVL_TRACE) <<
"Done writing fragment, but there's another writer. Not doing bookkeeping steps.";
261 TLOG(TLVL_TRACE) <<
"Done writing fragment, and no other writer. Doing bookkeeping steps.";
262 auto frag_count = GetFragmentCount(frag.sequence_id);
263 hdr->is_complete = frag_count == num_fragments_per_event_;
264 TLOG(TLVL_TRACE) <<
"DoneWritingFragment: Received Fragment with sequence ID " << frag.sequence_id <<
" and fragment id " << frag.fragment_id <<
", count/expected = " << frag_count <<
"/" << num_fragments_per_event_;
265 #if ART_SUPPORTS_DUPLICATE_EVENTS
266 if (!hdr->is_complete && released_incomplete_events_.count(frag.sequence_id))
268 hdr->is_complete = frag_count == released_incomplete_events_[frag.sequence_id] && buffer_writes_pending_[buffer] == 0;
273 complete_buffer_(buffer);
274 if (requests_) requests_->SendRequest(
true);
275 TLOG(TLVL_TRACE) <<
"DoneWritingFragment END";
280 return GetFragmentCountInBuffer(getBufferForSequenceID_(seqID,
false), type);
285 if (buffer == -1)
return 0;
286 ResetReadPos(buffer);
287 IncrementReadPos(buffer,
sizeof(detail::RawEventHeader));
291 while (MoreDataInBuffer(buffer))
293 auto fragHdr =
reinterpret_cast<artdaq::detail::RawFragmentHeader*
>(GetReadPos(buffer));
294 IncrementReadPos(buffer, fragHdr->word_count *
sizeof(RawDataType));
295 if (type != Fragment::InvalidFragmentType && fragHdr->type != type)
continue;
296 TLOG(TLVL_TRACE) <<
"Adding Fragment with size=" << fragHdr->word_count <<
" to Fragment count";
307 auto start_time = std::chrono::steady_clock::now();
309 TLOG(TLVL_INFO) <<
"Starting art process with config file " << config_file->getFileName();
315 char* filename =
new char[config_file->getFileName().length() + 1];
316 strcpy(filename, config_file->getFileName().c_str());
318 std::vector<char*> args{ (
char*)
"art", (
char*)
"-c", filename, NULL };
326 std::string envVarKey =
"ARTDAQ_PARTITION_NUMBER";
327 std::string envVarValue = std::to_string(GetPartitionNumber());
328 if (setenv(envVarKey.c_str(), envVarValue.c_str(), 1) != 0)
330 TLOG(TLVL_ERROR) <<
"Error setting environment variable \"" << envVarKey
331 <<
"\" in the environment of a child art process. "
332 <<
"This may result in incorrect TCP port number "
333 <<
"assignments or other issues, and data may "
334 <<
"not flow through the system correctly.";
337 execvp(
"art", &args[0]);
346 std::cout <<
"Please run the following command in a separate terminal:" << std::endl
347 <<
"art -c " << config_file->getFileName() << std::endl
348 <<
"Then, in a third terminal, execute: \"ps aux|grep [a]rt -c " << config_file->getFileName() <<
"\" and note the PID of the art process." << std::endl
349 <<
"Finally, return to this window and enter the pid: " << std::endl;
354 TLOG(TLVL_INFO) <<
"PID of new art process is " << pid;
356 std::unique_lock<std::mutex> lk(art_process_mutex_);
357 art_processes_.insert(pid);
360 auto sts = waitid(P_PID, pid, &status, WEXITED);
361 TLOG(TLVL_INFO) <<
"Removing PID " << pid <<
" from process list";
363 std::unique_lock<std::mutex> lk(art_process_mutex_);
364 art_processes_.erase(pid);
368 TLOG(TLVL_WARNING) <<
"Error occurred in waitid for art process " << pid <<
": " << errno <<
" (" << strerror(errno) <<
").";
370 else if (status.si_code == CLD_EXITED && status.si_status == 0)
372 TLOG(TLVL_INFO) <<
"art process " << pid <<
" exited normally, " << (restart_art_ ?
"restarting" :
"not restarting");
376 auto art_lifetime = TimeUtils::GetElapsedTime(start_time);
377 if (art_lifetime < minimum_art_lifetime_s_) restart_art_ =
false;
379 auto exit_type =
"exited with status code";
380 switch (status.si_code)
384 exit_type =
"was killed with signal";
391 TLOG((restart_art_ ? TLVL_WARNING : TLVL_ERROR))
392 <<
"art process " << pid <<
" " << exit_type <<
" " << status.si_status
393 << (status.si_code == CLD_DUMPED ?
" (core dumped)" :
"")
394 <<
" after running for " << std::setprecision(2) << std::fixed << art_lifetime <<
" seconds, "
395 << (restart_art_ ?
"restarting" :
"not restarting");
397 }
while (restart_art_);
402 restart_art_ = always_restart_art_;
403 if (num_art_processes_ == 0)
return;
404 for (
size_t ii = 0; ii < num_art_processes_; ++ii)
406 StartArtProcess(current_art_pset_);
412 static std::mutex start_art_mutex;
413 std::unique_lock<std::mutex> lk(start_art_mutex);
415 restart_art_ = always_restart_art_;
416 auto initialCount = GetAttachedCount();
417 auto startTime = std::chrono::steady_clock::now();
419 if (pset != current_art_pset_ || !current_art_config_file_)
421 current_art_pset_ = pset;
422 current_art_config_file_ = std::make_shared<art_config_file>(pset);
424 std::shared_ptr<std::atomic<pid_t>> pid(
new std::atomic<pid_t>(-1));
425 boost::thread thread([&] { RunArt(current_art_config_file_, pid); });
428 auto currentCount = GetAttachedCount() - initialCount;
429 while ((currentCount < 1 || *pid <= 0) && (TimeUtils::GetElapsedTime(startTime) < 5 || manual_art_))
432 currentCount = GetAttachedCount() - initialCount;
434 if ((currentCount < 1 || *pid <= 0) && manual_art_)
436 TLOG(TLVL_WARNING) <<
"Manually-started art process has not connected to shared memory or has bad PID: connected:" << currentCount <<
", PID:" << pid;
439 else if (currentCount < 1 || *pid <= 0)
441 TLOG(TLVL_WARNING) <<
"art process has not started after 5s. Check art configuration!"
442 <<
" (pid=" << *pid <<
", attachedCount=" << currentCount <<
")";
447 TLOG(TLVL_INFO) << std::setw(4) << std::fixed <<
"art initialization took "
448 << TimeUtils::GetElapsedTime(startTime) <<
" seconds.";
457 restart_art_ =
false;
461 auto check_pids = [&](
bool print) {
463 for (
auto pid = pids.begin(); pid != pids.end();)
467 std::unique_lock<std::mutex> lk(art_process_mutex_);
470 TLOG(TLVL_WARNING) <<
"Removing an invalid PID (" << *pid
471 <<
") from the shutdown list.";
472 pid = pids.erase(pid);
474 else if (kill(*pid, 0) < 0)
476 pid = pids.erase(pid);
480 if (print) std::cout << *pid <<
" ";
486 if (pids.size() == 0)
488 TLOG(14) <<
"All art processes already exited, nothing to do.";
495 TLOG(TLVL_TRACE) <<
"Gently informing art processes that it is time to shut down";
496 for (
auto pid : pids)
498 TLOG(TLVL_TRACE) <<
"Sending SIGQUIT to pid " << pid;
502 int graceful_wait_ms = 5000;
503 int int_wait_ms = 1000;
505 TLOG(TLVL_TRACE) <<
"Waiting up to " << graceful_wait_ms <<
" ms for all art processes to exit gracefully";
506 for (
int ii = 0; ii < graceful_wait_ms; ++ii)
511 if (pids.size() == 0)
513 TLOG(TLVL_TRACE) <<
"All art processes exited after " << ii <<
" ms.";
518 TLOG(TLVL_TRACE) <<
"Insisting that the art processes shut down";
519 for (
auto pid : pids)
524 TLOG(TLVL_TRACE) <<
"Waiting up to " << int_wait_ms <<
" ms for all art processes to exit";
525 for (
int ii = graceful_wait_ms; ii < graceful_wait_ms + int_wait_ms; ++ii)
531 if (pids.size() == 0)
533 TLOG(TLVL_TRACE) <<
"All art processes exited after " << ii <<
" ms.";
538 TLOG(TLVL_TRACE) <<
"Killing remaning art processes with extreme prejudice";
539 while (pids.size() > 0)
541 kill(*pids.begin(), SIGKILL);
549 std::cout <<
"Please shut down all art processes, then hit return/enter" << std::endl;
550 while (pids.size() > 0)
552 std::cout <<
"The following PIDs are running: ";
554 std::cout << std::endl;
563 TLOG(TLVL_DEBUG) <<
"ReconfigureArt BEGIN";
564 if (restart_art_ || !always_restart_art_)
568 for (
size_t ii = 0; ii < broadcasts_.size(); ++ii)
570 broadcasts_.MarkBufferEmpty(ii,
true);
572 if (newRun == 0) newRun = run_id_ + 1;
574 if (art_pset != current_art_pset_ || !current_art_config_file_)
576 current_art_pset_ = art_pset;
577 current_art_config_file_ = std::make_shared<art_config_file>(art_pset);
580 if (n_art_processes != -1)
582 TLOG(TLVL_INFO) <<
"Setting number of art processes to " << n_art_processes;
583 num_art_processes_ = n_art_processes;
586 TLOG(TLVL_DEBUG) <<
"ReconfigureArt END";
592 init_fragment_.reset(
nullptr);
593 TLOG(TLVL_DEBUG) <<
"SharedMemoryEventManager::endOfData";
594 restart_art_ =
false;
596 size_t initialStoreSize = GetIncompleteEventCount();
597 TLOG(TLVL_DEBUG) <<
"endOfData: Flushing " << initialStoreSize
598 <<
" stale events from the SharedMemoryEventManager.";
599 int counter = initialStoreSize;
600 while (active_buffers_.size() > 0 && counter > 0)
602 complete_buffer_(*active_buffers_.begin());
605 TLOG(TLVL_DEBUG) <<
"endOfData: Done flushing, there are now " << GetIncompleteEventCount()
606 <<
" stale events in the SharedMemoryEventManager.";
609 TLOG(TLVL_DEBUG) <<
"Waiting for " << (ReadReadyCount() + (size() - WriteReadyCount(overwrite_mode_))) <<
" outstanding buffers...";
610 auto start = std::chrono::steady_clock::now();
611 auto lastReadCount = ReadReadyCount() + (size() - WriteReadyCount(overwrite_mode_));
612 auto end_of_data_wait_us = art_event_processing_time_us_ * lastReadCount;
614 auto outstanding_buffer_wait_time = art_event_processing_time_us_ > 100000 ? 100000 : art_event_processing_time_us_;
617 while (lastReadCount > 0 && (end_of_data_wait_us == 0 || TimeUtils::GetElapsedTimeMicroseconds(start) < end_of_data_wait_us) && get_art_process_count_() > 0)
619 auto temp = ReadReadyCount() + (size() - WriteReadyCount(overwrite_mode_));
620 if (temp != lastReadCount)
622 TLOG(TLVL_TRACE) <<
"Waiting for " << temp <<
" outstanding buffers...";
623 lastReadCount = temp;
624 start = std::chrono::steady_clock::now();
626 if (lastReadCount > 0) {
627 TRACE(19,
"About to sleep %lu us - lastReadCount=%lu size=%lu end_of_data_wait_us=%lu", outstanding_buffer_wait_time, lastReadCount, size(), end_of_data_wait_us);
628 usleep(outstanding_buffer_wait_time);
632 TLOG(TLVL_DEBUG) <<
"endOfData: After wait for outstanding buffers. Still outstanding: " << lastReadCount <<
", time waited: "
633 << TimeUtils::GetElapsedTime(start) <<
" s / " << (end_of_data_wait_us / 1000000.0) <<
" s, art process count: " << get_art_process_count_();
635 TLOG(TLVL_DEBUG) <<
"endOfData: Broadcasting EndOfData Fragment";
636 FragmentPtr outFrag = Fragment::eodFrag(GetBufferCount());
637 bool success = broadcastFragment_(std::move(outFrag), outFrag);
640 TLOG(TLVL_DEBUG) <<
"endOfData: Clearing buffers to make room for EndOfData Fragment";
641 for (
size_t ii = 0; ii < broadcasts_.size(); ++ii)
643 broadcasts_.MarkBufferEmpty(ii,
true);
645 broadcastFragment_(std::move(outFrag), outFrag);
647 auto endOfDataProcessingStart = std::chrono::steady_clock::now();
649 if (get_art_process_count_() > 0)
651 TLOG(TLVL_DEBUG) <<
"Allowing " << get_art_process_count_() <<
" art processes the chance to end gracefully";
652 if (end_of_data_wait_us == 0)
654 TLOG(TLVL_DEBUG) <<
"Expected art event processing time not specified. Waiting up to 100s for art to end gracefully.";
655 end_of_data_wait_us = 100 * 1000000;
658 auto sleep_count = (end_of_data_wait_us / 10000) + 1;
659 for (
size_t ii = 0; ii < sleep_count; ++ii)
662 if (get_art_process_count_() == 0)
break;
666 while (get_art_process_count_() > 0)
668 TLOG(TLVL_DEBUG) <<
"There are " << get_art_process_count_() <<
" art processes remaining. Proceeding to shutdown.";
670 ShutdownArtProcesses(art_processes_);
672 TLOG(TLVL_DEBUG) <<
"It took " << TimeUtils::GetElapsedTime(endOfDataProcessingStart) <<
" s for all art processes to close after sending EndOfData Fragment";
674 ResetAttachedCount();
676 TLOG(TLVL_DEBUG) <<
"endOfData: Clearing buffers";
677 for (
size_t ii = 0; ii < size(); ++ii)
679 MarkBufferEmpty(ii,
true);
687 released_incomplete_events_.clear();
689 TLOG(TLVL_DEBUG) <<
"endOfData: Shutting down RequestReceiver";
690 requests_.reset(
nullptr);
692 TLOG(TLVL_DEBUG) <<
"endOfData END";
693 TLOG(TLVL_INFO) <<
"EndOfData Complete. There were " << GetLastSeenBufferID() <<
" buffers processed.";
700 init_fragment_.reset(
nullptr);
701 TLOG(TLVL_TRACE) <<
"startRun: Clearing broadcast buffers";
702 for (
size_t ii = 0; ii < broadcasts_.size(); ++ii)
704 broadcasts_.MarkBufferEmpty(ii,
true);
709 subrun_rollover_event_ = Fragment::InvalidSequenceID;
710 last_released_event_ = 0;
714 requests_->SendRoutingToken(queue_size_);
716 TLOG(TLVL_DEBUG) <<
"Starting run " << run_id_
717 <<
", max queue size = "
720 << GetLockedBufferCount();
723 double runSubrun = run_id_ + ((double)subrun_id_ / 10000);
724 metricMan->sendMetric(
"Run Number", runSubrun,
"Run:Subrun", 1, MetricMode::LastPoint);
731 subrun_rollover_event_ = Fragment::InvalidSequenceID;
734 double runSubrun = run_id_ + ((double)subrun_id_ / 10000);
735 metricMan->sendMetric(
"Run Number", runSubrun,
"Run:Subrun", 1, MetricMode::LastPoint);
741 TLOG(TLVL_INFO) <<
"Ending run " << run_id_;
742 FragmentPtr endOfRunFrag(
new
743 Fragment(static_cast<size_t>
744 (ceil(
sizeof(my_rank) /
745 static_cast<double>(
sizeof(Fragment::value_type))))));
747 TLOG(TLVL_DEBUG) <<
"Broadcasting EndOfRun Fragment";
748 endOfRunFrag->setSystemType(Fragment::EndOfRunFragmentType);
749 *endOfRunFrag->dataBegin() = my_rank;
750 broadcastFragment_(std::move(endOfRunFrag), endOfRunFrag);
752 TLOG(TLVL_INFO) <<
"Run " << run_id_ <<
" has ended. There were " << run_event_count_ <<
" events in this run.";
753 run_event_count_ = 0;
754 run_incomplete_event_count_ = 0;
755 oversize_fragment_count_ = 0;
761 TLOG(TLVL_INFO) <<
"Ending subrun " << subrun_id_;
762 std::unique_ptr<artdaq::Fragment>
764 Fragment(static_cast<size_t>
765 (ceil(
sizeof(my_rank) /
766 static_cast<double>(
sizeof(Fragment::value_type))))));
768 TLOG(TLVL_DEBUG) <<
"Broadcasting EndOfSubrun Fragment";
769 endOfSubrunFrag->setSequenceID(subrun_rollover_event_);
770 endOfSubrunFrag->setSystemType(Fragment::EndOfSubrunFragmentType);
771 *endOfSubrunFrag->dataBegin() = my_rank;
773 broadcastFragment_(std::move(endOfSubrunFrag), endOfSubrunFrag);
775 TLOG(TLVL_INFO) <<
"Subrun " << subrun_id_ <<
" in run " << run_id_ <<
" has ended. There were " << subrun_event_count_ <<
" events in this subrun.";
776 subrun_event_count_ = 0;
777 subrun_incomplete_event_count_ = 0;
785 if (boundary == 0 || boundary == Fragment::InvalidSequenceID)
return;
787 if (boundary < last_released_event_)
789 auto logLevel = TLVL_ERROR;
790 bool processAnyway =
false;
791 if (last_released_event_ - boundary < 100)
793 logLevel = TLVL_WARNING;
794 processAnyway =
true;
796 TLOG(logLevel) <<
"Subrun rollover requested for event that is in the past. (last_released_event="
797 << last_released_event_ <<
",requested_rollover_boundary=" << boundary <<
").";
798 if (!processAnyway)
return;
800 TLOG(TLVL_INFO) <<
"Will roll over when I reach Sequence ID " << boundary;
809 if (boundary == last_released_event_ + 1) {
810 TLOG(TLVL_INFO) <<
"rolloverSubrun: Last released event had sequence id " << last_released_event_ << \
811 ", boundary is sequence id " << boundary <<
", so will start a new subrun here";
814 subrun_rollover_event_ = std::numeric_limits<sequence_id_t>::max();
817 subrun_rollover_event_ = boundary;
825 metricMan->sendMetric(
"Incomplete Event Count", GetIncompleteEventCount(),
"events", 1, MetricMode::LastPoint);
826 metricMan->sendMetric(
"Pending Event Count", GetPendingEventCount(),
"events", 1, MetricMode::LastPoint);
829 if (incomplete_event_report_interval_ms_ > 0 && GetLockedBufferCount())
831 if (TimeUtils::GetElapsedTimeMilliseconds(last_incomplete_event_report_time_) < static_cast<size_t>(incomplete_event_report_interval_ms_))
834 last_incomplete_event_report_time_ = std::chrono::steady_clock::now();
835 std::ostringstream oss;
836 oss <<
"Incomplete Events (" << num_fragments_per_event_ <<
"): ";
837 for (
auto& ev : active_buffers_)
839 auto hdr = getEventHeader_(ev);
840 oss << hdr->sequence_id <<
" (" << GetFragmentCount(hdr->sequence_id) <<
"), ";
842 TLOG(TLVL_DEBUG) << oss.str();
846 bool artdaq::SharedMemoryEventManager::broadcastFragment_(FragmentPtr frag, FragmentPtr& outFrag)
848 TLOG(TLVL_DEBUG) <<
"Broadcasting Fragment with seqID=" << frag->sequenceID() <<
", type " << detail::RawFragmentHeader::SystemTypeToString(frag->type()) <<
", size=" << frag->sizeBytes() <<
"B.";
849 auto buffer = broadcasts_.GetBufferForWriting(
false);
850 TLOG(TLVL_DEBUG) <<
"broadcastFragment_: after getting buffer 1st buffer=" << buffer;
851 auto start_time = std::chrono::steady_clock::now();
852 while (buffer == -1 && TimeUtils::GetElapsedTimeMilliseconds(start_time) < static_cast<size_t>(broadcast_timeout_ms_))
855 buffer = broadcasts_.GetBufferForWriting(
false);
857 TLOG(TLVL_DEBUG) <<
"broadcastFragment_: after getting buffer w/timeout, buffer=" << buffer <<
", elapsed time=" << TimeUtils::GetElapsedTime(start_time) <<
" s.";
860 TLOG(TLVL_ERROR) <<
"Broadcast of fragment type " << frag->typeString() <<
" failed due to timeout waiting for buffer!";
865 TLOG(TLVL_DEBUG) <<
"broadcastFragment_: Filling in RawEventHeader";
866 auto hdr =
reinterpret_cast<detail::RawEventHeader*
>(broadcasts_.GetBufferStart(buffer));
867 hdr->run_id = run_id_;
868 hdr->subrun_id = subrun_id_;
869 hdr->sequence_id = frag->sequenceID();
870 hdr->is_complete =
true;
871 broadcasts_.IncrementWritePos(buffer,
sizeof(detail::RawEventHeader));
873 TLOG(TLVL_DEBUG) <<
"broadcastFragment_ before Write calls";
874 broadcasts_.Write(buffer, frag->headerAddress(), frag->size() *
sizeof(RawDataType));
876 TLOG(TLVL_DEBUG) <<
"broadcastFragment_ Marking buffer full";
877 broadcasts_.MarkBufferFull(buffer, -1);
879 TLOG(TLVL_DEBUG) <<
"broadcastFragment_ Complete";
883 artdaq::detail::RawEventHeader* artdaq::SharedMemoryEventManager::getEventHeader_(
int buffer)
885 return reinterpret_cast<detail::RawEventHeader*
>(GetBufferStart(buffer));
888 int artdaq::SharedMemoryEventManager::getBufferForSequenceID_(Fragment::sequence_id_t seqID,
bool create_new, Fragment::timestamp_t timestamp)
890 TLOG(14) <<
"getBufferForSequenceID " << seqID <<
" BEGIN";
891 std::unique_lock<std::mutex> lk(sequence_id_mutex_);
893 TLOG(14) <<
"getBufferForSequenceID obtained sequence_id_mutex for seqid=" << seqID;
895 auto buffers = GetBuffersOwnedByManager();
896 for (
auto& buf : buffers)
898 auto hdr = getEventHeader_(buf);
899 if (hdr->sequence_id == seqID)
901 TLOG(14) <<
"getBufferForSequenceID " << seqID <<
" returning " << buf;
906 #if !ART_SUPPORTS_DUPLICATE_EVENTS
907 if (released_incomplete_events_.count(seqID))
909 TLOG(TLVL_ERROR) <<
"Event " << seqID <<
" has already been marked \"Incomplete\" and sent to art!";
914 if (!create_new)
return -1;
916 check_pending_buffers_(lk);
917 int new_buffer = GetBufferForWriting(
false);
919 if (new_buffer == -1)
921 new_buffer = GetBufferForWriting(overwrite_mode_);
924 if (new_buffer == -1)
return -1;
925 TLOG(TLVL_BUFLCK) <<
"getBufferForSequenceID_: obtaining buffer_mutexes lock for buffer " << new_buffer;
926 std::unique_lock<std::mutex> buffer_lk(buffer_mutexes_[new_buffer]);
927 TLOG(TLVL_BUFLCK) <<
"getBufferForSequenceID_: obtained buffer_mutexes lock for buffer " << new_buffer;
929 auto hdr = getEventHeader_(new_buffer);
930 hdr->is_complete =
false;
931 hdr->run_id = run_id_;
932 hdr->subrun_id = subrun_id_;
933 hdr->event_id = use_sequence_id_for_event_number_ ?
static_cast<uint32_t
>(seqID) : static_cast<uint32_t>(timestamp);
934 hdr->sequence_id = seqID;
935 buffer_writes_pending_[new_buffer] = 0;
936 IncrementWritePos(new_buffer,
sizeof(detail::RawEventHeader));
937 SetMFIteration(
"Sequence ID " + std::to_string(seqID));
939 TLOG(TLVL_BUFFER) <<
"getBufferForSequenceID placing " << new_buffer <<
" to active.";
940 active_buffers_.insert(new_buffer);
941 TLOG(TLVL_BUFFER) <<
"Buffer occupancy now (total,full,reading,empty,pending,active)=("
943 << ReadReadyCount() <<
","
944 << WriteReadyCount(
true) - WriteReadyCount(
false) - ReadReadyCount() <<
","
945 << WriteReadyCount(
false) <<
","
946 << pending_buffers_.size() <<
","
947 << active_buffers_.size() <<
")";
951 if (timestamp != Fragment::InvalidTimestamp)
953 requests_->AddRequest(seqID, timestamp);
959 requests_->SendRequest();
962 TLOG(14) <<
"getBufferForSequenceID " << seqID <<
" returning newly initialized buffer " << new_buffer;
966 bool artdaq::SharedMemoryEventManager::hasFragments_(
int buffer)
968 if (buffer == -1)
return true;
969 if (!CheckBuffer(buffer, BufferSemaphoreFlags::Writing))
973 ResetReadPos(buffer);
974 IncrementReadPos(buffer,
sizeof(detail::RawEventHeader));
975 return MoreDataInBuffer(buffer);
978 void artdaq::SharedMemoryEventManager::complete_buffer_(
int buffer)
980 auto hdr = getEventHeader_(buffer);
981 if (hdr->is_complete)
983 TLOG(TLVL_DEBUG) <<
"complete_buffer_: This fragment completes event " << hdr->sequence_id <<
".";
986 TLOG(TLVL_BUFFER) <<
"complete_buffer_ moving " << buffer <<
" from active to pending.";
988 TLOG(TLVL_BUFLCK) <<
"complete_buffer_: obtaining sequence_id_mutex lock for seqid=" << hdr->sequence_id;
989 std::unique_lock<std::mutex> lk(sequence_id_mutex_);
990 TLOG(TLVL_BUFLCK) <<
"complete_buffer_: obtained sequence_id_mutex lock for seqid=" << hdr->sequence_id;
991 active_buffers_.erase(buffer);
992 pending_buffers_.insert(buffer);
994 TLOG(TLVL_BUFFER) <<
"Buffer occupancy now (total,full,reading,empty,pending,active)=("
996 << ReadReadyCount() <<
","
997 << WriteReadyCount(
true) - WriteReadyCount(
false) - ReadReadyCount() <<
","
998 << WriteReadyCount(
false) <<
","
999 << pending_buffers_.size() <<
","
1000 << active_buffers_.size() <<
")";
1004 requests_->RemoveRequest(hdr->sequence_id);
1007 CheckPendingBuffers();
1010 bool artdaq::SharedMemoryEventManager::bufferComparator(
int bufA,
int bufB)
1012 return getEventHeader_(bufA)->sequence_id < getEventHeader_(bufB)->sequence_id;
1017 TLOG(TLVL_BUFLCK) <<
"CheckPendingBuffers: Obtaining sequence_id_mutex_";
1018 std::unique_lock<std::mutex> lk(sequence_id_mutex_);
1019 TLOG(TLVL_BUFLCK) <<
"CheckPendingBuffers: Obtained sequence_id_mutex_";
1020 check_pending_buffers_(lk);
1023 void artdaq::SharedMemoryEventManager::check_pending_buffers_(std::unique_lock<std::mutex>
const& lock)
1025 TLOG(TLVL_TRACE) <<
"check_pending_buffers_ BEGIN Locked=" << std::boolalpha << lock.owns_lock();
1027 auto buffers = GetBuffersOwnedByManager();
1028 for (
auto buf : buffers)
1030 if (ResetBuffer(buf) && !pending_buffers_.count(buf))
1032 TLOG(15) <<
"check_pending_buffers_ Incomplete buffer detected, buf=" << buf <<
" active_bufers_.count(buf)=" << active_buffers_.count(buf) <<
" buffer_writes_pending_[buf]=" << buffer_writes_pending_[buf].load();
1033 auto hdr = getEventHeader_(buf);
1034 if (active_buffers_.count(buf) && (buffer_writes_pending_[buf].load() == 0 || !running_))
1038 requests_->RemoveRequest(hdr->sequence_id);
1040 TLOG(TLVL_BUFFER) <<
"check_pending_buffers_ moving buffer " << buf <<
" from active to pending";
1041 active_buffers_.erase(buf);
1042 pending_buffers_.insert(buf);
1043 TLOG(TLVL_BUFFER) <<
"Buffer occupancy now (total,full,reading,empty,pending,active)=("
1045 << ReadReadyCount() <<
","
1046 << WriteReadyCount(
true) - WriteReadyCount(
false) - ReadReadyCount() <<
","
1047 << WriteReadyCount(
false) <<
","
1048 << pending_buffers_.size() <<
","
1049 << active_buffers_.size() <<
")";
1051 subrun_incomplete_event_count_++;
1052 run_incomplete_event_count_++;
1053 if (metricMan) metricMan->sendMetric(
"Incomplete Event Rate", 1,
"events/s", 3, MetricMode::Rate);
1054 if (!released_incomplete_events_.count(hdr->sequence_id))
1056 released_incomplete_events_[hdr->sequence_id] = num_fragments_per_event_ - GetFragmentCountInBuffer(buf);
1060 released_incomplete_events_[hdr->sequence_id] -= GetFragmentCountInBuffer(buf);
1062 TLOG(TLVL_WARNING) <<
"Active event " << hdr->sequence_id <<
" is stale. Scheduling release of incomplete event (missing " << released_incomplete_events_[hdr->sequence_id] <<
" Fragments) to art.";
1068 std::list<int> sorted_buffers(pending_buffers_.begin(), pending_buffers_.end());
1069 sorted_buffers.sort([
this](
int a,
int b) {
return bufferComparator(a, b); });
1072 double eventSize = 0;
1073 for (
auto buf : sorted_buffers)
1075 auto hdr = getEventHeader_(buf);
1077 if (hdr->sequence_id >= subrun_rollover_event_)
1079 TLOG(TLVL_INFO) <<
"Subrun rollover reached at event " << hdr->sequence_id <<
" (boundary=" << subrun_rollover_event_ <<
"), last released event is " << last_released_event_ <<
".";
1083 if (hdr->sequence_id > last_released_event_) last_released_event_ = hdr->sequence_id;
1085 TLOG(TLVL_DEBUG) <<
"Releasing event " << std::to_string(hdr->sequence_id) <<
" in buffer " << buf <<
" to art, "
1086 <<
"event_size=" << BufferDataSize(buf) <<
", buffer_size=" << BufferSize();
1088 TLOG(TLVL_BUFFER) <<
"check_pending_buffers_ removing buffer " << buf <<
" moving from pending to full";
1089 MarkBufferFull(buf);
1090 subrun_event_count_++;
1093 eventSize += BufferDataSize(buf);
1094 pending_buffers_.erase(buf);
1095 TLOG(TLVL_BUFFER) <<
"Buffer occupancy now (total,full,reading,empty,pending,active)=("
1097 << ReadReadyCount() <<
","
1098 << WriteReadyCount(
true) - WriteReadyCount(
false) - ReadReadyCount() <<
","
1099 << WriteReadyCount(
false) <<
","
1100 << pending_buffers_.size() <<
","
1101 << active_buffers_.size() <<
")";
1106 auto outstanding_tokens = requests_->GetSentTokenCount() - run_event_count_;
1107 auto available_buffers = WriteReadyCount(overwrite_mode_);
1109 TLOG(TLVL_TRACE) <<
"check_pending_buffers_: outstanding_tokens: " << outstanding_tokens <<
", available_buffers: " << available_buffers
1110 <<
", tokens_to_send: " << available_buffers - outstanding_tokens;
1112 if (available_buffers > outstanding_tokens)
1114 auto tokens_to_send = available_buffers - outstanding_tokens;
1116 while (tokens_to_send > 0)
1118 TLOG(35) <<
"check_pending_buffers_: Sending a Routing Token";
1119 requests_->SendRoutingToken(1);
1125 metric_data_.event_count += counter;
1126 metric_data_.event_size += eventSize;
1128 if (metricMan && TimeUtils::GetElapsedTimeMilliseconds(last_shmem_buffer_metric_update_) > 500)
1130 TLOG(TLVL_TRACE) <<
"check_pending_buffers_: Sending Metrics";
1131 metricMan->sendMetric(
"Event Rate", metric_data_.event_count,
"Events/s", 1, MetricMode::Rate);
1132 if (metric_data_.event_count > 0) metricMan->sendMetric(
"Average Event Size", metric_data_.event_size / metric_data_.event_count,
"Bytes", 1, MetricMode::Average);
1133 metric_data_ = MetricData();
1135 metricMan->sendMetric(
"Events Released to art this run", run_event_count_,
"Events", 1, MetricMode::LastPoint);
1136 metricMan->sendMetric(
"Incomplete Events Released to art this run", run_incomplete_event_count_,
"Events", 1, MetricMode::LastPoint);
1137 metricMan->sendMetric(
"Events Released to art this subrun", subrun_event_count_,
"Events", 2, MetricMode::LastPoint);
1138 metricMan->sendMetric(
"Incomplete Events Released to art this subrun", subrun_incomplete_event_count_,
"Events", 2, MetricMode::LastPoint);
1139 if (requests_) metricMan->sendMetric(
"Tokens sent", requests_->GetSentTokenCount(),
"Tokens", 2, MetricMode::LastPoint);
1141 auto bufferReport = GetBufferReport();
1142 int full = std::count_if(bufferReport.begin(), bufferReport.end(), [](std::pair<int, BufferSemaphoreFlags> p) {
return p.second == BufferSemaphoreFlags::Full; });
1143 int empty = std::count_if(bufferReport.begin(), bufferReport.end(), [](std::pair<int, BufferSemaphoreFlags> p) {
return p.second == BufferSemaphoreFlags::Empty; });
1144 int writing = std::count_if(bufferReport.begin(), bufferReport.end(), [](std::pair<int, BufferSemaphoreFlags> p) {
return p.second == BufferSemaphoreFlags::Writing; });
1145 int reading = std::count_if(bufferReport.begin(), bufferReport.end(), [](std::pair<int, BufferSemaphoreFlags> p) {
return p.second == BufferSemaphoreFlags::Reading; });
1146 auto total = size();
1147 TLOG(TLVL_DEBUG) <<
"Buffer usage: full=" << full <<
", empty=" << empty <<
", writing=" << writing <<
", reading=" << reading <<
", total=" << total;
1149 metricMan->sendMetric(
"Shared Memory Full Buffers", full,
"buffers", 2, MetricMode::LastPoint);
1150 metricMan->sendMetric(
"Shared Memory Available Buffers", empty,
"buffers", 2, MetricMode::LastPoint);
1151 metricMan->sendMetric(
"Shared Memory Pending Buffers", writing,
"buffers", 2, MetricMode::LastPoint);
1152 metricMan->sendMetric(
"Shared Memory Reading Buffers", reading,
"buffers", 2, MetricMode::LastPoint);
1155 metricMan->sendMetric(
"Shared Memory Full %", full * 100 / static_cast<double>(total),
"%", 2, MetricMode::LastPoint);
1156 metricMan->sendMetric(
"Shared Memory Available %", empty * 100 / static_cast<double>(total),
"%", 2, MetricMode::LastPoint);
1159 last_shmem_buffer_metric_update_ = std::chrono::steady_clock::now();
1161 TLOG(TLVL_TRACE) <<
"check_pending_buffers_ END";
1164 void artdaq::SharedMemoryEventManager::send_init_frag_()
1166 if (init_fragment_ !=
nullptr)
1168 TLOG(TLVL_INFO) <<
"Broadcasting init fragment to all art subprocesses...";
1171 std::string fileName =
"receiveInitMessage_" + std::to_string(my_rank) +
".bin";
1172 std::fstream ostream(fileName.c_str(), std::ios::out | std::ios::binary);
1173 ostream.write(reinterpret_cast<char*>(init_fragment_->dataBeginBytes()), init_fragment_->dataSizeBytes());
1177 broadcastFragment_(std::move(init_fragment_), init_fragment_);
1178 TLOG(TLVL_TRACE) <<
"Init Fragment sent";
1180 else if (send_init_fragments_)
1182 TLOG(TLVL_WARNING) <<
"Cannot send init fragment because I haven't yet received one!";
1188 if (!init_fragment_ || init_fragment_ ==
nullptr)
1190 init_fragment_.swap(frag);
1197 TLOG(TLVL_DEBUG) <<
"UpdateArtConfiguration BEGIN";
1198 if (art_pset != current_art_pset_ || !current_art_config_file_)
1200 current_art_pset_ = art_pset;
1201 current_art_config_file_ = std::make_shared<art_config_file>(art_pset);
1203 TLOG(TLVL_DEBUG) <<
"UpdateArtConfiguration END";
1206 #if MESSAGEFACILITY_HEX_VERSION >= 0x20103
void RunArt(std::shared_ptr< art_config_file > config_file, std::shared_ptr< std::atomic< pid_t >> pid_out)
Run an art instance, recording the return codes and restarting it until the end flag is raised...
void ShutdownArtProcesses(std::set< pid_t > &pids)
Shutdown a set of art processes.
virtual ~SharedMemoryEventManager()
SharedMemoryEventManager Destructor.
The SharedMemoryEventManager is a SharedMemoryManger which tracks events as they are built...
Fragment::sequence_id_t sequence_id_t
Copy Fragment::sequence_id_t into local scope.
void ReconfigureArt(fhicl::ParameterSet art_pset, run_id_t newRun=0, int n_art_processes=-1)
Restart all art processes, using the given fhicl code to configure the new art processes.
The RequestSender contains methods used to send data requests and Routing tokens. ...
pid_t StartArtProcess(fhicl::ParameterSet pset)
Start one art process.
RawDataType * WriteFragmentHeader(detail::RawFragmentHeader frag, bool dropIfNoBuffersAvailable=false)
Get a pointer to a reserved memory area for the given Fragment header.
RawEvent::run_id_t run_id_t
Copy RawEvent::run_id_t into local scope.
size_t GetFragmentCount(Fragment::sequence_id_t seqID, Fragment::type_t type=Fragment::InvalidFragmentType)
Get the count of Fragments of a given type in an event.
void UpdateArtConfiguration(fhicl::ParameterSet art_pset)
Updates the internally-stored copy of the art configuration.
void StartArt()
Start all the art processes.
void SetInitFragment(FragmentPtr frag)
Set the stored Init fragment, if one has not yet been set already.
void rolloverSubrun(sequence_id_t boundary)
Rollover the subrun after the specified event.
void sendMetrics()
Send metrics to the MetricManager, if one has been instantiated in the application.
void startSubrun()
Start a new Subrun, incrementing the subrun number.
SharedMemoryEventManager(fhicl::ParameterSet pset, fhicl::ParameterSet art_pset)
SharedMemoryEventManager Constructor.
bool endSubrun()
Send an EndOfSubRunFragment to the art thread.
bool endRun()
Send an EndOfRunFragment to the art thread.
void DoneWritingFragment(detail::RawFragmentHeader frag)
Used to indicate that the given Fragment is now completely in the buffer. Will check for buffer compl...
bool endOfData()
Indicate that the end of input has been reached to the art processes.
void startRun(run_id_t runID)
Start a Run.
size_t GetFragmentCountInBuffer(int buffer, Fragment::type_t type=Fragment::InvalidFragmentType)
Get the count of Fragments of a given type in a buffer.
void CheckPendingBuffers()
Check for buffers which are ready to be marked incomplete and released to art and issue tokens for an...