$treeview $search $mathjax $extrastylesheet
artdaq
v3_04_00
$projectbrief
|
$projectbrief
|
$searchbox |
00001 #ifndef artdaq_ArtModules_detail_SharedMemoryReader_hh 00002 #define artdaq_ArtModules_detail_SharedMemoryReader_hh 00003 00004 #include "artdaq/DAQdata/Globals.hh" 00005 #include "artdaq-core/Utilities/ExceptionHandler.hh" 00006 #include "art/Framework/Core/Frameworkfwd.h" 00007 00008 #include "art/Framework/Core/FileBlock.h" 00009 #include "art/Framework/Core/ProductRegistryHelper.h" 00010 #include "art/Framework/IO/Sources/SourceHelper.h" 00011 #include "art/Framework/Principal/EventPrincipal.h" 00012 #include "art/Framework/Principal/RunPrincipal.h" 00013 #include "art/Framework/Principal/SubRunPrincipal.h" 00014 #include "artdaq-core/Core/SharedMemoryManager.hh" 00015 #include "artdaq-core/Utilities/TimeUtils.hh" 00016 #include "fhiclcpp/ParameterSet.h" 00017 #include "artdaq-core/Data/Fragment.hh" 00018 #include "artdaq-core/Data/ContainerFragment.hh" 00019 #include "artdaq-core/Core/SharedMemoryEventReceiver.hh" 00020 #include "art/Framework/IO/Sources/put_product_in_principal.h" 00021 #include "canvas/Persistency/Provenance/FileFormatVersion.h" 00022 #include <sys/time.h> 00023 00024 00025 #include <string> 00026 #include <map> 00027 #include "artdaq-core/Data/RawEvent.hh" 00028 00029 namespace artdaq 00030 { 00031 namespace detail 00032 { 00036 template<std::map<artdaq::Fragment::type_t, std::string> getDefaultTypes() = artdaq::Fragment::MakeSystemTypeMap > 00037 struct SharedMemoryReader 00038 { 00042 SharedMemoryReader(SharedMemoryReader const&) = delete; 00043 00048 SharedMemoryReader& operator=(SharedMemoryReader const&) = delete; 00049 00050 art::SourceHelper const& pmaker; 00051 std::unique_ptr<SharedMemoryEventReceiver> incoming_events; 00052 double waiting_time; 00053 bool resume_after_timeout; 00054 std::string pretend_module_name; 00055 std::string unidentified_instance_name; 00056 bool shutdownMsgReceived; 00057 bool outputFileCloseNeeded; 00058 size_t bytesRead; 00059 std::chrono::steady_clock::time_point last_read_time; 00060 //std::unique_ptr<SharedMemoryManager> data_shm; ///< SharedMemoryManager containing data 00061 //std::unique_ptr<SharedMemoryManager> broadcast_shm; ///< SharedMemoryManager containing broadcasts (control Fragments) 00062 00077 SharedMemoryReader(fhicl::ParameterSet const& ps, 00078 art::ProductRegistryHelper& help, 00079 art::SourceHelper const& pm) 00080 : pmaker(pm) 00081 , waiting_time(ps.get<double>("waiting_time", 86400.0)) 00082 , resume_after_timeout(ps.get<bool>("resume_after_timeout", true)) 00083 , pretend_module_name(ps.get<std::string>("raw_data_label", "daq")) 00084 , unidentified_instance_name("unidentified") 00085 , shutdownMsgReceived(false) 00086 , outputFileCloseNeeded(false) 00087 , bytesRead(0) 00088 , last_read_time(std::chrono::steady_clock::now()) 00089 , fragment_type_map_(getDefaultTypes()) 00090 , readNext_calls_(0) 00091 { 00092 00093 // For testing 00094 //if (ps.has_key("buffer_count") && (ps.has_key("max_event_size_bytes") || (ps.has_key("expected_fragments_per_event") && ps.has_key("max_fragment_size_bytes")))) 00095 //{ 00096 // data_shm.reset(new SharedMemoryManager(ps.get<uint32_t>("shared_memory_key", 0xBEE70000 + getppid()), ps.get<int>("buffer_count"), ps.has_key("max_event_size_bytes") ? ps.get<size_t>("max_event_size_bytes") : ps.get<size_t>("expected_fragments_per_event") * ps.get<size_t>("max_fragment_size_bytes"))); 00097 // broadcast_shm.reset(new SharedMemoryManager(ps.get<uint32_t>("broadcast_shared_memory_key", 0xCEE70000 + getppid()), ps.get<int>("broadcast_buffer_count", 5), ps.get<size_t>("broadcast_buffer_size", 0x100000))); 00098 //} 00099 incoming_events.reset(new SharedMemoryEventReceiver(ps.get<uint32_t>("shared_memory_key", 0xBEE70000 + getppid()), ps.get<uint32_t>("broadcast_shared_memory_key", 0xCEE70000 + getppid()))); 00100 my_rank = incoming_events->GetRank(); 00101 00102 char const* artapp_env = getenv("ARTDAQ_APPLICATION_NAME"); 00103 std::string artapp_str = ""; 00104 if (artapp_env != NULL) 00105 { 00106 artapp_str = std::string(artapp_env) + "_"; 00107 } 00108 00109 app_name = artapp_str + "art" + std::to_string(incoming_events->GetMyId()); 00110 00111 try { 00112 if (metricMan) 00113 { 00114 metricMan->initialize(ps.get<fhicl::ParameterSet>("metrics", fhicl::ParameterSet()), app_name); 00115 metricMan->do_start(); 00116 } 00117 } 00118 catch (...) 00119 { 00120 ExceptionHandler(ExceptionHandlerRethrow::no, "Error loading metrics in SharedMemoryReader()"); 00121 } 00122 00123 help.reconstitutes<Fragments, art::InEvent>(pretend_module_name, unidentified_instance_name); 00124 for (auto it = fragment_type_map_.begin(); it != fragment_type_map_.end(); ++it) 00125 { 00126 help.reconstitutes<Fragments, art::InEvent>(pretend_module_name, it->second); 00127 help.reconstitutes<Fragments, art::InEvent>(pretend_module_name, "Container" + it->second); 00128 } 00129 auto extraTypes = ps.get<std::vector<std::pair<Fragment::type_t, std::string>>>("fragment_type_map", std::vector<std::pair<Fragment::type_t, std::string>>()); 00130 for (auto it = extraTypes.begin(); it != extraTypes.end(); ++it) 00131 { 00132 fragment_type_map_[it->first] = it->second; 00133 help.reconstitutes<Fragments, art::InEvent>(pretend_module_name, it->second); 00134 help.reconstitutes<Fragments, art::InEvent>(pretend_module_name, "Container" + it->second); 00135 } 00136 TLOG_INFO("SharedMemoryReader") << "SharedMemoryReader initialized with ParameterSet: " << ps.to_string(); 00137 //for(auto& type : fragment_type_map_) 00138 //{ 00139 // TLOG_INFO("SharedMemoryReader") << "Fragment Type " << type.second << " has typeid " << type.first ; 00140 //} 00141 } 00142 00151 SharedMemoryReader(fhicl::ParameterSet const& ps, 00152 art::ProductRegistryHelper& help, 00153 art::SourceHelper const& pm, 00154 art::MasterProductRegistry&) : SharedMemoryReader(ps, help, pm) 00155 {} 00156 00160 virtual ~SharedMemoryReader() { 00161 artdaq::Globals::CleanUpGlobals(); 00162 } 00163 00167 void closeCurrentFile() {} 00168 00173 void readFile(std::string const&, art::FileBlock*& fb) 00174 { 00175 TLOG_ARB(5, "SharedMemoryReader") << "readFile enter/start"; 00176 fb = new art::FileBlock(art::FileFormatVersion(1, "RawEvent2011"), "nothing"); 00177 } 00178 00183 bool hasMoreData() const { return (!shutdownMsgReceived); } 00184 00195 bool readNext(art::RunPrincipal* const & inR, 00196 art::SubRunPrincipal* const & inSR, 00197 art::RunPrincipal*& outR, 00198 art::SubRunPrincipal*& outSR, 00199 art::EventPrincipal*& outE) 00200 { 00201 TLOG_DEBUG("SharedMemoryReader") << "readNext BEGIN"; 00202 /*if (outputFileCloseNeeded) { 00203 outputFileCloseNeeded = false; 00204 return false; 00205 }*/ 00206 // Establish default 'results' 00207 outR = 0; 00208 outSR = 0; 00209 outE = 0; 00210 // Try to get an event from the queue. We'll continuously loop, either until: 00211 // 1) we have read a RawEvent off the queue, or 00212 // 2) we have timed out, AND we are told the when we timeout we 00213 // should stop. 00214 // In any case, if we time out, we emit an informational message. 00215 00216 if (shutdownMsgReceived) 00217 { 00218 TLOG_INFO("SharedMemoryReader") << "Shutdown Message received, returning false (should exit art)"; 00219 return false; 00220 } 00221 00222 auto read_start_time = std::chrono::steady_clock::now(); 00223 start: 00224 bool keep_looping = true; 00225 bool got_event = false; 00226 auto sleepTimeUsec = waiting_time * 1000; // waiting_time * 1000000 us/s / 1000 reps = us/rep 00227 if (sleepTimeUsec > 100000) sleepTimeUsec = 100000; // Don't wait longer than 1/10th of a second 00228 while (keep_looping) 00229 { 00230 TLOG_TRACE("SharedMemoryReader") << "ReadyForRead loops BEGIN"; 00231 keep_looping = false; 00232 auto start_time = std::chrono::steady_clock::now(); 00233 while (!got_event && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 1000) 00234 { 00235 // BURN CPU for 1 ms! 00236 got_event = incoming_events->ReadyForRead(); 00237 } 00238 TLOG_TRACE("SharedMemoryReader") << "ReadyForRead spin end, poll begin"; 00239 while (!got_event && TimeUtils::GetElapsedTime(start_time) < waiting_time) 00240 { 00241 got_event = incoming_events->ReadyForRead(); 00242 if (!got_event) 00243 { 00244 usleep(sleepTimeUsec); 00245 //TLOG_INFO("SharedMemoryReader") << "Waited " << TimeUtils::GetElapsedTime(start_time) << " of " << waiting_time ; 00246 } 00247 } 00248 TLOG_TRACE("SharedMemoryReader") << "ReadyForRead loops END"; 00249 if (!got_event) 00250 { 00251 TLOG_INFO("SharedMemoryReader") 00252 << "InputFailure: Reading timed out in SharedMemoryReader::readNext()"; 00253 keep_looping = resume_after_timeout; 00254 } 00255 } 00256 00257 if (!got_event) 00258 { 00259 TLOG_INFO("SharedMemoryReader") << "Did not receive an event from Shared Memory, returning false"; 00260 shutdownMsgReceived = true; 00261 return false; 00262 } 00263 TLOG_DEBUG("SharedMemoryReader") << "Got Event!"; 00264 auto got_event_time = std::chrono::steady_clock::now(); 00265 00266 auto errflag = false; 00267 auto evtHeader = incoming_events->ReadHeader(errflag); 00268 if (errflag) goto start; // Buffer was changed out from under reader! 00269 auto fragmentTypes = incoming_events->GetFragmentTypes(errflag); 00270 if (errflag) goto start; // Buffer was changed out from under reader! 00271 if (fragmentTypes.size() == 0) 00272 { 00273 TLOG_ERROR("SharedMemoryReader") << "Event has no Fragments! Aborting!"; 00274 incoming_events->ReleaseBuffer(); 00275 return false; 00276 } 00277 auto firstFragmentType = *fragmentTypes.begin(); 00278 TLOG_DEBUG("SharedMemoryReader") << "First Fragment type is " << (int)firstFragmentType << " (" << fragment_type_map_[firstFragmentType] << ")"; 00279 00280 // We return false, indicating we're done reading, if: 00281 // 1) we did not obtain an event, because we timed out and were 00282 // configured NOT to keep trying after a timeout, or 00283 // 2) the event we read was the end-of-data marker: a null 00284 // pointer 00285 if (firstFragmentType == Fragment::EndOfDataFragmentType) 00286 { 00287 TLOG_DEBUG("SharedMemoryReader") << "Received shutdown message, returning false"; 00288 shutdownMsgReceived = true; 00289 incoming_events->ReleaseBuffer(); 00290 return false; 00291 } 00292 00293 size_t qsize = incoming_events->ReadReadyCount(); // save the qsize at this point 00294 00295 // Check the number of fragments in the RawEvent. If we have a single 00296 // fragment and that fragment is marked as EndRun or EndSubrun we'll create 00297 // the special principals for that. 00298 art::Timestamp currentTime = 0; 00299 #if 0 00300 art::TimeValue_t lo_res_time = time(0); 00301 TLOG_ARB(15, "SharedMemoryReader") << "lo_res_time = " << lo_res_time; 00302 currentTime = ((lo_res_time & 0xffffffff) << 32); 00303 #endif 00304 timespec hi_res_time; 00305 int retcode = clock_gettime(CLOCK_REALTIME, &hi_res_time); 00306 TLOG_ARB(15, "SharedMemoryReader") << "hi_res_time tv_sec = " << hi_res_time.tv_sec 00307 << " tv_nsec = " << hi_res_time.tv_nsec 00308 << " (retcode = " << retcode << ")"; 00309 if (retcode == 0) 00310 { 00311 currentTime = ((hi_res_time.tv_sec & 0xffffffff) << 32) | 00312 (hi_res_time.tv_nsec & 0xffffffff); 00313 } 00314 else 00315 { 00316 TLOG_ERROR("SharedMemoryReader") << "Unable to fetch a high-resolution time with clock_gettime for art::Event Timestamp. " 00317 << "The art::Event Timestamp will be zero for event " << evtHeader->event_id; 00318 } 00319 00320 // make new run if inR is 0 or if the run has changed 00321 if (inR == 0 || inR->run() != evtHeader->run_id) 00322 { 00323 outR = pmaker.makeRunPrincipal(evtHeader->run_id, 00324 currentTime); 00325 } 00326 00327 if (firstFragmentType == Fragment::EndOfRunFragmentType) 00328 { 00329 art::EventID const evid(art::EventID::flushEvent()); 00330 outR = pmaker.makeRunPrincipal(evid.runID(), currentTime); 00331 outSR = pmaker.makeSubRunPrincipal(evid.subRunID(), currentTime); 00332 outE = pmaker.makeEventPrincipal(evid, currentTime); 00333 incoming_events->ReleaseBuffer(); 00334 return true; 00335 } 00336 else if (firstFragmentType == Fragment::EndOfSubrunFragmentType) 00337 { 00338 // Check if inR == 0 or is a new run 00339 if (inR == 0 || inR->run() != evtHeader->run_id) 00340 { 00341 outSR = pmaker.makeSubRunPrincipal(evtHeader->run_id, 00342 evtHeader->subrun_id, 00343 currentTime); 00344 art::EventID const evid(art::EventID::flushEvent(outSR->id())); 00345 outE = pmaker.makeEventPrincipal(evid, currentTime); 00346 } 00347 else 00348 { 00349 // If the previous subrun was neither 0 nor flush and was identical with the current 00350 // subrun, then it must have been associated with a data event. In that case, we need 00351 // to generate a flush event with a valid run but flush subrun and event number in order 00352 // to end the subrun. 00353 if (inSR != 0 && !inSR->id().isFlush() && inSR->subRun() == evtHeader->subrun_id) 00354 { 00355 art::EventID const evid(art::EventID::flushEvent(inR->id())); 00356 outSR = pmaker.makeSubRunPrincipal(evid.subRunID(), currentTime); 00357 outE = pmaker.makeEventPrincipal(evid, currentTime); 00358 // If this is either a new or another empty subrun, then generate a flush event with 00359 // valid run and subrun numbers but flush event number 00360 //} else if(inSR==0 || inSR->id().isFlush()){ 00361 } 00362 else 00363 { 00364 outSR = pmaker.makeSubRunPrincipal(evtHeader->run_id, 00365 evtHeader->subrun_id, 00366 currentTime); 00367 art::EventID const evid(art::EventID::flushEvent(outSR->id())); 00368 outE = pmaker.makeEventPrincipal(evid, currentTime); 00369 // Possible error condition 00370 //} else { 00371 } 00372 outR = 0; 00373 } 00374 //outputFileCloseNeeded = true; 00375 incoming_events->ReleaseBuffer(); 00376 return true; 00377 } 00378 00379 // make new subrun if inSR is 0 or if the subrun has changed 00380 art::SubRunID subrun_check(evtHeader->run_id, evtHeader->subrun_id); 00381 if (inSR == 0 || subrun_check != inSR->id()) 00382 { 00383 outSR = pmaker.makeSubRunPrincipal(evtHeader->run_id, 00384 evtHeader->subrun_id, 00385 currentTime); 00386 } 00387 outE = pmaker.makeEventPrincipal(evtHeader->run_id, 00388 evtHeader->subrun_id, 00389 evtHeader->event_id, 00390 currentTime); 00391 00392 // insert the Fragments of each type into the EventPrincipal 00393 std::map<Fragment::type_t, std::string>::const_iterator iter_end = 00394 fragment_type_map_.end(); 00395 for (auto& type_code : fragmentTypes) 00396 { 00397 std::map<Fragment::type_t, std::string>::const_iterator iter = 00398 fragment_type_map_.find(type_code); 00399 TLOG_TRACE("SharedMemoryReader") << "Before GetFragmentsByType call, type is " << (int)type_code; 00400 auto product = incoming_events->GetFragmentsByType(errflag, type_code); 00401 TLOG_TRACE("SharedMemoryReader") << "After GetFragmentsByType call"; 00402 if (errflag) goto start; // Buffer was changed out from under reader! 00403 for (auto &frag : *product) 00404 bytesRead += frag.sizeBytes(); 00405 if (iter != iter_end) 00406 { 00407 if (type_code == artdaq::Fragment::ContainerFragmentType) 00408 { 00409 std::unordered_map<std::string, std::unique_ptr<Fragments>> derived_fragments; 00410 derived_fragments[iter->second] = std::make_unique<Fragments>(); 00411 00412 for (size_t ii = 0; ii < product->size(); ++ii) 00413 { 00414 ContainerFragment cf(product->at(ii)); 00415 auto contained_type = fragment_type_map_.find(cf.fragment_type()); 00416 if (contained_type != iter_end) 00417 { 00418 auto label = iter->second + contained_type->second; 00419 if (!derived_fragments.count(label)) 00420 { 00421 derived_fragments[label] = std::make_unique<Fragments>(); 00422 } 00423 derived_fragments[label]->emplace_back(std::move(product->at(ii))); 00424 } 00425 else 00426 { 00427 derived_fragments[iter->second]->emplace_back(std::move(product->at(ii))); 00428 } 00429 } 00430 00431 for (auto& type : derived_fragments) 00432 { 00433 put_product_in_principal(std::move(type.second), 00434 *outE, 00435 pretend_module_name, 00436 type.first); 00437 } 00438 00439 } 00440 else 00441 { 00442 put_product_in_principal(std::move(product), 00443 *outE, 00444 pretend_module_name, 00445 iter->second); 00446 } 00447 } 00448 else 00449 { 00450 put_product_in_principal(std::move(product), 00451 *outE, 00452 pretend_module_name, 00453 unidentified_instance_name); 00454 TLOG_WARNING("SharedMemoryReader") 00455 << "UnknownFragmentType: The product instance name mapping for fragment type \"" 00456 << ((int)type_code) << "\" is not known. Fragments of this " 00457 << "type will be stored in the event with an instance name of \"" 00458 << unidentified_instance_name << "\"."; 00459 } 00460 } 00461 TLOG_TRACE("SharedMemoryReader") << "After putting fragments in event"; 00462 00463 auto read_finish_time = std::chrono::steady_clock::now(); 00464 incoming_events->ReleaseBuffer(); 00465 auto qcap = incoming_events->size(); 00466 TLOG_ARB(10, "SharedMemoryReader") << "readNext: bytesRead=" << bytesRead << " qsize=" << qsize << " cap=" << qcap << " metricMan=" << (void*)metricMan.get(); 00467 if (metricMan) 00468 { 00469 metricMan->sendMetric("Avg Processing Time", artdaq::TimeUtils::GetElapsedTime(last_read_time, read_start_time), "s", 2, MetricMode::Average); 00470 metricMan->sendMetric("Avg Input Wait Time", artdaq::TimeUtils::GetElapsedTime(read_start_time, got_event_time), "s", 3, MetricMode::Average); 00471 metricMan->sendMetric("Avg Read Time", artdaq::TimeUtils::GetElapsedTime(got_event_time, read_finish_time), "s", 3, MetricMode::Average); 00472 metricMan->sendMetric("bytesRead", bytesRead, "B", 3, MetricMode::LastPoint); 00473 if (qcap > 0) metricMan->sendMetric("queue%Used", static_cast<unsigned long int>(qsize * 100 / qcap), "%", 5, MetricMode::LastPoint); 00474 } 00475 00476 TLOG_TRACE("SharedMemoryReader") << "Returning from readNext"; 00477 last_read_time = std::chrono::steady_clock::now(); 00478 return true; 00479 } 00480 00481 std::map<Fragment::type_t, std::string> fragment_type_map_; 00482 unsigned readNext_calls_; 00483 }; 00484 } // detail 00485 } // artdaq 00486 00487 00488 #endif /* artdaq_ArtModules_detail_SharedMemoryReader_hh */