00001 #ifndef artdaq_ArtModules_detail_SharedMemoryReader_hh
00002 #define artdaq_ArtModules_detail_SharedMemoryReader_hh
00003
00004 #include "artdaq/DAQdata/Globals.hh"
00005 #include "artdaq-core/Utilities/ExceptionHandler.hh"
00006 #include "art/Framework/Core/Frameworkfwd.h"
00007
00008 #include "art/Framework/Core/FileBlock.h"
00009 #include "art/Framework/Core/ProductRegistryHelper.h"
00010 #include "art/Framework/IO/Sources/SourceHelper.h"
00011 #include "art/Framework/Principal/EventPrincipal.h"
00012 #include "art/Framework/Principal/RunPrincipal.h"
00013 #include "art/Framework/Principal/SubRunPrincipal.h"
00014 #include "artdaq-core/Core/SharedMemoryManager.hh"
00015 #include "artdaq-core/Utilities/TimeUtils.hh"
00016 #include "fhiclcpp/ParameterSet.h"
00017 #include "artdaq-core/Data/Fragment.hh"
00018 #include "artdaq-core/Data/ContainerFragment.hh"
00019 #include "artdaq-core/Core/SharedMemoryEventReceiver.hh"
00020 #include "art/Framework/IO/Sources/put_product_in_principal.h"
00021 #include "canvas/Persistency/Provenance/FileFormatVersion.h"
00022 #include <sys/time.h>
00023
00024
00025 #include <string>
00026 #include <map>
00027 #include "artdaq-core/Data/RawEvent.hh"
00028
00029 namespace artdaq
00030 {
00031 namespace detail
00032 {
00036 template<std::map<artdaq::Fragment::type_t, std::string> getDefaultTypes() = artdaq::Fragment::MakeSystemTypeMap >
00037 struct SharedMemoryReader
00038 {
00042 SharedMemoryReader(SharedMemoryReader const&) = delete;
00043
00048 SharedMemoryReader& operator=(SharedMemoryReader const&) = delete;
00049
00050 art::SourceHelper const& pmaker;
00051 std::unique_ptr<SharedMemoryEventReceiver> incoming_events;
00052 double waiting_time;
00053 bool resume_after_timeout;
00054 std::string pretend_module_name;
00055 std::string unidentified_instance_name;
00056 bool shutdownMsgReceived;
00057 bool outputFileCloseNeeded;
00058 size_t bytesRead;
00059 std::chrono::steady_clock::time_point last_read_time;
00060
00061
00062
00077 SharedMemoryReader(fhicl::ParameterSet const& ps,
00078 art::ProductRegistryHelper& help,
00079 art::SourceHelper const& pm)
00080 : pmaker(pm)
00081 , waiting_time(ps.get<double>("waiting_time", 86400.0))
00082 , resume_after_timeout(ps.get<bool>("resume_after_timeout", true))
00083 , pretend_module_name(ps.get<std::string>("raw_data_label", "daq"))
00084 , unidentified_instance_name("unidentified")
00085 , shutdownMsgReceived(false)
00086 , outputFileCloseNeeded(false)
00087 , bytesRead(0)
00088 , last_read_time(std::chrono::steady_clock::now())
00089 , fragment_type_map_(getDefaultTypes())
00090 , readNext_calls_(0)
00091 {
00092
00093
00094
00095
00096
00097
00098
00099 incoming_events.reset(new SharedMemoryEventReceiver(ps.get<uint32_t>("shared_memory_key", 0xBEE70000 + getppid()), ps.get<uint32_t>("broadcast_shared_memory_key", 0xCEE70000 + getppid())));
00100 my_rank = incoming_events->GetRank();
00101
00102 char const* artapp_env = getenv("ARTDAQ_APPLICATION_NAME");
00103 std::string artapp_str = "";
00104 if (artapp_env != NULL)
00105 {
00106 artapp_str = std::string(artapp_env) + "_";
00107 }
00108
00109 app_name = artapp_str + "art" + std::to_string(incoming_events->GetMyId());
00110
00111 try {
00112 if (metricMan)
00113 {
00114 metricMan->initialize(ps.get<fhicl::ParameterSet>("metrics", fhicl::ParameterSet()), app_name);
00115 metricMan->do_start();
00116 }
00117 }
00118 catch (...)
00119 {
00120 ExceptionHandler(ExceptionHandlerRethrow::no, "Error loading metrics in SharedMemoryReader()");
00121 }
00122
00123 help.reconstitutes<Fragments, art::InEvent>(pretend_module_name, unidentified_instance_name);
00124 for (auto it = fragment_type_map_.begin(); it != fragment_type_map_.end(); ++it)
00125 {
00126 help.reconstitutes<Fragments, art::InEvent>(pretend_module_name, it->second);
00127 help.reconstitutes<Fragments, art::InEvent>(pretend_module_name, "Container" + it->second);
00128 }
00129 auto extraTypes = ps.get<std::vector<std::pair<Fragment::type_t, std::string>>>("fragment_type_map", std::vector<std::pair<Fragment::type_t, std::string>>());
00130 for (auto it = extraTypes.begin(); it != extraTypes.end(); ++it)
00131 {
00132 fragment_type_map_[it->first] = it->second;
00133 help.reconstitutes<Fragments, art::InEvent>(pretend_module_name, it->second);
00134 help.reconstitutes<Fragments, art::InEvent>(pretend_module_name, "Container" + it->second);
00135 }
00136 TLOG_INFO("SharedMemoryReader") << "SharedMemoryReader initialized with ParameterSet: " << ps.to_string();
00137
00138
00139
00140
00141 }
00142
00151 SharedMemoryReader(fhicl::ParameterSet const& ps,
00152 art::ProductRegistryHelper& help,
00153 art::SourceHelper const& pm,
00154 art::MasterProductRegistry&) : SharedMemoryReader(ps, help, pm)
00155 {}
00156
00160 virtual ~SharedMemoryReader() {
00161 artdaq::Globals::CleanUpGlobals();
00162 }
00163
00167 void closeCurrentFile() {}
00168
00173 void readFile(std::string const&, art::FileBlock*& fb)
00174 {
00175 TLOG_ARB(5, "SharedMemoryReader") << "readFile enter/start";
00176 fb = new art::FileBlock(art::FileFormatVersion(1, "RawEvent2011"), "nothing");
00177 }
00178
00183 bool hasMoreData() const { return (!shutdownMsgReceived); }
00184
00195 bool readNext(art::RunPrincipal* const & inR,
00196 art::SubRunPrincipal* const & inSR,
00197 art::RunPrincipal*& outR,
00198 art::SubRunPrincipal*& outSR,
00199 art::EventPrincipal*& outE)
00200 {
00201 TLOG_DEBUG("SharedMemoryReader") << "readNext BEGIN";
00202
00203
00204
00205
00206
00207 outR = 0;
00208 outSR = 0;
00209 outE = 0;
00210
00211
00212
00213
00214
00215
00216 if (shutdownMsgReceived)
00217 {
00218 TLOG_INFO("SharedMemoryReader") << "Shutdown Message received, returning false (should exit art)";
00219 return false;
00220 }
00221
00222 auto read_start_time = std::chrono::steady_clock::now();
00223 start:
00224 bool keep_looping = true;
00225 bool got_event = false;
00226 auto sleepTimeUsec = waiting_time * 1000;
00227 if (sleepTimeUsec > 100000) sleepTimeUsec = 100000;
00228 while (keep_looping)
00229 {
00230 TLOG_TRACE("SharedMemoryReader") << "ReadyForRead loops BEGIN";
00231 keep_looping = false;
00232 auto start_time = std::chrono::steady_clock::now();
00233 while (!got_event && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 1000)
00234 {
00235
00236 got_event = incoming_events->ReadyForRead();
00237 }
00238 TLOG_TRACE("SharedMemoryReader") << "ReadyForRead spin end, poll begin";
00239 while (!got_event && TimeUtils::GetElapsedTime(start_time) < waiting_time)
00240 {
00241 got_event = incoming_events->ReadyForRead();
00242 if (!got_event)
00243 {
00244 usleep(sleepTimeUsec);
00245
00246 }
00247 }
00248 TLOG_TRACE("SharedMemoryReader") << "ReadyForRead loops END";
00249 if (!got_event)
00250 {
00251 TLOG_INFO("SharedMemoryReader")
00252 << "InputFailure: Reading timed out in SharedMemoryReader::readNext()";
00253 keep_looping = resume_after_timeout;
00254 }
00255 }
00256
00257 if (!got_event)
00258 {
00259 TLOG_INFO("SharedMemoryReader") << "Did not receive an event from Shared Memory, returning false";
00260 shutdownMsgReceived = true;
00261 return false;
00262 }
00263 TLOG_DEBUG("SharedMemoryReader") << "Got Event!";
00264 auto got_event_time = std::chrono::steady_clock::now();
00265
00266 auto errflag = false;
00267 auto evtHeader = incoming_events->ReadHeader(errflag);
00268 if (errflag) goto start;
00269 auto fragmentTypes = incoming_events->GetFragmentTypes(errflag);
00270 if (errflag) goto start;
00271 if (fragmentTypes.size() == 0)
00272 {
00273 TLOG_ERROR("SharedMemoryReader") << "Event has no Fragments! Aborting!";
00274 incoming_events->ReleaseBuffer();
00275 return false;
00276 }
00277 auto firstFragmentType = *fragmentTypes.begin();
00278 TLOG_DEBUG("SharedMemoryReader") << "First Fragment type is " << (int)firstFragmentType << " (" << fragment_type_map_[firstFragmentType] << ")";
00279
00280
00281
00282
00283
00284
00285 if (firstFragmentType == Fragment::EndOfDataFragmentType)
00286 {
00287 TLOG_DEBUG("SharedMemoryReader") << "Received shutdown message, returning false";
00288 shutdownMsgReceived = true;
00289 incoming_events->ReleaseBuffer();
00290 return false;
00291 }
00292
00293 size_t qsize = incoming_events->ReadReadyCount();
00294
00295
00296
00297
00298 art::Timestamp currentTime = 0;
00299 #if 0
00300 art::TimeValue_t lo_res_time = time(0);
00301 TLOG_ARB(15, "SharedMemoryReader") << "lo_res_time = " << lo_res_time;
00302 currentTime = ((lo_res_time & 0xffffffff) << 32);
00303 #endif
00304 timespec hi_res_time;
00305 int retcode = clock_gettime(CLOCK_REALTIME, &hi_res_time);
00306 TLOG_ARB(15, "SharedMemoryReader") << "hi_res_time tv_sec = " << hi_res_time.tv_sec
00307 << " tv_nsec = " << hi_res_time.tv_nsec
00308 << " (retcode = " << retcode << ")";
00309 if (retcode == 0)
00310 {
00311 currentTime = ((hi_res_time.tv_sec & 0xffffffff) << 32) |
00312 (hi_res_time.tv_nsec & 0xffffffff);
00313 }
00314 else
00315 {
00316 TLOG_ERROR("SharedMemoryReader") << "Unable to fetch a high-resolution time with clock_gettime for art::Event Timestamp. "
00317 << "The art::Event Timestamp will be zero for event " << evtHeader->event_id;
00318 }
00319
00320
00321 if (inR == 0 || inR->run() != evtHeader->run_id)
00322 {
00323 outR = pmaker.makeRunPrincipal(evtHeader->run_id,
00324 currentTime);
00325 }
00326
00327 if (firstFragmentType == Fragment::EndOfRunFragmentType)
00328 {
00329 art::EventID const evid(art::EventID::flushEvent());
00330 outR = pmaker.makeRunPrincipal(evid.runID(), currentTime);
00331 outSR = pmaker.makeSubRunPrincipal(evid.subRunID(), currentTime);
00332 outE = pmaker.makeEventPrincipal(evid, currentTime);
00333 incoming_events->ReleaseBuffer();
00334 return true;
00335 }
00336 else if (firstFragmentType == Fragment::EndOfSubrunFragmentType)
00337 {
00338
00339 if (inR == 0 || inR->run() != evtHeader->run_id)
00340 {
00341 outSR = pmaker.makeSubRunPrincipal(evtHeader->run_id,
00342 evtHeader->subrun_id,
00343 currentTime);
00344 art::EventID const evid(art::EventID::flushEvent(outSR->id()));
00345 outE = pmaker.makeEventPrincipal(evid, currentTime);
00346 }
00347 else
00348 {
00349
00350
00351
00352
00353 if (inSR != 0 && !inSR->id().isFlush() && inSR->subRun() == evtHeader->subrun_id)
00354 {
00355 art::EventID const evid(art::EventID::flushEvent(inR->id()));
00356 outSR = pmaker.makeSubRunPrincipal(evid.subRunID(), currentTime);
00357 outE = pmaker.makeEventPrincipal(evid, currentTime);
00358
00359
00360
00361 }
00362 else
00363 {
00364 outSR = pmaker.makeSubRunPrincipal(evtHeader->run_id,
00365 evtHeader->subrun_id,
00366 currentTime);
00367 art::EventID const evid(art::EventID::flushEvent(outSR->id()));
00368 outE = pmaker.makeEventPrincipal(evid, currentTime);
00369
00370
00371 }
00372 outR = 0;
00373 }
00374
00375 incoming_events->ReleaseBuffer();
00376 return true;
00377 }
00378
00379
00380 art::SubRunID subrun_check(evtHeader->run_id, evtHeader->subrun_id);
00381 if (inSR == 0 || subrun_check != inSR->id())
00382 {
00383 outSR = pmaker.makeSubRunPrincipal(evtHeader->run_id,
00384 evtHeader->subrun_id,
00385 currentTime);
00386 }
00387 outE = pmaker.makeEventPrincipal(evtHeader->run_id,
00388 evtHeader->subrun_id,
00389 evtHeader->event_id,
00390 currentTime);
00391
00392
00393 std::map<Fragment::type_t, std::string>::const_iterator iter_end =
00394 fragment_type_map_.end();
00395 for (auto& type_code : fragmentTypes)
00396 {
00397 std::map<Fragment::type_t, std::string>::const_iterator iter =
00398 fragment_type_map_.find(type_code);
00399 auto product = incoming_events->GetFragmentsByType(errflag, type_code);
00400 if (errflag) goto start;
00401 for (auto &frag : *product)
00402 bytesRead += frag.sizeBytes();
00403 if (iter != iter_end)
00404 {
00405 if (type_code == artdaq::Fragment::ContainerFragmentType)
00406 {
00407 std::unordered_map<std::string, std::unique_ptr<Fragments>> derived_fragments;
00408 derived_fragments[iter->second] = std::make_unique<Fragments>();
00409
00410 for (size_t ii = 0; ii < product->size(); ++ii)
00411 {
00412 ContainerFragment cf(product->at(ii));
00413 auto contained_type = fragment_type_map_.find(cf.fragment_type());
00414 if (contained_type != iter_end)
00415 {
00416 auto label = iter->second + contained_type->second;
00417 if (!derived_fragments.count(label))
00418 {
00419 derived_fragments[label] = std::make_unique<Fragments>();
00420 }
00421 derived_fragments[label]->emplace_back(std::move(product->at(ii)));
00422 }
00423 else
00424 {
00425 derived_fragments[iter->second]->emplace_back(std::move(product->at(ii)));
00426 }
00427 }
00428
00429 for (auto& type : derived_fragments)
00430 {
00431 put_product_in_principal(std::move(type.second),
00432 *outE,
00433 pretend_module_name,
00434 type.first);
00435 }
00436
00437 }
00438 else
00439 {
00440 put_product_in_principal(std::move(product),
00441 *outE,
00442 pretend_module_name,
00443 iter->second);
00444 }
00445 }
00446 else
00447 {
00448 put_product_in_principal(std::move(product),
00449 *outE,
00450 pretend_module_name,
00451 unidentified_instance_name);
00452 TLOG_WARNING("SharedMemoryReader")
00453 << "UnknownFragmentType: The product instance name mapping for fragment type \""
00454 << ((int)type_code) << "\" is not known. Fragments of this "
00455 << "type will be stored in the event with an instance name of \""
00456 << unidentified_instance_name << "\".";
00457 }
00458 }
00459
00460 auto read_finish_time = std::chrono::steady_clock::now();
00461 incoming_events->ReleaseBuffer();
00462 auto qcap = incoming_events->size();
00463 TLOG_ARB(10, "SharedMemoryReader") << "readNext: bytesRead=" << bytesRead << " qsize=" << qsize << " cap=" << qcap << " metricMan=" << (void*)metricMan.get();
00464 if (metricMan)
00465 {
00466 metricMan->sendMetric("Avg Processing Time", artdaq::TimeUtils::GetElapsedTime(last_read_time, read_start_time), "s", 2, MetricMode::Average);
00467 metricMan->sendMetric("Avg Input Wait Time", artdaq::TimeUtils::GetElapsedTime(read_start_time, got_event_time), "s", 3, MetricMode::Average);
00468 metricMan->sendMetric("Avg Read Time", artdaq::TimeUtils::GetElapsedTime(got_event_time, read_finish_time), "s", 3, MetricMode::Average);
00469 metricMan->sendMetric("bytesRead", bytesRead, "B", 3, MetricMode::LastPoint);
00470 if (qcap > 0) metricMan->sendMetric("queue%Used", static_cast<unsigned long int>(qsize * 100 / qcap), "%", 5, MetricMode::LastPoint);
00471 }
00472
00473 last_read_time = std::chrono::steady_clock::now();
00474 return true;
00475 }
00476
00477 std::map<Fragment::type_t, std::string> fragment_type_map_;
00478 unsigned readNext_calls_;
00479 };
00480 }
00481 }
00482
00483
00484 #endif