00001 #ifndef artdaq_ArtModules_detail_SharedMemoryReader_hh
00002 #define artdaq_ArtModules_detail_SharedMemoryReader_hh
00003
00004 #include "artdaq/DAQdata/Globals.hh"
00005 #include "artdaq-core/Utilities/ExceptionHandler.hh"
00006 #include "art/Framework/Core/Frameworkfwd.h"
00007
00008 #include "art/Framework/Core/FileBlock.h"
00009 #include "art/Framework/Core/ProductRegistryHelper.h"
00010 #include "art/Framework/IO/Sources/SourceHelper.h"
00011 #include "art/Framework/Principal/EventPrincipal.h"
00012 #include "art/Framework/Principal/RunPrincipal.h"
00013 #include "art/Framework/Principal/SubRunPrincipal.h"
00014 #include "artdaq-core/Core/SharedMemoryManager.hh"
00015 #include "artdaq-core/Utilities/TimeUtils.hh"
00016 #include "fhiclcpp/ParameterSet.h"
00017 #include "artdaq-core/Data/Fragment.hh"
00018 #include "artdaq-core/Data/ContainerFragment.hh"
00019 #include "artdaq-core/Core/SharedMemoryEventReceiver.hh"
00020 #include "art/Framework/IO/Sources/put_product_in_principal.h"
00021 #include "canvas/Persistency/Provenance/FileFormatVersion.h"
00022 #include <sys/time.h>
00023
00024
00025 #include <string>
00026 #include <map>
00027 #include "artdaq-core/Data/RawEvent.hh"
00028
00029 namespace artdaq
00030 {
00031 namespace detail
00032 {
00036 template<std::map<artdaq::Fragment::type_t, std::string> getDefaultTypes() = artdaq::Fragment::MakeSystemTypeMap >
00037 struct SharedMemoryReader
00038 {
00042 SharedMemoryReader(SharedMemoryReader const&) = delete;
00043
00048 SharedMemoryReader& operator=(SharedMemoryReader const&) = delete;
00049
00050 art::SourceHelper const& pmaker;
00051 std::unique_ptr<SharedMemoryEventReceiver> incoming_events;
00052 double waiting_time;
00053 bool resume_after_timeout;
00054 std::string pretend_module_name;
00055 std::string unidentified_instance_name;
00056 bool shutdownMsgReceived;
00057 bool outputFileCloseNeeded;
00058 size_t bytesRead;
00059
00060
00061
00076 SharedMemoryReader(fhicl::ParameterSet const& ps,
00077 art::ProductRegistryHelper& help,
00078 art::SourceHelper const& pm)
00079 : pmaker(pm)
00080 , waiting_time(ps.get<double>("waiting_time", 86400.0))
00081 , resume_after_timeout(ps.get<bool>("resume_after_timeout", true))
00082 , pretend_module_name(ps.get<std::string>("raw_data_label", "daq"))
00083 , unidentified_instance_name("unidentified")
00084 , shutdownMsgReceived(false)
00085 , outputFileCloseNeeded(false)
00086 , bytesRead(0)
00087 , fragment_type_map_(getDefaultTypes())
00088 , readNext_calls_(0)
00089 {
00090 try {
00091 if (metricMan)
00092 {
00093 metricMan->initialize(ps.get<fhicl::ParameterSet>("metrics", fhicl::ParameterSet()), "artdaqart");
00094 metricMan->do_start();
00095 }
00096 }
00097 catch (...)
00098 {
00099 ExceptionHandler(ExceptionHandlerRethrow::no, "Error loading metrics in SharedMemoryReader()");
00100 }
00101
00102
00103
00104
00105
00106
00107
00108 incoming_events.reset(new SharedMemoryEventReceiver(ps.get<uint32_t>("shared_memory_key", 0xBEE70000 + getppid()), ps.get<uint32_t>("broadcast_shared_memory_key", 0xCEE70000 + getppid())));
00109 my_rank = incoming_events->GetRank();
00110
00111 help.reconstitutes<Fragments, art::InEvent>(pretend_module_name, unidentified_instance_name);
00112 for (auto it = fragment_type_map_.begin(); it != fragment_type_map_.end(); ++it)
00113 {
00114 help.reconstitutes<Fragments, art::InEvent>(pretend_module_name, it->second);
00115 help.reconstitutes<Fragments, art::InEvent>(pretend_module_name, "Container" + it->second);
00116 }
00117 auto extraTypes = ps.get<std::vector<std::pair<Fragment::type_t, std::string>>>("fragment_type_map", std::vector<std::pair<Fragment::type_t, std::string>>());
00118 for (auto it = extraTypes.begin(); it != extraTypes.end(); ++it)
00119 {
00120 fragment_type_map_[it->first] = it->second;
00121 help.reconstitutes<Fragments, art::InEvent>(pretend_module_name, it->second);
00122 help.reconstitutes<Fragments, art::InEvent>(pretend_module_name, "Container" + it->second);
00123 }
00124 TLOG_INFO("SharedMemoryReader") << "SharedMemoryReader initialized with ParameterSet: " << ps.to_string();
00125
00126
00127
00128
00129 }
00130
00139 SharedMemoryReader(fhicl::ParameterSet const& ps,
00140 art::ProductRegistryHelper& help,
00141 art::SourceHelper const& pm,
00142 art::MasterProductRegistry&) : SharedMemoryReader(ps, help, pm)
00143 {}
00144
00148 virtual ~SharedMemoryReader() {
00149 artdaq::Globals::CleanUpGlobals();
00150 }
00151
00155 void closeCurrentFile() {}
00156
00161 void readFile(std::string const&, art::FileBlock*& fb)
00162 {
00163 TLOG_ARB(5, "SharedMemoryReader") << "readFile enter/start";
00164 fb = new art::FileBlock(art::FileFormatVersion(1, "RawEvent2011"), "nothing");
00165 }
00166
00171 bool hasMoreData() const { return (!shutdownMsgReceived); }
00172
00183 bool readNext(art::RunPrincipal* const & inR,
00184 art::SubRunPrincipal* const & inSR,
00185 art::RunPrincipal*& outR,
00186 art::SubRunPrincipal*& outSR,
00187 art::EventPrincipal*& outE)
00188 {
00189 TLOG_DEBUG("SharedMemoryReader") << "readNext BEGIN";
00190
00191
00192
00193
00194
00195 outR = 0;
00196 outSR = 0;
00197 outE = 0;
00198
00199
00200
00201
00202
00203
00204 if (shutdownMsgReceived)
00205 {
00206 TLOG_INFO("SharedMemoryReader") << "Shutdown Message received, returning false (should exit art)";
00207 return false;
00208 }
00209
00210 start:
00211 bool keep_looping = true;
00212 bool got_event = false;
00213 auto sleepTimeUsec = waiting_time * 1000;
00214 if (sleepTimeUsec > 100000) sleepTimeUsec = 100000;
00215 while (keep_looping)
00216 {
00217 TLOG_TRACE("SharedMemoryReader") << "ReadyForRead loops BEGIN";
00218 keep_looping = false;
00219 auto start_time = std::chrono::steady_clock::now();
00220 while (!got_event && TimeUtils::GetElapsedTimeMicroseconds(start_time) < 1000)
00221 {
00222
00223 got_event = incoming_events->ReadyForRead();
00224 }
00225 TLOG_TRACE("SharedMemoryReader") << "ReadyForRead spin end, poll begin";
00226 while (!got_event && TimeUtils::GetElapsedTime(start_time) < waiting_time)
00227 {
00228 got_event = incoming_events->ReadyForRead();
00229 if (!got_event)
00230 {
00231 usleep(sleepTimeUsec);
00232
00233 }
00234 }
00235 TLOG_TRACE("SharedMemoryReader") << "ReadyForRead loops END";
00236 if (!got_event)
00237 {
00238 TLOG_INFO("SharedMemoryReader")
00239 << "InputFailure: Reading timed out in SharedMemoryReader::readNext()";
00240 keep_looping = resume_after_timeout;
00241 }
00242 }
00243
00244 if (!got_event)
00245 {
00246 TLOG_INFO("SharedMemoryReader") << "Did not receive an event from Shared Memory, returning false";
00247 shutdownMsgReceived = true;
00248 return false;
00249 }
00250 TLOG_DEBUG("SharedMemoryReader") << "Got Event!";
00251
00252 auto errflag = false;
00253 auto evtHeader = incoming_events->ReadHeader(errflag);
00254 if (errflag) goto start;
00255 auto fragmentTypes = incoming_events->GetFragmentTypes(errflag);
00256 if (errflag) goto start;
00257 if (fragmentTypes.size() == 0)
00258 {
00259 TLOG_ERROR("SharedMemoryReader") << "Event has no Fragments! Aborting!";
00260 incoming_events->ReleaseBuffer();
00261 return false;
00262 }
00263 auto firstFragmentType = *fragmentTypes.begin();
00264 TLOG_DEBUG("SharedMemoryReader") << "First Fragment type is " << (int)firstFragmentType << " (" << fragment_type_map_[firstFragmentType] << ")";
00265
00266
00267
00268
00269
00270
00271 if (firstFragmentType == Fragment::EndOfDataFragmentType)
00272 {
00273 TLOG_DEBUG("SharedMemoryReader") << "Received shutdown message, returning false";
00274 shutdownMsgReceived = true;
00275 incoming_events->ReleaseBuffer();
00276 return false;
00277 }
00278
00279 size_t qsize = incoming_events->ReadReadyCount();
00280
00281
00282
00283
00284 art::Timestamp currentTime = 0;
00285 #if 0
00286 art::TimeValue_t lo_res_time = time(0);
00287 TLOG_ARB(15, "SharedMemoryReader") << "lo_res_time = " << lo_res_time;
00288 currentTime = ((lo_res_time & 0xffffffff) << 32);
00289 #endif
00290 timespec hi_res_time;
00291 int retcode = clock_gettime(CLOCK_REALTIME, &hi_res_time);
00292 TLOG_ARB(15, "SharedMemoryReader") << "hi_res_time tv_sec = " << hi_res_time.tv_sec
00293 << " tv_nsec = " << hi_res_time.tv_nsec
00294 << " (retcode = " << retcode << ")";
00295 if (retcode == 0)
00296 {
00297 currentTime = ((hi_res_time.tv_sec & 0xffffffff) << 32) |
00298 (hi_res_time.tv_nsec & 0xffffffff);
00299 }
00300 else
00301 {
00302 TLOG_ERROR("SharedMemoryReader") << "Unable to fetch a high-resolution time with clock_gettime for art::Event Timestamp. "
00303 << "The art::Event Timestamp will be zero for event " << evtHeader->event_id;
00304 }
00305
00306
00307 if (inR == 0 || inR->run() != evtHeader->run_id)
00308 {
00309 outR = pmaker.makeRunPrincipal(evtHeader->run_id,
00310 currentTime);
00311 }
00312
00313 if (firstFragmentType == Fragment::EndOfRunFragmentType)
00314 {
00315 art::EventID const evid(art::EventID::flushEvent());
00316 outR = pmaker.makeRunPrincipal(evid.runID(), currentTime);
00317 outSR = pmaker.makeSubRunPrincipal(evid.subRunID(), currentTime);
00318 outE = pmaker.makeEventPrincipal(evid, currentTime);
00319 incoming_events->ReleaseBuffer();
00320 return true;
00321 }
00322 else if (firstFragmentType == Fragment::EndOfSubrunFragmentType)
00323 {
00324
00325 if (inR == 0 || inR->run() != evtHeader->run_id)
00326 {
00327 outSR = pmaker.makeSubRunPrincipal(evtHeader->run_id,
00328 evtHeader->subrun_id,
00329 currentTime);
00330 art::EventID const evid(art::EventID::flushEvent(outSR->id()));
00331 outE = pmaker.makeEventPrincipal(evid, currentTime);
00332 }
00333 else
00334 {
00335
00336
00337
00338
00339 if (inSR != 0 && !inSR->id().isFlush() && inSR->subRun() == evtHeader->subrun_id)
00340 {
00341 art::EventID const evid(art::EventID::flushEvent(inR->id()));
00342 outSR = pmaker.makeSubRunPrincipal(evid.subRunID(), currentTime);
00343 outE = pmaker.makeEventPrincipal(evid, currentTime);
00344
00345
00346
00347 }
00348 else
00349 {
00350 outSR = pmaker.makeSubRunPrincipal(evtHeader->run_id,
00351 evtHeader->subrun_id,
00352 currentTime);
00353 art::EventID const evid(art::EventID::flushEvent(outSR->id()));
00354 outE = pmaker.makeEventPrincipal(evid, currentTime);
00355
00356
00357 }
00358 outR = 0;
00359 }
00360
00361 incoming_events->ReleaseBuffer();
00362 return true;
00363 }
00364
00365
00366 art::SubRunID subrun_check(evtHeader->run_id, evtHeader->subrun_id);
00367 if (inSR == 0 || subrun_check != inSR->id())
00368 {
00369 outSR = pmaker.makeSubRunPrincipal(evtHeader->run_id,
00370 evtHeader->subrun_id,
00371 currentTime);
00372 }
00373 outE = pmaker.makeEventPrincipal(evtHeader->run_id,
00374 evtHeader->subrun_id,
00375 evtHeader->event_id,
00376 currentTime);
00377
00378
00379 std::map<Fragment::type_t, std::string>::const_iterator iter_end =
00380 fragment_type_map_.end();
00381 for (auto& type_code : fragmentTypes)
00382 {
00383 std::map<Fragment::type_t, std::string>::const_iterator iter =
00384 fragment_type_map_.find(type_code);
00385 auto product = incoming_events->GetFragmentsByType(errflag, type_code);
00386 if (errflag) goto start;
00387 for (auto &frag : *product)
00388 bytesRead += frag.sizeBytes();
00389 if (iter != iter_end)
00390 {
00391 if (type_code == artdaq::Fragment::ContainerFragmentType)
00392 {
00393 std::unordered_map<std::string, std::unique_ptr<Fragments>> derived_fragments;
00394 derived_fragments[iter->second] = std::make_unique<Fragments>();
00395
00396 for (size_t ii = 0; ii < product->size(); ++ii)
00397 {
00398 ContainerFragment cf(product->at(ii));
00399 auto contained_type = fragment_type_map_.find(cf.fragment_type());
00400 if (contained_type != iter_end)
00401 {
00402 auto label = iter->second + contained_type->second;
00403 if (!derived_fragments.count(label))
00404 {
00405 derived_fragments[label] = std::make_unique<Fragments>();
00406 }
00407 derived_fragments[label]->emplace_back(std::move(product->at(ii)));
00408 }
00409 else
00410 {
00411 derived_fragments[iter->second]->emplace_back(std::move(product->at(ii)));
00412 }
00413 }
00414
00415 for (auto& type : derived_fragments)
00416 {
00417 put_product_in_principal(std::move(type.second),
00418 *outE,
00419 pretend_module_name,
00420 type.first);
00421 }
00422
00423 }
00424 else
00425 {
00426 put_product_in_principal(std::move(product),
00427 *outE,
00428 pretend_module_name,
00429 iter->second);
00430 }
00431 }
00432 else
00433 {
00434 put_product_in_principal(std::move(product),
00435 *outE,
00436 pretend_module_name,
00437 unidentified_instance_name);
00438 TLOG_WARNING("SharedMemoryReader")
00439 << "UnknownFragmentType: The product instance name mapping for fragment type \""
00440 << ((int)type_code) << "\" is not known. Fragments of this "
00441 << "type will be stored in the event with an instance name of \""
00442 << unidentified_instance_name << "\".";
00443 }
00444 }
00445 incoming_events->ReleaseBuffer();
00446 TLOG_ARB(10, "SharedMemoryReader") << "readNext: bytesRead=" << bytesRead << " qsize=" << qsize << " cap=" << incoming_events->size() << " metricMan=" << (void*)metricMan.get();
00447 if (metricMan)
00448 {
00449 metricMan->sendMetric("bytesRead", bytesRead, "B", 5, MetricMode::Accumulate, "", true);
00450 metricMan->sendMetric("queue%Used", static_cast<unsigned long int>(qsize * 100 / incoming_events->size()), "%", 5, MetricMode::LastPoint, "", true);
00451 }
00452
00453 return true;
00454 }
00455
00456 std::map<Fragment::type_t, std::string> fragment_type_map_;
00457 unsigned readNext_calls_;
00458 };
00459 }
00460 }
00461
00462
00463 #endif