00001 #ifndef artdaq_ArtModules_detail_SharedMemoryReader_hh
00002 #define artdaq_ArtModules_detail_SharedMemoryReader_hh
00003
00004 #include "artdaq/DAQdata/Globals.hh"
00005 #include "art/Framework/Core/Frameworkfwd.h"
00006
00007 #include "art/Framework/Core/FileBlock.h"
00008 #include "art/Framework/Core/ProductRegistryHelper.h"
00009 #include "art/Framework/IO/Sources/SourceHelper.h"
00010 #include "art/Framework/Principal/EventPrincipal.h"
00011 #include "art/Framework/Principal/RunPrincipal.h"
00012 #include "art/Framework/Principal/SubRunPrincipal.h"
00013 #include "artdaq-core/Core/SharedMemoryManager.hh"
00014 #include "artdaq-core/Utilities/TimeUtils.hh"
00015 #include "fhiclcpp/ParameterSet.h"
00016 #include "artdaq-core/Data/Fragment.hh"
00017 #include "artdaq-core/Data/ContainerFragment.hh"
00018 #include "artdaq-core/Core/SharedMemoryEventReceiver.hh"
00019 #include "art/Framework/IO/Sources/put_product_in_principal.h"
00020 #include "canvas/Persistency/Provenance/FileFormatVersion.h"
00021 #include <sys/time.h>
00022
00023
00024 #include <string>
00025 #include <map>
00026 #include "artdaq-core/Data/RawEvent.hh"
00027
00028 namespace artdaq
00029 {
00030 namespace detail
00031 {
00035 template<std::map<artdaq::Fragment::type_t, std::string> getDefaultTypes() = artdaq::Fragment::MakeSystemTypeMap >
00036 struct SharedMemoryReader
00037 {
00041 SharedMemoryReader(SharedMemoryReader const&) = delete;
00042
00047 SharedMemoryReader& operator=(SharedMemoryReader const&) = delete;
00048
00049 art::SourceHelper const pmaker;
00050 std::unique_ptr<SharedMemoryEventReceiver> incoming_events;
00051 double waiting_time;
00052 bool resume_after_timeout;
00053 std::string pretend_module_name;
00054 std::string unidentified_instance_name;
00055 bool shutdownMsgReceived;
00056 bool outputFileCloseNeeded;
00057 size_t bytesRead;
00058
00059
00060
00075 SharedMemoryReader(fhicl::ParameterSet const& ps,
00076 art::ProductRegistryHelper& help,
00077 art::SourceHelper const& pm)
00078 : pmaker(pm)
00079 , waiting_time(ps.get<double>("waiting_time", 86400.0))
00080 , resume_after_timeout(ps.get<bool>("resume_after_timeout", true))
00081 , pretend_module_name(ps.get<std::string>("raw_data_label", "daq"))
00082 , unidentified_instance_name("unidentified")
00083 , shutdownMsgReceived(false)
00084 , outputFileCloseNeeded(false)
00085 , bytesRead(0)
00086 , fragment_type_map_(getDefaultTypes())
00087 , readNext_calls_(0)
00088 {
00089
00090
00091
00092
00093
00094
00095 incoming_events.reset(new SharedMemoryEventReceiver(ps.get<uint32_t>("shared_memory_key", 0xBEE70000 + getppid()), ps.get<uint32_t>("broadcast_shared_memory_key", 0xCEE70000 + getppid())));
00096
00097 help.reconstitutes<Fragments, art::InEvent>(pretend_module_name, unidentified_instance_name);
00098 for (auto it = fragment_type_map_.begin(); it != fragment_type_map_.end(); ++it)
00099 {
00100 help.reconstitutes<Fragments, art::InEvent>(pretend_module_name, it->second);
00101 help.reconstitutes<Fragments, art::InEvent>(pretend_module_name, "Container" + it->second);
00102 }
00103 auto extraTypes = ps.get<std::vector<std::pair<Fragment::type_t, std::string>>>("fragment_type_map", std::vector<std::pair<Fragment::type_t, std::string>>());
00104 for (auto it = extraTypes.begin(); it != extraTypes.end(); ++it)
00105 {
00106 fragment_type_map_[it->first] = it->second;
00107 help.reconstitutes<Fragments, art::InEvent>(pretend_module_name, it->second);
00108 help.reconstitutes<Fragments, art::InEvent>(pretend_module_name, "Container" + it->second);
00109 }
00110 TLOG_INFO("SharedMemoryReader") << "SharedMemoryReader initialized with ParameterSet: " << ps.to_string() << TLOG_ENDL;
00111
00112
00113
00114
00115 }
00116
00125 SharedMemoryReader(fhicl::ParameterSet const& ps,
00126 art::ProductRegistryHelper& help,
00127 art::SourceHelper const& pm,
00128 art::MasterProductRegistry&) : SharedMemoryReader(ps, help, pm)
00129 {}
00130
00134 virtual ~SharedMemoryReader() = default;
00135
00139 void closeCurrentFile() {}
00140
00145 void readFile(std::string const&, art::FileBlock*& fb)
00146 {
00147 TRACE(5, "SharedMemoryReader::readFile enter/start");
00148 fb = new art::FileBlock(art::FileFormatVersion(1, "RawEvent2011"), "nothing");
00149 }
00150
00155 bool hasMoreData() const { return (!shutdownMsgReceived); }
00156
00167 bool readNext(art::RunPrincipal* const & inR,
00168 art::SubRunPrincipal* const & inSR,
00169 art::RunPrincipal*& outR,
00170 art::SubRunPrincipal*& outSR,
00171 art::EventPrincipal*& outE)
00172 {
00173 TLOG_DEBUG("SharedMemoryReader") << "readNext BEGIN" << TLOG_ENDL;
00174
00175
00176
00177
00178
00179 outR = 0;
00180 outSR = 0;
00181 outE = 0;
00182
00183
00184
00185
00186
00187
00188 if (shutdownMsgReceived) return false;
00189
00190 start:
00191 bool keep_looping = true;
00192 bool got_event = false;
00193 auto sleepTimeUsec = waiting_time * 1000;
00194 if (sleepTimeUsec > 100000) sleepTimeUsec = 100000;
00195 while (keep_looping)
00196 {
00197 keep_looping = false;
00198 auto start_time = std::chrono::steady_clock::now();
00199 while (!got_event && TimeUtils::GetElapsedTime(start_time) < waiting_time)
00200 {
00201 got_event = incoming_events->ReadyForRead();
00202 if (!got_event)
00203 {
00204 usleep(sleepTimeUsec);
00205
00206 }
00207 }
00208 if (!got_event)
00209 {
00210 TLOG_INFO("SharedMemoryReader")
00211 << "InputFailure: Reading timed out in SharedMemoryReader::readNext()" << TLOG_ENDL;
00212 keep_looping = resume_after_timeout;
00213 }
00214 }
00215
00216 if (!got_event)
00217 {
00218 TLOG_INFO("SharedMemoryReader") << "Did not receive an event from Shared Memory, returning false" << TLOG_ENDL;
00219 shutdownMsgReceived = true;
00220 return false;
00221 }
00222 TLOG_DEBUG("SharedMemoryReader") << "Got Event!" << TLOG_ENDL;
00223
00224 auto errflag = false;
00225 auto evtHeader = incoming_events->ReadHeader(errflag);
00226 if (errflag) goto start;
00227 auto fragmentTypes = incoming_events->GetFragmentTypes(errflag);
00228 if (errflag) goto start;
00229 if (fragmentTypes.size() == 0)
00230 {
00231 TLOG_ERROR("SharedMemoryReader") << "Event has no Fragments! Aborting!" << TLOG_ENDL;
00232 incoming_events->ReleaseBuffer();
00233 return false;
00234 }
00235 auto firstFragmentType = *fragmentTypes.begin();
00236 TLOG_DEBUG("SharedMemoryReader") << "First Fragment type is " << std::to_string(firstFragmentType) << TLOG_ENDL;
00237
00238
00239
00240
00241
00242
00243 if (firstFragmentType == Fragment::EndOfDataFragmentType)
00244 {
00245 TLOG_DEBUG("SharedMemoryReader") << "Received shutdown message, returning false" << TLOG_ENDL;
00246 shutdownMsgReceived = true;
00247 incoming_events->ReleaseBuffer();
00248 return false;
00249 }
00250
00251 size_t qsize = incoming_events->ReadReadyCount();
00252
00253
00254
00255
00256 art::Timestamp currentTime = time(0);
00257
00258
00259 if (inR == 0 || inR->run() != evtHeader->run_id)
00260 {
00261 outR = pmaker.makeRunPrincipal(evtHeader->run_id,
00262 currentTime);
00263 }
00264
00265 if (firstFragmentType == Fragment::EndOfRunFragmentType)
00266 {
00267 art::EventID const evid(art::EventID::flushEvent());
00268 outR = pmaker.makeRunPrincipal(evid.runID(), currentTime);
00269 outSR = pmaker.makeSubRunPrincipal(evid.subRunID(), currentTime);
00270 outE = pmaker.makeEventPrincipal(evid, currentTime);
00271 incoming_events->ReleaseBuffer();
00272 return true;
00273 }
00274 else if (firstFragmentType == Fragment::EndOfSubrunFragmentType)
00275 {
00276
00277 if (inR == 0 || inR->run() != evtHeader->run_id)
00278 {
00279 outSR = pmaker.makeSubRunPrincipal(evtHeader->run_id,
00280 evtHeader->subrun_id,
00281 currentTime);
00282 art::EventID const evid(art::EventID::flushEvent(outSR->id()));
00283 outE = pmaker.makeEventPrincipal(evid, currentTime);
00284 }
00285 else
00286 {
00287
00288
00289
00290
00291 if (inSR != 0 && !inSR->id().isFlush() && inSR->subRun() == evtHeader->subrun_id)
00292 {
00293 art::EventID const evid(art::EventID::flushEvent(inR->id()));
00294 outSR = pmaker.makeSubRunPrincipal(evid.subRunID(), currentTime);
00295 outE = pmaker.makeEventPrincipal(evid, currentTime);
00296
00297
00298
00299 }
00300 else
00301 {
00302 outSR = pmaker.makeSubRunPrincipal(evtHeader->run_id,
00303 evtHeader->subrun_id,
00304 currentTime);
00305 art::EventID const evid(art::EventID::flushEvent(outSR->id()));
00306 outE = pmaker.makeEventPrincipal(evid, currentTime);
00307
00308
00309 }
00310 outR = 0;
00311 }
00312
00313 incoming_events->ReleaseBuffer();
00314 return true;
00315 }
00316
00317
00318 art::SubRunID subrun_check(evtHeader->run_id, evtHeader->subrun_id);
00319 if (inSR == 0 || subrun_check != inSR->id())
00320 {
00321 outSR = pmaker.makeSubRunPrincipal(evtHeader->run_id,
00322 evtHeader->subrun_id,
00323 currentTime);
00324 }
00325 outE = pmaker.makeEventPrincipal(evtHeader->run_id,
00326 evtHeader->subrun_id,
00327 evtHeader->sequence_id,
00328 currentTime);
00329
00330
00331 std::map<Fragment::type_t, std::string>::const_iterator iter_end =
00332 fragment_type_map_.end();
00333 for (auto& type_code : fragmentTypes)
00334 {
00335 std::map<Fragment::type_t, std::string>::const_iterator iter =
00336 fragment_type_map_.find(type_code);
00337 auto product = incoming_events->GetFragmentsByType(errflag, type_code);
00338 if (errflag) goto start;
00339 for (auto &frag : *product)
00340 bytesRead += frag.sizeBytes();
00341 if (iter != iter_end)
00342 {
00343 if (type_code == artdaq::Fragment::ContainerFragmentType)
00344 {
00345 std::unordered_map<std::string, std::unique_ptr<Fragments>> derived_fragments;
00346 derived_fragments[iter->second] = std::make_unique<Fragments>();
00347
00348 for (size_t ii = 0; ii < product->size(); ++ii)
00349 {
00350 ContainerFragment cf(product->at(ii));
00351 auto contained_type = fragment_type_map_.find(cf.fragment_type());
00352 if (contained_type != iter_end)
00353 {
00354 auto label = iter->second + contained_type->second;
00355 if (!derived_fragments.count(label))
00356 {
00357 derived_fragments[label] = std::make_unique<Fragments>();
00358 }
00359 derived_fragments[label]->emplace_back(std::move(product->at(ii)));
00360 }
00361 else
00362 {
00363 derived_fragments[iter->second]->emplace_back(std::move(product->at(ii)));
00364 }
00365 }
00366
00367 for (auto& type : derived_fragments)
00368 {
00369 put_product_in_principal(std::move(type.second),
00370 *outE,
00371 pretend_module_name,
00372 type.first);
00373 }
00374
00375 }
00376 else
00377 {
00378 put_product_in_principal(std::move(product),
00379 *outE,
00380 pretend_module_name,
00381 iter->second);
00382 }
00383 }
00384 else
00385 {
00386 put_product_in_principal(std::move(product),
00387 *outE,
00388 pretend_module_name,
00389 unidentified_instance_name);
00390 TLOG_WARNING("SharedMemoryReader")
00391 << "UnknownFragmentType: The product instance name mapping for fragment type \""
00392 << ((int)type_code) << "\" is not known. Fragments of this "
00393 << "type will be stored in the event with an instance name of \""
00394 << unidentified_instance_name << "\"." << TLOG_ENDL;
00395 }
00396 }
00397 incoming_events->ReleaseBuffer();
00398 TLOG_ARB(10, "SharedMemoryReader") << "readNext: bytesRead=" << std::to_string(bytesRead) << " qsize=" << std::to_string(qsize) << " cap=" << std::to_string(incoming_events->size()) << " metricMan=" << (void*)metricMan << TLOG_ENDL;
00399 if (metricMan)
00400 {
00401 metricMan->sendMetric("bytesRead", bytesRead, "B", 5, MetricMode::Accumulate, "", true);
00402 metricMan->sendMetric("queue%Used", static_cast<unsigned long int>(qsize * 100 / incoming_events->size()), "%", 5, MetricMode::LastPoint, "", true);
00403 }
00404
00405 return true;
00406 }
00407
00408 std::map<Fragment::type_t, std::string> fragment_type_map_;
00409 unsigned readNext_calls_;
00410 };
00411 }
00412 }
00413
00414
00415 #endif