artdaq  3.12.04
ArtdaqSharedMemoryService_service.cc
1 #include "TRACE/tracemf.h"
2 #define TRACE_NAME "ArtdaqSharedMemoryService"
3 
4 #include "artdaq/ArtModules/ArtdaqSharedMemoryServiceInterface.h"
5 #include "artdaq/DAQdata/Globals.hh"
6 
7 #include "artdaq-core/Core/SharedMemoryEventReceiver.hh"
8 #include "artdaq-core/Utilities/ExceptionHandler.hh"
9 
10 #include "art/Framework/Services/Registry/ServiceDefinitionMacros.h"
11 #include "art/Framework/Services/Registry/ServiceHandle.h"
12 #include "fhiclcpp/types/Atom.h"
13 #include "fhiclcpp/types/Comment.h"
14 #include "fhiclcpp/types/ConfigurationTable.h"
15 #include "fhiclcpp/types/Name.h"
16 
17 #include <cstdint>
18 #include <memory>
19 
20 // ----------------------------------------------------------------------
21 
28 {
29 public:
33  struct Config
34  {
36  fhicl::Atom<uint32_t> shared_memory_key{fhicl::Name{"shared_memory_key"}, fhicl::Comment{"Key to use when connecting to shared memory. Will default to 0xBEE70000 + getppid()."}, 0xBEE70000};
38  fhicl::Atom<uint32_t> broadcast_shared_memory_key{fhicl::Name{"broadcast_shared_memory_key"}, fhicl::Comment{"Key to use when connecting to broadcast shared memory. Will default to 0xCEE70000 + getppid()."}, 0xCEE70000};
40  fhicl::Atom<int> rank{fhicl::Name{"rank"}, fhicl::Comment{"Rank of this artdaq application. Used for data transfers"}};
41  };
43  using Parameters = fhicl::WrappedTable<Config>;
44 
49 
54  ArtdaqSharedMemoryService(fhicl::ParameterSet const& pset, art::ActivityRegistry&);
55 
61  std::unordered_map<artdaq::Fragment::type_t, std::unique_ptr<artdaq::Fragments>> ReceiveEvent(bool broadcast) override;
62 
67  size_t GetQueueSize() override { return incoming_events_->ReadReadyCount(); }
72  size_t GetQueueCapacity() override { return incoming_events_->size(); }
77  std::shared_ptr<artdaq::detail::RawEventHeader> GetEventHeader() override { return evtHeader_; }
78 
79 private:
82  ArtdaqSharedMemoryService& operator=(ArtdaqSharedMemoryService const&) = delete;
84 
85 private:
86  std::unique_ptr<artdaq::SharedMemoryEventReceiver> incoming_events_;
87  std::shared_ptr<artdaq::detail::RawEventHeader> evtHeader_;
88  size_t read_timeout_;
89  bool resume_after_timeout_;
90  bool printed_exit_message_{false};
91 };
92 
93 DECLARE_ART_SERVICE_INTERFACE_IMPL(ArtdaqSharedMemoryService, ArtdaqSharedMemoryServiceInterface, LEGACY)
94 
95 #define build_key(seed) ((seed) + ((GetPartitionNumber() + 1) << 16) + (getppid() & 0xFFFF))
96 
97 static fhicl::ParameterSet empty_pset;
98 
99 ArtdaqSharedMemoryService::ArtdaqSharedMemoryService(fhicl::ParameterSet const& pset, art::ActivityRegistry& /*unused*/)
100  : incoming_events_(nullptr)
101  , evtHeader_(nullptr)
102  , read_timeout_(pset.get<size_t>("read_timeout_us", static_cast<size_t>(pset.get<double>("waiting_time", 600.0) * 1000000)))
103  , resume_after_timeout_(pset.get<bool>("resume_after_timeout", true))
104 {
105  TLOG(TLVL_DEBUG + 33) << "ArtdaqSharedMemoryService CONSTRUCTOR";
106 
107  incoming_events_ = std::make_unique<artdaq::SharedMemoryEventReceiver>(
108  pset.get<int>("shared_memory_key", build_key(0xEE000000)),
109  pset.get<int>("broadcast_shared_memory_key", build_key(0xBB000000)));
110 
111  char const* artapp_env = getenv("ARTDAQ_APPLICATION_NAME");
112  std::string artapp_str;
113  if (artapp_env != nullptr)
114  {
115  artapp_str = std::string(artapp_env) + "_";
116  }
117 
118  TLOG(TLVL_DEBUG + 33) << "Setting app_name";
119  app_name = artapp_str + "art" + std::to_string(incoming_events_->GetMyId());
120  // artdaq::configureMessageFacility(app_name.c_str()); // ELF 11/20/2020: MessageFacility already configured by initialization pset
121 
122  artapp_env = getenv("ARTDAQ_RANK");
123  if (artapp_env != nullptr && my_rank < 0)
124  {
125  TLOG(TLVL_DEBUG + 33) << "Setting rank from envrionment";
126  my_rank = strtol(artapp_env, nullptr, 10);
127  }
128  else
129  {
130  TLOG(TLVL_DEBUG + 33) << "Setting my_rank from shared memory";
131  my_rank = incoming_events_->GetRank();
132  }
133 
134  try
135  {
136  if (metricMan)
137  {
138  metricMan->initialize(pset.get<fhicl::ParameterSet>("metrics", fhicl::ParameterSet()), app_name);
139  metricMan->do_start();
140  }
141  }
142  catch (...)
143  {
144  artdaq::ExceptionHandler(artdaq::ExceptionHandlerRethrow::no, "Error loading metrics in ArtdaqSharedMemoryService()");
145  }
146 
147  TLOG(TLVL_INFO) << "app_name is " << app_name << ", rank " << my_rank;
148 }
149 
151 {
153 }
154 
155 std::unordered_map<artdaq::Fragment::type_t, std::unique_ptr<artdaq::Fragments>> ArtdaqSharedMemoryService::ReceiveEvent(bool broadcast)
156 {
157  TLOG(TLVL_DEBUG + 33) << "ReceiveEvent BEGIN";
158  std::unordered_map<artdaq::Fragment::type_t, std::unique_ptr<artdaq::Fragments>> recvd_fragments;
159 
160  if(printed_exit_message_) {
161  return recvd_fragments;
162  }
163 
164  while (recvd_fragments.empty())
165  {
166  TLOG(TLVL_DEBUG + 33) << "ReceiveEvent: Waiting for available buffer";
167  bool got_event = false;
168  auto start_time = std::chrono::steady_clock::now();
169  auto read_timeout_to_use = read_timeout_ > 100000 ? 100000 : read_timeout_;
170  if (!resume_after_timeout_ || broadcast) read_timeout_to_use = read_timeout_;
171  while (!incoming_events_->IsEndOfData() && !got_event)
172  {
173  got_event = incoming_events_->ReadyForRead(broadcast, read_timeout_to_use);
174  if (!got_event && (!resume_after_timeout_ || broadcast)) // Only try broadcasts once!
175  {
176  TLOG(TLVL_ERROR) << "Timeout occurred! No data received after " << read_timeout_to_use << " us. Returning empty Fragment list!";
177  return recvd_fragments;
178  }
179  if (!got_event && artdaq::TimeUtils::GetElapsedTimeMicroseconds(start_time) > read_timeout_)
180  {
181  TLOG(TLVL_WARNING) << "Timeout occurred! No data received after " << artdaq::TimeUtils::GetElapsedTimeMicroseconds(start_time) << " us. Retrying.";
182  }
183  }
184  if (incoming_events_->IsEndOfData())
185  {
186  if (!printed_exit_message_)
187  {
188  TLOG(TLVL_INFO) << "End of Data signal received, exiting";
189  printed_exit_message_ = true;
190  }
191  return recvd_fragments;
192  }
193 
194  TLOG(TLVL_DEBUG + 33) << "ReceiveEvent: Reading buffer header";
195  auto errflag = false;
196  auto hdrPtr = incoming_events_->ReadHeader(errflag);
197  if (errflag || hdrPtr == nullptr)
198  { // Buffer was changed out from under reader!
199  incoming_events_->ReleaseBuffer();
200  continue; // retry
201  // return recvd_fragments;
202  }
203  evtHeader_ = std::make_shared<artdaq::detail::RawEventHeader>(*hdrPtr);
204  TLOG(TLVL_DEBUG + 33) << "ReceiveEvent: Getting Fragment types";
205  auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag);
206  if (errflag)
207  { // Buffer was changed out from under reader!
208  incoming_events_->ReleaseBuffer();
209  continue; // retry
210  // return recvd_fragments;
211  }
212  if (fragmentTypes.empty())
213  {
214  TLOG(TLVL_ERROR) << "Event has no Fragments! Aborting!";
215  incoming_events_->ReleaseBuffer();
216  return recvd_fragments;
217  }
218 
219  for (auto const& type : fragmentTypes)
220  {
221  TLOG(TLVL_DEBUG + 33) << "ReceiveEvent: Getting all Fragments of type " << static_cast<int>(type);
222  recvd_fragments[type] = incoming_events_->GetFragmentsByType(errflag, type);
223  if (!recvd_fragments[type])
224  {
225  TLOG(TLVL_WARNING) << "Error retrieving Fragments from shared memory! (Most likely due to a buffer overwrite) Retrying...";
226  incoming_events_->ReleaseBuffer();
227  recvd_fragments.clear();
228  continue;
229  }
230  /* Events coming out of the EventStore are not sorted but need to be
231  sorted by sequence ID before they can be passed to art.
232  */
233  std::sort(recvd_fragments[type]->begin(), recvd_fragments[type]->end(), artdaq::fragmentSequenceIDCompare);
234  }
235  TLOG(TLVL_DEBUG + 33) << "ReceiveEvent: Releasing buffer";
236  incoming_events_->ReleaseBuffer();
237  }
238 
239  TLOG(TLVL_DEBUG + 33) << "ReceiveEvent END";
240  return recvd_fragments;
241 }
242 
243 DEFINE_ART_SERVICE_INTERFACE_IMPL(ArtdaqSharedMemoryService, ArtdaqSharedMemoryServiceInterface)
fhicl::WrappedTable< Config > Parameters
Used for ParameterSet validation (if desired)
Interface for ArtdaqSharedMemoryService. This interface is declared to art as part of the required re...
static void CleanUpGlobals()
Clean up statically-allocated Manager class instances.
Definition: Globals.hh:156
size_t GetQueueSize() override
Get the number of events which are ready to be read.
virtual ~ArtdaqSharedMemoryService()
NetMonTransportService Destructor. Calls disconnect().
fhicl::Atom< int > rank
&quot;rank&quot; (OPTIONAL) : The rank of this applicaiton, for use by non - artdaq applications running NetMon...
ArtdaqSharedMemoryService(fhicl::ParameterSet const &pset, art::ActivityRegistry &)
NetMonTransportService Constructor.
fhicl::Atom< uint32_t > broadcast_shared_memory_key
&quot;shared_memory_key&quot; (Default: 0xCEE70000 + pid): Key to use when connecting to broadcast shared memor...
size_t GetQueueCapacity() override
Get the maximum number of events which can be stored in the shared memory.
Allowed Configuration parameters of NetMonTransportService. May be used for configuration validation ...
ArtdaqSharedMemoryService extends ArtdaqSharedMemoryServiceInterface. It receives events from shared ...
fhicl::Atom< uint32_t > shared_memory_key
&quot;shared_memory_key&quot; (Default: 0xBEE70000 + pid): Key to use when connecting to shared memory...
std::shared_ptr< artdaq::detail::RawEventHeader > GetEventHeader() override
Get a shared_ptr to the current event header, if any.
std::unordered_map< artdaq::Fragment::type_t, std::unique_ptr< artdaq::Fragments > > ReceiveEvent(bool broadcast) override
Receive an event from the shared memory.