artdaq  3.12.05
ArtdaqSharedMemoryService_service.cc
1 #include "TRACE/tracemf.h"
2 #define TRACE_NAME "ArtdaqSharedMemoryService"
3 
4 #include "artdaq/ArtModules/ArtdaqSharedMemoryServiceInterface.h"
5 #include "artdaq/DAQdata/Globals.hh"
6 
7 #include "artdaq-core/Core/SharedMemoryEventReceiver.hh"
8 #include "artdaq-core/Utilities/ExceptionHandler.hh"
9 
10 #include "art/Framework/Services/Registry/ServiceDefinitionMacros.h"
11 #include "art/Framework/Services/Registry/ServiceHandle.h"
12 #include "fhiclcpp/types/Atom.h"
13 #include "fhiclcpp/types/Comment.h"
14 #include "fhiclcpp/types/ConfigurationTable.h"
15 #include "fhiclcpp/types/Name.h"
16 
17 #include <cstdint>
18 #include <memory>
19 
20 // ----------------------------------------------------------------------
21 
28 {
29 public:
33  struct Config
34  {
36  fhicl::Atom<uint32_t> shared_memory_key{fhicl::Name{"shared_memory_key"}, fhicl::Comment{"Key to use when connecting to shared memory. Will default to 0xBEE70000 + getppid()."}, 0xBEE70000};
38  fhicl::Atom<uint32_t> broadcast_shared_memory_key{fhicl::Name{"broadcast_shared_memory_key"}, fhicl::Comment{"Key to use when connecting to broadcast shared memory. Will default to 0xCEE70000 + getppid()."}, 0xCEE70000};
40  fhicl::Atom<int> rank{fhicl::Name{"rank"}, fhicl::Comment{"Rank of this artdaq application. Used for data transfers"}};
41  };
43  using Parameters = fhicl::WrappedTable<Config>;
44 
49 
54  ArtdaqSharedMemoryService(fhicl::ParameterSet const& pset, art::ActivityRegistry&);
55 
61  std::unordered_map<artdaq::Fragment::type_t, std::unique_ptr<artdaq::Fragments>> ReceiveEvent(bool broadcast) override;
62 
67  size_t GetQueueSize() override { return incoming_events_->ReadReadyCount(); }
72  size_t GetQueueCapacity() override { return incoming_events_->size(); }
77  std::shared_ptr<artdaq::detail::RawEventHeader> GetEventHeader() override { return evtHeader_; }
78 
79 private:
82  ArtdaqSharedMemoryService& operator=(ArtdaqSharedMemoryService const&) = delete;
84 
85 private:
86  std::unique_ptr<artdaq::SharedMemoryEventReceiver> incoming_events_;
87  std::shared_ptr<artdaq::detail::RawEventHeader> evtHeader_;
88  size_t read_timeout_;
89  bool resume_after_timeout_;
90  bool printed_exit_message_{false};
91 };
92 
93 DECLARE_ART_SERVICE_INTERFACE_IMPL(ArtdaqSharedMemoryService, ArtdaqSharedMemoryServiceInterface, LEGACY)
94 
95 #define build_key(seed) ((seed) + ((GetPartitionNumber() + 1) << 16) + (getppid() & 0xFFFF))
96 
97 static fhicl::ParameterSet empty_pset;
98 
99 ArtdaqSharedMemoryService::ArtdaqSharedMemoryService(fhicl::ParameterSet const& pset, art::ActivityRegistry& /*unused*/)
100  : incoming_events_(nullptr)
101  , evtHeader_(nullptr)
102  , read_timeout_(pset.get<size_t>("read_timeout_us", static_cast<size_t>(pset.get<double>("waiting_time", 600.0) * 1000000)))
103  , resume_after_timeout_(pset.get<bool>("resume_after_timeout", true))
104 {
105  TLOG(TLVL_DEBUG + 33) << "ArtdaqSharedMemoryService CONSTRUCTOR";
106 
107  incoming_events_ = std::make_unique<artdaq::SharedMemoryEventReceiver>(
108  pset.get<int>("shared_memory_key", build_key(0xEE000000)),
109  pset.get<int>("broadcast_shared_memory_key", build_key(0xBB000000)));
110 
111  char const* artapp_env = getenv("ARTDAQ_APPLICATION_NAME");
112  std::string artapp_str;
113  if (artapp_env != nullptr)
114  {
115  artapp_str = std::string(artapp_env) + "_";
116  }
117 
118  TLOG(TLVL_DEBUG + 33) << "Setting app_name";
119  app_name = artapp_str + "art" + std::to_string(incoming_events_->GetMyId());
120  // artdaq::configureMessageFacility(app_name.c_str()); // ELF 11/20/2020: MessageFacility already configured by initialization pset
121 
122  artapp_env = getenv("ARTDAQ_RANK");
123  if (artapp_env != nullptr && my_rank < 0)
124  {
125  TLOG(TLVL_DEBUG + 33) << "Setting rank from envrionment";
126  my_rank = strtol(artapp_env, nullptr, 10);
127  }
128  else
129  {
130  TLOG(TLVL_DEBUG + 33) << "Setting my_rank from shared memory";
131  my_rank = incoming_events_->GetRank();
132  }
133 
134  try
135  {
136  if (metricMan)
137  {
138  metricMan->initialize(pset.get<fhicl::ParameterSet>("metrics", fhicl::ParameterSet()), app_name);
139  metricMan->do_start();
140  }
141  }
142  catch (...)
143  {
144  artdaq::ExceptionHandler(artdaq::ExceptionHandlerRethrow::no, "Error loading metrics in ArtdaqSharedMemoryService()");
145  }
146 
147  TLOG(TLVL_INFO) << "app_name is " << app_name << ", rank " << my_rank;
148 }
149 
151 {
153 }
154 
155 std::unordered_map<artdaq::Fragment::type_t, std::unique_ptr<artdaq::Fragments>> ArtdaqSharedMemoryService::ReceiveEvent(bool broadcast)
156 {
157  TLOG(TLVL_DEBUG + 33) << "ReceiveEvent BEGIN";
158  std::unordered_map<artdaq::Fragment::type_t, std::unique_ptr<artdaq::Fragments>> recvd_fragments;
159 
160  if (printed_exit_message_)
161  {
162  return recvd_fragments;
163  }
164 
165  while (recvd_fragments.empty())
166  {
167  TLOG(TLVL_DEBUG + 33) << "ReceiveEvent: Waiting for available buffer";
168  bool got_event = false;
169  auto start_time = std::chrono::steady_clock::now();
170  auto read_timeout_to_use = read_timeout_ > 100000 ? 100000 : read_timeout_;
171  if (!resume_after_timeout_ || broadcast) read_timeout_to_use = read_timeout_;
172  while (!incoming_events_->IsEndOfData() && !got_event)
173  {
174  got_event = incoming_events_->ReadyForRead(broadcast, read_timeout_to_use);
175  if (!got_event && (!resume_after_timeout_ || broadcast)) // Only try broadcasts once!
176  {
177  TLOG(TLVL_ERROR) << "Timeout occurred! No data received after " << read_timeout_to_use << " us. Returning empty Fragment list!";
178  return recvd_fragments;
179  }
180  if (!got_event && artdaq::TimeUtils::GetElapsedTimeMicroseconds(start_time) > read_timeout_)
181  {
182  TLOG(TLVL_WARNING) << "Timeout occurred! No data received after " << artdaq::TimeUtils::GetElapsedTimeMicroseconds(start_time) << " us. Retrying.";
183  }
184  }
185  if (incoming_events_->IsEndOfData())
186  {
187  if (!printed_exit_message_)
188  {
189  TLOG(TLVL_INFO) << "End of Data signal received, exiting";
190  printed_exit_message_ = true;
191  }
192  return recvd_fragments;
193  }
194 
195  TLOG(TLVL_DEBUG + 33) << "ReceiveEvent: Reading buffer header";
196  auto errflag = false;
197  auto hdrPtr = incoming_events_->ReadHeader(errflag);
198  if (errflag || hdrPtr == nullptr)
199  { // Buffer was changed out from under reader!
200  incoming_events_->ReleaseBuffer();
201  continue; // retry
202  // return recvd_fragments;
203  }
204  evtHeader_ = std::make_shared<artdaq::detail::RawEventHeader>(*hdrPtr);
205  TLOG(TLVL_DEBUG + 33) << "ReceiveEvent: Getting Fragment types";
206  auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag);
207  if (errflag)
208  { // Buffer was changed out from under reader!
209  incoming_events_->ReleaseBuffer();
210  continue; // retry
211  // return recvd_fragments;
212  }
213  if (fragmentTypes.empty())
214  {
215  TLOG(TLVL_ERROR) << "Event has no Fragments! Aborting!";
216  incoming_events_->ReleaseBuffer();
217  return recvd_fragments;
218  }
219 
220  for (auto const& type : fragmentTypes)
221  {
222  TLOG(TLVL_DEBUG + 33) << "ReceiveEvent: Getting all Fragments of type " << static_cast<int>(type);
223  recvd_fragments[type] = incoming_events_->GetFragmentsByType(errflag, type);
224  if (!recvd_fragments[type])
225  {
226  TLOG(TLVL_WARNING) << "Error retrieving Fragments from shared memory! (Most likely due to a buffer overwrite) Retrying...";
227  incoming_events_->ReleaseBuffer();
228  recvd_fragments.clear();
229  continue;
230  }
231  /* Events coming out of the EventStore are not sorted but need to be
232  sorted by sequence ID before they can be passed to art.
233  */
234  std::sort(recvd_fragments[type]->begin(), recvd_fragments[type]->end(), artdaq::fragmentSequenceIDCompare);
235  }
236  TLOG(TLVL_DEBUG + 33) << "ReceiveEvent: Releasing buffer";
237  incoming_events_->ReleaseBuffer();
238  }
239 
240  TLOG(TLVL_DEBUG + 33) << "ReceiveEvent END";
241  return recvd_fragments;
242 }
243 
244 DEFINE_ART_SERVICE_INTERFACE_IMPL(ArtdaqSharedMemoryService, ArtdaqSharedMemoryServiceInterface)
fhicl::WrappedTable< Config > Parameters
Used for ParameterSet validation (if desired)
Interface for ArtdaqSharedMemoryService. This interface is declared to art as part of the required re...
static void CleanUpGlobals()
Clean up statically-allocated Manager class instances.
Definition: Globals.hh:156
size_t GetQueueSize() override
Get the number of events which are ready to be read.
virtual ~ArtdaqSharedMemoryService()
NetMonTransportService Destructor. Calls disconnect().
fhicl::Atom< int > rank
&quot;rank&quot; (OPTIONAL) : The rank of this applicaiton, for use by non - artdaq applications running NetMon...
ArtdaqSharedMemoryService(fhicl::ParameterSet const &pset, art::ActivityRegistry &)
NetMonTransportService Constructor.
fhicl::Atom< uint32_t > broadcast_shared_memory_key
&quot;shared_memory_key&quot; (Default: 0xCEE70000 + pid): Key to use when connecting to broadcast shared memor...
size_t GetQueueCapacity() override
Get the maximum number of events which can be stored in the shared memory.
Allowed Configuration parameters of NetMonTransportService. May be used for configuration validation ...
ArtdaqSharedMemoryService extends ArtdaqSharedMemoryServiceInterface. It receives events from shared ...
fhicl::Atom< uint32_t > shared_memory_key
&quot;shared_memory_key&quot; (Default: 0xBEE70000 + pid): Key to use when connecting to shared memory...
std::shared_ptr< artdaq::detail::RawEventHeader > GetEventHeader() override
Get a shared_ptr to the current event header, if any.
std::unordered_map< artdaq::Fragment::type_t, std::unique_ptr< artdaq::Fragments > > ReceiveEvent(bool broadcast) override
Receive an event from the shared memory.