artdaq  3.12.07
ArtdaqSharedMemoryService_service.cc
1 #include "TRACE/tracemf.h"
2 #define TRACE_NAME "ArtdaqSharedMemoryService"
3 
4 #include "artdaq/ArtModules/ArtdaqSharedMemoryServiceInterface.h"
5 #include "artdaq/DAQdata/Globals.hh"
6 
7 #include "artdaq-core/Core/SharedMemoryEventReceiver.hh"
8 #include "artdaq-core/Utilities/ExceptionHandler.hh"
9 
10 #include "art/Framework/Services/Registry/ServiceDefinitionMacros.h"
11 #include "art/Framework/Services/Registry/ServiceHandle.h"
12 #include "fhiclcpp/types/Atom.h"
13 #include "fhiclcpp/types/Comment.h"
14 #include "fhiclcpp/types/ConfigurationTable.h"
15 #include "fhiclcpp/types/Name.h"
16 
17 #include <cstdint>
18 #include <memory>
19 
20 // ----------------------------------------------------------------------
21 
28 {
29 public:
33  struct Config
34  {
36  fhicl::Atom<uint32_t> shared_memory_key{fhicl::Name{"shared_memory_key"}, fhicl::Comment{"Key to use when connecting to shared memory. Will default to 0xBEE70000 + getppid()."}, 0xBEE70000};
38  fhicl::Atom<uint32_t> broadcast_shared_memory_key{fhicl::Name{"broadcast_shared_memory_key"}, fhicl::Comment{"Key to use when connecting to broadcast shared memory. Will default to 0xCEE70000 + getppid()."}, 0xCEE70000};
40  fhicl::Atom<int> rank{fhicl::Name{"rank"}, fhicl::Comment{"Rank of this artdaq application. Used for data transfers"}};
41  };
43  using Parameters = fhicl::WrappedTable<Config>;
44 
49 
54  ArtdaqSharedMemoryService(fhicl::ParameterSet const& pset, art::ActivityRegistry&);
55 
61  std::unordered_map<artdaq::Fragment::type_t, std::unique_ptr<artdaq::Fragments>> ReceiveEvent(bool broadcast) override;
62 
67  size_t GetQueueSize() override { return incoming_events_->ReadReadyCount(); }
72  size_t GetQueueCapacity() override { return incoming_events_->size(); }
77  std::shared_ptr<artdaq::detail::RawEventHeader> GetEventHeader() override { return evtHeader_; }
78 
83  size_t GetMyId() override { return incoming_events_->GetMyId(); }
84 
85 private:
88  ArtdaqSharedMemoryService& operator=(ArtdaqSharedMemoryService const&) = delete;
90 
91 private:
92  std::unique_ptr<artdaq::SharedMemoryEventReceiver> incoming_events_;
93  std::shared_ptr<artdaq::detail::RawEventHeader> evtHeader_;
94  size_t read_timeout_;
95  bool resume_after_timeout_;
96  bool printed_exit_message_{false};
97 };
98 
99 DECLARE_ART_SERVICE_INTERFACE_IMPL(ArtdaqSharedMemoryService, ArtdaqSharedMemoryServiceInterface, LEGACY)
100 
101 #define build_key(seed) ((seed) + ((GetPartitionNumber() + 1) << 16) + (getppid() & 0xFFFF))
102 
103 static fhicl::ParameterSet empty_pset;
104 
105 ArtdaqSharedMemoryService::ArtdaqSharedMemoryService(fhicl::ParameterSet const& pset, art::ActivityRegistry& /*unused*/)
106  : incoming_events_(nullptr)
107  , evtHeader_(nullptr)
108  , read_timeout_(pset.get<size_t>("read_timeout_us", static_cast<size_t>(pset.get<double>("waiting_time", 600.0) * 1000000)))
109  , resume_after_timeout_(pset.get<bool>("resume_after_timeout", true))
110 {
111  TLOG(TLVL_DEBUG + 33) << "ArtdaqSharedMemoryService CONSTRUCTOR";
112 
113  incoming_events_ = std::make_unique<artdaq::SharedMemoryEventReceiver>(
114  pset.get<int>("shared_memory_key", build_key(0xEE000000)),
115  pset.get<int>("broadcast_shared_memory_key", build_key(0xBB000000)));
116 
117  char const* artapp_env = getenv("ARTDAQ_APPLICATION_NAME");
118  std::string artapp_str;
119  if (artapp_env != nullptr)
120  {
121  artapp_str = std::string(artapp_env) + "_";
122  }
123 
124  TLOG(TLVL_DEBUG + 33) << "Setting app_name";
125  app_name = artapp_str + "art" + std::to_string(incoming_events_->GetMyId());
126  // artdaq::configureMessageFacility(app_name.c_str()); // ELF 11/20/2020: MessageFacility already configured by initialization pset
127 
128  artapp_env = getenv("ARTDAQ_RANK");
129  if (artapp_env != nullptr && my_rank < 0)
130  {
131  TLOG(TLVL_DEBUG + 33) << "Setting rank from envrionment";
132  my_rank = strtol(artapp_env, nullptr, 10);
133  }
134  else
135  {
136  TLOG(TLVL_DEBUG + 33) << "Setting my_rank from shared memory";
137  my_rank = incoming_events_->GetRank();
138  }
139 
140  try
141  {
142  if (metricMan)
143  {
144  metricMan->initialize(pset.get<fhicl::ParameterSet>("metrics", fhicl::ParameterSet()), app_name);
145  metricMan->do_start();
146  }
147  }
148  catch (...)
149  {
150  artdaq::ExceptionHandler(artdaq::ExceptionHandlerRethrow::no, "Error loading metrics in ArtdaqSharedMemoryService()");
151  }
152 
153  TLOG(TLVL_INFO) << "app_name is " << app_name << ", rank " << my_rank;
154 }
155 
157 {
159 }
160 
161 std::unordered_map<artdaq::Fragment::type_t, std::unique_ptr<artdaq::Fragments>> ArtdaqSharedMemoryService::ReceiveEvent(bool broadcast)
162 {
163  TLOG(TLVL_DEBUG + 33) << "ReceiveEvent BEGIN";
164  std::unordered_map<artdaq::Fragment::type_t, std::unique_ptr<artdaq::Fragments>> recvd_fragments;
165 
166  if (printed_exit_message_)
167  {
168  return recvd_fragments;
169  }
170 
171  while (recvd_fragments.empty())
172  {
173  TLOG(TLVL_DEBUG + 33) << "ReceiveEvent: Waiting for available buffer";
174  bool got_event = false;
175  auto start_time = std::chrono::steady_clock::now();
176  auto read_timeout_to_use = read_timeout_ > 100000 ? 100000 : read_timeout_;
177  if (!resume_after_timeout_ || broadcast) read_timeout_to_use = read_timeout_;
178  while (!incoming_events_->IsEndOfData() && !got_event)
179  {
180  got_event = incoming_events_->ReadyForRead(broadcast, read_timeout_to_use);
181  if (!got_event && (!resume_after_timeout_ || broadcast)) // Only try broadcasts once!
182  {
183  TLOG(TLVL_ERROR) << "Timeout occurred! No data received after " << read_timeout_to_use << " us. Returning empty Fragment list!";
184  return recvd_fragments;
185  }
186  if (!got_event && artdaq::TimeUtils::GetElapsedTimeMicroseconds(start_time) > read_timeout_)
187  {
188  TLOG(TLVL_WARNING) << "Timeout occurred! No data received after " << artdaq::TimeUtils::GetElapsedTimeMicroseconds(start_time) << " us. Retrying.";
189  }
190  }
191  if (incoming_events_->IsEndOfData())
192  {
193  if (!printed_exit_message_)
194  {
195  TLOG(TLVL_INFO) << "End of Data signal received, exiting";
196  printed_exit_message_ = true;
197  }
198  return recvd_fragments;
199  }
200 
201  TLOG(TLVL_DEBUG + 33) << "ReceiveEvent: Reading buffer header";
202  auto errflag = false;
203  auto hdrPtr = incoming_events_->ReadHeader(errflag);
204  if (errflag || hdrPtr == nullptr)
205  { // Buffer was changed out from under reader!
206  incoming_events_->ReleaseBuffer();
207  continue; // retry
208  // return recvd_fragments;
209  }
210  evtHeader_ = std::make_shared<artdaq::detail::RawEventHeader>(*hdrPtr);
211  TLOG(TLVL_DEBUG + 33) << "ReceiveEvent: Getting Fragment types";
212  auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag);
213  if (errflag)
214  { // Buffer was changed out from under reader!
215  incoming_events_->ReleaseBuffer();
216  continue; // retry
217  // return recvd_fragments;
218  }
219  if (fragmentTypes.empty())
220  {
221  TLOG(TLVL_ERROR) << "Event has no Fragments! Aborting!";
222  incoming_events_->ReleaseBuffer();
223  return recvd_fragments;
224  }
225 
226  for (auto const& type : fragmentTypes)
227  {
228  TLOG(TLVL_DEBUG + 33) << "ReceiveEvent: Getting all Fragments of type " << static_cast<int>(type);
229  recvd_fragments[type] = incoming_events_->GetFragmentsByType(errflag, type);
230  if (!recvd_fragments[type])
231  {
232  TLOG(TLVL_WARNING) << "Error retrieving Fragments from shared memory! (Most likely due to a buffer overwrite) Retrying...";
233  incoming_events_->ReleaseBuffer();
234  recvd_fragments.clear();
235  continue;
236  }
237  /* Events coming out of the EventStore are not sorted but need to be
238  sorted by sequence ID before they can be passed to art.
239  */
240  std::sort(recvd_fragments[type]->begin(), recvd_fragments[type]->end(), artdaq::fragmentSequenceIDCompare);
241  }
242  TLOG(TLVL_DEBUG + 33) << "ReceiveEvent: Releasing buffer";
243  incoming_events_->ReleaseBuffer();
244  }
245 
246  TLOG(TLVL_DEBUG + 33) << "ReceiveEvent END";
247  return recvd_fragments;
248 }
249 
250 DEFINE_ART_SERVICE_INTERFACE_IMPL(ArtdaqSharedMemoryService, ArtdaqSharedMemoryServiceInterface)
fhicl::WrappedTable< Config > Parameters
Used for ParameterSet validation (if desired)
Interface for ArtdaqSharedMemoryService. This interface is declared to art as part of the required re...
static void CleanUpGlobals()
Clean up statically-allocated Manager class instances.
Definition: Globals.hh:156
size_t GetQueueSize() override
Get the number of events which are ready to be read.
virtual ~ArtdaqSharedMemoryService()
NetMonTransportService Destructor. Calls disconnect().
fhicl::Atom< int > rank
&quot;rank&quot; (OPTIONAL) : The rank of this applicaiton, for use by non - artdaq applications running NetMon...
ArtdaqSharedMemoryService(fhicl::ParameterSet const &pset, art::ActivityRegistry &)
NetMonTransportService Constructor.
fhicl::Atom< uint32_t > broadcast_shared_memory_key
&quot;shared_memory_key&quot; (Default: 0xCEE70000 + pid): Key to use when connecting to broadcast shared memor...
size_t GetQueueCapacity() override
Get the maximum number of events which can be stored in the shared memory.
Allowed Configuration parameters of NetMonTransportService. May be used for configuration validation ...
size_t GetMyId() override
Get the ID of this art process.
ArtdaqSharedMemoryService extends ArtdaqSharedMemoryServiceInterface. It receives events from shared ...
fhicl::Atom< uint32_t > shared_memory_key
&quot;shared_memory_key&quot; (Default: 0xBEE70000 + pid): Key to use when connecting to shared memory...
std::shared_ptr< artdaq::detail::RawEventHeader > GetEventHeader() override
Get a shared_ptr to the current event header, if any.
std::unordered_map< artdaq::Fragment::type_t, std::unique_ptr< artdaq::Fragments > > ReceiveEvent(bool broadcast) override
Receive an event from the shared memory.