artdaq  v3_04_00
NetMonTransportService_service.cc
1 #define TRACE_NAME "NetMonTransportService"
2 
3 #include "artdaq/DAQdata/Globals.hh"
4 #include "artdaq/ArtModules/NetMonTransportService.h"
5 #include "artdaq/DAQrate/DataSenderManager.hh"
6 #include "artdaq-core/Core/SharedMemoryEventReceiver.hh"
7 
8 #include "artdaq-core/Data/Fragment.hh"
9 #include "artdaq/DAQdata/NetMonHeader.hh"
10 #include "artdaq-core/Data/RawEvent.hh"
11 #include "artdaq-core/Utilities/TimeUtils.hh"
12 
13 #include "art/Framework/Services/Registry/ActivityRegistry.h"
14 #include "canvas/Utilities/Exception.h"
15 #include "cetlib/container_algorithms.h"
16 #include "cetlib_except/exception.h"
17 #include "fhiclcpp/ParameterSet.h"
18 #include "fhiclcpp/ParameterSetRegistry.h"
19 
20 #include <TClass.h>
21 #include <TBufferFile.h>
22 
23 #include <iomanip>
24 #include <iostream>
25 #include <fstream>
26 #include <string>
27 #include <vector>
28 
29 
30 #define DUMP_SEND_MESSAGE 0
31 #define DUMP_RECEIVE_MESSAGE 0
32 
33 static fhicl::ParameterSet empty_pset;
34 
35 
37 NetMonTransportService(fhicl::ParameterSet const& pset, art::ActivityRegistry&)
39  , data_pset_(pset)
40  , init_received_(false)
41  , sender_ptr_(nullptr)
42  , incoming_events_(nullptr)
43  , recvd_fragments_(nullptr)
44 {
45  TLOG(TLVL_TRACE) << "NetMonTransportService CONSTRUCTOR" ;
46  if (pset.has_key("rank")) my_rank = pset.get<int>("rank");
47 
48  init_timeout_s_ = pset.get<double>("init_fragment_timeout_seconds", 1.0);
49 }
50 
53 {
55 }
56 
57 void
60 {
61  sender_ptr_.reset(new artdaq::DataSenderManager(data_pset_));
62 }
63 
64 void
67 {
68  if (!incoming_events_)
69  {
70  incoming_events_.reset(new artdaq::SharedMemoryEventReceiver(data_pset_.get<int>("shared_memory_key", 0xBEE70000 + getppid()), data_pset_.get<int>("broadcast_shared_memory_key", 0xCEE70000 + getppid())));
71 
72  char const* artapp_env = getenv("ARTDAQ_APPLICATION_NAME");
73  std::string artapp_str = "";
74  if (artapp_env != NULL)
75  {
76  artapp_str = std::string(artapp_env) + "_";
77  }
78 
79  app_name = artapp_str + "art" + std::to_string(incoming_events_->GetMyId());
80 
81  if (data_pset_.has_key("rank")) my_rank = data_pset_.get<int>("rank");
82  else my_rank = incoming_events_->GetRank();
83  }
84  return;
85 }
86 
87 void
90 {
91  if (sender_ptr_) sender_ptr_.reset(nullptr);
92 }
93 
94 void
96 sendMessage(uint64_t sequenceId, uint8_t messageType, TBufferFile& msg)
97 {
98  if (sender_ptr_ == nullptr)
99  {
100  TLOG(TLVL_DEBUG) << "Reconnecting DataSenderManager" ;
101  connect();
102  }
103 
104 #if DUMP_SEND_MESSAGE
105  std::string fileName = "sendMessage_" + std::to_string(my_rank) + "_" + std::to_string(getpid()) + "_" + std::to_string(sequenceId) + ".bin";
106  std::fstream ostream(fileName, std::ios::out | std::ios::binary);
107  ostream.write(msg.Buffer(), msg.Length());
108  ostream.close();
109 #endif
110 
111  TLOG(TLVL_DEBUG) << "Sending message with sequenceID=" << sequenceId << ", type=" << (int)messageType << ", length=" << msg.Length() ;
112  artdaq::NetMonHeader header;
113  header.data_length = static_cast<uint64_t>(msg.Length());
114  artdaq::Fragment
115  fragment(std::ceil(msg.Length() /
116  static_cast<double>(sizeof(artdaq::RawDataType))),
117  sequenceId, 0, messageType, header);
118 
119  memcpy(&*fragment.dataBegin(), msg.Buffer(), msg.Length());
120  sender_ptr_->sendFragment(std::move(fragment));
121 }
122 
123 void
125 receiveMessage(TBufferFile*& msg)
126 {
127  listen();
128  TLOG(TLVL_TRACE) << "receiveMessage BEGIN" ;
129  while (recvd_fragments_ == nullptr)
130  {
131  TLOG(TLVL_TRACE) << "receiveMessage: Waiting for available buffer" ;
132  bool keep_looping = true;
133  bool got_event = false;
134  while (keep_looping)
135  {
136  keep_looping = false;
137  got_event = incoming_events_->ReadyForRead();
138  if (!got_event)
139  {
140  keep_looping = true;
141  }
142  }
143 
144  TLOG(TLVL_TRACE) << "receiveMessage: Reading buffer header" ;
145  auto errflag = false;
146  incoming_events_->ReadHeader(errflag);
147  if (errflag) { // Buffer was changed out from under reader!
148  msg = nullptr;
149  return;
150  }
151  TLOG(TLVL_TRACE) << "receiveMessage: Getting Fragment types" ;
152  auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag);
153  if (errflag) { // Buffer was changed out from under reader!
154  incoming_events_->ReleaseBuffer();
155  msg = nullptr;
156  return;
157  }
158  if (fragmentTypes.size() == 0)
159  {
160  TLOG(TLVL_ERROR) << "Event has no Fragments! Aborting!" ;
161  incoming_events_->ReleaseBuffer();
162  msg = nullptr;
163  return;
164  }
165  TLOG(TLVL_TRACE) << "receiveMessage: Checking first Fragment type" ;
166  auto firstFragmentType = *fragmentTypes.begin();
167 
168  // We return false, indicating we're done reading, if:
169  // 1) we did not obtain an event, because we timed out and were
170  // configured NOT to keep trying after a timeout, or
171  // 2) the event we read was the end-of-data marker: a null
172  // pointer
173  if (!got_event || firstFragmentType == artdaq::Fragment::EndOfDataFragmentType)
174  {
175  TLOG(TLVL_DEBUG) << "Received shutdown message, returning from receiveMessage "
176  << "(debug: got_event=" << got_event << ",fragType=" << (int)firstFragmentType
177  << ",EODFragType=" << (int)artdaq::Fragment::EndOfDataFragmentType << ")";
178  incoming_events_->ReleaseBuffer();
179  msg = nullptr;
180  return;
181  }
182  if (firstFragmentType == artdaq::Fragment::InitFragmentType)
183  {
184  TLOG(TLVL_DEBUG) << "Cannot receive InitFragments here, retrying" ;
185  incoming_events_->ReleaseBuffer();
186  continue;
187  }
188  // EndOfRun and EndOfSubrun Fragments are ignored in NetMonTransportService
189  else if (firstFragmentType == artdaq::Fragment::EndOfRunFragmentType || firstFragmentType == artdaq::Fragment::EndOfSubrunFragmentType)
190  {
191  TLOG(TLVL_DEBUG) << "Ignoring EndOfRun or EndOfSubrun Fragment" ;
192  incoming_events_->ReleaseBuffer();
193  continue;
194  }
195 
196  TLOG(TLVL_TRACE) << "receiveMessage: Getting all Fragments" ;
197  recvd_fragments_ = incoming_events_->GetFragmentsByType(errflag, artdaq::Fragment::InvalidFragmentType);
198  if (!recvd_fragments_)
199  {
200  TLOG(TLVL_ERROR) << "Error retrieving Fragments from shared memory! Aborting!";
201  incoming_events_->ReleaseBuffer();
202  msg = nullptr;
203  return;
204 
205  }
206  /* Events coming out of the EventStore are not sorted but need to be
207  sorted by sequence ID before they can be passed to art.
208  */
209  std::sort(recvd_fragments_->begin(), recvd_fragments_->end(),
210  artdaq::fragmentSequenceIDCompare);
211 
212  TLOG(TLVL_TRACE) << "receiveMessage: Releasing buffer" ;
213  incoming_events_->ReleaseBuffer();
214  }
215 
216  // Do not process data until Init Fragment received!
217  auto start = std::chrono::steady_clock::now();
218  while (!init_received_ && artdaq::TimeUtils::GetElapsedTime(start) < init_timeout_s_)
219  {
220  usleep(init_timeout_s_ * 1000000 / 100); // Check 100 times
221  }
222  if (!init_received_) {
223  TLOG(TLVL_ERROR) << "Received data but no Init Fragment after " << init_timeout_s_ << " seconds. Art will crash." ;
224  }
225 
226  TLOG(TLVL_TRACE) << "receiveMessage: Returning top Fragment" ;
227  artdaq::Fragment topFrag = std::move(recvd_fragments_->at(0));
228  recvd_fragments_->erase(recvd_fragments_->begin());
229  if (recvd_fragments_->size() == 0)
230  {
231  recvd_fragments_.reset(nullptr);
232  }
233 
234  TLOG(TLVL_TRACE) << "receiveMessage: Copying Fragment into TBufferFile, length=" << topFrag.metadata<artdaq::NetMonHeader>()->data_length ;
235  auto header = topFrag.metadata<artdaq::NetMonHeader>();
236  auto buffer = static_cast<char *>(malloc(header->data_length));
237  memcpy(buffer, &*topFrag.dataBegin(), header->data_length);
238  msg = new TBufferFile(TBuffer::kRead, header->data_length, buffer, kTRUE, 0);
239 
240 #if DUMP_RECEIVE_MESSAGE
241  std::string fileName = "receiveMessage_" + std::to_string(my_rank) + "_" + std::to_string(getpid()) + "_" + std::to_string(topFrag.sequenceID()) + ".bin";
242  std::fstream ostream(fileName.c_str(), std::ios::out | std::ios::binary);
243  ostream.write(buffer, header->data_length);
244  ostream.close();
245 #endif
246 
247  TLOG(TLVL_TRACE) << "receiveMessage END" ;
248 }
249 
250 void
252 receiveInitMessage(TBufferFile*& msg)
253 {
254  listen();
255  TLOG(TLVL_TRACE) << "receiveInitMessage BEGIN" ;
256  if (recvd_fragments_ == nullptr)
257  {
258  TLOG(TLVL_TRACE) << "receiveInitMessage: Waiting for available buffer" ;
259 
260  bool got_init = false;
261  auto errflag = false;
262  while (!got_init) {
263 
264  bool got_event = false;
265  while (!got_event)
266  {
267  got_event = incoming_events_->ReadyForRead(true);
268  }
269 
270  TLOG(TLVL_TRACE) << "receiveInitMessage: Reading buffer header" ;
271  incoming_events_->ReadHeader(errflag);
272  if (errflag) { // Buffer was changed out from under reader!
273  TLOG(TLVL_ERROR) << "receiveInitMessage: Error receiving message!" ;
274  incoming_events_->ReleaseBuffer();
275  msg = nullptr;
276  return;
277  }
278  TLOG(TLVL_TRACE) << "receiveInitMessage: Getting Fragment types" ;
279  auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag);
280  if (errflag) { // Buffer was changed out from under reader!
281  incoming_events_->ReleaseBuffer();
282  msg = nullptr;
283  TLOG(TLVL_ERROR) << "receiveInitMessage: Error receiving message!" ;
284  return;
285  }
286  if (fragmentTypes.size() == 0)
287  {
288  TLOG(TLVL_ERROR) << "Event has no Fragments! Aborting!" ;
289  incoming_events_->ReleaseBuffer();
290  msg = nullptr;
291  return;
292  }
293  TLOG(TLVL_TRACE) << "receiveInitMessage: Checking first Fragment type" ;
294  auto firstFragmentType = *fragmentTypes.begin();
295 
296  // We return false, indicating we're done reading, if:
297  // 1) we did not obtain an event, because we timed out and were
298  // configured NOT to keep trying after a timeout, or
299  // 2) the event we read was the end-of-data marker: a null
300  // pointer
301  if (!got_event || firstFragmentType == artdaq::Fragment::EndOfDataFragmentType)
302  {
303  TLOG(TLVL_DEBUG) << "Received shutdown message, returning" ;
304  incoming_events_->ReleaseBuffer();
305  msg = nullptr;
306  return;
307  }
308  if (firstFragmentType != artdaq::Fragment::InitFragmentType)
309  {
310  TLOG(TLVL_WARNING) << "Did NOT receive Init Fragment as first broadcast! Type=" << artdaq::detail::RawFragmentHeader::SystemTypeToString(firstFragmentType) ;
311  incoming_events_->ReleaseBuffer();
312  }
313  got_init = true;
314  }
315  TLOG(TLVL_TRACE) << "receiveInitMessage: Getting all Fragments" ;
316  recvd_fragments_ = incoming_events_->GetFragmentsByType(errflag, artdaq::Fragment::InvalidFragmentType);
317  /* Events coming out of the EventStore are not sorted but need to be
318  sorted by sequence ID before they can be passed to art.
319  */
320  std::sort(recvd_fragments_->begin(), recvd_fragments_->end(),
321  artdaq::fragmentSequenceIDCompare);
322 
323  incoming_events_->ReleaseBuffer();
324  }
325 
326  TLOG(TLVL_TRACE) << "receiveInitMessage: Returning top Fragment" ;
327  artdaq::Fragment topFrag = std::move(recvd_fragments_->at(0));
328  recvd_fragments_->erase(recvd_fragments_->begin());
329  if (recvd_fragments_->size() == 0)
330  {
331  recvd_fragments_.reset(nullptr);
332  }
333 
334  auto header = topFrag.metadata<artdaq::NetMonHeader>();
335  TLOG(TLVL_TRACE) << "receiveInitMessage: Copying Fragment into TBufferFile: message length: " << header->data_length ;
336  auto buffer = new char[header->data_length];
337  //auto buffer = static_cast<char *>(malloc(header->data_length)); // Fix alloc-dealloc-mismatch
338  memcpy(buffer, &*topFrag.dataBegin(), header->data_length);
339 
340 #if DUMP_RECEIVE_MESSAGE
341  std::string fileName = "receiveInitMessage_" + std::to_string(getpid()) + ".bin";
342  std::fstream ostream(fileName.c_str(), std::ios::out | std::ios::binary);
343  ostream.write(buffer, header->data_length);
344  ostream.close();
345 #endif
346 
347  msg = new TBufferFile(TBuffer::kRead, header->data_length, buffer, kTRUE, 0);
348 
349  TLOG(TLVL_TRACE) << "receiveInitMessage END" ;
350  init_received_ = true;
351 }
352 DEFINE_ART_SERVICE_INTERFACE_IMPL(NetMonTransportService, NetMonTransportServiceInterface)
void receiveInitMessage(TBufferFile *&msg) override
Receive the init message.
Sends Fragment objects using TransferInterface plugins. Uses Routing Tables if confgiured, otherwise will Round-Robin Fragments to the destinations.
void sendMessage(uint64_t sequenceId, uint8_t messageType, TBufferFile &msg) override
Send ROOT data, wrapped in an artdaq::Fragment object.
NetMonTransportService extends NetMonTransportServiceInterface. It sends events using DataSenderManag...
void receiveMessage(TBufferFile *&msg) override
Receive data from the ConcurrentQueue.
Header with length information for NetMonTransport messages.
Definition: NetMonHeader.hh:14
void connect() override
Reconnect the NetMonTransportService.
virtual ~NetMonTransportService()
NetMonTransportService Destructor. Calls disconnect().
NetMonTransportService(fhicl::ParameterSet const &pset, art::ActivityRegistry &)
NetMonTransportService Constructor.
void disconnect() override
Disconnects the NetMonTranportService.
uint64_t data_length
The length of the message.
Definition: NetMonHeader.hh:16
Interface for NetMonTranportService. This interface is declared to art as part of the required regist...
void listen() override
Listen for connections. This method is a No-Op.