artdaq  v3_03_00
NetMonTransportService_service.cc
1 #define TRACE_NAME "NetMonTransportService"
2 
3 #include "artdaq/DAQdata/Globals.hh"
4 #include "artdaq/ArtModules/NetMonTransportService.h"
5 #include "artdaq/DAQrate/DataSenderManager.hh"
6 #include "artdaq-core/Core/SharedMemoryEventReceiver.hh"
7 
8 #include "artdaq-core/Data/Fragment.hh"
9 #include "artdaq/DAQdata/NetMonHeader.hh"
10 #include "artdaq-core/Data/RawEvent.hh"
11 #include "artdaq-core/Utilities/TimeUtils.hh"
12 
13 #include "art/Framework/Services/Registry/ActivityRegistry.h"
14 #include "canvas/Utilities/Exception.h"
15 #include "cetlib/container_algorithms.h"
16 #include "cetlib_except/exception.h"
17 #include "fhiclcpp/ParameterSet.h"
18 #include "fhiclcpp/ParameterSetRegistry.h"
19 
20 #include <TClass.h>
21 #include <TBufferFile.h>
22 
23 #include <iomanip>
24 #include <iostream>
25 #include <fstream>
26 #include <string>
27 #include <vector>
28 
29 
30 #define DUMP_SEND_MESSAGE 0
31 #define DUMP_RECEIVE_MESSAGE 0
32 
33 static fhicl::ParameterSet empty_pset;
34 
35 
37 NetMonTransportService(fhicl::ParameterSet const& pset, art::ActivityRegistry&)
39  , data_pset_(pset)
40  , init_received_(false)
41  , sender_ptr_(nullptr)
42  , incoming_events_(nullptr)
43  , recvd_fragments_(nullptr)
44 {
45  TLOG(TLVL_TRACE) << "NetMonTransportService CONSTRUCTOR" ;
46  if (pset.has_key("rank")) my_rank = pset.get<int>("rank");
47 
48  init_timeout_s_ = pset.get<double>("init_fragment_timeout_seconds", 1.0);
49 }
50 
53 {
55 }
56 
57 void
60 {
61  sender_ptr_.reset(new artdaq::DataSenderManager(data_pset_));
62 }
63 
64 void
67 {
68  if (!incoming_events_)
69  {
70  incoming_events_.reset(new artdaq::SharedMemoryEventReceiver(data_pset_.get<int>("shared_memory_key", 0xBEE70000 + getppid()), data_pset_.get<int>("broadcast_shared_memory_key", 0xCEE70000 + getppid())));
71  if (data_pset_.has_key("rank")) my_rank = data_pset_.get<int>("rank");
72  else my_rank = incoming_events_->GetRank();
73  }
74  return;
75 }
76 
77 void
80 {
81  if (sender_ptr_) sender_ptr_.reset(nullptr);
82 }
83 
84 void
86 sendMessage(uint64_t sequenceId, uint8_t messageType, TBufferFile& msg)
87 {
88  if (sender_ptr_ == nullptr)
89  {
90  TLOG(TLVL_DEBUG) << "Reconnecting DataSenderManager" ;
91  connect();
92  }
93 
94 #if DUMP_SEND_MESSAGE
95  std::string fileName = "sendMessage_" + std::to_string(my_rank) + "_" + std::to_string(getpid()) + "_" + std::to_string(sequenceId) + ".bin";
96  std::fstream ostream(fileName, std::ios::out | std::ios::binary);
97  ostream.write(msg.Buffer(), msg.Length());
98  ostream.close();
99 #endif
100 
101  TLOG(TLVL_DEBUG) << "Sending message with sequenceID=" << sequenceId << ", type=" << (int)messageType << ", length=" << msg.Length() ;
102  artdaq::NetMonHeader header;
103  header.data_length = static_cast<uint64_t>(msg.Length());
104  artdaq::Fragment
105  fragment(std::ceil(msg.Length() /
106  static_cast<double>(sizeof(artdaq::RawDataType))),
107  sequenceId, 0, messageType, header);
108 
109  memcpy(&*fragment.dataBegin(), msg.Buffer(), msg.Length());
110  sender_ptr_->sendFragment(std::move(fragment));
111 }
112 
113 void
115 receiveMessage(TBufferFile*& msg)
116 {
117  listen();
118  TLOG(TLVL_TRACE) << "receiveMessage BEGIN" ;
119  while (recvd_fragments_ == nullptr)
120  {
121  TLOG(TLVL_TRACE) << "receiveMessage: Waiting for available buffer" ;
122  bool keep_looping = true;
123  bool got_event = false;
124  while (keep_looping)
125  {
126  keep_looping = false;
127  got_event = incoming_events_->ReadyForRead();
128  if (!got_event)
129  {
130  keep_looping = true;
131  }
132  }
133 
134  TLOG(TLVL_TRACE) << "receiveMessage: Reading buffer header" ;
135  auto errflag = false;
136  incoming_events_->ReadHeader(errflag);
137  if (errflag) { // Buffer was changed out from under reader!
138  msg = nullptr;
139  return;
140  }
141  TLOG(TLVL_TRACE) << "receiveMessage: Getting Fragment types" ;
142  auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag);
143  if (errflag) { // Buffer was changed out from under reader!
144  incoming_events_->ReleaseBuffer();
145  msg = nullptr;
146  return;
147  }
148  if (fragmentTypes.size() == 0)
149  {
150  TLOG(TLVL_ERROR) << "Event has no Fragments! Aborting!" ;
151  incoming_events_->ReleaseBuffer();
152  msg = nullptr;
153  return;
154  }
155  TLOG(TLVL_TRACE) << "receiveMessage: Checking first Fragment type" ;
156  auto firstFragmentType = *fragmentTypes.begin();
157 
158  // We return false, indicating we're done reading, if:
159  // 1) we did not obtain an event, because we timed out and were
160  // configured NOT to keep trying after a timeout, or
161  // 2) the event we read was the end-of-data marker: a null
162  // pointer
163  if (!got_event || firstFragmentType == artdaq::Fragment::EndOfDataFragmentType)
164  {
165  TLOG(TLVL_DEBUG) << "Received shutdown message, returning from receiveMessage "
166  << "(debug: got_event=" << got_event << ",fragType=" << (int)firstFragmentType
167  << ",EODFragType=" << (int)artdaq::Fragment::EndOfDataFragmentType << ")";
168  incoming_events_->ReleaseBuffer();
169  msg = nullptr;
170  return;
171  }
172  if (firstFragmentType == artdaq::Fragment::InitFragmentType)
173  {
174  TLOG(TLVL_DEBUG) << "Cannot receive InitFragments here, retrying" ;
175  incoming_events_->ReleaseBuffer();
176  continue;
177  }
178  // EndOfRun and EndOfSubrun Fragments are ignored in NetMonTransportService
179  else if (firstFragmentType == artdaq::Fragment::EndOfRunFragmentType || firstFragmentType == artdaq::Fragment::EndOfSubrunFragmentType)
180  {
181  TLOG(TLVL_DEBUG) << "Ignoring EndOfRun or EndOfSubrun Fragment" ;
182  incoming_events_->ReleaseBuffer();
183  continue;
184  }
185 
186  TLOG(TLVL_TRACE) << "receiveMessage: Getting all Fragments" ;
187  recvd_fragments_ = incoming_events_->GetFragmentsByType(errflag, artdaq::Fragment::InvalidFragmentType);
188  if (!recvd_fragments_)
189  {
190  TLOG(TLVL_ERROR) << "Error retrieving Fragments from shared memory! Aborting!";
191  incoming_events_->ReleaseBuffer();
192  msg = nullptr;
193  return;
194 
195  }
196  /* Events coming out of the EventStore are not sorted but need to be
197  sorted by sequence ID before they can be passed to art.
198  */
199  std::sort(recvd_fragments_->begin(), recvd_fragments_->end(),
200  artdaq::fragmentSequenceIDCompare);
201 
202  TLOG(TLVL_TRACE) << "receiveMessage: Releasing buffer" ;
203  incoming_events_->ReleaseBuffer();
204  }
205 
206  // Do not process data until Init Fragment received!
207  auto start = std::chrono::steady_clock::now();
208  while (!init_received_ && artdaq::TimeUtils::GetElapsedTime(start) < init_timeout_s_)
209  {
210  usleep(init_timeout_s_ * 1000000 / 100); // Check 100 times
211  }
212  if (!init_received_) {
213  TLOG(TLVL_ERROR) << "Received data but no Init Fragment after " << init_timeout_s_ << " seconds. Art will crash." ;
214  }
215 
216  TLOG(TLVL_TRACE) << "receiveMessage: Returning top Fragment" ;
217  artdaq::Fragment topFrag = std::move(recvd_fragments_->at(0));
218  recvd_fragments_->erase(recvd_fragments_->begin());
219  if (recvd_fragments_->size() == 0)
220  {
221  recvd_fragments_.reset(nullptr);
222  }
223 
224  TLOG(TLVL_TRACE) << "receiveMessage: Copying Fragment into TBufferFile, length=" << topFrag.metadata<artdaq::NetMonHeader>()->data_length ;
225  auto header = topFrag.metadata<artdaq::NetMonHeader>();
226  auto buffer = static_cast<char *>(malloc(header->data_length));
227  memcpy(buffer, &*topFrag.dataBegin(), header->data_length);
228  msg = new TBufferFile(TBuffer::kRead, header->data_length, buffer, kTRUE, 0);
229 
230 #if DUMP_RECEIVE_MESSAGE
231  std::string fileName = "receiveMessage_" + std::to_string(my_rank) + "_" + std::to_string(getpid()) + "_" + std::to_string(topFrag.sequenceID()) + ".bin";
232  std::fstream ostream(fileName.c_str(), std::ios::out | std::ios::binary);
233  ostream.write(buffer, header->data_length);
234  ostream.close();
235 #endif
236 
237  TLOG(TLVL_TRACE) << "receiveMessage END" ;
238 }
239 
240 void
242 receiveInitMessage(TBufferFile*& msg)
243 {
244  listen();
245  TLOG(TLVL_TRACE) << "receiveInitMessage BEGIN" ;
246  if (recvd_fragments_ == nullptr)
247  {
248  TLOG(TLVL_TRACE) << "receiveInitMessage: Waiting for available buffer" ;
249 
250  bool got_init = false;
251  auto errflag = false;
252  while (!got_init) {
253 
254  bool got_event = false;
255  while (!got_event)
256  {
257  got_event = incoming_events_->ReadyForRead(true);
258  }
259 
260  TLOG(TLVL_TRACE) << "receiveInitMessage: Reading buffer header" ;
261  incoming_events_->ReadHeader(errflag);
262  if (errflag) { // Buffer was changed out from under reader!
263  TLOG(TLVL_ERROR) << "receiveInitMessage: Error receiving message!" ;
264  incoming_events_->ReleaseBuffer();
265  msg = nullptr;
266  return;
267  }
268  TLOG(TLVL_TRACE) << "receiveInitMessage: Getting Fragment types" ;
269  auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag);
270  if (errflag) { // Buffer was changed out from under reader!
271  incoming_events_->ReleaseBuffer();
272  msg = nullptr;
273  TLOG(TLVL_ERROR) << "receiveInitMessage: Error receiving message!" ;
274  return;
275  }
276  if (fragmentTypes.size() == 0)
277  {
278  TLOG(TLVL_ERROR) << "Event has no Fragments! Aborting!" ;
279  incoming_events_->ReleaseBuffer();
280  msg = nullptr;
281  return;
282  }
283  TLOG(TLVL_TRACE) << "receiveInitMessage: Checking first Fragment type" ;
284  auto firstFragmentType = *fragmentTypes.begin();
285 
286  // We return false, indicating we're done reading, if:
287  // 1) we did not obtain an event, because we timed out and were
288  // configured NOT to keep trying after a timeout, or
289  // 2) the event we read was the end-of-data marker: a null
290  // pointer
291  if (!got_event || firstFragmentType == artdaq::Fragment::EndOfDataFragmentType)
292  {
293  TLOG(TLVL_DEBUG) << "Received shutdown message, returning" ;
294  incoming_events_->ReleaseBuffer();
295  msg = nullptr;
296  return;
297  }
298  if (firstFragmentType != artdaq::Fragment::InitFragmentType)
299  {
300  TLOG(TLVL_WARNING) << "Did NOT receive Init Fragment as first broadcast! Type=" << artdaq::detail::RawFragmentHeader::SystemTypeToString(firstFragmentType) ;
301  incoming_events_->ReleaseBuffer();
302  }
303  got_init = true;
304  }
305  TLOG(TLVL_TRACE) << "receiveInitMessage: Getting all Fragments" ;
306  recvd_fragments_ = incoming_events_->GetFragmentsByType(errflag, artdaq::Fragment::InvalidFragmentType);
307  /* Events coming out of the EventStore are not sorted but need to be
308  sorted by sequence ID before they can be passed to art.
309  */
310  std::sort(recvd_fragments_->begin(), recvd_fragments_->end(),
311  artdaq::fragmentSequenceIDCompare);
312 
313  incoming_events_->ReleaseBuffer();
314  }
315 
316  TLOG(TLVL_TRACE) << "receiveInitMessage: Returning top Fragment" ;
317  artdaq::Fragment topFrag = std::move(recvd_fragments_->at(0));
318  recvd_fragments_->erase(recvd_fragments_->begin());
319  if (recvd_fragments_->size() == 0)
320  {
321  recvd_fragments_.reset(nullptr);
322  }
323 
324  auto header = topFrag.metadata<artdaq::NetMonHeader>();
325  TLOG(TLVL_TRACE) << "receiveInitMessage: Copying Fragment into TBufferFile: message length: " << header->data_length ;
326  auto buffer = new char[header->data_length];
327  //auto buffer = static_cast<char *>(malloc(header->data_length)); // Fix alloc-dealloc-mismatch
328  memcpy(buffer, &*topFrag.dataBegin(), header->data_length);
329 
330 #if DUMP_RECEIVE_MESSAGE
331  std::string fileName = "receiveInitMessage_" + std::to_string(getpid()) + ".bin";
332  std::fstream ostream(fileName.c_str(), std::ios::out | std::ios::binary);
333  ostream.write(buffer, header->data_length);
334  ostream.close();
335 #endif
336 
337  msg = new TBufferFile(TBuffer::kRead, header->data_length, buffer, kTRUE, 0);
338 
339  TLOG(TLVL_TRACE) << "receiveInitMessage END" ;
340  init_received_ = true;
341 }
342 DEFINE_ART_SERVICE_INTERFACE_IMPL(NetMonTransportService, NetMonTransportServiceInterface)
void receiveInitMessage(TBufferFile *&msg) override
Receive the init message.
Sends Fragment objects using TransferInterface plugins. Uses Routing Tables if confgiured, otherwise will Round-Robin Fragments to the destinations.
void sendMessage(uint64_t sequenceId, uint8_t messageType, TBufferFile &msg) override
Send ROOT data, wrapped in an artdaq::Fragment object.
NetMonTransportService extends NetMonTransportServiceInterface. It sends events using DataSenderManag...
void receiveMessage(TBufferFile *&msg) override
Receive data from the ConcurrentQueue.
Header with length information for NetMonTransport messages.
Definition: NetMonHeader.hh:14
void connect() override
Reconnect the NetMonTransportService.
virtual ~NetMonTransportService()
NetMonTransportService Destructor. Calls disconnect().
NetMonTransportService(fhicl::ParameterSet const &pset, art::ActivityRegistry &)
NetMonTransportService Constructor.
void disconnect() override
Disconnects the NetMonTranportService.
uint64_t data_length
The length of the message.
Definition: NetMonHeader.hh:16
Interface for NetMonTranportService. This interface is declared to art as part of the required regist...
void listen() override
Listen for connections. This method is a No-Op.