artdaq  v3_02_01
NetMonTransportService_service.cc
1 #define TRACE_NAME "NetMonTransportService"
2 
3 #include "artdaq/DAQdata/Globals.hh"
4 #include "artdaq/ArtModules/NetMonTransportService.h"
5 #include "artdaq/DAQrate/DataSenderManager.hh"
6 #include "artdaq-core/Core/SharedMemoryEventReceiver.hh"
7 
8 #include "artdaq-core/Data/Fragment.hh"
9 #include "artdaq/DAQdata/NetMonHeader.hh"
10 #include "artdaq-core/Data/RawEvent.hh"
11 #include "artdaq-core/Utilities/TimeUtils.hh"
12 
13 #include "art/Framework/Services/Registry/ActivityRegistry.h"
14 #include "canvas/Utilities/Exception.h"
15 #include "cetlib/container_algorithms.h"
16 #include "cetlib_except/exception.h"
17 #include "fhiclcpp/ParameterSet.h"
18 #include "fhiclcpp/ParameterSetRegistry.h"
19 
20 #include <TClass.h>
21 #include <TBufferFile.h>
22 
23 #include <iomanip>
24 #include <iostream>
25 #include <fstream>
26 #include <string>
27 #include <vector>
28 
29 
30 #define DUMP_SEND_MESSAGE 0
31 #define DUMP_RECEIVE_MESSAGE 0
32 
33 static fhicl::ParameterSet empty_pset;
34 
35 
37 NetMonTransportService(fhicl::ParameterSet const& pset, art::ActivityRegistry&)
39  , data_pset_(pset)
40  , init_received_(false)
41  , sender_ptr_(nullptr)
42  , incoming_events_(new artdaq::SharedMemoryEventReceiver(pset.get<int>("shared_memory_key", 0xBEE70000 + getppid()), pset.get<int>("broadcast_shared_memory_key", 0xCEE70000 + getppid())))
43  , recvd_fragments_(nullptr)
44 {
45  TLOG(TLVL_TRACE) << "NetMonTransportService CONSTRUCTOR" ;
46  if (pset.has_key("rank")) my_rank = pset.get<int>("rank");
47  else my_rank = incoming_events_->GetRank();
48 
49  init_timeout_s_ = pset.get<double>("init_fragment_timeout_seconds", 1.0);
50 }
51 
54 {
56 }
57 
58 void
61 {
62  sender_ptr_.reset(new artdaq::DataSenderManager(data_pset_));
63 }
64 
65 void
68 {
69  return;
70 }
71 
72 void
75 {
76  if (sender_ptr_) sender_ptr_.reset(nullptr);
77 }
78 
79 void
81 sendMessage(uint64_t sequenceId, uint8_t messageType, TBufferFile& msg)
82 {
83  if (sender_ptr_ == nullptr)
84  {
85  TLOG(TLVL_DEBUG) << "Reconnecting DataSenderManager" ;
86  connect();
87  }
88 
89 #if DUMP_SEND_MESSAGE
90  std::string fileName = "sendMessage_" + std::to_string(my_rank) + "_" + std::to_string(getpid()) + "_" + std::to_string(sequenceId) + ".bin";
91  std::fstream ostream(fileName, std::ios::out | std::ios::binary);
92  ostream.write(msg.Buffer(), msg.Length());
93  ostream.close();
94 #endif
95 
96  TLOG(TLVL_DEBUG) << "Sending message with sequenceID=" << sequenceId << ", type=" << (int)messageType << ", length=" << msg.Length() ;
97  artdaq::NetMonHeader header;
98  header.data_length = static_cast<uint64_t>(msg.Length());
99  artdaq::Fragment
100  fragment(std::ceil(msg.Length() /
101  static_cast<double>(sizeof(artdaq::RawDataType))),
102  sequenceId, 0, messageType, header);
103 
104  memcpy(&*fragment.dataBegin(), msg.Buffer(), msg.Length());
105  sender_ptr_->sendFragment(std::move(fragment));
106 }
107 
108 void
110 receiveMessage(TBufferFile*& msg)
111 {
112  TLOG(TLVL_TRACE) << "receiveMessage BEGIN" ;
113  while (recvd_fragments_ == nullptr)
114  {
115  TLOG(TLVL_TRACE) << "receiveMessage: Waiting for available buffer" ;
116  bool keep_looping = true;
117  bool got_event = false;
118  while (keep_looping)
119  {
120  keep_looping = false;
121  got_event = incoming_events_->ReadyForRead();
122  if (!got_event)
123  {
124  keep_looping = true;
125  }
126  }
127 
128  TLOG(TLVL_TRACE) << "receiveMessage: Reading buffer header" ;
129  auto errflag = false;
130  incoming_events_->ReadHeader(errflag);
131  if (errflag) { // Buffer was changed out from under reader!
132  msg = nullptr;
133  return;
134  }
135  TLOG(TLVL_TRACE) << "receiveMessage: Getting Fragment types" ;
136  auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag);
137  if (errflag) { // Buffer was changed out from under reader!
138  msg = nullptr;
139  return;
140  }
141  if (fragmentTypes.size() == 0)
142  {
143  TLOG(TLVL_ERROR) << "Event has no Fragments! Aborting!" ;
144  incoming_events_->ReleaseBuffer();
145  msg = nullptr;
146  return;
147  }
148  TLOG(TLVL_TRACE) << "receiveMessage: Checking first Fragment type" ;
149  auto firstFragmentType = *fragmentTypes.begin();
150 
151  // We return false, indicating we're done reading, if:
152  // 1) we did not obtain an event, because we timed out and were
153  // configured NOT to keep trying after a timeout, or
154  // 2) the event we read was the end-of-data marker: a null
155  // pointer
156  if (!got_event || firstFragmentType == artdaq::Fragment::EndOfDataFragmentType)
157  {
158  TLOG(TLVL_DEBUG) << "Received shutdown message, returning" ;
159  incoming_events_->ReleaseBuffer();
160  msg = nullptr;
161  return;
162  }
163  if (firstFragmentType == artdaq::Fragment::InitFragmentType)
164  {
165  TLOG(TLVL_DEBUG) << "Cannot receive InitFragments here, retrying" ;
166  incoming_events_->ReleaseBuffer();
167  continue;
168  }
169  // EndOfRun and EndOfSubrun Fragments are ignored in NetMonTransportService
170  else if (firstFragmentType == artdaq::Fragment::EndOfRunFragmentType || firstFragmentType == artdaq::Fragment::EndOfSubrunFragmentType)
171  {
172  TLOG(TLVL_DEBUG) << "Ignoring EndOfRun or EndOfSubrun Fragment" ;
173  incoming_events_->ReleaseBuffer();
174  continue;
175  }
176 
177  TLOG(TLVL_TRACE) << "receiveMessage: Getting all Fragments" ;
178  recvd_fragments_ = incoming_events_->GetFragmentsByType(errflag, artdaq::Fragment::InvalidFragmentType);
179  /* Events coming out of the EventStore are not sorted but need to be
180  sorted by sequence ID before they can be passed to art.
181  */
182  std::sort(recvd_fragments_->begin(), recvd_fragments_->end(),
183  artdaq::fragmentSequenceIDCompare);
184 
185  TLOG(TLVL_TRACE) << "receiveMessage: Releasing buffer" ;
186  incoming_events_->ReleaseBuffer();
187  }
188 
189  // Do not process data until Init Fragment received!
190  auto start = std::chrono::steady_clock::now();
191  while (!init_received_ && artdaq::TimeUtils::GetElapsedTime(start) < init_timeout_s_)
192  {
193  usleep(init_timeout_s_ * 1000000 / 100); // Check 100 times
194  }
195  if (!init_received_) {
196  TLOG(TLVL_ERROR) << "Received data but no Init Fragment after " << init_timeout_s_ << " seconds. Art will crash." ;
197  }
198 
199  TLOG(TLVL_TRACE) << "receiveMessage: Returning top Fragment" ;
200  artdaq::Fragment topFrag = std::move(recvd_fragments_->at(0));
201  recvd_fragments_->erase(recvd_fragments_->begin());
202  if (recvd_fragments_->size() == 0)
203  {
204  recvd_fragments_.reset(nullptr);
205  }
206 
207  TLOG(TLVL_TRACE) << "receiveMessage: Copying Fragment into TBufferFile, length=" << topFrag.metadata<artdaq::NetMonHeader>()->data_length ;
208  auto header = topFrag.metadata<artdaq::NetMonHeader>();
209  auto buffer = static_cast<char *>(malloc(header->data_length));
210  memcpy(buffer, &*topFrag.dataBegin(), header->data_length);
211  msg = new TBufferFile(TBuffer::kRead, header->data_length, buffer, kTRUE, 0);
212 
213 #if DUMP_RECEIVE_MESSAGE
214  std::string fileName = "receiveMessage_" + std::to_string(my_rank) + "_" + std::to_string(getpid()) + "_" + std::to_string(topFrag.sequenceID()) + ".bin";
215  std::fstream ostream(fileName.c_str(), std::ios::out | std::ios::binary);
216  ostream.write(buffer, header->data_length);
217  ostream.close();
218 #endif
219 
220  TLOG(TLVL_TRACE) << "receiveMessage END" ;
221 }
222 
223 void
225 receiveInitMessage(TBufferFile*& msg)
226 {
227  TLOG(TLVL_TRACE) << "receiveInitMessage BEGIN" ;
228  if (recvd_fragments_ == nullptr)
229  {
230  TLOG(TLVL_TRACE) << "receiveInitMessage: Waiting for available buffer" ;
231 
232  bool got_init = false;
233  auto errflag = false;
234  while (!got_init) {
235 
236  bool got_event = false;
237  while (!got_event)
238  {
239  got_event = incoming_events_->ReadyForRead(true);
240  }
241 
242  TLOG(TLVL_TRACE) << "receiveInitMessage: Reading buffer header" ;
243  incoming_events_->ReadHeader(errflag);
244  if (errflag) { // Buffer was changed out from under reader!
245  TLOG(TLVL_ERROR) << "receiveInitMessage: Error receiving message!" ;
246  msg = nullptr;
247  return;
248  }
249  TLOG(TLVL_TRACE) << "receiveInitMessage: Getting Fragment types" ;
250  auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag);
251  if (errflag) { // Buffer was changed out from under reader!
252  msg = nullptr;
253  TLOG(TLVL_ERROR) << "receiveInitMessage: Error receiving message!" ;
254  return;
255  }
256  if (fragmentTypes.size() == 0)
257  {
258  TLOG(TLVL_ERROR) << "Event has no Fragments! Aborting!" ;
259  incoming_events_->ReleaseBuffer();
260  msg = nullptr;
261  return;
262  }
263  TLOG(TLVL_TRACE) << "receiveInitMessage: Checking first Fragment type" ;
264  auto firstFragmentType = *fragmentTypes.begin();
265 
266  // We return false, indicating we're done reading, if:
267  // 1) we did not obtain an event, because we timed out and were
268  // configured NOT to keep trying after a timeout, or
269  // 2) the event we read was the end-of-data marker: a null
270  // pointer
271  if (!got_event || firstFragmentType == artdaq::Fragment::EndOfDataFragmentType)
272  {
273  TLOG(TLVL_DEBUG) << "Received shutdown message, returning" ;
274  incoming_events_->ReleaseBuffer();
275  msg = nullptr;
276  return;
277  }
278  if (firstFragmentType != artdaq::Fragment::InitFragmentType)
279  {
280  TLOG(TLVL_WARNING) << "Did NOT receive Init Fragment as first broadcast! Type=" << artdaq::detail::RawFragmentHeader::SystemTypeToString(firstFragmentType) ;
281  incoming_events_->ReleaseBuffer();
282  }
283  got_init = true;
284  }
285  TLOG(TLVL_TRACE) << "receiveInitMessage: Getting all Fragments" ;
286  recvd_fragments_ = incoming_events_->GetFragmentsByType(errflag, artdaq::Fragment::InvalidFragmentType);
287  /* Events coming out of the EventStore are not sorted but need to be
288  sorted by sequence ID before they can be passed to art.
289  */
290  std::sort(recvd_fragments_->begin(), recvd_fragments_->end(),
291  artdaq::fragmentSequenceIDCompare);
292  }
293 
294  TLOG(TLVL_TRACE) << "receiveInitMessage: Returning top Fragment" ;
295  artdaq::Fragment topFrag = std::move(recvd_fragments_->at(0));
296  recvd_fragments_->erase(recvd_fragments_->begin());
297  if (recvd_fragments_->size() == 0)
298  {
299  recvd_fragments_.reset(nullptr);
300  }
301 
302  auto header = topFrag.metadata<artdaq::NetMonHeader>();
303  TLOG(TLVL_TRACE) << "receiveInitMessage: Copying Fragment into TBufferFile: message length: " << header->data_length ;
304  auto buffer = static_cast<char *>(malloc(header->data_length));
305  memcpy(buffer, &*topFrag.dataBegin(), header->data_length);
306 
307 #if DUMP_RECEIVE_MESSAGE
308  std::string fileName = "receiveInitMessage_" + std::to_string(getpid()) + ".bin";
309  std::fstream ostream(fileName.c_str(), std::ios::out | std::ios::binary);
310  ostream.write(buffer, header->data_length);
311  ostream.close();
312 #endif
313 
314  msg = new TBufferFile(TBuffer::kRead, header->data_length, buffer, kTRUE, 0);
315 
316  TLOG(TLVL_TRACE) << "receiveInitMessage END" ;
317  init_received_ = true;
318 }
319 DEFINE_ART_SERVICE_INTERFACE_IMPL(NetMonTransportService, NetMonTransportServiceInterface)
void receiveInitMessage(TBufferFile *&msg) override
Receive the init message.
Sends Fragment objects using TransferInterface plugins. Uses Routing Tables if confgiured, otherwise will Round-Robin Fragments to the destinations.
void sendMessage(uint64_t sequenceId, uint8_t messageType, TBufferFile &msg) override
Send ROOT data, wrapped in an artdaq::Fragment object.
NetMonTransportService extends NetMonTransportServiceInterface. It sends events using DataSenderManag...
void receiveMessage(TBufferFile *&msg) override
Receive data from the ConcurrentQueue.
Header with length information for NetMonTransport messages.
Definition: NetMonHeader.hh:14
void connect() override
Reconnect the NetMonTransportService.
virtual ~NetMonTransportService()
NetMonTransportService Destructor. Calls disconnect().
NetMonTransportService(fhicl::ParameterSet const &pset, art::ActivityRegistry &)
NetMonTransportService Constructor.
void disconnect() override
Disconnects the NetMonTranportService.
uint64_t data_length
The length of the message.
Definition: NetMonHeader.hh:16
Interface for NetMonTranportService. This interface is declared to art as part of the required regist...
void listen() override
Listen for connections. This method is a No-Op.