artdaq  v3_00_03
NetMonTransportService_service.cc
1 #include "artdaq/DAQdata/Globals.hh"
2 #include "artdaq/ArtModules/NetMonTransportService.h"
3 #include "artdaq/DAQrate/DataSenderManager.hh"
4 #include "artdaq-core/Core/SharedMemoryEventReceiver.hh"
5 
6 #include "artdaq-core/Data/Fragment.hh"
7 #include "artdaq/DAQdata/NetMonHeader.hh"
8 #include "artdaq-core/Data/RawEvent.hh"
9 #include "artdaq-core/Utilities/TimeUtils.hh"
10 
11 #include "art/Framework/Services/Registry/ActivityRegistry.h"
12 #include "canvas/Utilities/Exception.h"
13 #include "cetlib/container_algorithms.h"
14 #include "cetlib_except/exception.h"
15 #include "fhiclcpp/ParameterSet.h"
16 #include "fhiclcpp/ParameterSetRegistry.h"
17 
18 #include <TClass.h>
19 #include <TBufferFile.h>
20 
21 #include <iomanip>
22 #include <iostream>
23 #include <fstream>
24 #include <string>
25 #include <vector>
26 
27 
28 #define DUMP_SEND_MESSAGE 0
29 #define DUMP_RECEIVE_MESSAGE 0
30 
31 static fhicl::ParameterSet empty_pset;
32 
33 
35 NetMonTransportService(fhicl::ParameterSet const& pset, art::ActivityRegistry&)
37  , data_pset_(pset)
38  , init_received_(false)
39  , sender_ptr_(nullptr)
40  , incoming_events_(new artdaq::SharedMemoryEventReceiver(pset.get<int>("shared_memory_key", 0xBEE70000 + getppid()), pset.get<int>("broadcast_shared_memory_key", 0xCEE70000 + getppid())))
41  , recvd_fragments_(nullptr)
42 {
43  TLOG_TRACE("NetMonTransportService") << "NetMonTransportService CONSTRUCTOR" << TLOG_ENDL;
44  if (pset.has_key("rank")) my_rank = pset.get<int>("rank");
45  else my_rank = incoming_events_->GetRank();
46 
47  init_timeout_s_ = pset.get<double>("init_fragment_timeout_seconds", 1.0);
48 }
49 
52 {
54 }
55 
56 void
59 {
60  sender_ptr_.reset(new artdaq::DataSenderManager(data_pset_));
61 }
62 
63 void
66 {
67  return;
68 }
69 
70 void
73 {
74  if (sender_ptr_) sender_ptr_.reset(nullptr);
75 }
76 
77 void
79 sendMessage(uint64_t sequenceId, uint8_t messageType, TBufferFile& msg)
80 {
81  if (sender_ptr_ == nullptr)
82  {
83  TLOG_DEBUG("NetMonTransportService") << "Reconnecting DataSenderManager" << TLOG_ENDL;
84  connect();
85  }
86 
87 #if DUMP_SEND_MESSAGE
88  std::string fileName = "sendMessage_" + std::to_string(my_rank) + "_" + std::to_string(getpid()) + "_" + std::to_string(sequenceId) + ".bin";
89  std::fstream ostream(fileName, std::ios::out | std::ios::binary);
90  ostream.write(msg.Buffer(), msg.Length());
91  ostream.close();
92 #endif
93 
94  TLOG_DEBUG("NetMonTransportService") << "Sending message with sequenceID=" << std::to_string(sequenceId) << ", type=" << std::to_string(messageType) << ", length=" << std::to_string(msg.Length()) << TLOG_ENDL;
95  artdaq::NetMonHeader header;
96  header.data_length = static_cast<uint64_t>(msg.Length());
97  artdaq::Fragment
98  fragment(std::ceil(msg.Length() /
99  static_cast<double>(sizeof(artdaq::RawDataType))),
100  sequenceId, 0, messageType, header);
101 
102  memcpy(&*fragment.dataBegin(), msg.Buffer(), msg.Length());
103  sender_ptr_->sendFragment(std::move(fragment));
104 }
105 
106 void
108 receiveMessage(TBufferFile*& msg)
109 {
110  TLOG_TRACE("NetMonTransportService") << "receiveMessage BEGIN" << TLOG_ENDL;
111  while (recvd_fragments_ == nullptr)
112  {
113  TLOG_TRACE("NetMonTransportService") << "receiveMessage: Waiting for available buffer" << TLOG_ENDL;
114  bool keep_looping = true;
115  bool got_event = false;
116  while (keep_looping)
117  {
118  keep_looping = false;
119  got_event = incoming_events_->ReadyForRead();
120  if (!got_event)
121  {
122  keep_looping = true;
123  }
124  }
125 
126  TLOG_TRACE("NetMonTransportService") << "receiveMessage: Reading buffer header" << TLOG_ENDL;
127  auto errflag = false;
128  incoming_events_->ReadHeader(errflag);
129  if (errflag) { // Buffer was changed out from under reader!
130  msg = nullptr;
131  return;
132  }
133  TLOG_TRACE("NetMonTransportService") << "receiveMessage: Getting Fragment types" << TLOG_ENDL;
134  auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag);
135  if (errflag) { // Buffer was changed out from under reader!
136  msg = nullptr;
137  return;
138  }
139  if (fragmentTypes.size() == 0)
140  {
141  TLOG_ERROR("NetMonTransportService") << "Event has no Fragments! Aborting!" << TLOG_ENDL;
142  incoming_events_->ReleaseBuffer();
143  msg = nullptr;
144  return;
145  }
146  TLOG_TRACE("NetMonTransportService") << "receiveMessage: Checking first Fragment type" << TLOG_ENDL;
147  auto firstFragmentType = *fragmentTypes.begin();
148 
149  // We return false, indicating we're done reading, if:
150  // 1) we did not obtain an event, because we timed out and were
151  // configured NOT to keep trying after a timeout, or
152  // 2) the event we read was the end-of-data marker: a null
153  // pointer
154  if (!got_event || firstFragmentType == artdaq::Fragment::EndOfDataFragmentType)
155  {
156  TLOG_DEBUG("NetMonTransportService") << "Received shutdown message, returning" << TLOG_ENDL;
157  incoming_events_->ReleaseBuffer();
158  msg = nullptr;
159  return;
160  }
161  if (firstFragmentType == artdaq::Fragment::InitFragmentType)
162  {
163  TLOG_DEBUG("NetMonTransportService") << "Cannot receive InitFragments here, retrying" << TLOG_ENDL;
164  incoming_events_->ReleaseBuffer();
165  continue;
166  }
167  // EndOfRun and EndOfSubrun Fragments are ignored in NetMonTransportService
168  else if (firstFragmentType == artdaq::Fragment::EndOfRunFragmentType || firstFragmentType == artdaq::Fragment::EndOfSubrunFragmentType)
169  {
170  TLOG_DEBUG("NetMonTransportService") << "Ignoring EndOfRun or EndOfSubrun Fragment" << TLOG_ENDL;
171  incoming_events_->ReleaseBuffer();
172  continue;
173  }
174 
175  TLOG_TRACE("NetMonTransportService") << "receiveMessage: Getting all Fragments" << TLOG_ENDL;
176  recvd_fragments_ = incoming_events_->GetFragmentsByType(errflag, artdaq::Fragment::InvalidFragmentType);
177  /* Events coming out of the EventStore are not sorted but need to be
178  sorted by sequence ID before they can be passed to art.
179  */
180  std::sort(recvd_fragments_->begin(), recvd_fragments_->end(),
181  artdaq::fragmentSequenceIDCompare);
182 
183  TLOG_TRACE("NetMonTransportService") << "receiveMessage: Releasing buffer" << TLOG_ENDL;
184  incoming_events_->ReleaseBuffer();
185  }
186 
187  // Do not process data until Init Fragment received!
188  auto start = std::chrono::steady_clock::now();
189  while (!init_received_ && artdaq::TimeUtils::GetElapsedTime(start) < init_timeout_s_)
190  {
191  usleep(init_timeout_s_ * 1000000 / 100); // Check 100 times
192  }
193  if (!init_received_) {
194  TLOG_ERROR("NetMonTransportService") << "Received data but no Init Fragment after " << init_timeout_s_ << " seconds. Art will crash." << TLOG_ENDL;
195  }
196 
197  TLOG_TRACE("NetMonTransportService") << "receiveMessage: Returning top Fragment" << TLOG_ENDL;
198  artdaq::Fragment topFrag = std::move(recvd_fragments_->at(0));
199  recvd_fragments_->erase(recvd_fragments_->begin());
200  if (recvd_fragments_->size() == 0)
201  {
202  recvd_fragments_.reset(nullptr);
203  }
204 
205  TLOG_TRACE("NetMonTransportService") << "receiveMessage: Copying Fragment into TBufferFile, length=" << topFrag.metadata<artdaq::NetMonHeader>()->data_length << TLOG_ENDL;
206  auto header = topFrag.metadata<artdaq::NetMonHeader>();
207  auto buffer = static_cast<char *>(malloc(header->data_length));
208  memcpy(buffer, &*topFrag.dataBegin(), header->data_length);
209  msg = new TBufferFile(TBuffer::kRead, header->data_length, buffer, kTRUE, 0);
210 
211 #if DUMP_RECEIVE_MESSAGE
212  std::string fileName = "receiveMessage_" + std::to_string(my_rank) + "_" + std::to_string(getpid()) + "_" + std::to_string(topFrag.sequenceID()) + ".bin";
213  std::fstream ostream(fileName.c_str(), std::ios::out | std::ios::binary);
214  ostream.write(buffer, header->data_length);
215  ostream.close();
216 #endif
217 
218  TLOG_TRACE("NetMonTransportService") << "receiveMessage END" << TLOG_ENDL;
219 }
220 
221 void
223 receiveInitMessage(TBufferFile*& msg)
224 {
225  TLOG_TRACE("NetMonTransportService") << "receiveInitMessage BEGIN" << TLOG_ENDL;
226  if (recvd_fragments_ == nullptr)
227  {
228  TLOG_TRACE("NetMonTransportService") << "receiveInitMessage: Waiting for available buffer" << TLOG_ENDL;
229 
230  bool got_init = false;
231  auto errflag = false;
232  while (!got_init) {
233 
234  bool got_event = false;
235  while (!got_event)
236  {
237  got_event = incoming_events_->ReadyForRead(true);
238  }
239 
240  TLOG_TRACE("NetMonTransportService") << "receiveInitMessage: Reading buffer header" << TLOG_ENDL;
241  incoming_events_->ReadHeader(errflag);
242  if (errflag) { // Buffer was changed out from under reader!
243  TLOG_ERROR("NetMonTransportService") << "receiveInitMessage: Error receiving message!" << TLOG_ENDL;
244  msg = nullptr;
245  return;
246  }
247  TLOG_TRACE("NetMonTransportService") << "receiveInitMessage: Getting Fragment types" << TLOG_ENDL;
248  auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag);
249  if (errflag) { // Buffer was changed out from under reader!
250  msg = nullptr;
251  TLOG_ERROR("NetMonTransportService") << "receiveInitMessage: Error receiving message!" << TLOG_ENDL;
252  return;
253  }
254  if (fragmentTypes.size() == 0)
255  {
256  TLOG_ERROR("NetMonTransportService") << "Event has no Fragments! Aborting!" << TLOG_ENDL;
257  incoming_events_->ReleaseBuffer();
258  msg = nullptr;
259  return;
260  }
261  TLOG_TRACE("NetMonTransportService") << "receiveInitMessage: Checking first Fragment type" << TLOG_ENDL;
262  auto firstFragmentType = *fragmentTypes.begin();
263 
264  // We return false, indicating we're done reading, if:
265  // 1) we did not obtain an event, because we timed out and were
266  // configured NOT to keep trying after a timeout, or
267  // 2) the event we read was the end-of-data marker: a null
268  // pointer
269  if (!got_event || firstFragmentType == artdaq::Fragment::EndOfDataFragmentType)
270  {
271  TLOG_DEBUG("NetMonTransportService") << "Received shutdown message, returning" << TLOG_ENDL;
272  incoming_events_->ReleaseBuffer();
273  msg = nullptr;
274  return;
275  }
276  if (firstFragmentType != artdaq::Fragment::InitFragmentType)
277  {
278  TLOG_WARNING("NetMonTransportService") << "Did NOT receive Init Fragment as first broadcast! Type=" << artdaq::detail::RawFragmentHeader::SystemTypeToString(firstFragmentType) << TLOG_ENDL;
279  incoming_events_->ReleaseBuffer();
280  }
281  got_init = true;
282  }
283  TLOG_TRACE("NetMonTransportService") << "receiveInitMessage: Getting all Fragments" << TLOG_ENDL;
284  recvd_fragments_ = incoming_events_->GetFragmentsByType(errflag, artdaq::Fragment::InvalidFragmentType);
285  /* Events coming out of the EventStore are not sorted but need to be
286  sorted by sequence ID before they can be passed to art.
287  */
288  std::sort(recvd_fragments_->begin(), recvd_fragments_->end(),
289  artdaq::fragmentSequenceIDCompare);
290  }
291 
292  TLOG_TRACE("NetMonTransportService") << "receiveInitMessage: Returning top Fragment" << TLOG_ENDL;
293  artdaq::Fragment topFrag = std::move(recvd_fragments_->at(0));
294  recvd_fragments_->erase(recvd_fragments_->begin());
295  if (recvd_fragments_->size() == 0)
296  {
297  recvd_fragments_.reset(nullptr);
298  }
299 
300  auto header = topFrag.metadata<artdaq::NetMonHeader>();
301  TLOG_TRACE("NetMonTransportService") << "receiveInitMessage: Copying Fragment into TBufferFile: message length: " << std::to_string(header->data_length) << TLOG_ENDL;
302  auto buffer = static_cast<char *>(malloc(header->data_length));
303  memcpy(buffer, &*topFrag.dataBegin(), header->data_length);
304 
305 #if DUMP_RECEIVE_MESSAGE
306  std::string fileName = "receiveInitMessage_" + std::to_string(getpid()) + ".bin";
307  std::fstream ostream(fileName.c_str(), std::ios::out | std::ios::binary);
308  ostream.write(buffer, header->data_length);
309  ostream.close();
310 #endif
311 
312  msg = new TBufferFile(TBuffer::kRead, header->data_length, buffer, kTRUE, 0);
313 
314  TLOG_TRACE("NetMonTransportService") << "receiveInitMessage END" << TLOG_ENDL;
315  init_received_ = true;
316 }
317 DEFINE_ART_SERVICE_INTERFACE_IMPL(NetMonTransportService, NetMonTransportServiceInterface)
void receiveInitMessage(TBufferFile *&msg) override
Receive the init message.
Sends Fragment objects using TransferInterface plugins. Uses Routing Tables if confgiured, otherwise will Round-Robin Fragments to the destinations.
void sendMessage(uint64_t sequenceId, uint8_t messageType, TBufferFile &msg) override
Send ROOT data, wrapped in an artdaq::Fragment object.
NetMonTransportService extends NetMonTransportServiceInterface. It sends events using DataSenderManag...
void receiveMessage(TBufferFile *&msg) override
Receive data from the ConcurrentQueue.
Header with length information for NetMonTransport messages.
Definition: NetMonHeader.hh:14
void connect() override
Reconnect the NetMonTransportService.
virtual ~NetMonTransportService()
NetMonTransportService Destructor. Calls disconnect().
NetMonTransportService(fhicl::ParameterSet const &pset, art::ActivityRegistry &)
NetMonTransportService Constructor.
void disconnect() override
Disconnects the NetMonTranportService.
uint64_t data_length
The length of the message.
Definition: NetMonHeader.hh:16
Interface for NetMonTranportService. This interface is declared to art as part of the required regist...
void listen() override
Listen for connections. This method is a No-Op.