artdaq  v3_00_01
NetMonTransportService_service.cc
1 #include "artdaq/DAQdata/Globals.hh"
2 #include "artdaq/ArtModules/NetMonTransportService.h"
3 #include "artdaq/DAQrate/DataSenderManager.hh"
4 #include "artdaq-core/Core/SharedMemoryEventReceiver.hh"
5 
6 #include "artdaq-core/Data/Fragment.hh"
7 #include "artdaq/DAQdata/NetMonHeader.hh"
8 #include "artdaq-core/Data/RawEvent.hh"
9 #include "artdaq-core/Utilities/TimeUtils.hh"
10 
11 #include "art/Framework/Services/Registry/ActivityRegistry.h"
12 #include "canvas/Utilities/Exception.h"
13 #include "cetlib/container_algorithms.h"
14 #include "cetlib_except/exception.h"
15 #include "fhiclcpp/ParameterSet.h"
16 #include "fhiclcpp/ParameterSetRegistry.h"
17 
18 #include <TClass.h>
19 #include <TBufferFile.h>
20 
21 #include <iomanip>
22 #include <iostream>
23 #include <fstream>
24 #include <string>
25 #include <vector>
26 
27 static fhicl::ParameterSet empty_pset;
28 
29 
31 NetMonTransportService(fhicl::ParameterSet const& pset, art::ActivityRegistry&)
33  , data_pset_(pset)
34  , init_received_(false)
35  , sender_ptr_(nullptr)
36  , incoming_events_(new artdaq::SharedMemoryEventReceiver(pset.get<int>("shared_memory_key", 0xBEE70000 + getppid()), pset.get<int>("broadcast_shared_memory_key", 0xCEE70000 + getppid())))
37  , recvd_fragments_(nullptr)
38 {
39  TLOG_TRACE("NetMonTransportService") << "NetMonTransportService CONSTRUCTOR" << TLOG_ENDL;
40  if (pset.has_key("rank")) my_rank = pset.get<int>("rank");
41  else my_rank = incoming_events_->GetRank();
42 
43  init_timeout_s_ = pset.get<double>("init_fragment_timeout_seconds", 1.0);
44 }
45 
48 {
50 }
51 
52 void
55 {
56  sender_ptr_.reset(new artdaq::DataSenderManager(data_pset_));
57 }
58 
59 void
62 {
63  return;
64 }
65 
66 void
69 {
70  if (sender_ptr_) sender_ptr_.reset(nullptr);
71 }
72 
73 void
75 sendMessage(uint64_t sequenceId, uint8_t messageType, TBufferFile& msg)
76 {
77  if (sender_ptr_ == nullptr)
78  {
79  TLOG_DEBUG("NetMonTransportService") << "Reconnecting DataSenderManager" << TLOG_ENDL;
80  connect();
81  }
82 
83 #if 0
84  if (messageType == artdaq::Fragment::InitFragmentType) {
85  std::fstream ostream("sendInitMessage.bin", std::ios::out | std::ios::binary);
86  ostream.write(msg.Buffer(), msg.Length());
87  ostream.close();
88  }
89 #endif
90 
91  TLOG_DEBUG("NetMonTransportService") << "Sending message with sequenceID=" << std::to_string(sequenceId) << ", type=" << std::to_string(messageType) << ", length=" << std::to_string(msg.Length()) << TLOG_ENDL;
92  artdaq::NetMonHeader header;
93  header.data_length = static_cast<uint64_t>(msg.Length());
94  artdaq::Fragment
95  fragment(std::ceil(msg.Length() /
96  static_cast<double>(sizeof(artdaq::RawDataType))),
97  sequenceId, 0, messageType, header);
98 
99  memcpy(&*fragment.dataBegin(), msg.Buffer(), msg.Length());
100  sender_ptr_->sendFragment(std::move(fragment));
101 }
102 
103 void
105 receiveMessage(TBufferFile*& msg)
106 {
107  TLOG_TRACE("NetMonTransportService") << "receiveMessage BEGIN" << TLOG_ENDL;
108  while (recvd_fragments_ == nullptr)
109  {
110  TLOG_TRACE("NetMonTransportService") << "receiveMessage: Waiting for available buffer" << TLOG_ENDL;
111  bool keep_looping = true;
112  bool got_event = false;
113  while (keep_looping)
114  {
115  keep_looping = false;
116  got_event = incoming_events_->ReadyForRead();
117  if (!got_event)
118  {
119  keep_looping = true;
120  }
121  }
122 
123  TLOG_TRACE("NetMonTransportService") << "receiveMessage: Reading buffer header" << TLOG_ENDL;
124  auto errflag = false;
125  incoming_events_->ReadHeader(errflag);
126  if (errflag) { // Buffer was changed out from under reader!
127  msg = nullptr;
128  return;
129  }
130  TLOG_TRACE("NetMonTransportService") << "receiveMessage: Getting Fragment types" << TLOG_ENDL;
131  auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag);
132  if (errflag) { // Buffer was changed out from under reader!
133  msg = nullptr;
134  return;
135  }
136  if (fragmentTypes.size() == 0)
137  {
138  TLOG_ERROR("NetMonTransportService") << "Event has no Fragments! Aborting!" << TLOG_ENDL;
139  incoming_events_->ReleaseBuffer();
140  msg = nullptr;
141  return;
142  }
143  TLOG_TRACE("NetMonTransportService") << "receiveMessage: Checking first Fragment type" << TLOG_ENDL;
144  auto firstFragmentType = *fragmentTypes.begin();
145 
146  // We return false, indicating we're done reading, if:
147  // 1) we did not obtain an event, because we timed out and were
148  // configured NOT to keep trying after a timeout, or
149  // 2) the event we read was the end-of-data marker: a null
150  // pointer
151  if (!got_event || firstFragmentType == artdaq::Fragment::EndOfDataFragmentType)
152  {
153  TLOG_DEBUG("NetMonTransportService") << "Received shutdown message, returning" << TLOG_ENDL;
154  incoming_events_->ReleaseBuffer();
155  msg = nullptr;
156  return;
157  }
158  if (firstFragmentType == artdaq::Fragment::InitFragmentType)
159  {
160  TLOG_DEBUG("NetMonTransportService") << "Cannot receive InitFragments here, retrying" << TLOG_ENDL;
161  incoming_events_->ReleaseBuffer();
162  continue;
163  }
164  // EndOfRun and EndOfSubrun Fragments are ignored in NetMonTransportService
165  else if (firstFragmentType == artdaq::Fragment::EndOfRunFragmentType || firstFragmentType == artdaq::Fragment::EndOfSubrunFragmentType)
166  {
167  TLOG_DEBUG("NetMonTransportService") << "Ignoring EndOfRun or EndOfSubrun Fragment" << TLOG_ENDL;
168  incoming_events_->ReleaseBuffer();
169  continue;
170  }
171 
172  TLOG_TRACE("NetMonTransportService") << "receiveMessage: Getting all Fragments" << TLOG_ENDL;
173  recvd_fragments_ = incoming_events_->GetFragmentsByType(errflag, artdaq::Fragment::InvalidFragmentType);
174  /* Events coming out of the EventStore are not sorted but need to be
175  sorted by sequence ID before they can be passed to art.
176  */
177  std::sort(recvd_fragments_->begin(), recvd_fragments_->end(),
178  artdaq::fragmentSequenceIDCompare);
179 
180  TLOG_TRACE("NetMonTransportService") << "receiveMessage: Releasing buffer" << TLOG_ENDL;
181  incoming_events_->ReleaseBuffer();
182  }
183 
184  // Do not process data until Init Fragment received!
185  auto start = std::chrono::steady_clock::now();
186  while (!init_received_ && artdaq::TimeUtils::GetElapsedTime(start) < init_timeout_s_)
187  {
188  usleep(init_timeout_s_ * 1000000 / 100); // Check 100 times
189  }
190  if (!init_received_) {
191  TLOG_ERROR("NetMonTransportService") << "Received data but no Init Fragment after " << init_timeout_s_ << " seconds. Art will crash." << TLOG_ENDL;
192  }
193 
194  TLOG_TRACE("NetMonTransportService") << "receiveMessage: Returning top Fragment" << TLOG_ENDL;
195  artdaq::Fragment topFrag = std::move(recvd_fragments_->at(0));
196  recvd_fragments_->erase(recvd_fragments_->begin());
197  if (recvd_fragments_->size() == 0)
198  {
199  recvd_fragments_.reset(nullptr);
200  }
201 
202  TLOG_TRACE("NetMonTransportService") << "receiveMessage: Copying Fragment into TBufferFile" << TLOG_ENDL;
203  auto header = topFrag.metadata<artdaq::NetMonHeader>();
204  auto buffer = static_cast<char *>(malloc(header->data_length));
205  memcpy(buffer, &*topFrag.dataBegin(), header->data_length);
206  msg = new TBufferFile(TBuffer::kRead, header->data_length, buffer, kTRUE, 0);
207 
208  TLOG_TRACE("NetMonTransportService") << "receiveMessage END" << TLOG_ENDL;
209 }
210 
211 void
213 receiveInitMessage(TBufferFile*& msg)
214 {
215  TLOG_TRACE("NetMonTransportService") << "receiveInitMessage BEGIN" << TLOG_ENDL;
216  if (recvd_fragments_ == nullptr)
217  {
218  TLOG_TRACE("NetMonTransportService") << "receiveInitMessage: Waiting for available buffer" << TLOG_ENDL;
219 
220  bool got_init = false;
221  auto errflag = false;
222  while (!got_init) {
223 
224  bool got_event = false;
225  while (!got_event)
226  {
227  got_event = incoming_events_->ReadyForRead(true);
228  }
229 
230  TLOG_TRACE("NetMonTransportService") << "receiveInitMessage: Reading buffer header" << TLOG_ENDL;
231  incoming_events_->ReadHeader(errflag);
232  if (errflag) { // Buffer was changed out from under reader!
233  TLOG_ERROR("NetMonTransportService") << "receiveInitMessage: Error receiving message!" << TLOG_ENDL;
234  msg = nullptr;
235  return;
236  }
237  TLOG_TRACE("NetMonTransportService") << "receiveInitMessage: Getting Fragment types" << TLOG_ENDL;
238  auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag);
239  if (errflag) { // Buffer was changed out from under reader!
240  msg = nullptr;
241  TLOG_ERROR("NetMonTransportService") << "receiveInitMessage: Error receiving message!" << TLOG_ENDL;
242  return;
243  }
244  if (fragmentTypes.size() == 0)
245  {
246  TLOG_ERROR("NetMonTransportService") << "Event has no Fragments! Aborting!" << TLOG_ENDL;
247  incoming_events_->ReleaseBuffer();
248  msg = nullptr;
249  return;
250  }
251  TLOG_TRACE("NetMonTransportService") << "receiveInitMessage: Checking first Fragment type" << TLOG_ENDL;
252  auto firstFragmentType = *fragmentTypes.begin();
253 
254  // We return false, indicating we're done reading, if:
255  // 1) we did not obtain an event, because we timed out and were
256  // configured NOT to keep trying after a timeout, or
257  // 2) the event we read was the end-of-data marker: a null
258  // pointer
259  if (!got_event || firstFragmentType == artdaq::Fragment::EndOfDataFragmentType)
260  {
261  TLOG_DEBUG("NetMonTransportService") << "Received shutdown message, returning" << TLOG_ENDL;
262  incoming_events_->ReleaseBuffer();
263  msg = nullptr;
264  return;
265  }
266  if (firstFragmentType != artdaq::Fragment::InitFragmentType)
267  {
268  TLOG_WARNING("NetMonTransportService") << "Did NOT receive Init Fragment as first broadcast! Type=" << artdaq::detail::RawFragmentHeader::SystemTypeToString(firstFragmentType) << TLOG_ENDL;
269  incoming_events_->ReleaseBuffer();
270  }
271  got_init = true;
272  }
273  TLOG_TRACE("NetMonTransportService") << "receiveInitMessage: Getting all Fragments" << TLOG_ENDL;
274  recvd_fragments_ = incoming_events_->GetFragmentsByType(errflag, artdaq::Fragment::InvalidFragmentType);
275  /* Events coming out of the EventStore are not sorted but need to be
276  sorted by sequence ID before they can be passed to art.
277  */
278  std::sort(recvd_fragments_->begin(), recvd_fragments_->end(),
279  artdaq::fragmentSequenceIDCompare);
280  }
281 
282  TLOG_TRACE("NetMonTransportService") << "receiveInitMessage: Returning top Fragment" << TLOG_ENDL;
283  artdaq::Fragment topFrag = std::move(recvd_fragments_->at(0));
284  recvd_fragments_->erase(recvd_fragments_->begin());
285  if (recvd_fragments_->size() == 0)
286  {
287  recvd_fragments_.reset(nullptr);
288  }
289 
290  auto header = topFrag.metadata<artdaq::NetMonHeader>();
291  TLOG_TRACE("NetMonTransportService") << "receiveInitMessage: Copying Fragment into TBufferFile: message length: " << std::to_string(header->data_length) << TLOG_ENDL;
292  auto buffer = static_cast<char *>(malloc(header->data_length));
293  memcpy(buffer, &*topFrag.dataBegin(), header->data_length);
294 
295 #if 0
296  std::string fileName = "receiveInitMessage_" + std::to_string(getpid()) + ".bin";
297  std::fstream ostream(fileName.c_str(), std::ios::out | std::ios::binary);
298  ostream.write(buffer, header->data_length);
299  ostream.close();
300 #endif
301 
302  msg = new TBufferFile(TBuffer::kRead, header->data_length, buffer, kTRUE, 0);
303 
304  TLOG_TRACE("NetMonTransportService") << "receiveInitMessage END" << TLOG_ENDL;
305  init_received_ = true;
306 }
307 DEFINE_ART_SERVICE_INTERFACE_IMPL(NetMonTransportService, NetMonTransportServiceInterface)
void receiveInitMessage(TBufferFile *&msg) override
Receive the init message.
Sends Fragment objects using TransferInterface plugins. Uses Routing Tables if confgiured, otherwise will Round-Robin Fragments to the destinations.
void sendMessage(uint64_t sequenceId, uint8_t messageType, TBufferFile &msg) override
Send ROOT data, wrapped in an artdaq::Fragment object.
NetMonTransportService extends NetMonTransportServiceInterface. It sends events using DataSenderManag...
void receiveMessage(TBufferFile *&msg) override
Receive data from the ConcurrentQueue.
Header with length information for NetMonTransport messages.
Definition: NetMonHeader.hh:14
void connect() override
Reconnect the NetMonTransportService.
virtual ~NetMonTransportService()
NetMonTransportService Destructor. Calls disconnect().
NetMonTransportService(fhicl::ParameterSet const &pset, art::ActivityRegistry &)
NetMonTransportService Constructor.
void disconnect() override
Disconnects the NetMonTranportService.
uint64_t data_length
The length of the message.
Definition: NetMonHeader.hh:16
Interface for NetMonTranportService. This interface is declared to art as part of the required regist...
void listen() override
Listen for connections. This method is a No-Op.