1 #include "artdaq/DAQdata/Globals.hh"
2 #include "artdaq/ArtModules/NetMonTransportService.h"
3 #include "artdaq/DAQrate/DataSenderManager.hh"
4 #include "artdaq-core/Core/SharedMemoryEventReceiver.hh"
6 #include "artdaq-core/Data/Fragment.hh"
7 #include "artdaq/DAQdata/NetMonHeader.hh"
8 #include "artdaq-core/Data/RawEvent.hh"
9 #include "artdaq-core/Utilities/TimeUtils.hh"
11 #include "art/Framework/Services/Registry/ActivityRegistry.h"
12 #include "canvas/Utilities/Exception.h"
13 #include "cetlib/container_algorithms.h"
14 #include "cetlib_except/exception.h"
15 #include "fhiclcpp/ParameterSet.h"
16 #include "fhiclcpp/ParameterSetRegistry.h"
19 #include <TBufferFile.h>
28 #define DUMP_SEND_MESSAGE 0
29 #define DUMP_RECEIVE_MESSAGE 0
31 static fhicl::ParameterSet empty_pset;
38 , init_received_(false)
39 , sender_ptr_(nullptr)
40 , incoming_events_(new artdaq::SharedMemoryEventReceiver(pset.get<int>(
"shared_memory_key", 0xBEE70000 + getppid()), pset.get<int>(
"broadcast_shared_memory_key", 0xCEE70000 + getppid())))
41 , recvd_fragments_(nullptr)
43 TLOG_TRACE(
"NetMonTransportService") <<
"NetMonTransportService CONSTRUCTOR" << TLOG_ENDL;
44 if (pset.has_key(
"rank")) my_rank = pset.get<
int>(
"rank");
45 else my_rank = incoming_events_->GetRank();
47 init_timeout_s_ = pset.get<
double>(
"init_fragment_timeout_seconds", 1.0);
74 if (sender_ptr_) sender_ptr_.reset(
nullptr);
79 sendMessage(uint64_t sequenceId, uint8_t messageType, TBufferFile& msg)
81 if (sender_ptr_ ==
nullptr)
83 TLOG_DEBUG(
"NetMonTransportService") <<
"Reconnecting DataSenderManager" << TLOG_ENDL;
88 std::string fileName =
"sendMessage_" + std::to_string(my_rank) +
"_" + std::to_string(getpid()) +
"_" + std::to_string(sequenceId) +
".bin";
89 std::fstream ostream(fileName, std::ios::out | std::ios::binary);
90 ostream.write(msg.Buffer(), msg.Length());
94 TLOG_DEBUG(
"NetMonTransportService") <<
"Sending message with sequenceID=" << std::to_string(sequenceId) <<
", type=" << std::to_string(messageType) <<
", length=" << std::to_string(msg.Length()) << TLOG_ENDL;
96 header.
data_length =
static_cast<uint64_t
>(msg.Length());
98 fragment(std::ceil(msg.Length() /
99 static_cast<double>(
sizeof(artdaq::RawDataType))),
100 sequenceId, 0, messageType, header);
102 memcpy(&*fragment.dataBegin(), msg.Buffer(), msg.Length());
103 sender_ptr_->sendFragment(std::move(fragment));
110 TLOG_TRACE(
"NetMonTransportService") <<
"receiveMessage BEGIN" << TLOG_ENDL;
111 while (recvd_fragments_ ==
nullptr)
113 TLOG_TRACE(
"NetMonTransportService") <<
"receiveMessage: Waiting for available buffer" << TLOG_ENDL;
114 bool keep_looping =
true;
115 bool got_event =
false;
118 keep_looping =
false;
119 got_event = incoming_events_->ReadyForRead();
126 TLOG_TRACE(
"NetMonTransportService") <<
"receiveMessage: Reading buffer header" << TLOG_ENDL;
127 auto errflag =
false;
128 incoming_events_->ReadHeader(errflag);
133 TLOG_TRACE(
"NetMonTransportService") <<
"receiveMessage: Getting Fragment types" << TLOG_ENDL;
134 auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag);
139 if (fragmentTypes.size() == 0)
141 TLOG_ERROR(
"NetMonTransportService") <<
"Event has no Fragments! Aborting!" << TLOG_ENDL;
142 incoming_events_->ReleaseBuffer();
146 TLOG_TRACE(
"NetMonTransportService") <<
"receiveMessage: Checking first Fragment type" << TLOG_ENDL;
147 auto firstFragmentType = *fragmentTypes.begin();
154 if (!got_event || firstFragmentType == artdaq::Fragment::EndOfDataFragmentType)
156 TLOG_DEBUG(
"NetMonTransportService") <<
"Received shutdown message, returning" << TLOG_ENDL;
157 incoming_events_->ReleaseBuffer();
161 if (firstFragmentType == artdaq::Fragment::InitFragmentType)
163 TLOG_DEBUG(
"NetMonTransportService") <<
"Cannot receive InitFragments here, retrying" << TLOG_ENDL;
164 incoming_events_->ReleaseBuffer();
168 else if (firstFragmentType == artdaq::Fragment::EndOfRunFragmentType || firstFragmentType == artdaq::Fragment::EndOfSubrunFragmentType)
170 TLOG_DEBUG(
"NetMonTransportService") <<
"Ignoring EndOfRun or EndOfSubrun Fragment" << TLOG_ENDL;
171 incoming_events_->ReleaseBuffer();
175 TLOG_TRACE(
"NetMonTransportService") <<
"receiveMessage: Getting all Fragments" << TLOG_ENDL;
176 recvd_fragments_ = incoming_events_->GetFragmentsByType(errflag, artdaq::Fragment::InvalidFragmentType);
180 std::sort(recvd_fragments_->begin(), recvd_fragments_->end(),
181 artdaq::fragmentSequenceIDCompare);
183 TLOG_TRACE(
"NetMonTransportService") <<
"receiveMessage: Releasing buffer" << TLOG_ENDL;
184 incoming_events_->ReleaseBuffer();
188 auto start = std::chrono::steady_clock::now();
189 while (!init_received_ && artdaq::TimeUtils::GetElapsedTime(start) < init_timeout_s_)
191 usleep(init_timeout_s_ * 1000000 / 100);
193 if (!init_received_) {
194 TLOG_ERROR(
"NetMonTransportService") <<
"Received data but no Init Fragment after " << init_timeout_s_ <<
" seconds. Art will crash." << TLOG_ENDL;
197 TLOG_TRACE(
"NetMonTransportService") <<
"receiveMessage: Returning top Fragment" << TLOG_ENDL;
198 artdaq::Fragment topFrag = std::move(recvd_fragments_->at(0));
199 recvd_fragments_->erase(recvd_fragments_->begin());
200 if (recvd_fragments_->size() == 0)
202 recvd_fragments_.reset(
nullptr);
205 TLOG_TRACE(
"NetMonTransportService") <<
"receiveMessage: Copying Fragment into TBufferFile, length=" << topFrag.metadata<
artdaq::NetMonHeader>()->data_length << TLOG_ENDL;
207 auto buffer =
static_cast<char *
>(malloc(header->
data_length));
208 memcpy(buffer, &*topFrag.dataBegin(), header->
data_length);
209 msg =
new TBufferFile(TBuffer::kRead, header->
data_length, buffer, kTRUE, 0);
211 #if DUMP_RECEIVE_MESSAGE
212 std::string fileName =
"receiveMessage_" + std::to_string(my_rank) +
"_" + std::to_string(getpid()) +
"_" + std::to_string(topFrag.sequenceID()) +
".bin";
213 std::fstream ostream(fileName.c_str(), std::ios::out | std::ios::binary);
218 TLOG_TRACE(
"NetMonTransportService") <<
"receiveMessage END" << TLOG_ENDL;
225 TLOG_TRACE(
"NetMonTransportService") <<
"receiveInitMessage BEGIN" << TLOG_ENDL;
226 if (recvd_fragments_ ==
nullptr)
228 TLOG_TRACE(
"NetMonTransportService") <<
"receiveInitMessage: Waiting for available buffer" << TLOG_ENDL;
230 bool got_init =
false;
231 auto errflag =
false;
234 bool got_event =
false;
237 got_event = incoming_events_->ReadyForRead(
true);
240 TLOG_TRACE(
"NetMonTransportService") <<
"receiveInitMessage: Reading buffer header" << TLOG_ENDL;
241 incoming_events_->ReadHeader(errflag);
243 TLOG_ERROR(
"NetMonTransportService") <<
"receiveInitMessage: Error receiving message!" << TLOG_ENDL;
247 TLOG_TRACE(
"NetMonTransportService") <<
"receiveInitMessage: Getting Fragment types" << TLOG_ENDL;
248 auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag);
251 TLOG_ERROR(
"NetMonTransportService") <<
"receiveInitMessage: Error receiving message!" << TLOG_ENDL;
254 if (fragmentTypes.size() == 0)
256 TLOG_ERROR(
"NetMonTransportService") <<
"Event has no Fragments! Aborting!" << TLOG_ENDL;
257 incoming_events_->ReleaseBuffer();
261 TLOG_TRACE(
"NetMonTransportService") <<
"receiveInitMessage: Checking first Fragment type" << TLOG_ENDL;
262 auto firstFragmentType = *fragmentTypes.begin();
269 if (!got_event || firstFragmentType == artdaq::Fragment::EndOfDataFragmentType)
271 TLOG_DEBUG(
"NetMonTransportService") <<
"Received shutdown message, returning" << TLOG_ENDL;
272 incoming_events_->ReleaseBuffer();
276 if (firstFragmentType != artdaq::Fragment::InitFragmentType)
278 TLOG_WARNING(
"NetMonTransportService") <<
"Did NOT receive Init Fragment as first broadcast! Type=" << artdaq::detail::RawFragmentHeader::SystemTypeToString(firstFragmentType) << TLOG_ENDL;
279 incoming_events_->ReleaseBuffer();
283 TLOG_TRACE(
"NetMonTransportService") <<
"receiveInitMessage: Getting all Fragments" << TLOG_ENDL;
284 recvd_fragments_ = incoming_events_->GetFragmentsByType(errflag, artdaq::Fragment::InvalidFragmentType);
288 std::sort(recvd_fragments_->begin(), recvd_fragments_->end(),
289 artdaq::fragmentSequenceIDCompare);
292 TLOG_TRACE(
"NetMonTransportService") <<
"receiveInitMessage: Returning top Fragment" << TLOG_ENDL;
293 artdaq::Fragment topFrag = std::move(recvd_fragments_->at(0));
294 recvd_fragments_->erase(recvd_fragments_->begin());
295 if (recvd_fragments_->size() == 0)
297 recvd_fragments_.reset(
nullptr);
301 TLOG_TRACE(
"NetMonTransportService") <<
"receiveInitMessage: Copying Fragment into TBufferFile: message length: " << std::to_string(header->data_length) << TLOG_ENDL;
302 auto buffer =
static_cast<char *
>(malloc(header->data_length));
303 memcpy(buffer, &*topFrag.dataBegin(), header->data_length);
305 #if DUMP_RECEIVE_MESSAGE
306 std::string fileName =
"receiveInitMessage_" + std::to_string(getpid()) +
".bin";
307 std::fstream ostream(fileName.c_str(), std::ios::out | std::ios::binary);
308 ostream.write(buffer, header->data_length);
312 msg =
new TBufferFile(TBuffer::kRead, header->data_length, buffer, kTRUE, 0);
314 TLOG_TRACE(
"NetMonTransportService") <<
"receiveInitMessage END" << TLOG_ENDL;
315 init_received_ =
true;
void receiveInitMessage(TBufferFile *&msg) override
Receive the init message.
Sends Fragment objects using TransferInterface plugins. Uses Routing Tables if confgiured, otherwise will Round-Robin Fragments to the destinations.
void sendMessage(uint64_t sequenceId, uint8_t messageType, TBufferFile &msg) override
Send ROOT data, wrapped in an artdaq::Fragment object.
NetMonTransportService extends NetMonTransportServiceInterface. It sends events using DataSenderManag...
void receiveMessage(TBufferFile *&msg) override
Receive data from the ConcurrentQueue.
void connect() override
Reconnect the NetMonTransportService.
virtual ~NetMonTransportService()
NetMonTransportService Destructor. Calls disconnect().
NetMonTransportService(fhicl::ParameterSet const &pset, art::ActivityRegistry &)
NetMonTransportService Constructor.
void disconnect() override
Disconnects the NetMonTranportService.
Interface for NetMonTranportService. This interface is declared to art as part of the required regist...
void listen() override
Listen for connections. This method is a No-Op.