1 #define TRACE_NAME "NetMonTransportService"
3 #include "artdaq/DAQdata/Globals.hh"
4 #include "artdaq/ArtModules/NetMonTransportService.h"
5 #include "artdaq/DAQrate/DataSenderManager.hh"
6 #include "artdaq-core/Core/SharedMemoryEventReceiver.hh"
8 #include "artdaq-core/Data/Fragment.hh"
9 #include "artdaq/DAQdata/NetMonHeader.hh"
10 #include "artdaq-core/Data/RawEvent.hh"
11 #include "artdaq-core/Utilities/TimeUtils.hh"
13 #include "art/Framework/Services/Registry/ActivityRegistry.h"
14 #include "canvas/Utilities/Exception.h"
15 #include "cetlib/container_algorithms.h"
16 #include "cetlib_except/exception.h"
17 #include "fhiclcpp/ParameterSet.h"
18 #include "fhiclcpp/ParameterSetRegistry.h"
21 #include <TBufferFile.h>
30 #define DUMP_SEND_MESSAGE 0
31 #define DUMP_RECEIVE_MESSAGE 0
33 static fhicl::ParameterSet empty_pset;
40 , init_received_(false)
41 , sender_ptr_(nullptr)
42 , incoming_events_(nullptr)
43 , recvd_fragments_(nullptr)
45 TLOG(TLVL_TRACE) <<
"NetMonTransportService CONSTRUCTOR" ;
46 if (pset.has_key(
"rank")) my_rank = pset.get<
int>(
"rank");
48 init_timeout_s_ = pset.get<
double>(
"init_fragment_timeout_seconds", 1.0);
68 if (!incoming_events_)
70 incoming_events_.reset(
new artdaq::SharedMemoryEventReceiver(data_pset_.get<
int>(
"shared_memory_key", 0xBEE70000 + getppid()), data_pset_.get<
int>(
"broadcast_shared_memory_key", 0xCEE70000 + getppid())));
71 if (data_pset_.has_key(
"rank")) my_rank = data_pset_.get<
int>(
"rank");
72 else my_rank = incoming_events_->GetRank();
81 if (sender_ptr_) sender_ptr_.reset(
nullptr);
86 sendMessage(uint64_t sequenceId, uint8_t messageType, TBufferFile& msg)
88 if (sender_ptr_ ==
nullptr)
90 TLOG(TLVL_DEBUG) <<
"Reconnecting DataSenderManager" ;
95 std::string fileName =
"sendMessage_" + std::to_string(my_rank) +
"_" + std::to_string(getpid()) +
"_" + std::to_string(sequenceId) +
".bin";
96 std::fstream ostream(fileName, std::ios::out | std::ios::binary);
97 ostream.write(msg.Buffer(), msg.Length());
101 TLOG(TLVL_DEBUG) <<
"Sending message with sequenceID=" << sequenceId <<
", type=" << (int)messageType <<
", length=" << msg.Length() ;
103 header.
data_length =
static_cast<uint64_t
>(msg.Length());
105 fragment(std::ceil(msg.Length() /
106 static_cast<double>(
sizeof(artdaq::RawDataType))),
107 sequenceId, 0, messageType, header);
109 memcpy(&*fragment.dataBegin(), msg.Buffer(), msg.Length());
110 sender_ptr_->sendFragment(std::move(fragment));
118 TLOG(TLVL_TRACE) <<
"receiveMessage BEGIN" ;
119 while (recvd_fragments_ ==
nullptr)
121 TLOG(TLVL_TRACE) <<
"receiveMessage: Waiting for available buffer" ;
122 bool keep_looping =
true;
123 bool got_event =
false;
126 keep_looping =
false;
127 got_event = incoming_events_->ReadyForRead();
134 TLOG(TLVL_TRACE) <<
"receiveMessage: Reading buffer header" ;
135 auto errflag =
false;
136 incoming_events_->ReadHeader(errflag);
141 TLOG(TLVL_TRACE) <<
"receiveMessage: Getting Fragment types" ;
142 auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag);
144 incoming_events_->ReleaseBuffer();
148 if (fragmentTypes.size() == 0)
150 TLOG(TLVL_ERROR) <<
"Event has no Fragments! Aborting!" ;
151 incoming_events_->ReleaseBuffer();
155 TLOG(TLVL_TRACE) <<
"receiveMessage: Checking first Fragment type" ;
156 auto firstFragmentType = *fragmentTypes.begin();
163 if (!got_event || firstFragmentType == artdaq::Fragment::EndOfDataFragmentType)
165 TLOG(TLVL_DEBUG) <<
"Received shutdown message, returning from receiveMessage "
166 <<
"(debug: got_event=" << got_event <<
",fragType=" << (int)firstFragmentType
167 <<
",EODFragType=" << (
int)artdaq::Fragment::EndOfDataFragmentType <<
")";
168 incoming_events_->ReleaseBuffer();
172 if (firstFragmentType == artdaq::Fragment::InitFragmentType)
174 TLOG(TLVL_DEBUG) <<
"Cannot receive InitFragments here, retrying" ;
175 incoming_events_->ReleaseBuffer();
179 else if (firstFragmentType == artdaq::Fragment::EndOfRunFragmentType || firstFragmentType == artdaq::Fragment::EndOfSubrunFragmentType)
181 TLOG(TLVL_DEBUG) <<
"Ignoring EndOfRun or EndOfSubrun Fragment" ;
182 incoming_events_->ReleaseBuffer();
186 TLOG(TLVL_TRACE) <<
"receiveMessage: Getting all Fragments" ;
187 recvd_fragments_ = incoming_events_->GetFragmentsByType(errflag, artdaq::Fragment::InvalidFragmentType);
188 if (!recvd_fragments_)
190 TLOG(TLVL_ERROR) <<
"Error retrieving Fragments from shared memory! Aborting!";
191 incoming_events_->ReleaseBuffer();
199 std::sort(recvd_fragments_->begin(), recvd_fragments_->end(),
200 artdaq::fragmentSequenceIDCompare);
202 TLOG(TLVL_TRACE) <<
"receiveMessage: Releasing buffer" ;
203 incoming_events_->ReleaseBuffer();
207 auto start = std::chrono::steady_clock::now();
208 while (!init_received_ && artdaq::TimeUtils::GetElapsedTime(start) < init_timeout_s_)
210 usleep(init_timeout_s_ * 1000000 / 100);
212 if (!init_received_) {
213 TLOG(TLVL_ERROR) <<
"Received data but no Init Fragment after " << init_timeout_s_ <<
" seconds. Art will crash." ;
216 TLOG(TLVL_TRACE) <<
"receiveMessage: Returning top Fragment" ;
217 artdaq::Fragment topFrag = std::move(recvd_fragments_->at(0));
218 recvd_fragments_->erase(recvd_fragments_->begin());
219 if (recvd_fragments_->size() == 0)
221 recvd_fragments_.reset(
nullptr);
224 TLOG(TLVL_TRACE) <<
"receiveMessage: Copying Fragment into TBufferFile, length=" << topFrag.metadata<
artdaq::NetMonHeader>()->data_length ;
226 auto buffer =
static_cast<char *
>(malloc(header->
data_length));
227 memcpy(buffer, &*topFrag.dataBegin(), header->
data_length);
228 msg =
new TBufferFile(TBuffer::kRead, header->
data_length, buffer, kTRUE, 0);
230 #if DUMP_RECEIVE_MESSAGE
231 std::string fileName =
"receiveMessage_" + std::to_string(my_rank) +
"_" + std::to_string(getpid()) +
"_" + std::to_string(topFrag.sequenceID()) +
".bin";
232 std::fstream ostream(fileName.c_str(), std::ios::out | std::ios::binary);
237 TLOG(TLVL_TRACE) <<
"receiveMessage END" ;
245 TLOG(TLVL_TRACE) <<
"receiveInitMessage BEGIN" ;
246 if (recvd_fragments_ ==
nullptr)
248 TLOG(TLVL_TRACE) <<
"receiveInitMessage: Waiting for available buffer" ;
250 bool got_init =
false;
251 auto errflag =
false;
254 bool got_event =
false;
257 got_event = incoming_events_->ReadyForRead(
true);
260 TLOG(TLVL_TRACE) <<
"receiveInitMessage: Reading buffer header" ;
261 incoming_events_->ReadHeader(errflag);
263 TLOG(TLVL_ERROR) <<
"receiveInitMessage: Error receiving message!" ;
264 incoming_events_->ReleaseBuffer();
268 TLOG(TLVL_TRACE) <<
"receiveInitMessage: Getting Fragment types" ;
269 auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag);
271 incoming_events_->ReleaseBuffer();
273 TLOG(TLVL_ERROR) <<
"receiveInitMessage: Error receiving message!" ;
276 if (fragmentTypes.size() == 0)
278 TLOG(TLVL_ERROR) <<
"Event has no Fragments! Aborting!" ;
279 incoming_events_->ReleaseBuffer();
283 TLOG(TLVL_TRACE) <<
"receiveInitMessage: Checking first Fragment type" ;
284 auto firstFragmentType = *fragmentTypes.begin();
291 if (!got_event || firstFragmentType == artdaq::Fragment::EndOfDataFragmentType)
293 TLOG(TLVL_DEBUG) <<
"Received shutdown message, returning" ;
294 incoming_events_->ReleaseBuffer();
298 if (firstFragmentType != artdaq::Fragment::InitFragmentType)
300 TLOG(TLVL_WARNING) <<
"Did NOT receive Init Fragment as first broadcast! Type=" << artdaq::detail::RawFragmentHeader::SystemTypeToString(firstFragmentType) ;
301 incoming_events_->ReleaseBuffer();
305 TLOG(TLVL_TRACE) <<
"receiveInitMessage: Getting all Fragments" ;
306 recvd_fragments_ = incoming_events_->GetFragmentsByType(errflag, artdaq::Fragment::InvalidFragmentType);
310 std::sort(recvd_fragments_->begin(), recvd_fragments_->end(),
311 artdaq::fragmentSequenceIDCompare);
313 incoming_events_->ReleaseBuffer();
316 TLOG(TLVL_TRACE) <<
"receiveInitMessage: Returning top Fragment" ;
317 artdaq::Fragment topFrag = std::move(recvd_fragments_->at(0));
318 recvd_fragments_->erase(recvd_fragments_->begin());
319 if (recvd_fragments_->size() == 0)
321 recvd_fragments_.reset(
nullptr);
325 TLOG(TLVL_TRACE) <<
"receiveInitMessage: Copying Fragment into TBufferFile: message length: " << header->
data_length ;
326 auto buffer =
new char[header->data_length];
328 memcpy(buffer, &*topFrag.dataBegin(), header->data_length);
330 #if DUMP_RECEIVE_MESSAGE
331 std::string fileName =
"receiveInitMessage_" + std::to_string(getpid()) +
".bin";
332 std::fstream ostream(fileName.c_str(), std::ios::out | std::ios::binary);
333 ostream.write(buffer, header->data_length);
337 msg =
new TBufferFile(TBuffer::kRead, header->data_length, buffer, kTRUE, 0);
339 TLOG(TLVL_TRACE) <<
"receiveInitMessage END" ;
340 init_received_ =
true;
void receiveInitMessage(TBufferFile *&msg) override
Receive the init message.
Sends Fragment objects using TransferInterface plugins. Uses Routing Tables if confgiured, otherwise will Round-Robin Fragments to the destinations.
void sendMessage(uint64_t sequenceId, uint8_t messageType, TBufferFile &msg) override
Send ROOT data, wrapped in an artdaq::Fragment object.
NetMonTransportService extends NetMonTransportServiceInterface. It sends events using DataSenderManag...
void receiveMessage(TBufferFile *&msg) override
Receive data from the ConcurrentQueue.
void connect() override
Reconnect the NetMonTransportService.
virtual ~NetMonTransportService()
NetMonTransportService Destructor. Calls disconnect().
NetMonTransportService(fhicl::ParameterSet const &pset, art::ActivityRegistry &)
NetMonTransportService Constructor.
void disconnect() override
Disconnects the NetMonTranportService.
Interface for NetMonTranportService. This interface is declared to art as part of the required regist...
void listen() override
Listen for connections. This method is a No-Op.