1 #define TRACE_NAME "NetMonTransportService"
3 #include "artdaq/DAQdata/Globals.hh"
4 #include "artdaq/ArtModules/NetMonTransportService.h"
5 #include "artdaq/DAQrate/DataSenderManager.hh"
6 #include "artdaq-core/Core/SharedMemoryEventReceiver.hh"
8 #include "artdaq-core/Data/Fragment.hh"
9 #include "artdaq/DAQdata/NetMonHeader.hh"
10 #include "artdaq-core/Data/RawEvent.hh"
11 #include "artdaq-core/Utilities/TimeUtils.hh"
13 #include "art/Framework/Services/Registry/ActivityRegistry.h"
14 #include "canvas/Utilities/Exception.h"
15 #include "cetlib/container_algorithms.h"
16 #include "cetlib_except/exception.h"
17 #include "fhiclcpp/ParameterSet.h"
18 #include "fhiclcpp/ParameterSetRegistry.h"
21 #include <TBufferFile.h>
30 #define DUMP_SEND_MESSAGE 0
31 #define DUMP_RECEIVE_MESSAGE 0
33 static fhicl::ParameterSet empty_pset;
40 , init_received_(false)
41 , sender_ptr_(nullptr)
42 , incoming_events_(new artdaq::SharedMemoryEventReceiver(pset.get<int>(
"shared_memory_key", 0xBEE70000 + getppid()), pset.get<int>(
"broadcast_shared_memory_key", 0xCEE70000 + getppid())))
43 , recvd_fragments_(nullptr)
45 TLOG(TLVL_TRACE) <<
"NetMonTransportService CONSTRUCTOR" ;
46 if (pset.has_key(
"rank")) my_rank = pset.get<
int>(
"rank");
47 else my_rank = incoming_events_->GetRank();
49 init_timeout_s_ = pset.get<
double>(
"init_fragment_timeout_seconds", 1.0);
76 if (sender_ptr_) sender_ptr_.reset(
nullptr);
81 sendMessage(uint64_t sequenceId, uint8_t messageType, TBufferFile& msg)
83 if (sender_ptr_ ==
nullptr)
85 TLOG(TLVL_DEBUG) <<
"Reconnecting DataSenderManager" ;
90 std::string fileName =
"sendMessage_" + std::to_string(my_rank) +
"_" + std::to_string(getpid()) +
"_" + std::to_string(sequenceId) +
".bin";
91 std::fstream ostream(fileName, std::ios::out | std::ios::binary);
92 ostream.write(msg.Buffer(), msg.Length());
96 TLOG(TLVL_DEBUG) <<
"Sending message with sequenceID=" << sequenceId <<
", type=" << (int)messageType <<
", length=" << msg.Length() ;
98 header.
data_length =
static_cast<uint64_t
>(msg.Length());
100 fragment(std::ceil(msg.Length() /
101 static_cast<double>(
sizeof(artdaq::RawDataType))),
102 sequenceId, 0, messageType, header);
104 memcpy(&*fragment.dataBegin(), msg.Buffer(), msg.Length());
105 sender_ptr_->sendFragment(std::move(fragment));
112 TLOG(TLVL_TRACE) <<
"receiveMessage BEGIN" ;
113 while (recvd_fragments_ ==
nullptr)
115 TLOG(TLVL_TRACE) <<
"receiveMessage: Waiting for available buffer" ;
116 bool keep_looping =
true;
117 bool got_event =
false;
120 keep_looping =
false;
121 got_event = incoming_events_->ReadyForRead();
128 TLOG(TLVL_TRACE) <<
"receiveMessage: Reading buffer header" ;
129 auto errflag =
false;
130 incoming_events_->ReadHeader(errflag);
135 TLOG(TLVL_TRACE) <<
"receiveMessage: Getting Fragment types" ;
136 auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag);
141 if (fragmentTypes.size() == 0)
143 TLOG(TLVL_ERROR) <<
"Event has no Fragments! Aborting!" ;
144 incoming_events_->ReleaseBuffer();
148 TLOG(TLVL_TRACE) <<
"receiveMessage: Checking first Fragment type" ;
149 auto firstFragmentType = *fragmentTypes.begin();
156 if (!got_event || firstFragmentType == artdaq::Fragment::EndOfDataFragmentType)
158 TLOG(TLVL_DEBUG) <<
"Received shutdown message, returning" ;
159 incoming_events_->ReleaseBuffer();
163 if (firstFragmentType == artdaq::Fragment::InitFragmentType)
165 TLOG(TLVL_DEBUG) <<
"Cannot receive InitFragments here, retrying" ;
166 incoming_events_->ReleaseBuffer();
170 else if (firstFragmentType == artdaq::Fragment::EndOfRunFragmentType || firstFragmentType == artdaq::Fragment::EndOfSubrunFragmentType)
172 TLOG(TLVL_DEBUG) <<
"Ignoring EndOfRun or EndOfSubrun Fragment" ;
173 incoming_events_->ReleaseBuffer();
177 TLOG(TLVL_TRACE) <<
"receiveMessage: Getting all Fragments" ;
178 recvd_fragments_ = incoming_events_->GetFragmentsByType(errflag, artdaq::Fragment::InvalidFragmentType);
182 std::sort(recvd_fragments_->begin(), recvd_fragments_->end(),
183 artdaq::fragmentSequenceIDCompare);
185 TLOG(TLVL_TRACE) <<
"receiveMessage: Releasing buffer" ;
186 incoming_events_->ReleaseBuffer();
190 auto start = std::chrono::steady_clock::now();
191 while (!init_received_ && artdaq::TimeUtils::GetElapsedTime(start) < init_timeout_s_)
193 usleep(init_timeout_s_ * 1000000 / 100);
195 if (!init_received_) {
196 TLOG(TLVL_ERROR) <<
"Received data but no Init Fragment after " << init_timeout_s_ <<
" seconds. Art will crash." ;
199 TLOG(TLVL_TRACE) <<
"receiveMessage: Returning top Fragment" ;
200 artdaq::Fragment topFrag = std::move(recvd_fragments_->at(0));
201 recvd_fragments_->erase(recvd_fragments_->begin());
202 if (recvd_fragments_->size() == 0)
204 recvd_fragments_.reset(
nullptr);
207 TLOG(TLVL_TRACE) <<
"receiveMessage: Copying Fragment into TBufferFile, length=" << topFrag.metadata<
artdaq::NetMonHeader>()->data_length ;
209 auto buffer =
static_cast<char *
>(malloc(header->
data_length));
210 memcpy(buffer, &*topFrag.dataBegin(), header->
data_length);
211 msg =
new TBufferFile(TBuffer::kRead, header->
data_length, buffer, kTRUE, 0);
213 #if DUMP_RECEIVE_MESSAGE
214 std::string fileName =
"receiveMessage_" + std::to_string(my_rank) +
"_" + std::to_string(getpid()) +
"_" + std::to_string(topFrag.sequenceID()) +
".bin";
215 std::fstream ostream(fileName.c_str(), std::ios::out | std::ios::binary);
220 TLOG(TLVL_TRACE) <<
"receiveMessage END" ;
227 TLOG(TLVL_TRACE) <<
"receiveInitMessage BEGIN" ;
228 if (recvd_fragments_ ==
nullptr)
230 TLOG(TLVL_TRACE) <<
"receiveInitMessage: Waiting for available buffer" ;
232 bool got_init =
false;
233 auto errflag =
false;
236 bool got_event =
false;
239 got_event = incoming_events_->ReadyForRead(
true);
242 TLOG(TLVL_TRACE) <<
"receiveInitMessage: Reading buffer header" ;
243 incoming_events_->ReadHeader(errflag);
245 TLOG(TLVL_ERROR) <<
"receiveInitMessage: Error receiving message!" ;
249 TLOG(TLVL_TRACE) <<
"receiveInitMessage: Getting Fragment types" ;
250 auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag);
253 TLOG(TLVL_ERROR) <<
"receiveInitMessage: Error receiving message!" ;
256 if (fragmentTypes.size() == 0)
258 TLOG(TLVL_ERROR) <<
"Event has no Fragments! Aborting!" ;
259 incoming_events_->ReleaseBuffer();
263 TLOG(TLVL_TRACE) <<
"receiveInitMessage: Checking first Fragment type" ;
264 auto firstFragmentType = *fragmentTypes.begin();
271 if (!got_event || firstFragmentType == artdaq::Fragment::EndOfDataFragmentType)
273 TLOG(TLVL_DEBUG) <<
"Received shutdown message, returning" ;
274 incoming_events_->ReleaseBuffer();
278 if (firstFragmentType != artdaq::Fragment::InitFragmentType)
280 TLOG(TLVL_WARNING) <<
"Did NOT receive Init Fragment as first broadcast! Type=" << artdaq::detail::RawFragmentHeader::SystemTypeToString(firstFragmentType) ;
281 incoming_events_->ReleaseBuffer();
285 TLOG(TLVL_TRACE) <<
"receiveInitMessage: Getting all Fragments" ;
286 recvd_fragments_ = incoming_events_->GetFragmentsByType(errflag, artdaq::Fragment::InvalidFragmentType);
290 std::sort(recvd_fragments_->begin(), recvd_fragments_->end(),
291 artdaq::fragmentSequenceIDCompare);
294 TLOG(TLVL_TRACE) <<
"receiveInitMessage: Returning top Fragment" ;
295 artdaq::Fragment topFrag = std::move(recvd_fragments_->at(0));
296 recvd_fragments_->erase(recvd_fragments_->begin());
297 if (recvd_fragments_->size() == 0)
299 recvd_fragments_.reset(
nullptr);
303 TLOG(TLVL_TRACE) <<
"receiveInitMessage: Copying Fragment into TBufferFile: message length: " << header->
data_length ;
304 auto buffer =
static_cast<char *
>(malloc(header->data_length));
305 memcpy(buffer, &*topFrag.dataBegin(), header->data_length);
307 #if DUMP_RECEIVE_MESSAGE
308 std::string fileName =
"receiveInitMessage_" + std::to_string(getpid()) +
".bin";
309 std::fstream ostream(fileName.c_str(), std::ios::out | std::ios::binary);
310 ostream.write(buffer, header->data_length);
314 msg =
new TBufferFile(TBuffer::kRead, header->data_length, buffer, kTRUE, 0);
316 TLOG(TLVL_TRACE) <<
"receiveInitMessage END" ;
317 init_received_ =
true;
void receiveInitMessage(TBufferFile *&msg) override
Receive the init message.
Sends Fragment objects using TransferInterface plugins. Uses Routing Tables if confgiured, otherwise will Round-Robin Fragments to the destinations.
void sendMessage(uint64_t sequenceId, uint8_t messageType, TBufferFile &msg) override
Send ROOT data, wrapped in an artdaq::Fragment object.
NetMonTransportService extends NetMonTransportServiceInterface. It sends events using DataSenderManag...
void receiveMessage(TBufferFile *&msg) override
Receive data from the ConcurrentQueue.
void connect() override
Reconnect the NetMonTransportService.
virtual ~NetMonTransportService()
NetMonTransportService Destructor. Calls disconnect().
NetMonTransportService(fhicl::ParameterSet const &pset, art::ActivityRegistry &)
NetMonTransportService Constructor.
void disconnect() override
Disconnects the NetMonTranportService.
Interface for NetMonTranportService. This interface is declared to art as part of the required regist...
void listen() override
Listen for connections. This method is a No-Op.