1 #define TRACE_NAME "NetMonTransportService"
3 #include "artdaq/DAQdata/Globals.hh"
4 #include "artdaq/ArtModules/NetMonTransportService.h"
5 #include "artdaq/DAQrate/DataSenderManager.hh"
6 #include "artdaq-core/Core/SharedMemoryEventReceiver.hh"
8 #include "artdaq-core/Data/Fragment.hh"
9 #include "artdaq/DAQdata/NetMonHeader.hh"
10 #include "artdaq-core/Data/RawEvent.hh"
11 #include "artdaq-core/Utilities/TimeUtils.hh"
13 #include "art/Framework/Services/Registry/ActivityRegistry.h"
14 #include "canvas/Utilities/Exception.h"
15 #include "cetlib/container_algorithms.h"
16 #include "cetlib_except/exception.h"
17 #include "fhiclcpp/ParameterSet.h"
18 #include "fhiclcpp/ParameterSetRegistry.h"
21 #include <TBufferFile.h>
30 #define DUMP_SEND_MESSAGE 0
31 #define DUMP_RECEIVE_MESSAGE 0
33 static fhicl::ParameterSet empty_pset;
40 , init_received_(false)
41 , sender_ptr_(nullptr)
42 , incoming_events_(nullptr)
43 , recvd_fragments_(nullptr)
45 TLOG(TLVL_TRACE) <<
"NetMonTransportService CONSTRUCTOR" ;
46 if (pset.has_key(
"rank")) my_rank = pset.get<
int>(
"rank");
48 init_timeout_s_ = pset.get<
double>(
"init_fragment_timeout_seconds", 1.0);
68 if (!incoming_events_)
70 incoming_events_.reset(
new artdaq::SharedMemoryEventReceiver(data_pset_.get<
int>(
"shared_memory_key", 0xBEE70000 + getppid()), data_pset_.get<
int>(
"broadcast_shared_memory_key", 0xCEE70000 + getppid())));
72 char const* artapp_env = getenv(
"ARTDAQ_APPLICATION_NAME");
73 std::string artapp_str =
"";
74 if (artapp_env != NULL)
76 artapp_str = std::string(artapp_env) +
"_";
79 app_name = artapp_str +
"art" + std::to_string(incoming_events_->GetMyId());
81 if (data_pset_.has_key(
"rank")) my_rank = data_pset_.get<
int>(
"rank");
82 else my_rank = incoming_events_->GetRank();
91 if (sender_ptr_) sender_ptr_.reset(
nullptr);
96 sendMessage(uint64_t sequenceId, uint8_t messageType, TBufferFile& msg)
98 if (sender_ptr_ ==
nullptr)
100 TLOG(TLVL_DEBUG) <<
"Reconnecting DataSenderManager" ;
104 #if DUMP_SEND_MESSAGE
105 std::string fileName =
"sendMessage_" + std::to_string(my_rank) +
"_" + std::to_string(getpid()) +
"_" + std::to_string(sequenceId) +
".bin";
106 std::fstream ostream(fileName, std::ios::out | std::ios::binary);
107 ostream.write(msg.Buffer(), msg.Length());
111 TLOG(TLVL_DEBUG) <<
"Sending message with sequenceID=" << sequenceId <<
", type=" << (int)messageType <<
", length=" << msg.Length() ;
113 header.
data_length =
static_cast<uint64_t
>(msg.Length());
115 fragment(std::ceil(msg.Length() /
116 static_cast<double>(
sizeof(artdaq::RawDataType))),
117 sequenceId, 0, messageType, header);
119 memcpy(&*fragment.dataBegin(), msg.Buffer(), msg.Length());
120 sender_ptr_->sendFragment(std::move(fragment));
128 TLOG(TLVL_TRACE) <<
"receiveMessage BEGIN" ;
129 while (recvd_fragments_ ==
nullptr)
131 TLOG(TLVL_TRACE) <<
"receiveMessage: Waiting for available buffer" ;
132 bool keep_looping =
true;
133 bool got_event =
false;
136 keep_looping =
false;
137 got_event = incoming_events_->ReadyForRead();
144 TLOG(TLVL_TRACE) <<
"receiveMessage: Reading buffer header" ;
145 auto errflag =
false;
146 incoming_events_->ReadHeader(errflag);
151 TLOG(TLVL_TRACE) <<
"receiveMessage: Getting Fragment types" ;
152 auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag);
154 incoming_events_->ReleaseBuffer();
158 if (fragmentTypes.size() == 0)
160 TLOG(TLVL_ERROR) <<
"Event has no Fragments! Aborting!" ;
161 incoming_events_->ReleaseBuffer();
165 TLOG(TLVL_TRACE) <<
"receiveMessage: Checking first Fragment type" ;
166 auto firstFragmentType = *fragmentTypes.begin();
173 if (!got_event || firstFragmentType == artdaq::Fragment::EndOfDataFragmentType)
175 TLOG(TLVL_DEBUG) <<
"Received shutdown message, returning from receiveMessage "
176 <<
"(debug: got_event=" << got_event <<
",fragType=" << (int)firstFragmentType
177 <<
",EODFragType=" << (
int)artdaq::Fragment::EndOfDataFragmentType <<
")";
178 incoming_events_->ReleaseBuffer();
182 if (firstFragmentType == artdaq::Fragment::InitFragmentType)
184 TLOG(TLVL_DEBUG) <<
"Cannot receive InitFragments here, retrying" ;
185 incoming_events_->ReleaseBuffer();
189 else if (firstFragmentType == artdaq::Fragment::EndOfRunFragmentType || firstFragmentType == artdaq::Fragment::EndOfSubrunFragmentType)
191 TLOG(TLVL_DEBUG) <<
"Ignoring EndOfRun or EndOfSubrun Fragment" ;
192 incoming_events_->ReleaseBuffer();
196 TLOG(TLVL_TRACE) <<
"receiveMessage: Getting all Fragments" ;
197 recvd_fragments_ = incoming_events_->GetFragmentsByType(errflag, artdaq::Fragment::InvalidFragmentType);
198 if (!recvd_fragments_)
200 TLOG(TLVL_ERROR) <<
"Error retrieving Fragments from shared memory! Aborting!";
201 incoming_events_->ReleaseBuffer();
209 std::sort(recvd_fragments_->begin(), recvd_fragments_->end(),
210 artdaq::fragmentSequenceIDCompare);
212 TLOG(TLVL_TRACE) <<
"receiveMessage: Releasing buffer" ;
213 incoming_events_->ReleaseBuffer();
217 auto start = std::chrono::steady_clock::now();
218 while (!init_received_ && artdaq::TimeUtils::GetElapsedTime(start) < init_timeout_s_)
220 usleep(init_timeout_s_ * 1000000 / 100);
222 if (!init_received_) {
223 TLOG(TLVL_ERROR) <<
"Received data but no Init Fragment after " << init_timeout_s_ <<
" seconds. Art will crash." ;
226 TLOG(TLVL_TRACE) <<
"receiveMessage: Returning top Fragment" ;
227 artdaq::Fragment topFrag = std::move(recvd_fragments_->at(0));
228 recvd_fragments_->erase(recvd_fragments_->begin());
229 if (recvd_fragments_->size() == 0)
231 recvd_fragments_.reset(
nullptr);
234 TLOG(TLVL_TRACE) <<
"receiveMessage: Copying Fragment into TBufferFile, length=" << topFrag.metadata<
artdaq::NetMonHeader>()->data_length ;
236 auto buffer =
static_cast<char *
>(malloc(header->
data_length));
237 memcpy(buffer, &*topFrag.dataBegin(), header->
data_length);
238 msg =
new TBufferFile(TBuffer::kRead, header->
data_length, buffer, kTRUE, 0);
240 #if DUMP_RECEIVE_MESSAGE
241 std::string fileName =
"receiveMessage_" + std::to_string(my_rank) +
"_" + std::to_string(getpid()) +
"_" + std::to_string(topFrag.sequenceID()) +
".bin";
242 std::fstream ostream(fileName.c_str(), std::ios::out | std::ios::binary);
247 TLOG(TLVL_TRACE) <<
"receiveMessage END" ;
255 TLOG(TLVL_TRACE) <<
"receiveInitMessage BEGIN" ;
256 if (recvd_fragments_ ==
nullptr)
258 TLOG(TLVL_TRACE) <<
"receiveInitMessage: Waiting for available buffer" ;
260 bool got_init =
false;
261 auto errflag =
false;
264 bool got_event =
false;
267 got_event = incoming_events_->ReadyForRead(
true);
270 TLOG(TLVL_TRACE) <<
"receiveInitMessage: Reading buffer header" ;
271 incoming_events_->ReadHeader(errflag);
273 TLOG(TLVL_ERROR) <<
"receiveInitMessage: Error receiving message!" ;
274 incoming_events_->ReleaseBuffer();
278 TLOG(TLVL_TRACE) <<
"receiveInitMessage: Getting Fragment types" ;
279 auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag);
281 incoming_events_->ReleaseBuffer();
283 TLOG(TLVL_ERROR) <<
"receiveInitMessage: Error receiving message!" ;
286 if (fragmentTypes.size() == 0)
288 TLOG(TLVL_ERROR) <<
"Event has no Fragments! Aborting!" ;
289 incoming_events_->ReleaseBuffer();
293 TLOG(TLVL_TRACE) <<
"receiveInitMessage: Checking first Fragment type" ;
294 auto firstFragmentType = *fragmentTypes.begin();
301 if (!got_event || firstFragmentType == artdaq::Fragment::EndOfDataFragmentType)
303 TLOG(TLVL_DEBUG) <<
"Received shutdown message, returning" ;
304 incoming_events_->ReleaseBuffer();
308 if (firstFragmentType != artdaq::Fragment::InitFragmentType)
310 TLOG(TLVL_WARNING) <<
"Did NOT receive Init Fragment as first broadcast! Type=" << artdaq::detail::RawFragmentHeader::SystemTypeToString(firstFragmentType) ;
311 incoming_events_->ReleaseBuffer();
315 TLOG(TLVL_TRACE) <<
"receiveInitMessage: Getting all Fragments" ;
316 recvd_fragments_ = incoming_events_->GetFragmentsByType(errflag, artdaq::Fragment::InvalidFragmentType);
320 std::sort(recvd_fragments_->begin(), recvd_fragments_->end(),
321 artdaq::fragmentSequenceIDCompare);
323 incoming_events_->ReleaseBuffer();
326 TLOG(TLVL_TRACE) <<
"receiveInitMessage: Returning top Fragment" ;
327 artdaq::Fragment topFrag = std::move(recvd_fragments_->at(0));
328 recvd_fragments_->erase(recvd_fragments_->begin());
329 if (recvd_fragments_->size() == 0)
331 recvd_fragments_.reset(
nullptr);
335 TLOG(TLVL_TRACE) <<
"receiveInitMessage: Copying Fragment into TBufferFile: message length: " << header->
data_length ;
336 auto buffer =
new char[header->data_length];
338 memcpy(buffer, &*topFrag.dataBegin(), header->data_length);
340 #if DUMP_RECEIVE_MESSAGE
341 std::string fileName =
"receiveInitMessage_" + std::to_string(getpid()) +
".bin";
342 std::fstream ostream(fileName.c_str(), std::ios::out | std::ios::binary);
343 ostream.write(buffer, header->data_length);
347 msg =
new TBufferFile(TBuffer::kRead, header->data_length, buffer, kTRUE, 0);
349 TLOG(TLVL_TRACE) <<
"receiveInitMessage END" ;
350 init_received_ =
true;
void receiveInitMessage(TBufferFile *&msg) override
Receive the init message.
Sends Fragment objects using TransferInterface plugins. Uses Routing Tables if confgiured, otherwise will Round-Robin Fragments to the destinations.
void sendMessage(uint64_t sequenceId, uint8_t messageType, TBufferFile &msg) override
Send ROOT data, wrapped in an artdaq::Fragment object.
NetMonTransportService extends NetMonTransportServiceInterface. It sends events using DataSenderManag...
void receiveMessage(TBufferFile *&msg) override
Receive data from the ConcurrentQueue.
void connect() override
Reconnect the NetMonTransportService.
virtual ~NetMonTransportService()
NetMonTransportService Destructor. Calls disconnect().
NetMonTransportService(fhicl::ParameterSet const &pset, art::ActivityRegistry &)
NetMonTransportService Constructor.
void disconnect() override
Disconnects the NetMonTranportService.
Interface for NetMonTranportService. This interface is declared to art as part of the required regist...
void listen() override
Listen for connections. This method is a No-Op.