1 #define TRACE_NAME "NetMonTransportService"
3 #include "artdaq-core/Core/SharedMemoryEventReceiver.hh"
4 #include "artdaq/ArtModules/NetMonTransportService.h"
5 #include "artdaq/DAQdata/Globals.hh"
6 #include "artdaq/DAQrate/DataSenderManager.hh"
8 #include "artdaq-core/Data/Fragment.hh"
9 #include "artdaq-core/Data/RawEvent.hh"
10 #include "artdaq-core/Utilities/TimeUtils.hh"
11 #include "artdaq/DAQdata/NetMonHeader.hh"
13 #include "art/Framework/Services/Registry/ActivityRegistry.h"
14 #include "canvas/Utilities/Exception.h"
15 #include "cetlib/container_algorithms.h"
16 #include "cetlib_except/exception.h"
17 #include "fhiclcpp/ParameterSet.h"
18 #include "fhiclcpp/ParameterSetRegistry.h"
20 #include <TBufferFile.h>
29 #define DUMP_SEND_MESSAGE 0
30 #define DUMP_RECEIVE_MESSAGE 0
32 #define build_key(seed) seed + ((GetPartitionNumber() + 1) << 16) + (getppid() & 0xFFFF)
34 static fhicl::ParameterSet empty_pset;
37 :
NetMonTransportServiceInterface(), data_pset_(pset), init_received_(false), sender_ptr_(nullptr), incoming_events_(nullptr), recvd_fragments_(nullptr)
39 TLOG(TLVL_TRACE) <<
"NetMonTransportService CONSTRUCTOR";
41 init_timeout_s_ = pset.get<
double>(
"init_fragment_timeout_seconds", 1.0);
48 auto start_time = std::chrono::steady_clock::now();
50 char const* artapp_env = getenv(
"ARTDAQ_RANK");
51 if (artapp_env != NULL && my_rank < 0)
52 my_rank = std::atoi(artapp_env);
54 while (my_rank == -1 && artdaq::TimeUtils::GetElapsedTime(start_time) < init_timeout_s_)
63 TLOG(TLVL_TRACE) <<
"listen() start";
64 if (!incoming_events_)
66 incoming_events_.reset(
new artdaq::SharedMemoryEventReceiver(
67 data_pset_.get<
int>(
"shared_memory_key", build_key(0xEE000000)),
68 data_pset_.get<
int>(
"broadcast_shared_memory_key", build_key(0xBB000000))));
70 char const* artapp_env = getenv(
"ARTDAQ_APPLICATION_NAME");
71 std::string artapp_str =
"";
72 if (artapp_env != NULL)
74 artapp_str = std::string(artapp_env) +
"_";
77 TLOG(TLVL_TRACE) <<
"Setting app_name";
78 app_name = artapp_str +
"art" + std::to_string(incoming_events_->GetMyId());
80 artapp_env = getenv(
"ARTDAQ_RANK");
81 if (artapp_env != NULL && my_rank < 0)
83 TLOG(TLVL_TRACE) <<
"Setting rank from envrionment";
84 my_rank = std::atoi(artapp_env);
88 TLOG(TLVL_TRACE) <<
"Setting my_rank from shared memory";
89 my_rank = incoming_events_->GetRank();
92 TLOG(TLVL_INFO) <<
"app_name is " << app_name <<
", rank " << my_rank;
94 TLOG(TLVL_TRACE) <<
"listen() end";
100 if (sender_ptr_) sender_ptr_.reset(
nullptr);
105 if (sender_ptr_ ==
nullptr)
107 TLOG(TLVL_DEBUG) <<
"Reconnecting DataSenderManager";
111 #if DUMP_SEND_MESSAGE
112 std::string fileName =
"sendMessage_" + std::to_string(my_rank) +
"_" + std::to_string(getpid()) +
"_" +
113 std::to_string(sequenceId) +
".bin";
114 std::fstream ostream(fileName, std::ios::out | std::ios::binary);
115 ostream.write(msg.Buffer(), msg.Length());
119 TLOG(TLVL_DEBUG) <<
"Sending message with sequenceID=" << sequenceId <<
", type=" << (int)messageType
120 <<
", length=" << msg.Length();
122 header.
data_length =
static_cast<uint64_t
>(msg.Length());
123 artdaq::Fragment fragment(std::ceil(msg.Length() /
static_cast<double>(
sizeof(artdaq::RawDataType))), sequenceId, 0,
124 messageType, header);
126 memcpy(&*fragment.dataBegin(), msg.Buffer(), msg.Length());
127 sender_ptr_->sendFragment(std::move(fragment));
129 sender_ptr_->RemoveRoutingTableEntry(sequenceId);
135 TLOG(TLVL_TRACE) <<
"receiveMessage BEGIN";
136 while (recvd_fragments_ ==
nullptr)
138 TLOG(TLVL_TRACE) <<
"receiveMessage: Waiting for available buffer";
139 bool keep_looping =
true;
140 bool got_event =
false;
143 keep_looping =
false;
144 got_event = incoming_events_->ReadyForRead();
151 TLOG(TLVL_TRACE) <<
"receiveMessage: Reading buffer header";
152 auto errflag =
false;
153 incoming_events_->ReadHeader(errflag);
159 TLOG(TLVL_TRACE) <<
"receiveMessage: Getting Fragment types";
160 auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag);
163 incoming_events_->ReleaseBuffer();
167 if (fragmentTypes.size() == 0)
169 TLOG(TLVL_ERROR) <<
"Event has no Fragments! Aborting!";
170 incoming_events_->ReleaseBuffer();
174 TLOG(TLVL_TRACE) <<
"receiveMessage: Checking first Fragment type";
175 auto firstFragmentType = *fragmentTypes.begin();
182 if (!got_event || firstFragmentType == artdaq::Fragment::EndOfDataFragmentType)
184 TLOG(TLVL_DEBUG) <<
"Received shutdown message, returning from receiveMessage "
185 <<
"(debug: got_event=" << got_event <<
",fragType=" << (int)firstFragmentType
186 <<
",EODFragType=" << (
int)artdaq::Fragment::EndOfDataFragmentType <<
")";
187 incoming_events_->ReleaseBuffer();
191 if (firstFragmentType == artdaq::Fragment::InitFragmentType)
193 TLOG(TLVL_DEBUG) <<
"Cannot receive InitFragments here, retrying";
194 incoming_events_->ReleaseBuffer();
198 else if (firstFragmentType == artdaq::Fragment::EndOfRunFragmentType ||
199 firstFragmentType == artdaq::Fragment::EndOfSubrunFragmentType)
201 TLOG(TLVL_DEBUG) <<
"Ignoring EndOfRun or EndOfSubrun Fragment";
202 incoming_events_->ReleaseBuffer();
206 TLOG(TLVL_TRACE) <<
"receiveMessage: Getting all Fragments";
207 recvd_fragments_ = incoming_events_->GetFragmentsByType(errflag, artdaq::Fragment::InvalidFragmentType);
208 if (!recvd_fragments_)
210 TLOG(TLVL_ERROR) <<
"Error retrieving Fragments from shared memory! Aborting!";
211 incoming_events_->ReleaseBuffer();
218 std::sort(recvd_fragments_->begin(), recvd_fragments_->end(), artdaq::fragmentSequenceIDCompare);
220 TLOG(TLVL_TRACE) <<
"receiveMessage: Releasing buffer";
221 incoming_events_->ReleaseBuffer();
225 auto start = std::chrono::steady_clock::now();
226 while (!init_received_ && artdaq::TimeUtils::GetElapsedTime(start) < init_timeout_s_)
228 usleep(init_timeout_s_ * 1000000 / 100);
232 TLOG(TLVL_ERROR) <<
"Received data but no Init Fragment after " << init_timeout_s_ <<
" seconds. Art will crash.";
235 TLOG(TLVL_TRACE) <<
"receiveMessage: Returning top Fragment";
236 artdaq::Fragment topFrag = std::move(recvd_fragments_->at(0));
237 recvd_fragments_->erase(recvd_fragments_->begin());
238 if (recvd_fragments_->size() == 0)
240 recvd_fragments_.reset(
nullptr);
243 TLOG(TLVL_TRACE) <<
"receiveMessage: Copying Fragment into TBufferFile, length="
246 auto buffer =
static_cast<char*
>(malloc(header->
data_length));
247 memcpy(buffer, &*topFrag.dataBegin(), header->
data_length);
248 msg =
new TBufferFile(TBuffer::kRead, header->
data_length, buffer, kTRUE, 0);
250 #if DUMP_RECEIVE_MESSAGE
251 std::string fileName =
"receiveMessage_" + std::to_string(my_rank) +
"_" + std::to_string(getpid()) +
"_" +
252 std::to_string(topFrag.sequenceID()) +
".bin";
253 std::fstream ostream(fileName.c_str(), std::ios::out | std::ios::binary);
258 TLOG(TLVL_TRACE) <<
"receiveMessage END";
264 TLOG(TLVL_TRACE) <<
"receiveInitMessage BEGIN";
265 if (recvd_fragments_ ==
nullptr)
267 TLOG(TLVL_TRACE) <<
"receiveInitMessage: Waiting for available buffer";
269 bool got_init =
false;
270 auto errflag =
false;
273 bool got_event =
false;
276 got_event = incoming_events_->ReadyForRead(
true);
279 TLOG(TLVL_TRACE) <<
"receiveInitMessage: Reading buffer header";
280 incoming_events_->ReadHeader(errflag);
283 TLOG(TLVL_ERROR) <<
"receiveInitMessage: Error receiving message!";
284 incoming_events_->ReleaseBuffer();
288 TLOG(TLVL_TRACE) <<
"receiveInitMessage: Getting Fragment types";
289 auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag);
292 incoming_events_->ReleaseBuffer();
294 TLOG(TLVL_ERROR) <<
"receiveInitMessage: Error receiving message!";
297 if (fragmentTypes.size() == 0)
299 TLOG(TLVL_ERROR) <<
"Event has no Fragments! Aborting!";
300 incoming_events_->ReleaseBuffer();
304 TLOG(TLVL_TRACE) <<
"receiveInitMessage: Checking first Fragment type";
305 auto firstFragmentType = *fragmentTypes.begin();
312 if (!got_event || firstFragmentType == artdaq::Fragment::EndOfDataFragmentType)
314 TLOG(TLVL_DEBUG) <<
"Received shutdown message, returning";
315 incoming_events_->ReleaseBuffer();
319 if (firstFragmentType != artdaq::Fragment::InitFragmentType)
321 TLOG(TLVL_WARNING) <<
"Did NOT receive Init Fragment as first broadcast! Type="
322 << artdaq::detail::RawFragmentHeader::SystemTypeToString(firstFragmentType);
323 incoming_events_->ReleaseBuffer();
327 TLOG(TLVL_TRACE) <<
"receiveInitMessage: Getting all Fragments";
328 recvd_fragments_ = incoming_events_->GetFragmentsByType(errflag, artdaq::Fragment::InvalidFragmentType);
332 std::sort(recvd_fragments_->begin(), recvd_fragments_->end(), artdaq::fragmentSequenceIDCompare);
334 incoming_events_->ReleaseBuffer();
337 TLOG(TLVL_TRACE) <<
"receiveInitMessage: Returning top Fragment";
338 artdaq::Fragment topFrag = std::move(recvd_fragments_->at(0));
339 recvd_fragments_->erase(recvd_fragments_->begin());
340 if (recvd_fragments_->size() == 0)
342 recvd_fragments_.reset(
nullptr);
346 TLOG(TLVL_TRACE) <<
"receiveInitMessage: Copying Fragment into TBufferFile: message length: " << header->
data_length;
347 auto buffer =
new char[header->data_length];
349 memcpy(buffer, &*topFrag.dataBegin(), header->data_length);
351 #if DUMP_RECEIVE_MESSAGE
352 std::string fileName =
"receiveInitMessage_" + std::to_string(getpid()) +
".bin";
353 std::fstream ostream(fileName.c_str(), std::ios::out | std::ios::binary);
354 ostream.write(buffer, header->data_length);
358 msg =
new TBufferFile(TBuffer::kRead, header->data_length, buffer, kTRUE, 0);
360 TLOG(TLVL_TRACE) <<
"receiveInitMessage END";
361 init_received_ =
true;
void receiveInitMessage(TBufferFile *&msg) override
Receive the init message.
Sends Fragment objects using TransferInterface plugins. Uses Routing Tables if confgiured, otherwise will Round-Robin Fragments to the destinations.
void sendMessage(uint64_t sequenceId, uint8_t messageType, TBufferFile &msg) override
Send ROOT data, wrapped in an artdaq::Fragment object.
NetMonTransportService extends NetMonTransportServiceInterface. It sends events using DataSenderManag...
void receiveMessage(TBufferFile *&msg) override
Receive data from the ConcurrentQueue.
void connect() override
Reconnect the NetMonTransportService.
virtual ~NetMonTransportService()
NetMonTransportService Destructor. Calls disconnect().
NetMonTransportService(fhicl::ParameterSet const &pset, art::ActivityRegistry &)
NetMonTransportService Constructor.
void disconnect() override
Disconnects the NetMonTranportService.
Interface for NetMonTranportService. This interface is declared to art as part of the required regist...
void listen() override
Listen for connections. This method is a No-Op.