1 #define TRACE_NAME "NetMonTransportService"
3 #include "artdaq-core/Core/SharedMemoryEventReceiver.hh"
4 #include "artdaq/ArtModules/NetMonTransportService.h"
5 #include "artdaq/DAQdata/Globals.hh"
6 #include "artdaq/DAQrate/DataSenderManager.hh"
8 #include "artdaq-core/Data/Fragment.hh"
9 #include "artdaq-core/Data/RawEvent.hh"
10 #include "artdaq-core/Utilities/TimeUtils.hh"
11 #include "artdaq/DAQdata/NetMonHeader.hh"
13 #include "art/Framework/Services/Registry/ActivityRegistry.h"
14 #include "canvas/Utilities/Exception.h"
15 #include "cetlib/container_algorithms.h"
16 #include "cetlib_except/exception.h"
17 #include "fhiclcpp/ParameterSet.h"
18 #include "fhiclcpp/ParameterSetRegistry.h"
20 #include <TBufferFile.h>
29 #define DUMP_SEND_MESSAGE 0
30 #define DUMP_RECEIVE_MESSAGE 0
32 static fhicl::ParameterSet empty_pset;
35 :
NetMonTransportServiceInterface(), data_pset_(pset), init_received_(false), sender_ptr_(nullptr), incoming_events_(nullptr), recvd_fragments_(nullptr)
37 TLOG(TLVL_TRACE) <<
"NetMonTransportService CONSTRUCTOR";
39 init_timeout_s_ = pset.get<
double>(
"init_fragment_timeout_seconds", 1.0);
46 auto start_time = std::chrono::steady_clock::now();
48 char const* artapp_env = getenv(
"ARTDAQ_RANK");
49 if (artapp_env != NULL && my_rank < 0)
50 my_rank = std::atoi(artapp_env);
52 while (my_rank == -1 && artdaq::TimeUtils::GetElapsedTime(start_time) < init_timeout_s_)
61 TLOG(TLVL_INFO) <<
"listen() start";
62 if (!incoming_events_)
64 incoming_events_.reset(
new artdaq::SharedMemoryEventReceiver(
65 data_pset_.get<
int>(
"shared_memory_key", 0xBEE70000 + getppid()),
66 data_pset_.get<
int>(
"broadcast_shared_memory_key", 0xCEE70000 + getppid())));
68 char const* artapp_env = getenv(
"ARTDAQ_APPLICATION_NAME");
69 std::string artapp_str =
"";
70 if (artapp_env != NULL)
72 artapp_str = std::string(artapp_env) +
"_";
75 TLOG(TLVL_TRACE) <<
"Setting app_name";
76 app_name = artapp_str +
"art" + std::to_string(incoming_events_->GetMyId());
78 artapp_env = getenv(
"ARTDAQ_RANK");
79 if (artapp_env != NULL && my_rank < 0)
81 TLOG(TLVL_TRACE) <<
"Setting rank from envrionment";
82 my_rank = std::atoi(artapp_env);
86 TLOG(TLVL_TRACE) <<
"Setting my_rank from shared memory";
87 my_rank = incoming_events_->GetRank();
90 TLOG(TLVL_INFO) <<
"app_name is " << app_name <<
", rank " << my_rank;
92 TLOG(TLVL_INFO) <<
"listen() end";
98 if (sender_ptr_) sender_ptr_.reset(
nullptr);
103 if (sender_ptr_ ==
nullptr)
105 TLOG(TLVL_DEBUG) <<
"Reconnecting DataSenderManager";
109 #if DUMP_SEND_MESSAGE
110 std::string fileName =
"sendMessage_" + std::to_string(my_rank) +
"_" + std::to_string(getpid()) +
"_" +
111 std::to_string(sequenceId) +
".bin";
112 std::fstream ostream(fileName, std::ios::out | std::ios::binary);
113 ostream.write(msg.Buffer(), msg.Length());
117 TLOG(TLVL_DEBUG) <<
"Sending message with sequenceID=" << sequenceId <<
", type=" << (int)messageType
118 <<
", length=" << msg.Length();
120 header.
data_length =
static_cast<uint64_t
>(msg.Length());
121 artdaq::Fragment fragment(std::ceil(msg.Length() /
static_cast<double>(
sizeof(artdaq::RawDataType))), sequenceId, 0,
122 messageType, header);
124 memcpy(&*fragment.dataBegin(), msg.Buffer(), msg.Length());
125 sender_ptr_->sendFragment(std::move(fragment));
131 TLOG(TLVL_TRACE) <<
"receiveMessage BEGIN";
132 while (recvd_fragments_ ==
nullptr)
134 TLOG(TLVL_TRACE) <<
"receiveMessage: Waiting for available buffer";
135 bool keep_looping =
true;
136 bool got_event =
false;
139 keep_looping =
false;
140 got_event = incoming_events_->ReadyForRead();
147 TLOG(TLVL_TRACE) <<
"receiveMessage: Reading buffer header";
148 auto errflag =
false;
149 incoming_events_->ReadHeader(errflag);
155 TLOG(TLVL_TRACE) <<
"receiveMessage: Getting Fragment types";
156 auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag);
159 incoming_events_->ReleaseBuffer();
163 if (fragmentTypes.size() == 0)
165 TLOG(TLVL_ERROR) <<
"Event has no Fragments! Aborting!";
166 incoming_events_->ReleaseBuffer();
170 TLOG(TLVL_TRACE) <<
"receiveMessage: Checking first Fragment type";
171 auto firstFragmentType = *fragmentTypes.begin();
178 if (!got_event || firstFragmentType == artdaq::Fragment::EndOfDataFragmentType)
180 TLOG(TLVL_DEBUG) <<
"Received shutdown message, returning from receiveMessage "
181 <<
"(debug: got_event=" << got_event <<
",fragType=" << (int)firstFragmentType
182 <<
",EODFragType=" << (
int)artdaq::Fragment::EndOfDataFragmentType <<
")";
183 incoming_events_->ReleaseBuffer();
187 if (firstFragmentType == artdaq::Fragment::InitFragmentType)
189 TLOG(TLVL_DEBUG) <<
"Cannot receive InitFragments here, retrying";
190 incoming_events_->ReleaseBuffer();
194 else if (firstFragmentType == artdaq::Fragment::EndOfRunFragmentType ||
195 firstFragmentType == artdaq::Fragment::EndOfSubrunFragmentType)
197 TLOG(TLVL_DEBUG) <<
"Ignoring EndOfRun or EndOfSubrun Fragment";
198 incoming_events_->ReleaseBuffer();
202 TLOG(TLVL_TRACE) <<
"receiveMessage: Getting all Fragments";
203 recvd_fragments_ = incoming_events_->GetFragmentsByType(errflag, artdaq::Fragment::InvalidFragmentType);
204 if (!recvd_fragments_)
206 TLOG(TLVL_ERROR) <<
"Error retrieving Fragments from shared memory! Aborting!";
207 incoming_events_->ReleaseBuffer();
214 std::sort(recvd_fragments_->begin(), recvd_fragments_->end(), artdaq::fragmentSequenceIDCompare);
216 TLOG(TLVL_TRACE) <<
"receiveMessage: Releasing buffer";
217 incoming_events_->ReleaseBuffer();
221 auto start = std::chrono::steady_clock::now();
222 while (!init_received_ && artdaq::TimeUtils::GetElapsedTime(start) < init_timeout_s_)
224 usleep(init_timeout_s_ * 1000000 / 100);
228 TLOG(TLVL_ERROR) <<
"Received data but no Init Fragment after " << init_timeout_s_ <<
" seconds. Art will crash.";
231 TLOG(TLVL_TRACE) <<
"receiveMessage: Returning top Fragment";
232 artdaq::Fragment topFrag = std::move(recvd_fragments_->at(0));
233 recvd_fragments_->erase(recvd_fragments_->begin());
234 if (recvd_fragments_->size() == 0)
236 recvd_fragments_.reset(
nullptr);
239 TLOG(TLVL_TRACE) <<
"receiveMessage: Copying Fragment into TBufferFile, length="
242 auto buffer =
static_cast<char*
>(malloc(header->
data_length));
243 memcpy(buffer, &*topFrag.dataBegin(), header->
data_length);
244 msg =
new TBufferFile(TBuffer::kRead, header->
data_length, buffer, kTRUE, 0);
246 #if DUMP_RECEIVE_MESSAGE
247 std::string fileName =
"receiveMessage_" + std::to_string(my_rank) +
"_" + std::to_string(getpid()) +
"_" +
248 std::to_string(topFrag.sequenceID()) +
".bin";
249 std::fstream ostream(fileName.c_str(), std::ios::out | std::ios::binary);
254 TLOG(TLVL_TRACE) <<
"receiveMessage END";
260 TLOG(TLVL_TRACE) <<
"receiveInitMessage BEGIN";
261 if (recvd_fragments_ ==
nullptr)
263 TLOG(TLVL_TRACE) <<
"receiveInitMessage: Waiting for available buffer";
265 bool got_init =
false;
266 auto errflag =
false;
269 bool got_event =
false;
272 got_event = incoming_events_->ReadyForRead(
true);
275 TLOG(TLVL_TRACE) <<
"receiveInitMessage: Reading buffer header";
276 incoming_events_->ReadHeader(errflag);
279 TLOG(TLVL_ERROR) <<
"receiveInitMessage: Error receiving message!";
280 incoming_events_->ReleaseBuffer();
284 TLOG(TLVL_TRACE) <<
"receiveInitMessage: Getting Fragment types";
285 auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag);
288 incoming_events_->ReleaseBuffer();
290 TLOG(TLVL_ERROR) <<
"receiveInitMessage: Error receiving message!";
293 if (fragmentTypes.size() == 0)
295 TLOG(TLVL_ERROR) <<
"Event has no Fragments! Aborting!";
296 incoming_events_->ReleaseBuffer();
300 TLOG(TLVL_TRACE) <<
"receiveInitMessage: Checking first Fragment type";
301 auto firstFragmentType = *fragmentTypes.begin();
308 if (!got_event || firstFragmentType == artdaq::Fragment::EndOfDataFragmentType)
310 TLOG(TLVL_DEBUG) <<
"Received shutdown message, returning";
311 incoming_events_->ReleaseBuffer();
315 if (firstFragmentType != artdaq::Fragment::InitFragmentType)
317 TLOG(TLVL_WARNING) <<
"Did NOT receive Init Fragment as first broadcast! Type="
318 << artdaq::detail::RawFragmentHeader::SystemTypeToString(firstFragmentType);
319 incoming_events_->ReleaseBuffer();
323 TLOG(TLVL_TRACE) <<
"receiveInitMessage: Getting all Fragments";
324 recvd_fragments_ = incoming_events_->GetFragmentsByType(errflag, artdaq::Fragment::InvalidFragmentType);
328 std::sort(recvd_fragments_->begin(), recvd_fragments_->end(), artdaq::fragmentSequenceIDCompare);
330 incoming_events_->ReleaseBuffer();
333 TLOG(TLVL_TRACE) <<
"receiveInitMessage: Returning top Fragment";
334 artdaq::Fragment topFrag = std::move(recvd_fragments_->at(0));
335 recvd_fragments_->erase(recvd_fragments_->begin());
336 if (recvd_fragments_->size() == 0)
338 recvd_fragments_.reset(
nullptr);
342 TLOG(TLVL_TRACE) <<
"receiveInitMessage: Copying Fragment into TBufferFile: message length: " << header->
data_length;
343 auto buffer =
new char[header->data_length];
345 memcpy(buffer, &*topFrag.dataBegin(), header->data_length);
347 #if DUMP_RECEIVE_MESSAGE
348 std::string fileName =
"receiveInitMessage_" + std::to_string(getpid()) +
".bin";
349 std::fstream ostream(fileName.c_str(), std::ios::out | std::ios::binary);
350 ostream.write(buffer, header->data_length);
354 msg =
new TBufferFile(TBuffer::kRead, header->data_length, buffer, kTRUE, 0);
356 TLOG(TLVL_TRACE) <<
"receiveInitMessage END";
357 init_received_ =
true;
void receiveInitMessage(TBufferFile *&msg) override
Receive the init message.
Sends Fragment objects using TransferInterface plugins. Uses Routing Tables if confgiured, otherwise will Round-Robin Fragments to the destinations.
void sendMessage(uint64_t sequenceId, uint8_t messageType, TBufferFile &msg) override
Send ROOT data, wrapped in an artdaq::Fragment object.
NetMonTransportService extends NetMonTransportServiceInterface. It sends events using DataSenderManag...
void receiveMessage(TBufferFile *&msg) override
Receive data from the ConcurrentQueue.
void connect() override
Reconnect the NetMonTransportService.
virtual ~NetMonTransportService()
NetMonTransportService Destructor. Calls disconnect().
NetMonTransportService(fhicl::ParameterSet const &pset, art::ActivityRegistry &)
NetMonTransportService Constructor.
void disconnect() override
Disconnects the NetMonTranportService.
Interface for NetMonTranportService. This interface is declared to art as part of the required regist...
void listen() override
Listen for connections. This method is a No-Op.