1 #include "artdaq/DAQdata/Globals.hh"
2 #include "artdaq/ArtModules/NetMonTransportService.h"
3 #include "artdaq/DAQrate/DataSenderManager.hh"
4 #include "artdaq-core/Core/SharedMemoryEventReceiver.hh"
6 #include "artdaq-core/Data/Fragment.hh"
7 #include "artdaq/DAQdata/NetMonHeader.hh"
8 #include "artdaq-core/Data/RawEvent.hh"
9 #include "artdaq-core/Utilities/TimeUtils.hh"
11 #include "art/Framework/Services/Registry/ActivityRegistry.h"
12 #include "canvas/Utilities/Exception.h"
13 #include "cetlib/container_algorithms.h"
14 #include "cetlib_except/exception.h"
15 #include "fhiclcpp/ParameterSet.h"
16 #include "fhiclcpp/ParameterSetRegistry.h"
19 #include <TBufferFile.h>
27 static fhicl::ParameterSet empty_pset;
34 , init_received_(false)
35 , sender_ptr_(nullptr)
36 , incoming_events_(new artdaq::SharedMemoryEventReceiver(pset.get<int>(
"shared_memory_key", 0xBEE70000 + getppid()), pset.get<int>(
"broadcast_shared_memory_key", 0xCEE70000 + getppid())))
37 , recvd_fragments_(nullptr)
39 TLOG_TRACE(
"NetMonTransportService") <<
"NetMonTransportService CONSTRUCTOR" << TLOG_ENDL;
40 if (pset.has_key(
"rank")) my_rank = pset.get<
int>(
"rank");
41 else my_rank = incoming_events_->GetRank();
43 init_timeout_s_ = pset.get<
double>(
"init_fragment_timeout_seconds", 1.0);
70 if (sender_ptr_) sender_ptr_.reset(
nullptr);
75 sendMessage(uint64_t sequenceId, uint8_t messageType, TBufferFile& msg)
77 if (sender_ptr_ ==
nullptr)
79 TLOG_DEBUG(
"NetMonTransportService") <<
"Reconnecting DataSenderManager" << TLOG_ENDL;
84 if (messageType == artdaq::Fragment::InitFragmentType) {
85 std::fstream ostream(
"sendInitMessage.bin", std::ios::out | std::ios::binary);
86 ostream.write(msg.Buffer(), msg.Length());
91 TLOG_DEBUG(
"NetMonTransportService") <<
"Sending message with sequenceID=" << std::to_string(sequenceId) <<
", type=" << std::to_string(messageType) <<
", length=" << std::to_string(msg.Length()) << TLOG_ENDL;
93 header.
data_length =
static_cast<uint64_t
>(msg.Length());
95 fragment(std::ceil(msg.Length() /
96 static_cast<double>(
sizeof(artdaq::RawDataType))),
97 sequenceId, 0, messageType, header);
99 memcpy(&*fragment.dataBegin(), msg.Buffer(), msg.Length());
100 sender_ptr_->sendFragment(std::move(fragment));
107 TLOG_TRACE(
"NetMonTransportService") <<
"receiveMessage BEGIN" << TLOG_ENDL;
108 while (recvd_fragments_ ==
nullptr)
110 TLOG_TRACE(
"NetMonTransportService") <<
"receiveMessage: Waiting for available buffer" << TLOG_ENDL;
111 bool keep_looping =
true;
112 bool got_event =
false;
115 keep_looping =
false;
116 got_event = incoming_events_->ReadyForRead();
123 TLOG_TRACE(
"NetMonTransportService") <<
"receiveMessage: Reading buffer header" << TLOG_ENDL;
124 auto errflag =
false;
125 incoming_events_->ReadHeader(errflag);
130 TLOG_TRACE(
"NetMonTransportService") <<
"receiveMessage: Getting Fragment types" << TLOG_ENDL;
131 auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag);
136 if (fragmentTypes.size() == 0)
138 TLOG_ERROR(
"NetMonTransportService") <<
"Event has no Fragments! Aborting!" << TLOG_ENDL;
139 incoming_events_->ReleaseBuffer();
143 TLOG_TRACE(
"NetMonTransportService") <<
"receiveMessage: Checking first Fragment type" << TLOG_ENDL;
144 auto firstFragmentType = *fragmentTypes.begin();
151 if (!got_event || firstFragmentType == artdaq::Fragment::EndOfDataFragmentType)
153 TLOG_DEBUG(
"NetMonTransportService") <<
"Received shutdown message, returning" << TLOG_ENDL;
154 incoming_events_->ReleaseBuffer();
158 if (firstFragmentType == artdaq::Fragment::InitFragmentType)
160 TLOG_DEBUG(
"NetMonTransportService") <<
"Cannot receive InitFragments here, retrying" << TLOG_ENDL;
161 incoming_events_->ReleaseBuffer();
165 else if (firstFragmentType == artdaq::Fragment::EndOfRunFragmentType || firstFragmentType == artdaq::Fragment::EndOfSubrunFragmentType)
167 TLOG_DEBUG(
"NetMonTransportService") <<
"Ignoring EndOfRun or EndOfSubrun Fragment" << TLOG_ENDL;
168 incoming_events_->ReleaseBuffer();
172 TLOG_TRACE(
"NetMonTransportService") <<
"receiveMessage: Getting all Fragments" << TLOG_ENDL;
173 recvd_fragments_ = incoming_events_->GetFragmentsByType(errflag, artdaq::Fragment::InvalidFragmentType);
177 std::sort(recvd_fragments_->begin(), recvd_fragments_->end(),
178 artdaq::fragmentSequenceIDCompare);
180 TLOG_TRACE(
"NetMonTransportService") <<
"receiveMessage: Releasing buffer" << TLOG_ENDL;
181 incoming_events_->ReleaseBuffer();
185 auto start = std::chrono::steady_clock::now();
186 while (!init_received_ && artdaq::TimeUtils::GetElapsedTime(start) < init_timeout_s_)
188 usleep(init_timeout_s_ * 1000000 / 100);
190 if (!init_received_) {
191 TLOG_ERROR(
"NetMonTransportService") <<
"Received data but no Init Fragment after " << init_timeout_s_ <<
" seconds. Art will crash." << TLOG_ENDL;
194 TLOG_TRACE(
"NetMonTransportService") <<
"receiveMessage: Returning top Fragment" << TLOG_ENDL;
195 artdaq::Fragment topFrag = std::move(recvd_fragments_->at(0));
196 recvd_fragments_->erase(recvd_fragments_->begin());
197 if (recvd_fragments_->size() == 0)
199 recvd_fragments_.reset(
nullptr);
202 TLOG_TRACE(
"NetMonTransportService") <<
"receiveMessage: Copying Fragment into TBufferFile" << TLOG_ENDL;
204 auto buffer =
static_cast<char *
>(malloc(header->data_length));
205 memcpy(buffer, &*topFrag.dataBegin(), header->
data_length);
206 msg =
new TBufferFile(TBuffer::kRead, header->data_length, buffer, kTRUE, 0);
208 TLOG_TRACE(
"NetMonTransportService") <<
"receiveMessage END" << TLOG_ENDL;
215 TLOG_TRACE(
"NetMonTransportService") <<
"receiveInitMessage BEGIN" << TLOG_ENDL;
216 if (recvd_fragments_ ==
nullptr)
218 TLOG_TRACE(
"NetMonTransportService") <<
"receiveInitMessage: Waiting for available buffer" << TLOG_ENDL;
220 bool got_init =
false;
221 auto errflag =
false;
224 bool got_event =
false;
227 got_event = incoming_events_->ReadyForRead(
true);
230 TLOG_TRACE(
"NetMonTransportService") <<
"receiveInitMessage: Reading buffer header" << TLOG_ENDL;
231 incoming_events_->ReadHeader(errflag);
233 TLOG_ERROR(
"NetMonTransportService") <<
"receiveInitMessage: Error receiving message!" << TLOG_ENDL;
237 TLOG_TRACE(
"NetMonTransportService") <<
"receiveInitMessage: Getting Fragment types" << TLOG_ENDL;
238 auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag);
241 TLOG_ERROR(
"NetMonTransportService") <<
"receiveInitMessage: Error receiving message!" << TLOG_ENDL;
244 if (fragmentTypes.size() == 0)
246 TLOG_ERROR(
"NetMonTransportService") <<
"Event has no Fragments! Aborting!" << TLOG_ENDL;
247 incoming_events_->ReleaseBuffer();
251 TLOG_TRACE(
"NetMonTransportService") <<
"receiveInitMessage: Checking first Fragment type" << TLOG_ENDL;
252 auto firstFragmentType = *fragmentTypes.begin();
259 if (!got_event || firstFragmentType == artdaq::Fragment::EndOfDataFragmentType)
261 TLOG_DEBUG(
"NetMonTransportService") <<
"Received shutdown message, returning" << TLOG_ENDL;
262 incoming_events_->ReleaseBuffer();
266 if (firstFragmentType != artdaq::Fragment::InitFragmentType)
268 TLOG_WARNING(
"NetMonTransportService") <<
"Did NOT receive Init Fragment as first broadcast! Type=" << artdaq::detail::RawFragmentHeader::SystemTypeToString(firstFragmentType) << TLOG_ENDL;
269 incoming_events_->ReleaseBuffer();
273 TLOG_TRACE(
"NetMonTransportService") <<
"receiveInitMessage: Getting all Fragments" << TLOG_ENDL;
274 recvd_fragments_ = incoming_events_->GetFragmentsByType(errflag, artdaq::Fragment::InvalidFragmentType);
278 std::sort(recvd_fragments_->begin(), recvd_fragments_->end(),
279 artdaq::fragmentSequenceIDCompare);
282 TLOG_TRACE(
"NetMonTransportService") <<
"receiveInitMessage: Returning top Fragment" << TLOG_ENDL;
283 artdaq::Fragment topFrag = std::move(recvd_fragments_->at(0));
284 recvd_fragments_->erase(recvd_fragments_->begin());
285 if (recvd_fragments_->size() == 0)
287 recvd_fragments_.reset(
nullptr);
291 TLOG_TRACE(
"NetMonTransportService") <<
"receiveInitMessage: Copying Fragment into TBufferFile: message length: " << std::to_string(header->data_length) << TLOG_ENDL;
292 auto buffer =
static_cast<char *
>(malloc(header->data_length));
293 memcpy(buffer, &*topFrag.dataBegin(), header->data_length);
296 std::string fileName =
"receiveInitMessage_" + std::to_string(getpid()) +
".bin";
297 std::fstream ostream(fileName.c_str(), std::ios::out | std::ios::binary);
298 ostream.write(buffer, header->data_length);
302 msg =
new TBufferFile(TBuffer::kRead, header->data_length, buffer, kTRUE, 0);
304 TLOG_TRACE(
"NetMonTransportService") <<
"receiveInitMessage END" << TLOG_ENDL;
305 init_received_ =
true;
void receiveInitMessage(TBufferFile *&msg) override
Receive the init message.
Sends Fragment objects using TransferInterface plugins. Uses Routing Tables if confgiured, otherwise will Round-Robin Fragments to the destinations.
void sendMessage(uint64_t sequenceId, uint8_t messageType, TBufferFile &msg) override
Send ROOT data, wrapped in an artdaq::Fragment object.
NetMonTransportService extends NetMonTransportServiceInterface. It sends events using DataSenderManag...
void receiveMessage(TBufferFile *&msg) override
Receive data from the ConcurrentQueue.
void connect() override
Reconnect the NetMonTransportService.
virtual ~NetMonTransportService()
NetMonTransportService Destructor. Calls disconnect().
NetMonTransportService(fhicl::ParameterSet const &pset, art::ActivityRegistry &)
NetMonTransportService Constructor.
void disconnect() override
Disconnects the NetMonTranportService.
Interface for NetMonTranportService. This interface is declared to art as part of the required regist...
void listen() override
Listen for connections. This method is a No-Op.