00001 #define TRACE_NAME "NetMonTransportService"
00002
00003 #include "artdaq/DAQdata/Globals.hh"
00004 #include "artdaq/ArtModules/NetMonTransportService.h"
00005 #include "artdaq/DAQrate/DataSenderManager.hh"
00006 #include "artdaq-core/Core/SharedMemoryEventReceiver.hh"
00007
00008 #include "artdaq-core/Data/Fragment.hh"
00009 #include "artdaq/DAQdata/NetMonHeader.hh"
00010 #include "artdaq-core/Data/RawEvent.hh"
00011 #include "artdaq-core/Utilities/TimeUtils.hh"
00012
00013 #include "art/Framework/Services/Registry/ActivityRegistry.h"
00014 #include "canvas/Utilities/Exception.h"
00015 #include "cetlib/container_algorithms.h"
00016 #include "cetlib_except/exception.h"
00017 #include "fhiclcpp/ParameterSet.h"
00018 #include "fhiclcpp/ParameterSetRegistry.h"
00019
00020 #include <TClass.h>
00021 #include <TBufferFile.h>
00022
00023 #include <iomanip>
00024 #include <iostream>
00025 #include <fstream>
00026 #include <string>
00027 #include <vector>
00028
00029
00030 #define DUMP_SEND_MESSAGE 0
00031 #define DUMP_RECEIVE_MESSAGE 0
00032
00033 static fhicl::ParameterSet empty_pset;
00034
00035
00036 NetMonTransportService::
00037 NetMonTransportService(fhicl::ParameterSet const& pset, art::ActivityRegistry&)
00038 : NetMonTransportServiceInterface()
00039 , data_pset_(pset)
00040 , init_received_(false)
00041 , sender_ptr_(nullptr)
00042 , incoming_events_(new artdaq::SharedMemoryEventReceiver(pset.get<int>("shared_memory_key", 0xBEE70000 + getppid()), pset.get<int>("broadcast_shared_memory_key", 0xCEE70000 + getppid())))
00043 , recvd_fragments_(nullptr)
00044 {
00045 TLOG(TLVL_TRACE) << "NetMonTransportService CONSTRUCTOR" ;
00046 if (pset.has_key("rank")) my_rank = pset.get<int>("rank");
00047 else my_rank = incoming_events_->GetRank();
00048
00049 init_timeout_s_ = pset.get<double>("init_fragment_timeout_seconds", 1.0);
00050 }
00051
00052 NetMonTransportService::
00053 ~NetMonTransportService()
00054 {
00055 NetMonTransportService::disconnect();
00056 }
00057
00058 void
00059 NetMonTransportService::
00060 connect()
00061 {
00062 sender_ptr_.reset(new artdaq::DataSenderManager(data_pset_));
00063 }
00064
00065 void
00066 NetMonTransportService::
00067 listen()
00068 {
00069 return;
00070 }
00071
00072 void
00073 NetMonTransportService::
00074 disconnect()
00075 {
00076 if (sender_ptr_) sender_ptr_.reset(nullptr);
00077 }
00078
00079 void
00080 NetMonTransportService::
00081 sendMessage(uint64_t sequenceId, uint8_t messageType, TBufferFile& msg)
00082 {
00083 if (sender_ptr_ == nullptr)
00084 {
00085 TLOG(TLVL_DEBUG) << "Reconnecting DataSenderManager" ;
00086 connect();
00087 }
00088
00089 #if DUMP_SEND_MESSAGE
00090 std::string fileName = "sendMessage_" + std::to_string(my_rank) + "_" + std::to_string(getpid()) + "_" + std::to_string(sequenceId) + ".bin";
00091 std::fstream ostream(fileName, std::ios::out | std::ios::binary);
00092 ostream.write(msg.Buffer(), msg.Length());
00093 ostream.close();
00094 #endif
00095
00096 TLOG(TLVL_DEBUG) << "Sending message with sequenceID=" << std::to_string(sequenceId) << ", type=" << std::to_string(messageType) << ", length=" << std::to_string(msg.Length()) ;
00097 artdaq::NetMonHeader header;
00098 header.data_length = static_cast<uint64_t>(msg.Length());
00099 artdaq::Fragment
00100 fragment(std::ceil(msg.Length() /
00101 static_cast<double>(sizeof(artdaq::RawDataType))),
00102 sequenceId, 0, messageType, header);
00103
00104 memcpy(&*fragment.dataBegin(), msg.Buffer(), msg.Length());
00105 sender_ptr_->sendFragment(std::move(fragment));
00106 }
00107
00108 void
00109 NetMonTransportService::
00110 receiveMessage(TBufferFile*& msg)
00111 {
00112 TLOG(TLVL_TRACE) << "receiveMessage BEGIN" ;
00113 while (recvd_fragments_ == nullptr)
00114 {
00115 TLOG(TLVL_TRACE) << "receiveMessage: Waiting for available buffer" ;
00116 bool keep_looping = true;
00117 bool got_event = false;
00118 while (keep_looping)
00119 {
00120 keep_looping = false;
00121 got_event = incoming_events_->ReadyForRead();
00122 if (!got_event)
00123 {
00124 keep_looping = true;
00125 }
00126 }
00127
00128 TLOG(TLVL_TRACE) << "receiveMessage: Reading buffer header" ;
00129 auto errflag = false;
00130 incoming_events_->ReadHeader(errflag);
00131 if (errflag) {
00132 msg = nullptr;
00133 return;
00134 }
00135 TLOG(TLVL_TRACE) << "receiveMessage: Getting Fragment types" ;
00136 auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag);
00137 if (errflag) {
00138 msg = nullptr;
00139 return;
00140 }
00141 if (fragmentTypes.size() == 0)
00142 {
00143 TLOG(TLVL_ERROR) << "Event has no Fragments! Aborting!" ;
00144 incoming_events_->ReleaseBuffer();
00145 msg = nullptr;
00146 return;
00147 }
00148 TLOG(TLVL_TRACE) << "receiveMessage: Checking first Fragment type" ;
00149 auto firstFragmentType = *fragmentTypes.begin();
00150
00151
00152
00153
00154
00155
00156 if (!got_event || firstFragmentType == artdaq::Fragment::EndOfDataFragmentType)
00157 {
00158 TLOG(TLVL_DEBUG) << "Received shutdown message, returning" ;
00159 incoming_events_->ReleaseBuffer();
00160 msg = nullptr;
00161 return;
00162 }
00163 if (firstFragmentType == artdaq::Fragment::InitFragmentType)
00164 {
00165 TLOG(TLVL_DEBUG) << "Cannot receive InitFragments here, retrying" ;
00166 incoming_events_->ReleaseBuffer();
00167 continue;
00168 }
00169
00170 else if (firstFragmentType == artdaq::Fragment::EndOfRunFragmentType || firstFragmentType == artdaq::Fragment::EndOfSubrunFragmentType)
00171 {
00172 TLOG(TLVL_DEBUG) << "Ignoring EndOfRun or EndOfSubrun Fragment" ;
00173 incoming_events_->ReleaseBuffer();
00174 continue;
00175 }
00176
00177 TLOG(TLVL_TRACE) << "receiveMessage: Getting all Fragments" ;
00178 recvd_fragments_ = incoming_events_->GetFragmentsByType(errflag, artdaq::Fragment::InvalidFragmentType);
00179
00180
00181
00182 std::sort(recvd_fragments_->begin(), recvd_fragments_->end(),
00183 artdaq::fragmentSequenceIDCompare);
00184
00185 TLOG(TLVL_TRACE) << "receiveMessage: Releasing buffer" ;
00186 incoming_events_->ReleaseBuffer();
00187 }
00188
00189
00190 auto start = std::chrono::steady_clock::now();
00191 while (!init_received_ && artdaq::TimeUtils::GetElapsedTime(start) < init_timeout_s_)
00192 {
00193 usleep(init_timeout_s_ * 1000000 / 100);
00194 }
00195 if (!init_received_) {
00196 TLOG(TLVL_ERROR) << "Received data but no Init Fragment after " << init_timeout_s_ << " seconds. Art will crash." ;
00197 }
00198
00199 TLOG(TLVL_TRACE) << "receiveMessage: Returning top Fragment" ;
00200 artdaq::Fragment topFrag = std::move(recvd_fragments_->at(0));
00201 recvd_fragments_->erase(recvd_fragments_->begin());
00202 if (recvd_fragments_->size() == 0)
00203 {
00204 recvd_fragments_.reset(nullptr);
00205 }
00206
00207 TLOG(TLVL_TRACE) << "receiveMessage: Copying Fragment into TBufferFile, length=" << topFrag.metadata<artdaq::NetMonHeader>()->data_length ;
00208 auto header = topFrag.metadata<artdaq::NetMonHeader>();
00209 auto buffer = static_cast<char *>(malloc(header->data_length));
00210 memcpy(buffer, &*topFrag.dataBegin(), header->data_length);
00211 msg = new TBufferFile(TBuffer::kRead, header->data_length, buffer, kTRUE, 0);
00212
00213 #if DUMP_RECEIVE_MESSAGE
00214 std::string fileName = "receiveMessage_" + std::to_string(my_rank) + "_" + std::to_string(getpid()) + "_" + std::to_string(topFrag.sequenceID()) + ".bin";
00215 std::fstream ostream(fileName.c_str(), std::ios::out | std::ios::binary);
00216 ostream.write(buffer, header->data_length);
00217 ostream.close();
00218 #endif
00219
00220 TLOG(TLVL_TRACE) << "receiveMessage END" ;
00221 }
00222
00223 void
00224 NetMonTransportService::
00225 receiveInitMessage(TBufferFile*& msg)
00226 {
00227 TLOG(TLVL_TRACE) << "receiveInitMessage BEGIN" ;
00228 if (recvd_fragments_ == nullptr)
00229 {
00230 TLOG(TLVL_TRACE) << "receiveInitMessage: Waiting for available buffer" ;
00231
00232 bool got_init = false;
00233 auto errflag = false;
00234 while (!got_init) {
00235
00236 bool got_event = false;
00237 while (!got_event)
00238 {
00239 got_event = incoming_events_->ReadyForRead(true);
00240 }
00241
00242 TLOG(TLVL_TRACE) << "receiveInitMessage: Reading buffer header" ;
00243 incoming_events_->ReadHeader(errflag);
00244 if (errflag) {
00245 TLOG(TLVL_ERROR) << "receiveInitMessage: Error receiving message!" ;
00246 msg = nullptr;
00247 return;
00248 }
00249 TLOG(TLVL_TRACE) << "receiveInitMessage: Getting Fragment types" ;
00250 auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag);
00251 if (errflag) {
00252 msg = nullptr;
00253 TLOG(TLVL_ERROR) << "receiveInitMessage: Error receiving message!" ;
00254 return;
00255 }
00256 if (fragmentTypes.size() == 0)
00257 {
00258 TLOG(TLVL_ERROR) << "Event has no Fragments! Aborting!" ;
00259 incoming_events_->ReleaseBuffer();
00260 msg = nullptr;
00261 return;
00262 }
00263 TLOG(TLVL_TRACE) << "receiveInitMessage: Checking first Fragment type" ;
00264 auto firstFragmentType = *fragmentTypes.begin();
00265
00266
00267
00268
00269
00270
00271 if (!got_event || firstFragmentType == artdaq::Fragment::EndOfDataFragmentType)
00272 {
00273 TLOG(TLVL_DEBUG) << "Received shutdown message, returning" ;
00274 incoming_events_->ReleaseBuffer();
00275 msg = nullptr;
00276 return;
00277 }
00278 if (firstFragmentType != artdaq::Fragment::InitFragmentType)
00279 {
00280 TLOG(TLVL_WARNING) << "Did NOT receive Init Fragment as first broadcast! Type=" << artdaq::detail::RawFragmentHeader::SystemTypeToString(firstFragmentType) ;
00281 incoming_events_->ReleaseBuffer();
00282 }
00283 got_init = true;
00284 }
00285 TLOG(TLVL_TRACE) << "receiveInitMessage: Getting all Fragments" ;
00286 recvd_fragments_ = incoming_events_->GetFragmentsByType(errflag, artdaq::Fragment::InvalidFragmentType);
00287
00288
00289
00290 std::sort(recvd_fragments_->begin(), recvd_fragments_->end(),
00291 artdaq::fragmentSequenceIDCompare);
00292 }
00293
00294 TLOG(TLVL_TRACE) << "receiveInitMessage: Returning top Fragment" ;
00295 artdaq::Fragment topFrag = std::move(recvd_fragments_->at(0));
00296 recvd_fragments_->erase(recvd_fragments_->begin());
00297 if (recvd_fragments_->size() == 0)
00298 {
00299 recvd_fragments_.reset(nullptr);
00300 }
00301
00302 auto header = topFrag.metadata<artdaq::NetMonHeader>();
00303 TLOG(TLVL_TRACE) << "receiveInitMessage: Copying Fragment into TBufferFile: message length: " << std::to_string(header->data_length) ;
00304 auto buffer = static_cast<char *>(malloc(header->data_length));
00305 memcpy(buffer, &*topFrag.dataBegin(), header->data_length);
00306
00307 #if DUMP_RECEIVE_MESSAGE
00308 std::string fileName = "receiveInitMessage_" + std::to_string(getpid()) + ".bin";
00309 std::fstream ostream(fileName.c_str(), std::ios::out | std::ios::binary);
00310 ostream.write(buffer, header->data_length);
00311 ostream.close();
00312 #endif
00313
00314 msg = new TBufferFile(TBuffer::kRead, header->data_length, buffer, kTRUE, 0);
00315
00316 TLOG(TLVL_TRACE) << "receiveInitMessage END" ;
00317 init_received_ = true;
00318 }
00319 DEFINE_ART_SERVICE_INTERFACE_IMPL(NetMonTransportService, NetMonTransportServiceInterface)