00001 #define TRACE_NAME "NetMonTransportService"
00002
00003 #include "artdaq/DAQdata/Globals.hh"
00004 #include "artdaq/ArtModules/NetMonTransportService.h"
00005 #include "artdaq/DAQrate/DataSenderManager.hh"
00006 #include "artdaq-core/Core/SharedMemoryEventReceiver.hh"
00007
00008 #include "artdaq-core/Data/Fragment.hh"
00009 #include "artdaq/DAQdata/NetMonHeader.hh"
00010 #include "artdaq-core/Data/RawEvent.hh"
00011 #include "artdaq-core/Utilities/TimeUtils.hh"
00012
00013 #include "art/Framework/Services/Registry/ActivityRegistry.h"
00014 #include "canvas/Utilities/Exception.h"
00015 #include "cetlib/container_algorithms.h"
00016 #include "cetlib_except/exception.h"
00017 #include "fhiclcpp/ParameterSet.h"
00018 #include "fhiclcpp/ParameterSetRegistry.h"
00019
00020 #include <TClass.h>
00021 #include <TBufferFile.h>
00022
00023 #include <iomanip>
00024 #include <iostream>
00025 #include <fstream>
00026 #include <string>
00027 #include <vector>
00028
00029
00030 #define DUMP_SEND_MESSAGE 0
00031 #define DUMP_RECEIVE_MESSAGE 0
00032
00033 static fhicl::ParameterSet empty_pset;
00034
00035
00036 NetMonTransportService::
00037 NetMonTransportService(fhicl::ParameterSet const& pset, art::ActivityRegistry&)
00038 : NetMonTransportServiceInterface()
00039 , data_pset_(pset)
00040 , init_received_(false)
00041 , sender_ptr_(nullptr)
00042 , incoming_events_(nullptr)
00043 , recvd_fragments_(nullptr)
00044 {
00045 TLOG(TLVL_TRACE) << "NetMonTransportService CONSTRUCTOR" ;
00046 if (pset.has_key("rank")) my_rank = pset.get<int>("rank");
00047
00048 init_timeout_s_ = pset.get<double>("init_fragment_timeout_seconds", 1.0);
00049 }
00050
00051 NetMonTransportService::
00052 ~NetMonTransportService()
00053 {
00054 NetMonTransportService::disconnect();
00055 }
00056
00057 void
00058 NetMonTransportService::
00059 connect()
00060 {
00061 sender_ptr_.reset(new artdaq::DataSenderManager(data_pset_));
00062 }
00063
00064 void
00065 NetMonTransportService::
00066 listen()
00067 {
00068 if (!incoming_events_)
00069 {
00070 incoming_events_.reset(new artdaq::SharedMemoryEventReceiver(data_pset_.get<int>("shared_memory_key", 0xBEE70000 + getppid()), data_pset_.get<int>("broadcast_shared_memory_key", 0xCEE70000 + getppid())));
00071 if (data_pset_.has_key("rank")) my_rank = data_pset_.get<int>("rank");
00072 else my_rank = incoming_events_->GetRank();
00073 }
00074 return;
00075 }
00076
00077 void
00078 NetMonTransportService::
00079 disconnect()
00080 {
00081 if (sender_ptr_) sender_ptr_.reset(nullptr);
00082 }
00083
00084 void
00085 NetMonTransportService::
00086 sendMessage(uint64_t sequenceId, uint8_t messageType, TBufferFile& msg)
00087 {
00088 if (sender_ptr_ == nullptr)
00089 {
00090 TLOG(TLVL_DEBUG) << "Reconnecting DataSenderManager" ;
00091 connect();
00092 }
00093
00094 #if DUMP_SEND_MESSAGE
00095 std::string fileName = "sendMessage_" + std::to_string(my_rank) + "_" + std::to_string(getpid()) + "_" + std::to_string(sequenceId) + ".bin";
00096 std::fstream ostream(fileName, std::ios::out | std::ios::binary);
00097 ostream.write(msg.Buffer(), msg.Length());
00098 ostream.close();
00099 #endif
00100
00101 TLOG(TLVL_DEBUG) << "Sending message with sequenceID=" << sequenceId << ", type=" << (int)messageType << ", length=" << msg.Length() ;
00102 artdaq::NetMonHeader header;
00103 header.data_length = static_cast<uint64_t>(msg.Length());
00104 artdaq::Fragment
00105 fragment(std::ceil(msg.Length() /
00106 static_cast<double>(sizeof(artdaq::RawDataType))),
00107 sequenceId, 0, messageType, header);
00108
00109 memcpy(&*fragment.dataBegin(), msg.Buffer(), msg.Length());
00110 sender_ptr_->sendFragment(std::move(fragment));
00111 }
00112
00113 void
00114 NetMonTransportService::
00115 receiveMessage(TBufferFile*& msg)
00116 {
00117 listen();
00118 TLOG(TLVL_TRACE) << "receiveMessage BEGIN" ;
00119 while (recvd_fragments_ == nullptr)
00120 {
00121 TLOG(TLVL_TRACE) << "receiveMessage: Waiting for available buffer" ;
00122 bool keep_looping = true;
00123 bool got_event = false;
00124 while (keep_looping)
00125 {
00126 keep_looping = false;
00127 got_event = incoming_events_->ReadyForRead();
00128 if (!got_event)
00129 {
00130 keep_looping = true;
00131 }
00132 }
00133
00134 TLOG(TLVL_TRACE) << "receiveMessage: Reading buffer header" ;
00135 auto errflag = false;
00136 incoming_events_->ReadHeader(errflag);
00137 if (errflag) {
00138 msg = nullptr;
00139 return;
00140 }
00141 TLOG(TLVL_TRACE) << "receiveMessage: Getting Fragment types" ;
00142 auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag);
00143 if (errflag) {
00144 incoming_events_->ReleaseBuffer();
00145 msg = nullptr;
00146 return;
00147 }
00148 if (fragmentTypes.size() == 0)
00149 {
00150 TLOG(TLVL_ERROR) << "Event has no Fragments! Aborting!" ;
00151 incoming_events_->ReleaseBuffer();
00152 msg = nullptr;
00153 return;
00154 }
00155 TLOG(TLVL_TRACE) << "receiveMessage: Checking first Fragment type" ;
00156 auto firstFragmentType = *fragmentTypes.begin();
00157
00158
00159
00160
00161
00162
00163 if (!got_event || firstFragmentType == artdaq::Fragment::EndOfDataFragmentType)
00164 {
00165 TLOG(TLVL_DEBUG) << "Received shutdown message, returning from receiveMessage "
00166 << "(debug: got_event=" << got_event << ",fragType=" << (int)firstFragmentType
00167 << ",EODFragType=" << (int)artdaq::Fragment::EndOfDataFragmentType << ")";
00168 incoming_events_->ReleaseBuffer();
00169 msg = nullptr;
00170 return;
00171 }
00172 if (firstFragmentType == artdaq::Fragment::InitFragmentType)
00173 {
00174 TLOG(TLVL_DEBUG) << "Cannot receive InitFragments here, retrying" ;
00175 incoming_events_->ReleaseBuffer();
00176 continue;
00177 }
00178
00179 else if (firstFragmentType == artdaq::Fragment::EndOfRunFragmentType || firstFragmentType == artdaq::Fragment::EndOfSubrunFragmentType)
00180 {
00181 TLOG(TLVL_DEBUG) << "Ignoring EndOfRun or EndOfSubrun Fragment" ;
00182 incoming_events_->ReleaseBuffer();
00183 continue;
00184 }
00185
00186 TLOG(TLVL_TRACE) << "receiveMessage: Getting all Fragments" ;
00187 recvd_fragments_ = incoming_events_->GetFragmentsByType(errflag, artdaq::Fragment::InvalidFragmentType);
00188 if (!recvd_fragments_)
00189 {
00190 TLOG(TLVL_ERROR) << "Error retrieving Fragments from shared memory! Aborting!";
00191 incoming_events_->ReleaseBuffer();
00192 msg = nullptr;
00193 return;
00194
00195 }
00196
00197
00198
00199 std::sort(recvd_fragments_->begin(), recvd_fragments_->end(),
00200 artdaq::fragmentSequenceIDCompare);
00201
00202 TLOG(TLVL_TRACE) << "receiveMessage: Releasing buffer" ;
00203 incoming_events_->ReleaseBuffer();
00204 }
00205
00206
00207 auto start = std::chrono::steady_clock::now();
00208 while (!init_received_ && artdaq::TimeUtils::GetElapsedTime(start) < init_timeout_s_)
00209 {
00210 usleep(init_timeout_s_ * 1000000 / 100);
00211 }
00212 if (!init_received_) {
00213 TLOG(TLVL_ERROR) << "Received data but no Init Fragment after " << init_timeout_s_ << " seconds. Art will crash." ;
00214 }
00215
00216 TLOG(TLVL_TRACE) << "receiveMessage: Returning top Fragment" ;
00217 artdaq::Fragment topFrag = std::move(recvd_fragments_->at(0));
00218 recvd_fragments_->erase(recvd_fragments_->begin());
00219 if (recvd_fragments_->size() == 0)
00220 {
00221 recvd_fragments_.reset(nullptr);
00222 }
00223
00224 TLOG(TLVL_TRACE) << "receiveMessage: Copying Fragment into TBufferFile, length=" << topFrag.metadata<artdaq::NetMonHeader>()->data_length ;
00225 auto header = topFrag.metadata<artdaq::NetMonHeader>();
00226 auto buffer = static_cast<char *>(malloc(header->data_length));
00227 memcpy(buffer, &*topFrag.dataBegin(), header->data_length);
00228 msg = new TBufferFile(TBuffer::kRead, header->data_length, buffer, kTRUE, 0);
00229
00230 #if DUMP_RECEIVE_MESSAGE
00231 std::string fileName = "receiveMessage_" + std::to_string(my_rank) + "_" + std::to_string(getpid()) + "_" + std::to_string(topFrag.sequenceID()) + ".bin";
00232 std::fstream ostream(fileName.c_str(), std::ios::out | std::ios::binary);
00233 ostream.write(buffer, header->data_length);
00234 ostream.close();
00235 #endif
00236
00237 TLOG(TLVL_TRACE) << "receiveMessage END" ;
00238 }
00239
00240 void
00241 NetMonTransportService::
00242 receiveInitMessage(TBufferFile*& msg)
00243 {
00244 listen();
00245 TLOG(TLVL_TRACE) << "receiveInitMessage BEGIN" ;
00246 if (recvd_fragments_ == nullptr)
00247 {
00248 TLOG(TLVL_TRACE) << "receiveInitMessage: Waiting for available buffer" ;
00249
00250 bool got_init = false;
00251 auto errflag = false;
00252 while (!got_init) {
00253
00254 bool got_event = false;
00255 while (!got_event)
00256 {
00257 got_event = incoming_events_->ReadyForRead(true);
00258 }
00259
00260 TLOG(TLVL_TRACE) << "receiveInitMessage: Reading buffer header" ;
00261 incoming_events_->ReadHeader(errflag);
00262 if (errflag) {
00263 TLOG(TLVL_ERROR) << "receiveInitMessage: Error receiving message!" ;
00264 incoming_events_->ReleaseBuffer();
00265 msg = nullptr;
00266 return;
00267 }
00268 TLOG(TLVL_TRACE) << "receiveInitMessage: Getting Fragment types" ;
00269 auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag);
00270 if (errflag) {
00271 incoming_events_->ReleaseBuffer();
00272 msg = nullptr;
00273 TLOG(TLVL_ERROR) << "receiveInitMessage: Error receiving message!" ;
00274 return;
00275 }
00276 if (fragmentTypes.size() == 0)
00277 {
00278 TLOG(TLVL_ERROR) << "Event has no Fragments! Aborting!" ;
00279 incoming_events_->ReleaseBuffer();
00280 msg = nullptr;
00281 return;
00282 }
00283 TLOG(TLVL_TRACE) << "receiveInitMessage: Checking first Fragment type" ;
00284 auto firstFragmentType = *fragmentTypes.begin();
00285
00286
00287
00288
00289
00290
00291 if (!got_event || firstFragmentType == artdaq::Fragment::EndOfDataFragmentType)
00292 {
00293 TLOG(TLVL_DEBUG) << "Received shutdown message, returning" ;
00294 incoming_events_->ReleaseBuffer();
00295 msg = nullptr;
00296 return;
00297 }
00298 if (firstFragmentType != artdaq::Fragment::InitFragmentType)
00299 {
00300 TLOG(TLVL_WARNING) << "Did NOT receive Init Fragment as first broadcast! Type=" << artdaq::detail::RawFragmentHeader::SystemTypeToString(firstFragmentType) ;
00301 incoming_events_->ReleaseBuffer();
00302 }
00303 got_init = true;
00304 }
00305 TLOG(TLVL_TRACE) << "receiveInitMessage: Getting all Fragments" ;
00306 recvd_fragments_ = incoming_events_->GetFragmentsByType(errflag, artdaq::Fragment::InvalidFragmentType);
00307
00308
00309
00310 std::sort(recvd_fragments_->begin(), recvd_fragments_->end(),
00311 artdaq::fragmentSequenceIDCompare);
00312
00313 incoming_events_->ReleaseBuffer();
00314 }
00315
00316 TLOG(TLVL_TRACE) << "receiveInitMessage: Returning top Fragment" ;
00317 artdaq::Fragment topFrag = std::move(recvd_fragments_->at(0));
00318 recvd_fragments_->erase(recvd_fragments_->begin());
00319 if (recvd_fragments_->size() == 0)
00320 {
00321 recvd_fragments_.reset(nullptr);
00322 }
00323
00324 auto header = topFrag.metadata<artdaq::NetMonHeader>();
00325 TLOG(TLVL_TRACE) << "receiveInitMessage: Copying Fragment into TBufferFile: message length: " << header->data_length ;
00326 auto buffer = new char[header->data_length];
00327
00328 memcpy(buffer, &*topFrag.dataBegin(), header->data_length);
00329
00330 #if DUMP_RECEIVE_MESSAGE
00331 std::string fileName = "receiveInitMessage_" + std::to_string(getpid()) + ".bin";
00332 std::fstream ostream(fileName.c_str(), std::ios::out | std::ios::binary);
00333 ostream.write(buffer, header->data_length);
00334 ostream.close();
00335 #endif
00336
00337 msg = new TBufferFile(TBuffer::kRead, header->data_length, buffer, kTRUE, 0);
00338
00339 TLOG(TLVL_TRACE) << "receiveInitMessage END" ;
00340 init_received_ = true;
00341 }
00342 DEFINE_ART_SERVICE_INTERFACE_IMPL(NetMonTransportService, NetMonTransportServiceInterface)