00001 #define TRACE_NAME "NetMonTransportService"
00002
00003 #include "artdaq/DAQdata/Globals.hh"
00004 #include "artdaq/ArtModules/NetMonTransportService.h"
00005 #include "artdaq/DAQrate/DataSenderManager.hh"
00006 #include "artdaq-core/Core/SharedMemoryEventReceiver.hh"
00007
00008 #include "artdaq-core/Data/Fragment.hh"
00009 #include "artdaq/DAQdata/NetMonHeader.hh"
00010 #include "artdaq-core/Data/RawEvent.hh"
00011 #include "artdaq-core/Utilities/TimeUtils.hh"
00012
00013 #include "art/Framework/Services/Registry/ActivityRegistry.h"
00014 #include "canvas/Utilities/Exception.h"
00015 #include "cetlib/container_algorithms.h"
00016 #include "cetlib_except/exception.h"
00017 #include "fhiclcpp/ParameterSet.h"
00018 #include "fhiclcpp/ParameterSetRegistry.h"
00019
00020 #include <TClass.h>
00021 #include <TBufferFile.h>
00022
00023 #include <iomanip>
00024 #include <iostream>
00025 #include <fstream>
00026 #include <string>
00027 #include <vector>
00028
00029
00030 #define DUMP_SEND_MESSAGE 0
00031 #define DUMP_RECEIVE_MESSAGE 0
00032
00033 static fhicl::ParameterSet empty_pset;
00034
00035
00036 NetMonTransportService::
00037 NetMonTransportService(fhicl::ParameterSet const& pset, art::ActivityRegistry&)
00038 : NetMonTransportServiceInterface()
00039 , data_pset_(pset)
00040 , init_received_(false)
00041 , sender_ptr_(nullptr)
00042 , incoming_events_(nullptr)
00043 , recvd_fragments_(nullptr)
00044 {
00045 TLOG(TLVL_TRACE) << "NetMonTransportService CONSTRUCTOR" ;
00046 if (pset.has_key("rank")) my_rank = pset.get<int>("rank");
00047
00048 init_timeout_s_ = pset.get<double>("init_fragment_timeout_seconds", 1.0);
00049 }
00050
00051 NetMonTransportService::
00052 ~NetMonTransportService()
00053 {
00054 NetMonTransportService::disconnect();
00055 }
00056
00057 void
00058 NetMonTransportService::
00059 connect()
00060 {
00061 sender_ptr_.reset(new artdaq::DataSenderManager(data_pset_));
00062 }
00063
00064 void
00065 NetMonTransportService::
00066 listen()
00067 {
00068 if (!incoming_events_)
00069 {
00070 incoming_events_.reset(new artdaq::SharedMemoryEventReceiver(data_pset_.get<int>("shared_memory_key", 0xBEE70000 + getppid()), data_pset_.get<int>("broadcast_shared_memory_key", 0xCEE70000 + getppid())));
00071
00072 char const* artapp_env = getenv("ARTDAQ_APPLICATION_NAME");
00073 std::string artapp_str = "";
00074 if (artapp_env != NULL)
00075 {
00076 artapp_str = std::string(artapp_env) + "_";
00077 }
00078
00079 app_name = artapp_str + "art" + std::to_string(incoming_events_->GetMyId());
00080
00081 if (data_pset_.has_key("rank")) my_rank = data_pset_.get<int>("rank");
00082 else my_rank = incoming_events_->GetRank();
00083 }
00084 return;
00085 }
00086
00087 void
00088 NetMonTransportService::
00089 disconnect()
00090 {
00091 if (sender_ptr_) sender_ptr_.reset(nullptr);
00092 }
00093
00094 void
00095 NetMonTransportService::
00096 sendMessage(uint64_t sequenceId, uint8_t messageType, TBufferFile& msg)
00097 {
00098 if (sender_ptr_ == nullptr)
00099 {
00100 TLOG(TLVL_DEBUG) << "Reconnecting DataSenderManager" ;
00101 connect();
00102 }
00103
00104 #if DUMP_SEND_MESSAGE
00105 std::string fileName = "sendMessage_" + std::to_string(my_rank) + "_" + std::to_string(getpid()) + "_" + std::to_string(sequenceId) + ".bin";
00106 std::fstream ostream(fileName, std::ios::out | std::ios::binary);
00107 ostream.write(msg.Buffer(), msg.Length());
00108 ostream.close();
00109 #endif
00110
00111 TLOG(TLVL_DEBUG) << "Sending message with sequenceID=" << sequenceId << ", type=" << (int)messageType << ", length=" << msg.Length() ;
00112 artdaq::NetMonHeader header;
00113 header.data_length = static_cast<uint64_t>(msg.Length());
00114 artdaq::Fragment
00115 fragment(std::ceil(msg.Length() /
00116 static_cast<double>(sizeof(artdaq::RawDataType))),
00117 sequenceId, 0, messageType, header);
00118
00119 memcpy(&*fragment.dataBegin(), msg.Buffer(), msg.Length());
00120 sender_ptr_->sendFragment(std::move(fragment));
00121 }
00122
00123 void
00124 NetMonTransportService::
00125 receiveMessage(TBufferFile*& msg)
00126 {
00127 listen();
00128 TLOG(TLVL_TRACE) << "receiveMessage BEGIN" ;
00129 while (recvd_fragments_ == nullptr)
00130 {
00131 TLOG(TLVL_TRACE) << "receiveMessage: Waiting for available buffer" ;
00132 bool keep_looping = true;
00133 bool got_event = false;
00134 while (keep_looping)
00135 {
00136 keep_looping = false;
00137 got_event = incoming_events_->ReadyForRead();
00138 if (!got_event)
00139 {
00140 keep_looping = true;
00141 }
00142 }
00143
00144 TLOG(TLVL_TRACE) << "receiveMessage: Reading buffer header" ;
00145 auto errflag = false;
00146 incoming_events_->ReadHeader(errflag);
00147 if (errflag) {
00148 msg = nullptr;
00149 return;
00150 }
00151 TLOG(TLVL_TRACE) << "receiveMessage: Getting Fragment types" ;
00152 auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag);
00153 if (errflag) {
00154 incoming_events_->ReleaseBuffer();
00155 msg = nullptr;
00156 return;
00157 }
00158 if (fragmentTypes.size() == 0)
00159 {
00160 TLOG(TLVL_ERROR) << "Event has no Fragments! Aborting!" ;
00161 incoming_events_->ReleaseBuffer();
00162 msg = nullptr;
00163 return;
00164 }
00165 TLOG(TLVL_TRACE) << "receiveMessage: Checking first Fragment type" ;
00166 auto firstFragmentType = *fragmentTypes.begin();
00167
00168
00169
00170
00171
00172
00173 if (!got_event || firstFragmentType == artdaq::Fragment::EndOfDataFragmentType)
00174 {
00175 TLOG(TLVL_DEBUG) << "Received shutdown message, returning from receiveMessage "
00176 << "(debug: got_event=" << got_event << ",fragType=" << (int)firstFragmentType
00177 << ",EODFragType=" << (int)artdaq::Fragment::EndOfDataFragmentType << ")";
00178 incoming_events_->ReleaseBuffer();
00179 msg = nullptr;
00180 return;
00181 }
00182 if (firstFragmentType == artdaq::Fragment::InitFragmentType)
00183 {
00184 TLOG(TLVL_DEBUG) << "Cannot receive InitFragments here, retrying" ;
00185 incoming_events_->ReleaseBuffer();
00186 continue;
00187 }
00188
00189 else if (firstFragmentType == artdaq::Fragment::EndOfRunFragmentType || firstFragmentType == artdaq::Fragment::EndOfSubrunFragmentType)
00190 {
00191 TLOG(TLVL_DEBUG) << "Ignoring EndOfRun or EndOfSubrun Fragment" ;
00192 incoming_events_->ReleaseBuffer();
00193 continue;
00194 }
00195
00196 TLOG(TLVL_TRACE) << "receiveMessage: Getting all Fragments" ;
00197 recvd_fragments_ = incoming_events_->GetFragmentsByType(errflag, artdaq::Fragment::InvalidFragmentType);
00198 if (!recvd_fragments_)
00199 {
00200 TLOG(TLVL_ERROR) << "Error retrieving Fragments from shared memory! Aborting!";
00201 incoming_events_->ReleaseBuffer();
00202 msg = nullptr;
00203 return;
00204
00205 }
00206
00207
00208
00209 std::sort(recvd_fragments_->begin(), recvd_fragments_->end(),
00210 artdaq::fragmentSequenceIDCompare);
00211
00212 TLOG(TLVL_TRACE) << "receiveMessage: Releasing buffer" ;
00213 incoming_events_->ReleaseBuffer();
00214 }
00215
00216
00217 auto start = std::chrono::steady_clock::now();
00218 while (!init_received_ && artdaq::TimeUtils::GetElapsedTime(start) < init_timeout_s_)
00219 {
00220 usleep(init_timeout_s_ * 1000000 / 100);
00221 }
00222 if (!init_received_) {
00223 TLOG(TLVL_ERROR) << "Received data but no Init Fragment after " << init_timeout_s_ << " seconds. Art will crash." ;
00224 }
00225
00226 TLOG(TLVL_TRACE) << "receiveMessage: Returning top Fragment" ;
00227 artdaq::Fragment topFrag = std::move(recvd_fragments_->at(0));
00228 recvd_fragments_->erase(recvd_fragments_->begin());
00229 if (recvd_fragments_->size() == 0)
00230 {
00231 recvd_fragments_.reset(nullptr);
00232 }
00233
00234 TLOG(TLVL_TRACE) << "receiveMessage: Copying Fragment into TBufferFile, length=" << topFrag.metadata<artdaq::NetMonHeader>()->data_length ;
00235 auto header = topFrag.metadata<artdaq::NetMonHeader>();
00236 auto buffer = static_cast<char *>(malloc(header->data_length));
00237 memcpy(buffer, &*topFrag.dataBegin(), header->data_length);
00238 msg = new TBufferFile(TBuffer::kRead, header->data_length, buffer, kTRUE, 0);
00239
00240 #if DUMP_RECEIVE_MESSAGE
00241 std::string fileName = "receiveMessage_" + std::to_string(my_rank) + "_" + std::to_string(getpid()) + "_" + std::to_string(topFrag.sequenceID()) + ".bin";
00242 std::fstream ostream(fileName.c_str(), std::ios::out | std::ios::binary);
00243 ostream.write(buffer, header->data_length);
00244 ostream.close();
00245 #endif
00246
00247 TLOG(TLVL_TRACE) << "receiveMessage END" ;
00248 }
00249
00250 void
00251 NetMonTransportService::
00252 receiveInitMessage(TBufferFile*& msg)
00253 {
00254 listen();
00255 TLOG(TLVL_TRACE) << "receiveInitMessage BEGIN" ;
00256 if (recvd_fragments_ == nullptr)
00257 {
00258 TLOG(TLVL_TRACE) << "receiveInitMessage: Waiting for available buffer" ;
00259
00260 bool got_init = false;
00261 auto errflag = false;
00262 while (!got_init) {
00263
00264 bool got_event = false;
00265 while (!got_event)
00266 {
00267 got_event = incoming_events_->ReadyForRead(true);
00268 }
00269
00270 TLOG(TLVL_TRACE) << "receiveInitMessage: Reading buffer header" ;
00271 incoming_events_->ReadHeader(errflag);
00272 if (errflag) {
00273 TLOG(TLVL_ERROR) << "receiveInitMessage: Error receiving message!" ;
00274 incoming_events_->ReleaseBuffer();
00275 msg = nullptr;
00276 return;
00277 }
00278 TLOG(TLVL_TRACE) << "receiveInitMessage: Getting Fragment types" ;
00279 auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag);
00280 if (errflag) {
00281 incoming_events_->ReleaseBuffer();
00282 msg = nullptr;
00283 TLOG(TLVL_ERROR) << "receiveInitMessage: Error receiving message!" ;
00284 return;
00285 }
00286 if (fragmentTypes.size() == 0)
00287 {
00288 TLOG(TLVL_ERROR) << "Event has no Fragments! Aborting!" ;
00289 incoming_events_->ReleaseBuffer();
00290 msg = nullptr;
00291 return;
00292 }
00293 TLOG(TLVL_TRACE) << "receiveInitMessage: Checking first Fragment type" ;
00294 auto firstFragmentType = *fragmentTypes.begin();
00295
00296
00297
00298
00299
00300
00301 if (!got_event || firstFragmentType == artdaq::Fragment::EndOfDataFragmentType)
00302 {
00303 TLOG(TLVL_DEBUG) << "Received shutdown message, returning" ;
00304 incoming_events_->ReleaseBuffer();
00305 msg = nullptr;
00306 return;
00307 }
00308 if (firstFragmentType != artdaq::Fragment::InitFragmentType)
00309 {
00310 TLOG(TLVL_WARNING) << "Did NOT receive Init Fragment as first broadcast! Type=" << artdaq::detail::RawFragmentHeader::SystemTypeToString(firstFragmentType) ;
00311 incoming_events_->ReleaseBuffer();
00312 }
00313 got_init = true;
00314 }
00315 TLOG(TLVL_TRACE) << "receiveInitMessage: Getting all Fragments" ;
00316 recvd_fragments_ = incoming_events_->GetFragmentsByType(errflag, artdaq::Fragment::InvalidFragmentType);
00317
00318
00319
00320 std::sort(recvd_fragments_->begin(), recvd_fragments_->end(),
00321 artdaq::fragmentSequenceIDCompare);
00322
00323 incoming_events_->ReleaseBuffer();
00324 }
00325
00326 TLOG(TLVL_TRACE) << "receiveInitMessage: Returning top Fragment" ;
00327 artdaq::Fragment topFrag = std::move(recvd_fragments_->at(0));
00328 recvd_fragments_->erase(recvd_fragments_->begin());
00329 if (recvd_fragments_->size() == 0)
00330 {
00331 recvd_fragments_.reset(nullptr);
00332 }
00333
00334 auto header = topFrag.metadata<artdaq::NetMonHeader>();
00335 TLOG(TLVL_TRACE) << "receiveInitMessage: Copying Fragment into TBufferFile: message length: " << header->data_length ;
00336 auto buffer = new char[header->data_length];
00337
00338 memcpy(buffer, &*topFrag.dataBegin(), header->data_length);
00339
00340 #if DUMP_RECEIVE_MESSAGE
00341 std::string fileName = "receiveInitMessage_" + std::to_string(getpid()) + ".bin";
00342 std::fstream ostream(fileName.c_str(), std::ios::out | std::ios::binary);
00343 ostream.write(buffer, header->data_length);
00344 ostream.close();
00345 #endif
00346
00347 msg = new TBufferFile(TBuffer::kRead, header->data_length, buffer, kTRUE, 0);
00348
00349 TLOG(TLVL_TRACE) << "receiveInitMessage END" ;
00350 init_received_ = true;
00351 }
00352 DEFINE_ART_SERVICE_INTERFACE_IMPL(NetMonTransportService, NetMonTransportServiceInterface)