00001 #include "artdaq/DAQdata/Globals.hh"
00002 #include "artdaq/ArtModules/NetMonTransportService.h"
00003 #include "artdaq/DAQrate/DataSenderManager.hh"
00004 #include "artdaq-core/Core/SharedMemoryEventReceiver.hh"
00005
00006 #include "artdaq-core/Data/Fragment.hh"
00007 #include "artdaq/DAQdata/NetMonHeader.hh"
00008 #include "artdaq-core/Data/RawEvent.hh"
00009 #include "artdaq-core/Utilities/TimeUtils.hh"
00010
00011 #include "art/Framework/Services/Registry/ActivityRegistry.h"
00012 #include "canvas/Utilities/Exception.h"
00013 #include "cetlib/container_algorithms.h"
00014 #include "cetlib_except/exception.h"
00015 #include "fhiclcpp/ParameterSet.h"
00016 #include "fhiclcpp/ParameterSetRegistry.h"
00017
00018 #include <TClass.h>
00019 #include <TBufferFile.h>
00020
00021 #include <iomanip>
00022 #include <iostream>
00023 #include <fstream>
00024 #include <string>
00025 #include <vector>
00026
00027 static fhicl::ParameterSet empty_pset;
00028
00029
00030 NetMonTransportService::
00031 NetMonTransportService(fhicl::ParameterSet const& pset, art::ActivityRegistry&)
00032 : NetMonTransportServiceInterface()
00033 , data_pset_(pset)
00034 , init_received_(false)
00035 , sender_ptr_(nullptr)
00036 , incoming_events_(new artdaq::SharedMemoryEventReceiver(pset.get<int>("shared_memory_key", 0xBEE70000 + getppid()), pset.get<int>("broadcast_shared_memory_key", 0xCEE70000 + getppid())))
00037 , recvd_fragments_(nullptr)
00038 {
00039 TLOG_TRACE("NetMonTransportService") << "NetMonTransportService CONSTRUCTOR" << TLOG_ENDL;
00040 if (pset.has_key("rank")) my_rank = pset.get<int>("rank");
00041 else my_rank = incoming_events_->GetRank();
00042
00043 init_timeout_s_ = pset.get<double>("init_fragment_timeout_seconds", 1.0);
00044 }
00045
00046 NetMonTransportService::
00047 ~NetMonTransportService()
00048 {
00049 NetMonTransportService::disconnect();
00050 }
00051
00052 void
00053 NetMonTransportService::
00054 connect()
00055 {
00056 sender_ptr_.reset(new artdaq::DataSenderManager(data_pset_));
00057 }
00058
00059 void
00060 NetMonTransportService::
00061 listen()
00062 {
00063 return;
00064 }
00065
00066 void
00067 NetMonTransportService::
00068 disconnect()
00069 {
00070 if (sender_ptr_) sender_ptr_.reset(nullptr);
00071 }
00072
00073 void
00074 NetMonTransportService::
00075 sendMessage(uint64_t sequenceId, uint8_t messageType, TBufferFile& msg)
00076 {
00077 if (sender_ptr_ == nullptr)
00078 {
00079 TLOG_DEBUG("NetMonTransportService") << "Reconnecting DataSenderManager" << TLOG_ENDL;
00080 connect();
00081 }
00082
00083 #if 0
00084 if (messageType == artdaq::Fragment::InitFragmentType) {
00085 std::fstream ostream("sendInitMessage.bin", std::ios::out | std::ios::binary);
00086 ostream.write(msg.Buffer(), msg.Length());
00087 ostream.close();
00088 }
00089 #endif
00090
00091 TLOG_DEBUG("NetMonTransportService") << "Sending message with sequenceID=" << std::to_string(sequenceId) << ", type=" << std::to_string(messageType) << ", length=" << std::to_string(msg.Length()) << TLOG_ENDL;
00092 artdaq::NetMonHeader header;
00093 header.data_length = static_cast<uint64_t>(msg.Length());
00094 artdaq::Fragment
00095 fragment(std::ceil(msg.Length() /
00096 static_cast<double>(sizeof(artdaq::RawDataType))),
00097 sequenceId, 0, messageType, header);
00098
00099 memcpy(&*fragment.dataBegin(), msg.Buffer(), msg.Length());
00100 sender_ptr_->sendFragment(std::move(fragment));
00101 }
00102
00103 void
00104 NetMonTransportService::
00105 receiveMessage(TBufferFile*& msg)
00106 {
00107 TLOG_TRACE("NetMonTransportService") << "receiveMessage BEGIN" << TLOG_ENDL;
00108 while (recvd_fragments_ == nullptr)
00109 {
00110 TLOG_TRACE("NetMonTransportService") << "receiveMessage: Waiting for available buffer" << TLOG_ENDL;
00111 bool keep_looping = true;
00112 bool got_event = false;
00113 while (keep_looping)
00114 {
00115 keep_looping = false;
00116 got_event = incoming_events_->ReadyForRead();
00117 if (!got_event)
00118 {
00119 keep_looping = true;
00120 }
00121 }
00122
00123 TLOG_TRACE("NetMonTransportService") << "receiveMessage: Reading buffer header" << TLOG_ENDL;
00124 auto errflag = false;
00125 incoming_events_->ReadHeader(errflag);
00126 if (errflag) {
00127 msg = nullptr;
00128 return;
00129 }
00130 TLOG_TRACE("NetMonTransportService") << "receiveMessage: Getting Fragment types" << TLOG_ENDL;
00131 auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag);
00132 if (errflag) {
00133 msg = nullptr;
00134 return;
00135 }
00136 if (fragmentTypes.size() == 0)
00137 {
00138 TLOG_ERROR("NetMonTransportService") << "Event has no Fragments! Aborting!" << TLOG_ENDL;
00139 incoming_events_->ReleaseBuffer();
00140 msg = nullptr;
00141 return;
00142 }
00143 TLOG_TRACE("NetMonTransportService") << "receiveMessage: Checking first Fragment type" << TLOG_ENDL;
00144 auto firstFragmentType = *fragmentTypes.begin();
00145
00146
00147
00148
00149
00150
00151 if (!got_event || firstFragmentType == artdaq::Fragment::EndOfDataFragmentType)
00152 {
00153 TLOG_DEBUG("NetMonTransportService") << "Received shutdown message, returning" << TLOG_ENDL;
00154 incoming_events_->ReleaseBuffer();
00155 msg = nullptr;
00156 return;
00157 }
00158 if (firstFragmentType == artdaq::Fragment::InitFragmentType)
00159 {
00160 TLOG_DEBUG("NetMonTransportService") << "Cannot receive InitFragments here, retrying" << TLOG_ENDL;
00161 incoming_events_->ReleaseBuffer();
00162 continue;
00163 }
00164
00165 else if (firstFragmentType == artdaq::Fragment::EndOfRunFragmentType || firstFragmentType == artdaq::Fragment::EndOfSubrunFragmentType)
00166 {
00167 TLOG_DEBUG("NetMonTransportService") << "Ignoring EndOfRun or EndOfSubrun Fragment" << TLOG_ENDL;
00168 incoming_events_->ReleaseBuffer();
00169 continue;
00170 }
00171
00172 TLOG_TRACE("NetMonTransportService") << "receiveMessage: Getting all Fragments" << TLOG_ENDL;
00173 recvd_fragments_ = incoming_events_->GetFragmentsByType(errflag, artdaq::Fragment::InvalidFragmentType);
00174
00175
00176
00177 std::sort(recvd_fragments_->begin(), recvd_fragments_->end(),
00178 artdaq::fragmentSequenceIDCompare);
00179
00180 TLOG_TRACE("NetMonTransportService") << "receiveMessage: Releasing buffer" << TLOG_ENDL;
00181 incoming_events_->ReleaseBuffer();
00182 }
00183
00184
00185 auto start = std::chrono::steady_clock::now();
00186 while (!init_received_ && artdaq::TimeUtils::GetElapsedTime(start) < init_timeout_s_)
00187 {
00188 usleep(init_timeout_s_ * 1000000 / 100);
00189 }
00190 if (!init_received_) {
00191 TLOG_ERROR("NetMonTransportService") << "Received data but no Init Fragment after " << init_timeout_s_ << " seconds. Art will crash." << TLOG_ENDL;
00192 }
00193
00194 TLOG_TRACE("NetMonTransportService") << "receiveMessage: Returning top Fragment" << TLOG_ENDL;
00195 artdaq::Fragment topFrag = std::move(recvd_fragments_->at(0));
00196 recvd_fragments_->erase(recvd_fragments_->begin());
00197 if (recvd_fragments_->size() == 0)
00198 {
00199 recvd_fragments_.reset(nullptr);
00200 }
00201
00202 TLOG_TRACE("NetMonTransportService") << "receiveMessage: Copying Fragment into TBufferFile" << TLOG_ENDL;
00203 auto header = topFrag.metadata<artdaq::NetMonHeader>();
00204 auto buffer = static_cast<char *>(malloc(header->data_length));
00205 memcpy(buffer, &*topFrag.dataBegin(), header->data_length);
00206 msg = new TBufferFile(TBuffer::kRead, header->data_length, buffer, kTRUE, 0);
00207
00208 TLOG_TRACE("NetMonTransportService") << "receiveMessage END" << TLOG_ENDL;
00209 }
00210
00211 void
00212 NetMonTransportService::
00213 receiveInitMessage(TBufferFile*& msg)
00214 {
00215 TLOG_TRACE("NetMonTransportService") << "receiveInitMessage BEGIN" << TLOG_ENDL;
00216 if (recvd_fragments_ == nullptr)
00217 {
00218 TLOG_TRACE("NetMonTransportService") << "receiveInitMessage: Waiting for available buffer" << TLOG_ENDL;
00219
00220 bool got_init = false;
00221 auto errflag = false;
00222 while (!got_init) {
00223
00224 bool got_event = false;
00225 while (!got_event)
00226 {
00227 got_event = incoming_events_->ReadyForRead(true);
00228 }
00229
00230 TLOG_TRACE("NetMonTransportService") << "receiveInitMessage: Reading buffer header" << TLOG_ENDL;
00231 incoming_events_->ReadHeader(errflag);
00232 if (errflag) {
00233 TLOG_ERROR("NetMonTransportService") << "receiveInitMessage: Error receiving message!" << TLOG_ENDL;
00234 msg = nullptr;
00235 return;
00236 }
00237 TLOG_TRACE("NetMonTransportService") << "receiveInitMessage: Getting Fragment types" << TLOG_ENDL;
00238 auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag);
00239 if (errflag) {
00240 msg = nullptr;
00241 TLOG_ERROR("NetMonTransportService") << "receiveInitMessage: Error receiving message!" << TLOG_ENDL;
00242 return;
00243 }
00244 if (fragmentTypes.size() == 0)
00245 {
00246 TLOG_ERROR("NetMonTransportService") << "Event has no Fragments! Aborting!" << TLOG_ENDL;
00247 incoming_events_->ReleaseBuffer();
00248 msg = nullptr;
00249 return;
00250 }
00251 TLOG_TRACE("NetMonTransportService") << "receiveInitMessage: Checking first Fragment type" << TLOG_ENDL;
00252 auto firstFragmentType = *fragmentTypes.begin();
00253
00254
00255
00256
00257
00258
00259 if (!got_event || firstFragmentType == artdaq::Fragment::EndOfDataFragmentType)
00260 {
00261 TLOG_DEBUG("NetMonTransportService") << "Received shutdown message, returning" << TLOG_ENDL;
00262 incoming_events_->ReleaseBuffer();
00263 msg = nullptr;
00264 return;
00265 }
00266 if (firstFragmentType != artdaq::Fragment::InitFragmentType)
00267 {
00268 TLOG_WARNING("NetMonTransportService") << "Did NOT receive Init Fragment as first broadcast! Type=" << artdaq::detail::RawFragmentHeader::SystemTypeToString(firstFragmentType) << TLOG_ENDL;
00269 incoming_events_->ReleaseBuffer();
00270 }
00271 got_init = true;
00272 }
00273 TLOG_TRACE("NetMonTransportService") << "receiveInitMessage: Getting all Fragments" << TLOG_ENDL;
00274 recvd_fragments_ = incoming_events_->GetFragmentsByType(errflag, artdaq::Fragment::InvalidFragmentType);
00275
00276
00277
00278 std::sort(recvd_fragments_->begin(), recvd_fragments_->end(),
00279 artdaq::fragmentSequenceIDCompare);
00280 }
00281
00282 TLOG_TRACE("NetMonTransportService") << "receiveInitMessage: Returning top Fragment" << TLOG_ENDL;
00283 artdaq::Fragment topFrag = std::move(recvd_fragments_->at(0));
00284 recvd_fragments_->erase(recvd_fragments_->begin());
00285 if (recvd_fragments_->size() == 0)
00286 {
00287 recvd_fragments_.reset(nullptr);
00288 }
00289
00290 auto header = topFrag.metadata<artdaq::NetMonHeader>();
00291 TLOG_TRACE("NetMonTransportService") << "receiveInitMessage: Copying Fragment into TBufferFile: message length: " << std::to_string(header->data_length) << TLOG_ENDL;
00292 auto buffer = static_cast<char *>(malloc(header->data_length));
00293 memcpy(buffer, &*topFrag.dataBegin(), header->data_length);
00294
00295 #if 0
00296 std::string fileName = "receiveInitMessage_" + std::to_string(getpid()) + ".bin";
00297 std::fstream ostream(fileName.c_str(), std::ios::out | std::ios::binary);
00298 ostream.write(buffer, header->data_length);
00299 ostream.close();
00300 #endif
00301
00302 msg = new TBufferFile(TBuffer::kRead, header->data_length, buffer, kTRUE, 0);
00303
00304 TLOG_TRACE("NetMonTransportService") << "receiveInitMessage END" << TLOG_ENDL;
00305 init_received_ = true;
00306 }
00307 DEFINE_ART_SERVICE_INTERFACE_IMPL(NetMonTransportService, NetMonTransportServiceInterface)