00001 #include "artdaq/DAQdata/Globals.hh"
00002 #include "artdaq/ArtModules/NetMonTransportService.h"
00003 #include "artdaq/DAQrate/DataSenderManager.hh"
00004 #include "artdaq-core/Core/SharedMemoryEventReceiver.hh"
00005
00006 #include "artdaq-core/Data/Fragment.hh"
00007 #include "artdaq/DAQdata/NetMonHeader.hh"
00008 #include "artdaq-core/Data/RawEvent.hh"
00009 #include "artdaq-core/Utilities/TimeUtils.hh"
00010
00011 #include "art/Framework/Services/Registry/ActivityRegistry.h"
00012 #include "canvas/Utilities/Exception.h"
00013 #include "cetlib/container_algorithms.h"
00014 #include "cetlib_except/exception.h"
00015 #include "fhiclcpp/ParameterSet.h"
00016 #include "fhiclcpp/ParameterSetRegistry.h"
00017
00018 #include <TClass.h>
00019 #include <TBufferFile.h>
00020
00021 #include <iomanip>
00022 #include <iostream>
00023 #include <fstream>
00024 #include <string>
00025 #include <vector>
00026
00027
00028 #define DUMP_SEND_MESSAGE 0
00029 #define DUMP_RECEIVE_MESSAGE 0
00030
00031 static fhicl::ParameterSet empty_pset;
00032
00033
00034 NetMonTransportService::
00035 NetMonTransportService(fhicl::ParameterSet const& pset, art::ActivityRegistry&)
00036 : NetMonTransportServiceInterface()
00037 , data_pset_(pset)
00038 , init_received_(false)
00039 , sender_ptr_(nullptr)
00040 , incoming_events_(new artdaq::SharedMemoryEventReceiver(pset.get<int>("shared_memory_key", 0xBEE70000 + getppid()), pset.get<int>("broadcast_shared_memory_key", 0xCEE70000 + getppid())))
00041 , recvd_fragments_(nullptr)
00042 {
00043 TLOG_TRACE("NetMonTransportService") << "NetMonTransportService CONSTRUCTOR" << TLOG_ENDL;
00044 if (pset.has_key("rank")) my_rank = pset.get<int>("rank");
00045 else my_rank = incoming_events_->GetRank();
00046
00047 init_timeout_s_ = pset.get<double>("init_fragment_timeout_seconds", 1.0);
00048 }
00049
00050 NetMonTransportService::
00051 ~NetMonTransportService()
00052 {
00053 NetMonTransportService::disconnect();
00054 }
00055
00056 void
00057 NetMonTransportService::
00058 connect()
00059 {
00060 sender_ptr_.reset(new artdaq::DataSenderManager(data_pset_));
00061 }
00062
00063 void
00064 NetMonTransportService::
00065 listen()
00066 {
00067 return;
00068 }
00069
00070 void
00071 NetMonTransportService::
00072 disconnect()
00073 {
00074 if (sender_ptr_) sender_ptr_.reset(nullptr);
00075 }
00076
00077 void
00078 NetMonTransportService::
00079 sendMessage(uint64_t sequenceId, uint8_t messageType, TBufferFile& msg)
00080 {
00081 if (sender_ptr_ == nullptr)
00082 {
00083 TLOG_DEBUG("NetMonTransportService") << "Reconnecting DataSenderManager" << TLOG_ENDL;
00084 connect();
00085 }
00086
00087 #if DUMP_SEND_MESSAGE
00088 std::string fileName = "sendMessage_" + std::to_string(my_rank) + "_" + std::to_string(getpid()) + "_" + std::to_string(sequenceId) + ".bin";
00089 std::fstream ostream(fileName, std::ios::out | std::ios::binary);
00090 ostream.write(msg.Buffer(), msg.Length());
00091 ostream.close();
00092 #endif
00093
00094 TLOG_DEBUG("NetMonTransportService") << "Sending message with sequenceID=" << std::to_string(sequenceId) << ", type=" << std::to_string(messageType) << ", length=" << std::to_string(msg.Length()) << TLOG_ENDL;
00095 artdaq::NetMonHeader header;
00096 header.data_length = static_cast<uint64_t>(msg.Length());
00097 artdaq::Fragment
00098 fragment(std::ceil(msg.Length() /
00099 static_cast<double>(sizeof(artdaq::RawDataType))),
00100 sequenceId, 0, messageType, header);
00101
00102 memcpy(&*fragment.dataBegin(), msg.Buffer(), msg.Length());
00103 sender_ptr_->sendFragment(std::move(fragment));
00104 }
00105
00106 void
00107 NetMonTransportService::
00108 receiveMessage(TBufferFile*& msg)
00109 {
00110 TLOG_TRACE("NetMonTransportService") << "receiveMessage BEGIN" << TLOG_ENDL;
00111 while (recvd_fragments_ == nullptr)
00112 {
00113 TLOG_TRACE("NetMonTransportService") << "receiveMessage: Waiting for available buffer" << TLOG_ENDL;
00114 bool keep_looping = true;
00115 bool got_event = false;
00116 while (keep_looping)
00117 {
00118 keep_looping = false;
00119 got_event = incoming_events_->ReadyForRead();
00120 if (!got_event)
00121 {
00122 keep_looping = true;
00123 }
00124 }
00125
00126 TLOG_TRACE("NetMonTransportService") << "receiveMessage: Reading buffer header" << TLOG_ENDL;
00127 auto errflag = false;
00128 incoming_events_->ReadHeader(errflag);
00129 if (errflag) {
00130 msg = nullptr;
00131 return;
00132 }
00133 TLOG_TRACE("NetMonTransportService") << "receiveMessage: Getting Fragment types" << TLOG_ENDL;
00134 auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag);
00135 if (errflag) {
00136 msg = nullptr;
00137 return;
00138 }
00139 if (fragmentTypes.size() == 0)
00140 {
00141 TLOG_ERROR("NetMonTransportService") << "Event has no Fragments! Aborting!" << TLOG_ENDL;
00142 incoming_events_->ReleaseBuffer();
00143 msg = nullptr;
00144 return;
00145 }
00146 TLOG_TRACE("NetMonTransportService") << "receiveMessage: Checking first Fragment type" << TLOG_ENDL;
00147 auto firstFragmentType = *fragmentTypes.begin();
00148
00149
00150
00151
00152
00153
00154 if (!got_event || firstFragmentType == artdaq::Fragment::EndOfDataFragmentType)
00155 {
00156 TLOG_DEBUG("NetMonTransportService") << "Received shutdown message, returning" << TLOG_ENDL;
00157 incoming_events_->ReleaseBuffer();
00158 msg = nullptr;
00159 return;
00160 }
00161 if (firstFragmentType == artdaq::Fragment::InitFragmentType)
00162 {
00163 TLOG_DEBUG("NetMonTransportService") << "Cannot receive InitFragments here, retrying" << TLOG_ENDL;
00164 incoming_events_->ReleaseBuffer();
00165 continue;
00166 }
00167
00168 else if (firstFragmentType == artdaq::Fragment::EndOfRunFragmentType || firstFragmentType == artdaq::Fragment::EndOfSubrunFragmentType)
00169 {
00170 TLOG_DEBUG("NetMonTransportService") << "Ignoring EndOfRun or EndOfSubrun Fragment" << TLOG_ENDL;
00171 incoming_events_->ReleaseBuffer();
00172 continue;
00173 }
00174
00175 TLOG_TRACE("NetMonTransportService") << "receiveMessage: Getting all Fragments" << TLOG_ENDL;
00176 recvd_fragments_ = incoming_events_->GetFragmentsByType(errflag, artdaq::Fragment::InvalidFragmentType);
00177
00178
00179
00180 std::sort(recvd_fragments_->begin(), recvd_fragments_->end(),
00181 artdaq::fragmentSequenceIDCompare);
00182
00183 TLOG_TRACE("NetMonTransportService") << "receiveMessage: Releasing buffer" << TLOG_ENDL;
00184 incoming_events_->ReleaseBuffer();
00185 }
00186
00187
00188 auto start = std::chrono::steady_clock::now();
00189 while (!init_received_ && artdaq::TimeUtils::GetElapsedTime(start) < init_timeout_s_)
00190 {
00191 usleep(init_timeout_s_ * 1000000 / 100);
00192 }
00193 if (!init_received_) {
00194 TLOG_ERROR("NetMonTransportService") << "Received data but no Init Fragment after " << init_timeout_s_ << " seconds. Art will crash." << TLOG_ENDL;
00195 }
00196
00197 TLOG_TRACE("NetMonTransportService") << "receiveMessage: Returning top Fragment" << TLOG_ENDL;
00198 artdaq::Fragment topFrag = std::move(recvd_fragments_->at(0));
00199 recvd_fragments_->erase(recvd_fragments_->begin());
00200 if (recvd_fragments_->size() == 0)
00201 {
00202 recvd_fragments_.reset(nullptr);
00203 }
00204
00205 TLOG_TRACE("NetMonTransportService") << "receiveMessage: Copying Fragment into TBufferFile, length=" << topFrag.metadata<artdaq::NetMonHeader>()->data_length << TLOG_ENDL;
00206 auto header = topFrag.metadata<artdaq::NetMonHeader>();
00207 auto buffer = static_cast<char *>(malloc(header->data_length));
00208 memcpy(buffer, &*topFrag.dataBegin(), header->data_length);
00209 msg = new TBufferFile(TBuffer::kRead, header->data_length, buffer, kTRUE, 0);
00210
00211 #if DUMP_RECEIVE_MESSAGE
00212 std::string fileName = "receiveMessage_" + std::to_string(my_rank) + "_" + std::to_string(getpid()) + "_" + std::to_string(topFrag.sequenceID()) + ".bin";
00213 std::fstream ostream(fileName.c_str(), std::ios::out | std::ios::binary);
00214 ostream.write(buffer, header->data_length);
00215 ostream.close();
00216 #endif
00217
00218 TLOG_TRACE("NetMonTransportService") << "receiveMessage END" << TLOG_ENDL;
00219 }
00220
00221 void
00222 NetMonTransportService::
00223 receiveInitMessage(TBufferFile*& msg)
00224 {
00225 TLOG_TRACE("NetMonTransportService") << "receiveInitMessage BEGIN" << TLOG_ENDL;
00226 if (recvd_fragments_ == nullptr)
00227 {
00228 TLOG_TRACE("NetMonTransportService") << "receiveInitMessage: Waiting for available buffer" << TLOG_ENDL;
00229
00230 bool got_init = false;
00231 auto errflag = false;
00232 while (!got_init) {
00233
00234 bool got_event = false;
00235 while (!got_event)
00236 {
00237 got_event = incoming_events_->ReadyForRead(true);
00238 }
00239
00240 TLOG_TRACE("NetMonTransportService") << "receiveInitMessage: Reading buffer header" << TLOG_ENDL;
00241 incoming_events_->ReadHeader(errflag);
00242 if (errflag) {
00243 TLOG_ERROR("NetMonTransportService") << "receiveInitMessage: Error receiving message!" << TLOG_ENDL;
00244 msg = nullptr;
00245 return;
00246 }
00247 TLOG_TRACE("NetMonTransportService") << "receiveInitMessage: Getting Fragment types" << TLOG_ENDL;
00248 auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag);
00249 if (errflag) {
00250 msg = nullptr;
00251 TLOG_ERROR("NetMonTransportService") << "receiveInitMessage: Error receiving message!" << TLOG_ENDL;
00252 return;
00253 }
00254 if (fragmentTypes.size() == 0)
00255 {
00256 TLOG_ERROR("NetMonTransportService") << "Event has no Fragments! Aborting!" << TLOG_ENDL;
00257 incoming_events_->ReleaseBuffer();
00258 msg = nullptr;
00259 return;
00260 }
00261 TLOG_TRACE("NetMonTransportService") << "receiveInitMessage: Checking first Fragment type" << TLOG_ENDL;
00262 auto firstFragmentType = *fragmentTypes.begin();
00263
00264
00265
00266
00267
00268
00269 if (!got_event || firstFragmentType == artdaq::Fragment::EndOfDataFragmentType)
00270 {
00271 TLOG_DEBUG("NetMonTransportService") << "Received shutdown message, returning" << TLOG_ENDL;
00272 incoming_events_->ReleaseBuffer();
00273 msg = nullptr;
00274 return;
00275 }
00276 if (firstFragmentType != artdaq::Fragment::InitFragmentType)
00277 {
00278 TLOG_WARNING("NetMonTransportService") << "Did NOT receive Init Fragment as first broadcast! Type=" << artdaq::detail::RawFragmentHeader::SystemTypeToString(firstFragmentType) << TLOG_ENDL;
00279 incoming_events_->ReleaseBuffer();
00280 }
00281 got_init = true;
00282 }
00283 TLOG_TRACE("NetMonTransportService") << "receiveInitMessage: Getting all Fragments" << TLOG_ENDL;
00284 recvd_fragments_ = incoming_events_->GetFragmentsByType(errflag, artdaq::Fragment::InvalidFragmentType);
00285
00286
00287
00288 std::sort(recvd_fragments_->begin(), recvd_fragments_->end(),
00289 artdaq::fragmentSequenceIDCompare);
00290 }
00291
00292 TLOG_TRACE("NetMonTransportService") << "receiveInitMessage: Returning top Fragment" << TLOG_ENDL;
00293 artdaq::Fragment topFrag = std::move(recvd_fragments_->at(0));
00294 recvd_fragments_->erase(recvd_fragments_->begin());
00295 if (recvd_fragments_->size() == 0)
00296 {
00297 recvd_fragments_.reset(nullptr);
00298 }
00299
00300 auto header = topFrag.metadata<artdaq::NetMonHeader>();
00301 TLOG_TRACE("NetMonTransportService") << "receiveInitMessage: Copying Fragment into TBufferFile: message length: " << std::to_string(header->data_length) << TLOG_ENDL;
00302 auto buffer = static_cast<char *>(malloc(header->data_length));
00303 memcpy(buffer, &*topFrag.dataBegin(), header->data_length);
00304
00305 #if DUMP_RECEIVE_MESSAGE
00306 std::string fileName = "receiveInitMessage_" + std::to_string(getpid()) + ".bin";
00307 std::fstream ostream(fileName.c_str(), std::ios::out | std::ios::binary);
00308 ostream.write(buffer, header->data_length);
00309 ostream.close();
00310 #endif
00311
00312 msg = new TBufferFile(TBuffer::kRead, header->data_length, buffer, kTRUE, 0);
00313
00314 TLOG_TRACE("NetMonTransportService") << "receiveInitMessage END" << TLOG_ENDL;
00315 init_received_ = true;
00316 }
00317 DEFINE_ART_SERVICE_INTERFACE_IMPL(NetMonTransportService, NetMonTransportServiceInterface)