$treeview $search $mathjax $extrastylesheet
artdaq
v3_04_01
$projectbrief
|
$projectbrief
|
$searchbox |
00001 #define TRACE_NAME "NetMonTransportService" 00002 00003 #include "artdaq/DAQdata/Globals.hh" 00004 #include "artdaq/ArtModules/NetMonTransportService.h" 00005 #include "artdaq/DAQrate/DataSenderManager.hh" 00006 #include "artdaq-core/Core/SharedMemoryEventReceiver.hh" 00007 00008 #include "artdaq-core/Data/Fragment.hh" 00009 #include "artdaq/DAQdata/NetMonHeader.hh" 00010 #include "artdaq-core/Data/RawEvent.hh" 00011 #include "artdaq-core/Utilities/TimeUtils.hh" 00012 00013 #include "art/Framework/Services/Registry/ActivityRegistry.h" 00014 #include "canvas/Utilities/Exception.h" 00015 #include "cetlib/container_algorithms.h" 00016 #include "cetlib_except/exception.h" 00017 #include "fhiclcpp/ParameterSet.h" 00018 #include "fhiclcpp/ParameterSetRegistry.h" 00019 00020 #include <TClass.h> 00021 #include <TBufferFile.h> 00022 00023 #include <iomanip> 00024 #include <iostream> 00025 #include <fstream> 00026 #include <string> 00027 #include <vector> 00028 00029 00030 #define DUMP_SEND_MESSAGE 0 00031 #define DUMP_RECEIVE_MESSAGE 0 00032 00033 static fhicl::ParameterSet empty_pset; 00034 00035 00036 NetMonTransportService:: 00037 NetMonTransportService(fhicl::ParameterSet const& pset, art::ActivityRegistry&) 00038 : NetMonTransportServiceInterface() 00039 , data_pset_(pset) 00040 , init_received_(false) 00041 , sender_ptr_(nullptr) 00042 , incoming_events_(nullptr) 00043 , recvd_fragments_(nullptr) 00044 { 00045 TLOG(TLVL_TRACE) << "NetMonTransportService CONSTRUCTOR" ; 00046 if (pset.has_key("rank")) my_rank = pset.get<int>("rank"); 00047 00048 init_timeout_s_ = pset.get<double>("init_fragment_timeout_seconds", 1.0); 00049 } 00050 00051 NetMonTransportService:: 00052 ~NetMonTransportService() 00053 { 00054 NetMonTransportService::disconnect(); 00055 } 00056 00057 void 00058 NetMonTransportService:: 00059 connect() 00060 { 00061 sender_ptr_.reset(new artdaq::DataSenderManager(data_pset_)); 00062 } 00063 00064 void 00065 NetMonTransportService:: 00066 listen() 00067 { 00068 if (!incoming_events_) 00069 { 00070 incoming_events_.reset(new artdaq::SharedMemoryEventReceiver(data_pset_.get<int>("shared_memory_key", 0xBEE70000 + getppid()), data_pset_.get<int>("broadcast_shared_memory_key", 0xCEE70000 + getppid()))); 00071 00072 char const* artapp_env = getenv("ARTDAQ_APPLICATION_NAME"); 00073 std::string artapp_str = ""; 00074 if (artapp_env != NULL) 00075 { 00076 artapp_str = std::string(artapp_env) + "_"; 00077 } 00078 00079 app_name = artapp_str + "art" + std::to_string(incoming_events_->GetMyId()); 00080 00081 if (data_pset_.has_key("rank")) my_rank = data_pset_.get<int>("rank"); 00082 else my_rank = incoming_events_->GetRank(); 00083 } 00084 return; 00085 } 00086 00087 void 00088 NetMonTransportService:: 00089 disconnect() 00090 { 00091 if (sender_ptr_) sender_ptr_.reset(nullptr); 00092 } 00093 00094 void 00095 NetMonTransportService:: 00096 sendMessage(uint64_t sequenceId, uint8_t messageType, TBufferFile& msg) 00097 { 00098 if (sender_ptr_ == nullptr) 00099 { 00100 TLOG(TLVL_DEBUG) << "Reconnecting DataSenderManager" ; 00101 connect(); 00102 } 00103 00104 #if DUMP_SEND_MESSAGE 00105 std::string fileName = "sendMessage_" + std::to_string(my_rank) + "_" + std::to_string(getpid()) + "_" + std::to_string(sequenceId) + ".bin"; 00106 std::fstream ostream(fileName, std::ios::out | std::ios::binary); 00107 ostream.write(msg.Buffer(), msg.Length()); 00108 ostream.close(); 00109 #endif 00110 00111 TLOG(TLVL_DEBUG) << "Sending message with sequenceID=" << sequenceId << ", type=" << (int)messageType << ", length=" << msg.Length() ; 00112 artdaq::NetMonHeader header; 00113 header.data_length = static_cast<uint64_t>(msg.Length()); 00114 artdaq::Fragment 00115 fragment(std::ceil(msg.Length() / 00116 static_cast<double>(sizeof(artdaq::RawDataType))), 00117 sequenceId, 0, messageType, header); 00118 00119 memcpy(&*fragment.dataBegin(), msg.Buffer(), msg.Length()); 00120 sender_ptr_->sendFragment(std::move(fragment)); 00121 } 00122 00123 void 00124 NetMonTransportService:: 00125 receiveMessage(TBufferFile*& msg) 00126 { 00127 listen(); 00128 TLOG(TLVL_TRACE) << "receiveMessage BEGIN" ; 00129 while (recvd_fragments_ == nullptr) 00130 { 00131 TLOG(TLVL_TRACE) << "receiveMessage: Waiting for available buffer" ; 00132 bool keep_looping = true; 00133 bool got_event = false; 00134 while (keep_looping) 00135 { 00136 keep_looping = false; 00137 got_event = incoming_events_->ReadyForRead(); 00138 if (!got_event) 00139 { 00140 keep_looping = true; 00141 } 00142 } 00143 00144 TLOG(TLVL_TRACE) << "receiveMessage: Reading buffer header" ; 00145 auto errflag = false; 00146 incoming_events_->ReadHeader(errflag); 00147 if (errflag) { // Buffer was changed out from under reader! 00148 msg = nullptr; 00149 return; 00150 } 00151 TLOG(TLVL_TRACE) << "receiveMessage: Getting Fragment types" ; 00152 auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag); 00153 if (errflag) { // Buffer was changed out from under reader! 00154 incoming_events_->ReleaseBuffer(); 00155 msg = nullptr; 00156 return; 00157 } 00158 if (fragmentTypes.size() == 0) 00159 { 00160 TLOG(TLVL_ERROR) << "Event has no Fragments! Aborting!" ; 00161 incoming_events_->ReleaseBuffer(); 00162 msg = nullptr; 00163 return; 00164 } 00165 TLOG(TLVL_TRACE) << "receiveMessage: Checking first Fragment type" ; 00166 auto firstFragmentType = *fragmentTypes.begin(); 00167 00168 // We return false, indicating we're done reading, if: 00169 // 1) we did not obtain an event, because we timed out and were 00170 // configured NOT to keep trying after a timeout, or 00171 // 2) the event we read was the end-of-data marker: a null 00172 // pointer 00173 if (!got_event || firstFragmentType == artdaq::Fragment::EndOfDataFragmentType) 00174 { 00175 TLOG(TLVL_DEBUG) << "Received shutdown message, returning from receiveMessage " 00176 << "(debug: got_event=" << got_event << ",fragType=" << (int)firstFragmentType 00177 << ",EODFragType=" << (int)artdaq::Fragment::EndOfDataFragmentType << ")"; 00178 incoming_events_->ReleaseBuffer(); 00179 msg = nullptr; 00180 return; 00181 } 00182 if (firstFragmentType == artdaq::Fragment::InitFragmentType) 00183 { 00184 TLOG(TLVL_DEBUG) << "Cannot receive InitFragments here, retrying" ; 00185 incoming_events_->ReleaseBuffer(); 00186 continue; 00187 } 00188 // EndOfRun and EndOfSubrun Fragments are ignored in NetMonTransportService 00189 else if (firstFragmentType == artdaq::Fragment::EndOfRunFragmentType || firstFragmentType == artdaq::Fragment::EndOfSubrunFragmentType) 00190 { 00191 TLOG(TLVL_DEBUG) << "Ignoring EndOfRun or EndOfSubrun Fragment" ; 00192 incoming_events_->ReleaseBuffer(); 00193 continue; 00194 } 00195 00196 TLOG(TLVL_TRACE) << "receiveMessage: Getting all Fragments" ; 00197 recvd_fragments_ = incoming_events_->GetFragmentsByType(errflag, artdaq::Fragment::InvalidFragmentType); 00198 if (!recvd_fragments_) 00199 { 00200 TLOG(TLVL_ERROR) << "Error retrieving Fragments from shared memory! Aborting!"; 00201 incoming_events_->ReleaseBuffer(); 00202 msg = nullptr; 00203 return; 00204 00205 } 00206 /* Events coming out of the EventStore are not sorted but need to be 00207 sorted by sequence ID before they can be passed to art. 00208 */ 00209 std::sort(recvd_fragments_->begin(), recvd_fragments_->end(), 00210 artdaq::fragmentSequenceIDCompare); 00211 00212 TLOG(TLVL_TRACE) << "receiveMessage: Releasing buffer" ; 00213 incoming_events_->ReleaseBuffer(); 00214 } 00215 00216 // Do not process data until Init Fragment received! 00217 auto start = std::chrono::steady_clock::now(); 00218 while (!init_received_ && artdaq::TimeUtils::GetElapsedTime(start) < init_timeout_s_) 00219 { 00220 usleep(init_timeout_s_ * 1000000 / 100); // Check 100 times 00221 } 00222 if (!init_received_) { 00223 TLOG(TLVL_ERROR) << "Received data but no Init Fragment after " << init_timeout_s_ << " seconds. Art will crash." ; 00224 } 00225 00226 TLOG(TLVL_TRACE) << "receiveMessage: Returning top Fragment" ; 00227 artdaq::Fragment topFrag = std::move(recvd_fragments_->at(0)); 00228 recvd_fragments_->erase(recvd_fragments_->begin()); 00229 if (recvd_fragments_->size() == 0) 00230 { 00231 recvd_fragments_.reset(nullptr); 00232 } 00233 00234 TLOG(TLVL_TRACE) << "receiveMessage: Copying Fragment into TBufferFile, length=" << topFrag.metadata<artdaq::NetMonHeader>()->data_length ; 00235 auto header = topFrag.metadata<artdaq::NetMonHeader>(); 00236 auto buffer = static_cast<char *>(malloc(header->data_length)); 00237 memcpy(buffer, &*topFrag.dataBegin(), header->data_length); 00238 msg = new TBufferFile(TBuffer::kRead, header->data_length, buffer, kTRUE, 0); 00239 00240 #if DUMP_RECEIVE_MESSAGE 00241 std::string fileName = "receiveMessage_" + std::to_string(my_rank) + "_" + std::to_string(getpid()) + "_" + std::to_string(topFrag.sequenceID()) + ".bin"; 00242 std::fstream ostream(fileName.c_str(), std::ios::out | std::ios::binary); 00243 ostream.write(buffer, header->data_length); 00244 ostream.close(); 00245 #endif 00246 00247 TLOG(TLVL_TRACE) << "receiveMessage END" ; 00248 } 00249 00250 void 00251 NetMonTransportService:: 00252 receiveInitMessage(TBufferFile*& msg) 00253 { 00254 listen(); 00255 TLOG(TLVL_TRACE) << "receiveInitMessage BEGIN" ; 00256 if (recvd_fragments_ == nullptr) 00257 { 00258 TLOG(TLVL_TRACE) << "receiveInitMessage: Waiting for available buffer" ; 00259 00260 bool got_init = false; 00261 auto errflag = false; 00262 while (!got_init) { 00263 00264 bool got_event = false; 00265 while (!got_event) 00266 { 00267 got_event = incoming_events_->ReadyForRead(true); 00268 } 00269 00270 TLOG(TLVL_TRACE) << "receiveInitMessage: Reading buffer header" ; 00271 incoming_events_->ReadHeader(errflag); 00272 if (errflag) { // Buffer was changed out from under reader! 00273 TLOG(TLVL_ERROR) << "receiveInitMessage: Error receiving message!" ; 00274 incoming_events_->ReleaseBuffer(); 00275 msg = nullptr; 00276 return; 00277 } 00278 TLOG(TLVL_TRACE) << "receiveInitMessage: Getting Fragment types" ; 00279 auto fragmentTypes = incoming_events_->GetFragmentTypes(errflag); 00280 if (errflag) { // Buffer was changed out from under reader! 00281 incoming_events_->ReleaseBuffer(); 00282 msg = nullptr; 00283 TLOG(TLVL_ERROR) << "receiveInitMessage: Error receiving message!" ; 00284 return; 00285 } 00286 if (fragmentTypes.size() == 0) 00287 { 00288 TLOG(TLVL_ERROR) << "Event has no Fragments! Aborting!" ; 00289 incoming_events_->ReleaseBuffer(); 00290 msg = nullptr; 00291 return; 00292 } 00293 TLOG(TLVL_TRACE) << "receiveInitMessage: Checking first Fragment type" ; 00294 auto firstFragmentType = *fragmentTypes.begin(); 00295 00296 // We return false, indicating we're done reading, if: 00297 // 1) we did not obtain an event, because we timed out and were 00298 // configured NOT to keep trying after a timeout, or 00299 // 2) the event we read was the end-of-data marker: a null 00300 // pointer 00301 if (!got_event || firstFragmentType == artdaq::Fragment::EndOfDataFragmentType) 00302 { 00303 TLOG(TLVL_DEBUG) << "Received shutdown message, returning" ; 00304 incoming_events_->ReleaseBuffer(); 00305 msg = nullptr; 00306 return; 00307 } 00308 if (firstFragmentType != artdaq::Fragment::InitFragmentType) 00309 { 00310 TLOG(TLVL_WARNING) << "Did NOT receive Init Fragment as first broadcast! Type=" << artdaq::detail::RawFragmentHeader::SystemTypeToString(firstFragmentType) ; 00311 incoming_events_->ReleaseBuffer(); 00312 } 00313 got_init = true; 00314 } 00315 TLOG(TLVL_TRACE) << "receiveInitMessage: Getting all Fragments" ; 00316 recvd_fragments_ = incoming_events_->GetFragmentsByType(errflag, artdaq::Fragment::InvalidFragmentType); 00317 /* Events coming out of the EventStore are not sorted but need to be 00318 sorted by sequence ID before they can be passed to art. 00319 */ 00320 std::sort(recvd_fragments_->begin(), recvd_fragments_->end(), 00321 artdaq::fragmentSequenceIDCompare); 00322 00323 incoming_events_->ReleaseBuffer(); 00324 } 00325 00326 TLOG(TLVL_TRACE) << "receiveInitMessage: Returning top Fragment" ; 00327 artdaq::Fragment topFrag = std::move(recvd_fragments_->at(0)); 00328 recvd_fragments_->erase(recvd_fragments_->begin()); 00329 if (recvd_fragments_->size() == 0) 00330 { 00331 recvd_fragments_.reset(nullptr); 00332 } 00333 00334 auto header = topFrag.metadata<artdaq::NetMonHeader>(); 00335 TLOG(TLVL_TRACE) << "receiveInitMessage: Copying Fragment into TBufferFile: message length: " << header->data_length ; 00336 auto buffer = new char[header->data_length]; 00337 //auto buffer = static_cast<char *>(malloc(header->data_length)); // Fix alloc-dealloc-mismatch 00338 memcpy(buffer, &*topFrag.dataBegin(), header->data_length); 00339 00340 #if DUMP_RECEIVE_MESSAGE 00341 std::string fileName = "receiveInitMessage_" + std::to_string(getpid()) + ".bin"; 00342 std::fstream ostream(fileName.c_str(), std::ios::out | std::ios::binary); 00343 ostream.write(buffer, header->data_length); 00344 ostream.close(); 00345 #endif 00346 00347 msg = new TBufferFile(TBuffer::kRead, header->data_length, buffer, kTRUE, 0); 00348 00349 TLOG(TLVL_TRACE) << "receiveInitMessage END" ; 00350 init_received_ = true; 00351 } 00352 DEFINE_ART_SERVICE_INTERFACE_IMPL(NetMonTransportService, NetMonTransportServiceInterface)