00001 #include "artdaq/DAQdata/Globals.hh"
00002 #include "artdaq/DAQrate/EventStore.hh"
00003 #include <utility>
00004 #include <cstring>
00005 #include <dlfcn.h>
00006 #include <iomanip>
00007 #include <fstream>
00008 #include <sstream>
00009 #include <thread>
00010 #include <chrono>
00011
00012 #include "cetlib/exception.h"
00013 #include "artdaq-core/Core/StatisticsCollection.hh"
00014 #include "artdaq-core/Core/SimpleQueueReader.hh"
00015 #include "artdaq/Application/Routing/RoutingPacket.hh"
00016 #include "artdaq/DAQdata/TCPConnect.hh"
00017
00018 using namespace std;
00019
00020 namespace artdaq
00021 {
00022 const std::string EventStore::EVENT_RATE_STAT_KEY("EventStoreEventRate");
00023 const std::string EventStore::INCOMPLETE_EVENT_STAT_KEY("EventStoreIncompleteEvents");
00024
00025 EventStore::EventStore(const fhicl::ParameterSet& pset, size_t num_fragments_per_event, run_id_t run,
00026 size_t event_queue_depth, size_t max_incomplete_event_count)
00027 : num_fragments_per_event_(num_fragments_per_event)
00028 , max_queue_size_(pset.get<size_t>("event_queue_depth", event_queue_depth))
00029 , max_incomplete_count_(pset.get<size_t>("max_incomplete_events", max_incomplete_event_count))
00030 , run_id_(run)
00031 , subrun_id_(0)
00032 , events_()
00033 , queue_(getGlobalQueue(max_queue_size_))
00034 , reader_thread_launch_time_(std::chrono::steady_clock::now())
00035 , send_requests_(pset.get<bool>("send_requests", false))
00036 , active_requests_()
00037 , request_port_(pset.get<int>("request_port", 3001))
00038 , request_delay_(pset.get<size_t>("request_delay_ms", 10))
00039 , multicast_out_addr_(pset.get<std::string>("output_address", "localhost"))
00040 , request_mode_(detail::RequestMessageMode::Normal)
00041 , seqIDModulus_(1)
00042 , lastFlushedSeqID_(0)
00043 , highestSeqIDSeen_(0)
00044 , enq_timeout_(pset.get<double>("event_queue_wait_time", 5.0))
00045 , enq_check_count_(pset.get<size_t>("event_queue_check_count", 5000))
00046 , printSummaryStats_(pset.get<bool>("print_event_store_stats", false))
00047 , incomplete_event_report_interval_ms_(pset.get<int>("incomplete_event_report_interval_ms", -1))
00048 , last_incomplete_event_report_time_(std::chrono::steady_clock::now())
00049 , token_socket_(-1)
00050 , art_thread_wait_ms_(pset.get<int>("art_thread_wait_ms", 4000))
00051 {
00052 TLOG_DEBUG("EventStore") << "EventStore CONSTRUCTOR" << TLOG_ENDL;
00053 initStatistics_();
00054 setup_requests_(pset.get<std::string>("request_address", "227.128.12.26"));
00055
00056 auto rmConfig = pset.get<fhicl::ParameterSet>("routing_token_config", fhicl::ParameterSet());
00057 send_routing_tokens_ = rmConfig.get<bool>("use_routing_master", false);
00058 token_port_ = rmConfig.get<int>("routing_token_port", 35555);
00059 token_address_ = rmConfig.get<std::string>("routing_master_hostname", "localhost");
00060 setup_tokens_();
00061 TRACE(12, "artdaq::EventStore::EventStore ctor - reader_thread_ initialized");
00062 }
00063
00064 EventStore::EventStore(const fhicl::ParameterSet& pset,
00065 size_t num_fragments_per_event,
00066 run_id_t run,
00067 int argc,
00068 char* argv[],
00069 ART_CMDLINE_FCN* reader)
00070 : EventStore(pset, num_fragments_per_event, run, 50, 50)
00071 {
00072 reader_thread_ = (std::async(std::launch::async, reader, argc, argv));
00073 }
00074
00075 EventStore::EventStore(const fhicl::ParameterSet& pset,
00076 size_t num_fragments_per_event,
00077 run_id_t run,
00078 const std::string& configString,
00079 ART_CFGSTRING_FCN* reader)
00080 : EventStore(pset, num_fragments_per_event, run, 20, 20)
00081 {
00082 reader_thread_ = (std::async(std::launch::async, reader, configString));
00083 }
00084
00085 EventStore::~EventStore()
00086 {
00087 TLOG_DEBUG("EventStore") << "Shutting down EventStore" << TLOG_ENDL;
00088 if (printSummaryStats_)
00089 {
00090 reportStatistics_();
00091 }
00092 shutdown(request_socket_, 2);
00093 close(request_socket_);
00094 shutdown(token_socket_, 2);
00095 close(token_socket_);
00096 }
00097
00098 void EventStore::insert(FragmentPtr pfrag,
00099 bool printWarningWhenFragmentIsDropped)
00100 {
00101
00102
00103 assert(pfrag != nullptr);
00104 assert(pfrag->fragmentID() != Fragment::InvalidFragmentID);
00105
00106
00107
00108
00109
00110
00111
00112
00113
00114 if (pfrag->sequenceID() > highestSeqIDSeen_)
00115 {
00116 highestSeqIDSeen_ = pfrag->sequenceID();
00117
00118
00119 Fragment::timestamp_t timestamp = pfrag->timestamp();
00120
00121
00122 if (send_requests_)
00123 {
00124 std::lock_guard<std::mutex> lk(request_mutex_);
00125 active_requests_[highestSeqIDSeen_] = timestamp;
00126 send_request_();
00127 }
00128 }
00129
00130
00131
00132 if (send_requests_ && request_mode_ == detail::RequestMessageMode::EndOfRun)
00133 {
00134 std::lock_guard<std::mutex> lk(request_mutex_);
00135 send_request_();
00136 }
00137 Fragment::sequence_id_t sequence_id = ((pfrag->sequenceID() - (1 + lastFlushedSeqID_)) / seqIDModulus_) + 1;
00138 TRACE(13, "EventStore::insert seq=%lu fragID=%d id=%d lastFlushed=%lu seqIDMod=%d seq=%lu"
00139 , pfrag->sequenceID(), pfrag->fragmentID(), my_rank, lastFlushedSeqID_, seqIDModulus_, sequence_id);
00140
00141
00142
00143
00144 EventMap::iterator loc = events_.lower_bound(sequence_id);
00145
00146 if (loc == events_.end() || events_.key_comp()(sequence_id, loc->first))
00147 {
00148
00149
00150 RawEvent_ptr newevent(new RawEvent(run_id_, subrun_id_, pfrag->sequenceID()));
00151 loc =
00152 events_.insert(loc, EventMap::value_type(sequence_id, newevent));
00153 }
00154
00155
00156 loc->second->insertFragment(std::move(pfrag));
00157 if (loc->second->numFragments() == num_fragments_per_event_)
00158 {
00159
00160
00161
00162 RawEvent_ptr complete_event(loc->second);
00163 complete_event->markComplete();
00164
00165 events_.erase(loc);
00166
00167 if (send_requests_)
00168 {
00169 std::lock_guard<std::mutex> lk(request_mutex_);
00170 active_requests_.erase(sequence_id);
00171 }
00172
00173
00174
00175 MonitoredQuantityPtr mqPtr = StatisticsCollection::getInstance().
00176 getMonitoredQuantity(EVENT_RATE_STAT_KEY);
00177 if (mqPtr.get() != 0)
00178 {
00179 mqPtr->addSample(complete_event->wordCount());
00180 }
00181 TRACE(14, "EventStore::insert seq=%lu enqTimedWait start", sequence_id);
00182 bool enqSuccess = queue_.enqTimedWait(complete_event, enq_timeout_);
00183 TRACE(enqSuccess ? 14 : 0, "EventStore::insert seq=%lu enqTimedWait complete", sequence_id);
00184 if (metricMan)
00185 {
00186 metricMan->sendMetric("Current Event Number", sequence_id, "id", 2, MetricMode::LastPoint);
00187 }
00188 if (!enqSuccess)
00189 {
00190
00191 if (printWarningWhenFragmentIsDropped)
00192 {
00193 TLOG_WARNING("EventStore") << "Enqueueing event " << sequence_id
00194 << " FAILED, queue size = "
00195 << queue_.size() <<
00196 "; apparently no events were removed from this process's queue during the " << std::to_string(enq_timeout_.count())
00197 << "-second timeout period" << TLOG_ENDL;
00198 }
00199 else
00200 {
00201 TLOG_DEBUG("EventStore") << "Enqueueing event " << sequence_id
00202 << " FAILED, queue size = "
00203 << queue_.size() <<
00204 "; apparently no events were removed from this process's queue during the " << std::to_string(enq_timeout_.count())
00205 << "-second timeout period" << TLOG_ENDL;
00206 }
00207 }
00208 else
00209 {
00210 send_routing_token_(1);
00211 }
00212 }
00213 MonitoredQuantityPtr mqPtr = StatisticsCollection::getInstance().
00214 getMonitoredQuantity(INCOMPLETE_EVENT_STAT_KEY);
00215 if (mqPtr.get() != 0)
00216 {
00217 mqPtr->addSample(events_.size());
00218 }
00219 }
00220
00221 EventStore::EventStoreInsertResult EventStore::insert(FragmentPtr pfrag, FragmentPtr& rejectedFragment)
00222 {
00223
00224
00225
00226
00227 TRACE(12, "EventStore: Testing if queue is full");
00228 if (queue_.full())
00229 {
00230 size_t sleepTime = 1000000 * (enq_timeout_.count() / enq_check_count_);
00231 TRACE(12, "EventStore: sleepTime is %lu.", sleepTime);
00232 size_t loopCount = 0;
00233 while (loopCount < enq_check_count_ && queue_.full())
00234 {
00235 ++loopCount;
00236 usleep(sleepTime);
00237 }
00238 if (queue_.full())
00239 {
00240 rejectedFragment = std::move(pfrag);
00241 return EventStoreInsertResult::REJECT_QUEUEFULL;
00242 }
00243 }
00244 TRACE(12, "EventStore: Testing if there's room in the EventStore");
00245 auto incomplete_full = events_.size() >= max_incomplete_count_;
00246 if (incomplete_full)
00247 {
00248 EventMap::iterator loc = events_.lower_bound(pfrag->sequenceID());
00249
00250 if (loc == events_.end() || events_.key_comp()(pfrag->sequenceID(), loc->first))
00251 {
00252 rejectedFragment = std::move(pfrag);
00253 return EventStoreInsertResult::REJECT_STOREFULL;
00254 }
00255 }
00256
00257 TRACE(12, "EventStore: Performing insert");
00258 insert(std::move(pfrag));
00259 return incomplete_full ? EventStoreInsertResult::SUCCESS_STOREFULL : EventStoreInsertResult::SUCCESS;
00260 }
00261
00262 bool
00263 EventStore::endOfData(int& readerReturnValue)
00264 {
00265 TLOG_DEBUG("EventStore") << "EventStore::endOfData" << TLOG_ENDL;
00266 RawEvent_ptr end_of_data(nullptr);
00267 TRACE(4, "EventStore::endOfData: Enqueuing end_of_data event");
00268 bool enqSuccess = queue_.enqTimedWait(end_of_data, enq_timeout_);
00269 if (!enqSuccess)
00270 {
00271 return false;
00272 }
00273 TRACE(4, "EventStore::endOfData: Getting return code from art thread");
00274 readerReturnValue = reader_thread_.get();
00275 return true;
00276 }
00277
00278 void EventStore::setSeqIDModulus(unsigned int seqIDModulus)
00279 {
00280 seqIDModulus_ = seqIDModulus;
00281 }
00282
00283 bool EventStore::flushData()
00284 {
00285 bool enqSuccess;
00286 size_t initialStoreSize = events_.size();
00287 TLOG_DEBUG("EventStore") << "Flushing " << initialStoreSize
00288 << " stale events from the EventStore." << TLOG_ENDL;
00289 EventMap::iterator loc;
00290 std::vector<sequence_id_t> flushList;
00291 for (loc = events_.begin(); loc != events_.end(); ++loc)
00292 {
00293 RawEvent_ptr complete_event(loc->second);
00294 MonitoredQuantityPtr mqPtr = StatisticsCollection::getInstance().
00295 getMonitoredQuantity(EVENT_RATE_STAT_KEY);
00296 if (mqPtr.get() != 0)
00297 {
00298 mqPtr->addSample(complete_event->wordCount());
00299 }
00300 enqSuccess = queue_.enqTimedWait(complete_event, enq_timeout_);
00301 if (!enqSuccess)
00302 {
00303 break;
00304 }
00305 else
00306 {
00307 flushList.push_back(loc->first);
00308 }
00309 }
00310 for (size_t idx = 0; idx < flushList.size(); ++idx)
00311 {
00312 events_.erase(flushList[idx]);
00313 }
00314 TLOG_DEBUG("EventStore") << "Done flushing " << flushList.size()
00315 << " stale events from the EventStore." << TLOG_ENDL;
00316
00317 lastFlushedSeqID_ = highestSeqIDSeen_;
00318 return (flushList.size() >= initialStoreSize);
00319 }
00320
00321 void EventStore::startRun(run_id_t runID)
00322 {
00323 if (!queue_.queueReaderIsReady())
00324 {
00325 TLOG_WARNING("EventStore") << "Run start requested, but the art thread is not yet ready, waiting up to " << art_thread_wait_ms_ << " msec..." << TLOG_ENDL;
00326 while (!queue_.queueReaderIsReady() && std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - reader_thread_launch_time_).count() < art_thread_wait_ms_)
00327 {
00328 usleep(1000);
00329 }
00330 if (queue_.queueReaderIsReady())
00331 {
00332 auto dur = std::chrono::duration_cast<std::chrono::milliseconds>(queue_.getReadyTime() - reader_thread_launch_time_).count();
00333 TLOG_INFO("EventStore") << "art initialization took (roughly) " << std::setw(4) << std::to_string(dur) << " ms." << TLOG_ENDL;
00334 }
00335 else
00336 {
00337 auto dur = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - reader_thread_launch_time_).count();
00338 TLOG_ERROR("EventStore") << "art thread still not ready after " << dur << " ms. Continuing to start..." << TLOG_ENDL;
00339 }
00340 }
00341 run_id_ = runID;
00342 subrun_id_ = 1;
00343 lastFlushedSeqID_ = 0;
00344 highestSeqIDSeen_ = 0;
00345 send_routing_token_(max_queue_size_);
00346 TLOG_DEBUG("EventStore") << "Starting run " << run_id_
00347 << ", max queue size = "
00348 << max_queue_size_
00349 << ", queue capacity = "
00350 << queue_.capacity()
00351 << ", queue size = "
00352 << queue_.size() << TLOG_ENDL;
00353 if (metricMan)
00354 {
00355 double runSubrun = run_id_ + ((double)subrun_id_ / 10000);
00356 metricMan->sendMetric("Run Number", runSubrun, "Run:Subrun", 1, MetricMode::LastPoint);
00357 }
00358 }
00359
00360 void EventStore::startSubrun()
00361 {
00362 ++subrun_id_;
00363 if (metricMan)
00364 {
00365 double runSubrun = run_id_ + ((double)subrun_id_ / 10000);
00366 metricMan->sendMetric("Run Number", runSubrun, "Run:Subrun", 1, MetricMode::LastPoint);
00367 }
00368 }
00369
00370 bool EventStore::endRun()
00371 {
00372 RawEvent_ptr endOfRunEvent(new RawEvent(run_id_, subrun_id_, 0));
00373 std::unique_ptr<artdaq::Fragment>
00374 endOfRunFrag(new
00375 Fragment(static_cast<size_t>
00376 (ceil(sizeof(my_rank) /
00377 static_cast<double>(sizeof(Fragment::value_type))))));
00378
00379 endOfRunFrag->setSystemType(Fragment::EndOfRunFragmentType);
00380 *endOfRunFrag->dataBegin() = my_rank;
00381 endOfRunEvent->insertFragment(std::move(endOfRunFrag));
00382
00383 return queue_.enqTimedWait(endOfRunEvent, enq_timeout_);
00384 }
00385
00386 bool EventStore::endSubrun()
00387 {
00388 RawEvent_ptr endOfSubrunEvent(new RawEvent(run_id_, subrun_id_, 0));
00389 std::unique_ptr<artdaq::Fragment>
00390 endOfSubrunFrag(new
00391 Fragment(static_cast<size_t>
00392 (ceil(sizeof(my_rank) /
00393 static_cast<double>(sizeof(Fragment::value_type))))));
00394
00395 endOfSubrunFrag->setSystemType(Fragment::EndOfSubrunFragmentType);
00396 *endOfSubrunFrag->dataBegin() = my_rank;
00397 endOfSubrunEvent->insertFragment(std::move(endOfSubrunFrag));
00398
00399 return queue_.enqTimedWait(endOfSubrunEvent, enq_timeout_);
00400 }
00401
00402 void EventStore::setRequestMode(detail::RequestMessageMode mode)
00403 {
00404 request_mode_ = mode;
00405 if (send_requests_ && request_mode_ == detail::RequestMessageMode::EndOfRun)
00406 {
00407 send_request_();
00408 }
00409 }
00410
00411 void
00412 EventStore::initStatistics_()
00413 {
00414 MonitoredQuantityPtr mqPtr = StatisticsCollection::getInstance().
00415 getMonitoredQuantity(EVENT_RATE_STAT_KEY);
00416 if (mqPtr.get() == 0)
00417 {
00418 mqPtr.reset(new MonitoredQuantity(3.0, 300.0));
00419 StatisticsCollection::getInstance().
00420 addMonitoredQuantity(EVENT_RATE_STAT_KEY, mqPtr);
00421 }
00422 mqPtr->reset();
00423
00424 mqPtr = StatisticsCollection::getInstance().
00425 getMonitoredQuantity(INCOMPLETE_EVENT_STAT_KEY);
00426 if (mqPtr.get() == 0)
00427 {
00428 mqPtr.reset(new MonitoredQuantity(3.0, 300.0));
00429 StatisticsCollection::getInstance().
00430 addMonitoredQuantity(INCOMPLETE_EVENT_STAT_KEY, mqPtr);
00431 }
00432 mqPtr->reset();
00433 }
00434
00435 void
00436 EventStore::reportStatistics_()
00437 {
00438 MonitoredQuantityPtr mqPtr = StatisticsCollection::getInstance().
00439 getMonitoredQuantity(EVENT_RATE_STAT_KEY);
00440 if (mqPtr.get() != 0)
00441 {
00442 ostringstream oss;
00443 oss << EVENT_RATE_STAT_KEY << "_" << setfill('0') << setw(4) << run_id_
00444 << "_" << setfill('0') << setw(4) << my_rank << ".txt";
00445 std::string filename = oss.str();
00446 ofstream outStream(filename.c_str());
00447 mqPtr->waitUntilAccumulatorsHaveBeenFlushed(3.0);
00448 artdaq::MonitoredQuantityStats stats;
00449 mqPtr->getStats(stats);
00450 outStream << "EventStore rank " << my_rank << ": events processed = "
00451 << stats.fullSampleCount << " at " << stats.fullSampleRate
00452 << " events/sec, data rate = "
00453 << (stats.fullValueRate * sizeof(RawDataType)
00454 / 1024.0 / 1024.0) << " MB/sec, duration = "
00455 << stats.fullDuration << " sec" << std::endl
00456 << " minimum event size = "
00457 << (stats.fullValueMin * sizeof(RawDataType)
00458 / 1024.0 / 1024.0)
00459 << " MB, maximum event size = "
00460 << (stats.fullValueMax * sizeof(RawDataType)
00461 / 1024.0 / 1024.0)
00462 << " MB" << std::endl;
00463 bool foundTheStart = false;
00464 for (int idx = 0; idx < (int)stats.recentBinnedDurations.size(); ++idx)
00465 {
00466 if (stats.recentBinnedDurations[idx] > 0.0)
00467 {
00468 foundTheStart = true;
00469 }
00470 if (foundTheStart)
00471 {
00472 outStream << " " << std::fixed << std::setprecision(3)
00473 << stats.recentBinnedEndTimes[idx]
00474 << ": " << stats.recentBinnedSampleCounts[idx]
00475 << " events at "
00476 << (stats.recentBinnedSampleCounts[idx] /
00477 stats.recentBinnedDurations[idx])
00478 << " events/sec, data rate = "
00479 << (stats.recentBinnedValueSums[idx] *
00480 sizeof(RawDataType) / 1024.0 / 1024.0 /
00481 stats.recentBinnedDurations[idx])
00482 << " MB/sec, bin size = "
00483 << stats.recentBinnedDurations[idx]
00484 << " sec" << std::endl;
00485 }
00486 }
00487 outStream.close();
00488 }
00489
00490 mqPtr = StatisticsCollection::getInstance().
00491 getMonitoredQuantity(INCOMPLETE_EVENT_STAT_KEY);
00492 if (mqPtr.get() != 0)
00493 {
00494 ostringstream oss;
00495 oss << INCOMPLETE_EVENT_STAT_KEY << "_" << setfill('0')
00496 << setw(4) << run_id_
00497 << "_" << setfill('0') << setw(4) << my_rank << ".txt";
00498 std::string filename = oss.str();
00499 ofstream outStream(filename.c_str());
00500 mqPtr->waitUntilAccumulatorsHaveBeenFlushed(3.0);
00501 artdaq::MonitoredQuantityStats stats;
00502 mqPtr->getStats(stats);
00503 outStream << "EventStore rank " << my_rank << ": fragments processed = "
00504 << stats.fullSampleCount << " at " << stats.fullSampleRate
00505 << " fragments/sec, average incomplete event count = "
00506 << stats.fullValueAverage << " duration = "
00507 << stats.fullDuration << " sec" << std::endl
00508 << " minimum incomplete event count = "
00509 << stats.fullValueMin << ", maximum incomplete event count = "
00510 << stats.fullValueMax << std::endl;
00511 bool foundTheStart = false;
00512 for (int idx = 0; idx < (int)stats.recentBinnedDurations.size(); ++idx)
00513 {
00514 if (stats.recentBinnedDurations[idx] > 0.0)
00515 {
00516 foundTheStart = true;
00517 }
00518 if (foundTheStart && stats.recentBinnedSampleCounts[idx] > 0.0)
00519 {
00520 outStream << " " << std::fixed << std::setprecision(3)
00521 << stats.recentBinnedEndTimes[idx]
00522 << ": " << stats.recentBinnedSampleCounts[idx]
00523 << " fragments at "
00524 << (stats.recentBinnedSampleCounts[idx] /
00525 stats.recentBinnedDurations[idx])
00526 << " fragments/sec, average incomplete event count = "
00527 << (stats.recentBinnedValueSums[idx] /
00528 stats.recentBinnedSampleCounts[idx])
00529 << ", bin size = "
00530 << stats.recentBinnedDurations[idx]
00531 << " sec" << std::endl;
00532 }
00533 }
00534 outStream << "Incomplete count now = " << events_.size() << std::endl;
00535 outStream.close();
00536 }
00537 }
00538
00539 void
00540 EventStore::setup_requests_(std::string request_address)
00541 {
00542 if (send_requests_)
00543 {
00544 request_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
00545 if (!request_socket_)
00546 {
00547 TLOG_ERROR("EventStore") << "I failed to create the socket for sending Data Requests!" << TLOG_ENDL;
00548 exit(1);
00549 }
00550 int sts = ResolveHost(request_address.c_str(), request_port_, request_addr_);
00551 if (sts == -1)
00552 {
00553 TLOG_ERROR("EventStore") << "Unable to resolve Data Request address" << TLOG_ENDL;
00554 exit(1);
00555 }
00556
00557 if (multicast_out_addr_ != "localhost")
00558 {
00559 struct in_addr addr;
00560 int sts = ResolveHost(multicast_out_addr_.c_str(), addr);
00561 if (sts == -1)
00562 {
00563 TLOG_ERROR("EventStore") << "Unable to resolve multicast interface address" << TLOG_ENDL;
00564 exit(1);
00565 }
00566
00567 int yes = 1;
00568 if (setsockopt(request_socket_, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) < 0)
00569 {
00570 TLOG_ERROR("EventStore") << "Unable to enable port reuse on request socket" << TLOG_ENDL;
00571 exit(1);
00572 }
00573 if (setsockopt(request_socket_, IPPROTO_IP, IP_MULTICAST_IF, &addr, sizeof(addr)) == -1)
00574 {
00575 TLOG_ERROR("EventStore") << "Cannot set outgoing interface." << TLOG_ENDL;
00576 exit(1);
00577 }
00578 }
00579 int yes = 1;
00580 if (setsockopt(request_socket_, SOL_SOCKET, SO_BROADCAST, (void*)&yes, sizeof(int)) == -1)
00581 {
00582 TLOG_ERROR("EventStore") << "Cannot set request socket to broadcast." << TLOG_ENDL;
00583 exit(1);
00584 }
00585 }
00586 }
00587
00588 void
00589 EventStore::setup_tokens_()
00590 {
00591 if (send_routing_tokens_)
00592 {
00593 TLOG_DEBUG("EventStore") << "Creating Routing Token sending socket" << TLOG_ENDL;
00594 token_socket_ = TCPConnect(token_address_.c_str(), token_port_);
00595 if (!token_socket_)
00596 {
00597 TLOG_ERROR("EventStore") << "I failed to create the socket for sending Routing Tokens!" << TLOG_ENDL;
00598 exit(1);
00599 }
00600 }
00601 }
00602
00603 void EventStore::do_send_request_()
00604 {
00605 std::this_thread::sleep_for(std::chrono::microseconds(request_delay_));
00606
00607 detail::RequestMessage message;
00608 {
00609 std::lock_guard<std::mutex> lk(request_mutex_);
00610 for (auto& req : active_requests_)
00611 {
00612 message.addRequest(req.first, req.second);
00613 }
00614 }
00615 message.header()->mode = request_mode_;
00616 char str[INET_ADDRSTRLEN];
00617 inet_ntop(AF_INET, &(request_addr_.sin_addr), str, INET_ADDRSTRLEN);
00618 TLOG_DEBUG("EventStore") << "Sending request for " << std::to_string(message.size()) << " events to multicast group " << str << TLOG_ENDL;
00619 if (sendto(request_socket_, message.header(), sizeof(detail::RequestHeader), 0, (struct sockaddr *)&request_addr_, sizeof(request_addr_)) < 0)
00620 {
00621 TLOG_ERROR("EventStore") << "Error sending request message header" << TLOG_ENDL;
00622 }
00623 if (sendto(request_socket_, message.buffer(), sizeof(detail::RequestPacket) * message.size(), 0, (struct sockaddr *)&request_addr_, sizeof(request_addr_)) < 0)
00624 {
00625 TLOG_ERROR("EventStore") << "Error sending request message data" << TLOG_ENDL;
00626 }
00627 }
00628
00629 void EventStore::send_routing_token_(int nSlots)
00630 {
00631 TLOG_DEBUG("EventStore") << "send_routing_token_ called, send_routing_tokens_=" << std::boolalpha << send_routing_tokens_ << TLOG_ENDL;
00632 if (!send_routing_tokens_) return;
00633 if (token_socket_ == -1) setup_tokens_();
00634 detail::RoutingToken token;
00635 token.header = TOKEN_MAGIC;
00636 token.rank = my_rank;
00637 token.new_slots_free = nSlots;
00638
00639 TLOG_DEBUG("EventStore") << "Sending RoutingToken to " << token_address_ << ":" << token_port_ << TLOG_ENDL;
00640 size_t sts = 0;
00641 while (sts < sizeof(detail::RoutingToken))
00642 {
00643 auto res = send(token_socket_, reinterpret_cast<uint8_t*>(&token) + sts, sizeof(detail::RoutingToken) - sts, 0);
00644 if (res == -1)
00645 {
00646 usleep(1000);
00647 continue;
00648 }
00649 sts += res;
00650 }
00651 TLOG_DEBUG("EventStore") << "Done sending RoutingToken to " << token_address_ << ":" << token_port_ << TLOG_ENDL;
00652 }
00653
00654 void
00655 EventStore::send_request_()
00656 {
00657 std::thread request([=] { do_send_request_(); });
00658 request.detach();
00659 }
00660
00661 void
00662 EventStore::sendMetrics()
00663 {
00664 if (metricMan)
00665 {
00666 metricMan->sendMetric("Incomplete Event Count", events_.size(), "events", 1, MetricMode::LastPoint);
00667
00668 MonitoredQuantityPtr mqPtr = StatisticsCollection::getInstance().
00669 getMonitoredQuantity(EVENT_RATE_STAT_KEY);
00670 if (mqPtr.get() != 0)
00671 {
00672 artdaq::MonitoredQuantityStats stats;
00673 mqPtr->getStats(stats);
00674
00675 metricMan->sendMetric("Event Count", static_cast<unsigned long>(stats.fullSampleCount), "events", 1, MetricMode::Accumulate);
00676 metricMan->sendMetric("Event Rate", stats.recentSampleRate, "events/sec", 1, MetricMode::Average);
00677 metricMan->sendMetric("Average Event Size", (stats.recentValueAverage * sizeof(artdaq::RawDataType)), "bytes/fragment", 2, MetricMode::Average);
00678 metricMan->sendMetric("Data Rate", (stats.recentValueRate * sizeof(artdaq::RawDataType)), "bytes/sec", 2, MetricMode::Average);
00679 }
00680 }
00681 if (incomplete_event_report_interval_ms_ > 0 && events_.size())
00682 {
00683 if (std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - last_incomplete_event_report_time_).count() < incomplete_event_report_interval_ms_) return;
00684 last_incomplete_event_report_time_ = std::chrono::steady_clock::now();
00685 std::ostringstream oss;
00686 oss << "Incomplete Events (" << num_fragments_per_event_ << "): ";
00687 for (auto& ev : events_)
00688 {
00689 oss << ev.first << " (" << ev.second->numFragments() << "), ";
00690 }
00691 TLOG_DEBUG("EventStore") << oss.str() << TLOG_ENDL;
00692 }
00693 }
00694 }