00001 #include "artdaq/DAQdata/Globals.hh"
00002 #include "artdaq/DAQrate/EventStore.hh"
00003 #include <utility>
00004 #include <cstring>
00005 #include <dlfcn.h>
00006 #include <iomanip>
00007 #include <fstream>
00008 #include <sstream>
00009 #include <thread>
00010 #include <chrono>
00011
00012 #include "cetlib/exception.h"
00013 #include "artdaq-core/Core/StatisticsCollection.hh"
00014 #include "artdaq-core/Core/SimpleQueueReader.hh"
00015 #include "artdaq/DAQrate/detail/RequestMessage.hh"
00016 #include "artdaq/Application/Routing/RoutingPacket.hh"
00017 #include "artdaq/DAQdata/TCPConnect.hh"
00018
00019 using namespace std;
00020
00021 namespace artdaq
00022 {
00023 const std::string EventStore::EVENT_RATE_STAT_KEY("EventStoreEventRate");
00024 const std::string EventStore::INCOMPLETE_EVENT_STAT_KEY("EventStoreIncompleteEvents");
00025
00026 EventStore::EventStore(const fhicl::ParameterSet& pset, size_t num_fragments_per_event, run_id_t run,
00027 size_t event_queue_depth, size_t max_incomplete_event_count)
00028 : num_fragments_per_event_(num_fragments_per_event)
00029 , max_queue_size_(pset.get<size_t>("event_queue_depth", event_queue_depth))
00030 , max_incomplete_count_(pset.get<size_t>("max_incomplete_events", max_incomplete_event_count))
00031 , run_id_(run)
00032 , subrun_id_(0)
00033 , events_()
00034 , queue_(getGlobalQueue(max_queue_size_))
00035 , reader_thread_launch_time_(std::chrono::steady_clock::now())
00036 , send_requests_(pset.get<bool>("send_requests", false))
00037 , active_requests_()
00038 , request_port_(pset.get<int>("request_port", 3001))
00039 , request_delay_(pset.get<size_t>("request_delay_ms", 10))
00040 , multicast_out_addr_(pset.get<std::string>("output_address", "localhost"))
00041 , seqIDModulus_(1)
00042 , lastFlushedSeqID_(0)
00043 , highestSeqIDSeen_(0)
00044 , enq_timeout_(pset.get<double>("event_queue_wait_time", 5.0))
00045 , enq_check_count_(pset.get<size_t>("event_queue_check_count", 5000))
00046 , printSummaryStats_(pset.get<bool>("print_event_store_stats", false))
00047 , incomplete_event_report_interval_ms_(pset.get<int>("incomplete_event_report_interval_ms", -1))
00048 , last_incomplete_event_report_time_(std::chrono::steady_clock::now())
00049 , token_socket_(-1)
00050 , art_thread_wait_ms_(pset.get<int>("art_thread_wait_ms", 4000))
00051 {
00052 TLOG_DEBUG("EventStore") << "EventStore CONSTRUCTOR" << TLOG_ENDL;
00053 initStatistics_();
00054 setup_requests_(pset.get<std::string>("request_address", "227.128.12.26"));
00055
00056 auto rmConfig = pset.get<fhicl::ParameterSet>("routing_token_config", fhicl::ParameterSet());
00057 send_routing_tokens_ = rmConfig.get<bool>("use_routing_master", false);
00058 token_port_ = rmConfig.get<int>("routing_token_port", 35555);
00059 token_address_ = rmConfig.get<std::string>("routing_master_hostname", "localhost");
00060 setup_tokens_();
00061 TRACE(12, "artdaq::EventStore::EventStore ctor - reader_thread_ initialized");
00062 }
00063
00064 EventStore::EventStore(const fhicl::ParameterSet& pset,
00065 size_t num_fragments_per_event,
00066 run_id_t run,
00067 int argc,
00068 char* argv[],
00069 ART_CMDLINE_FCN* reader)
00070 : EventStore(pset, num_fragments_per_event, run, 50, 50)
00071 { reader_thread_ = (std::async(std::launch::async, reader, argc, argv)); }
00072
00073 EventStore::EventStore(const fhicl::ParameterSet& pset,
00074 size_t num_fragments_per_event,
00075 run_id_t run,
00076 const std::string& configString,
00077 ART_CFGSTRING_FCN* reader)
00078 : EventStore(pset, num_fragments_per_event, run, 20, 20)
00079 { reader_thread_ = (std::async(std::launch::async, reader, configString)); }
00080
00081 EventStore::~EventStore()
00082 {
00083 TLOG_DEBUG("EventStore") << "Shutting down EventStore" << TLOG_ENDL;
00084 if (printSummaryStats_)
00085 {
00086 reportStatistics_();
00087 }
00088 shutdown(request_socket_, 2);
00089 close(request_socket_);
00090 shutdown(token_socket_, 2);
00091 close(token_socket_);
00092 }
00093
00094 void EventStore::insert(FragmentPtr pfrag,
00095 bool printWarningWhenFragmentIsDropped)
00096 {
00097
00098
00099 assert(pfrag != nullptr);
00100 assert(pfrag->fragmentID() != Fragment::InvalidFragmentID);
00101
00102
00103
00104
00105
00106
00107
00108
00109
00110 if (pfrag->sequenceID() > highestSeqIDSeen_)
00111 {
00112 highestSeqIDSeen_ = pfrag->sequenceID();
00113
00114
00115 Fragment::timestamp_t timestamp = pfrag->timestamp();
00116
00117
00118 if (send_requests_)
00119 {
00120 std::lock_guard<std::mutex> lk(request_mutex_);
00121 active_requests_[highestSeqIDSeen_] = timestamp;
00122 send_request_();
00123 }
00124 }
00125 Fragment::sequence_id_t sequence_id = ((pfrag->sequenceID() - (1 + lastFlushedSeqID_)) / seqIDModulus_) + 1;
00126 TRACE(13, "EventStore::insert seq=%lu fragID=%d id=%d lastFlushed=%lu seqIDMod=%d seq=%lu"
00127 , pfrag->sequenceID(), pfrag->fragmentID(), my_rank, lastFlushedSeqID_, seqIDModulus_, sequence_id);
00128
00129
00130
00131
00132 EventMap::iterator loc = events_.lower_bound(sequence_id);
00133
00134 if (loc == events_.end() || events_.key_comp()(sequence_id, loc->first))
00135 {
00136
00137
00138 RawEvent_ptr newevent(new RawEvent(run_id_, subrun_id_, pfrag->sequenceID()));
00139 loc =
00140 events_.insert(loc, EventMap::value_type(sequence_id, newevent));
00141 }
00142
00143
00144 loc->second->insertFragment(std::move(pfrag));
00145 if (loc->second->numFragments() == num_fragments_per_event_)
00146 {
00147
00148
00149
00150 RawEvent_ptr complete_event(loc->second);
00151 complete_event->markComplete();
00152
00153 events_.erase(loc);
00154
00155 if (send_requests_)
00156 {
00157 std::lock_guard<std::mutex> lk(request_mutex_);
00158 active_requests_.erase(sequence_id);
00159 }
00160
00161
00162
00163 MonitoredQuantityPtr mqPtr = StatisticsCollection::getInstance().
00164 getMonitoredQuantity(EVENT_RATE_STAT_KEY);
00165 if (mqPtr.get() != 0)
00166 {
00167 mqPtr->addSample(complete_event->wordCount());
00168 }
00169 TRACE(14, "EventStore::insert seq=%lu enqTimedWait start", sequence_id);
00170 bool enqSuccess = queue_.enqTimedWait(complete_event, enq_timeout_);
00171 TRACE(enqSuccess ? 14 : 0, "EventStore::insert seq=%lu enqTimedWait complete", sequence_id);
00172 if (!enqSuccess)
00173 {
00174
00175 if (printWarningWhenFragmentIsDropped)
00176 {
00177 TLOG_WARNING("EventStore") << "Enqueueing event " << sequence_id
00178 << " FAILED, queue size = "
00179 << queue_.size() <<
00180 "; apparently no events were removed from this process's queue during the " << std::to_string(enq_timeout_.count())
00181 << "-second timeout period" << TLOG_ENDL;
00182 }
00183 else
00184 {
00185 TLOG_DEBUG("EventStore") << "Enqueueing event " << sequence_id
00186 << " FAILED, queue size = "
00187 << queue_.size() <<
00188 "; apparently no events were removed from this process's queue during the " << std::to_string(enq_timeout_.count())
00189 << "-second timeout period" << TLOG_ENDL;
00190 }
00191 }
00192 else
00193 {
00194 send_routing_token_(1);
00195 }
00196 }
00197 MonitoredQuantityPtr mqPtr = StatisticsCollection::getInstance().
00198 getMonitoredQuantity(INCOMPLETE_EVENT_STAT_KEY);
00199 if (mqPtr.get() != 0)
00200 {
00201 mqPtr->addSample(events_.size());
00202 }
00203 }
00204
00205 EventStore::EventStoreInsertResult EventStore::insert(FragmentPtr pfrag, FragmentPtr& rejectedFragment)
00206 {
00207
00208
00209
00210
00211 TRACE(12, "EventStore: Testing if queue is full");
00212 if (queue_.full())
00213 {
00214 size_t sleepTime = 1000000 * (enq_timeout_.count() / enq_check_count_);
00215 TRACE(12, "EventStore: sleepTime is %lu.", sleepTime);
00216 size_t loopCount = 0;
00217 while (loopCount < enq_check_count_ && queue_.full())
00218 {
00219 ++loopCount;
00220 usleep(sleepTime);
00221 }
00222 if (queue_.full())
00223 {
00224 rejectedFragment = std::move(pfrag);
00225 return EventStoreInsertResult::REJECT_QUEUEFULL;
00226 }
00227 }
00228 TRACE(12, "EventStore: Testing if there's room in the EventStore");
00229 auto incomplete_full = events_.size() >= max_incomplete_count_;
00230 if (incomplete_full)
00231 {
00232 EventMap::iterator loc = events_.lower_bound(pfrag->sequenceID());
00233
00234 if (loc == events_.end() || events_.key_comp()(pfrag->sequenceID(), loc->first))
00235 {
00236 rejectedFragment = std::move(pfrag);
00237 return EventStoreInsertResult::REJECT_STOREFULL;
00238 }
00239 }
00240
00241 TRACE(12, "EventStore: Performing insert");
00242 insert(std::move(pfrag));
00243 return incomplete_full ? EventStoreInsertResult::SUCCESS_STOREFULL : EventStoreInsertResult::SUCCESS;
00244 }
00245
00246 bool
00247 EventStore::endOfData(int& readerReturnValue)
00248 {
00249 TLOG_DEBUG("EventStore") << "EventStore::endOfData" << TLOG_ENDL;
00250 RawEvent_ptr end_of_data(nullptr);
00251 TRACE(4, "EventStore::endOfData: Enqueuing end_of_data event");
00252 bool enqSuccess = queue_.enqTimedWait(end_of_data, enq_timeout_);
00253 if (!enqSuccess)
00254 {
00255 return false;
00256 }
00257 TRACE(4, "EventStore::endOfData: Getting return code from art thread");
00258 readerReturnValue = reader_thread_.get();
00259 return true;
00260 }
00261
00262 void EventStore::setSeqIDModulus(unsigned int seqIDModulus)
00263 {
00264 seqIDModulus_ = seqIDModulus;
00265 }
00266
00267 bool EventStore::flushData()
00268 {
00269 bool enqSuccess;
00270 size_t initialStoreSize = events_.size();
00271 TLOG_DEBUG("EventStore") << "Flushing " << initialStoreSize
00272 << " stale events from the EventStore." << TLOG_ENDL;
00273 EventMap::iterator loc;
00274 std::vector<sequence_id_t> flushList;
00275 for (loc = events_.begin(); loc != events_.end(); ++loc)
00276 {
00277 RawEvent_ptr complete_event(loc->second);
00278 MonitoredQuantityPtr mqPtr = StatisticsCollection::getInstance().
00279 getMonitoredQuantity(EVENT_RATE_STAT_KEY);
00280 if (mqPtr.get() != 0)
00281 {
00282 mqPtr->addSample(complete_event->wordCount());
00283 }
00284 enqSuccess = queue_.enqTimedWait(complete_event, enq_timeout_);
00285 if (!enqSuccess)
00286 {
00287 break;
00288 }
00289 else
00290 {
00291 flushList.push_back(loc->first);
00292 }
00293 }
00294 for (size_t idx = 0; idx < flushList.size(); ++idx)
00295 {
00296 events_.erase(flushList[idx]);
00297 }
00298 TLOG_DEBUG("EventStore") << "Done flushing " << flushList.size()
00299 << " stale events from the EventStore." << TLOG_ENDL;
00300
00301 lastFlushedSeqID_ = highestSeqIDSeen_;
00302 return (flushList.size() >= initialStoreSize);
00303 }
00304
00305 void EventStore::startRun(run_id_t runID)
00306 {
00307 if (!queue_.queueReaderIsReady())
00308 {
00309 TLOG_WARNING("EventStore") << "Run start requested, but the art thread is not yet ready, waiting up to " << art_thread_wait_ms_ << " msec..." << TLOG_ENDL;
00310 while (!queue_.queueReaderIsReady() && std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - reader_thread_launch_time_).count() < art_thread_wait_ms_)
00311 {
00312 usleep(1000);
00313 }
00314 if (queue_.queueReaderIsReady())
00315 {
00316 auto dur = std::chrono::duration_cast<std::chrono::milliseconds>(queue_.getReadyTime() - reader_thread_launch_time_).count();
00317 TLOG_INFO("EventStore") << "art initialization took (roughly) " << std::setw(4) << std::to_string(dur) << " ms." << TLOG_ENDL;
00318 }
00319 else
00320 {
00321 auto dur = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - reader_thread_launch_time_).count();
00322 TLOG_ERROR("EventStore") << "art thread still not ready after " << dur << " ms. Continuing to start..." << TLOG_ENDL;
00323 }
00324 }
00325 run_id_ = runID;
00326 subrun_id_ = 1;
00327 lastFlushedSeqID_ = 0;
00328 highestSeqIDSeen_ = 0;
00329 send_routing_token_(max_queue_size_);
00330 TLOG_DEBUG("EventStore") << "Starting run " << run_id_
00331 << ", max queue size = "
00332 << max_queue_size_
00333 << ", queue capacity = "
00334 << queue_.capacity()
00335 << ", queue size = "
00336 << queue_.size() << TLOG_ENDL;
00337 if (metricMan)
00338 {
00339 double runSubrun = run_id_ + ((double)subrun_id_ / 10000);
00340 metricMan->sendMetric("Run Number", runSubrun, "Run:Subrun", 1, false);
00341 }
00342 }
00343
00344 void EventStore::startSubrun()
00345 {
00346 ++subrun_id_;
00347 if (metricMan)
00348 {
00349 double runSubrun = run_id_ + ((double)subrun_id_ / 10000);
00350 metricMan->sendMetric("Run Number", runSubrun, "Run:Subrun", 1, false);
00351 }
00352 }
00353
00354 bool EventStore::endRun()
00355 {
00356 RawEvent_ptr endOfRunEvent(new RawEvent(run_id_, subrun_id_, 0));
00357 std::unique_ptr<artdaq::Fragment>
00358 endOfRunFrag(new
00359 Fragment(static_cast<size_t>
00360 (ceil(sizeof(my_rank) /
00361 static_cast<double>(sizeof(Fragment::value_type))))));
00362
00363 endOfRunFrag->setSystemType(Fragment::EndOfRunFragmentType);
00364 *endOfRunFrag->dataBegin() = my_rank;
00365 endOfRunEvent->insertFragment(std::move(endOfRunFrag));
00366
00367 return queue_.enqTimedWait(endOfRunEvent, enq_timeout_);
00368 }
00369
00370 bool EventStore::endSubrun()
00371 {
00372 RawEvent_ptr endOfSubrunEvent(new RawEvent(run_id_, subrun_id_, 0));
00373 std::unique_ptr<artdaq::Fragment>
00374 endOfSubrunFrag(new
00375 Fragment(static_cast<size_t>
00376 (ceil(sizeof(my_rank) /
00377 static_cast<double>(sizeof(Fragment::value_type))))));
00378
00379 endOfSubrunFrag->setSystemType(Fragment::EndOfSubrunFragmentType);
00380 *endOfSubrunFrag->dataBegin() = my_rank;
00381 endOfSubrunEvent->insertFragment(std::move(endOfSubrunFrag));
00382
00383 return queue_.enqTimedWait(endOfSubrunEvent, enq_timeout_);
00384 }
00385
00386 void
00387 EventStore::initStatistics_()
00388 {
00389 MonitoredQuantityPtr mqPtr = StatisticsCollection::getInstance().
00390 getMonitoredQuantity(EVENT_RATE_STAT_KEY);
00391 if (mqPtr.get() == 0)
00392 {
00393 mqPtr.reset(new MonitoredQuantity(3.0, 300.0));
00394 StatisticsCollection::getInstance().
00395 addMonitoredQuantity(EVENT_RATE_STAT_KEY, mqPtr);
00396 }
00397 mqPtr->reset();
00398
00399 mqPtr = StatisticsCollection::getInstance().
00400 getMonitoredQuantity(INCOMPLETE_EVENT_STAT_KEY);
00401 if (mqPtr.get() == 0)
00402 {
00403 mqPtr.reset(new MonitoredQuantity(3.0, 300.0));
00404 StatisticsCollection::getInstance().
00405 addMonitoredQuantity(INCOMPLETE_EVENT_STAT_KEY, mqPtr);
00406 }
00407 mqPtr->reset();
00408 }
00409
00410 void
00411 EventStore::reportStatistics_()
00412 {
00413 MonitoredQuantityPtr mqPtr = StatisticsCollection::getInstance().
00414 getMonitoredQuantity(EVENT_RATE_STAT_KEY);
00415 if (mqPtr.get() != 0)
00416 {
00417 ostringstream oss;
00418 oss << EVENT_RATE_STAT_KEY << "_" << setfill('0') << setw(4) << run_id_
00419 << "_" << setfill('0') << setw(4) << my_rank << ".txt";
00420 std::string filename = oss.str();
00421 ofstream outStream(filename.c_str());
00422 mqPtr->waitUntilAccumulatorsHaveBeenFlushed(3.0);
00423 artdaq::MonitoredQuantityStats stats;
00424 mqPtr->getStats(stats);
00425 outStream << "EventStore rank " << my_rank << ": events processed = "
00426 << stats.fullSampleCount << " at " << stats.fullSampleRate
00427 << " events/sec, data rate = "
00428 << (stats.fullValueRate * sizeof(RawDataType)
00429 / 1024.0 / 1024.0) << " MB/sec, duration = "
00430 << stats.fullDuration << " sec" << std::endl
00431 << " minimum event size = "
00432 << (stats.fullValueMin * sizeof(RawDataType)
00433 / 1024.0 / 1024.0)
00434 << " MB, maximum event size = "
00435 << (stats.fullValueMax * sizeof(RawDataType)
00436 / 1024.0 / 1024.0)
00437 << " MB" << std::endl;
00438 bool foundTheStart = false;
00439 for (int idx = 0; idx < (int)stats.recentBinnedDurations.size(); ++idx)
00440 {
00441 if (stats.recentBinnedDurations[idx] > 0.0)
00442 {
00443 foundTheStart = true;
00444 }
00445 if (foundTheStart)
00446 {
00447 outStream << " " << std::fixed << std::setprecision(3)
00448 << stats.recentBinnedEndTimes[idx]
00449 << ": " << stats.recentBinnedSampleCounts[idx]
00450 << " events at "
00451 << (stats.recentBinnedSampleCounts[idx] /
00452 stats.recentBinnedDurations[idx])
00453 << " events/sec, data rate = "
00454 << (stats.recentBinnedValueSums[idx] *
00455 sizeof(RawDataType) / 1024.0 / 1024.0 /
00456 stats.recentBinnedDurations[idx])
00457 << " MB/sec, bin size = "
00458 << stats.recentBinnedDurations[idx]
00459 << " sec" << std::endl;
00460 }
00461 }
00462 outStream.close();
00463 }
00464
00465 mqPtr = StatisticsCollection::getInstance().
00466 getMonitoredQuantity(INCOMPLETE_EVENT_STAT_KEY);
00467 if (mqPtr.get() != 0)
00468 {
00469 ostringstream oss;
00470 oss << INCOMPLETE_EVENT_STAT_KEY << "_" << setfill('0')
00471 << setw(4) << run_id_
00472 << "_" << setfill('0') << setw(4) << my_rank << ".txt";
00473 std::string filename = oss.str();
00474 ofstream outStream(filename.c_str());
00475 mqPtr->waitUntilAccumulatorsHaveBeenFlushed(3.0);
00476 artdaq::MonitoredQuantityStats stats;
00477 mqPtr->getStats(stats);
00478 outStream << "EventStore rank " << my_rank << ": fragments processed = "
00479 << stats.fullSampleCount << " at " << stats.fullSampleRate
00480 << " fragments/sec, average incomplete event count = "
00481 << stats.fullValueAverage << " duration = "
00482 << stats.fullDuration << " sec" << std::endl
00483 << " minimum incomplete event count = "
00484 << stats.fullValueMin << ", maximum incomplete event count = "
00485 << stats.fullValueMax << std::endl;
00486 bool foundTheStart = false;
00487 for (int idx = 0; idx < (int)stats.recentBinnedDurations.size(); ++idx)
00488 {
00489 if (stats.recentBinnedDurations[idx] > 0.0)
00490 {
00491 foundTheStart = true;
00492 }
00493 if (foundTheStart && stats.recentBinnedSampleCounts[idx] > 0.0)
00494 {
00495 outStream << " " << std::fixed << std::setprecision(3)
00496 << stats.recentBinnedEndTimes[idx]
00497 << ": " << stats.recentBinnedSampleCounts[idx]
00498 << " fragments at "
00499 << (stats.recentBinnedSampleCounts[idx] /
00500 stats.recentBinnedDurations[idx])
00501 << " fragments/sec, average incomplete event count = "
00502 << (stats.recentBinnedValueSums[idx] /
00503 stats.recentBinnedSampleCounts[idx])
00504 << ", bin size = "
00505 << stats.recentBinnedDurations[idx]
00506 << " sec" << std::endl;
00507 }
00508 }
00509 outStream << "Incomplete count now = " << events_.size() << std::endl;
00510 outStream.close();
00511 }
00512 }
00513
00514 void
00515 EventStore::setup_requests_(std::string request_address)
00516 {
00517 if (send_requests_)
00518 {
00519 request_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
00520 if (!request_socket_)
00521 {
00522 TLOG_ERROR("EventStore") << "I failed to create the socket for sending Data Requests!" << TLOG_ENDL;
00523 exit(1);
00524 }
00525 int sts = ResolveHost(request_address.c_str(), request_port_, request_addr_);
00526 if (sts == -1)
00527 {
00528 TLOG_ERROR("EventStore") << "Unable to resolve Data Request address" << TLOG_ENDL;
00529 exit(1);
00530 }
00531
00532 if (multicast_out_addr_ != "localhost") {
00533 struct in_addr addr;
00534 int sts = ResolveHost(multicast_out_addr_.c_str(), addr);
00535 if (sts == -1)
00536 {
00537 TLOG_ERROR("EventStore") << "Unable to resolve multicast interface address" << TLOG_ENDL;
00538 exit(1);
00539 }
00540
00541 int yes = 1;
00542 if (setsockopt(request_socket_, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) < 0)
00543 {
00544 TLOG_ERROR("EventStore") << "Unable to enable port reuse on request socket" << TLOG_ENDL;
00545 exit(1);
00546 }
00547 if (setsockopt(request_socket_, IPPROTO_IP, IP_MULTICAST_IF, &addr, sizeof(addr)) == -1)
00548 {
00549 TLOG_ERROR("EventStore") << "Cannot set outgoing interface." << TLOG_ENDL;
00550 exit(1);
00551 }
00552 }
00553 int yes = 1;
00554 if (setsockopt(request_socket_, SOL_SOCKET, SO_BROADCAST, (void*)&yes, sizeof(int)) == -1)
00555 {
00556 TLOG_ERROR("EventStore") << "Cannot set request socket to broadcast." << TLOG_ENDL;
00557 exit(1);
00558 }
00559 }
00560 }
00561
00562 void
00563 EventStore::setup_tokens_()
00564 {
00565 if (send_routing_tokens_)
00566 {
00567 TLOG_DEBUG("EventStore") << "Creating Routing Token sending socket" << TLOG_ENDL;
00568 token_socket_ = TCPConnect(token_address_.c_str(), token_port_);
00569 if (!token_socket_)
00570 {
00571 TLOG_ERROR("EventStore") << "I failed to create the socket for sending Routing Tokens!" << TLOG_ENDL;
00572 exit(1);
00573 }
00574 }
00575 }
00576
00577 void EventStore::do_send_request_()
00578 {
00579 std::this_thread::sleep_for(std::chrono::microseconds(request_delay_));
00580
00581 detail::RequestMessage message;
00582 {
00583 std::lock_guard<std::mutex> lk(request_mutex_);
00584 for (auto& req : active_requests_)
00585 {
00586 message.addRequest(req.first, req.second);
00587 }
00588 }
00589 char str[INET_ADDRSTRLEN];
00590 inet_ntop(AF_INET, &(request_addr_.sin_addr), str, INET_ADDRSTRLEN);
00591 TLOG_DEBUG("EventStore") << "Sending request for " << std::to_string(message.size()) << " events to multicast group " << str << TLOG_ENDL;
00592 if (sendto(request_socket_, message.header(), sizeof(detail::RequestHeader), 0, (struct sockaddr *)&request_addr_, sizeof(request_addr_)) < 0)
00593 {
00594 TLOG_ERROR("EventStore") << "Error sending request message header" << TLOG_ENDL;
00595 }
00596 if (sendto(request_socket_, message.buffer(), sizeof(detail::RequestPacket) * message.size(), 0, (struct sockaddr *)&request_addr_, sizeof(request_addr_)) < 0)
00597 {
00598 TLOG_ERROR("EventStore") << "Error sending request message data" << TLOG_ENDL;
00599 }
00600 }
00601
00602 void EventStore::send_routing_token_(int nSlots)
00603 {
00604 TLOG_DEBUG("EventStore") << "send_routing_token_ called, send_routing_tokens_=" << std::boolalpha << send_routing_tokens_ << TLOG_ENDL;
00605 if (!send_routing_tokens_) return;
00606 if (token_socket_ == -1) setup_tokens_();
00607 detail::RoutingToken token;
00608 token.header = TOKEN_MAGIC;
00609 token.rank = my_rank;
00610 token.new_slots_free = nSlots;
00611
00612 TLOG_DEBUG("EventStore") << "Sending RoutingToken to " << token_address_ << ":" << token_port_ << TLOG_ENDL;
00613 size_t sts = 0;
00614 while (sts < sizeof(detail::RoutingToken)) {
00615 auto res = send(token_socket_, reinterpret_cast<uint8_t*>(&token) + sts, sizeof(detail::RoutingToken) - sts, 0);
00616 if (res == -1) {
00617 usleep(1000);
00618 continue;
00619 }
00620 sts += res;
00621 }
00622 TLOG_DEBUG("EventStore") << "Done sending RoutingToken to " << token_address_ << ":" << token_port_ << TLOG_ENDL;
00623 }
00624
00625 void
00626 EventStore::send_request_()
00627 {
00628 std::thread request([=] { do_send_request_(); });
00629 request.detach();
00630 }
00631
00632 void
00633 EventStore::sendMetrics()
00634 {
00635 if (metricMan)
00636 {
00637 metricMan->sendMetric("Incomplete Event Count", events_.size(),
00638 "events", 1);
00639 }
00640 if (incomplete_event_report_interval_ms_ > 0 && events_.size())
00641 {
00642 if (std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - last_incomplete_event_report_time_).count() < incomplete_event_report_interval_ms_) return;
00643 last_incomplete_event_report_time_ = std::chrono::steady_clock::now();
00644 std::ostringstream oss;
00645 oss << "Incomplete Events (" << num_fragments_per_event_ << "): ";
00646 for (auto& ev : events_)
00647 {
00648 oss << ev.first << " (" << ev.second->numFragments() << "), ";
00649 }
00650 TLOG_DEBUG("EventStore") << oss.str() << TLOG_ENDL;
00651 }
00652 }
00653 }