00001 #include "artdaq/DAQdata/Globals.hh"
00002 #include "artdaq/DAQrate/EventStore.hh"
00003 #include <utility>
00004 #include <cstring>
00005 #include <dlfcn.h>
00006 #include <iomanip>
00007 #include <fstream>
00008 #include <sstream>
00009 #include <thread>
00010 #include <chrono>
00011
00012 #include "cetlib/exception.h"
00013 #include "artdaq-core/Core/StatisticsCollection.hh"
00014 #include "artdaq-core/Core/SimpleQueueReader.hh"
00015 #include "artdaq/Application/Routing/RoutingPacket.hh"
00016 #include "artdaq/DAQdata/TCPConnect.hh"
00017
00018 using namespace std;
00019
00020 namespace artdaq
00021 {
00022 const std::string EventStore::EVENT_RATE_STAT_KEY("EventStoreEventRate");
00023 const std::string EventStore::INCOMPLETE_EVENT_STAT_KEY("EventStoreIncompleteEvents");
00024
00025 EventStore::EventStore(const fhicl::ParameterSet& pset, size_t num_fragments_per_event, run_id_t run,
00026 size_t event_queue_depth, size_t max_incomplete_event_count)
00027 : num_fragments_per_event_(num_fragments_per_event)
00028 , max_queue_size_(pset.get<size_t>("event_queue_depth", event_queue_depth))
00029 , max_incomplete_count_(pset.get<size_t>("max_incomplete_events", max_incomplete_event_count))
00030 , run_id_(run)
00031 , subrun_id_(0)
00032 , events_()
00033 , queue_(getGlobalQueue(max_queue_size_))
00034 , reader_thread_launch_time_(std::chrono::steady_clock::now())
00035 , send_requests_(pset.get<bool>("send_requests", false))
00036 , active_requests_()
00037 , request_port_(pset.get<int>("request_port", 3001))
00038 , request_delay_(pset.get<size_t>("request_delay_ms", 10))
00039 , multicast_out_addr_(pset.get<std::string>("output_address", "localhost"))
00040 , request_mode_(detail::RequestMessageMode::Normal)
00041 , seqIDModulus_(1)
00042 , lastFlushedSeqID_(0)
00043 , highestSeqIDSeen_(0)
00044 , enq_timeout_(pset.get<double>("event_queue_wait_time", 5.0))
00045 , enq_check_count_(pset.get<size_t>("event_queue_check_count", 5000))
00046 , printSummaryStats_(pset.get<bool>("print_event_store_stats", false))
00047 , incomplete_event_report_interval_ms_(pset.get<int>("incomplete_event_report_interval_ms", -1))
00048 , last_incomplete_event_report_time_(std::chrono::steady_clock::now())
00049 , token_socket_(-1)
00050 , art_thread_wait_ms_(pset.get<int>("art_thread_wait_ms", 4000))
00051 {
00052 TLOG_DEBUG("EventStore") << "EventStore CONSTRUCTOR" << TLOG_ENDL;
00053 initStatistics_();
00054 setup_requests_(pset.get<std::string>("request_address", "227.128.12.26"));
00055
00056 auto rmConfig = pset.get<fhicl::ParameterSet>("routing_token_config", fhicl::ParameterSet());
00057 send_routing_tokens_ = rmConfig.get<bool>("use_routing_master", false);
00058 token_port_ = rmConfig.get<int>("routing_token_port", 35555);
00059 token_address_ = rmConfig.get<std::string>("routing_master_hostname", "localhost");
00060 setup_tokens_();
00061 TRACE(12, "artdaq::EventStore::EventStore ctor - reader_thread_ initialized");
00062 }
00063
00064 EventStore::EventStore(const fhicl::ParameterSet& pset,
00065 size_t num_fragments_per_event,
00066 run_id_t run,
00067 int argc,
00068 char* argv[],
00069 ART_CMDLINE_FCN* reader)
00070 : EventStore(pset, num_fragments_per_event, run, 50, 50)
00071 {
00072 reader_thread_ = (std::async(std::launch::async, reader, argc, argv));
00073 }
00074
00075 EventStore::EventStore(const fhicl::ParameterSet& pset,
00076 size_t num_fragments_per_event,
00077 run_id_t run,
00078 const std::string& configString,
00079 ART_CFGSTRING_FCN* reader)
00080 : EventStore(pset, num_fragments_per_event, run, 20, 20)
00081 {
00082 reader_thread_ = (std::async(std::launch::async, reader, configString));
00083 }
00084
00085 EventStore::~EventStore()
00086 {
00087 TLOG_DEBUG("EventStore") << "Shutting down EventStore" << TLOG_ENDL;
00088 if (printSummaryStats_)
00089 {
00090 reportStatistics_();
00091 }
00092 shutdown(request_socket_, 2);
00093 close(request_socket_);
00094 shutdown(token_socket_, 2);
00095 close(token_socket_);
00096 }
00097
00098 void EventStore::insert(FragmentPtr pfrag,
00099 bool printWarningWhenFragmentIsDropped)
00100 {
00101
00102
00103 assert(pfrag != nullptr);
00104 assert(pfrag->fragmentID() != Fragment::InvalidFragmentID);
00105
00106
00107
00108
00109
00110
00111
00112
00113
00114 if (pfrag->sequenceID() > highestSeqIDSeen_)
00115 {
00116 highestSeqIDSeen_ = pfrag->sequenceID();
00117
00118
00119 Fragment::timestamp_t timestamp = pfrag->timestamp();
00120
00121
00122 if (send_requests_)
00123 {
00124 std::lock_guard<std::mutex> lk(request_mutex_);
00125 active_requests_[highestSeqIDSeen_] = timestamp;
00126 send_request_();
00127 }
00128 }
00129
00130
00131
00132 if (send_requests_ && request_mode_ == detail::RequestMessageMode::EndOfRun)
00133 {
00134 std::lock_guard<std::mutex> lk(request_mutex_);
00135 send_request_();
00136 }
00137 Fragment::sequence_id_t sequence_id = ((pfrag->sequenceID() - (1 + lastFlushedSeqID_)) / seqIDModulus_) + 1;
00138 TRACE(13, "EventStore::insert seq=%lu fragID=%d id=%d lastFlushed=%lu seqIDMod=%d seq=%lu"
00139 , pfrag->sequenceID(), pfrag->fragmentID(), my_rank, lastFlushedSeqID_, seqIDModulus_, sequence_id);
00140
00141
00142
00143
00144 EventMap::iterator loc = events_.lower_bound(sequence_id);
00145
00146 if (loc == events_.end() || events_.key_comp()(sequence_id, loc->first))
00147 {
00148
00149
00150 RawEvent_ptr newevent(new RawEvent(run_id_, subrun_id_, pfrag->sequenceID()));
00151 loc =
00152 events_.insert(loc, EventMap::value_type(sequence_id, newevent));
00153 }
00154
00155
00156 loc->second->insertFragment(std::move(pfrag));
00157 if (loc->second->numFragments() == num_fragments_per_event_)
00158 {
00159
00160
00161
00162 RawEvent_ptr complete_event(loc->second);
00163 complete_event->markComplete();
00164
00165 events_.erase(loc);
00166
00167 if (send_requests_)
00168 {
00169 std::lock_guard<std::mutex> lk(request_mutex_);
00170 active_requests_.erase(sequence_id);
00171 }
00172
00173
00174
00175 MonitoredQuantityPtr mqPtr = StatisticsCollection::getInstance().
00176 getMonitoredQuantity(EVENT_RATE_STAT_KEY);
00177 if (mqPtr.get() != 0)
00178 {
00179 mqPtr->addSample(complete_event->wordCount());
00180 }
00181 TRACE(14, "EventStore::insert seq=%lu enqTimedWait start", sequence_id);
00182 bool enqSuccess = queue_.enqTimedWait(complete_event, enq_timeout_);
00183 TRACE(enqSuccess ? 14 : 0, "EventStore::insert seq=%lu enqTimedWait complete", sequence_id);
00184 if (!enqSuccess)
00185 {
00186
00187 if (printWarningWhenFragmentIsDropped)
00188 {
00189 TLOG_WARNING("EventStore") << "Enqueueing event " << sequence_id
00190 << " FAILED, queue size = "
00191 << queue_.size() <<
00192 "; apparently no events were removed from this process's queue during the " << std::to_string(enq_timeout_.count())
00193 << "-second timeout period" << TLOG_ENDL;
00194 }
00195 else
00196 {
00197 TLOG_DEBUG("EventStore") << "Enqueueing event " << sequence_id
00198 << " FAILED, queue size = "
00199 << queue_.size() <<
00200 "; apparently no events were removed from this process's queue during the " << std::to_string(enq_timeout_.count())
00201 << "-second timeout period" << TLOG_ENDL;
00202 }
00203 }
00204 else
00205 {
00206 send_routing_token_(1);
00207 }
00208 }
00209 MonitoredQuantityPtr mqPtr = StatisticsCollection::getInstance().
00210 getMonitoredQuantity(INCOMPLETE_EVENT_STAT_KEY);
00211 if (mqPtr.get() != 0)
00212 {
00213 mqPtr->addSample(events_.size());
00214 }
00215 }
00216
00217 EventStore::EventStoreInsertResult EventStore::insert(FragmentPtr pfrag, FragmentPtr& rejectedFragment)
00218 {
00219
00220
00221
00222
00223 TRACE(12, "EventStore: Testing if queue is full");
00224 if (queue_.full())
00225 {
00226 size_t sleepTime = 1000000 * (enq_timeout_.count() / enq_check_count_);
00227 TRACE(12, "EventStore: sleepTime is %lu.", sleepTime);
00228 size_t loopCount = 0;
00229 while (loopCount < enq_check_count_ && queue_.full())
00230 {
00231 ++loopCount;
00232 usleep(sleepTime);
00233 }
00234 if (queue_.full())
00235 {
00236 rejectedFragment = std::move(pfrag);
00237 return EventStoreInsertResult::REJECT_QUEUEFULL;
00238 }
00239 }
00240 TRACE(12, "EventStore: Testing if there's room in the EventStore");
00241 auto incomplete_full = events_.size() >= max_incomplete_count_;
00242 if (incomplete_full)
00243 {
00244 EventMap::iterator loc = events_.lower_bound(pfrag->sequenceID());
00245
00246 if (loc == events_.end() || events_.key_comp()(pfrag->sequenceID(), loc->first))
00247 {
00248 rejectedFragment = std::move(pfrag);
00249 return EventStoreInsertResult::REJECT_STOREFULL;
00250 }
00251 }
00252
00253 TRACE(12, "EventStore: Performing insert");
00254 insert(std::move(pfrag));
00255 return incomplete_full ? EventStoreInsertResult::SUCCESS_STOREFULL : EventStoreInsertResult::SUCCESS;
00256 }
00257
00258 bool
00259 EventStore::endOfData(int& readerReturnValue)
00260 {
00261 TLOG_DEBUG("EventStore") << "EventStore::endOfData" << TLOG_ENDL;
00262 RawEvent_ptr end_of_data(nullptr);
00263 TRACE(4, "EventStore::endOfData: Enqueuing end_of_data event");
00264 bool enqSuccess = queue_.enqTimedWait(end_of_data, enq_timeout_);
00265 if (!enqSuccess)
00266 {
00267 return false;
00268 }
00269 TRACE(4, "EventStore::endOfData: Getting return code from art thread");
00270 readerReturnValue = reader_thread_.get();
00271 return true;
00272 }
00273
00274 void EventStore::setSeqIDModulus(unsigned int seqIDModulus)
00275 {
00276 seqIDModulus_ = seqIDModulus;
00277 }
00278
00279 bool EventStore::flushData()
00280 {
00281 bool enqSuccess;
00282 size_t initialStoreSize = events_.size();
00283 TLOG_DEBUG("EventStore") << "Flushing " << initialStoreSize
00284 << " stale events from the EventStore." << TLOG_ENDL;
00285 EventMap::iterator loc;
00286 std::vector<sequence_id_t> flushList;
00287 for (loc = events_.begin(); loc != events_.end(); ++loc)
00288 {
00289 RawEvent_ptr complete_event(loc->second);
00290 MonitoredQuantityPtr mqPtr = StatisticsCollection::getInstance().
00291 getMonitoredQuantity(EVENT_RATE_STAT_KEY);
00292 if (mqPtr.get() != 0)
00293 {
00294 mqPtr->addSample(complete_event->wordCount());
00295 }
00296 enqSuccess = queue_.enqTimedWait(complete_event, enq_timeout_);
00297 if (!enqSuccess)
00298 {
00299 break;
00300 }
00301 else
00302 {
00303 flushList.push_back(loc->first);
00304 }
00305 }
00306 for (size_t idx = 0; idx < flushList.size(); ++idx)
00307 {
00308 events_.erase(flushList[idx]);
00309 }
00310 TLOG_DEBUG("EventStore") << "Done flushing " << flushList.size()
00311 << " stale events from the EventStore." << TLOG_ENDL;
00312
00313 lastFlushedSeqID_ = highestSeqIDSeen_;
00314 return (flushList.size() >= initialStoreSize);
00315 }
00316
00317 void EventStore::startRun(run_id_t runID)
00318 {
00319 if (!queue_.queueReaderIsReady())
00320 {
00321 TLOG_WARNING("EventStore") << "Run start requested, but the art thread is not yet ready, waiting up to " << art_thread_wait_ms_ << " msec..." << TLOG_ENDL;
00322 while (!queue_.queueReaderIsReady() && std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - reader_thread_launch_time_).count() < art_thread_wait_ms_)
00323 {
00324 usleep(1000);
00325 }
00326 if (queue_.queueReaderIsReady())
00327 {
00328 auto dur = std::chrono::duration_cast<std::chrono::milliseconds>(queue_.getReadyTime() - reader_thread_launch_time_).count();
00329 TLOG_INFO("EventStore") << "art initialization took (roughly) " << std::setw(4) << std::to_string(dur) << " ms." << TLOG_ENDL;
00330 }
00331 else
00332 {
00333 auto dur = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - reader_thread_launch_time_).count();
00334 TLOG_ERROR("EventStore") << "art thread still not ready after " << dur << " ms. Continuing to start..." << TLOG_ENDL;
00335 }
00336 }
00337 run_id_ = runID;
00338 subrun_id_ = 1;
00339 lastFlushedSeqID_ = 0;
00340 highestSeqIDSeen_ = 0;
00341 send_routing_token_(max_queue_size_);
00342 TLOG_DEBUG("EventStore") << "Starting run " << run_id_
00343 << ", max queue size = "
00344 << max_queue_size_
00345 << ", queue capacity = "
00346 << queue_.capacity()
00347 << ", queue size = "
00348 << queue_.size() << TLOG_ENDL;
00349 if (metricMan)
00350 {
00351 double runSubrun = run_id_ + ((double)subrun_id_ / 10000);
00352 metricMan->sendMetric("Run Number", runSubrun, "Run:Subrun", 1, false);
00353 }
00354 }
00355
00356 void EventStore::startSubrun()
00357 {
00358 ++subrun_id_;
00359 if (metricMan)
00360 {
00361 double runSubrun = run_id_ + ((double)subrun_id_ / 10000);
00362 metricMan->sendMetric("Run Number", runSubrun, "Run:Subrun", 1, false);
00363 }
00364 }
00365
00366 bool EventStore::endRun()
00367 {
00368 RawEvent_ptr endOfRunEvent(new RawEvent(run_id_, subrun_id_, 0));
00369 std::unique_ptr<artdaq::Fragment>
00370 endOfRunFrag(new
00371 Fragment(static_cast<size_t>
00372 (ceil(sizeof(my_rank) /
00373 static_cast<double>(sizeof(Fragment::value_type))))));
00374
00375 endOfRunFrag->setSystemType(Fragment::EndOfRunFragmentType);
00376 *endOfRunFrag->dataBegin() = my_rank;
00377 endOfRunEvent->insertFragment(std::move(endOfRunFrag));
00378
00379 return queue_.enqTimedWait(endOfRunEvent, enq_timeout_);
00380 }
00381
00382 bool EventStore::endSubrun()
00383 {
00384 RawEvent_ptr endOfSubrunEvent(new RawEvent(run_id_, subrun_id_, 0));
00385 std::unique_ptr<artdaq::Fragment>
00386 endOfSubrunFrag(new
00387 Fragment(static_cast<size_t>
00388 (ceil(sizeof(my_rank) /
00389 static_cast<double>(sizeof(Fragment::value_type))))));
00390
00391 endOfSubrunFrag->setSystemType(Fragment::EndOfSubrunFragmentType);
00392 *endOfSubrunFrag->dataBegin() = my_rank;
00393 endOfSubrunEvent->insertFragment(std::move(endOfSubrunFrag));
00394
00395 return queue_.enqTimedWait(endOfSubrunEvent, enq_timeout_);
00396 }
00397
00398 void
00399 EventStore::initStatistics_()
00400 {
00401 MonitoredQuantityPtr mqPtr = StatisticsCollection::getInstance().
00402 getMonitoredQuantity(EVENT_RATE_STAT_KEY);
00403 if (mqPtr.get() == 0)
00404 {
00405 mqPtr.reset(new MonitoredQuantity(3.0, 300.0));
00406 StatisticsCollection::getInstance().
00407 addMonitoredQuantity(EVENT_RATE_STAT_KEY, mqPtr);
00408 }
00409 mqPtr->reset();
00410
00411 mqPtr = StatisticsCollection::getInstance().
00412 getMonitoredQuantity(INCOMPLETE_EVENT_STAT_KEY);
00413 if (mqPtr.get() == 0)
00414 {
00415 mqPtr.reset(new MonitoredQuantity(3.0, 300.0));
00416 StatisticsCollection::getInstance().
00417 addMonitoredQuantity(INCOMPLETE_EVENT_STAT_KEY, mqPtr);
00418 }
00419 mqPtr->reset();
00420 }
00421
00422 void
00423 EventStore::reportStatistics_()
00424 {
00425 MonitoredQuantityPtr mqPtr = StatisticsCollection::getInstance().
00426 getMonitoredQuantity(EVENT_RATE_STAT_KEY);
00427 if (mqPtr.get() != 0)
00428 {
00429 ostringstream oss;
00430 oss << EVENT_RATE_STAT_KEY << "_" << setfill('0') << setw(4) << run_id_
00431 << "_" << setfill('0') << setw(4) << my_rank << ".txt";
00432 std::string filename = oss.str();
00433 ofstream outStream(filename.c_str());
00434 mqPtr->waitUntilAccumulatorsHaveBeenFlushed(3.0);
00435 artdaq::MonitoredQuantityStats stats;
00436 mqPtr->getStats(stats);
00437 outStream << "EventStore rank " << my_rank << ": events processed = "
00438 << stats.fullSampleCount << " at " << stats.fullSampleRate
00439 << " events/sec, data rate = "
00440 << (stats.fullValueRate * sizeof(RawDataType)
00441 / 1024.0 / 1024.0) << " MB/sec, duration = "
00442 << stats.fullDuration << " sec" << std::endl
00443 << " minimum event size = "
00444 << (stats.fullValueMin * sizeof(RawDataType)
00445 / 1024.0 / 1024.0)
00446 << " MB, maximum event size = "
00447 << (stats.fullValueMax * sizeof(RawDataType)
00448 / 1024.0 / 1024.0)
00449 << " MB" << std::endl;
00450 bool foundTheStart = false;
00451 for (int idx = 0; idx < (int)stats.recentBinnedDurations.size(); ++idx)
00452 {
00453 if (stats.recentBinnedDurations[idx] > 0.0)
00454 {
00455 foundTheStart = true;
00456 }
00457 if (foundTheStart)
00458 {
00459 outStream << " " << std::fixed << std::setprecision(3)
00460 << stats.recentBinnedEndTimes[idx]
00461 << ": " << stats.recentBinnedSampleCounts[idx]
00462 << " events at "
00463 << (stats.recentBinnedSampleCounts[idx] /
00464 stats.recentBinnedDurations[idx])
00465 << " events/sec, data rate = "
00466 << (stats.recentBinnedValueSums[idx] *
00467 sizeof(RawDataType) / 1024.0 / 1024.0 /
00468 stats.recentBinnedDurations[idx])
00469 << " MB/sec, bin size = "
00470 << stats.recentBinnedDurations[idx]
00471 << " sec" << std::endl;
00472 }
00473 }
00474 outStream.close();
00475 }
00476
00477 mqPtr = StatisticsCollection::getInstance().
00478 getMonitoredQuantity(INCOMPLETE_EVENT_STAT_KEY);
00479 if (mqPtr.get() != 0)
00480 {
00481 ostringstream oss;
00482 oss << INCOMPLETE_EVENT_STAT_KEY << "_" << setfill('0')
00483 << setw(4) << run_id_
00484 << "_" << setfill('0') << setw(4) << my_rank << ".txt";
00485 std::string filename = oss.str();
00486 ofstream outStream(filename.c_str());
00487 mqPtr->waitUntilAccumulatorsHaveBeenFlushed(3.0);
00488 artdaq::MonitoredQuantityStats stats;
00489 mqPtr->getStats(stats);
00490 outStream << "EventStore rank " << my_rank << ": fragments processed = "
00491 << stats.fullSampleCount << " at " << stats.fullSampleRate
00492 << " fragments/sec, average incomplete event count = "
00493 << stats.fullValueAverage << " duration = "
00494 << stats.fullDuration << " sec" << std::endl
00495 << " minimum incomplete event count = "
00496 << stats.fullValueMin << ", maximum incomplete event count = "
00497 << stats.fullValueMax << std::endl;
00498 bool foundTheStart = false;
00499 for (int idx = 0; idx < (int)stats.recentBinnedDurations.size(); ++idx)
00500 {
00501 if (stats.recentBinnedDurations[idx] > 0.0)
00502 {
00503 foundTheStart = true;
00504 }
00505 if (foundTheStart && stats.recentBinnedSampleCounts[idx] > 0.0)
00506 {
00507 outStream << " " << std::fixed << std::setprecision(3)
00508 << stats.recentBinnedEndTimes[idx]
00509 << ": " << stats.recentBinnedSampleCounts[idx]
00510 << " fragments at "
00511 << (stats.recentBinnedSampleCounts[idx] /
00512 stats.recentBinnedDurations[idx])
00513 << " fragments/sec, average incomplete event count = "
00514 << (stats.recentBinnedValueSums[idx] /
00515 stats.recentBinnedSampleCounts[idx])
00516 << ", bin size = "
00517 << stats.recentBinnedDurations[idx]
00518 << " sec" << std::endl;
00519 }
00520 }
00521 outStream << "Incomplete count now = " << events_.size() << std::endl;
00522 outStream.close();
00523 }
00524 }
00525
00526 void
00527 EventStore::setup_requests_(std::string request_address)
00528 {
00529 if (send_requests_)
00530 {
00531 request_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
00532 if (!request_socket_)
00533 {
00534 TLOG_ERROR("EventStore") << "I failed to create the socket for sending Data Requests!" << TLOG_ENDL;
00535 exit(1);
00536 }
00537 int sts = ResolveHost(request_address.c_str(), request_port_, request_addr_);
00538 if (sts == -1)
00539 {
00540 TLOG_ERROR("EventStore") << "Unable to resolve Data Request address" << TLOG_ENDL;
00541 exit(1);
00542 }
00543
00544 if (multicast_out_addr_ != "localhost") {
00545 struct in_addr addr;
00546 int sts = ResolveHost(multicast_out_addr_.c_str(), addr);
00547 if (sts == -1)
00548 {
00549 TLOG_ERROR("EventStore") << "Unable to resolve multicast interface address" << TLOG_ENDL;
00550 exit(1);
00551 }
00552
00553 int yes = 1;
00554 if (setsockopt(request_socket_, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) < 0)
00555 {
00556 TLOG_ERROR("EventStore") << "Unable to enable port reuse on request socket" << TLOG_ENDL;
00557 exit(1);
00558 }
00559 if (setsockopt(request_socket_, IPPROTO_IP, IP_MULTICAST_IF, &addr, sizeof(addr)) == -1)
00560 {
00561 TLOG_ERROR("EventStore") << "Cannot set outgoing interface." << TLOG_ENDL;
00562 exit(1);
00563 }
00564 }
00565 int yes = 1;
00566 if (setsockopt(request_socket_, SOL_SOCKET, SO_BROADCAST, (void*)&yes, sizeof(int)) == -1)
00567 {
00568 TLOG_ERROR("EventStore") << "Cannot set request socket to broadcast." << TLOG_ENDL;
00569 exit(1);
00570 }
00571 }
00572 }
00573
00574 void
00575 EventStore::setup_tokens_()
00576 {
00577 if (send_routing_tokens_)
00578 {
00579 TLOG_DEBUG("EventStore") << "Creating Routing Token sending socket" << TLOG_ENDL;
00580 token_socket_ = TCPConnect(token_address_.c_str(), token_port_);
00581 if (!token_socket_)
00582 {
00583 TLOG_ERROR("EventStore") << "I failed to create the socket for sending Routing Tokens!" << TLOG_ENDL;
00584 exit(1);
00585 }
00586 }
00587 }
00588
00589 void EventStore::do_send_request_()
00590 {
00591 std::this_thread::sleep_for(std::chrono::microseconds(request_delay_));
00592
00593 detail::RequestMessage message;
00594 {
00595 std::lock_guard<std::mutex> lk(request_mutex_);
00596 for (auto& req : active_requests_)
00597 {
00598 message.addRequest(req.first, req.second);
00599 }
00600 }
00601 message.header()->mode = request_mode_;
00602 char str[INET_ADDRSTRLEN];
00603 inet_ntop(AF_INET, &(request_addr_.sin_addr), str, INET_ADDRSTRLEN);
00604 TLOG_DEBUG("EventStore") << "Sending request for " << std::to_string(message.size()) << " events to multicast group " << str << TLOG_ENDL;
00605 if (sendto(request_socket_, message.header(), sizeof(detail::RequestHeader), 0, (struct sockaddr *)&request_addr_, sizeof(request_addr_)) < 0)
00606 {
00607 TLOG_ERROR("EventStore") << "Error sending request message header" << TLOG_ENDL;
00608 }
00609 if (sendto(request_socket_, message.buffer(), sizeof(detail::RequestPacket) * message.size(), 0, (struct sockaddr *)&request_addr_, sizeof(request_addr_)) < 0)
00610 {
00611 TLOG_ERROR("EventStore") << "Error sending request message data" << TLOG_ENDL;
00612 }
00613 }
00614
00615 void EventStore::send_routing_token_(int nSlots)
00616 {
00617 TLOG_DEBUG("EventStore") << "send_routing_token_ called, send_routing_tokens_=" << std::boolalpha << send_routing_tokens_ << TLOG_ENDL;
00618 if (!send_routing_tokens_) return;
00619 if (token_socket_ == -1) setup_tokens_();
00620 detail::RoutingToken token;
00621 token.header = TOKEN_MAGIC;
00622 token.rank = my_rank;
00623 token.new_slots_free = nSlots;
00624
00625 TLOG_DEBUG("EventStore") << "Sending RoutingToken to " << token_address_ << ":" << token_port_ << TLOG_ENDL;
00626 size_t sts = 0;
00627 while (sts < sizeof(detail::RoutingToken)) {
00628 auto res = send(token_socket_, reinterpret_cast<uint8_t*>(&token) + sts, sizeof(detail::RoutingToken) - sts, 0);
00629 if (res == -1) {
00630 usleep(1000);
00631 continue;
00632 }
00633 sts += res;
00634 }
00635 TLOG_DEBUG("EventStore") << "Done sending RoutingToken to " << token_address_ << ":" << token_port_ << TLOG_ENDL;
00636 }
00637
00638 void
00639 EventStore::send_request_()
00640 {
00641 std::thread request([=] { do_send_request_(); });
00642 request.detach();
00643 }
00644
00645 void
00646 EventStore::sendMetrics()
00647 {
00648 if (metricMan)
00649 {
00650 metricMan->sendMetric("Incomplete Event Count", events_.size(),
00651 "events", 1);
00652 }
00653 if (incomplete_event_report_interval_ms_ > 0 && events_.size())
00654 {
00655 if (std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - last_incomplete_event_report_time_).count() < incomplete_event_report_interval_ms_) return;
00656 last_incomplete_event_report_time_ = std::chrono::steady_clock::now();
00657 std::ostringstream oss;
00658 oss << "Incomplete Events (" << num_fragments_per_event_ << "): ";
00659 for (auto& ev : events_)
00660 {
00661 oss << ev.first << " (" << ev.second->numFragments() << "), ";
00662 }
00663 TLOG_DEBUG("EventStore") << oss.str() << TLOG_ENDL;
00664 }
00665 }
00666 }