00001 #include "artdaq/Application/CommandableFragmentGenerator.hh"
00002
00003 #include <boost/exception/all.hpp>
00004 #include <boost/throw_exception.hpp>
00005
00006 #include <limits>
00007 #include <iterator>
00008
00009 #include "canvas/Utilities/Exception.h"
00010 #include "cetlib/exception.h"
00011 #include "fhiclcpp/ParameterSet.h"
00012 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh"
00013 #include "artdaq-core/Data/Fragment.hh"
00014 #include "artdaq-core/Data/ContainerFragmentLoader.hh"
00015
00016 #include <fstream>
00017 #include <iomanip>
00018 #include <iterator>
00019 #include <iostream>
00020 #include <iomanip>
00021 #include <sys/poll.h>
00022 #include <artdaq/DAQdata/TCPConnect.hh>
00023
00024 artdaq::CommandableFragmentGenerator::CommandableFragmentGenerator()
00025 : mutex_()
00026 , request_port_(3001)
00027 , request_addr_("227.128.12.26")
00028 , requests_()
00029 , windowOffset_(0)
00030 , windowWidth_(0)
00031 , staleTimeout_(Fragment::InvalidTimestamp)
00032 , maxFragmentCount_(std::numeric_limits<size_t>::max())
00033 , uniqueWindows_(true)
00034 , useDataThread_(false)
00035 , dataBufferDepthFragments_(0)
00036 , dataBufferDepthBytes_(0)
00037 , maxDataBufferDepthFragments_(1000)
00038 , maxDataBufferDepthBytes_(1000)
00039 , useMonitoringThread_(false)
00040 , monitoringInterval_(0)
00041 , lastMonitoringCall_(std::chrono::steady_clock::now())
00042 , isHardwareOK_(true)
00043 , dataBuffer_()
00044 , newDataBuffer_()
00045 , run_number_(-1)
00046 , subrun_number_(-1)
00047 , timeout_(std::numeric_limits<uint64_t>::max())
00048 , timestamp_(std::numeric_limits<uint64_t>::max())
00049 , should_stop_(false)
00050 , exception_(false)
00051 , latest_exception_report_("none")
00052 , ev_counter_(1)
00053 , board_id_(-1)
00054 , instance_name_for_metrics_("FragmentGenerator")
00055 , sleep_on_stop_us_(0) {}
00056
00057 artdaq::CommandableFragmentGenerator::CommandableFragmentGenerator(const fhicl::ParameterSet& ps)
00058 : mutex_()
00059 , request_port_(ps.get<int>("request_port", 3001))
00060 , request_addr_(ps.get<std::string>("request_address", "227.128.12.26"))
00061 , requests_()
00062 , windowOffset_(ps.get<Fragment::timestamp_t>("request_window_offset", 0))
00063 , windowWidth_(ps.get<Fragment::timestamp_t>("request_window_width", 0))
00064 , staleTimeout_(ps.get<Fragment::timestamp_t>("stale_request_timeout", 0xFFFFFFFF))
00065 , uniqueWindows_(ps.get<bool>("request_windows_are_unique", true))
00066 , useDataThread_(ps.get<bool>("separate_data_thread", false))
00067 , dataBufferDepthFragments_(0)
00068 , dataBufferDepthBytes_(0)
00069 , maxDataBufferDepthFragments_(ps.get<int>("data_buffer_depth_fragments", 1000))
00070 , maxDataBufferDepthBytes_(ps.get<size_t>("data_buffer_depth_mb", 1000) * 1024 * 1024)
00071 , useMonitoringThread_(ps.get<bool>("separate_monitoring_thread", false))
00072 , monitoringInterval_(ps.get<int64_t>("hardware_poll_interval_us", 0))
00073 , lastMonitoringCall_(std::chrono::steady_clock::now())
00074 , isHardwareOK_(true)
00075 , dataBuffer_()
00076 , newDataBuffer_()
00077 , run_number_(-1)
00078 , subrun_number_(-1)
00079 , timeout_(std::numeric_limits<uint64_t>::max())
00080 , timestamp_(std::numeric_limits<uint64_t>::max())
00081 , should_stop_(false)
00082 , exception_(false)
00083 , latest_exception_report_("none")
00084 , ev_counter_(1)
00085 , board_id_(-1)
00086 , sleep_on_stop_us_(0)
00087 {
00088 board_id_ = ps.get<int>("board_id");
00089 instance_name_for_metrics_ = "BoardReader." + boost::lexical_cast<std::string>(board_id_);
00090
00091 fragment_ids_ = ps.get<std::vector<artdaq::Fragment::fragment_id_t>>("fragment_ids", std::vector<artdaq::Fragment::fragment_id_t>());
00092
00093 TRACE(24, "artdaq::CommandableFragmentGenerator::CommandableFragmentGenerator(ps)");
00094 int fragment_id = ps.get<int>("fragment_id", -99);
00095
00096 if (fragment_id != -99)
00097 {
00098 if (fragment_ids_.size() != 0)
00099 {
00100 latest_exception_report_ = "Error in CommandableFragmentGenerator: can't both define \"fragment_id\" and \"fragment_ids\" in FHiCL document";
00101 throw cet::exception(latest_exception_report_);
00102 }
00103 else
00104 {
00105 fragment_ids_.emplace_back(fragment_id);
00106 }
00107 }
00108
00109 sleep_on_stop_us_ = ps.get<int>("sleep_on_stop_us", 0);
00110
00111 dataBuffer_.emplace_back(FragmentPtr(new Fragment()));
00112 (*dataBuffer_.begin())->setSystemType(Fragment::EmptyFragmentType);
00113
00114 std::string modeString = ps.get<std::string>("request_mode", "ignored");
00115 if (modeString == "single" || modeString == "Single")
00116 {
00117 TRACE(3, "CommandableFragmentGenerator: RequestMode set to SINGLE");
00118 mode_ = RequestMode::Single;
00119 }
00120 else if (modeString.find("buffer") != std::string::npos || modeString.find("Buffer") != std::string::npos)
00121 {
00122 TRACE(3, "CommandableFragmentGenerator: RequestMode set to BUFFER");
00123 mode_ = RequestMode::Buffer;
00124 }
00125 else if (modeString == "window" || modeString == "Window")
00126 {
00127 TRACE(3, "CommandableFragmentGenerator: RequestMode set to WINDOW");
00128 mode_ = RequestMode::Window;
00129 }
00130 else if (modeString.find("ignore") != std::string::npos || modeString.find("Ignore") != std::string::npos)
00131 {
00132 TRACE(3, "CommandableFragmentGenerator: RequestMode set to IGNORE");
00133 mode_ = RequestMode::Ignored;
00134 }
00135 TLOG_DEBUG("CommandableFragmentGenerator") << "Request mode is " << printMode_() << TLOG_ENDL;
00136
00137 if (mode_ != RequestMode::Ignored)
00138 {
00139 if (!useDataThread_)
00140 {
00141 latest_exception_report_ = "Error in CommandableFragmentGenerator: use_data_thread must be true when request_mode is not \"Ignored\"!";
00142 throw cet::exception(latest_exception_report_);
00143 }
00144 setupRequestListener();
00145 }
00146 }
00147
00148 void artdaq::CommandableFragmentGenerator::setupRequestListener()
00149 {
00150 request_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
00151 if (!request_socket_)
00152 {
00153 throw art::Exception(art::errors::Configuration) << "CommandableFragmentGenerator: Error creating socket for receiving data requests!" << std::endl;
00154 exit(1);
00155 }
00156
00157 struct sockaddr_in si_me_request;
00158
00159 int yes = 1;
00160 if (setsockopt(request_socket_, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) < 0)
00161 {
00162 throw art::Exception(art::errors::Configuration) <<
00163 "RequestedFragmentGenrator: Unable to enable port reuse on request socket" << std::endl;
00164 exit(1);
00165 }
00166 memset(&si_me_request, 0, sizeof(si_me_request));
00167 si_me_request.sin_family = AF_INET;
00168 si_me_request.sin_port = htons(request_port_);
00169 si_me_request.sin_addr.s_addr = htonl(INADDR_ANY);
00170 if (bind(request_socket_, (struct sockaddr *)&si_me_request, sizeof(si_me_request)) == -1)
00171 {
00172 throw art::Exception(art::errors::Configuration) <<
00173 "CommandableFragmentGenerator: Cannot bind request socket to port " << request_port_ << std::endl;
00174 exit(1);
00175 }
00176
00177 struct ip_mreq mreq;
00178 int sts = ResolveHost(request_addr_.c_str(), mreq.imr_multiaddr);
00179 if(sts == -1)
00180 {
00181 throw art::Exception(art::errors::Configuration) << "Unable to resolve multicast request address" << std::endl;
00182 exit(1);
00183 }
00184 mreq.imr_interface.s_addr = htonl(INADDR_ANY);
00185 if (setsockopt(request_socket_, IPPROTO_IP, IP_ADD_MEMBERSHIP, &mreq, sizeof(mreq)) < 0)
00186 {
00187 throw art::Exception(art::errors::Configuration) <<
00188 "CommandableFragmentGenerator: Unable to join multicast group" << std::endl;
00189 exit(1);
00190 }
00191 }
00192
00193 artdaq::CommandableFragmentGenerator::~CommandableFragmentGenerator()
00194 {
00195 TLOG_DEBUG("CommandableFragmentGenerator") << "Joining dataThread" << TLOG_ENDL;
00196 if (dataThread_.joinable()) dataThread_.join();
00197 TLOG_DEBUG("CommandableFragmentGenerator") << "Joining monitoringThread" << TLOG_ENDL;
00198 if (monitoringThread_.joinable()) monitoringThread_.join();
00199 TLOG_DEBUG("CommandableFragmentGenerator") << "Joining requestThread" << TLOG_ENDL;
00200 if (requestThread_.joinable()) requestThread_.join();
00201 }
00202
00203 bool artdaq::CommandableFragmentGenerator::getNext(FragmentPtrs& output)
00204 {
00205 bool result = true;
00206
00207 if (check_stop()) usleep(sleep_on_stop_us_);
00208 if (exception()) return false;
00209
00210 if (!useMonitoringThread_ && monitoringInterval_ > 0)
00211 {
00212 TRACE(4, "CFG: Collecting Monitoring Data");
00213 auto now = std::chrono::steady_clock::now();
00214 if (std::chrono::duration_cast<std::chrono::microseconds>(now - lastMonitoringCall_).count() >= monitoringInterval_)
00215 {
00216 isHardwareOK_ = checkHWStatus_();
00217 lastMonitoringCall_ = now;
00218 }
00219 }
00220
00221 try
00222 {
00223 std::lock_guard<std::mutex> lk(mutex_);
00224 if (useDataThread_)
00225 {
00226 TRACE(4, "CFG: Calling applyRequests");
00227 result = applyRequests(output);
00228 TRACE(4, "CFG: Done with applyRequests");
00229 }
00230 else
00231 {
00232 TRACE(4, "CFG: Calling getNext_ %zu", ev_counter() );
00233 result = getNext_(output);
00234 TRACE(4, "CFG: Done with getNext_ %zu", ev_counter() );
00235 }
00236 }
00237 catch (const cet::exception& e)
00238 {
00239 latest_exception_report_ = "cet::exception caught in getNext(): ";
00240 latest_exception_report_.append(e.what());
00241 TLOG_ERROR("getNext") << "cet::exception caught: " << e << TLOG_ENDL;
00242 set_exception(true);
00243 return false;
00244 }
00245 catch (const boost::exception& e)
00246 {
00247 latest_exception_report_ = "boost::exception caught in getNext(): ";
00248 latest_exception_report_.append(boost::diagnostic_information(e));
00249 TLOG_ERROR("getNext") << "boost::exception caught: " << boost::diagnostic_information(e) << TLOG_ENDL;
00250 set_exception(true);
00251 return false;
00252 }
00253 catch (const std::exception& e)
00254 {
00255 latest_exception_report_ = "std::exception caught in getNext(): ";
00256 latest_exception_report_.append(e.what());
00257 TLOG_ERROR("getNext") << "std::exception caught: " << e.what() << TLOG_ENDL;
00258 set_exception(true);
00259 return false;
00260 }
00261 catch (...)
00262 {
00263 latest_exception_report_ = "Unknown exception caught in getNext().";
00264 TLOG_ERROR("getNext") << "unknown exception caught" << TLOG_ENDL;
00265 set_exception(true);
00266 return false;
00267 }
00268
00269 if (!result)
00270 {
00271 TRACE(4, "CFG:getNext Stopped");
00272 TLOG_DEBUG("getNext") << "stopped " << TLOG_ENDL;
00273 }
00274
00275 return result;
00276 }
00277
00278 bool artdaq::CommandableFragmentGenerator::check_stop()
00279 {
00280 TRACE(4, "CFG::check_stop: should_stop=%i, useDataThread_=%i, requests_.size()=%zu", should_stop(), useDataThread_, requests_.size());
00281 if (!should_stop()) return false;
00282 if (!useDataThread_ || mode_ == RequestMode::Ignored) return true;
00283
00284 return requests_.size() == 0;
00285 }
00286
00287 int artdaq::CommandableFragmentGenerator::fragment_id() const
00288 {
00289 if (fragment_ids_.size() != 1)
00290 {
00291 throw cet::exception("Error in CommandableFragmentGenerator: can't call fragment_id() unless member fragment_ids_ vector is length 1");
00292 }
00293 else
00294 {
00295 return fragment_ids_[0];
00296 }
00297 }
00298
00299 size_t artdaq::CommandableFragmentGenerator::ev_counter_inc(size_t step, bool force)
00300 {
00301 if (force || mode_ == RequestMode::Ignored)
00302 {
00303 return ev_counter_.fetch_add(step);
00304 }
00305 return ev_counter_.load();
00306 }
00307
00308 void artdaq::CommandableFragmentGenerator::StartCmd(int run, uint64_t timeout, uint64_t timestamp)
00309 {
00310 if (run < 0) throw cet::exception("CommandableFragmentGenerator") << "negative run number";
00311
00312 timeout_ = timeout;
00313 timestamp_ = timestamp;
00314 ev_counter_.store(1);
00315 should_stop_.store(false);
00316 exception_.store(false);
00317 run_number_ = run;
00318 subrun_number_ = 1;
00319 latest_exception_report_ = "none";
00320 dataBuffer_.clear();
00321 requests_.clear();
00322
00323 start();
00324
00325 std::unique_lock<std::mutex> lk(mutex_);
00326 if (useDataThread_) startDataThread();
00327 if (useMonitoringThread_) startMonitoringThread();
00328 if (mode_ != RequestMode::Ignored) startRequestReceiverThread();
00329 }
00330
00331 void artdaq::CommandableFragmentGenerator::StopCmd(uint64_t timeout, uint64_t timestamp)
00332 {
00333 TLOG_DEBUG("CommandableFragmentGenerator") << "Stop Command received." << TLOG_ENDL;
00334 TRACE(4, "CFG: Stop Command Received");
00335
00336 timeout_ = timeout;
00337 timestamp_ = timestamp;
00338
00339 stopNoMutex();
00340 should_stop_.store(true);
00341 std::unique_lock<std::mutex> lk(mutex_);
00342
00343 stop();
00344 }
00345
00346 void artdaq::CommandableFragmentGenerator::PauseCmd(uint64_t timeout, uint64_t timestamp)
00347 {
00348 timeout_ = timeout;
00349 timestamp_ = timestamp;
00350
00351 pauseNoMutex();
00352 should_stop_.store(true);
00353 std::unique_lock<std::mutex> lk(mutex_);
00354
00355 pause();
00356 }
00357
00358 void artdaq::CommandableFragmentGenerator::ResumeCmd(uint64_t timeout, uint64_t timestamp)
00359 {
00360 timeout_ = timeout;
00361 timestamp_ = timestamp;
00362
00363 subrun_number_ += 1;
00364 should_stop_ = false;
00365
00366 dataBuffer_.clear();
00367 requests_.clear();
00368
00369
00370 resume();
00371
00372 std::unique_lock<std::mutex> lk(mutex_);
00373 if (useDataThread_) startDataThread();
00374 if (useMonitoringThread_) startMonitoringThread();
00375 if (mode_ != RequestMode::Ignored) startRequestReceiverThread();
00376 }
00377
00378 std::string artdaq::CommandableFragmentGenerator::ReportCmd(std::string const& which)
00379 {
00380 std::lock_guard<std::mutex> lk(mutex_);
00381
00382
00383
00384
00385
00386
00387 std::string childReport = reportSpecific(which);
00388 if (childReport.length() > 0) { return childReport; }
00389
00390
00391 if (which == "latest_exception")
00392 {
00393 return latest_exception_report_;
00394 }
00395
00396
00397 childReport = report();
00398 if (childReport.length() > 0) { return childReport; }
00399
00400
00401 std::string tmpString = "The \"" + which + "\" command is not ";
00402 tmpString.append("currently supported by the ");
00403 tmpString.append(metricsReportingInstanceName());
00404 tmpString.append(" fragment generator.");
00405 return tmpString;
00406 }
00407
00408
00409 void artdaq::CommandableFragmentGenerator::pauseNoMutex()
00410 {
00411 #pragma message "Using default implementation of CommandableFragmentGenerator::pauseNoMutex()"
00412 }
00413
00414 void artdaq::CommandableFragmentGenerator::pause()
00415 {
00416 #pragma message "Using default implementation of CommandableFragmentGenerator::pause()"
00417 }
00418
00419 void artdaq::CommandableFragmentGenerator::resume()
00420 {
00421 #pragma message "Using default implementation of CommandableFragmentGenerator::resume()"
00422 }
00423
00424 std::string artdaq::CommandableFragmentGenerator::report()
00425 {
00426 #pragma message "Using default implementation of CommandableFragmentGenerator::report()"
00427 return "";
00428 }
00429
00430 std::string artdaq::CommandableFragmentGenerator::reportSpecific(std::string const&)
00431 {
00432 #pragma message "Using default implementation of CommandableFragmentGenerator::reportSpecific(std::string)"
00433 return "";
00434 }
00435
00436 bool artdaq::CommandableFragmentGenerator::checkHWStatus_()
00437 {
00438 #pragma message "Using default implementation of CommandableFragmentGenerator::checkHWStatus_()"
00439 return true;
00440 }
00441
00442 void artdaq::CommandableFragmentGenerator::startDataThread()
00443 {
00444 if (dataThread_.joinable()) dataThread_.join();
00445 TLOG_INFO("CommandableFragmentGenerator") << "Starting Data Receiver Thread" << TLOG_ENDL;
00446 dataThread_ = std::thread(&CommandableFragmentGenerator::getDataLoop, this);
00447 }
00448
00449 void artdaq::CommandableFragmentGenerator::startMonitoringThread()
00450 {
00451 if (monitoringThread_.joinable()) monitoringThread_.join();
00452 TLOG_INFO("CommandableFragmentGenerator") << "Starting Hardware Monitoring Thread" << TLOG_ENDL;
00453 monitoringThread_ = std::thread(&CommandableFragmentGenerator::getMonitoringDataLoop, this);
00454 }
00455
00456 void artdaq::CommandableFragmentGenerator::startRequestReceiverThread()
00457 {
00458 if (requestThread_.joinable()) requestThread_.join();
00459 TLOG_INFO("CommandableFragmentGenerator") << "Starting Request Reception Thread" << TLOG_ENDL;
00460 requestThread_ = std::thread(&CommandableFragmentGenerator::receiveRequestsLoop, this);
00461 }
00462
00463 std::string artdaq::CommandableFragmentGenerator::printMode_()
00464 {
00465 switch (mode_)
00466 {
00467 case RequestMode::Single:
00468 return "Single";
00469 case RequestMode::Buffer:
00470 return "Buffer";
00471 case RequestMode::Window:
00472 return "Window";
00473 case RequestMode::Ignored:
00474 return "Ignored";
00475 }
00476
00477 return "ERROR";
00478 }
00479
00480 void artdaq::CommandableFragmentGenerator::getDataLoop()
00481 {
00482 while (true)
00483 {
00484 if (should_stop() || !isHardwareOK_)
00485 {
00486 TLOG_DEBUG("CommandableFragmentGenerator") << "getDataLoop: should_stop is " << std::boolalpha << should_stop() << ", and isHardwareOK is " << isHardwareOK_ << TLOG_ENDL;
00487 return;
00488 }
00489
00490 TRACE(4, "CommandableFragmentGenerator::getDataLoop: calling getNext_");
00491 bool data = getNext_(newDataBuffer_);
00492
00493 auto startwait = std::chrono::steady_clock::now();
00494 bool first = true;
00495 auto lastwaittime = 0;
00496 while (dataBufferIsTooLarge())
00497 {
00498 if (should_stop())
00499 {
00500 TLOG_DEBUG("CommandaleFragmentGenerator") << "Run ended while waiting for buffer to shrink!" << TLOG_ENDL;
00501 std::unique_lock<std::mutex> lock(dataBufferMutex_);
00502 getDataBufferStats();
00503 dataCondition_.notify_all();
00504 return;
00505 }
00506 auto waittime = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - startwait).count();
00507
00508 if (first || (waittime != lastwaittime && waittime % 1000 == 0))
00509 {
00510 TLOG_WARNING("CommandableFragmentGenerator") << "Bad Omen: Data Buffer has exceeded its size limits. Check the connection between the BoardReader and the EventBuilders! (seq_id=" << ev_counter() << ")" << TLOG_ENDL;
00511 first = false;
00512 }
00513 if (waittime % 5 && waittime != lastwaittime)
00514 {
00515 TRACE(4, "CFG::getDataLoop: Data Retreival paused for %lu ms waiting for data buffer to drain", waittime);
00516 }
00517 lastwaittime = waittime;
00518 usleep(1000);
00519 }
00520
00521 if (data)
00522 {
00523 std::unique_lock<std::mutex> lock(dataBufferMutex_);
00524 switch (mode_)
00525 {
00526 case RequestMode::Single:
00527
00528 while (newDataBuffer_.size() >= fragment_ids_.size())
00529 {
00530 dataBuffer_.clear();
00531 auto it = newDataBuffer_.begin();
00532 std::advance(it, fragment_ids_.size());
00533 dataBuffer_.splice(dataBuffer_.end(), newDataBuffer_, newDataBuffer_.begin(), it);
00534 }
00535 break;
00536 case RequestMode::Buffer:
00537 case RequestMode::Ignored:
00538 case RequestMode::Window:
00539 default:
00540
00541 dataBuffer_.splice(dataBuffer_.end(), newDataBuffer_);
00542 break;
00543 }
00544 getDataBufferStats();
00545 }
00546
00547 {
00548 std::unique_lock<std::mutex> lock(dataBufferMutex_);
00549 if (dataBuffer_.size() > 0)
00550 {
00551 dataCondition_.notify_all();
00552 }
00553 }
00554 }
00555 }
00556
00557 bool artdaq::CommandableFragmentGenerator::dataBufferIsTooLarge()
00558 {
00559 return (maxDataBufferDepthFragments_ > 0 && dataBufferDepthFragments_ >= maxDataBufferDepthFragments_) || (maxDataBufferDepthBytes_ > 0 && dataBufferDepthBytes_ >= maxDataBufferDepthBytes_);
00560 }
00561
00562 void artdaq::CommandableFragmentGenerator::getDataBufferStats()
00563 {
00565 dataBufferDepthFragments_ = dataBuffer_.size();
00566 size_t acc = 0;
00567 for (auto i = dataBuffer_.begin(); i != dataBuffer_.end(); ++i)
00568 {
00569 acc += (*i)->sizeBytes();
00570 }
00571 dataBufferDepthBytes_ = acc;
00572
00573 if (metricMan)
00574 {
00575 metricMan->sendMetric("Buffer Depth Fragments", dataBufferDepthFragments_.load(), "fragments", 1);
00576 metricMan->sendMetric("Buffer Depth Bytes", dataBufferDepthBytes_.load(), "bytes", 1);
00577 }
00578 TRACE(4, "CFG::getDataBufferStats: frags=%i/%i, sz=%zd/%zd", dataBufferDepthFragments_.load(), maxDataBufferDepthFragments_, dataBufferDepthBytes_.load(), maxDataBufferDepthBytes_);
00579 }
00580
00581 void artdaq::CommandableFragmentGenerator::checkDataBuffer()
00582 {
00583 std::unique_lock<std::mutex> lock(dataBufferMutex_);
00584 dataCondition_.wait_for(lock, std::chrono::milliseconds(10));
00585 if (dataBufferDepthFragments_ > 0)
00586 {
00587 if ((mode_ == RequestMode::Buffer || mode_ == RequestMode::Window))
00588 {
00589
00590 while (dataBufferIsTooLarge())
00591 {
00592 dataBuffer_.erase(dataBuffer_.begin());
00593 getDataBufferStats();
00594 }
00595 if (dataBuffer_.size() > 0)
00596 {
00597 Fragment::timestamp_t last = dataBuffer_.back()->timestamp();
00598 Fragment::timestamp_t min = last > staleTimeout_ ? last - staleTimeout_ : 0;
00599 for (auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
00600 {
00601 if ((*it)->timestamp() < min)
00602 {
00603 it = dataBuffer_.erase(it);
00604 }
00605 else
00606 {
00607 ++it;
00608 }
00609 }
00610 getDataBufferStats();
00611 }
00612 }
00613 else if (mode_ == RequestMode::Single && dataBuffer_.size() > fragment_ids_.size())
00614 {
00615
00616 while (dataBuffer_.size() > fragment_ids_.size())
00617 {
00618 dataBuffer_.erase(dataBuffer_.begin());
00619 }
00620 }
00621 }
00622 }
00623
00624 void artdaq::CommandableFragmentGenerator::getMonitoringDataLoop()
00625 {
00626 while (true)
00627 {
00628 if (should_stop() || monitoringInterval_ <= 0)
00629 {
00630 TLOG_DEBUG("CommandableFragmentGenerator") << "getMonitoringDataLoop: should_stop() is " << std::boolalpha << should_stop()
00631 << " and monitoringInterval is " << monitoringInterval_ << ", returning" << TLOG_ENDL;
00632 return;
00633 }
00634 TRACE(4, "CFG::getMonitoringDataLoop Determining whether to call checkHWStatus_");
00635
00636 auto now = std::chrono::steady_clock::now();
00637 if (std::chrono::duration_cast<std::chrono::microseconds>(now - lastMonitoringCall_).count() >= monitoringInterval_)
00638 {
00639 isHardwareOK_ = checkHWStatus_();
00640 lastMonitoringCall_ = now;
00641 }
00642 usleep(monitoringInterval_ / 10);
00643 }
00644 }
00645
00646 void artdaq::CommandableFragmentGenerator::receiveRequestsLoop()
00647 {
00648 while (true)
00649 {
00650 if (should_stop() || !isHardwareOK_)
00651 {
00652 TLOG_DEBUG("CommandableFragmentGenerator") << "receiveRequestsLoop: should_stop is " << std::boolalpha << should_stop() << ", and isHardwareOK is " << isHardwareOK_ << TLOG_ENDL;
00653 return;
00654 }
00655
00656
00657 if (mode_ == RequestMode::Ignored) return;
00658 TRACE(4, "CFG::receiveRequestsLoop: Polling Request socket for new requests");
00659
00660 int ms_to_wait = 1000;
00661 struct pollfd ufds[1];
00662 ufds[0].fd = request_socket_;
00663 ufds[0].events = POLLIN | POLLPRI;
00664 int rv = poll(ufds, 1, ms_to_wait);
00665 if (rv > 0)
00666 {
00667 if (ufds[0].revents == POLLIN || ufds[0].revents == POLLPRI)
00668 {
00669 TRACE(4, "CFG: Recieved packet on Request channel");
00670 detail::RequestHeader hdr_buffer;
00671 recv(request_socket_, &hdr_buffer, sizeof(hdr_buffer), 0);
00672 TRACE(4, "CFG: Request header word: 0x%x", (int)hdr_buffer.header);
00673 if (hdr_buffer.isValid())
00674 {
00675 std::vector<detail::RequestPacket> pkt_buffer(hdr_buffer.packet_count);
00676 recv(request_socket_, &pkt_buffer[0], sizeof(detail::RequestPacket) * hdr_buffer.packet_count, 0);
00677 bool anyNew = false;
00678 for (auto& buffer : pkt_buffer)
00679 {
00680 if (!buffer.isValid()) continue;
00681 if (requests_.count(buffer.sequence_id) && requests_[buffer.sequence_id] != buffer.timestamp)
00682 {
00683 TLOG_ERROR("CommandableFragmentGenerator") << "Received conflicting request for SeqID "
00684 << std::to_string(buffer.sequence_id) << "!"
00685 << " Old ts=" << std::to_string(requests_[buffer.sequence_id])
00686 << ", new ts=" << std::to_string(buffer.timestamp) << ". Keeping OLD!" << TLOG_ENDL;
00687 }
00688 else if (!requests_.count(buffer.sequence_id))
00689 {
00690 int delta = buffer.sequence_id - ev_counter();
00691 TRACE(4, "CFG: Recieved request for sequence ID %llu and timestamp %lu (delta: %d)", (unsigned long long)buffer.sequence_id, (unsigned long)buffer.timestamp, delta);
00692 if (delta < 0)
00693 {
00694 TRACE(4, "CFG: Already serviced this request! Ignoring...");
00695 }
00696 else
00697 {
00698 std::unique_lock<std::mutex> tlk(request_mutex_);
00699 requests_[buffer.sequence_id] = buffer.timestamp;
00700 anyNew = true;
00701 }
00702 }
00703 }
00704 if (anyNew)
00705 {
00706 std::unique_lock<std::mutex> lock(request_mutex_);
00707 requestCondition_.notify_all();
00708 }
00709 }
00710 }
00711 }
00712 }
00713 }
00714
00715 bool artdaq::CommandableFragmentGenerator::applyRequests(artdaq::FragmentPtrs& frags)
00716 {
00717 if (check_stop())
00718 {
00719 return false;
00720 }
00721
00722 if (mode_ == RequestMode::Ignored)
00723 {
00724 while (dataBufferDepthFragments_ <= 0)
00725 {
00726 if (check_stop()) return false;
00727 std::unique_lock<std::mutex> lock(dataBufferMutex_);
00728 dataCondition_.wait_for(lock, std::chrono::milliseconds(10), [this]() { return dataBufferDepthFragments_ > 0; });
00729 }
00730 }
00731 else
00732 {
00733 while (requests_.size() <= 0)
00734 {
00735 if (check_stop()) return false;
00736
00737 checkDataBuffer();
00738
00739 std::unique_lock<std::mutex> lock(request_mutex_);
00740 requestCondition_.wait_for(lock, std::chrono::milliseconds(10), [this]() { return requests_.size() > 0; });
00741 }
00742 }
00743
00744 {
00745 std::unique_lock<std::mutex> dlk(dataBufferMutex_);
00746
00747 if (mode_ == RequestMode::Ignored)
00748 {
00749
00750 TRACE(4, "CFG: Mode is Ignored; Copying data to output");
00751 std::move(dataBuffer_.begin(), dataBuffer_.end(), std::inserter(frags, frags.end()));
00752 dataBuffer_.clear();
00753 }
00754 else if (mode_ == RequestMode::Single)
00755 {
00756
00757 sendEmptyFragments(frags);
00758
00759 if (dataBuffer_.size() > 0)
00760 {
00761 TRACE(4, "CFG: Mode is Single; Sending copy of last event");
00762 for (auto& fragptr : dataBuffer_)
00763 {
00764
00765 auto frag = fragptr.get();
00766 auto newfrag = std::unique_ptr<artdaq::Fragment>(new Fragment(ev_counter(), frag->fragmentID()));
00767 newfrag->resize(frag->size() - detail::RawFragmentHeader::num_words());
00768 memcpy(newfrag->headerAddress(), frag->headerAddress(), frag->sizeBytes());
00769 newfrag->setTimestamp(requests_[ev_counter()]);
00770 newfrag->setSequenceID(ev_counter());
00771 frags.push_back(std::move(newfrag));
00772 }
00773 }
00774 else
00775 {
00776 sendEmptyFragment(frags, ev_counter(), "No data for");
00777 }
00778 requests_.clear();
00779 ev_counter_inc(1, true);
00780 }
00781 else if (mode_ == RequestMode::Buffer || mode_ == RequestMode::Window)
00782 {
00783 if (mode_ == RequestMode::Buffer)
00784 {
00785
00786 sendEmptyFragments(frags);
00787 }
00788 for (auto req = requests_.begin(); req != requests_.end();)
00789 {
00790 auto ts = req->second;
00791 if (req->first < ev_counter())
00792 {
00793 req = requests_.erase(req);
00794 continue;
00795 }
00796 if (req->first > ev_counter())
00797 {
00798 ++req;
00799 continue;
00800 }
00801 TRACE(5, "CFG: ApplyRequestS: Checking that data exists for request window %llu (Buffered mode will always succeed)",(unsigned long long)req->first);
00802 Fragment::timestamp_t min = ts > windowOffset_ ? ts - windowOffset_ : 0;
00803 Fragment::timestamp_t max = min + windowWidth_;
00804 TRACE(5, "CFG::ApplyRequests: min is %lu and max is %lu and last point in buffer is %lu (sz=%zu)",
00805 (unsigned long)min,(unsigned long)max, (unsigned long)(dataBuffer_.size() > 0 ? dataBuffer_.back()->timestamp() : 0), dataBuffer_.size());
00806 bool windowClosed = mode_ != RequestMode::Window || (dataBuffer_.size() > 0 && dataBuffer_.back()->timestamp() >= max);
00807 if (windowClosed || should_stop())
00808 {
00809 TLOG_DEBUG("CommandableFragmentGenerator") << "Creating ContainerFragment for Buffered or Window-requested Fragments" << TLOG_ENDL;
00810 frags.emplace_back(new artdaq::Fragment(ev_counter(), fragment_id()));
00811 frags.back()->setTimestamp(ts);
00812 ContainerFragmentLoader cfl(*frags.back());
00813
00814 if (mode_ == RequestMode::Window && should_stop() && !windowClosed) cfl.set_missing_data(true);
00815 if (mode_ == RequestMode::Window && dataBuffer_.size() > 0 && dataBuffer_.front()->timestamp() < min)
00816 {
00817 TLOG_DEBUG("CommandableFragmentGenerator") << "Request Window covers data that is either before data collection began or has fallen off the end of the buffer" << TLOG_ENDL;
00818 cfl.set_missing_data(true);
00819 }
00820
00821
00822
00823 for (auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
00824 {
00825 if (mode_ == RequestMode::Window)
00826 {
00827 Fragment::timestamp_t fragT = (*it)->timestamp();
00828 if (fragT < min || fragT > max)
00829 {
00830 ++it;
00831 continue;
00832 }
00833 }
00834
00835 TRACE(5,"CFG::ApplyRequests: Adding Fragment with timestamp %llu to Container", (unsigned long long)(*it)->timestamp());
00836 cfl.addFragment(*it);
00837
00838 if (mode_ == RequestMode::Buffer || (mode_ == RequestMode::Window && uniqueWindows_))
00839 {
00840 it = dataBuffer_.erase(it);
00841 }
00842 else
00843 {
00844 ++it;
00845 }
00846 }
00847 req = requests_.erase(req);
00848 ev_counter_inc(1, true);
00849 }
00850 else
00851 {
00852
00853 break;
00854 }
00855 }
00856 }
00857 getDataBufferStats();
00858 }
00859
00860 if (frags.size() > 0)
00861 TRACE(4, "CFG: Finished Processing Event %lu for fragment_id %i.", ev_counter() + 1, fragment_id());
00862 return true;
00863 }
00864
00865 bool artdaq::CommandableFragmentGenerator::sendEmptyFragment(artdaq::FragmentPtrs& frags, size_t seqId, std::string desc)
00866 {
00867 TLOG_WARNING("CommandableFragmentGenerator") << desc << " request " << seqId << ", sending empty fragment" << TLOG_ENDL;
00868 auto frag = new Fragment();
00869 frag->setSequenceID(seqId);
00870 frag->setSystemType(Fragment::EmptyFragmentType);
00871 frags.emplace_back(FragmentPtr(frag));
00872 return true;
00873 }
00874
00875 void artdaq::CommandableFragmentGenerator::sendEmptyFragments(artdaq::FragmentPtrs& frags)
00876 {
00877 auto sequence_id = Fragment::InvalidSequenceID;
00878 auto timestamp = Fragment::InvalidTimestamp;
00879
00880 for (auto it = requests_.begin(); it != requests_.end();)
00881 {
00882 auto seq = it->first;
00883 auto ts = it->second;
00884
00885
00886 if (++it == requests_.end())
00887 {
00888 sequence_id = seq;
00889 timestamp = ts;
00890 break;
00891 }
00892 if (seq < ev_counter()) continue;
00893
00894
00895 sendEmptyFragment(frags, ev_counter(), "Missed request for");
00896 ev_counter_inc(1, true);
00897 }
00898 requests_.clear();
00899
00900 if (sequence_id < ev_counter()) return;
00901 requests_[sequence_id] = timestamp;
00902 }