artdaq  v2_03_02
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Pages
CommandableFragmentGenerator.cc
1 #include "artdaq/Application/CommandableFragmentGenerator.hh"
2 
3 #include <boost/exception/all.hpp>
4 #include <boost/throw_exception.hpp>
5 
6 #include <limits>
7 #include <iterator>
8 
9 #include "canvas/Utilities/Exception.h"
10 #include "cetlib/exception.h"
11 #include "fhiclcpp/ParameterSet.h"
12 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh"
13 #include "artdaq-core/Data/Fragment.hh"
14 #include "artdaq-core/Data/ContainerFragmentLoader.hh"
15 #include "artdaq-core/Utilities/ExceptionHandler.hh"
16 
17 #include <fstream>
18 #include <iomanip>
19 #include <iterator>
20 #include <iostream>
21 #include <iomanip>
22 #include <sys/poll.h>
24 
26  : mutex_()
27  , request_port_(3001)
28  , request_addr_("227.128.12.26")
29  , requests_()
30  , request_stop_requested_(false)
31 , end_of_run_timeout_ms_(1000)
32  , windowOffset_(0)
33  , windowWidth_(0)
34  , staleTimeout_(Fragment::InvalidTimestamp)
35  , maxFragmentCount_(std::numeric_limits<size_t>::max())
36  , uniqueWindows_(true)
37  , useDataThread_(false)
38  , data_thread_running_(false)
39  , dataBufferDepthFragments_(0)
40  , dataBufferDepthBytes_(0)
41  , maxDataBufferDepthFragments_(1000)
42  , maxDataBufferDepthBytes_(1000)
43  , useMonitoringThread_(false)
44  , monitoringInterval_(0)
45  , lastMonitoringCall_(std::chrono::steady_clock::now())
46  , isHardwareOK_(true)
47  , dataBuffer_()
48  , newDataBuffer_()
49  , run_number_(-1)
50  , subrun_number_(-1)
51  , timeout_(std::numeric_limits<uint64_t>::max())
52  , timestamp_(std::numeric_limits<uint64_t>::max())
53  , should_stop_(false)
54  , exception_(false)
55  , latest_exception_report_("none")
56  , ev_counter_(1)
57  , board_id_(-1)
58  , instance_name_for_metrics_("FragmentGenerator")
59  , sleep_on_stop_us_(0) {}
60 
62  : mutex_()
63  , request_port_(ps.get<int>("request_port", 3001))
64  , request_addr_(ps.get<std::string>("request_address", "227.128.12.26"))
65  , requests_()
66  , request_stop_requested_(false)
67  , end_of_run_timeout_ms_(ps.get<size_t>("end_of_run_quiet_timeout_ms", 1000))
68  , windowOffset_(ps.get<Fragment::timestamp_t>("request_window_offset", 0))
69  , windowWidth_(ps.get<Fragment::timestamp_t>("request_window_width", 0))
70  , staleTimeout_(ps.get<Fragment::timestamp_t>("stale_request_timeout", 0xFFFFFFFF))
71  , uniqueWindows_(ps.get<bool>("request_windows_are_unique", true))
72  , useDataThread_(ps.get<bool>("separate_data_thread", false))
73  , data_thread_running_(false)
74  , dataBufferDepthFragments_(0)
75  , dataBufferDepthBytes_(0)
76  , maxDataBufferDepthFragments_(ps.get<int>("data_buffer_depth_fragments", 1000))
77  , maxDataBufferDepthBytes_(ps.get<size_t>("data_buffer_depth_mb", 1000) * 1024 * 1024)
78  , useMonitoringThread_(ps.get<bool>("separate_monitoring_thread", false))
79  , monitoringInterval_(ps.get<int64_t>("hardware_poll_interval_us", 0))
80  , lastMonitoringCall_(std::chrono::steady_clock::now())
81  , isHardwareOK_(true)
82  , dataBuffer_()
83  , newDataBuffer_()
84  , run_number_(-1)
85  , subrun_number_(-1)
86  , timeout_(std::numeric_limits<uint64_t>::max())
87  , timestamp_(std::numeric_limits<uint64_t>::max())
88  , should_stop_(false)
89  , exception_(false)
90  , latest_exception_report_("none")
91  , ev_counter_(1)
92  , board_id_(-1)
93  , sleep_on_stop_us_(0)
94 {
95  board_id_ = ps.get<int>("board_id");
96  instance_name_for_metrics_ = "BoardReader." + boost::lexical_cast<std::string>(board_id_);
97 
98  fragment_ids_ = ps.get<std::vector<artdaq::Fragment::fragment_id_t>>("fragment_ids", std::vector<artdaq::Fragment::fragment_id_t>());
99 
100  TRACE(24, "artdaq::CommandableFragmentGenerator::CommandableFragmentGenerator(ps)");
101  int fragment_id = ps.get<int>("fragment_id", -99);
102 
103  if (fragment_id != -99)
104  {
105  if (fragment_ids_.size() != 0)
106  {
107  latest_exception_report_ = "Error in CommandableFragmentGenerator: can't both define \"fragment_id\" and \"fragment_ids\" in FHiCL document";
108  throw cet::exception(latest_exception_report_);
109  }
110  else
111  {
112  fragment_ids_.emplace_back(fragment_id);
113  }
114  }
115 
116  sleep_on_stop_us_ = ps.get<int>("sleep_on_stop_us", 0);
117 
118  dataBuffer_.emplace_back(FragmentPtr(new Fragment()));
119  (*dataBuffer_.begin())->setSystemType(Fragment::EmptyFragmentType);
120 
121  std::string modeString = ps.get<std::string>("request_mode", "ignored");
122  if (modeString == "single" || modeString == "Single")
123  {
124  TRACE(3, "CommandableFragmentGenerator: RequestMode set to SINGLE");
125  mode_ = RequestMode::Single;
126  }
127  else if (modeString.find("buffer") != std::string::npos || modeString.find("Buffer") != std::string::npos)
128  {
129  TRACE(3, "CommandableFragmentGenerator: RequestMode set to BUFFER");
130  mode_ = RequestMode::Buffer;
131  }
132  else if (modeString == "window" || modeString == "Window")
133  {
134  TRACE(3, "CommandableFragmentGenerator: RequestMode set to WINDOW");
135  mode_ = RequestMode::Window;
136  }
137  else if (modeString.find("ignore") != std::string::npos || modeString.find("Ignore") != std::string::npos)
138  {
139  TRACE(3, "CommandableFragmentGenerator: RequestMode set to IGNORE");
140  mode_ = RequestMode::Ignored;
141  }
142  TLOG_DEBUG("CommandableFragmentGenerator") << "Request mode is " << printMode_() << TLOG_ENDL;
143 
144  if (mode_ != RequestMode::Ignored)
145  {
146  if (!useDataThread_)
147  {
148  latest_exception_report_ = "Error in CommandableFragmentGenerator: use_data_thread must be true when request_mode is not \"Ignored\"!";
149  throw cet::exception(latest_exception_report_);
150  }
152  }
153 }
154 
156 {
157  request_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
158  if (!request_socket_)
159  {
160  throw art::Exception(art::errors::Configuration) << "CommandableFragmentGenerator: Error creating socket for receiving data requests!" << std::endl;
161  exit(1);
162  }
163 
164  struct sockaddr_in si_me_request;
165 
166  int yes = 1;
167  if (setsockopt(request_socket_, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) < 0)
168  {
169  throw art::Exception(art::errors::Configuration) <<
170  "RequestedFragmentGenrator: Unable to enable port reuse on request socket" << std::endl;
171  exit(1);
172  }
173  memset(&si_me_request, 0, sizeof(si_me_request));
174  si_me_request.sin_family = AF_INET;
175  si_me_request.sin_port = htons(request_port_);
176  si_me_request.sin_addr.s_addr = htonl(INADDR_ANY);
177  if (bind(request_socket_, (struct sockaddr *)&si_me_request, sizeof(si_me_request)) == -1)
178  {
179  throw art::Exception(art::errors::Configuration) <<
180  "CommandableFragmentGenerator: Cannot bind request socket to port " << request_port_ << std::endl;
181  exit(1);
182  }
183 
184  struct ip_mreq mreq;
185  int sts = ResolveHost(request_addr_.c_str(), mreq.imr_multiaddr);
186  if (sts == -1)
187  {
188  throw art::Exception(art::errors::Configuration) << "Unable to resolve multicast request address" << std::endl;
189  exit(1);
190  }
191  mreq.imr_interface.s_addr = htonl(INADDR_ANY);
192  if (setsockopt(request_socket_, IPPROTO_IP, IP_ADD_MEMBERSHIP, &mreq, sizeof(mreq)) < 0)
193  {
194  throw art::Exception(art::errors::Configuration) <<
195  "CommandableFragmentGenerator: Unable to join multicast group" << std::endl;
196  exit(1);
197  }
198 }
199 
201 {
202  TLOG_DEBUG("CommandableFragmentGenerator") << "Joining dataThread" << TLOG_ENDL;
203  if (dataThread_.joinable()) dataThread_.join();
204  TLOG_DEBUG("CommandableFragmentGenerator") << "Joining monitoringThread" << TLOG_ENDL;
205  if (monitoringThread_.joinable()) monitoringThread_.join();
206  TLOG_DEBUG("CommandableFragmentGenerator") << "Joining requestThread" << TLOG_ENDL;
207  if (requestThread_.joinable()) requestThread_.join();
208 }
209 
211 {
212  bool result = true;
213 
214  if (check_stop()) usleep(sleep_on_stop_us_);
215  if (exception()) return false;
216 
217  if (!useMonitoringThread_ && monitoringInterval_ > 0)
218  {
219  TRACE(4, "CFG: Collecting Monitoring Data");
220  auto now = std::chrono::steady_clock::now();
221  if (std::chrono::duration_cast<std::chrono::microseconds>(now - lastMonitoringCall_).count() >= monitoringInterval_)
222  {
223  isHardwareOK_ = checkHWStatus_();
224  lastMonitoringCall_ = now;
225  }
226  }
227 
228  try
229  {
230  std::lock_guard<std::mutex> lk(mutex_);
231  if (useDataThread_)
232  {
233  TRACE(4, "CFG: Calling applyRequests");
234  result = applyRequests(output);
235  TRACE(4, "CFG: Done with applyRequests");
236 
237  if (exception()) {
238  throw cet::exception("CommandableFragmentGenerator") << "Exception found in BoardReader with board ID " << board_id() << "; BoardReader will now return error status when queried";
239  }
240  }
241  else
242  {
243  TRACE(4, "CFG: Calling getNext_ %zu", ev_counter());
244  try {
245  result = getNext_(output);
246  } catch (...) {
247  throw;
248  }
249  TRACE(4, "CFG: Done with getNext_ %zu", ev_counter());
250  }
251  }
252  catch (const cet::exception& e)
253  {
254  latest_exception_report_ = "cet::exception caught in getNext(): ";
255  latest_exception_report_.append(e.what());
256  TLOG_ERROR("getNext") << "cet::exception caught: " << e << TLOG_ENDL;
257  set_exception(true);
258  return false;
259  }
260  catch (const boost::exception& e)
261  {
262  latest_exception_report_ = "boost::exception caught in getNext(): ";
263  latest_exception_report_.append(boost::diagnostic_information(e));
264  TLOG_ERROR("getNext") << "boost::exception caught: " << boost::diagnostic_information(e) << TLOG_ENDL;
265  set_exception(true);
266  return false;
267  }
268  catch (const std::exception& e)
269  {
270  latest_exception_report_ = "std::exception caught in getNext(): ";
271  latest_exception_report_.append(e.what());
272  TLOG_ERROR("getNext") << "std::exception caught: " << e.what() << TLOG_ENDL;
273  set_exception(true);
274  return false;
275  }
276  catch (...)
277  {
278  latest_exception_report_ = "Unknown exception caught in getNext().";
279  TLOG_ERROR("getNext") << "unknown exception caught" << TLOG_ENDL;
280  set_exception(true);
281  return false;
282  }
283 
284  if (!result)
285  {
286  TRACE(4, "CFG:getNext Stopped");
287  TLOG_DEBUG("getNext") << "stopped " << TLOG_ENDL;
288  }
289 
290  return result;
291 }
292 
294 {
295  TRACE(4, "CFG::check_stop: should_stop=%i, useDataThread_=%i, requests_.size()=%zu, exception status =%d", should_stop(), useDataThread_, requests_.size(), int(exception()));
296  if (!should_stop()) return false;
297  if (!useDataThread_ || mode_ == RequestMode::Ignored) return true;
298  if (!request_stop_requested_) return false;
299 
300  auto dur = std::chrono::steady_clock::now() - request_stop_timeout_;
301  return std::chrono::duration_cast<std::chrono::milliseconds>(dur).count() > static_cast<int>(end_of_run_timeout_ms_);// && requests_.size() == 0;
302 }
303 
305 {
306  if (fragment_ids_.size() != 1)
307  {
308  throw cet::exception("Error in CommandableFragmentGenerator: can't call fragment_id() unless member fragment_ids_ vector is length 1");
309  }
310  else
311  {
312  return fragment_ids_[0];
313  }
314 }
315 
317 {
318  if (force || mode_ == RequestMode::Ignored)
319  {
320  return ev_counter_.fetch_add(step);
321  }
322  return ev_counter_.load();
323 } // returns the prev value
324 
325 void artdaq::CommandableFragmentGenerator::StartCmd(int run, uint64_t timeout, uint64_t timestamp)
326 {
327  if (run < 0) throw cet::exception("CommandableFragmentGenerator") << "negative run number";
328 
329  timeout_ = timeout;
330  timestamp_ = timestamp;
331  ev_counter_.store(1);
332  should_stop_.store(false);
333  exception_.store(false);
334  run_number_ = run;
335  subrun_number_ = 1;
336  latest_exception_report_ = "none";
337  dataBuffer_.clear();
338  requests_.clear();
339 
340  start();
341 
342  std::unique_lock<std::mutex> lk(mutex_);
343  if (useDataThread_) startDataThread();
344  if (useMonitoringThread_) startMonitoringThread();
345  if (mode_ != RequestMode::Ignored) startRequestReceiverThread();
346 }
347 
348 void artdaq::CommandableFragmentGenerator::StopCmd(uint64_t timeout, uint64_t timestamp)
349 {
350  TLOG_DEBUG("CommandableFragmentGenerator") << "Stop Command received." << TLOG_ENDL;
351  TRACE(4, "CFG: Stop Command Received");
352 
353  timeout_ = timeout;
354  timestamp_ = timestamp;
355 
356  stopNoMutex();
357  should_stop_.store(true);
358  std::unique_lock<std::mutex> lk(mutex_);
359  stop();
360 }
361 
362 void artdaq::CommandableFragmentGenerator::PauseCmd(uint64_t timeout, uint64_t timestamp)
363 {
364  timeout_ = timeout;
365  timestamp_ = timestamp;
366 
367  pauseNoMutex();
368  should_stop_.store(true);
369  std::unique_lock<std::mutex> lk(mutex_);
370 
371  pause();
372 }
373 
374 void artdaq::CommandableFragmentGenerator::ResumeCmd(uint64_t timeout, uint64_t timestamp)
375 {
376  timeout_ = timeout;
377  timestamp_ = timestamp;
378 
379  subrun_number_ += 1;
380  should_stop_ = false;
381 
382  dataBuffer_.clear();
383  requests_.clear();
384 
385  // no lock required: thread not started yet
386  resume();
387 
388  std::unique_lock<std::mutex> lk(mutex_);
389  if (useDataThread_) startDataThread();
390  if (useMonitoringThread_) startMonitoringThread();
391  if (mode_ != RequestMode::Ignored) startRequestReceiverThread();
392 }
393 
394 std::string artdaq::CommandableFragmentGenerator::ReportCmd(std::string const& which)
395 {
396  std::lock_guard<std::mutex> lk(mutex_);
397 
398  // 14-May-2015, KAB: please see the comments associated with the report()
399  // methods in the CommandableFragmentGenerator.hh file for more information
400  // on the use of those methods in this method.
401 
402  // check if the child class has something meaningful for this request
403  std::string childReport = reportSpecific(which);
404  if (childReport.length() > 0) { return childReport; }
405 
406  // handle the requests that we can take care of at this level
407  if (which == "latest_exception")
408  {
409  return latest_exception_report_;
410  }
411 
412  // check if the child class has provided a catch-all report function
413  childReport = report();
414  if (childReport.length() > 0) { return childReport; }
415 
416  // if we haven't been able to come up with any report so far, say so
417  std::string tmpString = "The \"" + which + "\" command is not ";
418  tmpString.append("currently supported by the ");
419  tmpString.append(metricsReportingInstanceName());
420  tmpString.append(" fragment generator.");
421  return tmpString;
422 }
423 
424 // Default implemenetations of state functions
425 void artdaq::CommandableFragmentGenerator::pauseNoMutex()
426 {
427 #pragma message "Using default implementation of CommandableFragmentGenerator::pauseNoMutex()"
428 }
429 
430 void artdaq::CommandableFragmentGenerator::pause()
431 {
432 #pragma message "Using default implementation of CommandableFragmentGenerator::pause()"
433 }
434 
435 void artdaq::CommandableFragmentGenerator::resume()
436 {
437 #pragma message "Using default implementation of CommandableFragmentGenerator::resume()"
438 }
439 
440 std::string artdaq::CommandableFragmentGenerator::report()
441 {
442 #pragma message "Using default implementation of CommandableFragmentGenerator::report()"
443  return "";
444 }
445 
446 std::string artdaq::CommandableFragmentGenerator::reportSpecific(std::string const&)
447 {
448 #pragma message "Using default implementation of CommandableFragmentGenerator::reportSpecific(std::string)"
449  return "";
450 }
451 
452 bool artdaq::CommandableFragmentGenerator::checkHWStatus_()
453 {
454 #pragma message "Using default implementation of CommandableFragmentGenerator::checkHWStatus_()"
455  return true;
456 }
457 
459 {
460  if (dataThread_.joinable()) dataThread_.join();
461  TLOG_INFO("CommandableFragmentGenerator") << "Starting Data Receiver Thread" << TLOG_ENDL;
462  dataThread_ = std::thread(&CommandableFragmentGenerator::getDataLoop, this);
463 }
464 
466 {
467  if (monitoringThread_.joinable()) monitoringThread_.join();
468  TLOG_INFO("CommandableFragmentGenerator") << "Starting Hardware Monitoring Thread" << TLOG_ENDL;
469  monitoringThread_ = std::thread(&CommandableFragmentGenerator::getMonitoringDataLoop, this);
470 }
471 
473 {
474  if (requestThread_.joinable()) requestThread_.join();
475  TLOG_INFO("CommandableFragmentGenerator") << "Starting Request Reception Thread" << TLOG_ENDL;
476  requestThread_ = std::thread(&CommandableFragmentGenerator::receiveRequestsLoop, this);
477 }
478 
480 {
481  switch (mode_)
482  {
483  case RequestMode::Single:
484  return "Single";
485  case RequestMode::Buffer:
486  return "Buffer";
487  case RequestMode::Window:
488  return "Window";
489  case RequestMode::Ignored:
490  return "Ignored";
491  }
492 
493  return "ERROR";
494 }
495 
497 {
498  data_thread_running_ = true;
499  while (true)
500  {
501  if (!isHardwareOK_)
502  {
503  TLOG_DEBUG("CommandableFragmentGenerator") << "getDataLoop: isHardwareOK is " << isHardwareOK_ << ", aborting data thread" << TLOG_ENDL;
504  data_thread_running_ = false;
505  return;
506  }
507 
508  TRACE(4, "CommandableFragmentGenerator::getDataLoop: calling getNext_");
509 
510  bool data = false;
511 
512  try {
513  data = getNext_(newDataBuffer_);
514  } catch (...) {
515  ExceptionHandler(ExceptionHandlerRethrow::no,
516  "Exception thrown by fragment generator in CommandableFragmentGenerator::getDataLoop; setting exception state to \"true\"");
517  set_exception(true);
518 
519  data_thread_running_ = false;
520  return;
521  }
522 
523  auto startwait = std::chrono::steady_clock::now();
524  auto first = true;
525  auto lastwaittime = 0;
526  while (dataBufferIsTooLarge())
527  {
528  if (should_stop())
529  {
530  TLOG_DEBUG("CommandableFragmentGenerator") << "Run ended while waiting for buffer to shrink!" << TLOG_ENDL;
531  std::unique_lock<std::mutex> lock(dataBufferMutex_);
532  getDataBufferStats();
533  dataCondition_.notify_all();
534  data_thread_running_ = false;
535  return;
536  }
537  auto waittime = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - startwait).count();
538 
539  if (first || (waittime != lastwaittime && waittime % 1000 == 0))
540  {
541  TLOG_WARNING("CommandableFragmentGenerator") << "Bad Omen: Data Buffer has exceeded its size limits. Check the connection between the BoardReader and the EventBuilders! (seq_id=" << ev_counter() << ")" << TLOG_ENDL;
542  first = false;
543  }
544  if (waittime % 5 && waittime != lastwaittime)
545  {
546  TRACE(4, "CFG::getDataLoop: Data Retreival paused for %lu ms waiting for data buffer to drain", waittime);
547  }
548  lastwaittime = waittime;
549  usleep(1000);
550  }
551 
552  if (data)
553  {
554  std::unique_lock<std::mutex> lock(dataBufferMutex_);
555  switch (mode_)
556  {
557  case RequestMode::Single:
558  // While here, if for some strange reason more than one event's worth of data is returned from getNext_...
559  while (newDataBuffer_.size() >= fragment_ids_.size())
560  {
561  dataBuffer_.clear();
562  auto it = newDataBuffer_.begin();
563  std::advance(it, fragment_ids_.size());
564  dataBuffer_.splice(dataBuffer_.end(), newDataBuffer_, newDataBuffer_.begin(), it);
565  }
566  break;
567  case RequestMode::Buffer:
568  case RequestMode::Ignored:
569  case RequestMode::Window:
570  default:
571  //dataBuffer_.reserve(dataBuffer_.size() + newDataBuffer_.size());
572  dataBuffer_.splice(dataBuffer_.end(), newDataBuffer_);
573  break;
574  }
575  getDataBufferStats();
576  }
577 
578  {
579  std::unique_lock<std::mutex> lock(dataBufferMutex_);
580  if (dataBuffer_.size() > 0)
581  {
582  dataCondition_.notify_all();
583  }
584  }
585  if (!data)
586  {
587  TLOG_INFO("CommandableFragmentGenerator") << "Data flow has stopped. Ending data collection thread" << TLOG_ENDL;
588  data_thread_running_ = false;
589  return;
590  }
591  }
592 }
593 
595 {
596  return (maxDataBufferDepthFragments_ > 0 && dataBufferDepthFragments_ >= maxDataBufferDepthFragments_) || (maxDataBufferDepthBytes_ > 0 && dataBufferDepthBytes_ >= maxDataBufferDepthBytes_);
597 }
598 
600 {
602  dataBufferDepthFragments_ = dataBuffer_.size();
603  size_t acc = 0;
604  for (auto i = dataBuffer_.begin(); i != dataBuffer_.end(); ++i)
605  {
606  acc += (*i)->sizeBytes();
607  }
608  dataBufferDepthBytes_ = acc;
609 
610  if (metricMan)
611  {
612  metricMan->sendMetric("Buffer Depth Fragments", dataBufferDepthFragments_.load(), "fragments", 1);
613  metricMan->sendMetric("Buffer Depth Bytes", dataBufferDepthBytes_.load(), "bytes", 1);
614  }
615  TRACE(4, "CFG::getDataBufferStats: frags=%i/%i, sz=%zd/%zd", dataBufferDepthFragments_.load(), maxDataBufferDepthFragments_, dataBufferDepthBytes_.load(), maxDataBufferDepthBytes_);
616 }
617 
619 {
620  std::unique_lock<std::mutex> lock(dataBufferMutex_);
621  dataCondition_.wait_for(lock, std::chrono::milliseconds(10));
622  if (dataBufferDepthFragments_ > 0)
623  {
624  if ((mode_ == RequestMode::Buffer || mode_ == RequestMode::Window))
625  {
626  // Eliminate extra fragments
627  while (dataBufferIsTooLarge())
628  {
629  dataBuffer_.erase(dataBuffer_.begin());
630  getDataBufferStats();
631  }
632  if (dataBuffer_.size() > 0)
633  {
634  Fragment::timestamp_t last = dataBuffer_.back()->timestamp();
635  Fragment::timestamp_t min = last > staleTimeout_ ? last - staleTimeout_ : 0;
636  for (auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
637  {
638  if ((*it)->timestamp() < min)
639  {
640  it = dataBuffer_.erase(it);
641  }
642  else
643  {
644  ++it;
645  }
646  }
647  getDataBufferStats();
648  }
649  }
650  else if (mode_ == RequestMode::Single && dataBuffer_.size() > fragment_ids_.size())
651  {
652  // Eliminate extra fragments
653  while (dataBuffer_.size() > fragment_ids_.size())
654  {
655  dataBuffer_.erase(dataBuffer_.begin());
656  }
657  }
658  }
659 }
660 
662 {
663  while (true)
664  {
665  if (should_stop() || monitoringInterval_ <= 0)
666  {
667  TLOG_DEBUG("CommandableFragmentGenerator") << "getMonitoringDataLoop: should_stop() is " << std::boolalpha << should_stop()
668  << " and monitoringInterval is " << monitoringInterval_ << ", returning" << TLOG_ENDL;
669  return;
670  }
671  TRACE(4, "CFG::getMonitoringDataLoop Determining whether to call checkHWStatus_");
672 
673  auto now = std::chrono::steady_clock::now();
674  if (std::chrono::duration_cast<std::chrono::microseconds>(now - lastMonitoringCall_).count() >= monitoringInterval_)
675  {
676  isHardwareOK_ = checkHWStatus_();
677  lastMonitoringCall_ = now;
678  }
679  usleep(monitoringInterval_ / 10);
680  }
681 }
682 
684 {
685  while (true)
686  {
687  if (check_stop() || !isHardwareOK_ || exception())
688  {
689  TLOG_DEBUG("CommandableFragmentGenerator") << "receiveRequestsLoop: check_stop is " << std::boolalpha << check_stop()
690  << ", isHardwareOK_ is " << isHardwareOK_ << ", and exception state is " << exception() << ", aborting request reception thread." << TLOG_ENDL;
691  return;
692  }
693 
694  // Don't listen for requests when we're going to ignore them anyway
695  if (mode_ == RequestMode::Ignored) return;
696  TRACE(4, "CFG::receiveRequestsLoop: Polling Request socket for new requests");
697 
698  int ms_to_wait = 1000;
699  struct pollfd ufds[1];
700  ufds[0].fd = request_socket_;
701  ufds[0].events = POLLIN | POLLPRI;
702  int rv = poll(ufds, 1, ms_to_wait);
703  if (rv > 0)
704  {
705  if (ufds[0].revents == POLLIN || ufds[0].revents == POLLPRI)
706  {
707  TRACE(4, "CFG: Recieved packet on Request channel");
708  detail::RequestHeader hdr_buffer;
709  recv(request_socket_, &hdr_buffer, sizeof(hdr_buffer), 0);
710  TRACE(4, "CFG: Request header word: 0x%x", (int)hdr_buffer.header);
711  if (hdr_buffer.isValid())
712  {
713  if (hdr_buffer.mode == detail::RequestMessageMode::EndOfRun)
714  {
715  TLOG_INFO("CommandableFragmentGenerator") << "Received Request Message with the EndOfRun marker. (Re)Starting 1-second timeout for receiving all outstanding requests..." << TLOG_ENDL;
716  request_stop_timeout_ = std::chrono::steady_clock::now();
717  request_stop_requested_ = true;
718  }
719  std::vector<detail::RequestPacket> pkt_buffer(hdr_buffer.packet_count);
720  recv(request_socket_, &pkt_buffer[0], sizeof(detail::RequestPacket) * hdr_buffer.packet_count, 0);
721  bool anyNew = false;
722  for (auto& buffer : pkt_buffer)
723  {
724  if (!buffer.isValid()) continue;
725  if (requests_.count(buffer.sequence_id) && requests_[buffer.sequence_id] != buffer.timestamp)
726  {
727  TLOG_ERROR("CommandableFragmentGenerator") << "Received conflicting request for SeqID "
728  << std::to_string(buffer.sequence_id) << "!"
729  << " Old ts=" << std::to_string(requests_[buffer.sequence_id])
730  << ", new ts=" << std::to_string(buffer.timestamp) << ". Keeping OLD!" << TLOG_ENDL;
731  }
732  else if (!requests_.count(buffer.sequence_id))
733  {
734  int delta = buffer.sequence_id - ev_counter();
735  TRACE(4, "CFG: Recieved request for sequence ID %llu and timestamp %lu (delta: %d)", (unsigned long long)buffer.sequence_id, (unsigned long)buffer.timestamp, delta);
736  if (delta < 0)
737  {
738  TRACE(4, "CFG: Already serviced this request! Ignoring...");
739  }
740  else
741  {
742  std::unique_lock<std::mutex> tlk(request_mutex_);
743  requests_[buffer.sequence_id] = buffer.timestamp;
744  anyNew = true;
745  }
746  }
747  }
748  if (anyNew)
749  {
750  std::unique_lock<std::mutex> lock(request_mutex_);
751  requestCondition_.notify_all();
752  }
753  }
754  }
755  }
756  }
757 }
758 
760 {
761  if (check_stop() || exception())
762  {
763  return false;
764  }
765 
766  if (mode_ == RequestMode::Ignored)
767  {
768  while (dataBufferDepthFragments_ <= 0)
769  {
770  if (check_stop() || exception()) return false;
771  std::unique_lock<std::mutex> lock(dataBufferMutex_);
772  dataCondition_.wait_for(lock, std::chrono::milliseconds(10), [this]() { return dataBufferDepthFragments_ > 0; });
773  }
774  }
775  else
776  {
777  if ((check_stop() && requests_.size() <= 0) || exception()) return false;
778  checkDataBuffer();
779 
780  while (requests_.size() <= 0)
781  {
782  if (check_stop() || exception()) return false;
783 
784  checkDataBuffer();
785 
786  std::unique_lock<std::mutex> lock(request_mutex_);
787  requestCondition_.wait_for(lock, std::chrono::milliseconds(10), [this]() { return requests_.size() > 0; });
788  }
789  }
790 
791  {
792  std::unique_lock<std::mutex> dlk(dataBufferMutex_);
793  std::unique_lock<std::mutex> rlk(request_mutex_);
794 
795 
796  if (mode_ == RequestMode::Ignored)
797  {
798  // We just copy everything that's here into the output.
799  TRACE(4, "CFG: Mode is Ignored; Copying data to output");
800  std::move(dataBuffer_.begin(), dataBuffer_.end(), std::inserter(frags, frags.end()));
801  dataBuffer_.clear();
802  }
803  else if (mode_ == RequestMode::Single)
804  {
805  // We only care about the latest request received. Send empties for all others.
806  sendEmptyFragments(frags);
807 
808  if (dataBuffer_.size() > 0)
809  {
810  TRACE(4, "CFG: Mode is Single; Sending copy of last event");
811  for (auto& fragptr : dataBuffer_)
812  {
813  // Return the latest data point
814  auto frag = fragptr.get();
815  auto newfrag = std::unique_ptr<artdaq::Fragment>(new Fragment(ev_counter(), frag->fragmentID()));
816  newfrag->resize(frag->size() - detail::RawFragmentHeader::num_words());
817  memcpy(newfrag->headerAddress(), frag->headerAddress(), frag->sizeBytes());
818  newfrag->setTimestamp(requests_[ev_counter()]);
819  newfrag->setSequenceID(ev_counter());
820  frags.push_back(std::move(newfrag));
821  }
822  }
823  else
824  {
825  sendEmptyFragment(frags, ev_counter(), "No data for");
826  }
827  requests_.clear();
828  ev_counter_inc(1, true);
829  }
830  else if (mode_ == RequestMode::Buffer || mode_ == RequestMode::Window)
831  {
832  if (mode_ == RequestMode::Buffer)
833  {
834  // We only care about the latest request received. Send empties for all others.
835  sendEmptyFragments(frags);
836  }
837  for (auto req = requests_.begin(); req != requests_.end();)
838  {
839  auto ts = req->second;
840  if (req->first < ev_counter())
841  {
842  req = requests_.erase(req);
843  continue;
844  }
845  while (req->first > ev_counter() && request_stop_requested_ && std::chrono::duration_cast<std::chrono::seconds>(std::chrono::steady_clock::now() - request_stop_timeout_).count() > 1)
846  {
847  sendEmptyFragment(frags,ev_counter(), "Missing request for");
848  ev_counter_inc(1, true);
849  }
850  if (req->first > ev_counter())
851  {
852  ++req;
853  continue; // Will loop through all requests, means we're in Window mode and missing the correct one
854  }
855  TRACE(5, "CFG: ApplyRequestS: Checking that data exists for request window %llu (Buffered mode will always succeed)", (unsigned long long)req->first);
856  Fragment::timestamp_t min = ts > windowOffset_ ? ts - windowOffset_ : 0;
857  Fragment::timestamp_t max = min + windowWidth_;
858  TRACE(5, "CFG::ApplyRequests: min is %lu and max is %lu and last point in buffer is %lu (sz=%zu)",
859  (unsigned long)min, (unsigned long)max, (unsigned long)(dataBuffer_.size() > 0 ? dataBuffer_.back()->timestamp() : 0), dataBuffer_.size());
860  bool windowClosed = mode_ != RequestMode::Window || (dataBuffer_.size() > 0 && dataBuffer_.back()->timestamp() >= max);
861  if (windowClosed || !data_thread_running_)
862  {
863  TLOG_DEBUG("CommandableFragmentGenerator") << "Creating ContainerFragment for Buffered or Window-requested Fragments" << TLOG_ENDL;
864  frags.emplace_back(new artdaq::Fragment(ev_counter(), fragment_id()));
865  frags.back()->setTimestamp(ts);
866  ContainerFragmentLoader cfl(*frags.back());
867 
868  if (mode_ == RequestMode::Window && !data_thread_running_ && !windowClosed) cfl.set_missing_data(true);
869  if (mode_ == RequestMode::Window && dataBuffer_.size() > 0 && dataBuffer_.front()->timestamp() > min)
870  {
871  TLOG_DEBUG("CommandableFragmentGenerator") << "Request Window covers data that is either before data collection began or has fallen off the end of the buffer" << TLOG_ENDL;
872  cfl.set_missing_data(true);
873  }
874 
875  // Buffer mode TFGs should simply copy out the whole dataBuffer_ into a ContainerFragment
876  // Window mode TFGs must do a little bit more work to decide which fragments to send for a given request
877  for (auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
878  {
879  if (mode_ == RequestMode::Window)
880  {
881  Fragment::timestamp_t fragT = (*it)->timestamp();
882  if (fragT < min || fragT > max)
883  {
884  ++it;
885  continue;
886  }
887  }
888 
889  TRACE(5, "CFG::ApplyRequests: Adding Fragment with timestamp %llu to Container", (unsigned long long)(*it)->timestamp());
890  cfl.addFragment(*it);
891 
892  if (mode_ == RequestMode::Buffer || (mode_ == RequestMode::Window && uniqueWindows_))
893  {
894  it = dataBuffer_.erase(it);
895  }
896  else
897  {
898  ++it;
899  }
900  }
901  req = requests_.erase(req);
902  ev_counter_inc(1, true);
903  }
904  else
905  {
906  // Wait for the window to be closed for the current event
907  break;
908  }
909  }
910  }
911  getDataBufferStats();
912  }
913 
914  if (frags.size() > 0)
915  TRACE(4, "CFG: Finished Processing Event %lu for fragment_id %i.", ev_counter() + 1, fragment_id());
916  return true;
917 }
918 
919 bool artdaq::CommandableFragmentGenerator::sendEmptyFragment(artdaq::FragmentPtrs& frags, size_t seqId, std::string desc)
920 {
921  TLOG_WARNING("CommandableFragmentGenerator") << desc << " request " << seqId << ", sending empty fragment" << TLOG_ENDL;
922  auto frag = new Fragment();
923  frag->setSequenceID(seqId);
924  frag->setSystemType(Fragment::EmptyFragmentType);
925  frags.emplace_back(FragmentPtr(frag));
926  return true;
927 }
928 
930 {
931  auto sequence_id = Fragment::InvalidSequenceID;
932  auto timestamp = Fragment::InvalidTimestamp;
933  // Map is ordered by sequence ID!
934  for (auto it = requests_.begin(); it != requests_.end();)
935  {
936  auto seq = it->first;
937  auto ts = it->second;
938 
939  // Check if this is the one "true" request
940  if (++it == requests_.end())
941  {
942  sequence_id = seq;
943  timestamp = ts;
944  break;
945  }
946  if (seq < ev_counter()) continue;
947 
948  // Otherwise, this is just one we missed, send an empty
949  sendEmptyFragment(frags, ev_counter(), "Missed request for");
950  ev_counter_inc(1, true);
951  }
952  requests_.clear();
953 
954  if (sequence_id < ev_counter()) return; // No new requests received.
955  requests_[sequence_id] = timestamp;
956 }
int fragment_id() const
Get the current Fragment ID, if there is only one.
int ResolveHost(char const *host_in, in_addr &addr)
Convert a string hostname to a in_addr suitable for socket communication.
Definition: TCPConnect.cc:29
virtual ~CommandableFragmentGenerator()
CommandableFragmentGenerator Destructor.
bool sendEmptyFragment(FragmentPtrs &frags, size_t sequenceId, std::string desc)
Send an EmptyFragmentType Fragment.
void getMonitoringDataLoop()
This function regularly calls checkHWStatus_(), and sets the isHardwareOK flag accordingly.
void startDataThread()
Function that launches the data thread (getDataLoop())
std::string ReportCmd(std::string const &which="")
Get a report about a user-specified run-time quantity.
RequestMessageMode mode
Communicates additional information to the Request receiver.
bool isValid() const
Check the magic bytes of the packet.
bool dataBufferIsTooLarge()
Test the configured constraints on the data buffer.
void StopCmd(uint64_t timeout, uint64_t timestamp)
Stop the CommandableFragmentGenerator.
void StartCmd(int run, uint64_t timeout, uint64_t timestamp)
Start the CommandableFragmentGenerator.
bool check_stop()
Routine used by applyRequests to make sure that all outstanding requests have been fulfilled before r...
void ResumeCmd(uint64_t timeout, uint64_t timestamp)
Resume the CommandableFragmentGenerator.
CommandableFragmentGenerator()
CommandableFragmentGenerator default constructor.
bool getNext(FragmentPtrs &output) overridefinal
getNext calls either applyRequests or getNext_ to get any data that is ready to be sent to the EventB...
size_t ev_counter_inc(size_t step=1, bool force=false)
Increment the event counter, if the current RequestMode allows it.
Header of a RequestMessage. Contains magic bytes for validation and a count of expected RequestPacket...
uint32_t packet_count
The number of RequestPackets in this Request message.
void PauseCmd(uint64_t timeout, uint64_t timestamp)
Pause the CommandableFragmentGenerator.
void getDataLoop()
When separate_data_thread is set to true, this loop repeatedly calls getNext_ and adds returned Fragm...
The RequestPacket contains information about a single data request.
void startRequestReceiverThread()
Function that launches the data request receiver thread (receiveRequestsLoop())
void startMonitoringThread()
Function that launches the monitoring thread (getMonitoringDataLoop())
void checkDataBuffer()
Perform data buffer pruning operations. If the RequestMode is Single, removes all but the latest Frag...
std::string printMode_()
Return the string representation of the current RequestMode.
void sendEmptyFragments(FragmentPtrs &frags)
This function is for Buffered and Single request modes, as they can only respond to one data request ...
void getDataBufferStats()
Calculate the size of the dataBuffer and report appropriate metrics.
bool applyRequests(FragmentPtrs &output)
See if any requests have been received, and add the corresponding data Fragment objects to the output...
void receiveRequestsLoop()
This function receives data request packets, adding new requests to the request list.
void setupRequestListener()
Opens the socket used to listen for data requests.