artdaq  v2_03_00
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Pages
CommandableFragmentGenerator.cc
1 #include "artdaq/Application/CommandableFragmentGenerator.hh"
2 
3 #include <boost/exception/all.hpp>
4 #include <boost/throw_exception.hpp>
5 
6 #include <limits>
7 #include <iterator>
8 
9 #include "canvas/Utilities/Exception.h"
10 #include "cetlib/exception.h"
11 #include "fhiclcpp/ParameterSet.h"
12 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh"
13 #include "artdaq-core/Data/Fragment.hh"
14 #include "artdaq-core/Data/ContainerFragmentLoader.hh"
15 
16 #include <fstream>
17 #include <iomanip>
18 #include <iterator>
19 #include <iostream>
20 #include <iomanip>
21 #include <sys/poll.h>
23 
25  : mutex_()
26  , request_port_(3001)
27  , request_addr_("227.128.12.26")
28  , requests_()
29  , windowOffset_(0)
30  , windowWidth_(0)
31  , staleTimeout_(Fragment::InvalidTimestamp)
32  , maxFragmentCount_(std::numeric_limits<size_t>::max())
33  , uniqueWindows_(true)
34  , useDataThread_(false)
35  , dataBufferDepthFragments_(0)
36  , dataBufferDepthBytes_(0)
37  , maxDataBufferDepthFragments_(1000)
38  , maxDataBufferDepthBytes_(1000)
39  , useMonitoringThread_(false)
40  , monitoringInterval_(0)
41  , lastMonitoringCall_(std::chrono::steady_clock::now())
42  , isHardwareOK_(true)
43  , dataBuffer_()
44  , newDataBuffer_()
45  , run_number_(-1)
46  , subrun_number_(-1)
47  , timeout_(std::numeric_limits<uint64_t>::max())
48  , timestamp_(std::numeric_limits<uint64_t>::max())
49  , should_stop_(false)
50  , exception_(false)
51  , latest_exception_report_("none")
52  , ev_counter_(1)
53  , board_id_(-1)
54  , instance_name_for_metrics_("FragmentGenerator")
55  , sleep_on_stop_us_(0) {}
56 
58  : mutex_()
59  , request_port_(ps.get<int>("request_port", 3001))
60  , request_addr_(ps.get<std::string>("request_address", "227.128.12.26"))
61  , requests_()
62  , windowOffset_(ps.get<Fragment::timestamp_t>("request_window_offset", 0))
63  , windowWidth_(ps.get<Fragment::timestamp_t>("request_window_width", 0))
64  , staleTimeout_(ps.get<Fragment::timestamp_t>("stale_request_timeout", 0xFFFFFFFF))
65  , uniqueWindows_(ps.get<bool>("request_windows_are_unique", true))
66  , useDataThread_(ps.get<bool>("separate_data_thread", false))
67  , dataBufferDepthFragments_(0)
68  , dataBufferDepthBytes_(0)
69  , maxDataBufferDepthFragments_(ps.get<int>("data_buffer_depth_fragments", 1000))
70  , maxDataBufferDepthBytes_(ps.get<size_t>("data_buffer_depth_mb", 1000) * 1024 * 1024)
71  , useMonitoringThread_(ps.get<bool>("separate_monitoring_thread", false))
72  , monitoringInterval_(ps.get<int64_t>("hardware_poll_interval_us", 0))
73  , lastMonitoringCall_(std::chrono::steady_clock::now())
74  , isHardwareOK_(true)
75  , dataBuffer_()
76  , newDataBuffer_()
77  , run_number_(-1)
78  , subrun_number_(-1)
79  , timeout_(std::numeric_limits<uint64_t>::max())
80  , timestamp_(std::numeric_limits<uint64_t>::max())
81  , should_stop_(false)
82  , exception_(false)
83  , latest_exception_report_("none")
84  , ev_counter_(1)
85  , board_id_(-1)
86  , sleep_on_stop_us_(0)
87 {
88  board_id_ = ps.get<int>("board_id");
89  instance_name_for_metrics_ = "BoardReader." + boost::lexical_cast<std::string>(board_id_);
90 
91  fragment_ids_ = ps.get<std::vector<artdaq::Fragment::fragment_id_t>>("fragment_ids", std::vector<artdaq::Fragment::fragment_id_t>());
92 
93  TRACE(24, "artdaq::CommandableFragmentGenerator::CommandableFragmentGenerator(ps)");
94  int fragment_id = ps.get<int>("fragment_id", -99);
95 
96  if (fragment_id != -99)
97  {
98  if (fragment_ids_.size() != 0)
99  {
100  latest_exception_report_ = "Error in CommandableFragmentGenerator: can't both define \"fragment_id\" and \"fragment_ids\" in FHiCL document";
101  throw cet::exception(latest_exception_report_);
102  }
103  else
104  {
105  fragment_ids_.emplace_back(fragment_id);
106  }
107  }
108 
109  sleep_on_stop_us_ = ps.get<int>("sleep_on_stop_us", 0);
110 
111  dataBuffer_.emplace_back(FragmentPtr(new Fragment()));
112  (*dataBuffer_.begin())->setSystemType(Fragment::EmptyFragmentType);
113 
114  std::string modeString = ps.get<std::string>("request_mode", "ignored");
115  if (modeString == "single" || modeString == "Single")
116  {
117  TRACE(3, "CommandableFragmentGenerator: RequestMode set to SINGLE");
118  mode_ = RequestMode::Single;
119  }
120  else if (modeString.find("buffer") != std::string::npos || modeString.find("Buffer") != std::string::npos)
121  {
122  TRACE(3, "CommandableFragmentGenerator: RequestMode set to BUFFER");
123  mode_ = RequestMode::Buffer;
124  }
125  else if (modeString == "window" || modeString == "Window")
126  {
127  TRACE(3, "CommandableFragmentGenerator: RequestMode set to WINDOW");
128  mode_ = RequestMode::Window;
129  }
130  else if (modeString.find("ignore") != std::string::npos || modeString.find("Ignore") != std::string::npos)
131  {
132  TRACE(3, "CommandableFragmentGenerator: RequestMode set to IGNORE");
133  mode_ = RequestMode::Ignored;
134  }
135  TLOG_DEBUG("CommandableFragmentGenerator") << "Request mode is " << printMode_() << TLOG_ENDL;
136 
137  if (mode_ != RequestMode::Ignored)
138  {
139  if (!useDataThread_)
140  {
141  latest_exception_report_ = "Error in CommandableFragmentGenerator: use_data_thread must be true when request_mode is not \"Ignored\"!";
142  throw cet::exception(latest_exception_report_);
143  }
145  }
146 }
147 
149 {
150  request_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
151  if (!request_socket_)
152  {
153  throw art::Exception(art::errors::Configuration) << "CommandableFragmentGenerator: Error creating socket for receiving data requests!" << std::endl;
154  exit(1);
155  }
156 
157  struct sockaddr_in si_me_request;
158 
159  int yes = 1;
160  if (setsockopt(request_socket_, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) < 0)
161  {
162  throw art::Exception(art::errors::Configuration) <<
163  "RequestedFragmentGenrator: Unable to enable port reuse on request socket" << std::endl;
164  exit(1);
165  }
166  memset(&si_me_request, 0, sizeof(si_me_request));
167  si_me_request.sin_family = AF_INET;
168  si_me_request.sin_port = htons(request_port_);
169  si_me_request.sin_addr.s_addr = htonl(INADDR_ANY);
170  if (bind(request_socket_, (struct sockaddr *)&si_me_request, sizeof(si_me_request)) == -1)
171  {
172  throw art::Exception(art::errors::Configuration) <<
173  "CommandableFragmentGenerator: Cannot bind request socket to port " << request_port_ << std::endl;
174  exit(1);
175  }
176 
177  struct ip_mreq mreq;
178  int sts = ResolveHost(request_addr_.c_str(), mreq.imr_multiaddr);
179  if(sts == -1)
180  {
181  throw art::Exception(art::errors::Configuration) << "Unable to resolve multicast request address" << std::endl;
182  exit(1);
183  }
184  mreq.imr_interface.s_addr = htonl(INADDR_ANY);
185  if (setsockopt(request_socket_, IPPROTO_IP, IP_ADD_MEMBERSHIP, &mreq, sizeof(mreq)) < 0)
186  {
187  throw art::Exception(art::errors::Configuration) <<
188  "CommandableFragmentGenerator: Unable to join multicast group" << std::endl;
189  exit(1);
190  }
191 }
192 
194 {
195  TLOG_DEBUG("CommandableFragmentGenerator") << "Joining dataThread" << TLOG_ENDL;
196  if (dataThread_.joinable()) dataThread_.join();
197  TLOG_DEBUG("CommandableFragmentGenerator") << "Joining monitoringThread" << TLOG_ENDL;
198  if (monitoringThread_.joinable()) monitoringThread_.join();
199  TLOG_DEBUG("CommandableFragmentGenerator") << "Joining requestThread" << TLOG_ENDL;
200  if (requestThread_.joinable()) requestThread_.join();
201 }
202 
204 {
205  bool result = true;
206 
207  if (check_stop()) usleep(sleep_on_stop_us_);
208  if (exception()) return false;
209 
210  if (!useMonitoringThread_ && monitoringInterval_ > 0)
211  {
212  TRACE(4, "CFG: Collecting Monitoring Data");
213  auto now = std::chrono::steady_clock::now();
214  if (std::chrono::duration_cast<std::chrono::microseconds>(now - lastMonitoringCall_).count() >= monitoringInterval_)
215  {
216  isHardwareOK_ = checkHWStatus_();
217  lastMonitoringCall_ = now;
218  }
219  }
220 
221  try
222  {
223  std::lock_guard<std::mutex> lk(mutex_);
224  if (useDataThread_)
225  {
226  TRACE(4, "CFG: Calling applyRequests");
227  result = applyRequests(output);
228  TRACE(4, "CFG: Done with applyRequests");
229  }
230  else
231  {
232  TRACE(4, "CFG: Calling getNext_ %zu", ev_counter() );
233  result = getNext_(output);
234  TRACE(4, "CFG: Done with getNext_ %zu", ev_counter() );
235  }
236  }
237  catch (const cet::exception& e)
238  {
239  latest_exception_report_ = "cet::exception caught in getNext(): ";
240  latest_exception_report_.append(e.what());
241  TLOG_ERROR("getNext") << "cet::exception caught: " << e << TLOG_ENDL;
242  set_exception(true);
243  return false;
244  }
245  catch (const boost::exception& e)
246  {
247  latest_exception_report_ = "boost::exception caught in getNext(): ";
248  latest_exception_report_.append(boost::diagnostic_information(e));
249  TLOG_ERROR("getNext") << "boost::exception caught: " << boost::diagnostic_information(e) << TLOG_ENDL;
250  set_exception(true);
251  return false;
252  }
253  catch (const std::exception& e)
254  {
255  latest_exception_report_ = "std::exception caught in getNext(): ";
256  latest_exception_report_.append(e.what());
257  TLOG_ERROR("getNext") << "std::exception caught: " << e.what() << TLOG_ENDL;
258  set_exception(true);
259  return false;
260  }
261  catch (...)
262  {
263  latest_exception_report_ = "Unknown exception caught in getNext().";
264  TLOG_ERROR("getNext") << "unknown exception caught" << TLOG_ENDL;
265  set_exception(true);
266  return false;
267  }
268 
269  if (!result)
270  {
271  TRACE(4, "CFG:getNext Stopped");
272  TLOG_DEBUG("getNext") << "stopped " << TLOG_ENDL;
273  }
274 
275  return result;
276 }
277 
279 {
280  TRACE(4, "CFG::check_stop: should_stop=%i, useDataThread_=%i, requests_.size()=%zu", should_stop(), useDataThread_, requests_.size());
281  if (!should_stop()) return false;
282  if (!useDataThread_ || mode_ == RequestMode::Ignored) return true;
283 
284  return requests_.size() == 0;
285 }
286 
288 {
289  if (fragment_ids_.size() != 1)
290  {
291  throw cet::exception("Error in CommandableFragmentGenerator: can't call fragment_id() unless member fragment_ids_ vector is length 1");
292  }
293  else
294  {
295  return fragment_ids_[0];
296  }
297 }
298 
300 {
301  if (force || mode_ == RequestMode::Ignored)
302  {
303  return ev_counter_.fetch_add(step);
304  }
305  return ev_counter_.load();
306 } // returns the prev value
307 
308 void artdaq::CommandableFragmentGenerator::StartCmd(int run, uint64_t timeout, uint64_t timestamp)
309 {
310  if (run < 0) throw cet::exception("CommandableFragmentGenerator") << "negative run number";
311 
312  timeout_ = timeout;
313  timestamp_ = timestamp;
314  ev_counter_.store(1);
315  should_stop_.store(false);
316  exception_.store(false);
317  run_number_ = run;
318  subrun_number_ = 1;
319  latest_exception_report_ = "none";
320  dataBuffer_.clear();
321  requests_.clear();
322 
323  start();
324 
325  std::unique_lock<std::mutex> lk(mutex_);
326  if (useDataThread_) startDataThread();
327  if (useMonitoringThread_) startMonitoringThread();
328  if (mode_ != RequestMode::Ignored) startRequestReceiverThread();
329 }
330 
331 void artdaq::CommandableFragmentGenerator::StopCmd(uint64_t timeout, uint64_t timestamp)
332 {
333  TLOG_DEBUG("CommandableFragmentGenerator") << "Stop Command received." << TLOG_ENDL;
334  TRACE(4, "CFG: Stop Command Received");
335 
336  timeout_ = timeout;
337  timestamp_ = timestamp;
338 
339  stopNoMutex();
340  should_stop_.store(true);
341  std::unique_lock<std::mutex> lk(mutex_);
342 
343  stop();
344 }
345 
346 void artdaq::CommandableFragmentGenerator::PauseCmd(uint64_t timeout, uint64_t timestamp)
347 {
348  timeout_ = timeout;
349  timestamp_ = timestamp;
350 
351  pauseNoMutex();
352  should_stop_.store(true);
353  std::unique_lock<std::mutex> lk(mutex_);
354 
355  pause();
356 }
357 
358 void artdaq::CommandableFragmentGenerator::ResumeCmd(uint64_t timeout, uint64_t timestamp)
359 {
360  timeout_ = timeout;
361  timestamp_ = timestamp;
362 
363  subrun_number_ += 1;
364  should_stop_ = false;
365 
366  dataBuffer_.clear();
367  requests_.clear();
368 
369  // no lock required: thread not started yet
370  resume();
371 
372  std::unique_lock<std::mutex> lk(mutex_);
373  if (useDataThread_) startDataThread();
374  if (useMonitoringThread_) startMonitoringThread();
375  if (mode_ != RequestMode::Ignored) startRequestReceiverThread();
376 }
377 
378 std::string artdaq::CommandableFragmentGenerator::ReportCmd(std::string const& which)
379 {
380  std::lock_guard<std::mutex> lk(mutex_);
381 
382  // 14-May-2015, KAB: please see the comments associated with the report()
383  // methods in the CommandableFragmentGenerator.hh file for more information
384  // on the use of those methods in this method.
385 
386  // check if the child class has something meaningful for this request
387  std::string childReport = reportSpecific(which);
388  if (childReport.length() > 0) { return childReport; }
389 
390  // handle the requests that we can take care of at this level
391  if (which == "latest_exception")
392  {
393  return latest_exception_report_;
394  }
395 
396  // check if the child class has provided a catch-all report function
397  childReport = report();
398  if (childReport.length() > 0) { return childReport; }
399 
400  // if we haven't been able to come up with any report so far, say so
401  std::string tmpString = "The \"" + which + "\" command is not ";
402  tmpString.append("currently supported by the ");
403  tmpString.append(metricsReportingInstanceName());
404  tmpString.append(" fragment generator.");
405  return tmpString;
406 }
407 
408 // Default implemenetations of state functions
409 void artdaq::CommandableFragmentGenerator::pauseNoMutex()
410 {
411 #pragma message "Using default implementation of CommandableFragmentGenerator::pauseNoMutex()"
412 }
413 
414 void artdaq::CommandableFragmentGenerator::pause()
415 {
416 #pragma message "Using default implementation of CommandableFragmentGenerator::pause()"
417 }
418 
419 void artdaq::CommandableFragmentGenerator::resume()
420 {
421 #pragma message "Using default implementation of CommandableFragmentGenerator::resume()"
422 }
423 
424 std::string artdaq::CommandableFragmentGenerator::report()
425 {
426 #pragma message "Using default implementation of CommandableFragmentGenerator::report()"
427  return "";
428 }
429 
430 std::string artdaq::CommandableFragmentGenerator::reportSpecific(std::string const&)
431 {
432 #pragma message "Using default implementation of CommandableFragmentGenerator::reportSpecific(std::string)"
433  return "";
434 }
435 
436 bool artdaq::CommandableFragmentGenerator::checkHWStatus_()
437 {
438 #pragma message "Using default implementation of CommandableFragmentGenerator::checkHWStatus_()"
439  return true;
440 }
441 
443 {
444  if (dataThread_.joinable()) dataThread_.join();
445  TLOG_INFO("CommandableFragmentGenerator") << "Starting Data Receiver Thread" << TLOG_ENDL;
446  dataThread_ = std::thread(&CommandableFragmentGenerator::getDataLoop, this);
447 }
448 
450 {
451  if (monitoringThread_.joinable()) monitoringThread_.join();
452  TLOG_INFO("CommandableFragmentGenerator") << "Starting Hardware Monitoring Thread" << TLOG_ENDL;
453  monitoringThread_ = std::thread(&CommandableFragmentGenerator::getMonitoringDataLoop, this);
454 }
455 
457 {
458  if (requestThread_.joinable()) requestThread_.join();
459  TLOG_INFO("CommandableFragmentGenerator") << "Starting Request Reception Thread" << TLOG_ENDL;
460  requestThread_ = std::thread(&CommandableFragmentGenerator::receiveRequestsLoop, this);
461 }
462 
464 {
465  switch (mode_)
466  {
467  case RequestMode::Single:
468  return "Single";
469  case RequestMode::Buffer:
470  return "Buffer";
471  case RequestMode::Window:
472  return "Window";
473  case RequestMode::Ignored:
474  return "Ignored";
475  }
476 
477  return "ERROR";
478 }
479 
481 {
482  while (true)
483  {
484  if (should_stop() || !isHardwareOK_)
485  {
486  TLOG_DEBUG("CommandableFragmentGenerator") << "getDataLoop: should_stop is " << std::boolalpha << should_stop() << ", and isHardwareOK is " << isHardwareOK_ << TLOG_ENDL;
487  return;
488  }
489 
490  TRACE(4, "CommandableFragmentGenerator::getDataLoop: calling getNext_");
491  bool data = getNext_(newDataBuffer_);
492 
493  auto startwait = std::chrono::steady_clock::now();
494  bool first = true;
495  auto lastwaittime = 0;
496  while (dataBufferIsTooLarge())
497  {
498  if (should_stop())
499  {
500  TLOG_DEBUG("CommandaleFragmentGenerator") << "Run ended while waiting for buffer to shrink!" << TLOG_ENDL;
501  std::unique_lock<std::mutex> lock(dataBufferMutex_);
502  getDataBufferStats();
503  dataCondition_.notify_all();
504  return;
505  }
506  auto waittime = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - startwait).count();
507 
508  if (first || (waittime != lastwaittime && waittime % 1000 == 0))
509  {
510  TLOG_WARNING("CommandableFragmentGenerator") << "Bad Omen: Data Buffer has exceeded its size limits. Check the connection between the BoardReader and the EventBuilders! (seq_id=" << ev_counter() << ")" << TLOG_ENDL;
511  first = false;
512  }
513  if (waittime % 5 && waittime != lastwaittime)
514  {
515  TRACE(4, "CFG::getDataLoop: Data Retreival paused for %lu ms waiting for data buffer to drain", waittime);
516  }
517  lastwaittime = waittime;
518  usleep(1000);
519  }
520 
521  if (data)
522  {
523  std::unique_lock<std::mutex> lock(dataBufferMutex_);
524  switch (mode_)
525  {
526  case RequestMode::Single:
527  // While here, if for some strange reason more than one event's worth of data is returned from getNext_...
528  while (newDataBuffer_.size() >= fragment_ids_.size())
529  {
530  dataBuffer_.clear();
531  auto it = newDataBuffer_.begin();
532  std::advance(it, fragment_ids_.size());
533  dataBuffer_.splice(dataBuffer_.end(), newDataBuffer_, newDataBuffer_.begin(), it);
534  }
535  break;
536  case RequestMode::Buffer:
537  case RequestMode::Ignored:
538  case RequestMode::Window:
539  default:
540  //dataBuffer_.reserve(dataBuffer_.size() + newDataBuffer_.size());
541  dataBuffer_.splice(dataBuffer_.end(), newDataBuffer_);
542  break;
543  }
544  getDataBufferStats();
545  }
546 
547  {
548  std::unique_lock<std::mutex> lock(dataBufferMutex_);
549  if (dataBuffer_.size() > 0)
550  {
551  dataCondition_.notify_all();
552  }
553  }
554  }
555 }
556 
558 {
559  return (maxDataBufferDepthFragments_ > 0 && dataBufferDepthFragments_ >= maxDataBufferDepthFragments_) || (maxDataBufferDepthBytes_ > 0 && dataBufferDepthBytes_ >= maxDataBufferDepthBytes_);
560 }
561 
563 {
565  dataBufferDepthFragments_ = dataBuffer_.size();
566  size_t acc = 0;
567  for (auto i = dataBuffer_.begin(); i != dataBuffer_.end(); ++i)
568  {
569  acc += (*i)->sizeBytes();
570  }
571  dataBufferDepthBytes_ = acc;
572 
573  if (metricMan)
574  {
575  metricMan->sendMetric("Buffer Depth Fragments", dataBufferDepthFragments_.load(), "fragments", 1);
576  metricMan->sendMetric("Buffer Depth Bytes", dataBufferDepthBytes_.load(), "bytes", 1);
577  }
578  TRACE(4, "CFG::getDataBufferStats: frags=%i/%i, sz=%zd/%zd", dataBufferDepthFragments_.load(), maxDataBufferDepthFragments_, dataBufferDepthBytes_.load(), maxDataBufferDepthBytes_);
579 }
580 
582 {
583  std::unique_lock<std::mutex> lock(dataBufferMutex_);
584  dataCondition_.wait_for(lock, std::chrono::milliseconds(10));
585  if (dataBufferDepthFragments_ > 0)
586  {
587  if ((mode_ == RequestMode::Buffer || mode_ == RequestMode::Window))
588  {
589  // Eliminate extra fragments
590  while (dataBufferIsTooLarge())
591  {
592  dataBuffer_.erase(dataBuffer_.begin());
593  getDataBufferStats();
594  }
595  if (dataBuffer_.size() > 0)
596  {
597  Fragment::timestamp_t last = dataBuffer_.back()->timestamp();
598  Fragment::timestamp_t min = last > staleTimeout_ ? last - staleTimeout_ : 0;
599  for (auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
600  {
601  if ((*it)->timestamp() < min)
602  {
603  it = dataBuffer_.erase(it);
604  }
605  else
606  {
607  ++it;
608  }
609  }
610  getDataBufferStats();
611  }
612  }
613  else if (mode_ == RequestMode::Single && dataBuffer_.size() > fragment_ids_.size())
614  {
615  // Eliminate extra fragments
616  while (dataBuffer_.size() > fragment_ids_.size())
617  {
618  dataBuffer_.erase(dataBuffer_.begin());
619  }
620  }
621  }
622 }
623 
625 {
626  while (true)
627  {
628  if (should_stop() || monitoringInterval_ <= 0)
629  {
630  TLOG_DEBUG("CommandableFragmentGenerator") << "getMonitoringDataLoop: should_stop() is " << std::boolalpha << should_stop()
631  << " and monitoringInterval is " << monitoringInterval_ << ", returning" << TLOG_ENDL;
632  return;
633  }
634  TRACE(4, "CFG::getMonitoringDataLoop Determining whether to call checkHWStatus_");
635 
636  auto now = std::chrono::steady_clock::now();
637  if (std::chrono::duration_cast<std::chrono::microseconds>(now - lastMonitoringCall_).count() >= monitoringInterval_)
638  {
639  isHardwareOK_ = checkHWStatus_();
640  lastMonitoringCall_ = now;
641  }
642  usleep(monitoringInterval_ / 10);
643  }
644 }
645 
647 {
648  while (true)
649  {
650  if (should_stop() || !isHardwareOK_)
651  {
652  TLOG_DEBUG("CommandableFragmentGenerator") << "receiveRequestsLoop: should_stop is " << std::boolalpha << should_stop() << ", and isHardwareOK is " << isHardwareOK_ << TLOG_ENDL;
653  return;
654  }
655 
656  // Don't listen for requests when we're going to ignore them anyway
657  if (mode_ == RequestMode::Ignored) return;
658  TRACE(4, "CFG::receiveRequestsLoop: Polling Request socket for new requests");
659 
660  int ms_to_wait = 1000;
661  struct pollfd ufds[1];
662  ufds[0].fd = request_socket_;
663  ufds[0].events = POLLIN | POLLPRI;
664  int rv = poll(ufds, 1, ms_to_wait);
665  if (rv > 0)
666  {
667  if (ufds[0].revents == POLLIN || ufds[0].revents == POLLPRI)
668  {
669  TRACE(4, "CFG: Recieved packet on Request channel");
670  detail::RequestHeader hdr_buffer;
671  recv(request_socket_, &hdr_buffer, sizeof(hdr_buffer), 0);
672  TRACE(4, "CFG: Request header word: 0x%x", (int)hdr_buffer.header);
673  if (hdr_buffer.isValid())
674  {
675  std::vector<detail::RequestPacket> pkt_buffer(hdr_buffer.packet_count);
676  recv(request_socket_, &pkt_buffer[0], sizeof(detail::RequestPacket) * hdr_buffer.packet_count, 0);
677  bool anyNew = false;
678  for (auto& buffer : pkt_buffer)
679  {
680  if (!buffer.isValid()) continue;
681  if (requests_.count(buffer.sequence_id) && requests_[buffer.sequence_id] != buffer.timestamp)
682  {
683  TLOG_ERROR("CommandableFragmentGenerator") << "Received conflicting request for SeqID "
684  << std::to_string(buffer.sequence_id) << "!"
685  << " Old ts=" << std::to_string(requests_[buffer.sequence_id])
686  << ", new ts=" << std::to_string(buffer.timestamp) << ". Keeping OLD!" << TLOG_ENDL;
687  }
688  else if (!requests_.count(buffer.sequence_id))
689  {
690  int delta = buffer.sequence_id - ev_counter();
691  TRACE(4, "CFG: Recieved request for sequence ID %llu and timestamp %lu (delta: %d)", (unsigned long long)buffer.sequence_id, (unsigned long)buffer.timestamp, delta);
692  if (delta < 0)
693  {
694  TRACE(4, "CFG: Already serviced this request! Ignoring...");
695  }
696  else
697  {
698  std::unique_lock<std::mutex> tlk(request_mutex_);
699  requests_[buffer.sequence_id] = buffer.timestamp;
700  anyNew = true;
701  }
702  }
703  }
704  if (anyNew)
705  {
706  std::unique_lock<std::mutex> lock(request_mutex_);
707  requestCondition_.notify_all();
708  }
709  }
710  }
711  }
712  }
713 }
714 
716 {
717  if (check_stop())
718  {
719  return false;
720  }
721 
722  if (mode_ == RequestMode::Ignored)
723  {
724  while (dataBufferDepthFragments_ <= 0)
725  {
726  if (check_stop()) return false;
727  std::unique_lock<std::mutex> lock(dataBufferMutex_);
728  dataCondition_.wait_for(lock, std::chrono::milliseconds(10), [this]() { return dataBufferDepthFragments_ > 0; });
729  }
730  }
731  else
732  {
733  while (requests_.size() <= 0)
734  {
735  if (check_stop()) return false;
736 
737  checkDataBuffer();
738 
739  std::unique_lock<std::mutex> lock(request_mutex_);
740  requestCondition_.wait_for(lock, std::chrono::milliseconds(10), [this]() { return requests_.size() > 0; });
741  }
742  }
743 
744  {
745  std::unique_lock<std::mutex> dlk(dataBufferMutex_);
746 
747  if (mode_ == RequestMode::Ignored)
748  {
749  // We just copy everything that's here into the output.
750  TRACE(4, "CFG: Mode is Ignored; Copying data to output");
751  std::move(dataBuffer_.begin(), dataBuffer_.end(), std::inserter(frags, frags.end()));
752  dataBuffer_.clear();
753  }
754  else if (mode_ == RequestMode::Single)
755  {
756  // We only care about the latest request received. Send empties for all others.
757  sendEmptyFragments(frags);
758 
759  if (dataBuffer_.size() > 0)
760  {
761  TRACE(4, "CFG: Mode is Single; Sending copy of last event");
762  for (auto& fragptr : dataBuffer_)
763  {
764  // Return the latest data point
765  auto frag = fragptr.get();
766  auto newfrag = std::unique_ptr<artdaq::Fragment>(new Fragment(ev_counter(), frag->fragmentID()));
767  newfrag->resize(frag->size() - detail::RawFragmentHeader::num_words());
768  memcpy(newfrag->headerAddress(), frag->headerAddress(), frag->sizeBytes());
769  newfrag->setTimestamp(requests_[ev_counter()]);
770  newfrag->setSequenceID(ev_counter());
771  frags.push_back(std::move(newfrag));
772  }
773  }
774  else
775  {
776  sendEmptyFragment(frags, ev_counter(), "No data for");
777  }
778  requests_.clear();
779  ev_counter_inc(1, true);
780  }
781  else if (mode_ == RequestMode::Buffer || mode_ == RequestMode::Window)
782  {
783  if (mode_ == RequestMode::Buffer)
784  {
785  // We only care about the latest request received. Send empties for all others.
786  sendEmptyFragments(frags);
787  }
788  for (auto req = requests_.begin(); req != requests_.end();)
789  {
790  auto ts = req->second;
791  if (req->first < ev_counter())
792  {
793  req = requests_.erase(req);
794  continue;
795  }
796  if (req->first > ev_counter())
797  {
798  ++req;
799  continue; // Will loop through all requests, means we're in Window mode and missing the correct one
800  }
801  TRACE(5, "CFG: ApplyRequestS: Checking that data exists for request window %llu (Buffered mode will always succeed)",(unsigned long long)req->first);
802  Fragment::timestamp_t min = ts > windowOffset_ ? ts - windowOffset_ : 0;
803  Fragment::timestamp_t max = min + windowWidth_;
804  TRACE(5, "CFG::ApplyRequests: min is %lu and max is %lu and last point in buffer is %lu (sz=%zu)",
805  (unsigned long)min,(unsigned long)max, (unsigned long)(dataBuffer_.size() > 0 ? dataBuffer_.back()->timestamp() : 0), dataBuffer_.size());
806  bool windowClosed = mode_ != RequestMode::Window || (dataBuffer_.size() > 0 && dataBuffer_.back()->timestamp() >= max);
807  if (windowClosed || should_stop())
808  {
809  TLOG_DEBUG("CommandableFragmentGenerator") << "Creating ContainerFragment for Buffered or Window-requested Fragments" << TLOG_ENDL;
810  frags.emplace_back(new artdaq::Fragment(ev_counter(), fragment_id()));
811  frags.back()->setTimestamp(ts);
812  ContainerFragmentLoader cfl(*frags.back());
813 
814  if (mode_ == RequestMode::Window && should_stop() && !windowClosed) cfl.set_missing_data(true);
815  if (mode_ == RequestMode::Window && dataBuffer_.size() > 0 && dataBuffer_.front()->timestamp() < min)
816  {
817  TLOG_DEBUG("CommandableFragmentGenerator") << "Request Window covers data that is either before data collection began or has fallen off the end of the buffer" << TLOG_ENDL;
818  cfl.set_missing_data(true);
819  }
820 
821  // Buffer mode TFGs should simply copy out the whole dataBuffer_ into a ContainerFragment
822  // Window mode TFGs must do a little bit more work to decide which fragments to send for a given request
823  for (auto it = dataBuffer_.begin(); it != dataBuffer_.end();)
824  {
825  if (mode_ == RequestMode::Window)
826  {
827  Fragment::timestamp_t fragT = (*it)->timestamp();
828  if (fragT < min || fragT > max)
829  {
830  ++it;
831  continue;
832  }
833  }
834 
835  TRACE(5,"CFG::ApplyRequests: Adding Fragment with timestamp %llu to Container", (unsigned long long)(*it)->timestamp());
836  cfl.addFragment(*it);
837 
838  if (mode_ == RequestMode::Buffer || (mode_ == RequestMode::Window && uniqueWindows_))
839  {
840  it = dataBuffer_.erase(it);
841  }
842  else
843  {
844  ++it;
845  }
846  }
847  req = requests_.erase(req);
848  ev_counter_inc(1, true);
849  }
850  else
851  {
852  // Wait for the window to be closed for the current event
853  break;
854  }
855  }
856  }
857  getDataBufferStats();
858  }
859 
860  if (frags.size() > 0)
861  TRACE(4, "CFG: Finished Processing Event %lu for fragment_id %i.", ev_counter() + 1, fragment_id());
862  return true;
863 }
864 
865 bool artdaq::CommandableFragmentGenerator::sendEmptyFragment(artdaq::FragmentPtrs& frags, size_t seqId, std::string desc)
866 {
867  TLOG_WARNING("CommandableFragmentGenerator") << desc << " request " << seqId << ", sending empty fragment" << TLOG_ENDL;
868  auto frag = new Fragment();
869  frag->setSequenceID(seqId);
870  frag->setSystemType(Fragment::EmptyFragmentType);
871  frags.emplace_back(FragmentPtr(frag));
872  return true;
873 }
874 
876 {
877  auto sequence_id = Fragment::InvalidSequenceID;
878  auto timestamp = Fragment::InvalidTimestamp;
879  // Map is ordered by sequence ID!
880  for (auto it = requests_.begin(); it != requests_.end();)
881  {
882  auto seq = it->first;
883  auto ts = it->second;
884 
885  // Check if this is the one "true" request
886  if (++it == requests_.end())
887  {
888  sequence_id = seq;
889  timestamp = ts;
890  break;
891  }
892  if (seq < ev_counter()) continue;
893 
894  // Otherwise, this is just one we missed, send an empty
895  sendEmptyFragment(frags, ev_counter(), "Missed request for");
896  ev_counter_inc(1, true);
897  }
898  requests_.clear();
899 
900  if (sequence_id < ev_counter()) return; // No new requests received.
901  requests_[sequence_id] = timestamp;
902 }
int fragment_id() const
Get the current Fragment ID, if there is only one.
int ResolveHost(char const *host_in, in_addr &addr)
Convert a string hostname to a in_addr suitable for socket communication.
Definition: TCPConnect.cc:29
virtual ~CommandableFragmentGenerator()
CommandableFragmentGenerator Destructor.
bool sendEmptyFragment(FragmentPtrs &frags, size_t sequenceId, std::string desc)
Send an EmptyFragmentType Fragment.
void getMonitoringDataLoop()
This function regularly calls checkHWStatus_(), and sets the isHardwareOK flag accordingly.
void startDataThread()
Function that launches the data thread (getDataLoop())
std::string ReportCmd(std::string const &which="")
Get a report about a user-specified run-time quantity.
bool isValid() const
Check the magic bytes of the packet.
bool dataBufferIsTooLarge()
Test the configured constraints on the data buffer.
void StopCmd(uint64_t timeout, uint64_t timestamp)
Stop the CommandableFragmentGenerator.
void StartCmd(int run, uint64_t timeout, uint64_t timestamp)
Start the CommandableFragmentGenerator.
bool check_stop()
Routine used by applyRequests to make sure that all outstanding requests have been fulfilled before r...
void ResumeCmd(uint64_t timeout, uint64_t timestamp)
Resume the CommandableFragmentGenerator.
CommandableFragmentGenerator()
CommandableFragmentGenerator default constructor.
bool getNext(FragmentPtrs &output) overridefinal
getNext calls either applyRequests or getNext_ to get any data that is ready to be sent to the EventB...
size_t ev_counter_inc(size_t step=1, bool force=false)
Increment the event counter, if the current RequestMode allows it.
Header of a RequestMessage. Contains magic bytes for validation and a count of expected RequestPacket...
uint32_t packet_count
The number of RequestPackets in this Request message.
void PauseCmd(uint64_t timeout, uint64_t timestamp)
Pause the CommandableFragmentGenerator.
void getDataLoop()
When separate_data_thread is set to true, this loop repeatedly calls getNext_ and adds returned Fragm...
The RequestPacket contains information about a single data request.
void startRequestReceiverThread()
Function that launches the data request receiver thread (receiveRequestsLoop())
void startMonitoringThread()
Function that launches the monitoring thread (getMonitoringDataLoop())
void checkDataBuffer()
Perform data buffer pruning operations. If the RequestMode is Single, removes all but the latest Frag...
std::string printMode_()
Return the string representation of the current RequestMode.
void sendEmptyFragments(FragmentPtrs &frags)
This function is for Buffered and Single request modes, as they can only respond to one data request ...
void getDataBufferStats()
Calculate the size of the dataBuffer and report appropriate metrics.
bool applyRequests(FragmentPtrs &output)
See if any requests have been received, and add the corresponding data Fragment objects to the output...
void receiveRequestsLoop()
This function receives data request packets, adding new requests to the request list.
void setupRequestListener()
Opens the socket used to listen for data requests.