00001 #ifndef artdaq_Application_CommandableFragmentGenerator_hh
00002 #define artdaq_Application_CommandableFragmentGenerator_hh
00003
00004
00005 #include <arpa/inet.h>
00006 #include <netinet/in.h>
00007 #include <sys/types.h>
00008 #include <sys/socket.h>
00009 #include <unistd.h>
00010
00011 #include <atomic>
00012 #include <condition_variable>
00013 #include <mutex>
00014 #include <queue>
00015 #include <chrono>
00016 #include <array>
00017 #include <list>
00018
00019 #include "fhiclcpp/fwd.h"
00020 #include "fhiclcpp/ParameterSet.h"
00021
00022 #include "artdaq/DAQdata/Globals.hh"
00023 #include "artdaq-core/Data/Fragment.hh"
00024 #include "artdaq-core/Generators/FragmentGenerator.hh"
00025 #include "artdaq-utilities/Plugins/MetricManager.hh"
00026 #include "artdaq/DAQrate/RequestReceiver.hh"
00027
00028 namespace artdaq
00029 {
00033 enum class RequestMode
00034 {
00035 Single,
00036 Buffer,
00037 Window,
00038 Ignored
00039 };
00040
00083 class CommandableFragmentGenerator : public FragmentGenerator
00084 {
00085 public:
00086
00087 struct Config
00088 {
00089 fhicl::Atom<std::string> generator_type { fhicl::Name{"generator" }, fhicl::Comment{"Name of the CommandableFragmentGenerator plugin to load"} };
00090 fhicl::Atom<Fragment::timestamp_t> request_window_offset { fhicl::Name{"request_window_offset" }, fhicl::Comment{"Request messages contain a timestamp. For Window request mode, start the window this far before the timestamp in the request"}, 0 };
00091 fhicl::Atom<Fragment::timestamp_t> request_window_width { fhicl::Name{"request_window_width" }, fhicl::Comment{"For Window request mode, the window will be timestamp - offset to timestamp - offset + width"}, 0 };
00092 fhicl::Atom<Fragment::timestamp_t> stale_request_timeout { fhicl::Name{"stale_request_timeout" }, fhicl::Comment{"How long should request messages be retained"}, 0xFFFFFFFF };
00093 fhicl::Atom<Fragment::type_t> expected_fragment_type { fhicl::Name{"expected_fragment_type" }, fhicl::Comment{"The type of Fragments this CFG will be generating. \"Empty\" will auto-detect type based on Fragments generated."}, Fragment::type_t(Fragment::EmptyFragmentType) };
00094 fhicl::Atom<bool> request_windows_are_unique { fhicl::Name{"request_windows_are_unique" }, fhicl::Comment{"Whether Fragments should be removed from the buffer when matched to a request window"}, true };
00095 fhicl::Atom<size_t> missing_request_window_timeout_us { fhicl::Name{"missing_request_window_timeout_us"}, fhicl::Comment{"How long to wait for a missing request in Window mode (measured from the last time data was sent)"}, 1000000 };
00096 fhicl::Atom<size_t> window_close_timeout_us { fhicl::Name{"window_close_timeout_us" }, fhicl::Comment{"How long to wait for the end of the data buffer to pass the end of a request window (measured from the last time data was sent)"}, 2000000 };
00097 fhicl::Atom<bool> separate_data_thread { fhicl::Name{"separate_data_thread" }, fhicl::Comment{"Whether data collection should proceed on its own thread. Required for all data request processing"}, false };
00098 fhicl::Atom<size_t> sleep_on_no_data_us { fhicl::Name{"sleep_on_no_data_us" }, fhicl::Comment{"How long to sleep after calling getNext_ if no data is returned"}, 0 };
00099 fhicl::Atom<int> data_buffer_depth_fragments { fhicl::Name{"data_buffer_depth_fragments" }, fhicl::Comment{"How many Fragments to store in the buffer"}, 1000 };
00100 fhicl::Atom<size_t> data_buffer_depth_mb { fhicl::Name{"data_buffer_depth_mb" }, fhicl::Comment{"The maximum size of the data buffer in MB"}, 1000 };
00101 fhicl::Atom<bool> separate_monitoring_thread { fhicl::Name{"separate_monitoring_thread" }, fhicl::Comment{"Whether a thread that calls the checkHWStatus_ method should be created"}, false };
00102 fhicl::Atom<int64_t> hardware_poll_interval_us { fhicl::Name{"hardware_poll_interval_us" }, fhicl::Comment{"If a separate monitoring thread is used, how often should it call checkHWStatus_"}, 0 };
00103 fhicl::Atom<int> board_id { fhicl::Name{"board_id" }, fhicl::Comment{"The identification number for this CommandableFragmentGenerator"} };
00104 fhicl::Sequence<Fragment::fragment_id_t> fragment_ids { fhicl::Name("fragment_ids" ), fhicl::Comment("A list of Fragment IDs created by this CommandableFragmentGenerator") };
00105 fhicl::Atom<int> fragment_id { fhicl::Name{"fragment_id" }, fhicl::Comment{"The Fragment ID created by this CommandableFragmentGenerator"}, -99 };
00106 fhicl::Atom<int> sleep_on_stop_us { fhicl::Name{"sleep_on_stop_us" }, fhicl::Comment{"How long to sleep before returning when stop transition is called"}, 0 };
00107 fhicl::Atom<std::string> request_mode { fhicl::Name{"request_mode" }, fhicl::Comment{"The mode by which the CommandableFragmentGenerator will process reqeusts"}, "ignored" };
00108 fhicl::TableFragment<artdaq::RequestReceiver::Config> receiverConfig;
00109 };
00110 #if MESSAGEFACILITY_HEX_VERSION >= 0x20103
00111 using Parameters = fhicl::WrappedTable<Config>;
00112 #endif
00113
00118 CommandableFragmentGenerator();
00119
00154 explicit CommandableFragmentGenerator(const fhicl::ParameterSet& ps);
00155
00161 virtual ~CommandableFragmentGenerator();
00162
00163
00170 void joinThreads();
00171
00177 bool getNext(FragmentPtrs& output) override final;
00178
00179
00185 void applyRequestsIgnoredMode(artdaq::FragmentPtrs& frags);
00186
00192 void applyRequestsSingleMode(artdaq::FragmentPtrs& frags);
00193
00199 void applyRequestsBufferMode(artdaq::FragmentPtrs& frags);
00200
00206 void applyRequestsWindowMode(artdaq::FragmentPtrs& frags);
00207
00213 bool applyRequests(FragmentPtrs& output);
00214
00222 bool sendEmptyFragment(FragmentPtrs& frags, size_t sequenceId, std::string desc);
00223
00230 void sendEmptyFragments(FragmentPtrs& frags, std::map<Fragment::sequence_id_t, Fragment::timestamp_t>& requests);
00231
00235 void startDataThread();
00236
00240 void startMonitoringThread();
00241
00246 void getDataLoop();
00247
00252 bool waitForDataBufferReady();
00253
00258 bool dataBufferIsTooLarge();
00259
00263 void getDataBufferStats();
00264
00270 void checkDataBuffer();
00271
00275 void getMonitoringDataLoop();
00276
00281 std::vector<Fragment::fragment_id_t> fragmentIDs() override
00282 {
00283 return fragment_ids_;
00284 }
00285
00286
00287
00288
00289
00304 void StartCmd(int run, uint64_t timeout, uint64_t timestamp);
00305
00315 void StopCmd(uint64_t timeout, uint64_t timestamp);
00316
00326 void PauseCmd(uint64_t timeout, uint64_t timestamp);
00327
00336 void ResumeCmd(uint64_t timeout, uint64_t timestamp);
00337
00348 std::string ReportCmd(std::string const& which = "");
00349
00354 virtual std::string metricsReportingInstanceName() const
00355 {
00356 return instance_name_for_metrics_;
00357 }
00358
00359
00360
00361
00362
00363
00364
00365
00366
00367
00368
00369
00370
00371
00372
00373
00374
00379 bool exception() const { return exception_.load(); }
00380
00381
00388 virtual bool metaCommand(std::string const& command, std::string const& arg);
00389
00390 protected:
00391
00392
00393
00394
00395
00396
00397
00398
00399
00404 int run_number() const { return run_number_; }
00409 int subrun_number() const { return subrun_number_; }
00414 uint64_t timeout() const { return timeout_; }
00419 uint64_t timestamp() const { return timestamp_; }
00420
00425 bool should_stop() const { return should_stop_.load(); }
00426
00431 bool check_stop();
00432
00437 int board_id() const { return board_id_; }
00438
00444 int fragment_id() const;
00445
00450 size_t ev_counter() const { return ev_counter_.load(); }
00451
00458 size_t ev_counter_inc(size_t step = 1, bool force = false);
00459
00464 void set_exception(bool exception) { exception_.store(exception); }
00465
00470 void metricsReportingInstanceName(std::string const& name)
00471 {
00472 instance_name_for_metrics_ = name;
00473 }
00474
00479 std::string printMode_();
00480
00481
00482
00483
00484
00485
00486
00487
00488
00489 std::mutex mutex_;
00490
00491 private:
00492
00493
00494
00495
00496 std::unique_ptr<RequestReceiver> requestReceiver_;
00497
00498 RequestMode mode_;
00499 Fragment::timestamp_t windowOffset_;
00500 Fragment::timestamp_t windowWidth_;
00501 Fragment::timestamp_t staleTimeout_;
00502 Fragment::type_t expectedType_;
00503 size_t maxFragmentCount_;
00504 bool uniqueWindows_;
00505 bool missing_request_;
00506 std::chrono::steady_clock::time_point missing_request_time_;
00507 std::chrono::steady_clock::time_point last_window_send_time_;
00508 bool last_window_send_time_set_;
00509 std::set<Fragment::sequence_id_t> windows_sent_ooo_;
00510 size_t missing_request_window_timeout_us_;
00511 size_t window_close_timeout_us_;
00512
00513 bool useDataThread_;
00514 size_t sleep_on_no_data_us_;
00515 std::atomic<bool> data_thread_running_;
00516 boost::thread dataThread_;
00517
00518 std::condition_variable dataCondition_;
00519 std::atomic<int> dataBufferDepthFragments_;
00520 std::atomic<size_t> dataBufferDepthBytes_;
00521 int maxDataBufferDepthFragments_;
00522 size_t maxDataBufferDepthBytes_;
00523
00524 bool useMonitoringThread_;
00525 boost::thread monitoringThread_;
00526 int64_t monitoringInterval_;
00527 std::chrono::steady_clock::time_point lastMonitoringCall_;
00528 bool isHardwareOK_;
00529
00530 FragmentPtrs dataBuffer_;
00531 FragmentPtrs newDataBuffer_;
00532 std::mutex dataBufferMutex_;
00533
00534 std::vector<artdaq::Fragment::fragment_id_t> fragment_ids_;
00535
00536
00537
00538
00539 int run_number_, subrun_number_;
00540
00541
00542
00543
00544 uint64_t timeout_;
00545
00546
00547
00548
00549
00550
00551
00552 uint64_t timestamp_;
00553
00554 std::atomic<bool> should_stop_, exception_, force_stop_;
00555 std::string latest_exception_report_;
00556 std::atomic<size_t> ev_counter_;
00557
00558 int board_id_;
00559 std::string instance_name_for_metrics_;
00560
00561
00562
00563
00564 int sleep_on_stop_us_;
00565
00566 protected:
00567
00568
00569
00570
00571
00572 virtual bool getNext_(FragmentPtrs& output) = 0;
00573
00574
00575
00576
00577
00578 virtual bool checkHWStatus_();
00579
00580
00581
00582
00583
00584
00585
00586
00587
00588
00589
00590 virtual void start() = 0;
00591
00592
00593
00594 virtual void stopNoMutex() = 0;
00595
00596
00597
00598
00599
00600 virtual void stop() = 0;
00601
00602
00603
00604 virtual void pauseNoMutex();
00605
00606
00607
00608
00609 virtual void pause();
00610
00611
00612
00613
00614
00615 virtual void resume();
00616
00617
00618
00619
00620
00621
00622
00623
00624
00625 virtual std::string report();
00626
00627 virtual std::string reportSpecific(std::string const&);
00628
00629 };
00630 }
00631
00632 #endif