00001 #ifndef artdaq_Application_CommandableFragmentGenerator_hh
00002 #define artdaq_Application_CommandableFragmentGenerator_hh
00003
00004
00005 #include <arpa/inet.h>
00006 #include <netinet/in.h>
00007 #include <sys/types.h>
00008 #include <sys/socket.h>
00009 #include <unistd.h>
00010
00011 #include <atomic>
00012 #include <condition_variable>
00013 #include <mutex>
00014 #include <queue>
00015 #include <chrono>
00016 #include <array>
00017 #include <list>
00018
00019 #include "fhiclcpp/fwd.h"
00020 #include "fhiclcpp/ParameterSet.h"
00021
00022 #include "artdaq/DAQdata/Globals.hh"
00023 #include "artdaq-core/Data/Fragment.hh"
00024 #include "artdaq-core/Generators/FragmentGenerator.hh"
00025 #include "artdaq-utilities/Plugins/MetricManager.hh"
00026 #include "artdaq/DAQrate/RequestReceiver.hh"
00027
00028 namespace artdaq
00029 {
00033 enum class RequestMode
00034 {
00035 Single,
00036 Buffer,
00037 Window,
00038 Ignored
00039 };
00040
00083 class CommandableFragmentGenerator : public FragmentGenerator
00084 {
00085 public:
00089 struct Config
00090 {
00092 fhicl::Atom<std::string> generator_type { fhicl::Name{"generator" }, fhicl::Comment{"Name of the CommandableFragmentGenerator plugin to load"} };
00094 fhicl::Atom<Fragment::timestamp_t> request_window_offset { fhicl::Name{"request_window_offset" }, fhicl::Comment{"Request messages contain a timestamp. For Window request mode, start the window this far before the timestamp in the request"}, 0 };
00096 fhicl::Atom<Fragment::timestamp_t> request_window_width { fhicl::Name{"request_window_width" }, fhicl::Comment{"For Window request mode, the window will be timestamp - offset to timestamp - offset + width"}, 0 };
00098 fhicl::Atom<Fragment::timestamp_t> stale_request_timeout { fhicl::Name{"stale_request_timeout" }, fhicl::Comment{"How long should request messages be retained"}, 0xFFFFFFFF };
00100 fhicl::Atom<Fragment::type_t> expected_fragment_type { fhicl::Name{"expected_fragment_type" }, fhicl::Comment{"The type of Fragments this CFG will be generating. \"Empty\" will auto-detect type based on Fragments generated."}, Fragment::type_t(Fragment::EmptyFragmentType) };
00102 fhicl::Atom<bool> request_windows_are_unique { fhicl::Name{"request_windows_are_unique" }, fhicl::Comment{"Whether Fragments should be removed from the buffer when matched to a request window"}, true };
00104 fhicl::Atom<size_t> missing_request_window_timeout_us { fhicl::Name{"missing_request_window_timeout_us"}, fhicl::Comment{"How long to track missing requests in the \"out - of - order Windows\" list"}, 5000000 };
00106 fhicl::Atom<size_t> window_close_timeout_us { fhicl::Name{"window_close_timeout_us" }, fhicl::Comment{"How long to wait for the end of the data buffer to pass the end of a request window (measured from the time the request was received)"}, 2000000 };
00108 fhicl::Atom<bool> separate_data_thread { fhicl::Name{"separate_data_thread" }, fhicl::Comment{"Whether data collection should proceed on its own thread. Required for all data request processing"}, false };
00110 fhicl::Atom<bool> circular_buffer_mode { fhicl::Name{"circular_buffer_mode" }, fhicl::Comment{"Whether the data buffer should be treated as a circular buffer on the input side (i.e. old fragments are automatically discarded when the buffer is full to always call getNext_)."}, false };
00112 fhicl::Atom<size_t> sleep_on_no_data_us { fhicl::Name{"sleep_on_no_data_us" }, fhicl::Comment{"How long to sleep after calling getNext_ if no data is returned"}, 0 };
00114 fhicl::Atom<int> data_buffer_depth_fragments { fhicl::Name{"data_buffer_depth_fragments" }, fhicl::Comment{"How many Fragments to store in the buffer"}, 1000 };
00116 fhicl::Atom<size_t> data_buffer_depth_mb { fhicl::Name{"data_buffer_depth_mb" }, fhicl::Comment{"The maximum size of the data buffer in MB"}, 1000 };
00118 fhicl::Atom<bool> separate_monitoring_thread { fhicl::Name{"separate_monitoring_thread" }, fhicl::Comment{"Whether a thread that calls the checkHWStatus_ method should be created"}, false };
00120 fhicl::Atom<int64_t> hardware_poll_interval_us { fhicl::Name{"hardware_poll_interval_us" }, fhicl::Comment{"If a separate monitoring thread is used, how often should it call checkHWStatus_"}, 0 };
00122 fhicl::Atom<int> board_id { fhicl::Name{"board_id" }, fhicl::Comment{"The identification number for this CommandableFragmentGenerator"} };
00125 fhicl::Sequence<Fragment::fragment_id_t> fragment_ids { fhicl::Name("fragment_ids" ), fhicl::Comment("A list of Fragment IDs created by this CommandableFragmentGenerator") };
00128 fhicl::Atom<int> fragment_id { fhicl::Name{"fragment_id" }, fhicl::Comment{"The Fragment ID created by this CommandableFragmentGenerator"}, -99 };
00130 fhicl::Atom<int> sleep_on_stop_us { fhicl::Name{"sleep_on_stop_us" }, fhicl::Comment{"How long to sleep before returning when stop transition is called"}, 0 };
00138 fhicl::Atom<std::string> request_mode { fhicl::Name{"request_mode" }, fhicl::Comment{"The mode by which the CommandableFragmentGenerator will process reqeusts"}, "ignored" };
00139 fhicl::TableFragment<artdaq::RequestReceiver::Config> receiverConfig;
00140 };
00141 using Parameters = fhicl::WrappedTable<Config>;
00142
00148 CommandableFragmentGenerator();
00149
00154 explicit CommandableFragmentGenerator(const fhicl::ParameterSet& ps);
00155
00161 virtual ~CommandableFragmentGenerator();
00162
00163
00170 void joinThreads();
00171
00177 bool getNext(FragmentPtrs& output) override final;
00178
00179
00185 void applyRequestsIgnoredMode(artdaq::FragmentPtrs& frags);
00186
00192 void applyRequestsSingleMode(artdaq::FragmentPtrs& frags);
00193
00199 void applyRequestsBufferMode(artdaq::FragmentPtrs& frags);
00200
00206 void applyRequestsWindowMode(artdaq::FragmentPtrs& frags);
00207
00213 bool applyRequests(FragmentPtrs& output);
00214
00222 bool sendEmptyFragment(FragmentPtrs& frags, size_t sequenceId, std::string desc);
00223
00230 void sendEmptyFragments(FragmentPtrs& frags, std::map<Fragment::sequence_id_t, Fragment::timestamp_t>& requests);
00231
00236 void checkOutOfOrderWindows(Fragment::sequence_id_t seq);
00237
00242 std::map<Fragment::sequence_id_t, std::chrono::steady_clock::time_point> getOutOfOrderWindowList() const
00243 {
00244 return windows_sent_ooo_;
00245 }
00246
00250 void startDataThread();
00251
00255 void startMonitoringThread();
00256
00261 void getDataLoop();
00262
00267 bool waitForDataBufferReady();
00268
00273 bool dataBufferIsTooLarge();
00274
00278 void getDataBufferStats();
00279
00285 void checkDataBuffer();
00286
00290 void getMonitoringDataLoop();
00291
00296 std::vector<Fragment::fragment_id_t> fragmentIDs() override
00297 {
00298 return fragment_ids_;
00299 }
00300
00305 size_t ev_counter() const { return ev_counter_.load(); }
00306
00311 RequestMode request_mode() const { return mode_; }
00312
00313
00314
00315
00316
00331 void StartCmd(int run, uint64_t timeout, uint64_t timestamp);
00332
00342 void StopCmd(uint64_t timeout, uint64_t timestamp);
00343
00353 void PauseCmd(uint64_t timeout, uint64_t timestamp);
00354
00363 void ResumeCmd(uint64_t timeout, uint64_t timestamp);
00364
00375 std::string ReportCmd(std::string const& which = "");
00376
00381 virtual std::string metricsReportingInstanceName() const
00382 {
00383 return instance_name_for_metrics_;
00384 }
00385
00386
00387
00388
00389
00390
00391
00392
00393
00394
00395
00396
00397
00398
00399
00400
00401
00406 bool exception() const { return exception_.load(); }
00407
00408
00415 virtual bool metaCommand(std::string const& command, std::string const& arg);
00416
00417 protected:
00418
00419
00420
00421
00422
00423
00424
00425
00426
00431 int run_number() const { return run_number_; }
00436 int subrun_number() const { return subrun_number_; }
00441 uint64_t timeout() const { return timeout_; }
00446 uint64_t timestamp() const { return timestamp_; }
00447
00452 bool should_stop() const { return should_stop_.load(); }
00453
00458 bool check_stop();
00459
00464 int board_id() const { return board_id_; }
00465
00471 int fragment_id() const;
00472
00479 size_t ev_counter_inc(size_t step = 1, bool force = false);
00480
00485 void set_exception(bool exception) { exception_.store(exception); }
00486
00491 void metricsReportingInstanceName(std::string const& name)
00492 {
00493 instance_name_for_metrics_ = name;
00494 }
00495
00500 std::string printMode_();
00501
00502
00503
00504
00505
00506
00507
00508
00509
00510 std::mutex mutex_;
00511
00512 private:
00513
00514
00515
00516
00517 std::unique_ptr<RequestReceiver> requestReceiver_;
00518
00519 RequestMode mode_;
00520 Fragment::timestamp_t windowOffset_;
00521 Fragment::timestamp_t windowWidth_;
00522 Fragment::timestamp_t staleTimeout_;
00523 Fragment::type_t expectedType_;
00524 size_t maxFragmentCount_;
00525 bool uniqueWindows_;
00526 std::map<Fragment::sequence_id_t, std::chrono::steady_clock::time_point> windows_sent_ooo_;
00527 size_t missing_request_window_timeout_us_;
00528 size_t window_close_timeout_us_;
00529
00530 bool useDataThread_;
00531 bool circularDataBufferMode_;
00532 size_t sleep_on_no_data_us_;
00533 std::atomic<bool> data_thread_running_;
00534 boost::thread dataThread_;
00535
00536 std::condition_variable dataCondition_;
00537 std::atomic<int> dataBufferDepthFragments_;
00538 std::atomic<size_t> dataBufferDepthBytes_;
00539 int maxDataBufferDepthFragments_;
00540 size_t maxDataBufferDepthBytes_;
00541
00542 bool useMonitoringThread_;
00543 boost::thread monitoringThread_;
00544 int64_t monitoringInterval_;
00545 std::chrono::steady_clock::time_point lastMonitoringCall_;
00546 std::atomic<bool> isHardwareOK_;
00547
00548 FragmentPtrs dataBuffer_;
00549 FragmentPtrs newDataBuffer_;
00550 std::mutex dataBufferMutex_;
00551
00552 std::vector<artdaq::Fragment::fragment_id_t> fragment_ids_;
00553
00554
00555
00556
00557 int run_number_, subrun_number_;
00558
00559
00560
00561
00562 uint64_t timeout_;
00563
00564
00565
00566
00567
00568
00569
00570 uint64_t timestamp_;
00571
00572 std::atomic<bool> should_stop_, exception_, force_stop_;
00573 std::string latest_exception_report_;
00574 std::atomic<size_t> ev_counter_;
00575
00576 int board_id_;
00577 std::string instance_name_for_metrics_;
00578
00579
00580
00581
00582 int sleep_on_stop_us_;
00583
00584 protected:
00585
00593 virtual bool getNext_(FragmentPtrs& output) = 0;
00594
00595
00602 virtual bool checkHWStatus_();
00603
00604
00605
00606
00607
00618 virtual void start() = 0;
00619
00626 virtual void stopNoMutex() = 0;
00627
00636 virtual void stop() = 0;
00637
00642 virtual void pauseNoMutex();
00643
00649 virtual void pause();
00650
00654 virtual void resume();
00655
00667 virtual std::string report();
00668
00674 virtual std::string reportSpecific(std::string const& what);
00675
00676 };
00677 }
00678
00679 #endif