00001 #include "art/Framework/Core/ModuleMacros.h"
00002 #include "art/Framework/Core/OutputModule.h"
00003 #include "art/Framework/Principal/EventPrincipal.h"
00004 #include "art/Framework/Principal/RunPrincipal.h"
00005 #include "art/Framework/Principal/SubRunPrincipal.h"
00006 #include "art/Framework/Principal/Handle.h"
00007 #include "art/Persistency/Common/GroupQueryResult.h"
00008 #include "canvas/Utilities/DebugMacros.h"
00009 #include "canvas/Utilities/Exception.h"
00010 #include "fhiclcpp/ParameterSet.h"
00011
00012 #include "artdaq/DAQrate/DataSenderManager.hh"
00013 #include "artdaq/DAQdata/Globals.hh"
00014 #include "artdaq-core/Data/Fragment.hh"
00015
00016 #define TRACE_NAME "BinaryMPIOutput"
00017
00018 #include <iomanip>
00019 #include <iostream>
00020 #include <sstream>
00021 #include <string>
00022 #include <vector>
00023 #include <memory>
00024 #include "unistd.h"
00025
00026 namespace art
00027 {
00028 class BinaryMPIOutput;
00029 }
00030
00031 using art::BinaryMPIOutput;
00032 using fhicl::ParameterSet;
00033
00039 class art::BinaryMPIOutput final: public OutputModule
00040 {
00041 public:
00054 explicit BinaryMPIOutput(ParameterSet const& ps);
00055
00059 virtual ~BinaryMPIOutput();
00060
00061 private:
00062 void beginJob() override;
00063
00064 void endJob() override;
00065
00066 void write(EventPrincipal&) override;
00067
00068 void writeRun(RunPrincipal&) override {};
00069 void writeSubRun(SubRunPrincipal&) override {};
00070
00071 void initialize_MPI_();
00072
00073 void deinitialize_MPI_();
00074
00075 bool readParameterSet_(fhicl::ParameterSet const& pset);
00076
00077 private:
00078 ParameterSet data_pset_;
00079 std::string name_ = "BinaryMPIOutput";
00080 int rt_priority_ = 0;
00081 std::unique_ptr<artdaq::DataSenderManager> sender_ptr_ = {nullptr};
00082 };
00083
00084 art::BinaryMPIOutput::
00085 BinaryMPIOutput(ParameterSet const& ps)
00086 : OutputModule(ps)
00087 {
00088 FDEBUG(1) << "Begin: BinaryMPIOutput::BinaryMPIOutput(ParameterSet const& ps)\n";
00089 readParameterSet_(ps);
00090 FDEBUG(1) << "End: BinaryMPIOutput::BinaryMPIOutput(ParameterSet const& ps)\n";
00091 }
00092
00093 art::BinaryMPIOutput::
00094 ~BinaryMPIOutput()
00095 {
00096 FDEBUG(1) << "Begin/End: BinaryMPIOutput::~BinaryMPIOutput()\n";
00097 }
00098
00099 void
00100 art::BinaryMPIOutput::
00101 beginJob()
00102 {
00103 FDEBUG(1) << "Begin: BinaryMPIOutput::beginJob()\n";
00104 initialize_MPI_();
00105 FDEBUG(1) << "End: BinaryMPIOutput::beginJob()\n";
00106 }
00107
00108 void
00109 art::BinaryMPIOutput::
00110 endJob()
00111 {
00112 FDEBUG(1) << "Begin: BinaryMPIOutput::endJob()\n";
00113 deinitialize_MPI_();
00114 FDEBUG(1) << "End: BinaryMPIOutput::endJob()\n";
00115 }
00116
00117
00118 void
00119 art::BinaryMPIOutput::
00120 initialize_MPI_()
00121 {
00122 if (rt_priority_ > 0)
00123 {
00124 #pragma GCC diagnostic push
00125 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
00126 sched_param s_param = {};
00127 s_param.sched_priority = rt_priority_;
00128 int status = pthread_setschedparam(pthread_self(), SCHED_RR, &s_param);
00129 if (status != 0)
00130 {
00131 TLOG_ERROR(name_)
00132 << "Failed to set realtime priority to " << rt_priority_
00133 << ", return code = " << status << TLOG_ENDL;
00134 }
00135 #pragma GCC diagnostic pop
00136 }
00137
00138 sender_ptr_ = std::make_unique<artdaq::DataSenderManager>(data_pset_);
00139 assert(sender_ptr_);
00140 }
00141
00142 void
00143 art::BinaryMPIOutput::
00144 deinitialize_MPI_()
00145 {
00146 sender_ptr_.reset(nullptr);
00147 }
00148
00149 bool
00150 art::BinaryMPIOutput::
00151 readParameterSet_(fhicl::ParameterSet const& pset)
00152 {
00153 TLOG_DEBUG(name_) << "BinaryMPIOutput::readParameterSet_ method called with "
00154 << "ParameterSet = \"" << pset.to_string()
00155 << "\"." << TLOG_ENDL;
00156
00157
00158 data_pset_ = pset;
00159 name_ = pset.get<std::string>("module_name", "BinaryMPIOutput");
00160 rt_priority_ = pset.get<int>("rt_priority", 0);
00161
00162 TRACE(4, "BinaryMPIOutput::readParameterSet()");
00163
00164 return true;
00165 }
00166
00167 void
00168 art::BinaryMPIOutput::
00169 write(EventPrincipal& ep)
00170 {
00171 assert(sender_ptr_);
00172
00173 using RawEvent = artdaq::Fragments;;
00174 using RawEvents = std::vector<RawEvent>;
00175 using RawEventHandle = art::Handle<RawEvent>;
00176 using RawEventHandles = std::vector<RawEventHandle>;
00177
00178 auto result_handles = std::vector<art::GroupQueryResult>();
00179 ep.getManyByType(art::TypeID(typeid(RawEvent)), result_handles);
00180
00181 for (auto const& result_handle : result_handles)
00182 {
00183 auto const raw_event_handle = RawEventHandle(result_handle);
00184
00185 if (!raw_event_handle.isValid())
00186 continue;
00187
00188 for (auto const& fragment : *raw_event_handle)
00189 {
00190 auto fragment_copy = fragment;
00191 auto fragid_id = fragment_copy.fragmentID();
00192 auto sequence_id = fragment_copy.sequenceID();
00193 TRACE(1, "BinaryMPIOutput::write seq=%lu frag=%i start", sequence_id, fragid_id);
00194 sender_ptr_->sendFragment(std::move(fragment_copy));
00195 TRACE(2, "BinaryMPIOutput::write seq=%lu frag=%i done", sequence_id, fragid_id);
00196 }
00197 }
00198
00199 return;
00200 }
00201
00202 DEFINE_ART_MODULE(art::BinaryMPIOutput)