00001 #include "art/Framework/Art/artapp.h"
00002 #include "artdaq-core/Generators/FragmentGenerator.hh"
00003 #include "artdaq-core/Data/Fragment.hh"
00004 #include "artdaq-core/Generators/makeFragmentGenerator.hh"
00005 #include "Config.hh"
00006 #include "artdaq/DAQrate/EventStore.hh"
00007 #include "MPIProg.hh"
00008 #include "artdaq/DAQrate/DataSenderManager.hh"
00009 #include "artdaq/DAQrate/DataReceiverManager.hh"
00010 #include "artdaq-core/Core/SimpleQueueReader.hh"
00011 #include "artdaq/DAQrate/quiet_mpi.hh"
00012 #include "fhiclcpp/ParameterSet.h"
00013
00014 #include "boost/program_options.hpp"
00015 namespace bpo = boost::program_options;
00016
00017 #include <algorithm>
00018 #include <cmath>
00019 #include <cstdlib>
00020
00021 extern "C"
00022 {
00023 #include <unistd.h>
00024 }
00025
00026 #include <iostream>
00027 #include <memory>
00028 #include <utility>
00029
00030 extern "C"
00031 {
00032 #include <sys/time.h>
00033 #include <sys/resource.h>
00034 }
00035
00039 class Builder : public MPIProg
00040 {
00041 public:
00047 Builder(int argc, char* argv[]);
00048
00052 void go();
00053
00057 void source();
00058
00062 void sink();
00063
00067 void detector();
00068
00069 private:
00070 enum Color_t : int
00071 {
00072 DETECTOR,
00073 SOURCE,
00074 SINK
00075 };
00076
00077 void printHost(const std::string& functionName) const;
00078
00079 artdaq::Config conf_;
00080 fhicl::ParameterSet const daq_pset_;
00081 bool const want_sink_;
00082 bool const want_periodic_sync_;
00083 MPI_Comm local_group_comm_;
00084 };
00085
00086 Builder::Builder(int argc, char* argv[]) :
00087 MPIProg(argc, argv)
00088 , conf_(my_rank, procs_, 10, 10240, argc, argv)
00089 , daq_pset_(conf_.getArtPset())
00090 , want_sink_(daq_pset_.get<bool>("want_sink", true))
00091 , want_periodic_sync_(daq_pset_.get<bool>("want_periodic_sync", false))
00092 , local_group_comm_()
00093 {
00094 conf_.writeInfo();
00095 }
00096
00097 void Builder::go()
00098 {
00099 MPI_Barrier(MPI_COMM_WORLD);
00100
00101 MPI_Comm_split(MPI_COMM_WORLD, conf_.type_, 0, &local_group_comm_);
00102 switch (conf_.type_)
00103 {
00104 case artdaq::Config::TaskSink:
00105 if (want_sink_)
00106 {
00107 sink();
00108 }
00109 else
00110 {
00111 std::string
00112 msg("WARNING: a sink was instantiated despite want_sink being false:\n"
00113 "set nsinks to 0 in invocation of daqrate?\n");
00114 std::cerr << msg;
00115 MPI_Barrier(MPI_COMM_WORLD);
00116 }
00117 break;
00118 case artdaq::Config::TaskSource:
00119 source();
00120 break;
00121 case artdaq::Config::TaskDetector:
00122 detector();
00123 break;
00124 default:
00125 throw "No such node type";
00126 }
00127 }
00128
00129 void Builder::source()
00130 {
00131 printHost("source");
00132
00133 artdaq::FragmentPtr frag;
00134 {
00135 artdaq::DataReceiverManager from_d(conf_.makeParameterSet());
00136 from_d.start_threads();
00137 std::unique_ptr<artdaq::DataSenderManager> to_r(want_sink_ ? new artdaq::DataSenderManager(conf_.makeParameterSet()) : nullptr);
00138 int senderCount = from_d.enabled_sources().size();
00139 while (senderCount > 0)
00140 {
00141 int ignoredSender;
00142 frag = from_d.recvFragment(ignoredSender);
00143 if (!frag || ignoredSender == artdaq::TransferInterface::RECV_TIMEOUT) continue;
00144 std::cout << "Program::source: Received fragment " << frag->sequenceID() << " from sender " << ignoredSender << std::endl;
00145 if (want_sink_ && frag->type() != artdaq::Fragment::EndOfDataFragmentType)
00146 {
00147 to_r->sendFragment(std::move(*frag));
00148 }
00149 else if (frag->type() == artdaq::Fragment::EndOfDataFragmentType)
00150 {
00151 senderCount--;
00152 }
00153 }
00154 }
00155 TLOG_DEBUG("builder") << "source done " << conf_.rank_ << TLOG_ENDL;
00156 MPI_Barrier(MPI_COMM_WORLD);
00157 }
00158
00159 void Builder::detector()
00160 {
00161 printHost("detector");
00162 int detector_rank;
00163
00164 MPI_Comm_rank(local_group_comm_, &detector_rank);
00165 assert(!(detector_rank < 0));
00166 std::ostringstream det_ps_name_loc;
00167 std::vector<std::string> detectors;
00168 size_t detectors_size = 0;
00169 if (!(daq_pset_.get_if_present("detectors", detectors) &&
00170 (detectors_size = detectors.size())))
00171 {
00172 throw cet::exception("Configuration")
00173 << "Unable to find required sequence of detector "
00174 << "parameter set names, \"detectors\".";
00175 }
00176 fhicl::ParameterSet det_ps =
00177 daq_pset_.get<fhicl::ParameterSet>
00178 ((detectors_size > static_cast<size_t>(detector_rank)) ?
00179 detectors[detector_rank] :
00180 detectors[0]);
00181 std::unique_ptr<artdaq::FragmentGenerator> const
00182 gen(artdaq::makeFragmentGenerator
00183 (det_ps.get<std::string>("generator"),
00184 det_ps));
00185 {
00186 artdaq::DataSenderManager h(conf_.makeParameterSet());
00187 MPI_Barrier(local_group_comm_);
00188
00189
00190 size_t fragments_per_source = -1;
00191 daq_pset_.get_if_present("fragments_per_source", fragments_per_source);
00192 artdaq::FragmentPtrs frags;
00193 size_t fragments_sent = 0;
00194 while (fragments_sent < fragments_per_source && gen->getNext(frags))
00195 {
00196 if (!fragments_sent)
00197 {
00198
00199
00200 MPI_Barrier(local_group_comm_);
00201 }
00202 for (auto& fragPtr : frags)
00203 {
00204 std::cout << "Program::detector: Sending fragment " << fragments_sent + 1 << " of " << fragments_per_source << std::endl;
00205 h.sendFragment(std::move(*fragPtr));
00206 if (++fragments_sent == fragments_per_source) { break; }
00207 if (want_periodic_sync_ && (fragments_sent % 100) == 0)
00208 {
00209
00210 MPI_Barrier(local_group_comm_);
00211 }
00212 }
00213 frags.clear();
00214 }
00215 TLOG_DEBUG("builder") << "detector waiting " << conf_.rank_ << TLOG_ENDL;
00216 }
00217 TLOG_DEBUG("builder") << "detector done " << conf_.rank_ << TLOG_ENDL;
00218 MPI_Comm_free(&local_group_comm_);
00219 MPI_Barrier(MPI_COMM_WORLD);
00220 }
00221
00222 void Builder::sink()
00223 {
00224 printHost("sink");
00225 {
00226
00227 int sink_rank;
00228 bool useArt = daq_pset_.get<bool>("useArt", false);
00229 const char* dummyArgs[1]{"SimpleQueueReader"};
00230 MPI_Comm_rank(local_group_comm_, &sink_rank);
00231 artdaq::EventStore::ART_CMDLINE_FCN* reader =
00232 useArt ?
00233 &artapp :
00234 &artdaq::simpleQueueReaderApp;
00235 artdaq::EventStore events(daq_pset_, conf_.detectors_,
00236 conf_.run_,
00237 useArt ? conf_.art_argc_ : 1,
00238 useArt ? conf_.art_argv_ : const_cast<char**>(dummyArgs),
00239 reader);
00240 {
00241 artdaq::DataReceiverManager h(conf_.makeParameterSet());
00242 h.start_threads();
00243 int senderCount = h.enabled_sources().size();
00244 while (senderCount > 0)
00245 {
00246 artdaq::FragmentPtr pfragment(new artdaq::Fragment);
00247 int ignoredSource;
00248 pfragment = h.recvFragment(ignoredSource);
00249 if (!pfragment || ignoredSource == artdaq::TransferInterface::RECV_TIMEOUT) continue;
00250 std::cout << "Program::sink: Received fragment " << pfragment->sequenceID() << " from sender " << ignoredSource << std::endl;
00251 if (pfragment->type() != artdaq::Fragment::EndOfDataFragmentType)
00252 {
00253 events.insert(std::move(pfragment));
00254 }
00255 else
00256 {
00257 senderCount--;
00258 }
00259 }
00260 }
00261
00262
00263 int readerReturnValue;
00264 bool endSucceeded = false;
00265 int attemptsToEnd = 1;
00266 endSucceeded = events.endOfData(readerReturnValue);
00267 while (!endSucceeded && attemptsToEnd < 3)
00268 {
00269 ++attemptsToEnd;
00270 endSucceeded = events.endOfData(readerReturnValue);
00271 }
00272 if (endSucceeded)
00273 {
00274 TLOG_DEBUG("builder") << "Sink: reader is done, its exit status was: "
00275 << readerReturnValue << TLOG_ENDL;
00276 }
00277 else
00278 {
00279 TLOG_DEBUG("builder") << "Sink: reader failed to complete because the "
00280 << "endOfData marker could not be pushed onto the queue."
00281 << TLOG_ENDL;
00282 }
00283 }
00284 TLOG_DEBUG("builder") << "Sink done " << conf_.rank_ << TLOG_ENDL;
00285 MPI_Barrier(MPI_COMM_WORLD);
00286 }
00287
00288 void Builder::printHost(const std::string& functionName) const
00289 {
00290 char* doPrint = getenv("PRINT_HOST");
00291 if (doPrint == 0) { return; }
00292 const int ARRSIZE = 80;
00293 char hostname[ARRSIZE];
00294 std::string hostString;
00295 if (!gethostname(hostname, ARRSIZE))
00296 {
00297 hostString = hostname;
00298 }
00299 else
00300 {
00301 hostString = "unknown";
00302 }
00303 TLOG_DEBUG("builder") << "Running " << functionName
00304 << " on host " << hostString
00305 << " with rank " << my_rank << "."
00306 << TLOG_ENDL;
00307 }
00308
00309 void printUsage()
00310 {
00311 int myid = 0;
00312 struct rusage usage;
00313 getrusage(RUSAGE_SELF, &usage);
00314 std::cout << myid << ":"
00315 << " user=" << artdaq::Globals::timevalAsDouble(usage.ru_utime)
00316 << " sys=" << artdaq::Globals::timevalAsDouble(usage.ru_stime)
00317 << std::endl;
00318 }
00319
00320 int main(int argc, char* argv[])
00321 {
00322 artdaq::configureMessageFacility("builder");
00323 int rc = 1;
00324 try
00325 {
00326 Builder p(argc, argv);
00327 std::cerr << "Started process " << my_rank << " of " << p.procs_ << ".\n";
00328 p.go();
00329 rc = 0;
00330 }
00331 catch (std::string& x)
00332 {
00333 std::cerr << "Exception (type string) caught in driver: "
00334 << x
00335 << '\n';
00336 return 1;
00337 }
00338 catch (char const* m)
00339 {
00340 std::cerr << "Exception (type char const*) caught in driver: ";
00341 if (m)
00342 {
00343 std::cerr << m;
00344 }
00345 else
00346 {
00347 std::cerr << "[the value was a null pointer, so no message is available]";
00348 }
00349 std::cerr << '\n';
00350 }
00351 return rc;
00352 }