artdaq  v2_02_03
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Pages
builder.cc
1 #include "art/Framework/Art/artapp.h"
2 #include "artdaq-core/Generators/FragmentGenerator.hh"
3 #include "artdaq-core/Data/Fragment.hh"
4 #include "artdaq-core/Generators/makeFragmentGenerator.hh"
5 #include "Config.hh"
6 #include "artdaq/DAQrate/EventStore.hh"
7 #include "MPIProg.hh"
8 #include "artdaq/DAQrate/DataSenderManager.hh"
9 #include "artdaq/DAQrate/DataReceiverManager.hh"
10 #include "artdaq-core/Core/SimpleQueueReader.hh"
12 #include "fhiclcpp/ParameterSet.h"
13 
14 #include "boost/program_options.hpp"
15 namespace bpo = boost::program_options;
16 
17 #include <algorithm>
18 #include <cmath>
19 #include <cstdlib>
20 
21 extern "C"
22 {
23 #include <unistd.h>
24 }
25 
26 #include <iostream>
27 #include <memory>
28 #include <utility>
29 
30 extern "C"
31 {
32 #include <sys/time.h>
33 #include <sys/resource.h>
34 }
35 
39 class Builder : public MPIProg
40 {
41 public:
47  Builder(int argc, char* argv[]);
48 
52  void go();
53 
57  void source();
58 
62  void sink();
63 
67  void detector();
68 
69 private:
70  enum Color_t : int
71  {
72  DETECTOR,
73  SOURCE,
74  SINK
75  };
76 
77  void printHost(const std::string& functionName) const;
78 
79  artdaq::Config conf_;
80  fhicl::ParameterSet const daq_pset_;
81  bool const want_sink_;
82  bool const want_periodic_sync_;
83  MPI_Comm local_group_comm_;
84 };
85 
86 Builder::Builder(int argc, char* argv[]) :
87  MPIProg(argc, argv)
88  , conf_(my_rank, procs_, 10, 10240, argc, argv)
89  , daq_pset_(conf_.getArtPset())
90  , want_sink_(daq_pset_.get<bool>("want_sink", true))
91  , want_periodic_sync_(daq_pset_.get<bool>("want_periodic_sync", false))
92  , local_group_comm_()
93 {
94  conf_.writeInfo();
95 }
96 
98 {
99  MPI_Barrier(MPI_COMM_WORLD);
100  //std::cout << "daq_pset_: " << daq_pset_.to_string() << std::endl << "conf_.makeParameterSet(): " << conf_.makeParameterSet().to_string() << std::endl;
101  MPI_Comm_split(MPI_COMM_WORLD, conf_.type_, 0, &local_group_comm_);
102  switch (conf_.type_)
103  {
105  if (want_sink_)
106  {
107  sink();
108  }
109  else
110  {
111  std::string
112  msg("WARNING: a sink was instantiated despite want_sink being false:\n"
113  "set nsinks to 0 in invocation of daqrate?\n");
114  std::cerr << msg;
115  MPI_Barrier(MPI_COMM_WORLD);
116  }
117  break;
119  source();
120  break;
122  detector();
123  break;
124  default:
125  throw "No such node type";
126  }
127 }
128 
130 {
131  printHost("source");
132  // needs to get data from the detectors and send it to the sinks
133  artdaq::FragmentPtr frag;
134  { // Block to handle lifetime of to_r and from_d, below.
136  from_d.start_threads();
137  std::unique_ptr<artdaq::DataSenderManager> to_r(want_sink_ ? new artdaq::DataSenderManager(conf_.makeParameterSet()) : nullptr);
138  int senderCount = from_d.enabled_sources().size();
139  while (senderCount > 0)
140  {
141  int ignoredSender;
142  frag = from_d.recvFragment(ignoredSender);
143  if (!frag || ignoredSender == artdaq::TransferInterface::RECV_TIMEOUT) continue;
144  std::cout << "Program::source: Received fragment " << frag->sequenceID() << " from sender " << ignoredSender << std::endl;
145  if (want_sink_ && frag->type() != artdaq::Fragment::EndOfDataFragmentType)
146  {
147  to_r->sendFragment(std::move(*frag));
148  }
149  else if (frag->type() == artdaq::Fragment::EndOfDataFragmentType)
150  {
151  senderCount--;
152  }
153  }
154  }
155  TLOG_DEBUG("builder") << "source done " << conf_.rank_ << TLOG_ENDL;
156  MPI_Barrier(MPI_COMM_WORLD);
157 }
158 
160 {
161  printHost("detector");
162  int detector_rank;
163  // Should be zero-based, detectors only.
164  MPI_Comm_rank(local_group_comm_, &detector_rank);
165  assert(!(detector_rank < 0));
166  std::ostringstream det_ps_name_loc;
167  std::vector<std::string> detectors;
168  size_t detectors_size = 0;
169  if (!(daq_pset_.get_if_present("detectors", detectors) &&
170  (detectors_size = detectors.size())))
171  {
172  throw cet::exception("Configuration")
173  << "Unable to find required sequence of detector "
174  << "parameter set names, \"detectors\".";
175  }
176  fhicl::ParameterSet det_ps =
177  daq_pset_.get<fhicl::ParameterSet>
178  ((detectors_size > static_cast<size_t>(detector_rank)) ?
179  detectors[detector_rank] :
180  detectors[0]);
181  std::unique_ptr<artdaq::FragmentGenerator> const
182  gen(artdaq::makeFragmentGenerator
183  (det_ps.get<std::string>("generator"),
184  det_ps));
185  { // Block to handle lifetime of h, below.
187  MPI_Barrier(local_group_comm_);
188  // not using the run time method
189  // TimedLoop tl(conf_.run_time_);
190  size_t fragments_per_source = -1;
191  daq_pset_.get_if_present("fragments_per_source", fragments_per_source);
192  artdaq::FragmentPtrs frags;
193  size_t fragments_sent = 0;
194  while (fragments_sent < fragments_per_source && gen->getNext(frags))
195  {
196  if (!fragments_sent)
197  {
198  // Get the detectors lined up first time before we start the
199  // firehoses.
200  MPI_Barrier(local_group_comm_);
201  }
202  for (auto& fragPtr : frags)
203  {
204  std::cout << "Program::detector: Sending fragment " << fragments_sent + 1 << " of " << fragments_per_source << std::endl;
205  h.sendFragment(std::move(*fragPtr));
206  if (++fragments_sent == fragments_per_source) { break; }
207  if (want_periodic_sync_ && (fragments_sent % 100) == 0)
208  {
209  // Don't get too far out of sync.
210  MPI_Barrier(local_group_comm_);
211  }
212  }
213  frags.clear();
214  }
215  TLOG_DEBUG("builder") << "detector waiting " << conf_.rank_ << TLOG_ENDL;
216  }
217  TLOG_DEBUG("builder") << "detector done " << conf_.rank_ << TLOG_ENDL;
218  MPI_Comm_free(&local_group_comm_);
219  MPI_Barrier(MPI_COMM_WORLD);
220 }
221 
223 {
224  printHost("sink");
225  {
226  // This scope exists to control the lifetime of 'events'
227  int sink_rank;
228  bool useArt = daq_pset_.get<bool>("useArt", false);
229  const char* dummyArgs[1]{"SimpleQueueReader"};
230  MPI_Comm_rank(local_group_comm_, &sink_rank);
232  useArt ?
233  &artapp :
234  &artdaq::simpleQueueReaderApp;
235  artdaq::EventStore events(daq_pset_, conf_.detectors_,
236  conf_.run_,
237  useArt ? conf_.art_argc_ : 1,
238  useArt ? conf_.art_argv_ : const_cast<char**>(dummyArgs),
239  reader);
240  { // Block to handle scope of h, below.
242  h.start_threads();
243  int senderCount = h.enabled_sources().size();
244  while (senderCount > 0)
245  {
246  artdaq::FragmentPtr pfragment(new artdaq::Fragment);
247  int ignoredSource;
248  pfragment = h.recvFragment(ignoredSource);
249  if (!pfragment || ignoredSource == artdaq::TransferInterface::RECV_TIMEOUT) continue;
250  std::cout << "Program::sink: Received fragment " << pfragment->sequenceID() << " from sender " << ignoredSource << std::endl;
251  if (pfragment->type() != artdaq::Fragment::EndOfDataFragmentType)
252  {
253  events.insert(std::move(pfragment));
254  }
255  else
256  {
257  senderCount--;
258  }
259  }
260  }
261  // Make the reader application finish, and capture its return
262  // status.
263  int readerReturnValue;
264  bool endSucceeded = false;
265  int attemptsToEnd = 1;
266  endSucceeded = events.endOfData(readerReturnValue);
267  while (!endSucceeded && attemptsToEnd < 3)
268  {
269  ++attemptsToEnd;
270  endSucceeded = events.endOfData(readerReturnValue);
271  }
272  if (endSucceeded)
273  {
274  TLOG_DEBUG("builder") << "Sink: reader is done, its exit status was: "
275  << readerReturnValue << TLOG_ENDL;
276  }
277  else
278  {
279  TLOG_DEBUG("builder") << "Sink: reader failed to complete because the "
280  << "endOfData marker could not be pushed onto the queue."
281  << TLOG_ENDL;
282  }
283  } // end of lifetime of 'events'
284  TLOG_DEBUG("builder") << "Sink done " << conf_.rank_ << TLOG_ENDL;
285  MPI_Barrier(MPI_COMM_WORLD);
286 }
287 
288 void Builder::printHost(const std::string& functionName) const
289 {
290  char* doPrint = getenv("PRINT_HOST");
291  if (doPrint == 0) { return; }
292  const int ARRSIZE = 80;
293  char hostname[ARRSIZE];
294  std::string hostString;
295  if (!gethostname(hostname, ARRSIZE))
296  {
297  hostString = hostname;
298  }
299  else
300  {
301  hostString = "unknown";
302  }
303  TLOG_DEBUG("builder") << "Running " << functionName
304  << " on host " << hostString
305  << " with rank " << my_rank << "."
306  << TLOG_ENDL;
307 }
308 
309 void printUsage()
310 {
311  int myid = 0;
312  struct rusage usage;
313  getrusage(RUSAGE_SELF, &usage);
314  std::cout << myid << ":"
315  << " user=" << artdaq::Globals::timevalAsDouble(usage.ru_utime)
316  << " sys=" << artdaq::Globals::timevalAsDouble(usage.ru_stime)
317  << std::endl;
318 }
319 
320 int main(int argc, char* argv[])
321 {
323  int rc = 1;
324  try
325  {
326  Builder p(argc, argv);
327  std::cerr << "Started process " << my_rank << " of " << p.procs_ << ".\n";
328  p.go();
329  rc = 0;
330  }
331  catch (std::string& x)
332  {
333  std::cerr << "Exception (type string) caught in driver: "
334  << x
335  << '\n';
336  return 1;
337  }
338  catch (char const* m)
339  {
340  std::cerr << "Exception (type char const*) caught in driver: ";
341  if (m)
342  {
343  std::cerr << m;
344  }
345  else
346  {
347  std::cerr << "[the value was a null pointer, so no message is available]";
348  }
349  std::cerr << '\n';
350  }
351  return rc;
352 }
void insert(FragmentPtr pfrag, bool printWarningWhenFragmentIsDropped=true)
Give ownership of the Fragment to the EventStore.
Definition: EventStore.cc:94
Sends Fragment objects using TransferInterface plugins. Uses Routing Tables if confgiured, otherwise will Round-Robin Fragments to the destinations.
bool endOfData(int &readerReturnValue)
Indicate that the end of input has been reached to the art thread.
Definition: EventStore.cc:247
void go()
Start the Builder application, using the type configuration to select which method to run...
Definition: builder.cc:97
A wrapper for a MPI program. Similar to MPISentry.
Definition: MPIProg.hh:10
void configureMessageFacility(char const *progname, bool useConsole=true)
Configure and start the message facility. Provide the program name so that messages will be appropria...
TaskType type_
Type of this Builder application.
Definition: Config.hh:131
void source()
Receive data from detector via DataReceiverManager, and send to a sink using DataSenderManager.
Definition: builder.cc:129
void detector()
Generate data, and send it using DataSenderManager.
Definition: builder.cc:159
static const int RECV_TIMEOUT
Value to be returned upon receive timeout. Because receivers otherwise return rank, this is also the limit on the number of ranks that artdaq currently supports.
This Builder is a &quot;Detector&quot;.
Definition: Config.hh:36
This Builder is a &quot;Source&quot;.
Definition: Config.hh:35
fhicl::ParameterSet makeParameterSet() const
Write a ParameterSet using configuration.
Definition: Config.cc:148
int rank_
Rank of this application.
Definition: Config.hh:114
int sendFragment(Fragment &&frag)
Send the given Fragment. Return the rank of the destination to which the Fragment was sent...
char ** art_argv_
Arguments used for art.
Definition: Config.hh:136
The EventStore class collects Fragment objects, until it receives a complete event, at which point the event is handed over to the art thread.
Definition: EventStore.hh:48
This Builder is a &quot;Sink&quot;.
Definition: Config.hh:34
Helper class for configuring the builder test.
Definition: Config.hh:21
int( ART_CMDLINE_FCN)(int, char **)
An art function that accepts standard C main arguments.
Definition: EventStore.hh:55
The Builder class runs the builder test.
Definition: builder.cc:39
Receives Fragment objects from one or more DataSenderManager instances using TransferInterface plugin...
void sink()
Receive data from source via DataReceiverManager, send it to the EventStore (and art, if configured)
Definition: builder.cc:222
int run_
Run Number.
Definition: Config.hh:125
int art_argc_
Count of arguments used for art.
Definition: Config.hh:135
void writeInfo() const
Write information about this Config class to a file.
Definition: Config.cc:55
int detectors_
Count of detectors.
Definition: Config.hh:117
static double timevalAsDouble(struct timeval tv)
Convert a timeval value to a double.
Definition: Globals.cc:6
void start_threads()
Start receiver threads for all enabled sources.
Builder(int argc, char *argv[])
Builder Constructor.
Definition: builder.cc:86