artdaq  v3_00_03
builder.cc
1 #include "art/Framework/Art/artapp.h"
2 #include "artdaq-core/Generators/FragmentGenerator.hh"
3 #include "artdaq-core/Data/Fragment.hh"
4 #include "artdaq-core/Generators/makeFragmentGenerator.hh"
5 #include "MPIProg.hh"
6 #include "artdaq/DAQrate/DataSenderManager.hh"
7 #include "artdaq/DAQrate/DataReceiverManager.hh"
8 #include "artdaq-core/Core/SimpleMemoryReader.hh"
10 
11 #include <boost/program_options.hpp>
12 #include "fhiclcpp/make_ParameterSet.h"
13 namespace bpo = boost::program_options;
14 
15 #include <algorithm>
16 #include <cmath>
17 #include <cstdlib>
18 
19 extern "C"
20 {
21 #include <unistd.h>
22 }
23 
24 #include <iostream>
25 #include <memory>
26 #include <utility>
27 
28 extern "C"
29 {
30 #include <sys/time.h>
31 #include <sys/resource.h>
32 }
33 
37 class Builder : public MPIProg
38 {
39 public:
47  Builder(int argc, char* argv[], fhicl::ParameterSet pset, int key);
48 
52  void go();
53 
57  void sink();
58 
62  void detector();
63 
64 private:
65  enum class Role : int
66  {
67  DETECTOR,
68  SINK
69  };
70 
71  void printHost(const std::string& functionName) const;
72 
73  fhicl::ParameterSet daq_pset_;
74  bool const want_sink_;
75  bool const want_periodic_sync_;
76  MPI_Comm local_group_comm_;
77  Role builder_role_;
78 };
79 
80 Builder::Builder(int argc, char* argv[], fhicl::ParameterSet pset, int key) :
81  MPIProg(argc, argv)
82  , daq_pset_(pset)
83  , want_sink_(daq_pset_.get<bool>("want_sink", true))
84  , want_periodic_sync_(daq_pset_.get<bool>("want_periodic_sync", false))
85  , local_group_comm_()
86 {
87  std::vector<std::string> detectors;
88  daq_pset_.get_if_present("detectors", detectors);
89  if (static_cast<size_t>(my_rank) >= detectors.size())
90  {
91  builder_role_ = Role::SINK;
92  }
93  else
94  {
95  builder_role_ = Role::DETECTOR;
96  }
97  std::string type(pset.get<std::string>("transfer_plugin_type", "Shmem"));
98 
99  int senders = pset.get<int>("num_senders");
100  int receivers = pset.get<int>("num_receivers");
101  int buffer_count = pset.get<int>("buffer_count", 10);
102  int max_payload_size = pset.get<size_t>("fragment_size", 0x100000);
103 
104  std::string hostmap = "";
105  if (pset.has_key("hostmap"))
106  {
107  hostmap = " host_map: @local::hostmap";
108  }
109 
110  std::stringstream ss;
111  ss << pset.to_string();
112  ss << " sources: {";
113  for (int ii = 0; ii < senders; ++ii)
114  {
115  ss << "s" << ii << ": { transferPluginType: " << type << " source_rank: " << ii << " max_fragment_size_words: " << max_payload_size << " buffer_count: " << buffer_count << " shm_key_offset: " << std::to_string(key) << hostmap << "}";
116  }
117  ss << "} destinations: {";
118  for (int jj = senders; jj < senders + receivers; ++jj)
119  {
120  ss << "d" << jj << ": { transferPluginType: " << type << " destination_rank: " << jj << " max_fragment_size_words: " << max_payload_size << " buffer_count: " << buffer_count << " shm_key_offset: " << std::to_string(key) << hostmap << "}";
121  }
122  ss << "}";
123 
124  daq_pset_ = fhicl::ParameterSet();
125  make_ParameterSet(ss.str(), daq_pset_);
126 
127 
128 }
129 
131 {
132  //volatile bool loopForever = true;
133  //while(loopForever)
134  //{
135  // usleep(1000000);
136  //}
137 
138 
139  MPI_Barrier(MPI_COMM_WORLD);
140  //std::cout << "daq_pset_: " << daq_pset_.to_string() << std::endl << "conf_.makeParameterSet(): " << conf_.makeParameterSet().to_string() << std::endl;
141  MPI_Comm_split(MPI_COMM_WORLD, static_cast<int>(builder_role_), 0, &local_group_comm_);
142  switch (builder_role_)
143  {
144  case Role::SINK:
145  if (want_sink_)
146  {
147  sink();
148  }
149  else
150  {
151  std::string
152  msg("WARNING: a sink was instantiated despite want_sink being false:\n"
153  "set nsinks to 0 in invocation of daqrate?\n");
154  std::cerr << msg;
155  MPI_Barrier(MPI_COMM_WORLD);
156  }
157  break;
158  case Role::DETECTOR:
159  detector();
160  break;
161  default:
162  throw "No such node type";
163  }
164 }
165 
167 {
168  printHost("detector");
169  int detector_rank;
170  // Should be zero-based, detectors only.
171  MPI_Comm_rank(local_group_comm_, &detector_rank);
172  assert(!(detector_rank < 0));
173  std::ostringstream det_ps_name_loc;
174  std::vector<std::string> detectors;
175  bool detectors_present = daq_pset_.get_if_present("detectors", detectors);
176  size_t detectors_size = detectors.size();
177  if (!(detectors_present && detectors_size))
178  {
179  throw cet::exception("Configuration")
180  << "Unable to find required sequence of detector "
181  << "parameter set names, \"detectors\".";
182  }
183  fhicl::ParameterSet det_ps =
184  daq_pset_.get<fhicl::ParameterSet>(((detectors_size > static_cast<size_t>(detector_rank)) ? detectors[detector_rank] : detectors[0]));
185  std::unique_ptr<artdaq::FragmentGenerator> const
186  gen(artdaq::makeFragmentGenerator
187  (det_ps.get<std::string>("generator"),
188  det_ps));
189  { // Block to handle lifetime of h, below.
190  artdaq::DataSenderManager h(daq_pset_);
191  MPI_Barrier(local_group_comm_);
192  // not using the run time method
193  // TimedLoop tl(conf_.run_time_);
194  size_t fragments_per_source = -1;
195  daq_pset_.get_if_present("fragments_per_source", fragments_per_source);
196  artdaq::FragmentPtrs frags;
197  size_t fragments_sent = 0;
198  while (fragments_sent < fragments_per_source && gen->getNext(frags))
199  {
200  if (!fragments_sent)
201  {
202  // Get the detectors lined up first time before we start the
203  // firehoses.
204  MPI_Barrier(local_group_comm_);
205  }
206  for (auto& fragPtr : frags)
207  {
208  std::cout << "Program::detector: Sending fragment " << fragments_sent + 1 << " of " << fragments_per_source << std::endl;
209  TLOG_DEBUG("builder") << "Program::detector: Sending fragment " << fragments_sent + 1 << " of " << fragments_per_source << TLOG_ENDL;
210  h.sendFragment(std::move(*fragPtr));
211  if (++fragments_sent == fragments_per_source) { break; }
212  if (want_periodic_sync_ && (fragments_sent % 100) == 0)
213  {
214  // Don't get too far out of sync.
215  MPI_Barrier(local_group_comm_);
216  }
217  }
218  frags.clear();
219  }
220  TLOG_DEBUG("builder") << "detector waiting " << my_rank << TLOG_ENDL;
221  }
222  TLOG_DEBUG("builder") << "detector done " << my_rank << TLOG_ENDL;
223  MPI_Comm_free(&local_group_comm_);
224  MPI_Barrier(MPI_COMM_WORLD);
225 }
226 
228 {
229  printHost("sink");
230  {
231  usleep(1000 * my_rank);
232  // This scope exists to control the lifetime of 'events'
233  auto events = std::make_shared<artdaq::SharedMemoryEventManager>(daq_pset_, daq_pset_);
234  events->startRun(daq_pset_.get<int>("run_number", 100));
235  { // Block to handle scope of h, below.
236  artdaq::DataReceiverManager h(daq_pset_, events);
237  h.start_threads();
238  while (h.running_sources().size() > 0)
239  {
240  usleep(10000);
241  }
242  }
243 
244  TLOG_DEBUG("builder") << "All detectors are done, Sending endOfData Fragment" << TLOG_ENDL;
245  // Make the reader application finish, and capture its return
246  // status.
247  bool endSucceeded = false;
248  endSucceeded = events->endOfData();
249  if (endSucceeded)
250  {
251  TLOG_DEBUG("builder") << "Sink: reader is done" << TLOG_ENDL;
252  }
253  else
254  {
255  TLOG_DEBUG("builder") << "Sink: reader failed to complete because the "
256  << "endOfData marker could not be pushed onto the queue."
257  << TLOG_ENDL;
258  }
259  } // end of lifetime of 'events'
260  TLOG_DEBUG("builder") << "Sink done " << my_rank << TLOG_ENDL;
261  MPI_Barrier(MPI_COMM_WORLD);
262 }
263 
264 void Builder::printHost(const std::string& functionName) const
265 {
266  char* doPrint = getenv("PRINT_HOST");
267  if (doPrint == 0) { return; }
268  const int ARRSIZE = 80;
269  char hostname[ARRSIZE];
270  std::string hostString;
271  if (!gethostname(hostname, ARRSIZE))
272  {
273  hostString = hostname;
274  }
275  else
276  {
277  hostString = "unknown";
278  }
279  TLOG_DEBUG("builder") << "Running " << functionName
280  << " on host " << hostString
281  << " with rank " << my_rank << "."
282  << TLOG_ENDL;
283 }
284 
285 void printUsage()
286 {
287  int myid = 0;
288  struct rusage usage;
289  getrusage(RUSAGE_SELF, &usage);
290  std::cout << myid << ":"
291  << " user=" << artdaq::TimeUtils::convertUnixTimeToSeconds(usage.ru_utime)
292  << " sys=" << artdaq::TimeUtils::convertUnixTimeToSeconds(usage.ru_stime)
293  << std::endl;
294 }
295 
296 int main(int argc, char* argv[])
297 {
298  artdaq::configureMessageFacility("builder");
299 
300  std::ostringstream descstr;
301  descstr << argv[0]
302  << " <-c <config-file>> <other-options> [<source-file>]+";
303  bpo::options_description desc(descstr.str());
304  desc.add_options()
305  ("config,c", bpo::value<std::string>(), "Configuration file.")
306  ("key,k", bpo::value<int>(), "Shared Memory Key")
307  ("help,h", "produce help message");
308  bpo::variables_map vm;
309  try {
310  bpo::store(bpo::command_line_parser(argc, argv).options(desc).run(), vm);
311  bpo::notify(vm);
312  }
313  catch (bpo::error const & e) {
314  std::cerr << "Exception from command line processing in " << argv[0]
315  << ": " << e.what() << "\n";
316  return -1;
317  }
318  if (vm.count("help")) {
319  std::cout << desc << std::endl;
320  return 1;
321  }
322  if (!vm.count("config")) {
323  std::cerr << "Exception from command line processing in " << argv[0]
324  << ": no configuration file given.\n"
325  << "For usage and an options list, please do '"
326  << argv[0] << " --help"
327  << "'.\n";
328  return 2;
329  }
330  int key = 0;
331  if (vm.count("key"))
332  {
333  key = vm["key"].as<int>();
334  }
335  fhicl::ParameterSet pset;
336  if (getenv("FHICL_FILE_PATH") == nullptr) {
337  std::cerr
338  << "INFO: environment variable FHICL_FILE_PATH was not set. Using \".\"\n";
339  setenv("FHICL_FILE_PATH", ".", 0);
340  }
341  cet::filepath_lookup_after1 lookup_policy("FHICL_FILE_PATH");
342  fhicl::make_ParameterSet(vm["config"].as<std::string>(), lookup_policy, pset);
343 
344  int rc = 1;
345  try
346  {
347  Builder p(argc, argv, pset,key);
348  std::cerr << "Started process " << my_rank << " of " << p.procs_ << ".\n";
349  p.go();
350  rc = 0;
351  }
352  catch (std::string& x)
353  {
354  std::cerr << "Exception (type string) caught in driver: "
355  << x
356  << '\n';
357  return 1;
358  }
359  catch (char const* m)
360  {
361  std::cerr << "Exception (type char const*) caught in driver: ";
362  if (m)
363  {
364  std::cerr << m;
365  }
366  else
367  {
368  std::cerr << "[the value was a null pointer, so no message is available]";
369  }
370  std::cerr << '\n';
371  }
372  return rc;
373 }
Sends Fragment objects using TransferInterface plugins. Uses Routing Tables if confgiured, otherwise will Round-Robin Fragments to the destinations.
void go()
Start the Builder application, using the type configuration to select which method to run...
Definition: builder.cc:130
A wrapper for a MPI program. Similar to MPISentry.
Definition: MPIProg.hh:10
void detector()
Generate data, and send it using DataSenderManager.
Definition: builder.cc:166
int sendFragment(Fragment &&frag)
Send the given Fragment. Return the rank of the destination to which the Fragment was sent...
The Builder class runs the builder test.
Definition: builder.cc:37
Receives Fragment objects from one or more DataSenderManager instances using TransferInterface plugin...
void sink()
Receive data from source via DataReceiverManager, send it to the EventStore (and art, if configured)
Definition: builder.cc:227
std::set< int > running_sources() const
Get the list of sources which are still receiving data.
Builder(int argc, char *argv[], fhicl::ParameterSet pset, int key)
Builder Constructor.
Definition: builder.cc:80
void start_threads()
Start receiver threads for all enabled sources.