artdaq_mpich_plugin  v1_00_13
builder.cc
1 #define TRACE_NAME "builder"
2 
3 #include "MPIProg.hh"
4 #include "art/Framework/Art/artapp.h"
5 #include "artdaq-core/Core/SimpleMemoryReader.hh"
6 #include "artdaq-core/Data/Fragment.hh"
7 #include "artdaq-core/Generators/FragmentGenerator.hh"
8 #include "artdaq-core/Generators/makeFragmentGenerator.hh"
10 #include "artdaq/DAQrate/DataReceiverManager.hh"
11 #include "artdaq/DAQrate/DataSenderManager.hh"
12 
13 #include <boost/program_options.hpp>
14 #include "fhiclcpp/make_ParameterSet.h"
15 namespace bpo = boost::program_options;
16 
17 #include <algorithm>
18 #include <cmath>
19 #include <cstdlib>
20 
21 extern "C" {
22 #include <unistd.h>
23 }
24 
25 #include <iostream>
26 #include <memory>
27 #include <utility>
28 
29 extern "C" {
30 #include <sys/resource.h>
31 #include <sys/time.h>
32 }
33 
37 class Builder : public MPIProg
38 {
39 public:
47  Builder(int argc, char* argv[], fhicl::ParameterSet pset, int key);
48 
52  void go();
53 
57  void sink();
58 
62  void detector();
63 
64 private:
65  enum class Role : int
66  {
67  DETECTOR,
68  SINK
69  };
70 
71  void printHost(const std::string& functionName) const;
72 
73  fhicl::ParameterSet daq_pset_;
74  bool const want_sink_;
75  bool const want_periodic_sync_;
76  MPI_Comm local_group_comm_;
77  Role builder_role_;
78 };
79 
80 Builder::Builder(int argc, char* argv[], fhicl::ParameterSet pset, int key)
81  : MPIProg(argc, argv), daq_pset_(pset), want_sink_(daq_pset_.get<bool>("want_sink", true)), want_periodic_sync_(daq_pset_.get<bool>("want_periodic_sync", false)), local_group_comm_()
82 {
83  std::vector<std::string> detectors;
84  daq_pset_.get_if_present("detectors", detectors);
85  if (static_cast<size_t>(my_rank) >= detectors.size())
86  {
87  builder_role_ = Role::SINK;
88  }
89  else
90  {
91  builder_role_ = Role::DETECTOR;
92  }
93  std::string type(pset.get<std::string>("transfer_plugin_type", "Shmem"));
94 
95  int senders = pset.get<int>("num_senders");
96  int receivers = pset.get<int>("num_receivers");
97  int buffer_count = pset.get<int>("buffer_count", 10);
98  int max_payload_size = pset.get<size_t>("fragment_size", 0x100000);
99 
100  std::string hostmap = "";
101  if (pset.has_key("hostmap"))
102  {
103  hostmap = " host_map: @local::hostmap";
104  }
105 
106  std::stringstream ss;
107  ss << pset.to_string();
108  ss << " sources: {";
109  for (int ii = 0; ii < senders; ++ii)
110  {
111  ss << "s" << ii << ": { transferPluginType: " << type << " source_rank: " << ii
112  << " max_fragment_size_words: " << max_payload_size << " buffer_count: " << buffer_count
113  << " shm_key_offset: " << std::to_string(key) << hostmap << "}";
114  }
115  ss << "} destinations: {";
116  for (int jj = senders; jj < senders + receivers; ++jj)
117  {
118  ss << "d" << jj << ": { transferPluginType: " << type << " destination_rank: " << jj
119  << " max_fragment_size_words: " << max_payload_size << " buffer_count: " << buffer_count
120  << " shm_key_offset: " << std::to_string(key) << hostmap << "}";
121  }
122  ss << "}";
123 
124  daq_pset_ = fhicl::ParameterSet();
125  make_ParameterSet(ss.str(), daq_pset_);
126 }
127 
129 {
130  // volatile bool loopForever = true;
131  // while(loopForever)
132  //{
133  // usleep(1000000);
134  //}
135 
136  MPI_Barrier(MPI_COMM_WORLD);
137  // std::cout << "daq_pset_: " << daq_pset_.to_string() << std::endl << "conf_.makeParameterSet(): " <<
138  // conf_.makeParameterSet().to_string() << std::endl;
139  MPI_Comm_split(MPI_COMM_WORLD, static_cast<int>(builder_role_), 0, &local_group_comm_);
140  switch (builder_role_)
141  {
142  case Role::SINK:
143  if (want_sink_)
144  {
145  sink();
146  }
147  else
148  {
149  std::string msg(
150  "WARNING: a sink was instantiated despite want_sink being false:\n"
151  "set nsinks to 0 in invocation of daqrate?\n");
152  std::cerr << msg;
153  MPI_Barrier(MPI_COMM_WORLD);
154  }
155  break;
156  case Role::DETECTOR:
157  detector();
158  break;
159  default:
160  throw "No such node type";
161  }
162 }
163 
165 {
166  printHost("detector");
167  int detector_rank;
168  // Should be zero-based, detectors only.
169  MPI_Comm_rank(local_group_comm_, &detector_rank);
170  assert(!(detector_rank < 0));
171  std::ostringstream det_ps_name_loc;
172  std::vector<std::string> detectors;
173  bool detectors_present = daq_pset_.get_if_present("detectors", detectors);
174  size_t detectors_size = detectors.size();
175  if (!(detectors_present && detectors_size))
176  {
177  throw cet::exception("Configuration") << "Unable to find required sequence of detector "
178  << "parameter set names, \"detectors\".";
179  }
180  fhicl::ParameterSet det_ps = daq_pset_.get<fhicl::ParameterSet>(
181  ((detectors_size > static_cast<size_t>(detector_rank)) ? detectors[detector_rank] : detectors[0]));
182  std::unique_ptr<artdaq::FragmentGenerator> const gen(
183  artdaq::makeFragmentGenerator(det_ps.get<std::string>("generator"), det_ps));
184  { // Block to handle lifetime of h, below.
185  artdaq::DataSenderManager h(daq_pset_);
186  MPI_Barrier(local_group_comm_);
187  // not using the run time method
188  // TimedLoop tl(conf_.run_time_);
189  size_t fragments_per_source = -1;
190  daq_pset_.get_if_present("fragments_per_source", fragments_per_source);
191  artdaq::FragmentPtrs frags;
192  size_t fragments_sent = 0;
193  while (fragments_sent < fragments_per_source && gen->getNext(frags))
194  {
195  if (!fragments_sent)
196  {
197  // Get the detectors lined up first time before we start the
198  // firehoses.
199  MPI_Barrier(local_group_comm_);
200  }
201  for (auto& fragPtr : frags)
202  {
203  std::cout << "Program::detector: Sending fragment " << fragments_sent + 1 << " of " << fragments_per_source
204  << std::endl;
205  TLOG(TLVL_DEBUG) << "Program::detector: Sending fragment " << fragments_sent + 1 << " of "
206  << fragments_per_source;
207  auto sequence_id = fragPtr->sequenceID();
208  h.sendFragment(std::move(*fragPtr));
209  ;
210  if (h.GetSentSequenceIDCount(sequence_id) == gen->fragmentIDs().size())
211  {
212  h.RemoveRoutingTableEntry(sequence_id);
213  }
214 
215  if (++fragments_sent == fragments_per_source)
216  {
217  break;
218  }
219  if (want_periodic_sync_ && (fragments_sent % 100) == 0)
220  {
221  // Don't get too far out of sync.
222  MPI_Barrier(local_group_comm_);
223  }
224  }
225  frags.clear();
226  }
227  TLOG(TLVL_DEBUG) << "detector waiting " << my_rank;
228  }
229  TLOG(TLVL_DEBUG) << "detector done " << my_rank;
230  MPI_Comm_free(&local_group_comm_);
231  MPI_Barrier(MPI_COMM_WORLD);
232 }
233 
235 {
236  printHost("sink");
237  {
238  usleep(1000 * my_rank);
239  // This scope exists to control the lifetime of 'events'
240  auto events = std::make_shared<artdaq::SharedMemoryEventManager>(daq_pset_, daq_pset_);
241  events->startRun(daq_pset_.get<int>("run_number", 100));
242  { // Block to handle scope of h, below.
243  artdaq::DataReceiverManager h(daq_pset_, events);
244  h.start_threads();
245  while (h.running_sources().size() > 0)
246  {
247  usleep(10000);
248  }
249  }
250 
251  TLOG(TLVL_DEBUG) << "All detectors are done, Sending endOfData Fragment";
252  // Make the reader application finish, and capture its return
253  // status.
254  bool endSucceeded = false;
255  endSucceeded = events->endOfData();
256  if (endSucceeded)
257  {
258  TLOG(TLVL_DEBUG) << "Sink: reader is done";
259  }
260  else
261  {
262  TLOG(TLVL_DEBUG) << "Sink: reader failed to complete because the "
263  << "endOfData marker could not be pushed onto the queue.";
264  }
265  } // end of lifetime of 'events'
266  TLOG(TLVL_DEBUG) << "Sink done " << my_rank;
267  MPI_Barrier(MPI_COMM_WORLD);
268 }
269 
270 void Builder::printHost(const std::string& functionName) const
271 {
272  char* doPrint = getenv("PRINT_HOST");
273  if (doPrint == 0)
274  {
275  return;
276  }
277  const int ARRSIZE = 80;
278  char hostname[ARRSIZE];
279  std::string hostString;
280  if (!gethostname(hostname, ARRSIZE))
281  {
282  hostString = hostname;
283  }
284  else
285  {
286  hostString = "unknown";
287  }
288  TLOG(TLVL_DEBUG) << "Running " << functionName << " on host " << hostString << " with rank " << my_rank << ".";
289 }
290 
291 void printUsage()
292 {
293  int myid = 0;
294  struct rusage usage;
295  getrusage(RUSAGE_SELF, &usage);
296  std::cout << myid << ":"
297  << " user=" << artdaq::TimeUtils::convertUnixTimeToSeconds(usage.ru_utime)
298  << " sys=" << artdaq::TimeUtils::convertUnixTimeToSeconds(usage.ru_stime) << std::endl;
299 }
300 
301 int main(int argc, char* argv[])
302 {
303  artdaq::configureMessageFacility("builder");
304 
305  std::ostringstream descstr;
306  descstr << argv[0] << " <-c <config-file>> <other-options> [<source-file>]+";
307  bpo::options_description desc(descstr.str());
308  desc.add_options()("config,c", bpo::value<std::string>(), "Configuration file.")(
309  "key,k", bpo::value<int>(), "Shared Memory Key")("help,h", "produce help message");
310  bpo::variables_map vm;
311  try
312  {
313  bpo::store(bpo::command_line_parser(argc, argv).options(desc).run(), vm);
314  bpo::notify(vm);
315  }
316  catch (bpo::error const& e)
317  {
318  std::cerr << "Exception from command line processing in " << argv[0] << ": " << e.what() << "\n";
319  return -1;
320  }
321  if (vm.count("help"))
322  {
323  std::cout << desc << std::endl;
324  return 1;
325  }
326  if (!vm.count("config"))
327  {
328  std::cerr << "Exception from command line processing in " << argv[0] << ": no configuration file given.\n"
329  << "For usage and an options list, please do '" << argv[0] << " --help"
330  << "'.\n";
331  return 2;
332  }
333  int key = 0;
334  if (vm.count("key"))
335  {
336  key = vm["key"].as<int>();
337  }
338  fhicl::ParameterSet pset;
339  if (getenv("FHICL_FILE_PATH") == nullptr)
340  {
341  std::cerr << "INFO: environment variable FHICL_FILE_PATH was not set. Using \".\"\n";
342  setenv("FHICL_FILE_PATH", ".", 0);
343  }
344  cet::filepath_lookup_after1 lookup_policy("FHICL_FILE_PATH");
345  fhicl::make_ParameterSet(vm["config"].as<std::string>(), lookup_policy, pset);
346 
347  int rc = 1;
348  try
349  {
350  Builder p(argc, argv, pset, key);
351  std::cerr << "Started process " << my_rank << " of " << p.procs_ << ".\n";
352  p.go();
353  rc = 0;
354  }
355  catch (std::string& x)
356  {
357  std::cerr << "Exception (type string) caught in driver: " << x << '\n';
358  return 1;
359  }
360  catch (char const* m)
361  {
362  std::cerr << "Exception (type char const*) caught in driver: ";
363  if (m)
364  {
365  std::cerr << m;
366  }
367  else
368  {
369  std::cerr << "[the value was a null pointer, so no message is available]";
370  }
371  std::cerr << '\n';
372  }
373  return rc;
374 }
void go()
Start the Builder application, using the type configuration to select which method to run...
Definition: builder.cc:128
A wrapper for a MPI program. Similar to MPISentry.
Definition: MPIProg.hh:10
void detector()
Generate data, and send it using DataSenderManager.
Definition: builder.cc:164
The Builder class runs the builder test.
Definition: builder.cc:37
void sink()
Receive data from source via DataReceiverManager, send it to the EventStore (and art, if configured)
Definition: builder.cc:234
Builder(int argc, char *argv[], fhicl::ParameterSet pset, int key)
Builder Constructor.
Definition: builder.cc:80