artdaq_mpich_plugin  v1_00_08a
builder.cc
1 #define TRACE_NAME "builder"
2 
3 #include "MPIProg.hh"
4 #include "art/Framework/Art/artapp.h"
5 #include "artdaq-core/Core/SimpleMemoryReader.hh"
6 #include "artdaq-core/Data/Fragment.hh"
7 #include "artdaq-core/Generators/FragmentGenerator.hh"
8 #include "artdaq-core/Generators/makeFragmentGenerator.hh"
10 #include "artdaq/DAQrate/DataReceiverManager.hh"
11 #include "artdaq/DAQrate/DataSenderManager.hh"
12 
13 #include <boost/program_options.hpp>
14 #include "fhiclcpp/make_ParameterSet.h"
15 namespace bpo = boost::program_options;
16 
17 #include <algorithm>
18 #include <cmath>
19 #include <cstdlib>
20 
21 extern "C" {
22 #include <unistd.h>
23 }
24 
25 #include <iostream>
26 #include <memory>
27 #include <utility>
28 
29 extern "C" {
30 #include <sys/resource.h>
31 #include <sys/time.h>
32 }
33 
37 class Builder : public MPIProg {
38  public:
46  Builder(int argc, char* argv[], fhicl::ParameterSet pset, int key);
47 
51  void go();
52 
56  void sink();
57 
61  void detector();
62 
63  private:
64  enum class Role : int { DETECTOR, SINK };
65 
66  void printHost(const std::string& functionName) const;
67 
68  fhicl::ParameterSet daq_pset_;
69  bool const want_sink_;
70  bool const want_periodic_sync_;
71  MPI_Comm local_group_comm_;
72  Role builder_role_;
73 };
74 
75 Builder::Builder(int argc, char* argv[], fhicl::ParameterSet pset, int key)
76  : MPIProg(argc, argv),
77  daq_pset_(pset),
78  want_sink_(daq_pset_.get<bool>("want_sink", true)),
79  want_periodic_sync_(daq_pset_.get<bool>("want_periodic_sync", false)),
80  local_group_comm_() {
81  std::vector<std::string> detectors;
82  daq_pset_.get_if_present("detectors", detectors);
83  if (static_cast<size_t>(my_rank) >= detectors.size()) {
84  builder_role_ = Role::SINK;
85  } else {
86  builder_role_ = Role::DETECTOR;
87  }
88  std::string type(pset.get<std::string>("transfer_plugin_type", "Shmem"));
89 
90  int senders = pset.get<int>("num_senders");
91  int receivers = pset.get<int>("num_receivers");
92  int buffer_count = pset.get<int>("buffer_count", 10);
93  int max_payload_size = pset.get<size_t>("fragment_size", 0x100000);
94 
95  std::string hostmap = "";
96  if (pset.has_key("hostmap")) {
97  hostmap = " host_map: @local::hostmap";
98  }
99 
100  std::stringstream ss;
101  ss << pset.to_string();
102  ss << " sources: {";
103  for (int ii = 0; ii < senders; ++ii) {
104  ss << "s" << ii << ": { transferPluginType: " << type << " source_rank: " << ii
105  << " max_fragment_size_words: " << max_payload_size << " buffer_count: " << buffer_count
106  << " shm_key_offset: " << std::to_string(key) << hostmap << "}";
107  }
108  ss << "} destinations: {";
109  for (int jj = senders; jj < senders + receivers; ++jj) {
110  ss << "d" << jj << ": { transferPluginType: " << type << " destination_rank: " << jj
111  << " max_fragment_size_words: " << max_payload_size << " buffer_count: " << buffer_count
112  << " shm_key_offset: " << std::to_string(key) << hostmap << "}";
113  }
114  ss << "}";
115 
116  daq_pset_ = fhicl::ParameterSet();
117  make_ParameterSet(ss.str(), daq_pset_);
118 }
119 
120 void Builder::go() {
121  // volatile bool loopForever = true;
122  // while(loopForever)
123  //{
124  // usleep(1000000);
125  //}
126 
127  MPI_Barrier(MPI_COMM_WORLD);
128  // std::cout << "daq_pset_: " << daq_pset_.to_string() << std::endl << "conf_.makeParameterSet(): " <<
129  // conf_.makeParameterSet().to_string() << std::endl;
130  MPI_Comm_split(MPI_COMM_WORLD, static_cast<int>(builder_role_), 0, &local_group_comm_);
131  switch (builder_role_) {
132  case Role::SINK:
133  if (want_sink_) {
134  sink();
135  } else {
136  std::string msg(
137  "WARNING: a sink was instantiated despite want_sink being false:\n"
138  "set nsinks to 0 in invocation of daqrate?\n");
139  std::cerr << msg;
140  MPI_Barrier(MPI_COMM_WORLD);
141  }
142  break;
143  case Role::DETECTOR:
144  detector();
145  break;
146  default:
147  throw "No such node type";
148  }
149 }
150 
152  printHost("detector");
153  int detector_rank;
154  // Should be zero-based, detectors only.
155  MPI_Comm_rank(local_group_comm_, &detector_rank);
156  assert(!(detector_rank < 0));
157  std::ostringstream det_ps_name_loc;
158  std::vector<std::string> detectors;
159  bool detectors_present = daq_pset_.get_if_present("detectors", detectors);
160  size_t detectors_size = detectors.size();
161  if (!(detectors_present && detectors_size)) {
162  throw cet::exception("Configuration") << "Unable to find required sequence of detector "
163  << "parameter set names, \"detectors\".";
164  }
165  fhicl::ParameterSet det_ps = daq_pset_.get<fhicl::ParameterSet>(
166  ((detectors_size > static_cast<size_t>(detector_rank)) ? detectors[detector_rank] : detectors[0]));
167  std::unique_ptr<artdaq::FragmentGenerator> const gen(
168  artdaq::makeFragmentGenerator(det_ps.get<std::string>("generator"), det_ps));
169  { // Block to handle lifetime of h, below.
170  artdaq::DataSenderManager h(daq_pset_);
171  MPI_Barrier(local_group_comm_);
172  // not using the run time method
173  // TimedLoop tl(conf_.run_time_);
174  size_t fragments_per_source = -1;
175  daq_pset_.get_if_present("fragments_per_source", fragments_per_source);
176  artdaq::FragmentPtrs frags;
177  size_t fragments_sent = 0;
178  while (fragments_sent < fragments_per_source && gen->getNext(frags)) {
179  if (!fragments_sent) {
180  // Get the detectors lined up first time before we start the
181  // firehoses.
182  MPI_Barrier(local_group_comm_);
183  }
184  for (auto& fragPtr : frags) {
185  std::cout << "Program::detector: Sending fragment " << fragments_sent + 1 << " of " << fragments_per_source
186  << std::endl;
187  TLOG(TLVL_DEBUG) << "Program::detector: Sending fragment " << fragments_sent + 1 << " of "
188  << fragments_per_source;
189  auto sequence_id = fragPtr->sequenceID();
190  h.sendFragment(std::move(*fragPtr));
191  ;
192  if (h.GetSentSequenceIDCount(sequence_id) == gen->fragmentIDs().size()) {
193  h.RemoveRoutingTableEntry(sequence_id);
194  }
195 
196  if (++fragments_sent == fragments_per_source) {
197  break;
198  }
199  if (want_periodic_sync_ && (fragments_sent % 100) == 0) {
200  // Don't get too far out of sync.
201  MPI_Barrier(local_group_comm_);
202  }
203  }
204  frags.clear();
205  }
206  TLOG(TLVL_DEBUG) << "detector waiting " << my_rank;
207  }
208  TLOG(TLVL_DEBUG) << "detector done " << my_rank;
209  MPI_Comm_free(&local_group_comm_);
210  MPI_Barrier(MPI_COMM_WORLD);
211 }
212 
214  printHost("sink");
215  {
216  usleep(1000 * my_rank);
217  // This scope exists to control the lifetime of 'events'
218  auto events = std::make_shared<artdaq::SharedMemoryEventManager>(daq_pset_, daq_pset_);
219  events->startRun(daq_pset_.get<int>("run_number", 100));
220  { // Block to handle scope of h, below.
221  artdaq::DataReceiverManager h(daq_pset_, events);
222  h.start_threads();
223  while (h.running_sources().size() > 0) {
224  usleep(10000);
225  }
226  }
227 
228  TLOG(TLVL_DEBUG) << "All detectors are done, Sending endOfData Fragment";
229  // Make the reader application finish, and capture its return
230  // status.
231  bool endSucceeded = false;
232  endSucceeded = events->endOfData();
233  if (endSucceeded) {
234  TLOG(TLVL_DEBUG) << "Sink: reader is done";
235  } else {
236  TLOG(TLVL_DEBUG) << "Sink: reader failed to complete because the "
237  << "endOfData marker could not be pushed onto the queue.";
238  }
239  } // end of lifetime of 'events'
240  TLOG(TLVL_DEBUG) << "Sink done " << my_rank;
241  MPI_Barrier(MPI_COMM_WORLD);
242 }
243 
244 void Builder::printHost(const std::string& functionName) const {
245  char* doPrint = getenv("PRINT_HOST");
246  if (doPrint == 0) {
247  return;
248  }
249  const int ARRSIZE = 80;
250  char hostname[ARRSIZE];
251  std::string hostString;
252  if (!gethostname(hostname, ARRSIZE)) {
253  hostString = hostname;
254  } else {
255  hostString = "unknown";
256  }
257  TLOG(TLVL_DEBUG) << "Running " << functionName << " on host " << hostString << " with rank " << my_rank << ".";
258 }
259 
260 void printUsage() {
261  int myid = 0;
262  struct rusage usage;
263  getrusage(RUSAGE_SELF, &usage);
264  std::cout << myid << ":"
265  << " user=" << artdaq::TimeUtils::convertUnixTimeToSeconds(usage.ru_utime)
266  << " sys=" << artdaq::TimeUtils::convertUnixTimeToSeconds(usage.ru_stime) << std::endl;
267 }
268 
269 int main(int argc, char* argv[]) {
270  artdaq::configureMessageFacility("builder");
271 
272  std::ostringstream descstr;
273  descstr << argv[0] << " <-c <config-file>> <other-options> [<source-file>]+";
274  bpo::options_description desc(descstr.str());
275  desc.add_options()("config,c", bpo::value<std::string>(), "Configuration file.")(
276  "key,k", bpo::value<int>(), "Shared Memory Key")("help,h", "produce help message");
277  bpo::variables_map vm;
278  try {
279  bpo::store(bpo::command_line_parser(argc, argv).options(desc).run(), vm);
280  bpo::notify(vm);
281  } catch (bpo::error const& e) {
282  std::cerr << "Exception from command line processing in " << argv[0] << ": " << e.what() << "\n";
283  return -1;
284  }
285  if (vm.count("help")) {
286  std::cout << desc << std::endl;
287  return 1;
288  }
289  if (!vm.count("config")) {
290  std::cerr << "Exception from command line processing in " << argv[0] << ": no configuration file given.\n"
291  << "For usage and an options list, please do '" << argv[0] << " --help"
292  << "'.\n";
293  return 2;
294  }
295  int key = 0;
296  if (vm.count("key")) {
297  key = vm["key"].as<int>();
298  }
299  fhicl::ParameterSet pset;
300  if (getenv("FHICL_FILE_PATH") == nullptr) {
301  std::cerr << "INFO: environment variable FHICL_FILE_PATH was not set. Using \".\"\n";
302  setenv("FHICL_FILE_PATH", ".", 0);
303  }
304  cet::filepath_lookup_after1 lookup_policy("FHICL_FILE_PATH");
305  fhicl::make_ParameterSet(vm["config"].as<std::string>(), lookup_policy, pset);
306 
307  int rc = 1;
308  try {
309  Builder p(argc, argv, pset, key);
310  std::cerr << "Started process " << my_rank << " of " << p.procs_ << ".\n";
311  p.go();
312  rc = 0;
313  } catch (std::string& x) {
314  std::cerr << "Exception (type string) caught in driver: " << x << '\n';
315  return 1;
316  } catch (char const* m) {
317  std::cerr << "Exception (type char const*) caught in driver: ";
318  if (m) {
319  std::cerr << m;
320  } else {
321  std::cerr << "[the value was a null pointer, so no message is available]";
322  }
323  std::cerr << '\n';
324  }
325  return rc;
326 }
void go()
Start the Builder application, using the type configuration to select which method to run...
Definition: builder.cc:120
A wrapper for a MPI program. Similar to MPISentry.
Definition: MPIProg.hh:10
void detector()
Generate data, and send it using DataSenderManager.
Definition: builder.cc:151
The Builder class runs the builder test.
Definition: builder.cc:37
void sink()
Receive data from source via DataReceiverManager, send it to the EventStore (and art, if configured)
Definition: builder.cc:213
Builder(int argc, char *argv[], fhicl::ParameterSet pset, int key)
Builder Constructor.
Definition: builder.cc:75