1 #define TRACE_NAME "builder"
4 #include "art/Framework/Art/artapp.h"
5 #include "artdaq-core/Core/SimpleMemoryReader.hh"
6 #include "artdaq-core/Data/Fragment.hh"
7 #include "artdaq-core/Generators/FragmentGenerator.hh"
8 #include "artdaq-core/Generators/makeFragmentGenerator.hh"
10 #include "artdaq/DAQrate/DataReceiverManager.hh"
11 #include "artdaq/DAQrate/DataSenderManager.hh"
13 #include <boost/program_options.hpp>
14 #include "fhiclcpp/make_ParameterSet.h"
15 namespace bpo = boost::program_options;
30 #include <sys/resource.h>
47 Builder(
int argc,
char* argv[], fhicl::ParameterSet pset,
int key);
71 void printHost(
const std::string& functionName)
const;
73 fhicl::ParameterSet daq_pset_;
74 bool const want_sink_;
75 bool const want_periodic_sync_;
76 MPI_Comm local_group_comm_;
81 :
MPIProg(argc, argv), daq_pset_(pset), want_sink_(daq_pset_.get<bool>(
"want_sink", true)), want_periodic_sync_(daq_pset_.get<bool>(
"want_periodic_sync", false)), local_group_comm_()
83 std::vector<std::string> detectors;
84 daq_pset_.get_if_present(
"detectors", detectors);
85 if (static_cast<size_t>(my_rank) >= detectors.size())
87 builder_role_ = Role::SINK;
91 builder_role_ = Role::DETECTOR;
93 std::string type(pset.get<std::string>(
"transfer_plugin_type",
"Shmem"));
95 int senders = pset.get<
int>(
"num_senders");
96 int receivers = pset.get<
int>(
"num_receivers");
97 int buffer_count = pset.get<
int>(
"buffer_count", 10);
98 int max_payload_size = pset.get<
size_t>(
"fragment_size", 0x100000);
100 std::string hostmap =
"";
101 if (pset.has_key(
"hostmap"))
103 hostmap =
" host_map: @local::hostmap";
106 std::stringstream ss;
107 ss << pset.to_string();
109 for (
int ii = 0; ii < senders; ++ii)
111 ss <<
"s" << ii <<
": { transferPluginType: " << type <<
" source_rank: " << ii
112 <<
" max_fragment_size_words: " << max_payload_size <<
" buffer_count: " << buffer_count
113 <<
" shm_key_offset: " << std::to_string(key) << hostmap <<
"}";
115 ss <<
"} destinations: {";
116 for (
int jj = senders; jj < senders + receivers; ++jj)
118 ss <<
"d" << jj <<
": { transferPluginType: " << type <<
" destination_rank: " << jj
119 <<
" max_fragment_size_words: " << max_payload_size <<
" buffer_count: " << buffer_count
120 <<
" shm_key_offset: " << std::to_string(key) << hostmap <<
"}";
124 daq_pset_ = fhicl::ParameterSet();
125 make_ParameterSet(ss.str(), daq_pset_);
136 MPI_Barrier(MPI_COMM_WORLD);
139 MPI_Comm_split(MPI_COMM_WORLD, static_cast<int>(builder_role_), 0, &local_group_comm_);
140 switch (builder_role_)
150 "WARNING: a sink was instantiated despite want_sink being false:\n"
151 "set nsinks to 0 in invocation of daqrate?\n");
153 MPI_Barrier(MPI_COMM_WORLD);
160 throw "No such node type";
166 printHost(
"detector");
169 MPI_Comm_rank(local_group_comm_, &detector_rank);
170 assert(!(detector_rank < 0));
171 std::ostringstream det_ps_name_loc;
172 std::vector<std::string> detectors;
173 bool detectors_present = daq_pset_.get_if_present(
"detectors", detectors);
174 size_t detectors_size = detectors.size();
175 if (!(detectors_present && detectors_size))
177 throw cet::exception(
"Configuration") <<
"Unable to find required sequence of detector "
178 <<
"parameter set names, \"detectors\".";
180 fhicl::ParameterSet det_ps = daq_pset_.get<fhicl::ParameterSet>(
181 ((detectors_size >
static_cast<size_t>(detector_rank)) ? detectors[detector_rank] : detectors[0]));
182 std::unique_ptr<artdaq::FragmentGenerator>
const gen(
183 artdaq::makeFragmentGenerator(det_ps.get<std::string>(
"generator"), det_ps));
185 artdaq::DataSenderManager h(daq_pset_);
186 MPI_Barrier(local_group_comm_);
189 size_t fragments_per_source = -1;
190 daq_pset_.get_if_present(
"fragments_per_source", fragments_per_source);
191 artdaq::FragmentPtrs frags;
192 size_t fragments_sent = 0;
193 while (fragments_sent < fragments_per_source && gen->getNext(frags))
199 MPI_Barrier(local_group_comm_);
201 for (
auto& fragPtr : frags)
203 std::cout <<
"Program::detector: Sending fragment " << fragments_sent + 1 <<
" of " << fragments_per_source
205 TLOG(TLVL_DEBUG) <<
"Program::detector: Sending fragment " << fragments_sent + 1 <<
" of "
206 << fragments_per_source;
207 auto sequence_id = fragPtr->sequenceID();
208 h.sendFragment(std::move(*fragPtr));
210 if (h.GetSentSequenceIDCount(sequence_id) == gen->fragmentIDs().size())
212 h.RemoveRoutingTableEntry(sequence_id);
215 if (++fragments_sent == fragments_per_source)
219 if (want_periodic_sync_ && (fragments_sent % 100) == 0)
222 MPI_Barrier(local_group_comm_);
227 TLOG(TLVL_DEBUG) <<
"detector waiting " << my_rank;
229 TLOG(TLVL_DEBUG) <<
"detector done " << my_rank;
230 MPI_Comm_free(&local_group_comm_);
231 MPI_Barrier(MPI_COMM_WORLD);
238 usleep(1000 * my_rank);
240 auto events = std::make_shared<artdaq::SharedMemoryEventManager>(daq_pset_, daq_pset_);
241 events->startRun(daq_pset_.get<
int>(
"run_number", 100));
243 artdaq::DataReceiverManager h(daq_pset_, events);
245 while (h.running_sources().size() > 0)
251 TLOG(TLVL_DEBUG) <<
"All detectors are done, Sending endOfData Fragment";
254 bool endSucceeded =
false;
255 endSucceeded = events->endOfData();
258 TLOG(TLVL_DEBUG) <<
"Sink: reader is done";
262 TLOG(TLVL_DEBUG) <<
"Sink: reader failed to complete because the "
263 <<
"endOfData marker could not be pushed onto the queue.";
266 TLOG(TLVL_DEBUG) <<
"Sink done " << my_rank;
267 MPI_Barrier(MPI_COMM_WORLD);
270 void Builder::printHost(
const std::string& functionName)
const
272 char* doPrint = getenv(
"PRINT_HOST");
277 const int ARRSIZE = 80;
278 char hostname[ARRSIZE];
279 std::string hostString;
280 if (!gethostname(hostname, ARRSIZE))
282 hostString = hostname;
286 hostString =
"unknown";
288 TLOG(TLVL_DEBUG) <<
"Running " << functionName <<
" on host " << hostString <<
" with rank " << my_rank <<
".";
295 getrusage(RUSAGE_SELF, &usage);
296 std::cout << myid <<
":"
297 <<
" user=" << artdaq::TimeUtils::convertUnixTimeToSeconds(usage.ru_utime)
298 <<
" sys=" << artdaq::TimeUtils::convertUnixTimeToSeconds(usage.ru_stime) << std::endl;
301 int main(
int argc,
char* argv[])
303 artdaq::configureMessageFacility(
"builder");
305 std::ostringstream descstr;
306 descstr << argv[0] <<
" <-c <config-file>> <other-options> [<source-file>]+";
307 bpo::options_description desc(descstr.str());
308 desc.add_options()(
"config,c", bpo::value<std::string>(),
"Configuration file.")(
309 "key,k", bpo::value<int>(),
"Shared Memory Key")(
"help,h",
"produce help message");
310 bpo::variables_map vm;
313 bpo::store(bpo::command_line_parser(argc, argv).options(desc).run(), vm);
316 catch (bpo::error
const& e)
318 std::cerr <<
"Exception from command line processing in " << argv[0] <<
": " << e.what() <<
"\n";
321 if (vm.count(
"help"))
323 std::cout << desc << std::endl;
326 if (!vm.count(
"config"))
328 std::cerr <<
"Exception from command line processing in " << argv[0] <<
": no configuration file given.\n"
329 <<
"For usage and an options list, please do '" << argv[0] <<
" --help"
336 key = vm[
"key"].as<
int>();
338 fhicl::ParameterSet pset;
339 if (getenv(
"FHICL_FILE_PATH") ==
nullptr)
341 std::cerr <<
"INFO: environment variable FHICL_FILE_PATH was not set. Using \".\"\n";
342 setenv(
"FHICL_FILE_PATH",
".", 0);
344 cet::filepath_lookup_after1 lookup_policy(
"FHICL_FILE_PATH");
345 fhicl::make_ParameterSet(vm[
"config"].as<std::string>(), lookup_policy, pset);
350 Builder p(argc, argv, pset, key);
351 std::cerr <<
"Started process " << my_rank <<
" of " << p.procs_ <<
".\n";
355 catch (std::string& x)
357 std::cerr <<
"Exception (type string) caught in driver: " << x <<
'\n';
360 catch (
char const* m)
362 std::cerr <<
"Exception (type char const*) caught in driver: ";
369 std::cerr <<
"[the value was a null pointer, so no message is available]";
void go()
Start the Builder application, using the type configuration to select which method to run...
A wrapper for a MPI program. Similar to MPISentry.
void detector()
Generate data, and send it using DataSenderManager.
The Builder class runs the builder test.
void sink()
Receive data from source via DataReceiverManager, send it to the EventStore (and art, if configured)
Builder(int argc, char *argv[], fhicl::ParameterSet pset, int key)
Builder Constructor.