1 #define TRACE_NAME "builder"
4 #include "art/Framework/Art/artapp.h"
5 #include "artdaq-core/Core/SimpleMemoryReader.hh"
6 #include "artdaq-core/Data/Fragment.hh"
7 #include "artdaq-core/Generators/FragmentGenerator.hh"
8 #include "artdaq-core/Generators/makeFragmentGenerator.hh"
10 #include "artdaq/DAQrate/DataReceiverManager.hh"
11 #include "artdaq/DAQrate/DataSenderManager.hh"
13 #include <boost/program_options.hpp>
14 #include "fhiclcpp/make_ParameterSet.h"
15 namespace bpo = boost::program_options;
30 #include <sys/resource.h>
46 Builder(
int argc,
char* argv[], fhicl::ParameterSet pset,
int key);
64 enum class Role : int { DETECTOR, SINK };
66 void printHost(
const std::string& functionName)
const;
68 fhicl::ParameterSet daq_pset_;
69 bool const want_sink_;
70 bool const want_periodic_sync_;
71 MPI_Comm local_group_comm_;
78 want_sink_(daq_pset_.get<bool>(
"want_sink", true)),
79 want_periodic_sync_(daq_pset_.get<bool>(
"want_periodic_sync", false)),
81 std::vector<std::string> detectors;
82 daq_pset_.get_if_present(
"detectors", detectors);
83 if (static_cast<size_t>(my_rank) >= detectors.size()) {
84 builder_role_ = Role::SINK;
86 builder_role_ = Role::DETECTOR;
88 std::string type(pset.get<std::string>(
"transfer_plugin_type",
"Shmem"));
90 int senders = pset.get<
int>(
"num_senders");
91 int receivers = pset.get<
int>(
"num_receivers");
92 int buffer_count = pset.get<
int>(
"buffer_count", 10);
93 int max_payload_size = pset.get<
size_t>(
"fragment_size", 0x100000);
95 std::string hostmap =
"";
96 if (pset.has_key(
"hostmap")) {
97 hostmap =
" host_map: @local::hostmap";
100 std::stringstream ss;
101 ss << pset.to_string();
103 for (
int ii = 0; ii < senders; ++ii) {
104 ss <<
"s" << ii <<
": { transferPluginType: " << type <<
" source_rank: " << ii
105 <<
" max_fragment_size_words: " << max_payload_size <<
" buffer_count: " << buffer_count
106 <<
" shm_key_offset: " << std::to_string(key) << hostmap <<
"}";
108 ss <<
"} destinations: {";
109 for (
int jj = senders; jj < senders + receivers; ++jj) {
110 ss <<
"d" << jj <<
": { transferPluginType: " << type <<
" destination_rank: " << jj
111 <<
" max_fragment_size_words: " << max_payload_size <<
" buffer_count: " << buffer_count
112 <<
" shm_key_offset: " << std::to_string(key) << hostmap <<
"}";
116 daq_pset_ = fhicl::ParameterSet();
117 make_ParameterSet(ss.str(), daq_pset_);
127 MPI_Barrier(MPI_COMM_WORLD);
130 MPI_Comm_split(MPI_COMM_WORLD, static_cast<int>(builder_role_), 0, &local_group_comm_);
131 switch (builder_role_) {
137 "WARNING: a sink was instantiated despite want_sink being false:\n"
138 "set nsinks to 0 in invocation of daqrate?\n");
140 MPI_Barrier(MPI_COMM_WORLD);
147 throw "No such node type";
152 printHost(
"detector");
155 MPI_Comm_rank(local_group_comm_, &detector_rank);
156 assert(!(detector_rank < 0));
157 std::ostringstream det_ps_name_loc;
158 std::vector<std::string> detectors;
159 bool detectors_present = daq_pset_.get_if_present(
"detectors", detectors);
160 size_t detectors_size = detectors.size();
161 if (!(detectors_present && detectors_size)) {
162 throw cet::exception(
"Configuration") <<
"Unable to find required sequence of detector "
163 <<
"parameter set names, \"detectors\".";
165 fhicl::ParameterSet det_ps = daq_pset_.get<fhicl::ParameterSet>(
166 ((detectors_size >
static_cast<size_t>(detector_rank)) ? detectors[detector_rank] : detectors[0]));
167 std::unique_ptr<artdaq::FragmentGenerator>
const gen(
168 artdaq::makeFragmentGenerator(det_ps.get<std::string>(
"generator"), det_ps));
170 artdaq::DataSenderManager h(daq_pset_);
171 MPI_Barrier(local_group_comm_);
174 size_t fragments_per_source = -1;
175 daq_pset_.get_if_present(
"fragments_per_source", fragments_per_source);
176 artdaq::FragmentPtrs frags;
177 size_t fragments_sent = 0;
178 while (fragments_sent < fragments_per_source && gen->getNext(frags)) {
179 if (!fragments_sent) {
182 MPI_Barrier(local_group_comm_);
184 for (
auto& fragPtr : frags) {
185 std::cout <<
"Program::detector: Sending fragment " << fragments_sent + 1 <<
" of " << fragments_per_source
187 TLOG(TLVL_DEBUG) <<
"Program::detector: Sending fragment " << fragments_sent + 1 <<
" of "
188 << fragments_per_source;
189 auto sequence_id = fragPtr->sequenceID();
190 h.sendFragment(std::move(*fragPtr));
192 if (h.GetSentSequenceIDCount(sequence_id) == gen->fragmentIDs().size()) {
193 h.RemoveRoutingTableEntry(sequence_id);
196 if (++fragments_sent == fragments_per_source) {
199 if (want_periodic_sync_ && (fragments_sent % 100) == 0) {
201 MPI_Barrier(local_group_comm_);
206 TLOG(TLVL_DEBUG) <<
"detector waiting " << my_rank;
208 TLOG(TLVL_DEBUG) <<
"detector done " << my_rank;
209 MPI_Comm_free(&local_group_comm_);
210 MPI_Barrier(MPI_COMM_WORLD);
216 usleep(1000 * my_rank);
218 auto events = std::make_shared<artdaq::SharedMemoryEventManager>(daq_pset_, daq_pset_);
219 events->startRun(daq_pset_.get<
int>(
"run_number", 100));
221 artdaq::DataReceiverManager h(daq_pset_, events);
223 while (h.running_sources().size() > 0) {
228 TLOG(TLVL_DEBUG) <<
"All detectors are done, Sending endOfData Fragment";
231 bool endSucceeded =
false;
232 endSucceeded = events->endOfData();
234 TLOG(TLVL_DEBUG) <<
"Sink: reader is done";
236 TLOG(TLVL_DEBUG) <<
"Sink: reader failed to complete because the "
237 <<
"endOfData marker could not be pushed onto the queue.";
240 TLOG(TLVL_DEBUG) <<
"Sink done " << my_rank;
241 MPI_Barrier(MPI_COMM_WORLD);
244 void Builder::printHost(
const std::string& functionName)
const {
245 char* doPrint = getenv(
"PRINT_HOST");
249 const int ARRSIZE = 80;
250 char hostname[ARRSIZE];
251 std::string hostString;
252 if (!gethostname(hostname, ARRSIZE)) {
253 hostString = hostname;
255 hostString =
"unknown";
257 TLOG(TLVL_DEBUG) <<
"Running " << functionName <<
" on host " << hostString <<
" with rank " << my_rank <<
".";
263 getrusage(RUSAGE_SELF, &usage);
264 std::cout << myid <<
":"
265 <<
" user=" << artdaq::TimeUtils::convertUnixTimeToSeconds(usage.ru_utime)
266 <<
" sys=" << artdaq::TimeUtils::convertUnixTimeToSeconds(usage.ru_stime) << std::endl;
269 int main(
int argc,
char* argv[]) {
270 artdaq::configureMessageFacility(
"builder");
272 std::ostringstream descstr;
273 descstr << argv[0] <<
" <-c <config-file>> <other-options> [<source-file>]+";
274 bpo::options_description desc(descstr.str());
275 desc.add_options()(
"config,c", bpo::value<std::string>(),
"Configuration file.")(
276 "key,k", bpo::value<int>(),
"Shared Memory Key")(
"help,h",
"produce help message");
277 bpo::variables_map vm;
279 bpo::store(bpo::command_line_parser(argc, argv).options(desc).run(), vm);
281 }
catch (bpo::error
const& e) {
282 std::cerr <<
"Exception from command line processing in " << argv[0] <<
": " << e.what() <<
"\n";
285 if (vm.count(
"help")) {
286 std::cout << desc << std::endl;
289 if (!vm.count(
"config")) {
290 std::cerr <<
"Exception from command line processing in " << argv[0] <<
": no configuration file given.\n"
291 <<
"For usage and an options list, please do '" << argv[0] <<
" --help"
296 if (vm.count(
"key")) {
297 key = vm[
"key"].as<
int>();
299 fhicl::ParameterSet pset;
300 if (getenv(
"FHICL_FILE_PATH") ==
nullptr) {
301 std::cerr <<
"INFO: environment variable FHICL_FILE_PATH was not set. Using \".\"\n";
302 setenv(
"FHICL_FILE_PATH",
".", 0);
304 cet::filepath_lookup_after1 lookup_policy(
"FHICL_FILE_PATH");
305 fhicl::make_ParameterSet(vm[
"config"].as<std::string>(), lookup_policy, pset);
309 Builder p(argc, argv, pset, key);
310 std::cerr <<
"Started process " << my_rank <<
" of " << p.procs_ <<
".\n";
313 }
catch (std::string& x) {
314 std::cerr <<
"Exception (type string) caught in driver: " << x <<
'\n';
316 }
catch (
char const* m) {
317 std::cerr <<
"Exception (type char const*) caught in driver: ";
321 std::cerr <<
"[the value was a null pointer, so no message is available]";
void go()
Start the Builder application, using the type configuration to select which method to run...
A wrapper for a MPI program. Similar to MPISentry.
void detector()
Generate data, and send it using DataSenderManager.
The Builder class runs the builder test.
void sink()
Receive data from source via DataReceiverManager, send it to the EventStore (and art, if configured)
Builder(int argc, char *argv[], fhicl::ParameterSet pset, int key)
Builder Constructor.