1 #include "art/Framework/Art/artapp.h"
2 #include "artdaq-core/Generators/FragmentGenerator.hh"
3 #include "artdaq-core/Data/Fragment.hh"
4 #include "artdaq-core/Generators/makeFragmentGenerator.hh"
6 #include "artdaq/DAQrate/DataSenderManager.hh"
7 #include "artdaq/DAQrate/DataReceiverManager.hh"
8 #include "artdaq-core/Core/SimpleMemoryReader.hh"
11 #include <boost/program_options.hpp>
12 #include "fhiclcpp/make_ParameterSet.h"
13 namespace bpo = boost::program_options;
31 #include <sys/resource.h>
47 Builder(
int argc,
char* argv[], fhicl::ParameterSet pset,
int key);
71 void printHost(
const std::string& functionName)
const;
73 fhicl::ParameterSet daq_pset_;
74 bool const want_sink_;
75 bool const want_periodic_sync_;
76 MPI_Comm local_group_comm_;
83 , want_sink_(daq_pset_.get<bool>(
"want_sink", true))
84 , want_periodic_sync_(daq_pset_.get<bool>(
"want_periodic_sync", false))
87 std::vector<std::string> detectors;
88 daq_pset_.get_if_present(
"detectors", detectors);
89 if (static_cast<size_t>(my_rank) >= detectors.size())
91 builder_role_ = Role::SINK;
95 builder_role_ = Role::DETECTOR;
97 std::string type(pset.get<std::string>(
"transfer_plugin_type",
"Shmem"));
99 int senders = pset.get<
int>(
"num_senders");
100 int receivers = pset.get<
int>(
"num_receivers");
101 int buffer_count = pset.get<
int>(
"buffer_count", 10);
102 int max_payload_size = pset.get<
size_t>(
"fragment_size", 0x100000);
104 std::string hostmap =
"";
105 if (pset.has_key(
"hostmap"))
107 hostmap =
" host_map: @local::hostmap";
110 std::stringstream ss;
111 ss << pset.to_string();
113 for (
int ii = 0; ii < senders; ++ii)
115 ss <<
"s" << ii <<
": { transferPluginType: " << type <<
" source_rank: " << ii <<
" max_fragment_size_words: " << max_payload_size <<
" buffer_count: " << buffer_count <<
" shm_key_offset: " << std::to_string(key) << hostmap <<
"}";
117 ss <<
"} destinations: {";
118 for (
int jj = senders; jj < senders + receivers; ++jj)
120 ss <<
"d" << jj <<
": { transferPluginType: " << type <<
" destination_rank: " << jj <<
" max_fragment_size_words: " << max_payload_size <<
" buffer_count: " << buffer_count <<
" shm_key_offset: " << std::to_string(key) << hostmap <<
"}";
124 daq_pset_ = fhicl::ParameterSet();
125 make_ParameterSet(ss.str(), daq_pset_);
139 MPI_Barrier(MPI_COMM_WORLD);
141 MPI_Comm_split(MPI_COMM_WORLD, static_cast<int>(builder_role_), 0, &local_group_comm_);
142 switch (builder_role_)
152 msg(
"WARNING: a sink was instantiated despite want_sink being false:\n"
153 "set nsinks to 0 in invocation of daqrate?\n");
155 MPI_Barrier(MPI_COMM_WORLD);
162 throw "No such node type";
168 printHost(
"detector");
171 MPI_Comm_rank(local_group_comm_, &detector_rank);
172 assert(!(detector_rank < 0));
173 std::ostringstream det_ps_name_loc;
174 std::vector<std::string> detectors;
175 bool detectors_present = daq_pset_.get_if_present(
"detectors", detectors);
176 size_t detectors_size = detectors.size();
177 if (!(detectors_present && detectors_size))
179 throw cet::exception(
"Configuration")
180 <<
"Unable to find required sequence of detector "
181 <<
"parameter set names, \"detectors\".";
183 fhicl::ParameterSet det_ps =
184 daq_pset_.get<fhicl::ParameterSet>(((detectors_size >
static_cast<size_t>(detector_rank)) ? detectors[detector_rank] : detectors[0]));
185 std::unique_ptr<artdaq::FragmentGenerator>
const
186 gen(artdaq::makeFragmentGenerator
187 (det_ps.get<std::string>(
"generator"),
191 MPI_Barrier(local_group_comm_);
194 size_t fragments_per_source = -1;
195 daq_pset_.get_if_present(
"fragments_per_source", fragments_per_source);
196 artdaq::FragmentPtrs frags;
197 size_t fragments_sent = 0;
198 while (fragments_sent < fragments_per_source && gen->getNext(frags))
204 MPI_Barrier(local_group_comm_);
206 for (
auto& fragPtr : frags)
208 std::cout <<
"Program::detector: Sending fragment " << fragments_sent + 1 <<
" of " << fragments_per_source << std::endl;
209 TLOG_DEBUG(
"builder") <<
"Program::detector: Sending fragment " << fragments_sent + 1 <<
" of " << fragments_per_source << TLOG_ENDL;
211 if (++fragments_sent == fragments_per_source) {
break; }
212 if (want_periodic_sync_ && (fragments_sent % 100) == 0)
215 MPI_Barrier(local_group_comm_);
220 TLOG_DEBUG(
"builder") <<
"detector waiting " << my_rank << TLOG_ENDL;
222 TLOG_DEBUG(
"builder") <<
"detector done " << my_rank << TLOG_ENDL;
223 MPI_Comm_free(&local_group_comm_);
224 MPI_Barrier(MPI_COMM_WORLD);
231 usleep(1000 * my_rank);
233 auto events = std::make_shared<artdaq::SharedMemoryEventManager>(daq_pset_, daq_pset_);
234 events->startRun(daq_pset_.get<
int>(
"run_number", 100));
244 TLOG_DEBUG(
"builder") <<
"All detectors are done, Sending endOfData Fragment" << TLOG_ENDL;
247 bool endSucceeded =
false;
248 endSucceeded = events->endOfData();
251 TLOG_DEBUG(
"builder") <<
"Sink: reader is done" << TLOG_ENDL;
255 TLOG_DEBUG(
"builder") <<
"Sink: reader failed to complete because the "
256 <<
"endOfData marker could not be pushed onto the queue."
260 TLOG_DEBUG(
"builder") <<
"Sink done " << my_rank << TLOG_ENDL;
261 MPI_Barrier(MPI_COMM_WORLD);
264 void Builder::printHost(
const std::string& functionName)
const
266 char* doPrint = getenv(
"PRINT_HOST");
267 if (doPrint == 0) {
return; }
268 const int ARRSIZE = 80;
269 char hostname[ARRSIZE];
270 std::string hostString;
271 if (!gethostname(hostname, ARRSIZE))
273 hostString = hostname;
277 hostString =
"unknown";
279 TLOG_DEBUG(
"builder") <<
"Running " << functionName
280 <<
" on host " << hostString
281 <<
" with rank " << my_rank <<
"."
289 getrusage(RUSAGE_SELF, &usage);
290 std::cout << myid <<
":"
291 <<
" user=" << artdaq::TimeUtils::convertUnixTimeToSeconds(usage.ru_utime)
292 <<
" sys=" << artdaq::TimeUtils::convertUnixTimeToSeconds(usage.ru_stime)
296 int main(
int argc,
char* argv[])
298 artdaq::configureMessageFacility(
"builder");
300 std::ostringstream descstr;
302 <<
" <-c <config-file>> <other-options> [<source-file>]+";
303 bpo::options_description desc(descstr.str());
305 (
"config,c", bpo::value<std::string>(),
"Configuration file.")
306 (
"key,k", bpo::value<int>(),
"Shared Memory Key")
307 (
"help,h",
"produce help message");
308 bpo::variables_map vm;
310 bpo::store(bpo::command_line_parser(argc, argv).options(desc).run(), vm);
313 catch (bpo::error
const & e) {
314 std::cerr <<
"Exception from command line processing in " << argv[0]
315 <<
": " << e.what() <<
"\n";
318 if (vm.count(
"help")) {
319 std::cout << desc << std::endl;
322 if (!vm.count(
"config")) {
323 std::cerr <<
"Exception from command line processing in " << argv[0]
324 <<
": no configuration file given.\n"
325 <<
"For usage and an options list, please do '"
326 << argv[0] <<
" --help"
333 key = vm[
"key"].as<
int>();
335 fhicl::ParameterSet pset;
336 if (getenv(
"FHICL_FILE_PATH") ==
nullptr) {
338 <<
"INFO: environment variable FHICL_FILE_PATH was not set. Using \".\"\n";
339 setenv(
"FHICL_FILE_PATH",
".", 0);
341 cet::filepath_lookup_after1 lookup_policy(
"FHICL_FILE_PATH");
342 fhicl::make_ParameterSet(vm[
"config"].as<std::string>(), lookup_policy, pset);
347 Builder p(argc, argv, pset,key);
348 std::cerr <<
"Started process " << my_rank <<
" of " << p.procs_ <<
".\n";
352 catch (std::string& x)
354 std::cerr <<
"Exception (type string) caught in driver: "
359 catch (
char const* m)
361 std::cerr <<
"Exception (type char const*) caught in driver: ";
368 std::cerr <<
"[the value was a null pointer, so no message is available]";
Sends Fragment objects using TransferInterface plugins. Uses Routing Tables if confgiured, otherwise will Round-Robin Fragments to the destinations.
void go()
Start the Builder application, using the type configuration to select which method to run...
A wrapper for a MPI program. Similar to MPISentry.
void detector()
Generate data, and send it using DataSenderManager.
int sendFragment(Fragment &&frag)
Send the given Fragment. Return the rank of the destination to which the Fragment was sent...
The Builder class runs the builder test.
Receives Fragment objects from one or more DataSenderManager instances using TransferInterface plugin...
void sink()
Receive data from source via DataReceiverManager, send it to the EventStore (and art, if configured)
std::set< int > running_sources() const
Get the list of sources which are still receiving data.
Builder(int argc, char *argv[], fhicl::ParameterSet pset, int key)
Builder Constructor.
void start_threads()
Start receiver threads for all enabled sources.