1 #include "art/Framework/Art/artapp.h"
2 #include "artdaq-core/Generators/FragmentGenerator.hh"
3 #include "artdaq-core/Data/Fragment.hh"
4 #include "artdaq-core/Generators/makeFragmentGenerator.hh"
6 #include "artdaq/DAQrate/EventStore.hh"
8 #include "artdaq/DAQrate/DataSenderManager.hh"
9 #include "artdaq/DAQrate/DataReceiverManager.hh"
10 #include "artdaq-core/Core/SimpleQueueReader.hh"
12 #include "fhiclcpp/ParameterSet.h"
14 #include <boost/program_options.hpp>
15 namespace bpo = boost::program_options;
33 #include <sys/resource.h>
47 Builder(
int argc,
char* argv[]);
77 void printHost(
const std::string& functionName)
const;
80 fhicl::ParameterSet
const daq_pset_;
81 bool const want_sink_;
82 bool const want_periodic_sync_;
83 MPI_Comm local_group_comm_;
88 , conf_(my_rank, procs_, 10, 10240, argc, argv)
89 , daq_pset_(conf_.getArtPset())
90 , want_sink_(daq_pset_.get<bool>(
"want_sink", true))
91 , want_periodic_sync_(daq_pset_.get<bool>(
"want_periodic_sync", false))
99 MPI_Barrier(MPI_COMM_WORLD);
101 MPI_Comm_split(MPI_COMM_WORLD, conf_.
type_, 0, &local_group_comm_);
112 msg(
"WARNING: a sink was instantiated despite want_sink being false:\n"
113 "set nsinks to 0 in invocation of daqrate?\n");
115 MPI_Barrier(MPI_COMM_WORLD);
125 throw "No such node type";
133 artdaq::FragmentPtr frag;
138 int senderCount = from_d.enabled_sources().size();
139 while (senderCount > 0)
142 frag = from_d.recvFragment(ignoredSender);
144 std::cout <<
"Program::source: Received fragment " << frag->sequenceID() <<
" from sender " << ignoredSender << std::endl;
145 if (want_sink_ && frag->type() != artdaq::Fragment::EndOfDataFragmentType)
149 else if (frag->type() == artdaq::Fragment::EndOfDataFragmentType)
155 TLOG_DEBUG(
"builder") <<
"source done " << conf_.
rank_ << TLOG_ENDL;
156 MPI_Barrier(MPI_COMM_WORLD);
161 printHost(
"detector");
164 MPI_Comm_rank(local_group_comm_, &detector_rank);
165 assert(!(detector_rank < 0));
166 std::ostringstream det_ps_name_loc;
167 std::vector<std::string> detectors;
168 size_t detectors_size = 0;
169 if (!(daq_pset_.get_if_present(
"detectors", detectors) &&
170 (detectors_size = detectors.size())))
172 throw cet::exception(
"Configuration")
173 <<
"Unable to find required sequence of detector "
174 <<
"parameter set names, \"detectors\".";
176 fhicl::ParameterSet det_ps =
177 daq_pset_.get<fhicl::ParameterSet>
178 ((detectors_size >
static_cast<size_t>(detector_rank)) ?
179 detectors[detector_rank] :
181 std::unique_ptr<artdaq::FragmentGenerator>
const
182 gen(artdaq::makeFragmentGenerator
183 (det_ps.get<std::string>(
"generator"),
187 MPI_Barrier(local_group_comm_);
190 size_t fragments_per_source = -1;
191 daq_pset_.get_if_present(
"fragments_per_source", fragments_per_source);
192 artdaq::FragmentPtrs frags;
193 size_t fragments_sent = 0;
194 while (fragments_sent < fragments_per_source && gen->getNext(frags))
200 MPI_Barrier(local_group_comm_);
202 for (
auto& fragPtr : frags)
204 std::cout <<
"Program::detector: Sending fragment " << fragments_sent + 1 <<
" of " << fragments_per_source << std::endl;
205 h.sendFragment(std::move(*fragPtr));
206 if (++fragments_sent == fragments_per_source) {
break; }
207 if (want_periodic_sync_ && (fragments_sent % 100) == 0)
210 MPI_Barrier(local_group_comm_);
215 TLOG_DEBUG(
"builder") <<
"detector waiting " << conf_.
rank_ << TLOG_ENDL;
217 TLOG_DEBUG(
"builder") <<
"detector done " << conf_.
rank_ << TLOG_ENDL;
218 MPI_Comm_free(&local_group_comm_);
219 MPI_Barrier(MPI_COMM_WORLD);
228 bool useArt = daq_pset_.get<
bool>(
"useArt",
false);
229 const char* dummyArgs[1]{
"SimpleQueueReader"};
230 MPI_Comm_rank(local_group_comm_, &sink_rank);
234 &artdaq::simpleQueueReaderApp;
238 useArt ? conf_.
art_argv_ : const_cast<char**>(dummyArgs),
243 int senderCount = h.enabled_sources().size();
244 while (senderCount > 0)
246 artdaq::FragmentPtr pfragment(
new artdaq::Fragment);
248 pfragment = h.recvFragment(ignoredSource);
250 std::cout <<
"Program::sink: Received fragment " << pfragment->sequenceID() <<
" from sender " << ignoredSource << std::endl;
251 if (pfragment->type() != artdaq::Fragment::EndOfDataFragmentType)
253 events.
insert(std::move(pfragment));
263 int readerReturnValue;
264 bool endSucceeded =
false;
265 int attemptsToEnd = 1;
266 endSucceeded = events.
endOfData(readerReturnValue);
267 while (!endSucceeded && attemptsToEnd < 3)
270 endSucceeded = events.
endOfData(readerReturnValue);
274 TLOG_DEBUG(
"builder") <<
"Sink: reader is done, its exit status was: "
275 << readerReturnValue << TLOG_ENDL;
279 TLOG_DEBUG(
"builder") <<
"Sink: reader failed to complete because the "
280 <<
"endOfData marker could not be pushed onto the queue."
284 TLOG_DEBUG(
"builder") <<
"Sink done " << conf_.
rank_ << TLOG_ENDL;
285 MPI_Barrier(MPI_COMM_WORLD);
288 void Builder::printHost(
const std::string& functionName)
const
290 char* doPrint = getenv(
"PRINT_HOST");
291 if (doPrint == 0) {
return; }
292 const int ARRSIZE = 80;
293 char hostname[ARRSIZE];
294 std::string hostString;
295 if (!gethostname(hostname, ARRSIZE))
297 hostString = hostname;
301 hostString =
"unknown";
303 TLOG_DEBUG(
"builder") <<
"Running " << functionName
304 <<
" on host " << hostString
305 <<
" with rank " << my_rank <<
"."
313 getrusage(RUSAGE_SELF, &usage);
314 std::cout << myid <<
":"
320 int main(
int argc,
char* argv[])
327 std::cerr <<
"Started process " << my_rank <<
" of " << p.procs_ <<
".\n";
331 catch (std::string& x)
333 std::cerr <<
"Exception (type string) caught in driver: "
338 catch (
char const* m)
340 std::cerr <<
"Exception (type char const*) caught in driver: ";
347 std::cerr <<
"[the value was a null pointer, so no message is available]";
void insert(FragmentPtr pfrag, bool printWarningWhenFragmentIsDropped=true)
Give ownership of the Fragment to the EventStore.
Sends Fragment objects using TransferInterface plugins. Uses Routing Tables if confgiured, otherwise will Round-Robin Fragments to the destinations.
bool endOfData(int &readerReturnValue)
Indicate that the end of input has been reached to the art thread.
void go()
Start the Builder application, using the type configuration to select which method to run...
A wrapper for a MPI program. Similar to MPISentry.
void configureMessageFacility(char const *progname, bool useConsole=true)
Configure and start the message facility. Provide the program name so that messages will be appropria...
TaskType type_
Type of this Builder application.
void source()
Receive data from detector via DataReceiverManager, and send to a sink using DataSenderManager.
void detector()
Generate data, and send it using DataSenderManager.
static const int RECV_TIMEOUT
Value to be returned upon receive timeout. Because receivers otherwise return rank, this is also the limit on the number of ranks that artdaq currently supports.
This Builder is a "Detector".
This Builder is a "Source".
fhicl::ParameterSet makeParameterSet() const
Write a ParameterSet using configuration.
int rank_
Rank of this application.
int sendFragment(Fragment &&frag)
Send the given Fragment. Return the rank of the destination to which the Fragment was sent...
char ** art_argv_
Arguments used for art.
The EventStore class collects Fragment objects, until it receives a complete event, at which point the event is handed over to the art thread.
This Builder is a "Sink".
Helper class for configuring the builder test.
int( ART_CMDLINE_FCN)(int, char **)
An art function that accepts standard C main arguments.
The Builder class runs the builder test.
Receives Fragment objects from one or more DataSenderManager instances using TransferInterface plugin...
void sink()
Receive data from source via DataReceiverManager, send it to the EventStore (and art, if configured)
int art_argc_
Count of arguments used for art.
void writeInfo() const
Write information about this Config class to a file.
int detectors_
Count of detectors.
static double timevalAsDouble(struct timeval tv)
Convert a timeval value to a double.
void start_threads()
Start receiver threads for all enabled sources.
Builder(int argc, char *argv[])
Builder Constructor.