1 #define TRACE_NAME "routing_master_t"
3 #include "artdaq/DAQrate/detail/RoutingPacket.hh"
4 #include "artdaq/DAQdata/TCPConnect.hh"
6 #include "cetlib/filepath_maker.h"
7 #include "fhiclcpp/ParameterSet.h"
8 #include "fhiclcpp/make_ParameterSet.h"
10 #include <boost/program_options.hpp>
11 #include "artdaq/Application/RoutingMasterCore.hh"
12 #include "artdaq/Application/RoutingMasterApp.hh"
14 namespace bpo = boost::program_options;
28 #include <arpa/inet.h>
29 #include <netinet/in.h>
30 #include <sys/types.h>
31 #include <sys/socket.h>
36 #include <sys/resource.h>
87 fhicl::ParameterSet
getPset(
int argc,
char* argv[])
const;
90 enum class TestRole_t : int
97 void printHost(
const std::string& functionName)
const;
99 fhicl::ParameterSet
const pset_;
100 fhicl::ParameterSet
const daq_pset_;
103 std::string routing_master_address_;
104 std::string multicast_address_;
108 std::vector<int> eb_ranks_;
110 size_t token_interval_us_;
115 , pset_(getPset(argc, argv))
116 , daq_pset_(pset_.get<fhicl::ParameterSet>(
"daq"))
117 , routing_master_address_(daq_pset_.get<std::string>(
"routing_master_hostname",
"localhost"))
118 , multicast_address_(daq_pset_.get<std::string>(
"table_update_address",
"227.128.12.28"))
119 , token_port_(daq_pset_.get<int>(
"routing_token_port", 35555))
120 , table_port_(daq_pset_.get<int>(
"table_update_port", 35556))
121 , ack_port_(daq_pset_.get<int>(
"table_acknowledge_port", 35557))
122 , token_count_(pset_.get<int>(
"token_count", 1000))
123 , token_interval_us_(pset_.get<size_t>(
"token_interval_us", 5000))
125 assert(!(my_rank < 0));
129 role_ = TestRole_t::TOKEN_GEN;
132 role_ = TestRole_t::ROUTING_MASTER;
135 role_ = TestRole_t::TABLE_RECEIVER;
138 auto policy_pset = daq_pset_.get<fhicl::ParameterSet>(
"policy");
139 eb_ranks_ = policy_pset.get<std::vector<int>>(
"receiver_ranks");
145 std::ostringstream descstr;
146 descstr <<
"-- <-c <config-file>>";
147 bpo::options_description desc(descstr.str());
149 (
"config,c", bpo::value<std::string>(),
"Configuration file.");
150 bpo::variables_map vm;
153 bpo::store(bpo::command_line_parser(argc, argv).
154 options(desc).allow_unregistered().run(), vm);
157 catch (bpo::error
const& e)
159 std::cerr <<
"Exception from command line processing in Config::getArtPset: " << e.what() <<
"\n";
160 throw "cmdline parsing error.";
162 if (!vm.count(
"config"))
164 std::cerr <<
"Expected \"-- -c <config-file>\" fhicl file specification.\n";
165 throw "cmdline parsing error.";
167 fhicl::ParameterSet pset;
168 cet::filepath_lookup lookup_policy(
"FHICL_FILE_PATH");
169 fhicl::make_ParameterSet(vm[
"config"].as<std::string>(), lookup_policy, pset);
176 MPI_Barrier(MPI_COMM_WORLD);
181 case TestRole_t::TABLE_RECEIVER:
184 case TestRole_t::ROUTING_MASTER:
187 case TestRole_t::TOKEN_GEN:
191 throw "No such node type";
193 TLOG(TLVL_DEBUG) <<
"Rank " << my_rank <<
" complete." ;
198 TLOG(TLVL_DEBUG) <<
"generate_tokens(): Init" ;
199 printHost(
"generate_tokens");
202 int token_socket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
203 if (token_socket < 0)
205 TLOG(TLVL_ERROR) <<
"generate_tokens(): I failed to create the socket for sending Routing Tokens!" ;
208 struct sockaddr_in token_addr;
209 auto sts = ResolveHost(routing_master_address_.c_str(), token_port_, token_addr);
212 TLOG(TLVL_ERROR) <<
"generate_tokens(): Could not resolve host name" ;
215 connect(token_socket, (
struct sockaddr*)&token_addr,
sizeof(token_addr));
218 std::map<int, int> token_counter;
219 for(
auto rank : eb_ranks_)
221 token_counter[rank] = 0;
223 while (sent_tokens < token_count_) {
224 int this_rank = eb_ranks_[seedAndRandom() % eb_ranks_.size()];
225 token_counter[this_rank]++;
226 artdaq::detail::RoutingToken token;
227 token.header = TOKEN_MAGIC;
228 token.rank = this_rank;
229 token.new_slots_free = 1;
231 TLOG(TLVL_DEBUG) <<
"generate_tokens(): Sending RoutingToken " << std::to_string(++sent_tokens) <<
" for rank " << this_rank <<
" to " << routing_master_address_ ;
232 send(token_socket, &token,
sizeof(artdaq::detail::RoutingToken), 0);
233 usleep(token_interval_us_);
236 for(
auto rank : token_counter)
238 if (rank.second > max_rank) max_rank = rank.second;
240 for(
auto rank : token_counter)
242 artdaq::detail::RoutingToken token;
243 token.header = TOKEN_MAGIC;
244 token.rank = rank.first;
245 token.new_slots_free = max_rank - rank.second;
247 TLOG(TLVL_DEBUG) <<
"generate_tokens(): Sending RoutingToken " << std::to_string(++sent_tokens) <<
" for rank " << rank.first <<
" to " << routing_master_address_ ;
248 send(token_socket, &token,
sizeof(artdaq::detail::RoutingToken), 0);
249 usleep(token_interval_us_);
253 TLOG(TLVL_INFO) <<
"generate_tokens(): Waiting at MPI_Barrier" ;
254 MPI_Barrier(MPI_COMM_WORLD);
255 TLOG(TLVL_INFO) <<
"generate_tokens(): Done with MPI_Barrier" ;
260 TLOG(TLVL_DEBUG) <<
"table_receiver(): Init" ;
261 printHost(
"table_receiver");
264 auto table_socket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
265 if (table_socket < 0)
267 TLOG(TLVL_ERROR) <<
"table_receiver(): Error creating socket for receiving data requests!" ;
271 struct sockaddr_in si_me_request;
274 if (setsockopt(table_socket, SOL_SOCKET, SO_REUSEADDR, &yes,
sizeof(yes)) < 0)
276 TLOG(TLVL_ERROR) <<
"table_receiver(): Unable to enable port reuse on request socket" ;
279 memset(&si_me_request, 0,
sizeof(si_me_request));
280 si_me_request.sin_family = AF_INET;
281 si_me_request.sin_port = htons(table_port_);
282 si_me_request.sin_addr.s_addr = htonl(INADDR_ANY);
283 if (bind(table_socket, (
struct sockaddr *)&si_me_request,
sizeof(si_me_request)) == -1)
285 TLOG(TLVL_ERROR) <<
"table_receiver(): Cannot bind request socket to port " << table_port_ ;
290 long int sts = ResolveHost(multicast_address_.c_str(), mreq.imr_multiaddr);
293 TLOG(TLVL_ERROR) <<
"table_receiver(): Unable to resolve multicast hostname" ;
296 mreq.imr_interface.s_addr = htonl(INADDR_ANY);
297 if (setsockopt(table_socket, IPPROTO_IP, IP_ADD_MEMBERSHIP, &mreq,
sizeof(mreq)) < 0)
299 TLOG(TLVL_ERROR) <<
"table_receiver(): Unable to join multicast group" ;
303 struct epoll_event ev;
304 int table_epoll_fd = epoll_create1(0);
305 ev.events = EPOLLIN | EPOLLPRI;
306 ev.data.fd = table_socket;
307 if (epoll_ctl(table_epoll_fd, EPOLL_CTL_ADD, table_socket, &ev) == -1)
309 TLOG(TLVL_ERROR) <<
"table_receiver(): Could not register listen socket to epoll fd" ;
313 auto ack_socket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
314 struct sockaddr_in ack_addr;
315 sts = ResolveHost(routing_master_address_.c_str(), ack_port_, ack_addr);
318 TLOG(TLVL_ERROR) <<
"table_receiver(): Unable to resolve routing master hostname" ;
322 if (table_socket == -1 || table_epoll_fd == -1 || ack_socket == -1)
324 TLOG(TLVL_DEBUG) <<
"table_receiver(): One of the listen sockets was not opened successfully." ;
327 artdaq::Fragment::sequence_id_t max_sequence_id = token_count_;
328 artdaq::Fragment::sequence_id_t current_sequence_id = 0;
329 std::map<artdaq::Fragment::sequence_id_t, int> routing_table;
330 TLOG(TLVL_INFO) <<
"table_receiver(): Expecting " << std::to_string(max_sequence_id) <<
" as the last Sequence ID in this run" ;
331 while (current_sequence_id < max_sequence_id)
333 std::vector<epoll_event> table_events_(4);
334 TLOG(TLVL_DEBUG) <<
"table_receiver(): Waiting for event on table socket" ;
335 auto nfds = epoll_wait(table_epoll_fd, &table_events_[0], table_events_.size(), -1);
337 perror(
"epoll_wait");
341 TLOG(TLVL_DEBUG) <<
"table_receiver(): Received " << nfds <<
" table update(s)" ;
342 for (
auto n = 0; n < nfds; ++n) {
343 auto first = artdaq::Fragment::InvalidSequenceID;
344 auto last = artdaq::Fragment::InvalidSequenceID;
345 artdaq::detail::RoutingPacketHeader hdr;
346 recv(table_events_[n].data.fd, &hdr,
sizeof(artdaq::detail::RoutingPacketHeader), 0);
348 TLOG(TLVL_DEBUG) <<
"table_receiver(): Checking for valid header" ;
349 if (hdr.header == ROUTING_MAGIC) {
350 artdaq::detail::RoutingPacket buffer(hdr.nEntries);
351 TLOG(TLVL_DEBUG) <<
"table_receiver(): Receiving data buffer" ;
352 sts = recv(table_events_[n].data.fd, &buffer[0],
sizeof(artdaq::detail::RoutingPacketEntry) * hdr.nEntries, 0);
353 assert(static_cast<size_t>(sts) ==
sizeof(artdaq::detail::RoutingPacketEntry) * hdr.nEntries);
355 first = buffer[0].sequence_id;
356 last = buffer[buffer.size() - 1].sequence_id;
358 for (
auto entry : buffer)
360 if (routing_table.count(entry.sequence_id))
362 assert(routing_table[entry.sequence_id] == entry.destination_rank);
365 routing_table[entry.sequence_id] = entry.destination_rank;
366 TLOG(TLVL_DEBUG) <<
"table_receiver(): table_receiver " << std::to_string(my_rank) <<
": received update: SeqID " << std::to_string(entry.sequence_id) <<
" -> Rank " << std::to_string(entry.destination_rank) ;
369 artdaq::detail::RoutingAckPacket ack;
371 ack.first_sequence_id = first;
372 ack.last_sequence_id = last;
374 TLOG(TLVL_DEBUG) <<
"table_receiver(): Sending RoutingAckPacket with first= " << std::to_string(first) <<
" and last= " << std::to_string(last) <<
" to " << routing_master_address_ <<
", port " << ack_port_ ;
375 sendto(ack_socket, &ack,
sizeof(artdaq::detail::RoutingAckPacket), 0, (
struct sockaddr *)&ack_addr,
sizeof(ack_addr));
376 current_sequence_id = last;
381 TLOG(TLVL_INFO) <<
"table_receiver(): Waiting at MPI_Barrier" ;
382 MPI_Barrier(MPI_COMM_WORLD);
383 TLOG(TLVL_INFO) <<
"table_receiver(): Done with MPI_Barrier" ;
388 TLOG(TLVL_DEBUG) <<
"routing_master: Init" ;
389 printHost(
"routing_master");
391 app_name =
"RoutingMaster";
393 auto app = std::make_unique<artdaq::RoutingMasterApp>();
395 app->initialize(pset_, 0, 0);
396 app->do_start(art::RunID(1), 0, 0);
397 TLOG(TLVL_INFO) <<
"routing_master: Waiting at MPI_Barrier" ;
398 MPI_Barrier(MPI_COMM_WORLD);
399 TLOG(TLVL_INFO) <<
"routing_master: Done with MPI_Barrier, calling RoutingMasterCore::stop" ;
401 TLOG(TLVL_INFO) <<
"routing_master: Done with RoutingMasterCore::stop, calling shutdown" ;
403 TLOG(TLVL_INFO) <<
"routing_master: Done with RoutingMasterCore::shutdown" ;
406 void RoutingMasterTest::printHost(
const std::string& functionName)
const
408 char* doPrint = getenv(
"PRINT_HOST");
409 if (doPrint == 0) {
return; }
410 const int ARRSIZE = 80;
411 char hostname[ARRSIZE];
412 std::string hostString;
413 if (!gethostname(hostname, ARRSIZE))
415 hostString = hostname;
419 hostString =
"unknown";
421 TLOG(TLVL_DEBUG) <<
"Running " << functionName
422 <<
" on host " << hostString
423 <<
" with rank " << my_rank <<
"."
431 getrusage(RUSAGE_SELF, &usage);
432 std::cout << myid <<
":"
433 <<
" user=" << artdaq::TimeUtils::convertUnixTimeToSeconds(usage.ru_utime)
434 <<
" sys=" << artdaq::TimeUtils::convertUnixTimeToSeconds(usage.ru_stime)
438 int main(
int argc,
char* argv[])
440 artdaq::configureMessageFacility(
"routing_master",
false);
445 std::cerr <<
"Started process " << my_rank <<
" of " << p.procs_ <<
".\n";
449 catch (std::string& x)
451 std::cerr <<
"Exception (type string) caught in routing_master: "
456 catch (
char const* m)
458 std::cerr <<
"Exception (type char const*) caught in routing_master: ";
465 std::cerr <<
"[the value was a null pointer, so no message is available]";
The RoutingMasterTest class runs the routing_master test.
void routing_master()
Load a RoutingMasterCore instance, receive tokens from the token generators, and send table updates t...
A wrapper for a MPI program. Similar to MPISentry.
void go()
Start the test, using the role assigned.
RoutingMasterTest(int argc, char *argv[])
RoutingMasterTest Constructor.
void table_receiver()
Receive Routing Tables from the Routing Master and send acknowledgement packets back.
fhicl::ParameterSet getPset(int argc, char *argv[]) const
Parse the command line arguments and load a configuration FHiCL file.
void generate_tokens()
Generate tokens and send them to the Routing Master.