2 #include "artdaq/Application/Routing/RoutingPacket.hh"
5 #include "cetlib/filepath_maker.h"
6 #include "fhiclcpp/ParameterSet.h"
7 #include "fhiclcpp/make_ParameterSet.h"
9 #include <boost/program_options.hpp>
10 #include <boost/filesystem.hpp>
11 #include "artdaq/Application/RoutingMasterCore.hh"
12 #include "artdaq/Application/RoutingMasterApp.hh"
14 namespace bpo = boost::program_options;
28 #include <arpa/inet.h>
29 #include <netinet/in.h>
30 #include <sys/types.h>
31 #include <sys/socket.h>
36 #include <sys/resource.h>
49 explicit LockFile(std::string path) : fileName_(path)
51 std::ofstream fstream(fileName_);
52 fstream <<
"Locked" << std::endl;
60 if(
IsLocked(fileName_))
remove(fileName_.c_str());
69 return boost::filesystem::exists(path);
73 std::string fileName_;
124 fhicl::ParameterSet
getPset(
int argc,
char* argv[])
const;
127 enum class TestRole_t : int
134 void printHost(
const std::string& functionName)
const;
136 fhicl::ParameterSet
const pset_;
137 fhicl::ParameterSet
const daq_pset_;
138 MPI_Comm local_group_comm_;
141 std::string routing_master_address_;
142 std::string multicast_address_;
146 std::vector<int> eb_ranks_;
148 size_t token_interval_us_;
153 , pset_(getPset(argc, argv))
154 , daq_pset_(pset_.get<fhicl::ParameterSet>(
"daq"))
155 , local_group_comm_()
156 , routing_master_address_(daq_pset_.get<std::string>(
"routing_master_hostname",
"localhost"))
157 , multicast_address_(daq_pset_.get<std::string>(
"table_update_address",
"227.128.12.28"))
158 , token_port_(daq_pset_.get<int>(
"routing_token_port", 35555))
159 , table_port_(daq_pset_.get<int>(
"table_update_port", 35556))
160 , ack_port_(daq_pset_.get<int>(
"table_acknowledge_port", 35557))
161 , token_count_(pset_.get<int>(
"token_count", 1000))
162 , token_interval_us_(pset_.get<size_t>(
"token_interval_us", 5000))
164 assert(!(my_rank < 0));
168 role_ = TestRole_t::TOKEN_GEN;
171 role_ = TestRole_t::ROUTING_MASTER;
174 role_ = TestRole_t::TABLE_RECEIVER;
177 auto policy_pset = daq_pset_.get<fhicl::ParameterSet>(
"policy");
178 eb_ranks_ = policy_pset.get<std::vector<int>>(
"receiver_ranks");
184 std::ostringstream descstr;
185 descstr <<
"-- <-c <config-file>>";
186 bpo::options_description desc(descstr.str());
188 (
"config,c", bpo::value<std::string>(),
"Configuration file.");
189 bpo::variables_map vm;
192 bpo::store(bpo::command_line_parser(argc, argv).
193 options(desc).allow_unregistered().run(), vm);
196 catch (bpo::error
const& e)
198 std::cerr <<
"Exception from command line processing in Config::getArtPset: " << e.what() <<
"\n";
199 throw "cmdline parsing error.";
201 if (!vm.count(
"config"))
203 std::cerr <<
"Expected \"-- -c <config-file>\" fhicl file specification.\n";
204 throw "cmdline parsing error.";
206 fhicl::ParameterSet pset;
207 cet::filepath_lookup lookup_policy(
"FHICL_FILE_PATH");
208 fhicl::make_ParameterSet(vm[
"config"].as<std::string>(), lookup_policy, pset);
216 MPI_Barrier(MPI_COMM_WORLD);
217 std::unique_ptr<LockFile> lock;
219 lock = std::make_unique<LockFile>(
"/tmp/routing_master_t.lock");
222 MPI_Comm_split(MPI_COMM_WORLD, static_cast<int>(role_), 0, &local_group_comm_);
225 case TestRole_t::TABLE_RECEIVER:
228 case TestRole_t::ROUTING_MASTER:
231 case TestRole_t::TOKEN_GEN:
235 throw "No such node type";
237 TLOG_DEBUG(
"routing_master") <<
"Rank " << my_rank <<
" complete." << TLOG_ENDL;
242 TLOG_DEBUG(
"generate_tokens") <<
"Init" << TLOG_ENDL;
243 printHost(
"generate_tokens");
246 int token_socket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
249 TLOG_ERROR(
"generate_tokens") <<
"I failed to create the socket for sending Routing Tokens!" << TLOG_ENDL;
252 struct sockaddr_in token_addr;
253 auto sts =
ResolveHost(routing_master_address_.c_str(), token_port_, token_addr);
256 TLOG_ERROR(
"generate_tokens") <<
"Could not resolve host name" << TLOG_ENDL;
259 connect(token_socket, (
struct sockaddr*)&token_addr,
sizeof(token_addr));
262 std::map<int, int> token_counter;
263 for(
auto rank : eb_ranks_)
265 token_counter[rank] = 0;
267 while (sent_tokens < token_count_) {
268 int this_rank = eb_ranks_[rand() % eb_ranks_.size()];
269 token_counter[this_rank]++;
271 token.
header = TOKEN_MAGIC;
272 token.
rank = this_rank;
275 TLOG_DEBUG(
"generate_tokens") <<
"Sending RoutingToken " << std::to_string(++sent_tokens) <<
" for rank " << this_rank <<
" to " << routing_master_address_ << TLOG_ENDL;
277 usleep(token_interval_us_);
280 for(
auto rank : token_counter)
282 if (rank.second > max_rank) max_rank = rank.second;
284 for(
auto rank : token_counter)
287 token.
header = TOKEN_MAGIC;
288 token.
rank = rank.first;
291 TLOG_DEBUG(
"generate_tokens") <<
"Sending RoutingToken " << std::to_string(++sent_tokens) <<
" for rank " << rank.first <<
" to " << routing_master_address_ << TLOG_ENDL;
293 usleep(token_interval_us_);
297 MPI_Comm_free(&local_group_comm_);
298 TLOG_INFO(
"generate_tokens") <<
"Waiting at MPI_Barrier" << TLOG_ENDL;
299 MPI_Barrier(MPI_COMM_WORLD);
300 TLOG_INFO(
"generate_tokens") <<
"Done with MPI_Barrier" << TLOG_ENDL;
305 TLOG_DEBUG(
"table_receiver") <<
"Init" << TLOG_ENDL;
306 printHost(
"table_receiver");
309 auto table_socket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
312 TLOG_ERROR(
"table_receiver") <<
"Error creating socket for receiving data requests!" << TLOG_ENDL;
316 struct sockaddr_in si_me_request;
319 if (setsockopt(table_socket, SOL_SOCKET, SO_REUSEADDR, &yes,
sizeof(yes)) < 0)
321 TLOG_ERROR(
"table_receiver") <<
" Unable to enable port reuse on request socket" << TLOG_ENDL;
324 memset(&si_me_request, 0,
sizeof(si_me_request));
325 si_me_request.sin_family = AF_INET;
326 si_me_request.sin_port = htons(table_port_);
327 si_me_request.sin_addr.s_addr = htonl(INADDR_ANY);
328 if (bind(table_socket, (
struct sockaddr *)&si_me_request,
sizeof(si_me_request)) == -1)
330 TLOG_ERROR(
"table_receiver") <<
"Cannot bind request socket to port " << table_port_ << TLOG_ENDL;
335 long int sts =
ResolveHost(multicast_address_.c_str(), mreq.imr_multiaddr);
338 TLOG_ERROR(
"table_Receiver") <<
"Unable to resolve multicast hostname" << TLOG_ENDL;
341 mreq.imr_interface.s_addr = htonl(INADDR_ANY);
342 if (setsockopt(table_socket, IPPROTO_IP, IP_ADD_MEMBERSHIP, &mreq,
sizeof(mreq)) < 0)
344 TLOG_ERROR(
"table_receiver") <<
"Unable to join multicast group" << TLOG_ENDL;
348 struct epoll_event ev;
349 int table_epoll_fd = epoll_create1(0);
350 ev.events = EPOLLIN | EPOLLPRI;
351 ev.data.fd = table_socket;
352 if (epoll_ctl(table_epoll_fd, EPOLL_CTL_ADD, table_socket, &ev) == -1)
354 TLOG_ERROR(
"table_receiver") <<
"Could not register listen socket to epoll fd" << TLOG_ENDL;
358 auto ack_socket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
359 struct sockaddr_in ack_addr;
360 sts =
ResolveHost(routing_master_address_.c_str(), ack_port_, ack_addr);
363 TLOG_ERROR(
"table_Receiver") <<
"Unable to resolve routing master hostname" << TLOG_ENDL;
367 if (table_socket == -1 || table_epoll_fd == -1 || ack_socket == -1)
369 TLOG_DEBUG(
"table_receiver") <<
"One of the listen sockets was not opened successfully." << TLOG_ENDL;
372 artdaq::Fragment::sequence_id_t max_sequence_id = token_count_;
373 artdaq::Fragment::sequence_id_t current_sequence_id = 0;
374 std::map<artdaq::Fragment::sequence_id_t, int> routing_table;
375 TLOG_INFO(
"table_receiver") <<
"Expecting " << std::to_string(max_sequence_id) <<
" as the last Sequence ID in this run" << TLOG_ENDL;
376 while (current_sequence_id < max_sequence_id)
378 std::vector<epoll_event> table_events_(4);
379 TLOG_DEBUG(
"table_receiver") <<
"Waiting for event on table socket" << TLOG_ENDL;
380 auto nfds = epoll_wait(table_epoll_fd, &table_events_[0], table_events_.size(), -1);
382 perror(
"epoll_wait");
386 TLOG_DEBUG(
"table_receiver") <<
"Received " << nfds <<
" table update(s)" << TLOG_ENDL;
387 for (
auto n = 0; n < nfds; ++n) {
388 auto first = artdaq::Fragment::InvalidSequenceID;
389 auto last = artdaq::Fragment::InvalidSequenceID;
393 TLOG_DEBUG(
"table_receiver") <<
"Checking for valid header" << TLOG_ENDL;
394 if (hdr.
header == ROUTING_MAGIC) {
396 TLOG_DEBUG(
"table_receiver") <<
"Receiving data buffer" << TLOG_ENDL;
400 first = buffer[0].sequence_id;
401 last = buffer[buffer.size() - 1].sequence_id;
403 for (
auto entry : buffer)
405 if (routing_table.count(entry.sequence_id))
407 assert(routing_table[entry.sequence_id] == entry.destination_rank);
410 routing_table[entry.sequence_id] = entry.destination_rank;
411 TLOG_DEBUG(
"table_receiver") <<
"table_receiver " << std::to_string(my_rank) <<
": received update: SeqID " << std::to_string(entry.sequence_id) <<
" -> Rank " << std::to_string(entry.destination_rank) << TLOG_ENDL;
419 TLOG_DEBUG(
"table_receiver") <<
"Sending RoutingAckPacket with first= " << std::to_string(first) <<
" and last= " << std::to_string(last) <<
" to " << routing_master_address_ <<
", port " << ack_port_ << TLOG_ENDL
421 current_sequence_id = last;
426 MPI_Comm_free(&local_group_comm_);
427 TLOG_INFO(
"table_receiver") <<
"Waiting at MPI_Barrier" << TLOG_ENDL;
428 MPI_Barrier(MPI_COMM_WORLD);
429 TLOG_INFO(
"table_receiver") <<
"Done with MPI_Barrier" << TLOG_ENDL;
434 TLOG_DEBUG(
"routing_master") <<
"Init" << TLOG_ENDL;
435 printHost(
"routing_master");
437 auto app = std::make_unique<artdaq::RoutingMasterApp>(local_group_comm_,
"RoutingMaster");
439 app->initialize(pset_, 0, 0);
440 app->do_start(art::RunID(1), 0, 0);
441 TLOG_INFO(
"routing_master") <<
"Waiting at MPI_Barrier" << TLOG_ENDL;
442 MPI_Barrier(MPI_COMM_WORLD);
443 TLOG_INFO(
"routing_master") <<
"Done with MPI_Barrier, calling RoutingMasterCore::stop" << TLOG_ENDL;
445 TLOG_INFO(
"routing_master") <<
"Done with RoutingMasterCore::stop, calling shutdown" << TLOG_ENDL;
447 TLOG_INFO(
"routing_master") <<
"Done with RoutingMasterCore::shutdown" << TLOG_ENDL;
448 MPI_Comm_free(&local_group_comm_);
451 void RoutingMasterTest::printHost(
const std::string& functionName)
const
453 char* doPrint = getenv(
"PRINT_HOST");
454 if (doPrint == 0) {
return; }
455 const int ARRSIZE = 80;
456 char hostname[ARRSIZE];
457 std::string hostString;
458 if (!gethostname(hostname, ARRSIZE))
460 hostString = hostname;
464 hostString =
"unknown";
466 TLOG_DEBUG(
"routing_master") <<
"Running " << functionName
467 <<
" on host " << hostString
468 <<
" with rank " << my_rank <<
"."
476 getrusage(RUSAGE_SELF, &usage);
477 std::cout << myid <<
":"
483 int main(
int argc,
char* argv[])
490 std::cerr <<
"Started process " << my_rank <<
" of " << p.procs_ <<
".\n";
494 catch (std::string& x)
496 std::cerr <<
"Exception (type string) caught in routing_master: "
501 catch (
char const* m)
503 std::cerr <<
"Exception (type char const*) caught in routing_master: ";
510 std::cerr <<
"[the value was a null pointer, so no message is available]";
The RoutingMasterTest class runs the routing_master test.
int ResolveHost(char const *host_in, in_addr &addr)
Convert a string hostname to a in_addr suitable for socket communication.
A row of the Routing Table.
void routing_master()
Load a RoutingMasterCore instance, receive tokens from the token generators, and send table updates t...
A wrapper for a MPI program. Similar to MPISentry.
A RoutingAckPacket contains the rank of the table receiver, plus the first and last sequence IDs in t...
void configureMessageFacility(char const *progname, bool useConsole=true)
Configure and start the message facility. Provide the program name so that messages will be appropria...
void go()
Start the test, using the role assigned.
unsigned new_slots_free
The number of slots free in the token sender (usually 1)
The RoutingToken contains the magic bytes, the rank of the token sender, and the number of slots free...
RoutingMasterTest(int argc, char *argv[])
RoutingMasterTest Constructor.
void table_receiver()
Receive Routing Tables from the Routing Master and send acknowledgement packets back.
Fragment::sequence_id_t first_sequence_id
The first sequence ID in the received RoutingPacket.
fhicl::ParameterSet getPset(int argc, char *argv[]) const
Parse the command line arguments and load a configuration FHiCL file.
Fragment::sequence_id_t last_sequence_id
The last sequence ID in the received RoutingPacket.
std::vector< RoutingPacketEntry > RoutingPacket
A RoutingPacket is simply a vector of RoutingPacketEntry objects. It is not suitable for network tran...
int rank
The rank from which the RoutingToken came.
int rank
The rank from which the RoutingAckPacket came.
void generate_tokens()
Generate tokens and send them to the Routing Master.
static bool IsLocked(std::string path)
Check if the given lock file exists.
Create a "lock file", removing it upon class destruction.
LockFile(std::string path)
Create a lock file with the given path.
static double timevalAsDouble(struct timeval tv)
Convert a timeval value to a double.
uint32_t header
The magic bytes that help validate the RoutingToken.
~LockFile()
LockFile Destructor, removes the lock file.