1 #include "artdaq/DAQdata/Globals.hh"
2 #define TRACE_NAME (app_name + "_DataSenderManager").c_str()
3 #include "artdaq/DAQrate/DataSenderManager.hh"
4 #include "artdaq/TransferPlugins/MakeTransferPlugin.hh"
5 #include "artdaq/TransferPlugins/detail/HostMap.hh"
8 #include <netinet/in.h>
10 #include <sys/socket.h>
11 #include <sys/types.h>
14 #include "canvas/Utilities/Exception.h"
18 , broadcast_sends_(pset.get<bool>(
"broadcast_sends", false))
19 , non_blocking_mode_(pset.get<bool>(
"nonblocking_sends", false))
20 , send_timeout_us_(pset.get<size_t>(
"send_timeout_usec", 5000000))
21 , send_retry_count_(pset.get<size_t>(
"send_retry_count", 2))
22 , routing_manager_mode_(detail::RoutingManagerMode::INVALID)
26 , routing_table_last_(0)
27 , routing_table_max_size_(pset.get<size_t>(
"routing_table_max_size", 1000))
28 , highest_sequence_id_routed_(0)
30 TLOG(TLVL_DEBUG) <<
"Received pset: " << pset.to_string();
33 if (send_timeout_us_ == 0)
35 send_timeout_us_ = std::numeric_limits<size_t>::max();
38 auto rmConfig = pset.get<fhicl::ParameterSet>(
"routing_table_config", fhicl::ParameterSet());
39 use_routing_manager_ = rmConfig.get<
bool>(
"use_routing_manager",
false);
40 table_port_ = rmConfig.get<
int>(
"table_update_port", 35556);
41 table_address_ = rmConfig.get<std::string>(
"table_update_address",
"227.128.12.28");
42 table_multicast_interface_ = rmConfig.get<std::string>(
"table_update_multicast_interface",
"localhost");
43 ack_port_ = rmConfig.get<
int>(
"table_acknowledge_port", 35557);
44 ack_address_ = rmConfig.get<std::string>(
"routing_manager_hostname",
"localhost");
45 routing_timeout_ms_ = (rmConfig.get<
int>(
"routing_timeout_ms", 1000));
46 routing_retry_count_ = rmConfig.get<
int>(
"routing_retry_count", 5);
49 auto tcp_send_buffer_size = pset.get<
size_t>(
"tcp_send_buffer_size", 0);
50 auto max_fragment_size_words = pset.get<
size_t>(
"max_fragment_size_words", 0);
52 auto dests = pset.get<fhicl::ParameterSet>(
"destinations", fhicl::ParameterSet());
53 for (
auto& d : dests.get_pset_names())
55 auto dest_pset = dests.get<fhicl::ParameterSet>(d);
59 fhicl::ParameterSet dests_mod;
60 for (
auto& d : dests.get_pset_names())
62 auto dest_pset = dests.get<fhicl::ParameterSet>(d);
63 dest_pset.erase(
"host_map");
64 dest_pset.put<std::vector<fhicl::ParameterSet>>(
"host_map", host_map_pset);
66 if (tcp_send_buffer_size != 0 && !dest_pset.has_key(
"tcp_send_buffer_size"))
68 dest_pset.put<
size_t>(
"tcp_send_buffer_size", tcp_send_buffer_size);
70 if (max_fragment_size_words != 0 && !dest_pset.has_key(
"max_fragment_size_words"))
72 dest_pset.put<
size_t>(
"max_fragment_size_words", max_fragment_size_words);
75 dests_mod.put<fhicl::ParameterSet>(d, dest_pset);
78 for (
auto& d : dests_mod.get_pset_names())
83 auto destination_rank = transfer->destination_rank();
84 destinations_.emplace(destination_rank, std::move(transfer));
86 catch (
const std::invalid_argument&)
88 TLOG(TLVL_DEBUG) <<
"Invalid destination specification: " << d;
90 catch (
const cet::exception& ex)
92 TLOG(TLVL_WARNING) <<
"Caught cet::exception: " << ex.what();
96 TLOG(TLVL_WARNING) <<
"Non-cet exception while setting up TransferPlugin: " << d <<
".";
99 if (destinations_.empty())
101 TLOG(TLVL_ERROR) <<
"No destinations specified!";
105 auto enabled_dests = pset.get<std::vector<size_t>>(
"enabled_destinations", std::vector<size_t>());
106 if (enabled_dests.empty())
108 TLOG(TLVL_INFO) <<
"enabled_destinations not specified, assuming all destinations enabled.";
109 for (
auto& d : destinations_)
111 enabled_destinations_.insert(d.first);
116 for (
auto& d : enabled_dests)
118 enabled_destinations_.insert(d);
122 if (use_routing_manager_)
124 startTableReceiverThread_();
130 TLOG(TLVL_DEBUG) <<
"Shutting down DataSenderManager BEGIN";
132 for (
auto& dest : enabled_destinations_)
134 if (destinations_.count(dest) != 0u)
136 auto sts = destinations_[dest]->transfer_fragment_reliable_mode(std::move(*Fragment::eodFrag(sent_frag_count_.slotCount(dest))));
139 TLOG(TLVL_ERROR) <<
"Error sending EOD Fragment to sender rank " << dest;
146 if (routing_thread_.joinable())
148 routing_thread_.join();
154 TLOG(TLVL_DEBUG) <<
"Shutting down DataSenderManager END. Sent " << count() <<
" fragments.";
157 void artdaq::DataSenderManager::setupTableListener_()
160 table_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
161 if (table_socket_ < 0)
163 TLOG(TLVL_ERROR) <<
"Error creating socket for receiving table updates!";
167 struct sockaddr_in si_me_request;
170 if (setsockopt(table_socket_, SOL_SOCKET, SO_REUSEADDR, &yes,
sizeof(yes)) < 0)
172 TLOG(TLVL_ERROR) <<
" Unable to enable port reuse on request socket";
175 memset(&si_me_request, 0,
sizeof(si_me_request));
176 si_me_request.sin_family = AF_INET;
177 si_me_request.sin_port = htons(table_port_);
179 struct in_addr in_addr_s;
180 sts = inet_aton(table_address_.c_str(), &in_addr_s);
183 TLOG(TLVL_ERROR) <<
"inet_aton says table_address " << table_address_ <<
" is invalid";
185 si_me_request.sin_addr.s_addr = in_addr_s.s_addr;
186 if (bind(table_socket_, reinterpret_cast<struct sockaddr*>(&si_me_request),
sizeof(si_me_request)) == -1)
188 TLOG(TLVL_ERROR) <<
"Cannot bind request socket to port " << table_port_;
193 sts =
ResolveHost(table_address_.c_str(), mreq.imr_multiaddr);
196 TLOG(TLVL_ERROR) <<
"Unable to resolve multicast address for table updates";
202 TLOG(TLVL_ERROR) <<
"Unable to determine the multicast interface for table updates using " << table_multicast_interface_;
205 char addr_str[INET_ADDRSTRLEN];
206 inet_ntop(AF_INET, &(mreq.imr_interface), addr_str, INET_ADDRSTRLEN);
207 TLOG(TLVL_INFO) <<
"Successfully determined the multicast network interface for " << table_multicast_interface_ <<
": " << addr_str <<
" (DataSenderManager receiving routing table updates)";
208 if (setsockopt(table_socket_, IPPROTO_IP, IP_ADD_MEMBERSHIP, &mreq,
sizeof(mreq)) < 0)
210 TLOG(TLVL_ERROR) <<
"Unable to join multicast group";
214 void artdaq::DataSenderManager::startTableReceiverThread_()
216 if (routing_thread_.joinable())
218 routing_thread_.join();
220 TLOG(TLVL_INFO) <<
"Starting Routing Thread";
223 routing_thread_ = boost::thread(&DataSenderManager::receiveTableUpdatesLoop_,
this);
225 catch (
const boost::exception& e)
227 TLOG(TLVL_ERROR) <<
"Caught boost::exception starting Routing Table Receive thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
228 std::cerr <<
"Caught boost::exception starting Routing Table Receive thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
232 void artdaq::DataSenderManager::receiveTableUpdatesLoop_()
238 TLOG(TLVL_DEBUG) << __func__ <<
": should_stop is " << std::boolalpha << should_stop_ <<
", stopping";
241 TLOG(TLVL_DEBUG) << __func__ <<
": Sending RoutingAckPacket with end of run markers to " << ack_address_ <<
", port " << ack_port_ <<
" (my_rank = " << my_rank <<
")";
246 TLOG(TLVL_TRACE) << __func__ <<
": Polling table socket for new routes (interface,address,port = "
247 << table_multicast_interface_ <<
"," << table_address_ <<
"," << table_port_ <<
")";
248 if (table_socket_ == -1)
250 TLOG(TLVL_DEBUG) << __func__ <<
": Opening table listener socket";
251 setupTableListener_();
253 if (table_socket_ == -1)
255 TLOG(TLVL_DEBUG) << __func__ <<
": The listen socket was not opened successfully.";
258 if (ack_socket_ == -1)
260 ack_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
261 auto sts =
ResolveHost(ack_address_.c_str(), ack_port_, ack_addr_);
264 TLOG(TLVL_ERROR) << __func__ <<
": Unable to resolve routing_manager_address";
267 TLOG(TLVL_DEBUG) << __func__ <<
": Ack socket is fd " << ack_socket_;
268 char addr_str[INET_ADDRSTRLEN];
269 inet_ntop(AF_INET, &(ack_addr_.sin_addr), addr_str, INET_ADDRSTRLEN);
270 TLOG(TLVL_INFO) <<
"Successfully determined the network interface for " << ack_address_ <<
": " << addr_str <<
" (DataSenderManager sending table update acknowledgements)";
274 fd.fd = table_socket_;
275 fd.events = POLLIN | POLLPRI;
277 auto res = poll(&fd, 1, 1000);
280 auto first = artdaq::Fragment::InvalidSequenceID;
281 auto last = artdaq::Fragment::InvalidSequenceID;
282 std::vector<uint8_t> buf(MAX_ROUTING_TABLE_SIZE);
285 TLOG(TLVL_DEBUG) << __func__ <<
": Going to receive RoutingPacketHeader";
286 struct sockaddr_in from;
287 socklen_t len =
sizeof(from);
288 auto stss = recvfrom(table_socket_, &buf[0], MAX_ROUTING_TABLE_SIZE, 0, reinterpret_cast<struct sockaddr*>(&from), &len);
289 TLOG(TLVL_DEBUG) << __func__ <<
": Received " << stss <<
" bytes from " << inet_ntoa(from.sin_addr) <<
":" << from.sin_port;
291 if (stss > static_cast<ssize_t>(
sizeof(hdr)))
297 TLOG(TLVL_TRACE) << __func__ <<
": Incorrect size received. Discarding.";
301 TLOG(TLVL_DEBUG) <<
"receiveTableUpdatesLoop_: Checking for valid header with nEntries=" << hdr.
nEntries <<
"header=" << std::hex << hdr.
header;
302 if (hdr.
header != ROUTING_MAGIC)
304 TLOG(TLVL_TRACE) << __func__ <<
": non-RoutingPacket received. No ROUTING_MAGIC. size(bytes)=" << stss;
308 if (routing_manager_mode_ != detail::RoutingManagerMode::INVALID && routing_manager_mode_ != hdr.
mode)
310 TLOG(TLVL_ERROR) << __func__ <<
": Received table has different RoutingManagerMode than expected!";
313 routing_manager_mode_ = hdr.
mode;
319 first = buffer.front().sequence_id;
320 last = buffer.back().sequence_id;
322 if (first + hdr.
nEntries - 1 != last)
324 TLOG(TLVL_ERROR) << __func__ <<
": Skipping this RoutingPacket because the first (" << first <<
") and last (" << last <<
") entries are inconsistent (sz=" << hdr.
nEntries <<
")!";
327 auto thisSeqID = first;
330 std::unique_lock<std::mutex> lck(routing_mutex_);
331 if (routing_table_.count(last) == 0)
333 for (
auto entry : buffer)
335 if (thisSeqID != entry.sequence_id)
337 TLOG(TLVL_ERROR) << __func__ <<
": Aborting processing of this RoutingPacket because I encountered an inconsistent entry (seqid=" << entry.sequence_id <<
", expected=" << thisSeqID <<
")!";
338 last = thisSeqID - 1;
342 if (routing_table_.count(entry.sequence_id) != 0u)
344 if (routing_table_[entry.sequence_id] != entry.destination_rank)
346 TLOG(TLVL_ERROR) << __func__ <<
": Detected routing table corruption! Recevied update specifying that sequence ID " << entry.sequence_id
347 <<
" should go to rank " << entry.destination_rank <<
", but I had already been told to send it to " << routing_table_[entry.sequence_id] <<
"!"
348 <<
" I will use the original value!";
352 if (entry.sequence_id < routing_table_last_)
356 routing_table_[entry.sequence_id] = entry.destination_rank;
357 TLOG(TLVL_DEBUG) << __func__ <<
": (my_rank=" << my_rank <<
") received update: SeqID " << entry.sequence_id
358 <<
" -> Rank " << entry.destination_rank;
362 TLOG(TLVL_DEBUG) << __func__ <<
": There are now " << routing_table_.size() <<
" entries in the Routing Table";
363 if (!routing_table_.empty())
365 TLOG(TLVL_DEBUG) << __func__ <<
": Last routing table entry is seqID=" << routing_table_.rbegin()->first;
369 for (
auto& entry : routing_table_)
371 TLOG(45) <<
"Routing Table Entry" << counter <<
": " << entry.first <<
" -> " << entry.second;
381 if (last > routing_table_last_)
383 routing_table_last_ = last;
388 TLOG(TLVL_DEBUG) << __func__ <<
": Skipping RoutingAckPacket since this Routing Table Update has already been acknowledged (my_rank = " << my_rank <<
")";
392 TLOG(TLVL_DEBUG) << __func__ <<
": Sending RoutingAckPacket with first= " << first <<
" and last= " << last <<
" to " << ack_address_ <<
", port " << ack_port_ <<
" (my_rank = " << my_rank <<
")";
402 std::unique_lock<std::mutex> lck(routing_mutex_);
403 return routing_table_.size();
408 std::unique_lock<std::mutex> lck(routing_mutex_);
410 size_t dist = std::distance(routing_table_.upper_bound(highest_sequence_id_routed_), routing_table_.end());
414 int artdaq::DataSenderManager::calcDest_(Fragment::sequence_id_t sequence_id)
const
416 if (enabled_destinations_.empty())
420 if (!use_routing_manager_ && enabled_destinations_.size() == 1)
422 return *enabled_destinations_.begin();
425 if (use_routing_manager_)
427 auto start = std::chrono::steady_clock::now();
428 TLOG(15) <<
"calcDest_ use_routing_manager check for routing info for seqID=" << sequence_id <<
" routing_timeout_ms=" << routing_timeout_ms_ <<
" should_stop_=" << should_stop_;
429 while (!should_stop_ && (routing_timeout_ms_ <= 0 || TimeUtils::GetElapsedTimeMilliseconds(start) < static_cast<size_t>(routing_timeout_ms_)))
432 std::unique_lock<std::mutex> lck(routing_mutex_);
435 if (sequence_id > highest_sequence_id_routed_)
437 highest_sequence_id_routed_ = sequence_id;
439 routing_wait_time_.fetch_add(TimeUtils::GetElapsedTimeMicroseconds(start));
440 return routing_table_.at(sequence_id);
444 if (sent_frag_count_.count() + 1 > highest_sequence_id_routed_)
446 highest_sequence_id_routed_ = sent_frag_count_.count() + 1;
448 routing_wait_time_.fetch_add(TimeUtils::GetElapsedTimeMicroseconds(start));
449 return routing_table_.at(sent_frag_count_.count() + 1);
452 usleep(routing_timeout_ms_ * 10);
454 routing_wait_time_.fetch_add(TimeUtils::GetElapsedTimeMicroseconds(start));
457 TLOG(TLVL_WARNING) <<
"Bad Omen: I don't have routing information for seqID " << sequence_id
458 <<
" and the Routing Manager did not send a table update in routing_timeout_ms window (" << routing_timeout_ms_ <<
" ms)!";
462 TLOG(TLVL_WARNING) <<
"Bad Omen: I don't have routing information for send number " << sent_frag_count_.count()
463 <<
" and the Routing Manager did not send a table update in routing_timeout_ms window (" << routing_timeout_ms_ <<
" ms)!";
468 auto index = sequence_id % enabled_destinations_.size();
469 auto it = enabled_destinations_.begin();
470 for (; index > 0; --index)
473 if (it == enabled_destinations_.end())
475 it = enabled_destinations_.begin();
485 TLOG(15) <<
"RemoveRoutingTableEntry: Removing sequence ID " << seq <<
" from routing table. Sent " << GetSentSequenceIDCount(seq) <<
" Fragments with this Sequence ID.";
486 std::unique_lock<std::mutex> lck(routing_mutex_);
491 if (routing_table_.find(seq) != routing_table_.end())
493 routing_table_.erase(routing_table_.find(seq));
496 if (sent_sequence_id_count_.find(seq) != sent_sequence_id_count_.end())
498 sent_sequence_id_count_.erase(sent_sequence_id_count_.find(seq));
504 std::unique_lock<std::mutex> lck(routing_mutex_);
505 if (sent_sequence_id_count_.count(seq) == 0u)
509 return sent_sequence_id_count_[seq];
516 auto start_time = std::chrono::steady_clock::now();
517 if (frag.type() == Fragment::EndOfDataFragmentType)
519 throw cet::exception(
"LogicError")
520 <<
"EOD fragments should not be sent on as received: "
521 <<
"use sendEODFrag() instead.";
523 size_t seqID = frag.sequenceID();
524 size_t fragSize = frag.sizeBytes();
525 auto latency_s = frag.getLatency(
true);
526 auto isSystemBroadcast = frag.type() == Fragment::EndOfRunFragmentType || frag.type() == Fragment::EndOfSubrunFragmentType || frag.type() == Fragment::InitFragmentType;
528 double latency = latency_s.tv_sec + (latency_s.tv_nsec / 1000000000.0);
529 TLOG(13) <<
"sendFragment start frag.fragmentHeader()=" << std::hex << static_cast<void*>(frag.headerBeginBytes()) <<
", szB=" << std::dec << fragSize
530 <<
", seqID=" << seqID <<
", fragID=" << frag.fragmentID() <<
", type=" << frag.typeString();
533 if (broadcast_sends_ || isSystemBroadcast)
535 for (
auto& bdest : enabled_destinations_)
537 TLOG(TLVL_TRACE) <<
"sendFragment: Sending fragment with seqId " << seqID <<
" to destination " << bdest <<
" (broadcast)";
543 if (!non_blocking_mode_)
545 sts = destinations_[bdest]->transfer_fragment_reliable_mode(Fragment(frag));
549 sts = destinations_[bdest]->transfer_fragment_min_blocking_mode(frag, send_timeout_us_);
557 sent_frag_count_.incSlot(bdest);
560 else if (non_blocking_mode_)
562 auto count = routing_retry_count_;
565 dest = calcDest_(seqID);
569 TLOG(TLVL_WARNING) <<
"Could not get destination for seqID " << seqID << (count > 0 ?
", retrying." :
".");
574 TLOG(TLVL_TRACE) <<
"sendFragment: Sending fragment with seqId " << seqID <<
" to destination " << dest;
576 auto lastWarnTime = std::chrono::steady_clock::now();
580 sts = destinations_[dest]->transfer_fragment_min_blocking_mode(frag, send_timeout_us_);
583 TLOG(TLVL_WARNING) <<
"sendFragment: Sending fragment " << seqID <<
" to destination " << dest <<
" failed! Retrying...";
584 lastWarnTime = std::chrono::steady_clock::now();
593 sent_frag_count_.incSlot(dest);
595 else if (!should_stop_)
597 TLOG(TLVL_ERROR) <<
"(in non_blocking) calcDest returned invalid destination rank " << dest <<
"! This event has been lost: " << seqID
598 <<
". enabled_destinantions_.size()=" << enabled_destinations_.size();
603 auto start = std::chrono::steady_clock::now();
606 dest = calcDest_(seqID);
609 TLOG(TLVL_WARNING) <<
"Could not get destination for seqID " << seqID <<
", send number " << sent_frag_count_.count() <<
", retrying. Waited " << TimeUtils::GetElapsedTime(start) <<
" s for routing information.";
615 TLOG(5) <<
"DataSenderManager::sendFragment: Sending fragment with seqId " << seqID <<
" to destination " << dest;
618 sts = destinations_[dest]->transfer_fragment_reliable_mode(std::move(frag));
621 TLOG(TLVL_ERROR) <<
"sendFragment: Sending fragment " << seqID <<
" to destination "
622 << dest <<
" failed! Data has been lost!";
626 sent_frag_count_.incSlot(dest);
629 else if (!should_stop_)
631 TLOG(TLVL_ERROR) <<
"calcDest returned invalid destination rank " << dest <<
"! This event has been lost: " << seqID
632 <<
". enabled_destinantions_.size()=" << enabled_destinations_.size();
636 if (!isSystemBroadcast)
638 std::unique_lock<std::mutex> lck(routing_mutex_);
639 sent_sequence_id_count_[seqID]++;
642 auto delta_t = TimeUtils::GetElapsedTime(start_time);
646 TLOG(5) <<
"sendFragment: sending metrics";
647 metricMan->sendMetric(
"Data Send Time to Rank " + std::to_string(dest), delta_t,
"s", 5, MetricMode::Accumulate);
648 metricMan->sendMetric(
"Data Send Size to Rank " + std::to_string(dest), fragSize,
"B", 5, MetricMode::Accumulate);
649 metricMan->sendMetric(
"Data Send Rate to Rank " + std::to_string(dest), fragSize / delta_t,
"B/s", 5, MetricMode::Average);
650 metricMan->sendMetric(
"Data Send Count to Rank " + std::to_string(dest), sent_frag_count_.slotCount(dest),
"fragments", 3, MetricMode::LastPoint);
651 metricMan->sendMetric(
"Fragment Latency at Send", latency,
"s", 4, MetricMode::Average | MetricMode::Maximum);
653 if (use_routing_manager_)
655 metricMan->sendMetric(
"Routing Table Size", GetRoutingTableEntryCount(),
"events", 2, MetricMode::LastPoint);
656 if (routing_wait_time_ > 0)
658 metricMan->sendMetric(
"Routing Wait Time", static_cast<double>(routing_wait_time_.load()) / 1000000,
"s", 2, MetricMode::Average);
659 routing_wait_time_ = 0;
663 TLOG(5) <<
"sendFragment: Done sending fragment " << seqID <<
" to dest=" << dest;
664 return std::make_pair(dest, outsts);
void RemoveRoutingTableEntry(Fragment::sequence_id_t seq)
Remove the given sequence ID from the routing table and sent_count lists.
int ResolveHost(char const *host_in, in_addr &addr)
Convert a string hostname to a in_addr suitable for socket communication.
A row of the Routing Table.
Events should be routed by sequence ID (BR -> EB)
The send operation timed out.
virtual ~DataSenderManager()
DataSenderManager Destructor.
A RoutingAckPacket contains the rank of the table receiver, plus the first and last sequence IDs in t...
int GetInterfaceForNetwork(char const *host_in, in_addr &addr)
Convert an IP address to the network address of the interface sharing the subnet mask.
std::unique_ptr< artdaq::TransferInterface > MakeTransferPlugin(const fhicl::ParameterSet &pset, const std::string &plugin_label, TransferInterface::Role role)
Load a TransferInterface plugin.
size_t GetSentSequenceIDCount(Fragment::sequence_id_t seq)
Get the number of Fragments sent with a given Sequence ID.
std::pair< int, TransferInterface::CopyStatus > sendFragment(Fragment &&frag)
Send the given Fragment. Return the rank of the destination to which the Fragment was sent...
size_t GetRemainingRoutingTableEntries() const
Gets the number of sends remaining in the routing table, in case other parts of the system want to us...
DataSenderManager(const fhicl::ParameterSet &ps)
DataSenderManager Constructor.
Fragment::sequence_id_t first_sequence_id
The first sequence ID in the received RoutingPacket.
std::vector< fhicl::ParameterSet > MakeHostMapPset(std::map< int, std::string > input)
Create a list of HostMap::HostConfig ParameterSets from a hostMap_t map
size_t GetRoutingTableEntryCount() const
Gets the current size of the Routing Table, in case other parts of the system want to use this inform...
This TransferInterface is a Sender.
Fragment::sequence_id_t last_sequence_id
The last sequence ID in the received RoutingPacket.
std::vector< RoutingPacketEntry > RoutingPacket
A RoutingPacket is simply a vector of RoutingPacketEntry objects. It is not suitable for network tran...
int rank
The rank from which the RoutingAckPacket came.
Events should be routed by send count (EB -> Agg)
Some error occurred, but no exception was thrown.
The send operation completed successfully.
Value to be returned upon receive timeout.
std::map< int, std::string > hostMap_t
The host_map is a map associating ranks with artdaq::DestinationInfo objects.
hostMap_t MakeHostMap(fhicl::ParameterSet const &pset, hostMap_t map=hostMap_t())
Make a hostMap_t from a HostMap::Config ParameterSet
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.