00001 #define TRACE_NAME "DataSenderManager"
00002 #include "artdaq/DAQdata/Globals.hh"
00003 #include "artdaq/DAQrate/DataSenderManager.hh"
00004 #include "artdaq/TransferPlugins/MakeTransferPlugin.hh"
00005 #include "artdaq/TransferPlugins/detail/HostMap.hh"
00006
00007 #include <chrono>
00008 #include "canvas/Utilities/Exception.h"
00009 #include <arpa/inet.h>
00010 #include <netinet/in.h>
00011 #include <sys/types.h>
00012 #include <poll.h>
00013 #include <sys/socket.h>
00014 #include "artdaq/DAQdata/TCPConnect.hh"
00015
00016 artdaq::DataSenderManager::DataSenderManager(const fhicl::ParameterSet& pset)
00017 : destinations_()
00018 , enabled_destinations_()
00019 , sent_frag_count_()
00020 , broadcast_sends_(pset.get<bool>("broadcast_sends", false))
00021 , non_blocking_mode_(pset.get<bool>("nonblocking_sends", false))
00022 , send_timeout_us_(pset.get<size_t>("send_timeout_usec", 5000000))
00023 , send_retry_count_(pset.get<size_t>("send_retry_count", 2))
00024 , routing_master_mode_(detail::RoutingMasterMode::INVALID)
00025 , should_stop_(false)
00026 , ack_socket_(-1)
00027 , table_socket_(-1)
00028 {
00029 TLOG(TLVL_DEBUG) << "Received pset: " << pset.to_string();
00030
00031
00032 if (send_timeout_us_ == 0) send_timeout_us_ = std::numeric_limits<size_t>::max();
00033
00034 auto rmConfig = pset.get<fhicl::ParameterSet>("routing_table_config", fhicl::ParameterSet());
00035 use_routing_master_ = rmConfig.get<bool>("use_routing_master", false);
00036 table_port_ = rmConfig.get<int>("table_update_port", 35556);
00037 table_address_ = rmConfig.get<std::string>("table_update_address", "227.128.12.28");
00038 ack_port_ = rmConfig.get<int>("table_acknowledge_port", 35557);
00039 ack_address_ = rmConfig.get<std::string>("routing_master_hostname", "localhost");
00040 routing_timeout_ms_ = (rmConfig.get<int>("routing_timeout_ms", 1000));
00041 routing_retry_count_ = rmConfig.get<int>("routing_retry_count", 5);
00042
00043
00044 hostMap_t host_map = MakeHostMap(pset);
00045
00046 auto dests = pset.get<fhicl::ParameterSet>("destinations", fhicl::ParameterSet());
00047 for (auto& d : dests.get_pset_names())
00048 {
00049 auto dest_pset = dests.get<fhicl::ParameterSet>(d);
00050 host_map = MakeHostMap(dest_pset, 0, host_map);
00051 }
00052 auto host_map_pset = MakeHostMapPset(host_map);
00053 fhicl::ParameterSet dests_mod;
00054 for (auto& d : dests.get_pset_names())
00055 {
00056 auto dest_pset = dests.get<fhicl::ParameterSet>(d);
00057 dest_pset.erase("host_map");
00058 dest_pset.put<std::vector<fhicl::ParameterSet>>("host_map", host_map_pset);
00059 dests_mod.put<fhicl::ParameterSet>(d, dest_pset);
00060 }
00061
00062 for (auto& d : dests.get_pset_names())
00063 {
00064 try
00065 {
00066 auto transfer = MakeTransferPlugin(dests, d, TransferInterface::Role::kSend);
00067 auto destination_rank = transfer->destination_rank();
00068 destinations_.emplace(destination_rank, std::move(transfer));
00069 }
00070 catch (std::invalid_argument)
00071 {
00072 TLOG(TLVL_DEBUG) << "Invalid destination specification: " << d;
00073 }
00074 catch (cet::exception ex)
00075 {
00076 TLOG(TLVL_WARNING) << "Caught cet::exception: " << ex.what();
00077 }
00078 catch (...)
00079 {
00080 TLOG(TLVL_WARNING) << "Non-cet exception while setting up TransferPlugin: " << d << ".";
00081 }
00082 }
00083 if (destinations_.size() == 0)
00084 {
00085 TLOG(TLVL_ERROR) << "No destinations specified!";
00086 }
00087 else
00088 {
00089 auto enabled_dests = pset.get<std::vector<size_t>>("enabled_destinations", std::vector<size_t>());
00090 if (enabled_dests.size() == 0)
00091 {
00092 TLOG(TLVL_INFO) << "enabled_destinations not specified, assuming all destinations enabled.";
00093 for (auto& d : destinations_)
00094 {
00095 enabled_destinations_.insert(d.first);
00096 }
00097 }
00098 else
00099 {
00100 for (auto& d : enabled_dests)
00101 {
00102 enabled_destinations_.insert(d);
00103 }
00104 }
00105 }
00106 if (use_routing_master_) startTableReceiverThread_();
00107 }
00108
00109 artdaq::DataSenderManager::~DataSenderManager()
00110 {
00111 TLOG(TLVL_DEBUG) << "Shutting down DataSenderManager BEGIN";
00112 should_stop_ = true;
00113 for (auto& dest : enabled_destinations_)
00114 {
00115 if (destinations_.count(dest))
00116 {
00117 auto sts = destinations_[dest]->moveFragment(std::move(*Fragment::eodFrag(sent_frag_count_.slotCount(dest))));
00118 if (sts != TransferInterface::CopyStatus::kSuccess) TLOG(TLVL_ERROR) << "Error sending EOD Fragment to sender rank " << dest;
00119
00120 }
00121 }
00122 if (routing_thread_.joinable()) routing_thread_.join();
00123 TLOG(TLVL_DEBUG) << "Shutting down DataSenderManager END. Sent " << count() << " fragments.";
00124 }
00125
00126
00127 void artdaq::DataSenderManager::setupTableListener_()
00128 {
00129 table_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
00130 if (table_socket_ < 0)
00131 {
00132 TLOG(TLVL_ERROR) << "Error creating socket for receiving table updates!";
00133 exit(1);
00134 }
00135
00136 struct sockaddr_in si_me_request;
00137
00138 int yes = 1;
00139 if (setsockopt(table_socket_, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) < 0)
00140 {
00141 TLOG(TLVL_ERROR) << " Unable to enable port reuse on request socket";
00142 exit(1);
00143 }
00144 memset(&si_me_request, 0, sizeof(si_me_request));
00145 si_me_request.sin_family = AF_INET;
00146 si_me_request.sin_port = htons(table_port_);
00147 si_me_request.sin_addr.s_addr = htonl(INADDR_ANY);
00148 if (bind(table_socket_, (struct sockaddr *)&si_me_request, sizeof(si_me_request)) == -1)
00149 {
00150 TLOG(TLVL_ERROR) << "Cannot bind request socket to port " << table_port_;
00151 exit(1);
00152 }
00153
00154 struct ip_mreq mreq;
00155 int sts = ResolveHost(table_address_.c_str(), mreq.imr_multiaddr);
00156 if (sts == -1)
00157 {
00158 TLOG(TLVL_ERROR) << "Unable to resolve multicast address for table updates";
00159 exit(1);
00160 }
00161 mreq.imr_interface.s_addr = htonl(INADDR_ANY);
00162 if (setsockopt(table_socket_, IPPROTO_IP, IP_ADD_MEMBERSHIP, &mreq, sizeof(mreq)) < 0)
00163 {
00164 TLOG(TLVL_ERROR) << "Unable to join multicast group";
00165 exit(1);
00166 }
00167 }
00168 void artdaq::DataSenderManager::startTableReceiverThread_()
00169 {
00170 if (routing_thread_.joinable()) routing_thread_.join();
00171 TLOG(TLVL_INFO) << "Starting Routing Thread";
00172 routing_thread_ = boost::thread(&DataSenderManager::receiveTableUpdatesLoop_, this);
00173 }
00174 void artdaq::DataSenderManager::receiveTableUpdatesLoop_()
00175 {
00176 while (true)
00177 {
00178 if (should_stop_)
00179 {
00180 TLOG(TLVL_DEBUG) << "receiveTableUpdatesLoop: should_stop is " << std::boolalpha << should_stop_ << ", stopping";
00181 return;
00182 }
00183
00184 TLOG(TLVL_TRACE) << "DataSenderManager::receiveTableUpdatesLoop: Polling Request socket for new requests";
00185 if (table_socket_ == -1)
00186 {
00187 TLOG(TLVL_DEBUG) << "Opening table listener socket";
00188 setupTableListener_();
00189 }
00190 if (table_socket_ == -1)
00191 {
00192 TLOG(TLVL_DEBUG) << "The listen socket was not opened successfully.";
00193 return;
00194 }
00195 if (ack_socket_ == -1)
00196 {
00197 ack_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
00198 auto sts = ResolveHost(ack_address_.c_str(), ack_port_, ack_addr_);
00199 if (sts == -1)
00200 {
00201 TLOG(TLVL_ERROR) << "Unable to resolve routing_master_address";
00202 exit(1);
00203 }
00204 TLOG(TLVL_DEBUG) << "Ack socket is fd " << ack_socket_;
00205 }
00206
00207 struct pollfd fd;
00208 fd.fd = table_socket_;
00209 fd.events = POLLIN | POLLPRI;
00210
00211 auto res = poll(&fd, 1, 1000);
00212 if (res > 0)
00213 {
00214 auto first = artdaq::Fragment::InvalidSequenceID;
00215 auto last = artdaq::Fragment::InvalidSequenceID;
00216 artdaq::detail::RoutingPacketHeader hdr;
00217
00218 TLOG(TLVL_DEBUG) << "Going to receive RoutingPacketHeader";
00219 auto stss = recvfrom(table_socket_, &hdr, sizeof(artdaq::detail::RoutingPacketHeader), 0, NULL, NULL);
00220 TLOG(TLVL_DEBUG) << "Received " << std::to_string(stss) << " bytes. (sizeof(RoutingPacketHeader) == " << std::to_string(sizeof(detail::RoutingPacketHeader));
00221
00222 TLOG(TLVL_DEBUG) << "Checking for valid header";
00223 if (hdr.header == ROUTING_MAGIC)
00224 {
00225 if (routing_master_mode_ != detail::RoutingMasterMode::INVALID && routing_master_mode_ != hdr.mode)
00226 {
00227 TLOG(TLVL_ERROR) << "Received table has different RoutingMasterMode than expected!";
00228 exit(1);
00229 }
00230 routing_master_mode_ = hdr.mode;
00231
00232 artdaq::detail::RoutingPacket buffer(hdr.nEntries);
00233 TLOG(TLVL_DEBUG) << "Receiving data buffer";
00234 auto sts = recv(table_socket_, &buffer[0], sizeof(artdaq::detail::RoutingPacketEntry) * hdr.nEntries, 0);
00235 assert(static_cast<size_t>(sts) == sizeof(artdaq::detail::RoutingPacketEntry) * hdr.nEntries);
00236 TLOG(6) << "Received a packet of " << sts << " bytes";
00237
00238 first = buffer[0].sequence_id;
00239 last = buffer[buffer.size() - 1].sequence_id;
00240
00241 if (first + hdr.nEntries - 1 != last)
00242 {
00243 TLOG(TLVL_ERROR) << "Skipping this RoutingPacket because the first (" << first << ") and last (" << last << ") entries are inconsistent (sz=" << hdr.nEntries << ")!";
00244 continue;
00245 }
00246 auto thisSeqID = first;
00247
00248 if (routing_table_.count(last) == 0)
00249 {
00250 for (auto entry : buffer)
00251 {
00252 if (thisSeqID != entry.sequence_id)
00253 {
00254 TLOG(TLVL_ERROR) << "Aborting processing of this RoutingPacket because I encountered an inconsistent entry (seqid=" << entry.sequence_id << ", expected=" << thisSeqID << ")!";
00255 last = thisSeqID - 1;
00256 break;
00257 }
00258 thisSeqID++;
00259 if (routing_table_.count(entry.sequence_id))
00260 {
00261 if (routing_table_[entry.sequence_id] != entry.destination_rank)
00262 {
00263 TLOG(TLVL_ERROR) << "Detected routing table corruption! Recevied update specifying that sequence ID " << entry.sequence_id
00264 << " should go to rank " << entry.destination_rank << ", but I had already been told to send it to " << routing_table_[entry.sequence_id] << "!"
00265 << " I will use the original value!";
00266 }
00267 continue;
00268 }
00269 routing_table_[entry.sequence_id] = entry.destination_rank;
00270 TLOG(TLVL_DEBUG) << "DataSenderManager " << std::to_string(my_rank) << ": received update: SeqID " << std::to_string(entry.sequence_id) << " -> Rank " << std::to_string(entry.destination_rank);
00271 }
00272 }
00273
00274 artdaq::detail::RoutingAckPacket ack;
00275 ack.rank = my_rank;
00276 ack.first_sequence_id = first;
00277 ack.last_sequence_id = last;
00278
00279 TLOG(TLVL_DEBUG) << "Sending RoutingAckPacket with first= " << std::to_string(first) << " and last= " << std::to_string(last) << " to " << ack_address_ << ", port " << ack_port_ << " (my_rank = " << my_rank << ")";
00280 TLOG(TLVL_DEBUG) << "There are now " << routing_table_.size() << " entries in the Routing Table";
00281 sendto(ack_socket_, &ack, sizeof(artdaq::detail::RoutingAckPacket), 0, (struct sockaddr *)&ack_addr_, sizeof(ack_addr_));
00282 }
00283 }
00284 }
00285 }
00286
00287 size_t artdaq::DataSenderManager::GetRoutingTableEntryCount() const
00288 {
00289 std::unique_lock<std::mutex> lck(routing_mutex_);
00290 return routing_table_.size();
00291 }
00292
00293 int artdaq::DataSenderManager::calcDest_(Fragment::sequence_id_t sequence_id) const
00294 {
00295 if (enabled_destinations_.size() == 0) return TransferInterface::RECV_TIMEOUT;
00296 if (enabled_destinations_.size() == 1) return *enabled_destinations_.begin();
00297
00298 if (use_routing_master_)
00299 {
00300 auto start = std::chrono::steady_clock::now();
00301 while (routing_timeout_ms_ <= 0 || TimeUtils::GetElapsedTimeMilliseconds(start) < static_cast<size_t>(routing_timeout_ms_))
00302 {
00303 std::unique_lock<std::mutex> lck(routing_mutex_);
00304 if (routing_master_mode_ == detail::RoutingMasterMode::RouteBySequenceID && routing_table_.count(sequence_id))
00305 {
00306 routing_wait_time_.fetch_add(TimeUtils::GetElapsedTimeMicroseconds(start));
00307 return routing_table_.at(sequence_id);
00308 }
00309 else if (routing_master_mode_ == detail::RoutingMasterMode::RouteBySendCount && routing_table_.count(sent_frag_count_.count()))
00310 {
00311 routing_wait_time_.fetch_add(TimeUtils::GetElapsedTimeMicroseconds(start));
00312 return routing_table_.at(sent_frag_count_.count());
00313 }
00314 usleep(routing_timeout_ms_ * 10);
00315 }
00316 routing_wait_time_.fetch_add(TimeUtils::GetElapsedTimeMicroseconds(start));
00317 if (routing_master_mode_ == detail::RoutingMasterMode::RouteBySequenceID)
00318 {
00319 TLOG(TLVL_ERROR) << "Bad Omen: I don't have routing information for seqID " << std::to_string(sequence_id)
00320 << " and the Routing Master did not send a table update in routing_timeout (" << std::to_string(routing_timeout_ms_) << ")!";
00321 }
00322 else
00323 {
00324 TLOG(TLVL_ERROR) << "Bad Omen: I don't have routing information for send number " << std::to_string(sent_frag_count_.count())
00325 << " and the Routing Master did not send a table update in routing_timeout (" << std::to_string(routing_timeout_ms_) << ")!";
00326 }
00327 }
00328 else
00329 {
00330 auto index = sequence_id % enabled_destinations_.size();
00331 auto it = enabled_destinations_.begin();
00332 for (; index > 0; --index)
00333 {
00334 ++it;
00335 if (it == enabled_destinations_.end()) it = enabled_destinations_.begin();
00336 }
00337 return *it;
00338 }
00339 return TransferInterface::RECV_TIMEOUT;
00340 }
00341
00342 std::pair<int, artdaq::TransferInterface::CopyStatus> artdaq::DataSenderManager::sendFragment(Fragment&& frag)
00343 {
00344
00345
00346 auto start_time = std::chrono::steady_clock::now();
00347 if (frag.type() == Fragment::EndOfDataFragmentType)
00348 {
00349 throw cet::exception("LogicError")
00350 << "EOD fragments should not be sent on as received: "
00351 << "use sendEODFrag() instead.";
00352 }
00353 size_t seqID = frag.sequenceID();
00354 size_t fragSize = frag.sizeBytes();
00355 TLOG(13) << "sendFragment start frag.fragmentHeader()=" << std::hex << (void*)(frag.headerBeginBytes()) << ", szB=" << std::dec << std::to_string(fragSize)
00356 << ", seqID=" << std::to_string(seqID) << ", type=" << frag.typeString();
00357 int dest = TransferInterface::RECV_TIMEOUT;
00358 auto outsts = TransferInterface::CopyStatus::kSuccess;
00359 if (broadcast_sends_ || frag.type() == Fragment::EndOfRunFragmentType || frag.type() == Fragment::EndOfSubrunFragmentType || frag.type() == Fragment::InitFragmentType)
00360 {
00361 for (auto& bdest : enabled_destinations_)
00362 {
00363 TLOG(TLVL_TRACE) << "sendFragment: Sending fragment with seqId " << std::to_string(seqID) << " to destination " << bdest << " (broadcast)";
00364
00365 Fragment fragCopy(frag);
00366 auto sts = destinations_[bdest]->copyFragment(fragCopy, send_timeout_us_);
00367 size_t retries = 0;
00368 while (sts == TransferInterface::CopyStatus::kTimeout && retries < send_retry_count_)
00369 {
00370 sts = destinations_[bdest]->copyFragment(fragCopy, send_timeout_us_);
00371 retries++;
00372 }
00373 if (sts != TransferInterface::CopyStatus::kSuccess) outsts = sts;
00374 sent_frag_count_.incSlot(bdest);
00375 }
00376 }
00377 else if (non_blocking_mode_)
00378 {
00379 auto count = routing_retry_count_;
00380 while (dest == TransferInterface::RECV_TIMEOUT && count > 0)
00381 {
00382 dest = calcDest_(seqID);
00383 if (dest == TransferInterface::RECV_TIMEOUT)
00384 {
00385 count--;
00386 TLOG(TLVL_WARNING) << "Could not get destination for seqID " << std::to_string(seqID) << (count > 0 ? ", retrying." : ".");
00387 }
00388 }
00389 if (dest != TransferInterface::RECV_TIMEOUT && destinations_.count(dest) && enabled_destinations_.count(dest))
00390 {
00391 TLOG(TLVL_TRACE) << "sendFragment: Sending fragment with seqId " << std::to_string(seqID) << " to destination " << dest;
00392 TransferInterface::CopyStatus sts = TransferInterface::CopyStatus::kErrorNotRequiringException;
00393 auto lastWarnTime = std::chrono::steady_clock::now();
00394 size_t retries = 0;
00395 while (sts != TransferInterface::CopyStatus::kSuccess && retries <= send_retry_count_)
00396 {
00397 sts = destinations_[dest]->copyFragment(frag, send_timeout_us_);
00398 if (sts != TransferInterface::CopyStatus::kSuccess && TimeUtils::GetElapsedTime(lastWarnTime) >= 1)
00399 {
00400 TLOG(TLVL_ERROR) << "sendFragment: Sending fragment " << seqID << " to destination " << dest << " failed! Retrying...";
00401 lastWarnTime = std::chrono::steady_clock::now();
00402 }
00403 }
00404 if (sts != TransferInterface::CopyStatus::kSuccess) outsts = sts;
00405
00406 sent_frag_count_.incSlot(dest);
00407 }
00408 else
00409 {
00410 TLOG(TLVL_WARNING) << "calcDest returned invalid destination rank " << dest << "! This event has been lost: " << seqID;
00411 }
00412 }
00413 else
00414 {
00415 auto count = routing_retry_count_;
00416 while (dest == TransferInterface::RECV_TIMEOUT && count > 0)
00417 {
00418 dest = calcDest_(seqID);
00419 if (dest == TransferInterface::RECV_TIMEOUT)
00420 {
00421 count--;
00422 TLOG(TLVL_WARNING) << "Could not get destination for seqID "
00423 << std::to_string(seqID) << ", send number " << sent_frag_count_.count()
00424 << (count > 0 ? ", retrying." : ".");
00425 }
00426 }
00427 if (dest != TransferInterface::RECV_TIMEOUT && destinations_.count(dest) && enabled_destinations_.count(dest))
00428 {
00429 TLOG(5) << "DataSenderManager::sendFragment: Sending fragment with seqId " << seqID << " to destination " << dest;
00430 TransferInterface::CopyStatus sts = TransferInterface::CopyStatus::kErrorNotRequiringException;
00431
00432 sts = destinations_[dest]->moveFragment(std::move(frag));
00433 if (sts != TransferInterface::CopyStatus::kSuccess)
00434 TLOG(TLVL_ERROR) << "sendFragment: Sending fragment " << seqID << " to destination "
00435 << dest << " failed! Data has been lost!";
00436
00437
00438 sent_frag_count_.incSlot(dest);
00439 outsts = sts;
00440 }
00441 else
00442 TLOG(TLVL_WARNING) << "calcDest returned invalid destination rank " << dest
00443 << "! This event has been lost: " << seqID;
00444 }
00445 if (routing_master_mode_ == detail::RoutingMasterMode::RouteBySequenceID
00446 && routing_table_.find(seqID - 1) != routing_table_.end())
00447 {
00448 std::unique_lock<std::mutex> lck(routing_mutex_);
00449 routing_table_.erase(routing_table_.begin(), routing_table_.find(seqID - 1));
00450 }
00451 else if (routing_master_mode_ == detail::RoutingMasterMode::RouteBySendCount)
00452 {
00453 std::unique_lock<std::mutex> lck(routing_mutex_);
00454 routing_table_.erase(routing_table_.begin(), routing_table_.find(sent_frag_count_.count()));
00455 }
00456 if (metricMan)
00457 {
00458 TLOG(5) << "sendFragment: sending metrics";
00459 auto delta_t = TimeUtils::GetElapsedTime(start_time);
00460 metricMan->sendMetric("Data Send Time to Rank " + std::to_string(dest), delta_t, "s", 5, MetricMode::Accumulate);
00461 metricMan->sendMetric("Data Send Size to Rank " + std::to_string(dest), fragSize, "B", 5, MetricMode::Accumulate);
00462 metricMan->sendMetric("Data Send Rate to Rank " + std::to_string(dest), fragSize / delta_t, "B/s", 5, MetricMode::Average);
00463 metricMan->sendMetric("Data Send Count to Rank " + std::to_string(dest), sent_frag_count_.slotCount(dest),
00464 "fragments", 3, MetricMode::LastPoint);
00465 if (use_routing_master_)
00466 {
00467 metricMan->sendMetric("Routing Table Size", routing_table_.size(), "events", 2, MetricMode::LastPoint);
00468 if (routing_wait_time_ > 0)
00469 metricMan->sendMetric("Routing Wait Time", static_cast<double>(routing_wait_time_.load()) / 1000000, "s", 2,
00470 MetricMode::Average);
00471 }
00472 }
00473 TLOG(5) << "sendFragment: Done sending fragment " << seqID;
00474 return std::make_pair(dest, outsts);
00475 }