00001 #define TRACE_NAME "DataSenderManager"
00002 #include "artdaq/DAQdata/Globals.hh"
00003 #include "artdaq/DAQrate/DataSenderManager.hh"
00004 #include "artdaq/TransferPlugins/MakeTransferPlugin.hh"
00005 #include "artdaq/TransferPlugins/detail/HostMap.hh"
00006
00007 #include <chrono>
00008 #include "canvas/Utilities/Exception.h"
00009 #include <arpa/inet.h>
00010 #include <netinet/in.h>
00011 #include <sys/types.h>
00012 #include <poll.h>
00013 #include <sys/socket.h>
00014 #include "artdaq/DAQdata/TCPConnect.hh"
00015
00016 artdaq::DataSenderManager::DataSenderManager(const fhicl::ParameterSet& pset)
00017 : destinations_()
00018 , destination_metric_data_()
00019 , destination_metric_send_time_()
00020 , enabled_destinations_()
00021 , sent_frag_count_()
00022 , broadcast_sends_(pset.get<bool>("broadcast_sends", false))
00023 , non_blocking_mode_(pset.get<bool>("nonblocking_sends", false))
00024 , send_timeout_us_(pset.get<size_t>("send_timeout_usec", 5000000))
00025 , send_retry_count_(pset.get<size_t>("send_retry_count", 2))
00026 , routing_master_mode_(detail::RoutingMasterMode::INVALID)
00027 , should_stop_(false)
00028 , ack_socket_(-1)
00029 , table_socket_(-1)
00030 {
00031 TLOG(TLVL_DEBUG) << "Received pset: " << pset.to_string();
00032
00033
00034 if (send_timeout_us_ == 0) send_timeout_us_ = std::numeric_limits<size_t>::max();
00035
00036 auto rmConfig = pset.get<fhicl::ParameterSet>("routing_table_config", fhicl::ParameterSet());
00037 use_routing_master_ = rmConfig.get<bool>("use_routing_master", false);
00038 table_port_ = rmConfig.get<int>("table_update_port", 35556);
00039 table_address_ = rmConfig.get<std::string>("table_update_address", "227.128.12.28");
00040 ack_port_ = rmConfig.get<int>("table_acknowledge_port", 35557);
00041 ack_address_ = rmConfig.get<std::string>("routing_master_hostname", "localhost");
00042 routing_timeout_ms_ = (rmConfig.get<int>("routing_timeout_ms", 1000));
00043 routing_retry_count_ = rmConfig.get<int>("routing_retry_count", 5);
00044
00045
00046 hostMap_t host_map = MakeHostMap(pset);
00047
00048 auto dests = pset.get<fhicl::ParameterSet>("destinations", fhicl::ParameterSet());
00049 for (auto& d : dests.get_pset_names())
00050 {
00051 auto dest_pset = dests.get<fhicl::ParameterSet>(d);
00052 host_map = MakeHostMap(dest_pset, 0, host_map);
00053 }
00054 auto host_map_pset = MakeHostMapPset(host_map);
00055 fhicl::ParameterSet dests_mod;
00056 for (auto& d : dests.get_pset_names())
00057 {
00058 auto dest_pset = dests.get<fhicl::ParameterSet>(d);
00059 dest_pset.erase("host_map");
00060 dest_pset.put<std::vector<fhicl::ParameterSet>>("host_map", host_map_pset);
00061 dests_mod.put<fhicl::ParameterSet>(d, dest_pset);
00062 }
00063
00064 for (auto& d : dests.get_pset_names())
00065 {
00066 try
00067 {
00068 auto transfer = MakeTransferPlugin(dests, d, TransferInterface::Role::kSend);
00069 auto destination_rank = transfer->destination_rank();
00070 destinations_.emplace(destination_rank, std::move(transfer));
00071 destination_metric_data_[destination_rank] = std::pair<size_t, double>();
00072 destination_metric_send_time_[destination_rank] = std::chrono::steady_clock::now();
00073 }
00074 catch (std::invalid_argument)
00075 {
00076 TLOG(TLVL_DEBUG) << "Invalid destination specification: " << d;
00077 }
00078 catch (cet::exception ex)
00079 {
00080 TLOG(TLVL_WARNING) << "Caught cet::exception: " << ex.what();
00081 }
00082 catch (...)
00083 {
00084 TLOG(TLVL_WARNING) << "Non-cet exception while setting up TransferPlugin: " << d << ".";
00085 }
00086 }
00087 if (destinations_.size() == 0)
00088 {
00089 TLOG(TLVL_ERROR) << "No destinations specified!";
00090 }
00091 else
00092 {
00093 auto enabled_dests = pset.get<std::vector<size_t>>("enabled_destinations", std::vector<size_t>());
00094 if (enabled_dests.size() == 0)
00095 {
00096 TLOG(TLVL_INFO) << "enabled_destinations not specified, assuming all destinations enabled.";
00097 for (auto& d : destinations_)
00098 {
00099 enabled_destinations_.insert(d.first);
00100 }
00101 }
00102 else
00103 {
00104 for (auto& d : enabled_dests)
00105 {
00106 enabled_destinations_.insert(d);
00107 }
00108 }
00109 }
00110 if (use_routing_master_) startTableReceiverThread_();
00111 }
00112
00113 artdaq::DataSenderManager::~DataSenderManager()
00114 {
00115 TLOG(TLVL_DEBUG) << "Shutting down DataSenderManager BEGIN";
00116 should_stop_ = true;
00117 for (auto& dest : enabled_destinations_)
00118 {
00119 if (destinations_.count(dest))
00120 {
00121 auto sts = destinations_[dest]->moveFragment(std::move(*Fragment::eodFrag(sent_frag_count_.slotCount(dest))));
00122 if (sts != TransferInterface::CopyStatus::kSuccess) TLOG(TLVL_ERROR) << "Error sending EOD Fragment to sender rank " << dest;
00123
00124 }
00125 }
00126 if (routing_thread_.joinable()) routing_thread_.join();
00127 TLOG(TLVL_DEBUG) << "Shutting down DataSenderManager END. Sent " << count() << " fragments.";
00128 }
00129
00130
00131 void artdaq::DataSenderManager::setupTableListener_()
00132 {
00133 table_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
00134 if (table_socket_ < 0)
00135 {
00136 TLOG(TLVL_ERROR) << "Error creating socket for receiving table updates!";
00137 exit(1);
00138 }
00139
00140 struct sockaddr_in si_me_request;
00141
00142 int yes = 1;
00143 if (setsockopt(table_socket_, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) < 0)
00144 {
00145 TLOG(TLVL_ERROR) << " Unable to enable port reuse on request socket";
00146 exit(1);
00147 }
00148 memset(&si_me_request, 0, sizeof(si_me_request));
00149 si_me_request.sin_family = AF_INET;
00150 si_me_request.sin_port = htons(table_port_);
00151 si_me_request.sin_addr.s_addr = htonl(INADDR_ANY);
00152 if (bind(table_socket_, (struct sockaddr *)&si_me_request, sizeof(si_me_request)) == -1)
00153 {
00154 TLOG(TLVL_ERROR) << "Cannot bind request socket to port " << table_port_;
00155 exit(1);
00156 }
00157
00158 struct ip_mreq mreq;
00159 int sts = ResolveHost(table_address_.c_str(), mreq.imr_multiaddr);
00160 if (sts == -1)
00161 {
00162 TLOG(TLVL_ERROR) << "Unable to resolve multicast address for table updates";
00163 exit(1);
00164 }
00165 mreq.imr_interface.s_addr = htonl(INADDR_ANY);
00166 if (setsockopt(table_socket_, IPPROTO_IP, IP_ADD_MEMBERSHIP, &mreq, sizeof(mreq)) < 0)
00167 {
00168 TLOG(TLVL_ERROR) << "Unable to join multicast group";
00169 exit(1);
00170 }
00171 }
00172 void artdaq::DataSenderManager::startTableReceiverThread_()
00173 {
00174 if (routing_thread_.joinable()) routing_thread_.join();
00175 TLOG(TLVL_INFO) << "Starting Routing Thread";
00176 routing_thread_ = boost::thread(&DataSenderManager::receiveTableUpdatesLoop_, this);
00177 }
00178 void artdaq::DataSenderManager::receiveTableUpdatesLoop_()
00179 {
00180 while (true)
00181 {
00182 if (should_stop_)
00183 {
00184 TLOG(TLVL_DEBUG) << "receiveTableUpdatesLoop: should_stop is " << std::boolalpha << should_stop_ << ", stopping";
00185 return;
00186 }
00187
00188 TLOG(TLVL_TRACE) << "DataSenderManager::receiveTableUpdatesLoop: Polling Request socket for new requests";
00189 if (table_socket_ == -1)
00190 {
00191 TLOG(TLVL_DEBUG) << "Opening table listener socket";
00192 setupTableListener_();
00193 }
00194 if (table_socket_ == -1)
00195 {
00196 TLOG(TLVL_DEBUG) << "The listen socket was not opened successfully.";
00197 return;
00198 }
00199 if (ack_socket_ == -1)
00200 {
00201 ack_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
00202 auto sts = ResolveHost(ack_address_.c_str(), ack_port_, ack_addr_);
00203 if (sts == -1)
00204 {
00205 TLOG(TLVL_ERROR) << "Unable to resolve routing_master_address";
00206 exit(1);
00207 }
00208 TLOG(TLVL_DEBUG) << "Ack socket is fd " << ack_socket_;
00209 }
00210
00211 struct pollfd fd;
00212 fd.fd = table_socket_;
00213 fd.events = POLLIN | POLLPRI;
00214
00215 auto res = poll(&fd, 1, 1000);
00216 if (res > 0)
00217 {
00218 auto first = artdaq::Fragment::InvalidSequenceID;
00219 auto last = artdaq::Fragment::InvalidSequenceID;
00220 artdaq::detail::RoutingPacketHeader hdr;
00221
00222 TLOG(TLVL_DEBUG) << "Going to receive RoutingPacketHeader";
00223 auto stss = recvfrom(table_socket_, &hdr, sizeof(artdaq::detail::RoutingPacketHeader), 0, NULL, NULL);
00224 TLOG(TLVL_DEBUG) << "Received " << stss << " bytes. (sizeof(RoutingPacketHeader) == " << sizeof(detail::RoutingPacketHeader);
00225
00226 TLOG(TLVL_DEBUG) << "Checking for valid header";
00227 if (hdr.header == ROUTING_MAGIC)
00228 {
00229 if (routing_master_mode_ != detail::RoutingMasterMode::INVALID && routing_master_mode_ != hdr.mode)
00230 {
00231 TLOG(TLVL_ERROR) << "Received table has different RoutingMasterMode than expected!";
00232 exit(1);
00233 }
00234 routing_master_mode_ = hdr.mode;
00235
00236 artdaq::detail::RoutingPacket buffer(hdr.nEntries);
00237 TLOG(TLVL_DEBUG) << "Receiving data buffer";
00238 auto sts = recv(table_socket_, &buffer[0], sizeof(artdaq::detail::RoutingPacketEntry) * hdr.nEntries, 0);
00239 assert(static_cast<size_t>(sts) == sizeof(artdaq::detail::RoutingPacketEntry) * hdr.nEntries);
00240 TLOG(6) << "Received a packet of " << sts << " bytes";
00241
00242 first = buffer[0].sequence_id;
00243 last = buffer[buffer.size() - 1].sequence_id;
00244
00245 if (first + hdr.nEntries - 1 != last)
00246 {
00247 TLOG(TLVL_ERROR) << "Skipping this RoutingPacket because the first (" << first << ") and last (" << last << ") entries are inconsistent (sz=" << hdr.nEntries << ")!";
00248 continue;
00249 }
00250 auto thisSeqID = first;
00251
00252 if (routing_table_.count(last) == 0)
00253 {
00254 for (auto entry : buffer)
00255 {
00256 if (thisSeqID != entry.sequence_id)
00257 {
00258 TLOG(TLVL_ERROR) << "Aborting processing of this RoutingPacket because I encountered an inconsistent entry (seqid=" << entry.sequence_id << ", expected=" << thisSeqID << ")!";
00259 last = thisSeqID - 1;
00260 break;
00261 }
00262 thisSeqID++;
00263 if (routing_table_.count(entry.sequence_id))
00264 {
00265 if (routing_table_[entry.sequence_id] != entry.destination_rank)
00266 {
00267 TLOG(TLVL_ERROR) << "Detected routing table corruption! Recevied update specifying that sequence ID " << entry.sequence_id
00268 << " should go to rank " << entry.destination_rank << ", but I had already been told to send it to " << routing_table_[entry.sequence_id] << "!"
00269 << " I will use the original value!";
00270 }
00271 continue;
00272 }
00273 routing_table_[entry.sequence_id] = entry.destination_rank;
00274 TLOG(TLVL_DEBUG) << "DataSenderManager " << my_rank << ": received update: SeqID " << entry.sequence_id << " -> Rank " << entry.destination_rank;
00275 }
00276 }
00277
00278 artdaq::detail::RoutingAckPacket ack;
00279 ack.rank = my_rank;
00280 ack.first_sequence_id = first;
00281 ack.last_sequence_id = last;
00282
00283 TLOG(TLVL_DEBUG) << "Sending RoutingAckPacket with first= " << first << " and last= " << last << " to " << ack_address_ << ", port " << ack_port_ << " (my_rank = " << my_rank << ")";
00284 TLOG(TLVL_DEBUG) << "There are now " << routing_table_.size() << " entries in the Routing Table";
00285 sendto(ack_socket_, &ack, sizeof(artdaq::detail::RoutingAckPacket), 0, (struct sockaddr *)&ack_addr_, sizeof(ack_addr_));
00286 }
00287 }
00288 }
00289 }
00290
00291 size_t artdaq::DataSenderManager::GetRoutingTableEntryCount() const
00292 {
00293 std::unique_lock<std::mutex> lck(routing_mutex_);
00294 return routing_table_.size();
00295 }
00296
00297 int artdaq::DataSenderManager::calcDest_(Fragment::sequence_id_t sequence_id) const
00298 {
00299 if (enabled_destinations_.size() == 0) return TransferInterface::RECV_TIMEOUT;
00300 if (enabled_destinations_.size() == 1) return *enabled_destinations_.begin();
00301
00302 if (use_routing_master_)
00303 {
00304 auto start = std::chrono::steady_clock::now();
00305 while (routing_timeout_ms_ <= 0 || TimeUtils::GetElapsedTimeMilliseconds(start) < static_cast<size_t>(routing_timeout_ms_))
00306 {
00307 std::unique_lock<std::mutex> lck(routing_mutex_);
00308 if (routing_master_mode_ == detail::RoutingMasterMode::RouteBySequenceID && routing_table_.count(sequence_id))
00309 {
00310 routing_wait_time_.fetch_add(TimeUtils::GetElapsedTimeMicroseconds(start));
00311 return routing_table_.at(sequence_id);
00312 }
00313 else if (routing_master_mode_ == detail::RoutingMasterMode::RouteBySendCount && routing_table_.count(sent_frag_count_.count()))
00314 {
00315 routing_wait_time_.fetch_add(TimeUtils::GetElapsedTimeMicroseconds(start));
00316 return routing_table_.at(sent_frag_count_.count());
00317 }
00318 usleep(routing_timeout_ms_ * 10);
00319 }
00320 routing_wait_time_.fetch_add(TimeUtils::GetElapsedTimeMicroseconds(start));
00321 if (routing_master_mode_ == detail::RoutingMasterMode::RouteBySequenceID)
00322 {
00323 TLOG(TLVL_ERROR) << "Bad Omen: I don't have routing information for seqID " << sequence_id
00324 << " and the Routing Master did not send a table update in routing_timeout (" << routing_timeout_ms_ << ")!";
00325 }
00326 else
00327 {
00328 TLOG(TLVL_ERROR) << "Bad Omen: I don't have routing information for send number " << sent_frag_count_.count()
00329 << " and the Routing Master did not send a table update in routing_timeout (" << routing_timeout_ms_ << ")!";
00330 }
00331 }
00332 else
00333 {
00334 auto index = sequence_id % enabled_destinations_.size();
00335 auto it = enabled_destinations_.begin();
00336 for (; index > 0; --index)
00337 {
00338 ++it;
00339 if (it == enabled_destinations_.end()) it = enabled_destinations_.begin();
00340 }
00341 return *it;
00342 }
00343 return TransferInterface::RECV_TIMEOUT;
00344 }
00345
00346 std::pair<int, artdaq::TransferInterface::CopyStatus> artdaq::DataSenderManager::sendFragment(Fragment&& frag)
00347 {
00348
00349
00350 auto start_time = std::chrono::steady_clock::now();
00351 if (frag.type() == Fragment::EndOfDataFragmentType)
00352 {
00353 throw cet::exception("LogicError")
00354 << "EOD fragments should not be sent on as received: "
00355 << "use sendEODFrag() instead.";
00356 }
00357 size_t seqID = frag.sequenceID();
00358 size_t fragSize = frag.sizeBytes();
00359 TLOG(13) << "sendFragment start frag.fragmentHeader()=" << std::hex << (void*)(frag.headerBeginBytes()) << ", szB=" << std::dec << fragSize
00360 << ", seqID=" << seqID << ", type=" << frag.typeString();
00361 int dest = TransferInterface::RECV_TIMEOUT;
00362 auto outsts = TransferInterface::CopyStatus::kSuccess;
00363 if (broadcast_sends_ || frag.type() == Fragment::EndOfRunFragmentType || frag.type() == Fragment::EndOfSubrunFragmentType || frag.type() == Fragment::InitFragmentType)
00364 {
00365 for (auto& bdest : enabled_destinations_)
00366 {
00367 TLOG(TLVL_TRACE) << "sendFragment: Sending fragment with seqId " << seqID << " to destination " << bdest << " (broadcast)";
00368
00369 Fragment fragCopy(frag);
00370 auto sts = destinations_[bdest]->copyFragment(fragCopy, send_timeout_us_);
00371 size_t retries = 0;
00372 while (sts == TransferInterface::CopyStatus::kTimeout && retries < send_retry_count_)
00373 {
00374 sts = destinations_[bdest]->copyFragment(fragCopy, send_timeout_us_);
00375 retries++;
00376 }
00377 if (sts != TransferInterface::CopyStatus::kSuccess) outsts = sts;
00378 sent_frag_count_.incSlot(bdest);
00379 }
00380 }
00381 else if (non_blocking_mode_)
00382 {
00383 auto count = routing_retry_count_;
00384 while (dest == TransferInterface::RECV_TIMEOUT && count > 0)
00385 {
00386 dest = calcDest_(seqID);
00387 if (dest == TransferInterface::RECV_TIMEOUT)
00388 {
00389 count--;
00390 TLOG(TLVL_WARNING) << "Could not get destination for seqID " << seqID << (count > 0 ? ", retrying." : ".");
00391 }
00392 }
00393 if (dest != TransferInterface::RECV_TIMEOUT && destinations_.count(dest) && enabled_destinations_.count(dest))
00394 {
00395 TLOG(TLVL_TRACE) << "sendFragment: Sending fragment with seqId " << seqID << " to destination " << dest;
00396 TransferInterface::CopyStatus sts = TransferInterface::CopyStatus::kErrorNotRequiringException;
00397 auto lastWarnTime = std::chrono::steady_clock::now();
00398 size_t retries = 0;
00399 while (sts != TransferInterface::CopyStatus::kSuccess && retries <= send_retry_count_)
00400 {
00401 sts = destinations_[dest]->copyFragment(frag, send_timeout_us_);
00402 if (sts != TransferInterface::CopyStatus::kSuccess && TimeUtils::GetElapsedTime(lastWarnTime) >= 1)
00403 {
00404 TLOG(TLVL_ERROR) << "sendFragment: Sending fragment " << seqID << " to destination " << dest << " failed! Retrying...";
00405 lastWarnTime = std::chrono::steady_clock::now();
00406 }
00407 }
00408 if (sts != TransferInterface::CopyStatus::kSuccess) outsts = sts;
00409
00410 sent_frag_count_.incSlot(dest);
00411 }
00412 else
00413 {
00414 TLOG(TLVL_WARNING) << "calcDest returned invalid destination rank " << dest << "! This event has been lost: " << seqID;
00415 }
00416 }
00417 else
00418 {
00419 auto count = routing_retry_count_;
00420 while (dest == TransferInterface::RECV_TIMEOUT && count > 0)
00421 {
00422 dest = calcDest_(seqID);
00423 if (dest == TransferInterface::RECV_TIMEOUT)
00424 {
00425 count--;
00426 TLOG(TLVL_WARNING) << "Could not get destination for seqID "
00427 << seqID << ", send number " << sent_frag_count_.count()
00428 << (count > 0 ? ", retrying." : ".");
00429 }
00430 }
00431 if (dest != TransferInterface::RECV_TIMEOUT && destinations_.count(dest) && enabled_destinations_.count(dest))
00432 {
00433 TLOG(5) << "DataSenderManager::sendFragment: Sending fragment with seqId " << seqID << " to destination " << dest;
00434 TransferInterface::CopyStatus sts = TransferInterface::CopyStatus::kErrorNotRequiringException;
00435
00436 sts = destinations_[dest]->moveFragment(std::move(frag));
00437 if (sts != TransferInterface::CopyStatus::kSuccess)
00438 TLOG(TLVL_ERROR) << "sendFragment: Sending fragment " << seqID << " to destination "
00439 << dest << " failed! Data has been lost!";
00440
00441
00442 sent_frag_count_.incSlot(dest);
00443 outsts = sts;
00444 }
00445 else
00446 TLOG(TLVL_WARNING) << "calcDest returned invalid destination rank " << dest
00447 << "! This event has been lost: " << seqID;
00448 }
00449 if (routing_master_mode_ == detail::RoutingMasterMode::RouteBySequenceID
00450 && routing_table_.find(seqID - 1) != routing_table_.end())
00451 {
00452 std::unique_lock<std::mutex> lck(routing_mutex_);
00453 routing_table_.erase(routing_table_.begin(), routing_table_.find(seqID - 1));
00454 }
00455 else if (routing_master_mode_ == detail::RoutingMasterMode::RouteBySendCount)
00456 {
00457 std::unique_lock<std::mutex> lck(routing_mutex_);
00458 routing_table_.erase(routing_table_.begin(), routing_table_.find(sent_frag_count_.count()));
00459 }
00460
00461
00462 auto delta_t = TimeUtils::GetElapsedTime(start_time);
00463 destination_metric_data_[dest].first += fragSize;
00464 destination_metric_data_[dest].second += delta_t;
00465
00466 if (metricMan && TimeUtils::GetElapsedTime(destination_metric_send_time_[dest]) > 1)
00467 {
00468 TLOG(5) << "sendFragment: sending metrics";
00469 metricMan->sendMetric("Data Send Time to Rank " + std::to_string(dest), destination_metric_data_[dest].second, "s", 5, MetricMode::Accumulate);
00470 metricMan->sendMetric("Data Send Size to Rank " + std::to_string(dest), destination_metric_data_[dest].first, "B", 5, MetricMode::Accumulate);
00471 metricMan->sendMetric("Data Send Rate to Rank " + std::to_string(dest), destination_metric_data_[dest].first / destination_metric_data_[dest].second, "B/s", 5, MetricMode::Average);
00472 metricMan->sendMetric("Data Send Count to Rank " + std::to_string(dest), sent_frag_count_.slotCount(dest), "fragments", 3, MetricMode::LastPoint);
00473
00474 destination_metric_send_time_[dest] = std::chrono::steady_clock::now();
00475 destination_metric_data_[dest].first = 0;
00476 destination_metric_data_[dest].second = 0.0;
00477
00478 if (use_routing_master_)
00479 {
00480 metricMan->sendMetric("Routing Table Size", routing_table_.size(), "events", 2, MetricMode::LastPoint);
00481 if (routing_wait_time_ > 0)
00482 metricMan->sendMetric("Routing Wait Time", static_cast<double>(routing_wait_time_.load()) / 1000000, "s", 2, MetricMode::Average);
00483 }
00484 }
00485 TLOG(5) << "sendFragment: Done sending fragment " << seqID;
00486 return std::make_pair(dest, outsts);
00487 }