00001 #include "artdaq/DAQdata/Globals.hh"
00002 #include "artdaq/DAQrate/RequestSender.hh"
00003 #include <utility>
00004 #include <cstring>
00005 #include <dlfcn.h>
00006 #include <iomanip>
00007 #include <fstream>
00008 #include <sstream>
00009 #include <chrono>
00010
00011 #include "cetlib_except/exception.h"
00012 #include "artdaq-core/Core/StatisticsCollection.hh"
00013 #include "artdaq-core/Core/SimpleMemoryReader.hh"
00014 #include "artdaq/DAQrate/detail/RoutingPacket.hh"
00015 #include "artdaq/DAQdata/TCPConnect.hh"
00016
00017 namespace artdaq
00018 {
00019
00020 RequestSender::RequestSender(const fhicl::ParameterSet& pset)
00021 : send_requests_(pset.get<bool>("send_requests", false))
00022 , active_requests_()
00023 , request_port_(pset.get<int>("request_port", 3001))
00024 , request_delay_(pset.get<size_t>("request_delay_ms", 10) * 1000)
00025 , request_shutdown_timeout_us_(pset.get<size_t>("request_shutdown_timeout_us", 100000))
00026 , multicast_out_addr_(pset.get<std::string>("output_address", "0.0.0.0"))
00027 , request_mode_(detail::RequestMessageMode::Normal)
00028 , token_socket_(-1)
00029 {
00030 TLOG_DEBUG("RequestSender") << "RequestSender CONSTRUCTOR" << TLOG_ENDL;
00031 setup_requests_(pset.get<std::string>("request_address", "227.128.12.26"));
00032
00033 auto rmConfig = pset.get<fhicl::ParameterSet>("routing_token_config", fhicl::ParameterSet());
00034 send_routing_tokens_ = rmConfig.get<bool>("use_routing_master", false);
00035 token_port_ = rmConfig.get<int>("routing_token_port", 35555);
00036 token_address_ = rmConfig.get<std::string>("routing_master_hostname", "localhost");
00037 setup_tokens_();
00038 TRACE(12, "artdaq::RequestSender::RequestSender ctor - reader_thread_ initialized");
00039 }
00040
00041
00042 RequestSender::~RequestSender()
00043 {
00044 TLOG_TRACE("RequestSender") << "Shutting down RequestSender: Waiting for requests to be sent" << TLOG_ENDL;
00045
00046 auto start_time = std::chrono::steady_clock::now();
00047
00048 while (request_sending_ && request_shutdown_timeout_us_ > TimeUtils::GetElapsedTimeMicroseconds(start_time))
00049 {
00050 usleep(1000);
00051 }
00052 TLOG_TRACE("RequestSender") << "Shutting down RequestSender" << TLOG_ENDL;
00053 if (request_socket_ > 0)
00054 {
00055 shutdown(request_socket_, 2);
00056 close(request_socket_);
00057 }
00058 if (token_socket_ > 0)
00059 {
00060 shutdown(token_socket_, 2);
00061 close(token_socket_);
00062 }
00063 }
00064
00065
00066 void RequestSender::SetRequestMode(detail::RequestMessageMode mode)
00067 {
00068 request_mode_ = mode;
00069 SendRequest(true);
00070 }
00071
00072 void
00073 RequestSender::setup_requests_(std::string request_address)
00074 {
00075 if (send_requests_)
00076 {
00077 request_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
00078 if (request_socket_ < 0)
00079 {
00080 TLOG_ERROR("RequestSender") << "I failed to create the socket for sending Data Requests! err=" << strerror(errno) << TLOG_ENDL;
00081 exit(1);
00082 }
00083 int sts = ResolveHost(request_address.c_str(), request_port_, request_addr_);
00084 if (sts == -1)
00085 {
00086 TLOG_ERROR("RequestSender") << "Unable to resolve Data Request address, err=" << strerror(errno) << TLOG_ENDL;
00087 exit(1);
00088 }
00089
00090 if (multicast_out_addr_ == "0.0.0.0")
00091 {
00092 multicast_out_addr_.reserve(HOST_NAME_MAX);
00093 sts = gethostname(&multicast_out_addr_[0], HOST_NAME_MAX);
00094 if (sts < 0)
00095 {
00096 TLOG_ERROR("RequestSender") << "Could not get current hostname, err=" << strerror(errno) << TLOG_ENDL;
00097 exit(1);
00098 }
00099 }
00100
00101 if (multicast_out_addr_ != "localhost")
00102 {
00103 struct in_addr addr;
00104 sts = ResolveHost(multicast_out_addr_.c_str(), addr);
00105 if (sts == -1)
00106 {
00107 TLOG_ERROR("RequestSender") << "Unable to resolve multicast interface address, err=" << strerror(errno) << TLOG_ENDL;
00108 exit(1);
00109 }
00110
00111 int yes = 1;
00112 if (setsockopt(request_socket_, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) < 0)
00113 {
00114 TLOG_ERROR("RequestSender") << "Unable to enable port reuse on request socket, err=" << strerror(errno) << TLOG_ENDL;
00115 exit(1);
00116 }
00117 if (setsockopt(request_socket_, IPPROTO_IP, IP_MULTICAST_LOOP, &yes, sizeof(yes)) < 0)
00118 {
00119 TLOG_ERROR("RequestSender") << "Unable to enable multicast loopback on request socket, err=" << strerror(errno) << TLOG_ENDL;
00120 exit(1);
00121 }
00122 if (setsockopt(request_socket_, IPPROTO_IP, IP_MULTICAST_IF, &addr, sizeof(addr)) == -1)
00123 {
00124 TLOG_ERROR("RequestSender") << "Cannot set outgoing interface, err=" << strerror(errno) << TLOG_ENDL;
00125 exit(1);
00126 }
00127 }
00128 int yes = 1;
00129 if (setsockopt(request_socket_, SOL_SOCKET, SO_BROADCAST, (void*)&yes, sizeof(int)) == -1)
00130 {
00131 TLOG_ERROR("RequestSender") << "Cannot set request socket to broadcast, err=" << strerror(errno) << TLOG_ENDL;
00132 exit(1);
00133 }
00134 }
00135 }
00136
00137 void
00138 RequestSender::setup_tokens_()
00139 {
00140 if (send_routing_tokens_)
00141 {
00142 TLOG_DEBUG("RequestSender") << "Creating Routing Token sending socket" << TLOG_ENDL;
00143 int retry = 5;
00144 while (retry > 0 && token_socket_ < 0) {
00145 token_socket_ = TCPConnect(token_address_.c_str(), token_port_);
00146 if (token_socket_ < 0) usleep(100000);
00147 retry--;
00148 }
00149 if (token_socket_ < 0)
00150 {
00151 TLOG_ERROR("RequestSender") << "I failed to create the socket for sending Routing Tokens! err=" << strerror(errno) << TLOG_ENDL;
00152 exit(1);
00153 }
00154 TLOG_DEBUG("RequestSender") << "Routing Token sending socket created successfully" << TLOG_ENDL;
00155 }
00156 }
00157
00158 void RequestSender::do_send_request_()
00159 {
00160 if (!send_requests_) return;
00161 TLOG_TRACE("RequestSender") << "Waiting for " << request_delay_ << " microseconds." << TLOG_ENDL;
00162 std::this_thread::sleep_for(std::chrono::microseconds(request_delay_));
00163
00164 TLOG_TRACE("RequestSender") << "Creating RequestMessage" << TLOG_ENDL;
00165 detail::RequestMessage message;
00166 {
00167 std::lock_guard<std::mutex> lk(request_mutex_);
00168 for (auto& req : active_requests_)
00169 {
00170 message.addRequest(req.first, req.second);
00171 }
00172 }
00173 TLOG_TRACE("RequestSender") << "Setting mode flag in Message Header" << TLOG_ENDL;
00174 message.header()->mode = request_mode_;
00175 char str[INET_ADDRSTRLEN];
00176 inet_ntop(AF_INET, &(request_addr_.sin_addr), str, INET_ADDRSTRLEN);
00177 std::lock_guard<std::mutex> lk2(request_send_mutex_);
00178 TLOG_TRACE("RequestSender") << "Sending request for " << std::to_string(message.size()) << " events to multicast group " << str << TLOG_ENDL;
00179 if (sendto(request_socket_, message.header(), sizeof(detail::RequestHeader), 0, (struct sockaddr *)&request_addr_, sizeof(request_addr_)) < 0)
00180 {
00181 TLOG_ERROR("RequestSender") << "Error sending request message header err=" << strerror(errno) << TLOG_ENDL;
00182 }
00183 if (sendto(request_socket_, message.buffer(), sizeof(detail::RequestPacket) * message.size(), 0, (struct sockaddr *)&request_addr_, sizeof(request_addr_)) < 0)
00184 {
00185 TLOG_ERROR("RequestSender") << "Error sending request message data err=" << strerror(errno) << TLOG_ENDL;
00186 }
00187 request_sending_ = false;
00188 }
00189
00190 void RequestSender::send_routing_token_(int nSlots)
00191 {
00192 TLOG_TRACE("RequestSender") << "send_routing_token_ called, send_routing_tokens_=" << std::boolalpha << send_routing_tokens_ << TLOG_ENDL;
00193 if (!send_routing_tokens_) return;
00194 if (token_socket_ == -1) setup_tokens_();
00195 detail::RoutingToken token;
00196 token.header = TOKEN_MAGIC;
00197 token.rank = my_rank;
00198 token.new_slots_free = nSlots;
00199
00200 TLOG_TRACE("RequestSender") << "Sending RoutingToken to " << token_address_ << ":" << token_port_ << TLOG_ENDL;
00201 size_t sts = 0;
00202 while (sts < sizeof(detail::RoutingToken))
00203 {
00204 auto res = send(token_socket_, reinterpret_cast<uint8_t*>(&token) + sts, sizeof(detail::RoutingToken) - sts, 0);
00205 if (res == -1)
00206 {
00207 usleep(1000);
00208 continue;
00209 }
00210 sts += res;
00211 }
00212 TLOG_TRACE("RequestSender") << "Done sending RoutingToken to " << token_address_ << ":" << token_port_ << TLOG_ENDL;
00213 }
00214
00215 void RequestSender::SendRoutingToken(int nSlots)
00216 {
00217 if (!send_routing_tokens_) return;
00218 boost::thread token([=] {send_routing_token_(nSlots); });
00219 token.detach();
00220 usleep(0);
00221 }
00222
00223 void RequestSender::SendRequest(bool endOfRunOnly)
00224 {
00225 if (!send_requests_) return;
00226 if (endOfRunOnly && request_mode_ != detail::RequestMessageMode::EndOfRun) return;
00227 request_sending_ = true;
00228 boost::thread request([=] { do_send_request_(); });
00229 request.detach();
00230 }
00231
00232 void RequestSender::AddRequest(Fragment::sequence_id_t seqID, Fragment::timestamp_t timestamp)
00233 {
00234 {
00235 std::lock_guard<std::mutex> lk(request_mutex_);
00236 if (!active_requests_.count(seqID)) active_requests_[seqID] = timestamp;
00237 }
00238 SendRequest();
00239 }
00240
00241 void RequestSender::RemoveRequest(Fragment::sequence_id_t seqID)
00242 {
00243 std::lock_guard<std::mutex> lk(request_mutex_);
00244 active_requests_.erase(seqID);
00245 }
00246 }