00001 #define TRACE_NAME "RequestSender"
00002 #include "artdaq/DAQdata/Globals.hh"
00003 #include "artdaq/DAQrate/RequestSender.hh"
00004 #include <utility>
00005 #include <cstring>
00006 #include <dlfcn.h>
00007 #include <iomanip>
00008 #include <fstream>
00009 #include <sstream>
00010 #include <chrono>
00011
00012 #include "cetlib_except/exception.h"
00013 #include "artdaq-core/Core/StatisticsCollection.hh"
00014 #include "artdaq-core/Core/SimpleMemoryReader.hh"
00015 #include "artdaq/DAQrate/detail/RoutingPacket.hh"
00016 #include "artdaq/DAQdata/TCPConnect.hh"
00017
00018 namespace artdaq
00019 {
00020 RequestSender::RequestSender(const fhicl::ParameterSet& pset)
00021 : send_requests_(pset.get<bool>("send_requests", false))
00022 , active_requests_()
00023 , request_address_(pset.get<std::string>("request_address", "227.128.12.26"))
00024 , request_port_(pset.get<int>("request_port", 3001))
00025 , request_delay_(pset.get<size_t>("request_delay_ms", 10) * 1000)
00026 , request_shutdown_timeout_us_(pset.get<size_t>("request_shutdown_timeout_us", 100000))
00027 , multicast_out_addr_(pset.get<std::string>("multicast_interface_ip", pset.get<std::string>("output_address", "0.0.0.0")))
00028 , request_mode_(detail::RequestMessageMode::Normal)
00029 , token_socket_(-1)
00030 , request_sending_(0)
00031 {
00032 TLOG(TLVL_DEBUG) << "RequestSender CONSTRUCTOR";
00033 setup_requests_();
00034
00035 auto rmConfig = pset.get<fhicl::ParameterSet>("routing_token_config", fhicl::ParameterSet());
00036 send_routing_tokens_ = rmConfig.get<bool>("use_routing_master", false);
00037 token_port_ = rmConfig.get<int>("routing_token_port", 35555);
00038 token_address_ = rmConfig.get<std::string>("routing_master_hostname", "localhost");
00039 setup_tokens_();
00040 TLOG(12) << "artdaq::RequestSender::RequestSender ctor - reader_thread_ initialized";
00041 }
00042
00043
00044 RequestSender::~RequestSender()
00045 {
00046 TLOG(TLVL_INFO) << "Shutting down RequestSender: Waiting for requests to be sent";
00047
00048 auto start_time = std::chrono::steady_clock::now();
00049
00050 while (request_sending_ > 0 && request_shutdown_timeout_us_ > TimeUtils::GetElapsedTimeMicroseconds(start_time))
00051 {
00052 usleep(1000);
00053 }
00054 TLOG(TLVL_INFO) << "Shutting down RequestSender";
00055 if (request_socket_ > 0)
00056 {
00057 shutdown(request_socket_, 2);
00058 close(request_socket_);
00059 }
00060 if (token_socket_ > 0)
00061 {
00062 shutdown(token_socket_, 2);
00063 close(token_socket_);
00064 }
00065 }
00066
00067
00068 void RequestSender::SetRequestMode(detail::RequestMessageMode mode)
00069 {
00070 request_mode_ = mode;
00071 SendRequest(true);
00072 }
00073
00074 void
00075 RequestSender::setup_requests_()
00076 {
00077 if (send_requests_)
00078 {
00079 request_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
00080 if (request_socket_ < 0)
00081 {
00082 TLOG(TLVL_ERROR) << "I failed to create the socket for sending Data Requests! err=" << strerror(errno);
00083 exit(1);
00084 }
00085 int sts = ResolveHost(request_address_.c_str(), request_port_, request_addr_);
00086 if (sts == -1)
00087 {
00088 TLOG(TLVL_ERROR) << "Unable to resolve Data Request address, err=" << strerror(errno);
00089 exit(1);
00090 }
00091
00092 if (multicast_out_addr_ == "0.0.0.0")
00093 {
00094 multicast_out_addr_.reserve(HOST_NAME_MAX);
00095 sts = gethostname(&multicast_out_addr_[0], HOST_NAME_MAX);
00096 if (sts < 0)
00097 {
00098 TLOG(TLVL_ERROR) << "Could not get current hostname, err=" << strerror(errno);
00099 exit(1);
00100 }
00101 }
00102
00103 if (multicast_out_addr_ != "localhost")
00104 {
00105 struct in_addr addr;
00106 sts = GetInterfaceForNetwork(multicast_out_addr_.c_str(), addr);
00107
00108 if (sts == -1)
00109 {
00110 TLOG(TLVL_ERROR) << "Unable to resolve multicast interface address, err=" << strerror(errno);
00111 exit(1);
00112 }
00113
00114 if (setsockopt(request_socket_, IPPROTO_IP, IP_MULTICAST_IF, &addr, sizeof(addr)) == -1)
00115 {
00116 TLOG(TLVL_ERROR) << "Cannot set outgoing interface, err=" << strerror(errno);
00117 exit(1);
00118 }
00119 }
00120 int yes = 1;
00121 if (setsockopt(request_socket_, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) < 0)
00122 {
00123 TLOG(TLVL_ERROR) << "Unable to enable port reuse on request socket, err=" << strerror(errno);
00124 exit(1);
00125 }
00126 if (setsockopt(request_socket_, IPPROTO_IP, IP_MULTICAST_LOOP, &yes, sizeof(yes)) < 0)
00127 {
00128 TLOG(TLVL_ERROR) << "Unable to enable multicast loopback on request socket, err=" << strerror(errno);
00129 exit(1);
00130 }
00131 if (setsockopt(request_socket_, SOL_SOCKET, SO_BROADCAST, (void*)&yes, sizeof(int)) == -1)
00132 {
00133 TLOG(TLVL_ERROR) << "Cannot set request socket to broadcast, err=" << strerror(errno);
00134 exit(1);
00135 }
00136 }
00137 }
00138
00139 void
00140 RequestSender::setup_tokens_()
00141 {
00142 if (send_routing_tokens_)
00143 {
00144 TLOG(TLVL_DEBUG) << "Creating Routing Token sending socket";
00145 int retry = 5;
00146 while (retry > 0 && token_socket_ < 0)
00147 {
00148 token_socket_ = TCPConnect(token_address_.c_str(), token_port_);
00149 if (token_socket_ < 0) usleep(100000);
00150 retry--;
00151 }
00152 if (token_socket_ < 0)
00153 {
00154 TLOG(TLVL_ERROR) << "I failed to create the socket for sending Routing Tokens! err=" << strerror(errno);
00155 exit(1);
00156 }
00157 TLOG(TLVL_DEBUG) << "Routing Token sending socket created successfully";
00158 }
00159 }
00160
00161 void RequestSender::do_send_request_()
00162 {
00163 if (!send_requests_) return;
00164 if (request_socket_ == -1) setup_requests_();
00165
00166 TLOG(TLVL_TRACE) << "Waiting for " << request_delay_ << " microseconds.";
00167 std::this_thread::sleep_for(std::chrono::microseconds(request_delay_));
00168
00169 TLOG(TLVL_TRACE) << "Creating RequestMessage";
00170 detail::RequestMessage message;
00171 {
00172 std::lock_guard<std::mutex> lk(request_mutex_);
00173 for (auto& req : active_requests_)
00174 {
00175 TLOG(12, "RequestSender") << "Adding a request with sequence ID " << req.first << ", timestamp " << req.second;
00176 message.addRequest(req.first, req.second);
00177 }
00178 }
00179 TLOG(TLVL_TRACE) << "Setting mode flag in Message Header";
00180 message.header()->mode = request_mode_;
00181 char str[INET_ADDRSTRLEN];
00182 inet_ntop(AF_INET, &(request_addr_.sin_addr), str, INET_ADDRSTRLEN);
00183 std::lock_guard<std::mutex> lk2(request_send_mutex_);
00184 TLOG(TLVL_TRACE) << "Sending request for " << std::to_string(message.size()) << " events to multicast group " << str;
00185 if (sendto(request_socket_, message.header(), sizeof(detail::RequestHeader), 0, (struct sockaddr *)&request_addr_, sizeof(request_addr_)) < 0)
00186 {
00187 TLOG(TLVL_ERROR) << "Error sending request message header err=" << strerror(errno);
00188 request_socket_ = 1;
00189 request_sending_--;
00190 return;
00191 }
00192 size_t sent = 0;
00193 while (sent < sizeof(detail::RequestPacket) * message.size())
00194 {
00195 ssize_t thisSent = sendto(request_socket_, reinterpret_cast<uint8_t*>(message.buffer()) + sent, sizeof(detail::RequestPacket) * message.size() - sent, 0, (struct sockaddr *)&request_addr_, sizeof(request_addr_));
00196 if (thisSent < 0)
00197 {
00198 TLOG(TLVL_ERROR) << "Error sending request message data err=" << strerror(errno);
00199 request_socket_ = -1;
00200 request_sending_--;
00201 return;
00202 }
00203 sent += thisSent;
00204 }
00205 TLOG(TLVL_TRACE) << "Done sending request";
00206 request_sending_--;
00207 }
00208
00209 void RequestSender::send_routing_token_(int nSlots)
00210 {
00211 TLOG(TLVL_TRACE) << "send_routing_token_ called, send_routing_tokens_=" << std::boolalpha << send_routing_tokens_;
00212 if (!send_routing_tokens_) return;
00213 if (token_socket_ == -1) setup_tokens_();
00214 detail::RoutingToken token;
00215 token.header = TOKEN_MAGIC;
00216 token.rank = my_rank;
00217 token.new_slots_free = nSlots;
00218
00219 TLOG(TLVL_TRACE) << "Sending RoutingToken to " << token_address_ << ":" << token_port_;
00220 size_t sts = 0;
00221 while (sts < sizeof(detail::RoutingToken))
00222 {
00223 auto res = send(token_socket_, reinterpret_cast<uint8_t*>(&token) + sts, sizeof(detail::RoutingToken) - sts, 0);
00224 if (res == -1)
00225 {
00226 usleep(1000);
00227 continue;
00228 }
00229 sts += res;
00230 }
00231 TLOG(TLVL_TRACE) << "Done sending RoutingToken to " << token_address_ << ":" << token_port_;
00232 }
00233
00234 void RequestSender::SendRoutingToken(int nSlots)
00235 {
00236 if (!send_routing_tokens_) return;
00237 boost::thread token([=] { send_routing_token_(nSlots); });
00238 token.detach();
00239 usleep(0);
00240 }
00241
00242 void RequestSender::SendRequest(bool endOfRunOnly)
00243 {
00244 if (!send_requests_) return;
00245 if (endOfRunOnly && request_mode_ != detail::RequestMessageMode::EndOfRun) return;
00246 request_sending_++;
00247 boost::thread request([=] { do_send_request_(); });
00248 request.detach();
00249 }
00250
00251 void RequestSender::AddRequest(Fragment::sequence_id_t seqID, Fragment::timestamp_t timestamp)
00252 {
00253 {
00254 std::lock_guard<std::mutex> lk(request_mutex_);
00255 if (!active_requests_.count(seqID)) active_requests_[seqID] = timestamp;
00256 }
00257 SendRequest();
00258 }
00259
00260 void RequestSender::RemoveRequest(Fragment::sequence_id_t seqID)
00261 {
00262 std::lock_guard<std::mutex> lk(request_mutex_);
00263 active_requests_.erase(seqID);
00264 }
00265 }