00001 #define TRACE_NAME "RequestSender"
00002 #include "artdaq/DAQdata/Globals.hh"
00003 #include "artdaq/DAQrate/RequestSender.hh"
00004 #include <utility>
00005 #include <cstring>
00006 #include <dlfcn.h>
00007 #include <iomanip>
00008 #include <fstream>
00009 #include <sstream>
00010 #include <chrono>
00011
00012 #include "cetlib_except/exception.h"
00013 #include "artdaq-core/Core/StatisticsCollection.hh"
00014 #include "artdaq-core/Core/SimpleMemoryReader.hh"
00015 #include "artdaq/DAQrate/detail/RoutingPacket.hh"
00016 #include "artdaq/DAQdata/TCPConnect.hh"
00017
00018 namespace artdaq
00019 {
00020
00021 RequestSender::RequestSender(const fhicl::ParameterSet& pset)
00022 : send_requests_(pset.get<bool>("send_requests", false))
00023 , active_requests_()
00024 , request_port_(pset.get<int>("request_port", 3001))
00025 , request_delay_(pset.get<size_t>("request_delay_ms", 10) * 1000)
00026 , request_shutdown_timeout_us_(pset.get<size_t>("request_shutdown_timeout_us", 100000))
00027 , multicast_out_addr_(pset.get<std::string>("output_address", "0.0.0.0"))
00028 , request_mode_(detail::RequestMessageMode::Normal)
00029 , token_socket_(-1)
00030 {
00031 TLOG_DEBUG("RequestSender") << "RequestSender CONSTRUCTOR" << TLOG_ENDL;
00032 setup_requests_(pset.get<std::string>("request_address", "227.128.12.26"));
00033
00034 auto rmConfig = pset.get<fhicl::ParameterSet>("routing_token_config", fhicl::ParameterSet());
00035 send_routing_tokens_ = rmConfig.get<bool>("use_routing_master", false);
00036 token_port_ = rmConfig.get<int>("routing_token_port", 35555);
00037 token_address_ = rmConfig.get<std::string>("routing_master_hostname", "localhost");
00038 setup_tokens_();
00039 TRACE(12, "artdaq::RequestSender::RequestSender ctor - reader_thread_ initialized");
00040 }
00041
00042
00043 RequestSender::~RequestSender()
00044 {
00045 TLOG_TRACE("RequestSender") << "Shutting down RequestSender: Waiting for requests to be sent" << TLOG_ENDL;
00046
00047 auto start_time = std::chrono::steady_clock::now();
00048
00049 while (request_sending_ && request_shutdown_timeout_us_ > TimeUtils::GetElapsedTimeMicroseconds(start_time))
00050 {
00051 usleep(1000);
00052 }
00053 TLOG_TRACE("RequestSender") << "Shutting down RequestSender" << TLOG_ENDL;
00054 if (request_socket_ > 0)
00055 {
00056 shutdown(request_socket_, 2);
00057 close(request_socket_);
00058 }
00059 if (token_socket_ > 0)
00060 {
00061 shutdown(token_socket_, 2);
00062 close(token_socket_);
00063 }
00064 }
00065
00066
00067 void RequestSender::SetRequestMode(detail::RequestMessageMode mode)
00068 {
00069 request_mode_ = mode;
00070 SendRequest(true);
00071 }
00072
00073 void
00074 RequestSender::setup_requests_(std::string request_address)
00075 {
00076 if (send_requests_)
00077 {
00078 request_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
00079 if (request_socket_ < 0)
00080 {
00081 TLOG_ERROR("RequestSender") << "I failed to create the socket for sending Data Requests! err=" << strerror(errno) << TLOG_ENDL;
00082 exit(1);
00083 }
00084 int sts = ResolveHost(request_address.c_str(), request_port_, request_addr_);
00085 if (sts == -1)
00086 {
00087 TLOG_ERROR("RequestSender") << "Unable to resolve Data Request address, err=" << strerror(errno) << TLOG_ENDL;
00088 exit(1);
00089 }
00090
00091 if (multicast_out_addr_ == "0.0.0.0")
00092 {
00093 multicast_out_addr_.reserve(HOST_NAME_MAX);
00094 sts = gethostname(&multicast_out_addr_[0], HOST_NAME_MAX);
00095 if (sts < 0)
00096 {
00097 TLOG_ERROR("RequestSender") << "Could not get current hostname, err=" << strerror(errno) << TLOG_ENDL;
00098 exit(1);
00099 }
00100 }
00101
00102 if (multicast_out_addr_ != "localhost")
00103 {
00104 struct in_addr addr;
00105 sts = ResolveHost(multicast_out_addr_.c_str(), addr);
00106 if (sts == -1)
00107 {
00108 TLOG_ERROR("RequestSender") << "Unable to resolve multicast interface address, err=" << strerror(errno) << TLOG_ENDL;
00109 exit(1);
00110 }
00111
00112 int yes = 1;
00113 if (setsockopt(request_socket_, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) < 0)
00114 {
00115 TLOG_ERROR("RequestSender") << "Unable to enable port reuse on request socket, err=" << strerror(errno) << TLOG_ENDL;
00116 exit(1);
00117 }
00118 if (setsockopt(request_socket_, IPPROTO_IP, IP_MULTICAST_LOOP, &yes, sizeof(yes)) < 0)
00119 {
00120 TLOG_ERROR("RequestSender") << "Unable to enable multicast loopback on request socket, err=" << strerror(errno) << TLOG_ENDL;
00121 exit(1);
00122 }
00123 if (setsockopt(request_socket_, IPPROTO_IP, IP_MULTICAST_IF, &addr, sizeof(addr)) == -1)
00124 {
00125 TLOG_ERROR("RequestSender") << "Cannot set outgoing interface, err=" << strerror(errno) << TLOG_ENDL;
00126 exit(1);
00127 }
00128 }
00129 int yes = 1;
00130 if (setsockopt(request_socket_, SOL_SOCKET, SO_BROADCAST, (void*)&yes, sizeof(int)) == -1)
00131 {
00132 TLOG_ERROR("RequestSender") << "Cannot set request socket to broadcast, err=" << strerror(errno) << TLOG_ENDL;
00133 exit(1);
00134 }
00135 }
00136 }
00137
00138 void
00139 RequestSender::setup_tokens_()
00140 {
00141 if (send_routing_tokens_)
00142 {
00143 TLOG_DEBUG("RequestSender") << "Creating Routing Token sending socket" << TLOG_ENDL;
00144 int retry = 5;
00145 while (retry > 0 && token_socket_ < 0) {
00146 token_socket_ = TCPConnect(token_address_.c_str(), token_port_);
00147 if (token_socket_ < 0) usleep(100000);
00148 retry--;
00149 }
00150 if (token_socket_ < 0)
00151 {
00152 TLOG_ERROR("RequestSender") << "I failed to create the socket for sending Routing Tokens! err=" << strerror(errno) << TLOG_ENDL;
00153 exit(1);
00154 }
00155 TLOG_DEBUG("RequestSender") << "Routing Token sending socket created successfully" << TLOG_ENDL;
00156 }
00157 }
00158
00159 void RequestSender::do_send_request_()
00160 {
00161 if (!send_requests_) return;
00162 TLOG_TRACE("RequestSender") << "Waiting for " << request_delay_ << " microseconds." << TLOG_ENDL;
00163 std::this_thread::sleep_for(std::chrono::microseconds(request_delay_));
00164
00165 TLOG_TRACE("RequestSender") << "Creating RequestMessage" << TLOG_ENDL;
00166 detail::RequestMessage message;
00167 {
00168 std::lock_guard<std::mutex> lk(request_mutex_);
00169 for (auto& req : active_requests_)
00170 {
00171 message.addRequest(req.first, req.second);
00172 }
00173 }
00174 TLOG_TRACE("RequestSender") << "Setting mode flag in Message Header" << TLOG_ENDL;
00175 message.header()->mode = request_mode_;
00176 char str[INET_ADDRSTRLEN];
00177 inet_ntop(AF_INET, &(request_addr_.sin_addr), str, INET_ADDRSTRLEN);
00178 std::lock_guard<std::mutex> lk2(request_send_mutex_);
00179 TLOG_TRACE("RequestSender") << "Sending request for " << std::to_string(message.size()) << " events to multicast group " << str << TLOG_ENDL;
00180 if (sendto(request_socket_, message.header(), sizeof(detail::RequestHeader), 0, (struct sockaddr *)&request_addr_, sizeof(request_addr_)) < 0)
00181 {
00182 TLOG_ERROR("RequestSender") << "Error sending request message header err=" << strerror(errno) << TLOG_ENDL;
00183 }
00184 if (sendto(request_socket_, message.buffer(), sizeof(detail::RequestPacket) * message.size(), 0, (struct sockaddr *)&request_addr_, sizeof(request_addr_)) < 0)
00185 {
00186 TLOG_ERROR("RequestSender") << "Error sending request message data err=" << strerror(errno) << TLOG_ENDL;
00187 }
00188 request_sending_ = false;
00189 }
00190
00191 void RequestSender::send_routing_token_(int nSlots)
00192 {
00193 TLOG_TRACE("RequestSender") << "send_routing_token_ called, send_routing_tokens_=" << std::boolalpha << send_routing_tokens_ << TLOG_ENDL;
00194 if (!send_routing_tokens_) return;
00195 if (token_socket_ == -1) setup_tokens_();
00196 detail::RoutingToken token;
00197 token.header = TOKEN_MAGIC;
00198 token.rank = my_rank;
00199 token.new_slots_free = nSlots;
00200
00201 TLOG_TRACE("RequestSender") << "Sending RoutingToken to " << token_address_ << ":" << token_port_ << TLOG_ENDL;
00202 size_t sts = 0;
00203 while (sts < sizeof(detail::RoutingToken))
00204 {
00205 auto res = send(token_socket_, reinterpret_cast<uint8_t*>(&token) + sts, sizeof(detail::RoutingToken) - sts, 0);
00206 if (res == -1)
00207 {
00208 usleep(1000);
00209 continue;
00210 }
00211 sts += res;
00212 }
00213 TLOG_TRACE("RequestSender") << "Done sending RoutingToken to " << token_address_ << ":" << token_port_ << TLOG_ENDL;
00214 }
00215
00216 void RequestSender::SendRoutingToken(int nSlots)
00217 {
00218 if (!send_routing_tokens_) return;
00219 boost::thread token([=] {send_routing_token_(nSlots); });
00220 token.detach();
00221 usleep(0);
00222 }
00223
00224 void RequestSender::SendRequest(bool endOfRunOnly)
00225 {
00226 if (!send_requests_) return;
00227 if (endOfRunOnly && request_mode_ != detail::RequestMessageMode::EndOfRun) return;
00228 request_sending_ = true;
00229 boost::thread request([=] { do_send_request_(); });
00230 request.detach();
00231 }
00232
00233 void RequestSender::AddRequest(Fragment::sequence_id_t seqID, Fragment::timestamp_t timestamp)
00234 {
00235 {
00236 std::lock_guard<std::mutex> lk(request_mutex_);
00237 if (!active_requests_.count(seqID)) active_requests_[seqID] = timestamp;
00238 }
00239 SendRequest();
00240 }
00241
00242 void RequestSender::RemoveRequest(Fragment::sequence_id_t seqID)
00243 {
00244 std::lock_guard<std::mutex> lk(request_mutex_);
00245 active_requests_.erase(seqID);
00246 }
00247 }