00001 #define TRACE_NAME (app_name + "_RequestSender").c_str()
00002 #include "artdaq/DAQdata/Globals.hh"
00003 #include "artdaq/DAQrate/RequestSender.hh"
00004 #include <utility>
00005 #include <cstring>
00006 #include <dlfcn.h>
00007 #include <iomanip>
00008 #include <fstream>
00009 #include <sstream>
00010 #include <chrono>
00011
00012 #include "cetlib_except/exception.h"
00013 #include "artdaq-core/Core/StatisticsCollection.hh"
00014 #include "artdaq-core/Core/SimpleMemoryReader.hh"
00015 #include "artdaq/DAQrate/detail/RoutingPacket.hh"
00016 #include "artdaq/DAQdata/TCPConnect.hh"
00017
00018 namespace artdaq
00019 {
00020 RequestSender::RequestSender(const fhicl::ParameterSet& pset)
00021 : send_requests_(pset.get<bool>("send_requests", false))
00022 , initialized_(false)
00023 , active_requests_()
00024 , request_address_(pset.get<std::string>("request_address", "227.128.12.26"))
00025 , request_port_(pset.get<int>("request_port", 3001))
00026 , request_delay_(pset.get<size_t>("request_delay_ms", 0) * 1000)
00027 , request_shutdown_timeout_us_(pset.get<size_t>("request_shutdown_timeout_us", 100000))
00028 , multicast_out_addr_(pset.get<std::string>("multicast_interface_ip", pset.get<std::string>("output_address", "0.0.0.0")))
00029 , request_mode_(detail::RequestMessageMode::Normal)
00030 , token_socket_(-1)
00031 , request_sending_(0)
00032 , tokens_sent_(0)
00033 {
00034 TLOG(TLVL_DEBUG) << "RequestSender CONSTRUCTOR";
00035 setup_requests_();
00036
00037 auto rmConfig = pset.get<fhicl::ParameterSet>("routing_token_config", fhicl::ParameterSet());
00038 send_routing_tokens_ = rmConfig.get<bool>("use_routing_master", false);
00039 token_port_ = rmConfig.get<int>("routing_token_port", 35555);
00040 token_address_ = rmConfig.get<std::string>("routing_master_hostname", "localhost");
00041 setup_tokens_();
00042 TLOG(12) << "artdaq::RequestSender::RequestSender ctor - reader_thread_ initialized";
00043 initialized_ = true;
00044 }
00045
00046
00047 RequestSender::~RequestSender()
00048 {
00049 TLOG(TLVL_INFO) << "Shutting down RequestSender: Waiting for " << request_sending_.load() << " requests to be sent";
00050
00051 auto start_time = std::chrono::steady_clock::now();
00052
00053 while (request_sending_.load() > 0 && request_shutdown_timeout_us_ + request_delay_ > TimeUtils::GetElapsedTimeMicroseconds(start_time))
00054 {
00055 usleep(1000);
00056 }
00057 {
00058 std::unique_lock<std::mutex> lk(request_mutex_);
00059 std::unique_lock<std::mutex> lk2(request_send_mutex_);
00060 }
00061 TLOG(TLVL_INFO) << "Shutting down RequestSender";
00062 if (request_socket_ > 0)
00063 {
00064 shutdown(request_socket_, 2);
00065 close(request_socket_);
00066 }
00067 if (token_socket_ > 0)
00068 {
00069 shutdown(token_socket_, 2);
00070 close(token_socket_);
00071 }
00072 }
00073
00074
00075 void RequestSender::SetRequestMode(detail::RequestMessageMode mode)
00076 {
00077 request_mode_ = mode;
00078 SendRequest(true);
00079 }
00080
00081 void RequestSender::setup_requests_()
00082 {
00083 if (send_requests_)
00084 {
00085 request_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
00086 if (request_socket_ < 0)
00087 {
00088 TLOG(TLVL_ERROR) << "I failed to create the socket for sending Data Requests! err=" << strerror(errno);
00089 exit(1);
00090 }
00091 int sts = ResolveHost(request_address_.c_str(), request_port_, request_addr_);
00092 if (sts == -1)
00093 {
00094 TLOG(TLVL_ERROR) << "Unable to resolve Data Request address, err=" << strerror(errno);
00095 exit(1);
00096 }
00097
00098 if (multicast_out_addr_ == "0.0.0.0")
00099 {
00100 multicast_out_addr_.reserve(HOST_NAME_MAX);
00101 sts = gethostname(&multicast_out_addr_[0], HOST_NAME_MAX);
00102 if (sts < 0)
00103 {
00104 TLOG(TLVL_ERROR) << "Could not get current hostname, err=" << strerror(errno);
00105 exit(1);
00106 }
00107 }
00108
00109 if (multicast_out_addr_ != "localhost")
00110 {
00111 struct in_addr addr;
00112 sts = GetInterfaceForNetwork(multicast_out_addr_.c_str(), addr);
00113
00114 if (sts == -1)
00115 {
00116 TLOG(TLVL_ERROR) << "Unable to resolve multicast interface address, err=" << strerror(errno);
00117 exit(1);
00118 }
00119
00120 if (setsockopt(request_socket_, IPPROTO_IP, IP_MULTICAST_IF, &addr, sizeof(addr)) == -1)
00121 {
00122 TLOG(TLVL_ERROR) << "Cannot set outgoing interface, err=" << strerror(errno);
00123 exit(1);
00124 }
00125 }
00126 int yes = 1;
00127 if (setsockopt(request_socket_, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) < 0)
00128 {
00129 TLOG(TLVL_ERROR) << "Unable to enable port reuse on request socket, err=" << strerror(errno);
00130 exit(1);
00131 }
00132 if (setsockopt(request_socket_, IPPROTO_IP, IP_MULTICAST_LOOP, &yes, sizeof(yes)) < 0)
00133 {
00134 TLOG(TLVL_ERROR) << "Unable to enable multicast loopback on request socket, err=" << strerror(errno);
00135 exit(1);
00136 }
00137 if (setsockopt(request_socket_, SOL_SOCKET, SO_BROADCAST, (void*)&yes, sizeof(int)) == -1)
00138 {
00139 TLOG(TLVL_ERROR) << "Cannot set request socket to broadcast, err=" << strerror(errno);
00140 exit(1);
00141 }
00142 }
00143 }
00144
00145 void
00146 RequestSender::setup_tokens_()
00147 {
00148 if (send_routing_tokens_)
00149 {
00150 TLOG(TLVL_DEBUG) << "Creating Routing Token sending socket";
00151 int retry = 5;
00152 while (retry > 0 && token_socket_ < 0)
00153 {
00154 token_socket_ = TCPConnect(token_address_.c_str(), token_port_, 0, sizeof(detail::RoutingToken));
00155 if (token_socket_ < 0) usleep(100000);
00156 retry--;
00157 }
00158 if (token_socket_ < 0)
00159 {
00160 TLOG(TLVL_ERROR) << "I failed to create the socket for sending Routing Tokens! err=" << strerror(errno);
00161 exit(1);
00162 }
00163 TLOG(TLVL_DEBUG) << "Routing Token sending socket created successfully";
00164 }
00165 }
00166
00167 void RequestSender::do_send_request_()
00168 {
00169 if (!send_requests_) {
00170 request_sending_--;
00171 return;
00172 }
00173 if (request_socket_ == -1) setup_requests_();
00174
00175 TLOG(TLVL_TRACE) << "Waiting for " << request_delay_ << " microseconds.";
00176 std::this_thread::sleep_for(std::chrono::microseconds(request_delay_));
00177
00178 TLOG(TLVL_TRACE) << "Creating RequestMessage";
00179 detail::RequestMessage message;
00180 {
00181 std::unique_lock<std::mutex> lk(request_mutex_);
00182 for (auto& req : active_requests_)
00183 {
00184 TLOG(12) << "Adding a request with sequence ID " << req.first << ", timestamp " << req.second << " to request message";
00185 message.addRequest(req.first, req.second);
00186 }
00187 }
00188 TLOG(TLVL_TRACE) << "Setting mode flag in Message Header";
00189 message.setMode(request_mode_);
00190 char str[INET_ADDRSTRLEN];
00191 inet_ntop(AF_INET, &(request_addr_.sin_addr), str, INET_ADDRSTRLEN);
00192 std::unique_lock<std::mutex> lk2(request_send_mutex_);
00193 TLOG(TLVL_TRACE) << "Sending request for " << message.size() << " events to multicast group " << str;
00194 auto buf = message.GetMessage();
00195 auto sts=sendto(request_socket_, &buf[0], buf.size(), 0, (struct sockaddr *)&request_addr_, sizeof(request_addr_));
00196 if (sts < 0 || static_cast<size_t>(sts) != buf.size())
00197 {
00198 TLOG(TLVL_ERROR) << "Error sending request message err=" << strerror(errno)<< "sts="<<sts;
00199 request_socket_ = -1;
00200 request_sending_--;
00201 return;
00202 }
00203 TLOG(TLVL_TRACE) << "Done sending request sts="<<sts;
00204 request_sending_--;
00205 }
00206
00207 void RequestSender::send_routing_token_(int nSlots)
00208 {
00209 TLOG(TLVL_TRACE) << "send_routing_token_ called, send_routing_tokens_=" << std::boolalpha << send_routing_tokens_;
00210 if (!send_routing_tokens_) return;
00211 if (token_socket_ == -1) setup_tokens_();
00212 detail::RoutingToken token;
00213 token.header = TOKEN_MAGIC;
00214 token.rank = my_rank;
00215 token.new_slots_free = nSlots;
00216
00217 TLOG(TLVL_TRACE) << "Sending RoutingToken to " << token_address_ << ":" << token_port_;
00218 size_t sts = 0;
00219 while (sts < sizeof(detail::RoutingToken))
00220 {
00221 auto res = send(token_socket_, reinterpret_cast<uint8_t*>(&token) + sts, sizeof(detail::RoutingToken) - sts, 0);
00222 if (res == -1)
00223 {
00224 close(token_socket_);
00225 token_socket_ = -1;
00226 sts = 0;
00227 setup_tokens_();
00228 continue;
00229 }
00230 sts += res;
00231 }
00232 tokens_sent_ += nSlots;
00233 TLOG(TLVL_TRACE) << "Done sending RoutingToken to " << token_address_ << ":" << token_port_;
00234 }
00235
00236 void RequestSender::SendRoutingToken(int nSlots)
00237 {
00238 while (!initialized_) usleep(1000);
00239 if (!send_routing_tokens_) return;
00240 boost::thread token([=] { send_routing_token_(nSlots); });
00241 token.detach();
00242 usleep(0);
00243 }
00244
00245 void RequestSender::SendRequest(bool endOfRunOnly)
00246 {
00247 while (!initialized_) usleep(1000);
00248
00249 if (!send_requests_) return;
00250 if (endOfRunOnly && request_mode_ != detail::RequestMessageMode::EndOfRun) return;
00251 request_sending_++;
00252 boost::thread request([=] { do_send_request_(); });
00253 request.detach();
00254 }
00255
00256 void RequestSender::AddRequest(Fragment::sequence_id_t seqID, Fragment::timestamp_t timestamp)
00257 {
00258 while (!initialized_) usleep(1000);
00259
00260 {
00261 std::lock_guard<std::mutex> lk(request_mutex_);
00262 if (!active_requests_.count(seqID)) {
00263 TLOG(12) << "Adding request for sequence ID " << seqID << " and timestamp " << timestamp << " to request list.";
00264 active_requests_[seqID] = timestamp;
00265 }
00266 }
00267 SendRequest();
00268 }
00269
00270 void RequestSender::RemoveRequest(Fragment::sequence_id_t seqID)
00271 {
00272 while (!initialized_) usleep(1000);
00273 std::lock_guard<std::mutex> lk(request_mutex_);
00274 TLOG(12) << "Removing request for sequence ID " << seqID << " from request list.";
00275 active_requests_.erase(seqID);
00276 }
00277 }