$treeview $search $mathjax $extrastylesheet
artdaq
v3_04_00
$projectbrief
|
$projectbrief
|
$searchbox |
00001 #define TRACE_NAME (app_name + "_RequestSender").c_str() 00002 #include "artdaq/DAQdata/Globals.hh" // Before trace.h gets included in ConcurrentQueue (from GlobalQueue) 00003 #include "artdaq/DAQrate/RequestSender.hh" 00004 #include <utility> 00005 #include <cstring> 00006 #include <dlfcn.h> 00007 #include <iomanip> 00008 #include <fstream> 00009 #include <sstream> 00010 #include <chrono> 00011 00012 #include "cetlib_except/exception.h" 00013 #include "artdaq-core/Core/StatisticsCollection.hh" 00014 #include "artdaq-core/Core/SimpleMemoryReader.hh" 00015 #include "artdaq/DAQrate/detail/RoutingPacket.hh" 00016 #include "artdaq/DAQdata/TCPConnect.hh" 00017 00018 namespace artdaq 00019 { 00020 RequestSender::RequestSender(const fhicl::ParameterSet& pset) 00021 : send_requests_(pset.get<bool>("send_requests", false)) 00022 , initialized_(false) 00023 , active_requests_() 00024 , request_address_(pset.get<std::string>("request_address", "227.128.12.26")) 00025 , request_port_(pset.get<int>("request_port", 3001)) 00026 , request_delay_(pset.get<size_t>("request_delay_ms", 0) * 1000) 00027 , request_shutdown_timeout_us_(pset.get<size_t>("request_shutdown_timeout_us", 100000)) 00028 , multicast_out_addr_(pset.get<std::string>("multicast_interface_ip", pset.get<std::string>("output_address", "0.0.0.0"))) 00029 , request_mode_(detail::RequestMessageMode::Normal) 00030 , token_socket_(-1) 00031 , request_sending_(0) 00032 , tokens_sent_(0) 00033 , run_number_(0) 00034 { 00035 TLOG(TLVL_DEBUG) << "RequestSender CONSTRUCTOR"; 00036 setup_requests_(); 00037 00038 auto rmConfig = pset.get<fhicl::ParameterSet>("routing_token_config", fhicl::ParameterSet()); 00039 send_routing_tokens_ = rmConfig.get<bool>("use_routing_master", false); 00040 token_port_ = rmConfig.get<int>("routing_token_port", 35555); 00041 token_address_ = rmConfig.get<std::string>("routing_master_hostname", "localhost"); 00042 setup_tokens_(); 00043 TLOG(12) << "artdaq::RequestSender::RequestSender ctor - reader_thread_ initialized"; 00044 initialized_ = true; 00045 } 00046 00047 00048 RequestSender::~RequestSender() 00049 { 00050 TLOG(TLVL_INFO) << "Shutting down RequestSender: Waiting for " << request_sending_.load() << " requests to be sent"; 00051 00052 auto start_time = std::chrono::steady_clock::now(); 00053 00054 while (request_sending_.load() > 0 && request_shutdown_timeout_us_ + request_delay_ > TimeUtils::GetElapsedTimeMicroseconds(start_time)) 00055 { 00056 usleep(1000); 00057 } 00058 { 00059 std::unique_lock<std::mutex> lk(request_mutex_); 00060 std::unique_lock<std::mutex> lk2(request_send_mutex_); 00061 } 00062 TLOG(TLVL_INFO) << "Shutting down RequestSender"; 00063 if (request_socket_ > 0) 00064 { 00065 shutdown(request_socket_, 2); 00066 close(request_socket_); 00067 } 00068 if (token_socket_ > 0) 00069 { 00070 shutdown(token_socket_, 2); 00071 close(token_socket_); 00072 } 00073 } 00074 00075 00076 void RequestSender::SetRequestMode(detail::RequestMessageMode mode) 00077 { 00078 request_mode_ = mode; 00079 SendRequest(true); 00080 } 00081 00082 void RequestSender::setup_requests_() 00083 { 00084 if (send_requests_) 00085 { 00086 request_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); 00087 if (request_socket_ < 0) 00088 { 00089 TLOG(TLVL_ERROR) << "I failed to create the socket for sending Data Requests! err=" << strerror(errno); 00090 exit(1); 00091 } 00092 int sts = ResolveHost(request_address_.c_str(), request_port_, request_addr_); 00093 if (sts == -1) 00094 { 00095 TLOG(TLVL_ERROR) << "Unable to resolve Data Request address, err=" << strerror(errno); 00096 exit(1); 00097 } 00098 00099 if (multicast_out_addr_ == "0.0.0.0") 00100 { 00101 multicast_out_addr_.reserve(HOST_NAME_MAX); 00102 sts = gethostname(&multicast_out_addr_[0], HOST_NAME_MAX); 00103 if (sts < 0) 00104 { 00105 TLOG(TLVL_ERROR) << "Could not get current hostname, err=" << strerror(errno); 00106 exit(1); 00107 } 00108 } 00109 00110 if (multicast_out_addr_ != "localhost") 00111 { 00112 struct in_addr addr; 00113 sts = GetInterfaceForNetwork(multicast_out_addr_.c_str(), addr); 00114 //sts = ResolveHost(multicast_out_addr_.c_str(), addr); 00115 if (sts == -1) 00116 { 00117 TLOG(TLVL_ERROR) << "Unable to resolve multicast interface address, err=" << strerror(errno); 00118 exit(1); 00119 } 00120 00121 if (setsockopt(request_socket_, IPPROTO_IP, IP_MULTICAST_IF, &addr, sizeof(addr)) == -1) 00122 { 00123 TLOG(TLVL_ERROR) << "Cannot set outgoing interface, err=" << strerror(errno); 00124 exit(1); 00125 } 00126 } 00127 int yes = 1; 00128 if (setsockopt(request_socket_, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) < 0) 00129 { 00130 TLOG(TLVL_ERROR) << "Unable to enable port reuse on request socket, err=" << strerror(errno); 00131 exit(1); 00132 } 00133 if (setsockopt(request_socket_, IPPROTO_IP, IP_MULTICAST_LOOP, &yes, sizeof(yes)) < 0) 00134 { 00135 TLOG(TLVL_ERROR) << "Unable to enable multicast loopback on request socket, err=" << strerror(errno); 00136 exit(1); 00137 } 00138 if (setsockopt(request_socket_, SOL_SOCKET, SO_BROADCAST, (void*)&yes, sizeof(int)) == -1) 00139 { 00140 TLOG(TLVL_ERROR) << "Cannot set request socket to broadcast, err=" << strerror(errno); 00141 exit(1); 00142 } 00143 } 00144 } 00145 00146 void 00147 RequestSender::setup_tokens_() 00148 { 00149 if (send_routing_tokens_) 00150 { 00151 TLOG(TLVL_DEBUG) << "Creating Routing Token sending socket"; 00152 auto start_time = std::chrono::steady_clock::now(); 00153 while (token_socket_ < 0 && TimeUtils::GetElapsedTime(start_time) < 30) 00154 { 00155 token_socket_ = TCPConnect(token_address_.c_str(), token_port_, 0, sizeof(detail::RoutingToken)); 00156 if (token_socket_ < 0) 00157 { 00158 TLOG(TLVL_TRACE) << "Waited " << TimeUtils::GetElapsedTime(start_time) << " s for Routing Master to open token socket"; 00159 usleep(100000); 00160 } 00161 } 00162 if (token_socket_ < 0) 00163 { 00164 TLOG(TLVL_ERROR) << "I failed to create the socket for sending Routing Tokens! err=" << strerror(errno); 00165 exit(1); 00166 } 00167 TLOG(TLVL_DEBUG) << "Routing Token sending socket created successfully"; 00168 } 00169 } 00170 00171 void RequestSender::do_send_request_() 00172 { 00173 if (!send_requests_) { 00174 request_sending_--; 00175 return; 00176 } 00177 if (request_socket_ == -1) setup_requests_(); 00178 00179 TLOG(TLVL_TRACE) << "Waiting for " << request_delay_ << " microseconds."; 00180 std::this_thread::sleep_for(std::chrono::microseconds(request_delay_)); 00181 00182 TLOG(TLVL_TRACE) << "Creating RequestMessage"; 00183 detail::RequestMessage message; 00184 message.setRank(my_rank); 00185 message.setRunNumber(run_number_); 00186 { 00187 std::unique_lock<std::mutex> lk(request_mutex_); 00188 for (auto& req : active_requests_) 00189 { 00190 TLOG(12) << "Adding a request with sequence ID " << req.first << ", timestamp " << req.second << " to request message"; 00191 message.addRequest(req.first, req.second); 00192 } 00193 } 00194 TLOG(TLVL_TRACE) << "Setting mode flag in Message Header to " << request_mode_; 00195 message.setMode(request_mode_); 00196 char str[INET_ADDRSTRLEN]; 00197 inet_ntop(AF_INET, &(request_addr_.sin_addr), str, INET_ADDRSTRLEN); 00198 std::unique_lock<std::mutex> lk2(request_send_mutex_); 00199 TLOG(TLVL_TRACE) << "Sending request for " << message.size() << " events to multicast group " << str; 00200 auto buf = message.GetMessage(); 00201 auto sts=sendto(request_socket_, &buf[0], buf.size(), 0, (struct sockaddr *)&request_addr_, sizeof(request_addr_)); 00202 if (sts < 0 || static_cast<size_t>(sts) != buf.size()) 00203 { 00204 TLOG(TLVL_ERROR) << "Error sending request message err=" << strerror(errno)<< "sts="<<sts; 00205 request_socket_ = -1; 00206 request_sending_--; 00207 return; 00208 } 00209 TLOG(TLVL_TRACE) << "Done sending request sts="<<sts; 00210 request_sending_--; 00211 } 00212 00213 void RequestSender::send_routing_token_(int nSlots, int run_number) 00214 { 00215 TLOG(TLVL_TRACE) << "send_routing_token_ called, send_routing_tokens_=" << std::boolalpha << send_routing_tokens_; 00216 if (!send_routing_tokens_) return; 00217 if (token_socket_ == -1) setup_tokens_(); 00218 detail::RoutingToken token; 00219 token.header = TOKEN_MAGIC; 00220 token.rank = my_rank; 00221 token.new_slots_free = nSlots; 00222 token.run_number = run_number; 00223 00224 TLOG(TLVL_TRACE) << "Sending RoutingToken to " << token_address_ << ":" << token_port_; 00225 size_t sts = 0; 00226 while (sts < sizeof(detail::RoutingToken)) 00227 { 00228 auto res = send(token_socket_, reinterpret_cast<uint8_t*>(&token) + sts, sizeof(detail::RoutingToken) - sts, 0); 00229 if (res < 0) 00230 { 00231 close(token_socket_); 00232 token_socket_ = -1; 00233 sts = 0; 00234 setup_tokens_(); 00235 continue; 00236 } 00237 sts += res; 00238 } 00239 tokens_sent_ += nSlots; 00240 TLOG(TLVL_TRACE) << "Done sending RoutingToken to " << token_address_ << ":" << token_port_; 00241 } 00242 00243 void RequestSender::SendRoutingToken(int nSlots, int run_number) 00244 { 00245 while (!initialized_) usleep(1000); 00246 if (!send_routing_tokens_) return; 00247 boost::thread token([=] { send_routing_token_(nSlots, run_number); }); 00248 token.detach(); 00249 usleep(0); // Give up time slice 00250 } 00251 00252 void RequestSender::SendRequest(bool endOfRunOnly) 00253 { 00254 while (!initialized_) usleep(1000); 00255 00256 if (!send_requests_) return; 00257 if (endOfRunOnly && request_mode_ != detail::RequestMessageMode::EndOfRun) return; 00258 request_sending_++; 00259 boost::thread request([=] { do_send_request_(); }); 00260 request.detach(); 00261 } 00262 00263 void RequestSender::AddRequest(Fragment::sequence_id_t seqID, Fragment::timestamp_t timestamp) 00264 { 00265 while (!initialized_) usleep(1000); 00266 00267 { 00268 std::lock_guard<std::mutex> lk(request_mutex_); 00269 if (!active_requests_.count(seqID)) { 00270 TLOG(12) << "Adding request for sequence ID " << seqID << " and timestamp " << timestamp << " to request list."; 00271 active_requests_[seqID] = timestamp; 00272 } 00273 } 00274 SendRequest(); 00275 } 00276 00277 void RequestSender::RemoveRequest(Fragment::sequence_id_t seqID) 00278 { 00279 while (!initialized_) usleep(1000); 00280 std::lock_guard<std::mutex> lk(request_mutex_); 00281 TLOG(12) << "Removing request for sequence ID " << seqID << " from request list."; 00282 active_requests_.erase(seqID); 00283 } 00284 }