00001 #define TRACE_NAME "RequestReceiver"
00002
00003 #include "artdaq/DAQrate/RequestReceiver.hh"
00004 #include "artdaq/DAQdata/Globals.hh"
00005 #include "artdaq/DAQrate/detail/RequestMessage.hh"
00006
00007 #include <boost/exception/all.hpp>
00008 #include <boost/throw_exception.hpp>
00009
00010 #include <limits>
00011 #include <iterator>
00012
00013 #include "canvas/Utilities/Exception.h"
00014 #include "cetlib_except/exception.h"
00015 #include "fhiclcpp/ParameterSet.h"
00016
00017 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh"
00018 #include "artdaq-core/Data/Fragment.hh"
00019 #include "artdaq-core/Data/ContainerFragmentLoader.hh"
00020 #include "artdaq-core/Utilities/ExceptionHandler.hh"
00021 #include "artdaq-core/Utilities/TimeUtils.hh"
00022
00023 #include <fstream>
00024 #include <iomanip>
00025 #include <iterator>
00026 #include <iostream>
00027 #include <iomanip>
00028 #include <algorithm>
00029 #include <sys/poll.h>
00030 #include "artdaq/DAQdata/TCPConnect.hh"
00031
00032 artdaq::RequestReceiver::RequestReceiver()
00033 : request_port_(3001)
00034 , request_addr_("227.128.12.26")
00035 , running_(false)
00036 , requests_()
00037 , request_stop_requested_(false)
00038 , request_received_(false)
00039 , end_of_run_timeout_ms_(1000)
00040 , should_stop_(false)
00041 , highest_seen_request_(0)
00042 {}
00043
00044 artdaq::RequestReceiver::RequestReceiver(const fhicl::ParameterSet& ps)
00045 : request_port_(ps.get<int>("request_port", 3001))
00046 , request_addr_(ps.get<std::string>("request_address", "227.128.12.26"))
00047 , running_(false)
00048 , requests_()
00049 , request_stop_requested_(false)
00050 , request_received_(false)
00051 , end_of_run_timeout_ms_(ps.get<size_t>("end_of_run_quiet_timeout_ms", 1000))
00052 , should_stop_(false)
00053 , highest_seen_request_(0)
00054 {
00055 setupRequestListener();
00056 }
00057
00058 void artdaq::RequestReceiver::setupRequestListener()
00059 {
00060 TLOG_INFO("RequestReceiver") << "Setting up request listen socket, rank=" << my_rank << ", address=" << request_addr_ << ":" << request_port_ << TLOG_ENDL;
00061 request_socket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
00062 if (request_socket_ < 0)
00063 {
00064 TLOG_ERROR("RequestReceiver") << "Error creating socket for receiving data requests! err=" << strerror(errno) << TLOG_ENDL;
00065 exit(1);
00066 }
00067
00068 struct sockaddr_in si_me_request;
00069
00070 int yes = 1;
00071 if (setsockopt(request_socket_, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) < 0)
00072 {
00073 TLOG_ERROR("RequestReceiver") << "Unable to enable port reuse on request socket, err=" << strerror(errno) << TLOG_ENDL;
00074 exit(1);
00075 }
00076 memset(&si_me_request, 0, sizeof(si_me_request));
00077 si_me_request.sin_family = AF_INET;
00078 si_me_request.sin_port = htons(request_port_);
00079 si_me_request.sin_addr.s_addr = htonl(INADDR_ANY);
00080 if (bind(request_socket_, (struct sockaddr *)&si_me_request, sizeof(si_me_request)) == -1)
00081 {
00082 TLOG_ERROR("RequestReceiver") << "Cannot bind request socket to port " << request_port_ << ", err=" << strerror(errno) << TLOG_ENDL;
00083 exit(1);
00084 }
00085
00086 if (request_addr_ != "localhost")
00087 {
00088 struct ip_mreq mreq;
00089 int sts = ResolveHost(request_addr_.c_str(), mreq.imr_multiaddr);
00090 if (sts == -1)
00091 {
00092 TLOG_ERROR("RequestReceiver") << "Unable to resolve multicast request address, err=" << strerror(errno) << TLOG_ENDL;
00093 exit(1);
00094 }
00095 mreq.imr_interface.s_addr = htonl(INADDR_ANY);
00096 if (setsockopt(request_socket_, IPPROTO_IP, IP_ADD_MEMBERSHIP, &mreq, sizeof(mreq)) < 0)
00097 {
00098 TLOG_ERROR("RequestReceiver") << "Unable to join multicast group, err=" << strerror(errno) << TLOG_ENDL;
00099 exit(1);
00100 }
00101 }
00102 TLOG_INFO("RequestReceiver") << "Done setting up request socket, rank=" << my_rank << TLOG_ENDL;
00103 }
00104
00105 artdaq::RequestReceiver::~RequestReceiver()
00106 {
00107 if (!request_received_)
00108 {
00109 TLOG_ERROR("RequestReceiver") << "Stop request received by RequestReceiver, but no requests have ever been received." << std::endl
00110 << "Check that UDP port " << request_port_ << " is open in the firewall config." << TLOG_ENDL;
00111 }
00112 should_stop_ = true;
00113 TLOG_DEBUG("RequestReceiver") << "Joining requestThread" << TLOG_ENDL;
00114 if (requestThread_.joinable()) requestThread_.join();
00115 if (request_socket_ != -1) close(request_socket_);
00116 }
00117
00118 void artdaq::RequestReceiver::startRequestReceiverThread()
00119 {
00120 if (requestThread_.joinable()) requestThread_.join();
00121 TLOG_INFO("RequestReceiver") << "Starting Request Reception Thread" << TLOG_ENDL;
00122 requestThread_ = boost::thread(&RequestReceiver::receiveRequestsLoop, this);
00123 running_ = true;
00124 }
00125
00126 void artdaq::RequestReceiver::receiveRequestsLoop()
00127 {
00128 while (!should_stop_)
00129 {
00130 TLOG_ARB(16, "RequestReceiver") << "receiveRequestsLoop: Polling Request socket for new requests" << TLOG_ENDL;
00131
00132 int ms_to_wait = 100;
00133 struct pollfd ufds[1];
00134 ufds[0].fd = request_socket_;
00135 ufds[0].events = POLLIN | POLLPRI;
00136 int rv = poll(ufds, 1, ms_to_wait);
00137
00138
00139 if (rv <= 0 || (ufds[0].revents != POLLIN && ufds[0].revents != POLLPRI))
00140 {
00141 if (request_stop_requested_ && TimeUtils::GetElapsedTimeMilliseconds(request_stop_timeout_) > end_of_run_timeout_ms_)
00142 {
00143 break;
00144 }
00145 continue;
00146 }
00147
00148 TLOG_ARB(11, "RequestReceiver") << "Recieved packet on Request channel" << TLOG_ENDL;
00149 artdaq::detail::RequestHeader hdr_buffer;
00150 recv(request_socket_, &hdr_buffer, sizeof(hdr_buffer), 0);
00151 TLOG_ARB(11, "RequestReceiver") << "Request header word: 0x" << std::hex << hdr_buffer.header << TLOG_ENDL;
00152 if (!hdr_buffer.isValid()) continue;
00153
00154 request_received_ = true;
00155 if (hdr_buffer.mode == artdaq::detail::RequestMessageMode::EndOfRun)
00156 {
00157 TLOG_INFO("RequestReceiver") << "Received Request Message with the EndOfRun marker. (Re)Starting 1-second timeout for receiving all outstanding requests..." << TLOG_ENDL;
00158 request_stop_timeout_ = std::chrono::steady_clock::now();
00159 request_stop_requested_ = true;
00160 }
00161
00162 std::vector<artdaq::detail::RequestPacket> pkt_buffer(hdr_buffer.packet_count);
00163 recv(request_socket_, &pkt_buffer[0], sizeof(artdaq::detail::RequestPacket) * hdr_buffer.packet_count, 0);
00164 bool anyNew = false;
00165 for (auto& buffer : pkt_buffer)
00166 {
00167 if (!buffer.isValid()) continue;
00168 if (requests_.count(buffer.sequence_id) && requests_[buffer.sequence_id] != buffer.timestamp)
00169 {
00170 TLOG_ERROR("RequestReceiver") << "Received conflicting request for SeqID "
00171 << std::to_string(buffer.sequence_id) << "!"
00172 << " Old ts=" << std::to_string(requests_[buffer.sequence_id])
00173 << ", new ts=" << std::to_string(buffer.timestamp) << ". Keeping OLD!" << TLOG_ENDL;
00174 }
00175 else if (!requests_.count(buffer.sequence_id))
00176 {
00177 int delta = buffer.sequence_id - highest_seen_request_;
00178 TLOG_ARB(11, "RequestReceiver") << "Recieved request for sequence ID " << std::to_string(buffer.sequence_id)
00179 << " and timestamp " << std::to_string(buffer.timestamp) << " (delta: " << delta << ")" << TLOG_ENDL;
00180 if (delta < 0)
00181 {
00182 TLOG_ARB(11, "RequestReceiver") << "Already serviced this request! Ignoring..." << TLOG_ENDL;
00183 }
00184 else
00185 {
00186 std::unique_lock<std::mutex> tlk(request_mutex_);
00187 requests_[buffer.sequence_id] = buffer.timestamp;
00188 anyNew = true;
00189 }
00190 }
00191 }
00192 if (anyNew) {
00193 request_cv_.notify_all();
00194 }
00195 }
00196 running_ = false;
00197 }
00198
00199 void artdaq::RequestReceiver::RemoveRequest(artdaq::Fragment::sequence_id_t reqID)
00200 {
00201 std::unique_lock<std::mutex> lk(request_mutex_);
00202 requests_.erase(reqID);
00203 if (reqID > highest_seen_request_) highest_seen_request_ = reqID;
00204 }