00001 #include "artdaq-ots/Generators/UDPReceiver.hh"
00002
00003 #include "canvas/Utilities/Exception.h"
00004 #include "artdaq/Application/GeneratorMacros.hh"
00005 #include "cetlib/exception.h"
00006 #include "fhiclcpp/ParameterSet.h"
00007 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh"
00008 #include "artdaq-ots/Overlays/UDPFragmentWriter.hh"
00009 #include "artdaq-ots/Overlays/FragmentType.hh"
00010 #include "otsdaq-core/Macros/CoutHeaderMacros.h"
00011
00012
00013 #include <fstream>
00014 #include <iomanip>
00015 #include <iterator>
00016 #include <iostream>
00017 #include <sys/poll.h>
00018
00019 ots::UDPReceiver::UDPReceiver(fhicl::ParameterSet const & ps)
00020 : CommandableFragmentGenerator(ps)
00021 , rawOutput_(ps.get<bool>("raw_output_enabled", false))
00022 , rawPath_(ps.get<std::string>("raw_output_path", "/tmp"))
00023 , dataport_(ps.get<int>("port", 6343))
00024 , ip_(ps.get<std::string>("ip", "127.0.0.1"))
00025 , rcvbuf_(ps.get<int>("rcvbuf", 0x1000000))
00026 , expectedPacketNumber_(0)
00027 , sendCommands_(ps.get<bool>("send_OtsUDP_commands", false))
00028 , fragmentWindow_(ps.get<double>("fragment_time_window_ms", 1000))
00029 , lastFrag_(std::chrono::high_resolution_clock::now())
00030 {
00031 datasocket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
00032 if (!datasocket_)
00033 {
00034 throw art::Exception(art::errors::Configuration) << "UDPReceiver: Error creating socket!";
00035 exit(1);
00036 }
00037
00038 struct sockaddr_in si_me_data;
00039 si_me_data.sin_family = AF_INET;
00040 si_me_data.sin_port = htons(dataport_);
00041 si_me_data.sin_addr.s_addr = htonl(INADDR_ANY);
00042 if (bind(datasocket_, (struct sockaddr *)&si_me_data, sizeof(si_me_data)) == -1)
00043 {
00044 throw art::Exception(art::errors::Configuration) << "UDPReceiver: Cannot bind data socket to port " << dataport_;
00045 exit(1);
00046 }
00047
00048
00049
00050
00051
00052
00053 if (rcvbuf_ > 0 && setsockopt(datasocket_, SOL_SOCKET, SO_RCVBUF, &rcvbuf_, sizeof(rcvbuf_)))
00054 {
00055 throw art::Exception(art::errors::Configuration) << "UDPReceiver: Could not set receive buffer size: " << rcvbuf_;
00056 exit(1);
00057 }
00058
00059 si_data_.sin_family = AF_INET;
00060 si_data_.sin_port = htons(dataport_);
00061 if (inet_aton(ip_.c_str(), &si_data_.sin_addr) == 0)
00062 {
00063 throw art::Exception(art::errors::Configuration) << "UDPReceiver: Could not translate provided IP Address: " << ip_;
00064 exit(1);
00065 }
00066 TLOG_INFO("UDPReceiver") << "UDP Receiver Construction Complete!" << TLOG_ENDL;
00067 }
00068
00069 ots::UDPReceiver::~UDPReceiver()
00070 {
00071 receiverThread_.join();
00072 }
00073
00074 void ots::UDPReceiver::start() {
00075 TLOG_INFO("UDPReceiver") << "Starting..." << TLOG_ENDL;
00076
00077 receiverThread_ = std::thread(&UDPReceiver::receiveLoop_, this);
00078 start_();
00079 }
00080
00081 void ots::UDPReceiver::receiveLoop_()
00082 {
00083 while (!should_stop())
00084 {
00085 struct pollfd ufds[1];
00086 ufds[0].fd = datasocket_;
00087 ufds[0].events = POLLIN | POLLPRI;
00088
00089 int rv = poll(ufds, 1, 1000);
00090 if (rv > 0)
00091 {
00092 TLOG_TRACE("UDPReceiver") << "revents: " << ufds[0].revents << ", " << TLOG_ENDL;
00093 if (ufds[0].revents == POLLIN || ufds[0].revents == POLLPRI)
00094 {
00095
00096
00097
00098
00099 uint8_t peekBuffer[4];
00100 socklen_t dataSz = sizeof(si_data_);
00101 recvfrom(datasocket_, peekBuffer, sizeof(peekBuffer), MSG_PEEK,
00102 (struct sockaddr *) &si_data_, &dataSz);
00103
00104 TLOG_TRACE("UDPReceiver") << "Received UDP Datagram with sequence number " << std::hex << "0x" << static_cast<int>(peekBuffer[1]) << "!" << std::dec << TLOG_ENDL;
00105 TLOG_TRACE("UDPReceiver") << "peekBuffer[1] == expectedPacketNumber_: " << std::hex << static_cast<int>(peekBuffer[1]) << " =?= " << (int)expectedPacketNumber_ << TLOG_ENDL;
00106 TLOG_TRACE("UDPReceiver") << "peekBuffer: 0: " << std::hex << static_cast<int>(peekBuffer[0])
00107 << ", 1: " << std::hex << static_cast<int>(peekBuffer[1])
00108 << ", 2: " << std::hex << static_cast<int>(peekBuffer[2])
00109 << ", 3: " << std::hex << static_cast<int>(peekBuffer[3]) << TLOG_ENDL;
00110
00111 uint8_t seqNum = peekBuffer[1];
00112
00113 if (seqNum >= expectedPacketNumber_ || (seqNum < 64 && expectedPacketNumber_ > 192))
00114 {
00115 if (seqNum != expectedPacketNumber_) {
00116 int delta = seqNum - expectedPacketNumber_;
00117 TLOG_WARNING("UDPReceiver") << std::dec << "Sequence Number different than expected! (delta: " << delta << ")" << TLOG_ENDL;
00118 expectedPacketNumber_ = seqNum;
00119 }
00120
00121 packetBuffer_t receiveBuffer;
00122 receiveBuffer.resize(1500);
00123 int sts = recvfrom(datasocket_, &receiveBuffer[0], receiveBuffer.size(), 0, (struct sockaddr *) &si_data_, &dataSz);
00124 if (sts == -1)
00125 {
00126 TLOG_WARNING("UDPReceiver") << "Error on socket: " << strerror(errno) << TLOG_ENDL;
00127 }
00128 else
00129 {
00130 TLOG_TRACE("UDPReceiver") << "Received " << sts << " bytes." << TLOG_ENDL;
00131 }
00132
00133 std::unique_lock<std::mutex> lock(receiveBufferLock_);
00134 TLOG_TRACE("UDPReceiver") << "Now placing UDP datagram with sequence number " << std::hex << (int)seqNum << " into buffer." << std::dec << TLOG_ENDL;
00135 receiveBuffers_.push_back(receiveBuffer);
00136
00137 ++expectedPacketNumber_;
00138 }
00139 else {
00140
00141 TLOG_WARNING("UDPReceiver") << "Received out-of-order datagram: " << seqNum << " != " << expectedPacketNumber_ << " (expected)" << TLOG_ENDL;
00142 packetBuffer_t receiveBuffer;
00143 receiveBuffer.resize(1500);
00144 int sts = recvfrom(datasocket_, &receiveBuffer[0], receiveBuffer.size(), 0, (struct sockaddr *) &si_data_, &dataSz);
00145 }
00146
00147 }
00148 }
00149 }
00150 TLOG_INFO("UDPReceiver") << "receive Loop exiting..." << TLOG_ENDL;
00151 }
00152
00153 bool ots::UDPReceiver::getNext_(artdaq::FragmentPtrs & output)
00154 {
00155 if (should_stop()) {
00156 return false;
00157 }
00158
00159 {
00160 std::unique_lock<std::mutex> lock(receiveBufferLock_);
00161 std::move(receiveBuffers_.begin(), receiveBuffers_.end(), std::inserter(packetBuffers_, packetBuffers_.end()));
00162 receiveBuffers_.clear();
00163 }
00164
00165 if (packetBuffers_.size() > 0) {
00166 size_t packetBufferSize = 0;
00167 for (auto& buf : packetBuffers_) {
00168 packetBufferSize += buf.size();
00169 }
00170 TLOG_TRACE("UDPReceiver") << "Calling ProcessData, packetBuffers_.size() == " << std::to_string(packetBuffers_.size()) << ", sz = " << std::to_string(packetBufferSize) << TLOG_ENDL;
00171 ProcessData_(output);
00172
00173 packetBuffers_.clear();
00174 TLOG_TRACE("UDPReceiver") << "Returning output of size " << output.size() << TLOG_ENDL;
00175 }
00176 else {
00177
00178 usleep(100000);
00179 }
00180 return true;
00181 }
00182
00183 void ots::UDPReceiver::ProcessData_(artdaq::FragmentPtrs & output) {
00184
00185 ots::UDPFragment::Metadata metadata;
00186 metadata.port = dataport_;
00187 metadata.address = si_data_.sin_addr.s_addr;
00188
00189 std::size_t initial_payload_size = 0;
00190
00191 output.emplace_back(artdaq::Fragment::FragmentBytes(initial_payload_size,
00192 ev_counter(), fragment_id(),
00193 ots::detail::FragmentType::UDP, metadata));
00194
00195 ots::UDPFragmentWriter thisFrag(*output.back());
00196
00197 TLOG_TRACE("UDPReceiver") << "Received data, now placing data with UDP sequence number "
00198 << std::hex << static_cast<int>((packetBuffers_.front()).at(1))
00199 << " into UDPFragment" << TLOG_ENDL;
00200 thisFrag.resize(64050 * packetBuffers_.size() + 1);
00201 std::ofstream rawOutput;
00202 if (rawOutput_) {
00203 std::string outputPath = rawPath_ + "/UDPReceiver-" + ip_ + ":" + std::to_string(dataport_) + ".bin";
00204 rawOutput.open(outputPath, std::ios::out | std::ios::app | std::ios::binary);
00205 }
00206
00207 DataType dataType = getDataType((packetBuffers_.front()).at(0));
00208 thisFrag.set_hdr_type((int)dataType);
00209 int pos = 0;
00210 for (auto jj = packetBuffers_.begin(); jj != packetBuffers_.end(); ++jj) {
00211 for (int ii = 0; ii < 64050; ++ii) {
00212
00213 if ((jj)->at(ii) == 0 && (dataType == DataType::JSON || dataType == DataType::String)) { break; }
00214
00215 if (rawOutput_) rawOutput.write((char*)&((jj)->at(ii)), sizeof(uint8_t));
00216 *(thisFrag.dataBegin() + pos) = (jj)->at(ii);
00217 ++pos;
00218 }
00219 }
00220
00221 if (dataType == DataType::JSON || dataType == DataType::String) {
00222 *(thisFrag.dataBegin() + pos) = 0;
00223 char zero = 0;
00224 if (rawOutput_) rawOutput.write(&zero, sizeof(char));
00225 }
00226 if (rawOutput_) rawOutput.close();
00227 }
00228
00229 void ots::UDPReceiver::send(CommandType command)
00230 {
00231 if (sendCommands_) {
00232 CommandPacket packet;
00233 packet.type = command;
00234 packet.dataSize = 0;
00235 sendto(datasocket_, &packet, sizeof(packet), 0, (struct sockaddr *) &si_data_, sizeof(si_data_));
00236 }
00237 }
00238
00239 bool ots::UDPReceiver::isTimerExpired_()
00240 {
00241 auto now = std::chrono::high_resolution_clock::now();
00242 auto diff = std::chrono::duration<double, std::milli>(now - lastFrag_).count();
00243 return diff > fragmentWindow_;
00244 }
00245
00246 void ots::UDPReceiver::stop()
00247 {
00248 #pragma message "Using default implementation of UDPReceiver::stop()"
00249 }
00250
00251 void ots::UDPReceiver::stopNoMutex()
00252 {
00253 #pragma message "Using default implementation of UDPReceiver::stopNoMutex()"
00254 }
00255
00256 void ots::UDPReceiver::start_()
00257 {
00258 #pragma message "Using default implementation of UDPReceiver::start_()"
00259 }
00260
00261
00262 DEFINE_ARTDAQ_COMMANDABLE_GENERATOR(ots::UDPReceiver)