00001 #include "artdaq-ots/Generators/UDPReceiver.hh"
00002
00003 #include "canvas/Utilities/Exception.h"
00004 #include "artdaq/Application/GeneratorMacros.hh"
00005 #include "cetlib/exception.h"
00006 #include "fhiclcpp/ParameterSet.h"
00007 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh"
00008 #include "artdaq-ots/Overlays/UDPFragmentWriter.hh"
00009 #include "artdaq-ots/Overlays/FragmentType.hh"
00010 #include "otsdaq-core/Macros/CoutMacros.h"
00011
00012
00013 #include <fstream>
00014 #include <iomanip>
00015 #include <iterator>
00016 #include <iostream>
00017 #include <sys/poll.h>
00018
00019 ots::UDPReceiver::UDPReceiver(fhicl::ParameterSet const & ps)
00020 : CommandableFragmentGenerator(ps)
00021 , rawOutput_(ps.get<bool>("raw_output_enabled", false))
00022 , rawPath_(ps.get<std::string>("raw_output_path", "/tmp"))
00023 , dataport_(ps.get<int>("port", 6343))
00024 , ip_(ps.get<std::string>("ip", "127.0.0.1"))
00025 , rcvbuf_(ps.get<int>("rcvbuf", 0x1000000))
00026 , expectedPacketNumber_(0)
00027 , sendCommands_(ps.get<bool>("send_OtsUDP_commands", false))
00028 , fragmentWindow_(ps.get<double>("fragment_time_window_ms", 1000))
00029 , lastFrag_(std::chrono::high_resolution_clock::now())
00030 {
00031 datasocket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
00032 if (!datasocket_)
00033 {
00034 throw art::Exception(art::errors::Configuration) << "UDPReceiver: Error creating socket!";
00035 exit(1);
00036 }
00037
00038 struct sockaddr_in si_me_data;
00039 si_me_data.sin_family = AF_INET;
00040 si_me_data.sin_port = htons(dataport_);
00041 si_me_data.sin_addr.s_addr = htonl(INADDR_ANY);
00042 if (bind(datasocket_, (struct sockaddr *)&si_me_data, sizeof(si_me_data)) == -1)
00043 {
00044 throw art::Exception(art::errors::Configuration) << "UDPReceiver: Cannot bind data socket to port " << dataport_;
00045 exit(1);
00046 }
00047
00048
00049
00050
00051
00052 if (rcvbuf_ > 0 && setsockopt(datasocket_, SOL_SOCKET, SO_RCVBUF, &rcvbuf_, sizeof(rcvbuf_)))
00053 {
00054 throw art::Exception(art::errors::Configuration) << "UDPReceiver: Could not set receive buffer size: " << rcvbuf_;
00055 exit(1);
00056 }
00057
00058 si_data_.sin_family = AF_INET;
00059 si_data_.sin_port = htons(dataport_);
00060 if (inet_aton(ip_.c_str(), &si_data_.sin_addr) == 0)
00061 {
00062 throw art::Exception(art::errors::Configuration) << "UDPReceiver: Could not translate provided IP Address: " << ip_;
00063 exit(1);
00064 }
00065 TLOG_INFO("UDPReceiver") << "UDP Receiver Construction Complete!" << TLOG_ENDL;
00066 }
00067
00068 ots::UDPReceiver::~UDPReceiver()
00069 {
00070 receiverThread_.join();
00071 }
00072
00073 void ots::UDPReceiver::start() {
00074 TLOG_INFO("UDPReceiver") << "Starting..." << TLOG_ENDL;
00075
00076 receiverThread_ = std::thread(&UDPReceiver::receiveLoop_, this);
00077 start_();
00078 }
00079
00080 void ots::UDPReceiver::receiveLoop_()
00081 {
00082 while (!should_stop())
00083 {
00084 struct pollfd ufds[1];
00085 ufds[0].fd = datasocket_;
00086 ufds[0].events = POLLIN | POLLPRI;
00087
00088 int rv = poll(ufds, 1, 1000);
00089 if (rv > 0)
00090 {
00091 TLOG_TRACE("UDPReceiver") << "revents: " << ufds[0].revents << ", " << TLOG_ENDL;
00092 if (ufds[0].revents == POLLIN || ufds[0].revents == POLLPRI)
00093 {
00094
00095
00096
00097
00098 uint8_t peekBuffer[4];
00099 socklen_t dataSz = sizeof(si_data_);
00100 recvfrom(datasocket_, peekBuffer, sizeof(peekBuffer), MSG_PEEK,
00101 (struct sockaddr *) &si_data_, &dataSz);
00102
00103 TLOG_TRACE("UDPReceiver") << "Received UDP Datagram with sequence number " << std::hex << "0x" << static_cast<int>(peekBuffer[1]) << "!" << std::dec << TLOG_ENDL;
00104 TLOG_TRACE("UDPReceiver") << "peekBuffer[1] == expectedPacketNumber_: " << std::hex << static_cast<int>(peekBuffer[1]) << " =?= " << (int)expectedPacketNumber_ << TLOG_ENDL;
00105 TLOG_TRACE("UDPReceiver") << "peekBuffer: 0: " << std::hex << static_cast<int>(peekBuffer[0])
00106 << ", 1: " << std::hex << static_cast<int>(peekBuffer[1])
00107 << ", 2: " << std::hex << static_cast<int>(peekBuffer[2])
00108 << ", 3: " << std::hex << static_cast<int>(peekBuffer[3]) << TLOG_ENDL;
00109
00110 uint8_t seqNum = peekBuffer[1];
00111
00112 if (seqNum >= expectedPacketNumber_ || (seqNum < 64 && expectedPacketNumber_ > 192))
00113 {
00114 if (seqNum != expectedPacketNumber_) {
00115 int delta = seqNum - expectedPacketNumber_;
00116 TLOG_WARNING("UDPReceiver") << std::dec << "Sequence Number different than expected! (delta: " << delta << ")" << TLOG_ENDL;
00117 expectedPacketNumber_ = seqNum;
00118 }
00119
00120 packetBuffer_t receiveBuffer;
00121 receiveBuffer.resize(1500);
00122 int sts = recvfrom(datasocket_, &receiveBuffer[0], receiveBuffer.size(), 0, (struct sockaddr *) &si_data_, &dataSz);
00123 if (sts == -1)
00124 {
00125 TLOG_WARNING("UDPReceiver") << "Error on socket: " << strerror(errno) << TLOG_ENDL;
00126 }
00127 else
00128 {
00129 TLOG_TRACE("UDPReceiver") << "Received " << sts << " bytes." << TLOG_ENDL;
00130 }
00131
00132 std::unique_lock<std::mutex> lock(receiveBufferLock_);
00133 TLOG_TRACE("UDPReceiver") << "Now placing UDP datagram with sequence number " << std::hex << (int)seqNum << " into buffer." << std::dec << TLOG_ENDL;
00134 receiveBuffers_.push_back(receiveBuffer);
00135
00136 ++expectedPacketNumber_;
00137 }
00138 else {
00139
00140 TLOG_WARNING("UDPReceiver") << "Received out-of-order datagram: " << seqNum << " != " << expectedPacketNumber_ << " (expected)" << TLOG_ENDL;
00141 packetBuffer_t receiveBuffer;
00142 receiveBuffer.resize(1500);
00143 int sts = recvfrom(datasocket_, &receiveBuffer[0], receiveBuffer.size(), 0, (struct sockaddr *) &si_data_, &dataSz);
00144 }
00145
00146 }
00147 }
00148 }
00149 TLOG_INFO("UDPReceiver") << "receive Loop exiting..." << TLOG_ENDL;
00150 }
00151
00152 bool ots::UDPReceiver::getNext_(artdaq::FragmentPtrs & output)
00153 {
00154 if (should_stop()) {
00155 return false;
00156 }
00157
00158 {
00159 std::unique_lock<std::mutex> lock(receiveBufferLock_);
00160 std::move(receiveBuffers_.begin(), receiveBuffers_.end(), std::inserter(packetBuffers_, packetBuffers_.end()));
00161 receiveBuffers_.clear();
00162 }
00163
00164 if (packetBuffers_.size() > 0) {
00165 size_t packetBufferSize = 0;
00166 for (auto& buf : packetBuffers_) {
00167 packetBufferSize += buf.size();
00168 }
00169 TLOG_TRACE("UDPReceiver") << "Calling ProcessData, packetBuffers_.size() == " << std::to_string(packetBuffers_.size()) << ", sz = " << std::to_string(packetBufferSize) << TLOG_ENDL;
00170 ProcessData_(output);
00171
00172 packetBuffers_.clear();
00173 TLOG_TRACE("UDPReceiver") << "Returning output of size " << output.size() << TLOG_ENDL;
00174 }
00175 else {
00176
00177 usleep(100000);
00178 }
00179 return true;
00180 }
00181
00182 void ots::UDPReceiver::ProcessData_(artdaq::FragmentPtrs & output) {
00183
00184 ots::UDPFragment::Metadata metadata;
00185 metadata.port = dataport_;
00186 metadata.address = si_data_.sin_addr.s_addr;
00187
00188 std::size_t initial_payload_size = 0;
00189
00190 output.emplace_back(artdaq::Fragment::FragmentBytes(initial_payload_size,
00191 ev_counter(), fragment_id(),
00192 ots::detail::FragmentType::UDP, metadata));
00193
00194 ots::UDPFragmentWriter thisFrag(*output.back());
00195
00196 TLOG_TRACE("UDPReceiver") << "Received data, now placing data with UDP sequence number "
00197 << std::hex << static_cast<int>((packetBuffers_.front()).at(1))
00198 << " into UDPFragment" << TLOG_ENDL;
00199 thisFrag.resize(64050 * packetBuffers_.size() + 1);
00200 std::ofstream rawOutput;
00201 if (rawOutput_) {
00202 std::string outputPath = rawPath_ + "/UDPReceiver-" + ip_ + ":" + std::to_string(dataport_) + ".bin";
00203 rawOutput.open(outputPath, std::ios::out | std::ios::app | std::ios::binary);
00204 }
00205
00206 DataType dataType = getDataType((packetBuffers_.front()).at(0));
00207 thisFrag.set_hdr_type((int)dataType);
00208 int pos = 0;
00209 for (auto jj = packetBuffers_.begin(); jj != packetBuffers_.end(); ++jj) {
00210 for (int ii = 0; ii < 64050; ++ii) {
00211
00212 if ((jj)->at(ii) == 0 && (dataType == DataType::JSON || dataType == DataType::String)) { break; }
00213
00214 if (rawOutput_) rawOutput.write((char*)&((jj)->at(ii)), sizeof(uint8_t));
00215 *(thisFrag.dataBegin() + pos) = (jj)->at(ii);
00216 ++pos;
00217 }
00218 }
00219
00220 if (dataType == DataType::JSON || dataType == DataType::String) {
00221 *(thisFrag.dataBegin() + pos) = 0;
00222 char zero = 0;
00223 if (rawOutput_) rawOutput.write(&zero, sizeof(char));
00224 }
00225 if (rawOutput_) rawOutput.close();
00226 }
00227
00228 void ots::UDPReceiver::send(CommandType command)
00229 {
00230 if (sendCommands_) {
00231 CommandPacket packet;
00232 packet.type = command;
00233 packet.dataSize = 0;
00234 sendto(datasocket_, &packet, sizeof(packet), 0, (struct sockaddr *) &si_data_, sizeof(si_data_));
00235 }
00236 }
00237
00238 bool ots::UDPReceiver::isTimerExpired_()
00239 {
00240 auto now = std::chrono::high_resolution_clock::now();
00241 auto diff = std::chrono::duration<double, std::milli>(now - lastFrag_).count();
00242 return diff > fragmentWindow_;
00243 }
00244
00245 void ots::UDPReceiver::stop()
00246 {
00247 #pragma message "Using default implementation of UDPReceiver::stop()"
00248 }
00249
00250 void ots::UDPReceiver::stopNoMutex()
00251 {
00252 #pragma message "Using default implementation of UDPReceiver::stopNoMutex()"
00253 }
00254
00255 void ots::UDPReceiver::start_()
00256 {
00257 #pragma message "Using default implementation of UDPReceiver::start_()"
00258 }
00259
00260
00261 DEFINE_ARTDAQ_COMMANDABLE_GENERATOR(ots::UDPReceiver)