00001 #include "artdaq-ots/Generators/UDPReceiver.hh"
00002
00003 #include "canvas/Utilities/Exception.h"
00004 #include "artdaq/Application/GeneratorMacros.hh"
00005 #include "cetlib/exception.h"
00006 #include "fhiclcpp/ParameterSet.h"
00007 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh"
00008 #include "artdaq-ots/Overlays/UDPFragmentWriter.hh"
00009 #include "artdaq-ots/Overlays/FragmentType.hh"
00010 #include "otsdaq-core/Macros/CoutHeaderMacros.h"
00011
00012
00013 #include <fstream>
00014 #include <iomanip>
00015 #include <iterator>
00016 #include <iostream>
00017 #include <sys/poll.h>
00018
00019 ots::UDPReceiver::UDPReceiver(fhicl::ParameterSet const & ps)
00020 : CommandableFragmentGenerator(ps)
00021 , rawOutput_(ps.get<bool>("raw_output_enabled",false))
00022 , rawPath_(ps.get<std::string>("raw_output_path", "/tmp"))
00023 , dataport_(ps.get<int>("port",6343))
00024 , ip_(ps.get<std::string>("ip","127.0.0.1"))
00025 , expectedPacketNumber_(0)
00026 , sendCommands_(ps.get<bool>("send_OtsUDP_commands",false))
00027 , fragmentWindow_(ps.get<double>("fragment_time_window_ms", 1000))
00028 , lastFrag_(std::chrono::high_resolution_clock::now())
00029 {
00030 datasocket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
00031 if(!datasocket_)
00032 {
00033 throw art::Exception(art::errors::Configuration) << "UDPReceiver: Error creating socket!" << std::endl;
00034 exit(1);
00035 }
00036
00037 struct sockaddr_in si_me_data;
00038 si_me_data.sin_family = AF_INET;
00039 si_me_data.sin_port = htons(dataport_);
00040 si_me_data.sin_addr.s_addr = htonl(INADDR_ANY);
00041 if(bind(datasocket_, (struct sockaddr *)&si_me_data, sizeof(si_me_data)) == -1)
00042 {
00043 throw art::Exception(art::errors::Configuration) <<
00044 "UDPReceiver: Cannot bind data socket to port " << dataport_ << std::endl;
00045 exit(1);
00046 }
00047
00048 si_data_.sin_family = AF_INET;
00049 si_data_.sin_port = htons(dataport_);
00050 if(inet_aton(ip_.c_str(), &si_data_.sin_addr) == 0)
00051 {
00052 throw art::Exception(art::errors::Configuration) <<
00053 "UDPReceiver: Could not translate provided IP Address: " << ip_ << "\n";
00054 exit(1);
00055 }
00056 mf::LogInfo("UDPReceiver") << "UDP Receiver Construction Complete!" << std::endl;
00057 }
00058
00059 ots::UDPReceiver::~UDPReceiver()
00060 {
00061 receiverThread_.join();
00062 }
00063
00064 void ots::UDPReceiver::start() {
00065 receiverThread_ = std::thread(&UDPReceiver::receiveLoop_,this);
00066 }
00067
00068 void ots::UDPReceiver::receiveLoop_() {
00069
00070 uint8_t droppedPackets = 0;
00071 while(!should_stop()) {
00072 struct pollfd ufds[1];
00073 ufds[0].fd = datasocket_;
00074 ufds[0].events = POLLIN | POLLPRI;
00075
00076 int rv = poll(ufds, 1, 1000);
00077 if(rv > 0)
00078 {
00079
00080 if(ufds[0].revents == POLLIN || ufds[0].revents == POLLPRI)
00081 {
00082
00083
00084
00085
00086 uint32_t peekBuffer[6];
00087 recvfrom(datasocket_, peekBuffer, sizeof(peekBuffer), MSG_PEEK,
00088 (struct sockaddr *) &si_data_, (socklen_t*)sizeof(si_data_));
00089
00090 mf::LogInfo("UDPReceiver") << "Received UDP Packet with sequence number " << std::hex << "0x" << (int)peekBuffer[1] << "!" << std::dec;
00091
00092
00093 uint32_t seqNum = peekBuffer[5];
00094
00095 if(seqNum >= expectedPacketNumber_ || (seqNum < 10 && expectedPacketNumber_ > 240) || droppedPackets > 0 || expectedPacketNumber_ - seqNum > 20)
00096 {
00097 if(seqNum != expectedPacketNumber_ && (seqNum >= expectedPacketNumber_ || (seqNum < 10 && expectedPacketNumber_ > 200)))
00098 {
00099 int deltaHi = seqNum - expectedPacketNumber_;
00100 int deltaLo = 4294967295 + seqNum - expectedPacketNumber_;
00101 droppedPackets += deltaLo;
00102 mf::LogWarning("UDPReceiver") << "Dropped/Delayed packets detected: " << std::dec << std::to_string(droppedPackets);
00103 expectedPacketNumber_ = seqNum;
00104 } else if (seqNum != expectedPacketNumber_) {
00105 int delta = expectedPacketNumber_ - seqNum;
00106 mf::LogWarning("UDPReceiver") << std::dec << "Sequence Number significantly different than expected! (delta: " << delta << ")";
00107 }
00108
00109 packetBuffer_t* buffer = new packetBuffer_t();
00110 memset(&((*buffer).at(0)),0,sizeof(packetBuffer_t));
00111 int DJN= recvfrom(datasocket_, &(*buffer).at(0), sizeof(packetBuffer_t), 0,(struct sockaddr *) &si_data_, (socklen_t*)sizeof(si_data_));
00112 mf::LogInfo("DJN UDPReceiver") << "UDP packet recvfrom got Nbytes=" << (int)DJN << " into buffer.";
00113
00114 if(droppedPackets == 0) {
00115 std::lock_guard<std::mutex> lock(receiveBufferLock_);
00116 receiveBuffers_.emplace_back(std::unique_ptr<packetBuffer_t>(buffer));
00117 }
00118 else {
00119 bool found = false;
00120 for(packetBuffer_list_t::reverse_iterator it = packetBuffers_.rbegin(); it != packetBuffers_.rend(); ++it) {
00121 if(seqNum < (*it)->at(1)) {
00122 std::lock_guard<std::mutex> lock(receiveBufferLock_);
00123 receiveBuffers_.emplace(it.base(), std::unique_ptr<packetBuffer_t>(buffer));
00124 droppedPackets--;
00125 expectedPacketNumber_--;
00126 found = true;
00127 }
00128 }
00129 if(!found) {
00130 std::lock_guard<std::mutex> lock(receiveBufferLock_);
00131 receiveBuffers_.emplace_back(std::unique_ptr<packetBuffer_t>(buffer));
00132 }
00133 }
00134 mf::LogInfo("UDPReceiver") << "Now placing UDP packet with sequence number " << std::hex << (int)seqNum << " into buffer." << std::dec;
00135 }
00136
00137 ++expectedPacketNumber_;
00138 }
00139 }
00140 }
00141 }
00142
00143 bool ots::UDPReceiver::getNext_(artdaq::FragmentPtrs & output)
00144 {
00145 if (should_stop()) {
00146 return false;
00147 }
00148
00149 while(receiveBuffers_.size() == 0 && isTimerExpired_()) { usleep(1000); }
00150
00151 {
00152 std::lock_guard<std::mutex> lock(receiveBufferLock_);
00153 std::move(receiveBuffers_.begin(), receiveBuffers_.end(), std::inserter(packetBuffers_,packetBuffers_.end()));
00154 receiveBuffers_.clear();
00155 }
00156 mf::LogInfo("UDPReceiver") << "Calling ProcessData";
00157 ProcessData_(output);
00158
00159 mf::LogInfo("UDPReceiver") << "Returning output of size " << output.size() << " to TriggeredFragmentGenerator";
00160 return true;
00161 }
00162
00163 void ots::UDPReceiver::ProcessData_(artdaq::FragmentPtrs & output) {
00164
00165 ots::UDPFragment::Metadata metadata;
00166 metadata.port = dataport_;
00167 metadata.address = si_data_.sin_addr.s_addr;
00168
00169 std::size_t initial_payload_size = 0;
00170
00171 output.emplace_back( artdaq::Fragment::FragmentBytes(initial_payload_size,
00172 ev_counter(), fragment_id(),
00173 ots::detail::FragmentType::UDP, metadata) );
00174
00175 ots::UDPFragmentWriter thisFrag(*output.back());
00176
00177 std::cout << __COUT_HDR_FL__ << "Received data, now placing data with UDP sequence number "
00178 << std::hex << static_cast<int>((*packetBuffers_.front()).at(1))
00179 << " into UDPFragment" << std::endl;
00180 thisFrag.resize(64050 * packetBuffers_.size() + 1);
00181 std::ofstream rawOutput;
00182 if(rawOutput_) {
00183 std::string outputPath = rawPath_ + "/UDPReceiver-"+ ip_ + ":" + std::to_string(dataport_) + ".bin";
00184 rawOutput.open(outputPath, std::ios::out | std::ios::app | std::ios::binary );
00185 }
00186
00187 DataType dataType = getDataType((*packetBuffers_.front()).at(0));
00188 thisFrag.set_hdr_type((int)dataType);
00189 int pos = 0;
00190 for(auto jj = packetBuffers_.begin(); jj != packetBuffers_.end(); ++jj) {
00191 for(int ii = 0; ii < 64050; ++ii) {
00192
00193 if((*jj)->at(ii) == 0 && (dataType == DataType::JSON || dataType == DataType::String)) { break; }
00194
00195 if(rawOutput_) rawOutput.write((char*)&((*jj)->at(ii)), sizeof(uint8_t));
00196 *(thisFrag.dataBegin() + pos) = (*jj)->at(ii);
00197 ++pos;
00198 }
00199 }
00200 packetBuffers_.clear();
00201
00202 if(dataType == DataType::JSON || dataType == DataType::String) {
00203 *(thisFrag.dataBegin() + pos) = 0;
00204 char zero =0;
00205 if(rawOutput_) rawOutput.write(&zero, sizeof(char));
00206 }
00207 if(rawOutput_) rawOutput.close();
00208 }
00209
00210 void ots::UDPReceiver::send(CommandType command)
00211 {
00212 if(sendCommands_) {
00213 CommandPacket packet;
00214 packet.type = command;
00215 packet.dataSize = 0;
00216 sendto(datasocket_,&packet,sizeof(packet),0, (struct sockaddr *) &si_data_, sizeof(si_data_));
00217 }
00218 }
00219
00220 bool ots::UDPReceiver::isTimerExpired_()
00221 {
00222 auto now = std::chrono::high_resolution_clock::now();
00223 auto diff = std::chrono::duration<double, std::milli>(now - lastFrag_).count();
00224 return diff > fragmentWindow_;
00225 }
00226
00227 void ots::UDPReceiver::stop()
00228 {
00229 #pragma message "Using default implementation of UDPReceiver::stop()"
00230 }
00231
00232 void ots::UDPReceiver::stopNoMutex()
00233 {
00234 #pragma message "Using default implementation of UDPReceiver::stopNoMutex()"
00235 }
00236
00237
00238 DEFINE_ARTDAQ_COMMANDABLE_GENERATOR(ots::UDPReceiver)