$treeview $search $mathjax $extrastylesheet
otsdaq
v2_03_00
$projectbrief
|
$projectbrief
|
$searchbox |
00001 #include "artdaq-ots/Generators/UDPReceiver.hh" 00002 00003 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh" 00004 #include "artdaq-ots/Overlays/FragmentType.hh" 00005 #include "artdaq-ots/Overlays/UDPFragmentWriter.hh" 00006 #include "artdaq/Application/GeneratorMacros.hh" 00007 #include "canvas/Utilities/Exception.h" 00008 #include "fhiclcpp/ParameterSet.h" 00009 #include "otsdaq-core/Macros/CoutMacros.h" 00010 00011 #include <sys/poll.h> 00012 #include <fstream> 00013 #include <iomanip> 00014 #include <iostream> 00015 #include <iterator> 00016 00017 00018 //======================================================================================================================== 00019 ots::UDPReceiver::UDPReceiver(fhicl::ParameterSet const& ps) 00020 : CommandableFragmentGenerator(ps) 00021 , rawOutput_(ps.get<bool>("raw_output_enabled", false)) 00022 , rawPath_(ps.get<std::string>("raw_output_path", "/tmp")) 00023 , dataport_(ps.get<int>("port", 6343)) 00024 , ip_(ps.get<std::string>("ip", "127.0.0.1")) 00025 , rcvbuf_(ps.get<int>("rcvbuf", 0x1000000)) 00026 , expectedPacketNumber_(0) 00027 , sendCommands_(ps.get<bool>("send_OtsUDP_commands", false)) 00028 , fragmentWindow_(ps.get<double>("fragment_time_window_ms", 1000)) 00029 , lastFrag_(std::chrono::high_resolution_clock::now()) 00030 { 00031 __COUT__ << "Constructor." << __E__; 00032 00033 datasocket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP); 00034 if(!datasocket_) 00035 { 00036 throw art::Exception(art::errors::Configuration) 00037 << "UDPReceiver: Error creating socket!"; 00038 exit(1); 00039 } 00040 00041 struct sockaddr_in si_me_data; 00042 si_me_data.sin_family = AF_INET; 00043 si_me_data.sin_port = htons(dataport_); 00044 si_me_data.sin_addr.s_addr = htonl(INADDR_ANY); 00045 if(bind(datasocket_, (struct sockaddr*)&si_me_data, sizeof(si_me_data)) == -1) 00046 { 00047 throw art::Exception(art::errors::Configuration) 00048 << "UDPReceiver: Cannot bind data socket to port " << dataport_; 00049 exit(1); 00050 } 00051 /*if(fcntl(datasocket_, F_SETFL, O_NONBLOCK) == -1) { 00052 00053 throw art::Exception(art::errors::Configuration) << "UDPReceiver: Cannot set 00054 socket to nonblocking!" << TLOG_ENDL; 00055 }*/ 00056 00057 if(rcvbuf_ > 0 && 00058 setsockopt(datasocket_, SOL_SOCKET, SO_RCVBUF, &rcvbuf_, sizeof(rcvbuf_))) 00059 { 00060 throw art::Exception(art::errors::Configuration) 00061 << "UDPReceiver: Could not set receive buffer size: " << rcvbuf_; 00062 exit(1); 00063 } 00064 00065 si_data_.sin_family = AF_INET; 00066 si_data_.sin_port = htons(dataport_); 00067 if(inet_aton(ip_.c_str(), &si_data_.sin_addr) == 0) 00068 { 00069 throw art::Exception(art::errors::Configuration) 00070 << "UDPReceiver: Could not translate provided IP Address: " << ip_; 00071 exit(1); 00072 } 00073 TLOG_INFO("UDPReceiver") << "UDP Receiver Construction Complete!" << TLOG_ENDL; 00074 00075 __COUT__ << "Constructed." << __E__; 00076 } //end constructor() 00077 00078 //======================================================================================================================== 00079 ots::UDPReceiver::~UDPReceiver() 00080 { 00081 __COUT__ << "Destructor." << __E__; 00082 00083 //join waits for thread to complete 00084 if(receiverThread_.joinable()) //only join if thread has started 00085 receiverThread_.join(); 00086 00087 if(datasocket_) 00088 { 00089 close(datasocket_); 00090 datasocket_ = -1; 00091 } 00092 00093 __COUT__ << "Destructed." << __E__; 00094 } //end destructor() 00095 00096 //======================================================================================================================== 00097 void ots::UDPReceiver::start() 00098 { 00099 __COUT__ << "Starting..." << __E__; 00100 00101 TLOG_INFO("UDPReceiver") << "Starting..." << TLOG_ENDL; 00102 00103 receiverThread_ = std::thread(&UDPReceiver::receiveLoop_, this); 00104 start_(); 00105 00106 __COUT__ << "Started." << __E__; 00107 } //end start() 00108 00109 //======================================================================================================================== 00110 void ots::UDPReceiver::receiveLoop_() 00111 { 00112 while(!should_stop()) 00113 { 00114 struct pollfd ufds[1]; 00115 ufds[0].fd = datasocket_; 00116 ufds[0].events = POLLIN | POLLPRI; 00117 00118 int rv = poll(ufds, 1, 1000); 00119 if(rv > 0) 00120 { 00121 TLOG_TRACE("UDPReceiver") << "revents: " << ufds[0].revents << ", " 00122 << TLOG_ENDL; // ufds[1].revents << TLOG_ENDL; 00123 if(ufds[0].revents == POLLIN || ufds[0].revents == POLLPRI) 00124 { 00125 // FIXME -> IN THE STIB GENERATOR WE DON'T HAVE A HEADER 00126 // FIXME -> IN THE STIB GENERATOR WE DON'T HAVE A HEADER 00127 // FIXME -> IN THE STIB GENERATOR WE DON'T HAVE A HEADER 00128 uint8_t peekBuffer[4]; 00129 socklen_t dataSz = sizeof(si_data_); 00130 recvfrom(datasocket_, 00131 peekBuffer, 00132 sizeof(peekBuffer), 00133 MSG_PEEK, 00134 (struct sockaddr*)&si_data_, 00135 &dataSz); 00136 00137 TLOG_TRACE("UDPReceiver") 00138 << "Received UDP Datagram with sequence number " << std::hex << "0x" 00139 << static_cast<int>(peekBuffer[1]) << "!" << std::dec << TLOG_ENDL; 00140 TLOG_TRACE("UDPReceiver") 00141 << "peekBuffer[1] == expectedPacketNumber_: " << std::hex 00142 << static_cast<int>(peekBuffer[1]) 00143 << " =?= " << (int)expectedPacketNumber_ << TLOG_ENDL; 00144 TLOG_TRACE("UDPReceiver") 00145 << "peekBuffer: 0: " << std::hex << static_cast<int>(peekBuffer[0]) 00146 << ", 1: " << std::hex << static_cast<int>(peekBuffer[1]) 00147 << ", 2: " << std::hex << static_cast<int>(peekBuffer[2]) 00148 << ", 3: " << std::hex << static_cast<int>(peekBuffer[3]) 00149 << TLOG_ENDL; 00150 00151 uint8_t seqNum = peekBuffer[1]; 00152 // ReturnCode dataCode = getReturnCode(peekBuffer[0]); 00153 if(seqNum >= expectedPacketNumber_ || 00154 (seqNum < 64 && expectedPacketNumber_ > 192)) 00155 { 00156 if(seqNum != expectedPacketNumber_) 00157 { 00158 int delta = seqNum - expectedPacketNumber_; 00159 TLOG_WARNING("UDPReceiver") 00160 << std::dec 00161 << "Sequence Number different than expected! (delta: " 00162 << delta << ")" << TLOG_ENDL; 00163 expectedPacketNumber_ = seqNum; 00164 } 00165 00166 packetBuffer_t receiveBuffer; 00167 receiveBuffer.resize(1500); 00168 int sts = recvfrom(datasocket_, 00169 &receiveBuffer[0], 00170 receiveBuffer.size(), 00171 0, 00172 (struct sockaddr*)&si_data_, 00173 &dataSz); 00174 if(sts == -1) 00175 { 00176 TLOG_WARNING("UDPReceiver") 00177 << "Error on socket: " << strerror(errno) << TLOG_ENDL; 00178 } 00179 else 00180 { 00181 TLOG_TRACE("UDPReceiver") 00182 << "Received " << sts << " bytes." << TLOG_ENDL; 00183 } 00184 00185 std::unique_lock<std::mutex> lock(receiveBufferLock_); 00186 TLOG_TRACE("UDPReceiver") 00187 << "Now placing UDP datagram with sequence number " << std::hex 00188 << (int)seqNum << " into buffer." << std::dec << TLOG_ENDL; 00189 receiveBuffers_.push_back(receiveBuffer); 00190 00191 ++expectedPacketNumber_; 00192 } 00193 else 00194 { 00195 // Receiving out-of-order datagram, then moving on... 00196 TLOG_WARNING("UDPReceiver") 00197 << "Received out-of-order datagram: " << seqNum 00198 << " != " << expectedPacketNumber_ << " (expected)" << TLOG_ENDL; 00199 packetBuffer_t receiveBuffer; 00200 receiveBuffer.resize(1500); 00201 int sts = recvfrom(datasocket_, 00202 &receiveBuffer[0], 00203 receiveBuffer.size(), 00204 0, 00205 (struct sockaddr*)&si_data_, 00206 &dataSz); 00207 } 00208 } 00209 } 00210 } 00211 TLOG_INFO("UDPReceiver") << "receive Loop exiting..." << TLOG_ENDL; 00212 } 00213 00214 //======================================================================================================================== 00215 bool ots::UDPReceiver::getNext_(artdaq::FragmentPtrs& output) 00216 { 00217 if(should_stop()) 00218 { 00219 return false; 00220 } 00221 00222 { 00223 std::unique_lock<std::mutex> lock(receiveBufferLock_); 00224 std::move(receiveBuffers_.begin(), 00225 receiveBuffers_.end(), 00226 std::inserter(packetBuffers_, packetBuffers_.end())); 00227 receiveBuffers_.clear(); 00228 } 00229 00230 if(packetBuffers_.size() > 0) 00231 { 00232 size_t packetBufferSize = 0; 00233 for(auto& buf : packetBuffers_) 00234 { 00235 packetBufferSize += buf.size(); 00236 } 00237 TLOG_TRACE("UDPReceiver") 00238 << "Calling ProcessData, packetBuffers_.size() == " 00239 << std::to_string(packetBuffers_.size()) 00240 << ", sz = " << std::to_string(packetBufferSize) << TLOG_ENDL; 00241 ProcessData_(output); 00242 00243 packetBuffers_.clear(); 00244 TLOG_TRACE("UDPReceiver") 00245 << "Returning output of size " << output.size() << TLOG_ENDL; 00246 } 00247 else 00248 { 00249 // Sleep 10 times per poll timeout 00250 usleep(100000); 00251 } 00252 return true; 00253 } 00254 00255 //======================================================================================================================== 00256 void ots::UDPReceiver::ProcessData_(artdaq::FragmentPtrs& output) 00257 { 00258 ots::UDPFragment::Metadata metadata; 00259 metadata.port = dataport_; 00260 metadata.address = si_data_.sin_addr.s_addr; 00261 00262 std::size_t initial_payload_size = 0; 00263 00264 output.emplace_back(artdaq::Fragment::FragmentBytes(initial_payload_size, 00265 ev_counter(), 00266 fragment_id(), 00267 ots::detail::FragmentType::UDP, 00268 metadata)); 00269 // We now have a fragment to contain this event: 00270 ots::UDPFragmentWriter thisFrag(*output.back()); 00271 00272 TLOG_TRACE("UDPReceiver") 00273 << "Received data, now placing data with UDP sequence number " << std::hex 00274 << static_cast<int>((packetBuffers_.front()).at(1)) << " into UDPFragment" 00275 << TLOG_ENDL; 00276 thisFrag.resize(64050 * packetBuffers_.size() + 1); 00277 std::ofstream rawOutput; 00278 if(rawOutput_) 00279 { 00280 std::string outputPath = 00281 rawPath_ + "/UDPReceiver-" + ip_ + ":" + std::to_string(dataport_) + ".bin"; 00282 rawOutput.open(outputPath, std::ios::out | std::ios::app | std::ios::binary); 00283 } 00284 00285 DataType dataType = getDataType((packetBuffers_.front()).at(0)); 00286 thisFrag.set_hdr_type((int)dataType); 00287 int pos = 0; 00288 for(auto jj = packetBuffers_.begin(); jj != packetBuffers_.end(); ++jj) 00289 { 00290 for(int ii = 0; ii < 64050; ++ii) 00291 { 00292 // Null-terminate string types 00293 if((jj)->at(ii) == 0 && 00294 (dataType == DataType::JSON || dataType == DataType::String)) 00295 { 00296 break; 00297 } 00298 00299 if(rawOutput_) 00300 rawOutput.write((char*)&((jj)->at(ii)), sizeof(uint8_t)); 00301 *(thisFrag.dataBegin() + pos) = (jj)->at(ii); 00302 ++pos; 00303 } 00304 } 00305 00306 if(dataType == DataType::JSON || dataType == DataType::String) 00307 { 00308 *(thisFrag.dataBegin() + pos) = 0; 00309 char zero = 0; 00310 if(rawOutput_) 00311 rawOutput.write(&zero, sizeof(char)); 00312 } 00313 if(rawOutput_) 00314 rawOutput.close(); 00315 } 00316 00317 //======================================================================================================================== 00318 void ots::UDPReceiver::send(CommandType command) 00319 { 00320 if(sendCommands_) 00321 { 00322 CommandPacket packet; 00323 packet.type = command; 00324 packet.dataSize = 0; 00325 sendto(datasocket_, 00326 &packet, 00327 sizeof(packet), 00328 0, 00329 (struct sockaddr*)&si_data_, 00330 sizeof(si_data_)); 00331 } 00332 } 00333 00334 //======================================================================================================================== 00335 bool ots::UDPReceiver::isTimerExpired_() 00336 { 00337 auto now = std::chrono::high_resolution_clock::now(); 00338 auto diff = std::chrono::duration<double, std::milli>(now - lastFrag_).count(); 00339 return diff > fragmentWindow_; 00340 } 00341 00342 //======================================================================================================================== 00343 void ots::UDPReceiver::stop() 00344 { 00345 __COUT__ << "Stopping..." << __E__; 00346 //#pragma message "Using default implementation of UDPReceiver::stop()" 00347 } 00348 00349 //======================================================================================================================== 00350 void ots::UDPReceiver::stopNoMutex() 00351 { 00352 //#pragma message "Using default implementation of UDPReceiver::stopNoMutex()" 00353 } 00354 00355 //======================================================================================================================== 00356 void ots::UDPReceiver::start_() 00357 { 00358 //#pragma message "Using default implementation of UDPReceiver::start_()" 00359 } 00360 00361 // The following macro is defined in artdaq's GeneratorMacros.hh header 00362 DEFINE_ARTDAQ_COMMANDABLE_GENERATOR(ots::UDPReceiver)