1 #include "artdaq-ots/Generators/UDPReceiver.hh"
3 #include "canvas/Utilities/Exception.h"
4 #include "artdaq/Application/GeneratorMacros.hh"
5 #include "cetlib/exception.h"
6 #include "fhiclcpp/ParameterSet.h"
7 #include "artdaq-core/Utilities/SimpleLookupPolicy.hh"
8 #include "artdaq-ots/Overlays/UDPFragmentWriter.hh"
9 #include "artdaq-ots/Overlays/FragmentType.hh"
10 #include "otsdaq-core/Macros/CoutHeaderMacros.h"
19 ots::UDPReceiver::UDPReceiver(fhicl::ParameterSet
const & ps)
20 : CommandableFragmentGenerator(ps)
21 , rawOutput_(ps.get<bool>(
"raw_output_enabled", false))
22 , rawPath_(ps.get<std::string>(
"raw_output_path",
"/tmp"))
23 , dataport_(ps.get<int>(
"port", 6343))
24 , ip_(ps.get<std::string>(
"ip",
"127.0.0.1"))
25 , rcvbuf_(ps.get<int>(
"rcvbuf", 0x1000000))
26 , expectedPacketNumber_(0)
27 , sendCommands_(ps.get<bool>(
"send_OtsUDP_commands", false))
28 , fragmentWindow_(ps.get<double>(
"fragment_time_window_ms", 1000))
29 , lastFrag_(std::chrono::high_resolution_clock::now())
31 datasocket_ = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
34 throw art::Exception(art::errors::Configuration) <<
"UDPReceiver: Error creating socket!";
38 struct sockaddr_in si_me_data;
39 si_me_data.sin_family = AF_INET;
40 si_me_data.sin_port = htons(dataport_);
41 si_me_data.sin_addr.s_addr = htonl(INADDR_ANY);
42 if (bind(datasocket_, (
struct sockaddr *)&si_me_data,
sizeof(si_me_data)) == -1)
44 throw art::Exception(art::errors::Configuration) <<
"UDPReceiver: Cannot bind data socket to port " << dataport_;
53 if (rcvbuf_ > 0 && setsockopt(datasocket_, SOL_SOCKET, SO_RCVBUF, &rcvbuf_,
sizeof(rcvbuf_)))
55 throw art::Exception(art::errors::Configuration) <<
"UDPReceiver: Could not set receive buffer size: " << rcvbuf_;
59 si_data_.sin_family = AF_INET;
60 si_data_.sin_port = htons(dataport_);
61 if (inet_aton(ip_.c_str(), &si_data_.sin_addr) == 0)
63 throw art::Exception(art::errors::Configuration) <<
"UDPReceiver: Could not translate provided IP Address: " << ip_;
66 TLOG_INFO(
"UDPReceiver") <<
"UDP Receiver Construction Complete!" << TLOG_ENDL;
69 ots::UDPReceiver::~UDPReceiver()
71 receiverThread_.join();
74 void ots::UDPReceiver::start() {
75 TLOG_INFO(
"UDPReceiver") <<
"Starting..." << TLOG_ENDL;
77 receiverThread_ = std::thread(&UDPReceiver::receiveLoop_,
this);
81 void ots::UDPReceiver::receiveLoop_()
83 while (!should_stop())
85 struct pollfd ufds[1];
86 ufds[0].fd = datasocket_;
87 ufds[0].events = POLLIN | POLLPRI;
89 int rv = poll(ufds, 1, 1000);
92 TLOG_TRACE(
"UDPReceiver") <<
"revents: " << ufds[0].revents <<
", " << TLOG_ENDL;
93 if (ufds[0].revents == POLLIN || ufds[0].revents == POLLPRI)
99 uint8_t peekBuffer[4];
100 socklen_t dataSz =
sizeof(si_data_);
101 recvfrom(datasocket_, peekBuffer,
sizeof(peekBuffer), MSG_PEEK,
102 (
struct sockaddr *) &si_data_, &dataSz);
104 TLOG_TRACE(
"UDPReceiver") <<
"Received UDP Datagram with sequence number " << std::hex <<
"0x" <<
static_cast<int>(peekBuffer[1]) <<
"!" << std::dec << TLOG_ENDL;
105 TLOG_TRACE(
"UDPReceiver") <<
"peekBuffer[1] == expectedPacketNumber_: " << std::hex << static_cast<int>(peekBuffer[1]) <<
" =?= " << (
int)expectedPacketNumber_ << TLOG_ENDL;
106 TLOG_TRACE("UDPReceiver") << "peekBuffer: 0: " << std::hex << static_cast<
int>(peekBuffer[0])
107 << ", 1: " << std::hex << static_cast<
int>(peekBuffer[1])
108 << ", 2: " << std::hex << static_cast<
int>(peekBuffer[2])
109 << ", 3: " << std::hex << static_cast<
int>(peekBuffer[3]) << TLOG_ENDL;
111 uint8_t seqNum = peekBuffer[1];
113 if (seqNum >= expectedPacketNumber_ || (seqNum < 64 && expectedPacketNumber_ > 192))
115 if (seqNum != expectedPacketNumber_) {
116 int delta = seqNum - expectedPacketNumber_;
117 TLOG_WARNING(
"UDPReceiver") << std::dec <<
"Sequence Number different than expected! (delta: " << delta <<
")" << TLOG_ENDL;
118 expectedPacketNumber_ = seqNum;
121 packetBuffer_t receiveBuffer;
122 receiveBuffer.resize(1500);
123 int sts = recvfrom(datasocket_, &receiveBuffer[0], receiveBuffer.size(), 0, (
struct sockaddr *) &si_data_, &dataSz);
126 TLOG_WARNING(
"UDPReceiver") <<
"Error on socket: " << strerror(errno) << TLOG_ENDL;
130 TLOG_TRACE(
"UDPReceiver") <<
"Received " << sts <<
" bytes." << TLOG_ENDL;
133 std::unique_lock<std::mutex> lock(receiveBufferLock_);
134 TLOG_TRACE(
"UDPReceiver") <<
"Now placing UDP datagram with sequence number " << std::hex << (int)seqNum <<
" into buffer." << std::dec << TLOG_ENDL;
135 receiveBuffers_.push_back(receiveBuffer);
137 ++expectedPacketNumber_;
141 TLOG_WARNING(
"UDPReceiver") <<
"Received out-of-order datagram: " << seqNum <<
" != " << expectedPacketNumber_ <<
" (expected)" << TLOG_ENDL;
142 packetBuffer_t receiveBuffer;
143 receiveBuffer.resize(1500);
144 int sts = recvfrom(datasocket_, &receiveBuffer[0], receiveBuffer.size(), 0, (
struct sockaddr *) &si_data_, &dataSz);
150 TLOG_INFO(
"UDPReceiver") <<
"receive Loop exiting..." << TLOG_ENDL;
153 bool ots::UDPReceiver::getNext_(artdaq::FragmentPtrs & output)
160 std::unique_lock<std::mutex> lock(receiveBufferLock_);
161 std::move(receiveBuffers_.begin(), receiveBuffers_.end(), std::inserter(packetBuffers_, packetBuffers_.end()));
162 receiveBuffers_.clear();
165 if (packetBuffers_.size() > 0) {
166 size_t packetBufferSize = 0;
167 for (
auto& buf : packetBuffers_) {
168 packetBufferSize += buf.size();
170 TLOG_TRACE(
"UDPReceiver") <<
"Calling ProcessData, packetBuffers_.size() == " << std::to_string(packetBuffers_.size()) <<
", sz = " << std::to_string(packetBufferSize) << TLOG_ENDL;
171 ProcessData_(output);
173 packetBuffers_.clear();
174 TLOG_TRACE(
"UDPReceiver") <<
"Returning output of size " << output.size() << TLOG_ENDL;
183 void ots::UDPReceiver::ProcessData_(artdaq::FragmentPtrs & output) {
186 metadata.port = dataport_;
187 metadata.address = si_data_.sin_addr.s_addr;
189 std::size_t initial_payload_size = 0;
191 output.emplace_back(artdaq::Fragment::FragmentBytes(initial_payload_size,
192 ev_counter(), fragment_id(),
193 ots::detail::FragmentType::UDP, metadata));
197 TLOG_TRACE(
"UDPReceiver") <<
"Received data, now placing data with UDP sequence number "
198 << std::hex << static_cast<int>((packetBuffers_.front()).at(1))
199 <<
" into UDPFragment" << TLOG_ENDL;
200 thisFrag.resize(64050 * packetBuffers_.size() + 1);
201 std::ofstream rawOutput;
203 std::string outputPath = rawPath_ +
"/UDPReceiver-" + ip_ +
":" + std::to_string(dataport_) +
".bin";
204 rawOutput.open(outputPath, std::ios::out | std::ios::app | std::ios::binary);
207 DataType dataType = getDataType((packetBuffers_.front()).at(0));
208 thisFrag.set_hdr_type((
int)dataType);
210 for (
auto jj = packetBuffers_.begin(); jj != packetBuffers_.end(); ++jj) {
211 for (
int ii = 0; ii < 64050; ++ii) {
213 if ((jj)->at(ii) == 0 && (dataType == DataType::JSON || dataType == DataType::String)) {
break; }
215 if (rawOutput_) rawOutput.write((
char*)&((jj)->at(ii)),
sizeof(uint8_t));
216 *(thisFrag.dataBegin() + pos) = (jj)->at(ii);
221 if (dataType == DataType::JSON || dataType == DataType::String) {
222 *(thisFrag.dataBegin() + pos) = 0;
224 if (rawOutput_) rawOutput.write(&zero,
sizeof(
char));
226 if (rawOutput_) rawOutput.close();
229 void ots::UDPReceiver::send(CommandType command)
232 CommandPacket packet;
233 packet.type = command;
235 sendto(datasocket_, &packet,
sizeof(packet), 0, (
struct sockaddr *) &si_data_,
sizeof(si_data_));
239 bool ots::UDPReceiver::isTimerExpired_()
241 auto now = std::chrono::high_resolution_clock::now();
242 auto diff = std::chrono::duration<double, std::milli>(now - lastFrag_).count();
243 return diff > fragmentWindow_;
246 void ots::UDPReceiver::stop()
248 #pragma message "Using default implementation of UDPReceiver::stop()"
251 void ots::UDPReceiver::stopNoMutex()
253 #pragma message "Using default implementation of UDPReceiver::stopNoMutex()"
256 void ots::UDPReceiver::start_()
258 #pragma message "Using default implementation of UDPReceiver::start_()"