3 #define TRACE_NAME (app_name + "_DataReceiverManager").c_str()
4 #include "artdaq/DAQdata/Globals.hh"
5 #include "artdaq/DAQrate/DataReceiverManager.hh"
6 #include "artdaq/TransferPlugins/MakeTransferPlugin.hh"
7 #include "artdaq/TransferPlugins/detail/HostMap.hh"
8 #include "cetlib_except/exception.h"
12 : stop_requested_(false)
13 , stop_requested_time_(0)
16 , source_metric_data_()
17 , source_metric_send_time_()
23 , receive_timeout_(pset.get<size_t>(
"receive_timeout_usec", 100000))
24 , stop_timeout_ms_(pset.get<size_t>(
"stop_timeout_ms", 1500))
26 , non_reliable_mode_enabled_(pset.get<bool>(
"non_reliable_mode", false))
27 , non_reliable_mode_retry_count_(pset.get<size_t>(
"non_reliable_mode_retry_count", -1))
29 TLOG(TLVL_DEBUG) <<
"Constructor";
30 auto enabled_srcs = pset.get<std::vector<int>>(
"enabled_sources", std::vector<int>());
31 auto enabled_srcs_empty = enabled_srcs.size() == 0;
33 if (non_reliable_mode_enabled_)
35 TLOG(TLVL_WARNING) <<
"DataReceiverManager is configured to drop data after " << non_reliable_mode_retry_count_
36 <<
" failed attempts to put data into the SharedMemoryEventManager! If this is unexpected, please check your configuration!";
39 if (enabled_srcs_empty)
41 TLOG(TLVL_INFO) <<
"enabled_sources not specified, assuming all sources enabled.";
45 for (
auto& s : enabled_srcs)
47 enabled_sources_[s] =
true;
52 size_t tcp_receive_buffer_size = pset.get<
size_t>(
"tcp_receive_buffer_size", 0);
53 size_t max_fragment_size_words = pset.get<
size_t>(
"max_fragment_size_words", 0);
55 auto srcs = pset.get<fhicl::ParameterSet>(
"sources", fhicl::ParameterSet());
56 for (
auto& s : srcs.get_pset_names())
58 auto src_pset = srcs.get<fhicl::ParameterSet>(s);
62 fhicl::ParameterSet srcs_mod;
63 for (
auto& s : srcs.get_pset_names())
65 auto src_pset = srcs.get<fhicl::ParameterSet>(s);
66 src_pset.erase(
"host_map");
67 src_pset.put<std::vector<fhicl::ParameterSet>>(
"host_map", host_map_pset);
69 if (tcp_receive_buffer_size != 0 && !src_pset.has_key(
"tcp_receive_buffer_size"))
71 src_pset.put<
size_t>(
"tcp_receive_buffer_size", tcp_receive_buffer_size);
73 if (max_fragment_size_words != 0 && !src_pset.has_key(
"max_fragment_size_words"))
75 src_pset.put<
size_t>(
"max_fragment_size_words", max_fragment_size_words);
78 srcs_mod.put<fhicl::ParameterSet>(s, src_pset);
81 for (
auto& s : srcs_mod.get_pset_names())
87 auto source_rank = transfer->source_rank();
88 if (enabled_srcs_empty) enabled_sources_[source_rank] =
true;
89 else if (!enabled_sources_.count(source_rank)) enabled_sources_[source_rank] =
false;
90 running_sources_[source_rank] =
false;
91 source_plugins_[source_rank] = std::move(transfer);
92 source_metric_send_time_[source_rank] = std::chrono::steady_clock::now();
93 source_metric_data_[source_rank] = source_metric_data();
95 catch (cet::exception ex)
97 TLOG(TLVL_WARNING) <<
"cet::exception caught while setting up source " << s <<
": " << ex.what();
99 catch (std::exception ex)
101 TLOG(TLVL_WARNING) <<
"std::exception caught while setting up source " << s <<
": " << ex.what();
105 TLOG(TLVL_WARNING) <<
"Non-cet exception caught while setting up source " << s <<
".";
108 if (srcs.get_pset_names().size() == 0)
110 TLOG(TLVL_ERROR) <<
"No sources configured!";
116 TLOG(TLVL_TRACE) <<
"~DataReceiverManager: BEGIN";
118 shm_manager_.reset();
119 TLOG(TLVL_TRACE) <<
"Destructor END";
125 stop_requested_ =
false;
127 for (
auto& source : source_plugins_)
129 auto& rank = source.first;
130 if (enabled_sources_.count(rank) && enabled_sources_[rank].load())
132 running_sources_[rank] =
true;
133 boost::thread::attributes attrs;
134 attrs.set_stack_size(4096 * 2000);
136 source_threads_[rank] = boost::thread(attrs, boost::bind(&DataReceiverManager::runReceiver_,
this, rank));
138 catch (
const boost::exception& e)
140 TLOG(TLVL_ERROR) <<
"Caught boost::exception starting Receiver " << rank <<
" thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
141 std::cerr <<
"Caught boost::exception starting Receiver " << rank <<
" thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
150 TLOG(TLVL_TRACE) <<
"stop_threads: BEGIN: Setting stop_requested to true, frags=" << count() <<
", bytes=" << byteCount();
152 stop_requested_time_ = TimeUtils::gettimeofday_us();
153 stop_requested_ =
true;
155 TLOG(TLVL_TRACE) <<
"stop_threads: Joining all threads";
156 for (
auto& s : source_threads_)
158 auto& thread = s.second;
159 if (thread.joinable()) thread.join();
165 std::set<int> output;
166 for (
auto& src : enabled_sources_)
168 if (src.second) output.insert(src.first);
175 std::set<int> output;
176 for (
auto& src : running_sources_)
178 if (src.second) output.insert(src.first);
183 void artdaq::DataReceiverManager::runReceiver_(
int source_rank)
185 std::chrono::steady_clock::time_point start_time, after_header, before_body,after_body, end_time = std::chrono::steady_clock::now();
187 detail::RawFragmentHeader header;
188 size_t endOfDataCount = -1;
189 auto sleep_time = receive_timeout_ / 100 > 100000 ? 100000 : receive_timeout_ / 100;
190 if (sleep_time < 5000) sleep_time = 5000;
191 auto max_retries = non_reliable_mode_retry_count_ * ceil(receive_timeout_ / sleep_time);
193 while (!(stop_requested_ && TimeUtils::gettimeofday_us() - stop_requested_time_ > stop_timeout_ms_ * 1000) && enabled_sources_.count(source_rank))
195 TLOG(16) <<
"runReceiver_: Begin loop";
196 std::this_thread::yield();
199 if (endOfDataCount <= recv_frag_count_.slotCount(source_rank) && !source_plugins_[source_rank]->isRunning())
201 TLOG(TLVL_DEBUG) <<
"runReceiver_: End of Data conditions met, ending runReceiver loop";
205 start_time = std::chrono::steady_clock::now();
207 TLOG(16) <<
"runReceiver_: Calling receiveFragmentHeader tmo=" << receive_timeout_;
208 ret = source_plugins_[source_rank]->receiveFragmentHeader(header, receive_timeout_);
209 TLOG(16) <<
"runReceiver_: Done with receiveFragmentHeader, ret=" << ret <<
" (should be " << source_rank <<
")";
210 if (ret != source_rank)
213 TLOG(TLVL_WARNING) <<
"Received Fragment from rank " << ret <<
", but was expecting one from rank " << source_rank <<
"!";
217 TLOG(TLVL_ERROR) <<
"Transfer Plugin returned DATA_END, ending receive loop!";
220 if ((*running_sources_.begin()).first == source_rank)
222 TLOG(TLVL_DEBUG) <<
"Calling SMEM::CheckPendingBuffers from DRM receiver thread for " << source_rank <<
" to make sure that things aren't stuck";
223 shm_manager_->CheckPendingBuffers();
230 after_header = std::chrono::steady_clock::now();
232 if (Fragment::isUserFragmentType(header.type) || header.type == Fragment::DataFragmentType || header.type == Fragment::EmptyFragmentType || header.type == Fragment::ContainerFragmentType) {
233 TLOG(TLVL_TRACE) <<
"Received Fragment Header from rank " << source_rank <<
", sequence ID " << header.sequence_id <<
", timestamp " << header.timestamp;
234 RawDataType* loc =
nullptr;
236 while (loc ==
nullptr)
238 loc = shm_manager_->WriteFragmentHeader(header);
239 if (loc ==
nullptr && stop_requested_)
return;
240 if (loc ==
nullptr) usleep(sleep_time);
242 if (non_reliable_mode_enabled_ && retries > max_retries)
244 loc = shm_manager_->WriteFragmentHeader(header,
true);
250 TLOG(TLVL_ERROR) <<
"runReceiver_: Could not get data location for event " << header.sequence_id;
253 before_body = std::chrono::steady_clock::now();
255 auto hdrLoc =
reinterpret_cast<artdaq::detail::RawFragmentHeader*
>(loc - artdaq::detail::RawFragmentHeader::num_words());
256 TLOG(16) <<
"runReceiver_: Calling receiveFragmentData from rank " << source_rank <<
", sequence ID " << header.sequence_id <<
", timestamp " << header.timestamp;
257 auto ret2 = source_plugins_[source_rank]->receiveFragmentData(loc, header.word_count - header.num_words());
258 TLOG(16) <<
"runReceiver_: Done with receiveFragmentData, ret2=" << ret2 <<
" (should be " << source_rank <<
")";
261 TLOG(TLVL_ERROR) <<
"Unexpected return code from receiveFragmentData after receiveFragmentHeader! (Expected: " << ret <<
", Got: " << ret2 <<
")";
262 TLOG(TLVL_ERROR) <<
"Error receiving data from rank " << source_rank <<
", data has been lost! Event " << header.sequence_id <<
" will most likely be Incomplete!";
266 hdrLoc->type = Fragment::ErrorFragmentType;
268 shm_manager_->DoneWritingFragment(header);
273 shm_manager_->DoneWritingFragment(header);
274 TLOG(TLVL_TRACE) <<
"Done receiving fragment with sequence ID " << header.sequence_id <<
" from rank " << source_rank;
276 recv_frag_count_.incSlot(source_rank);
277 recv_frag_size_.incSlot(source_rank, header.word_count *
sizeof(RawDataType));
278 recv_seq_count_.setSlot(source_rank, header.sequence_id);
279 if (endOfDataCount != static_cast<size_t>(-1))
281 TLOG(TLVL_DEBUG) <<
"Received fragment " << header.sequence_id <<
" from rank " << source_rank
282 <<
" (" << recv_frag_count_.slotCount(source_rank) <<
"/" << endOfDataCount <<
")";
285 after_body = std::chrono::steady_clock::now();
287 source_metric_data_[source_rank].hdr_delta_t += TimeUtils::GetElapsedTime(start_time, after_header);
288 source_metric_data_[source_rank].store_delta_t += TimeUtils::GetElapsedTime(after_header, before_body);
289 source_metric_data_[source_rank].data_delta_t += TimeUtils::GetElapsedTime(before_body, after_body);
290 source_metric_data_[source_rank].delta_t += TimeUtils::GetElapsedTime(start_time, after_body);
291 source_metric_data_[source_rank].dead_t += TimeUtils::GetElapsedTime(end_time, start_time);
293 source_metric_data_[source_rank].data_size += header.word_count *
sizeof(RawDataType);
294 source_metric_data_[source_rank].header_size += header.num_words() *
sizeof(RawDataType);
295 source_metric_data_[source_rank].data_point_count++;
297 if (metricMan && TimeUtils::GetElapsedTime(source_metric_send_time_[source_rank]) > 1)
299 TLOG(6) <<
"runReceiver_: Sending receive stats for rank " << source_rank;
300 metricMan->sendMetric(
"Total Receive Time From Rank " + std::to_string(source_rank), source_metric_data_[source_rank].delta_t,
"s", 5, MetricMode::Accumulate);
301 metricMan->sendMetric(
"Total Receive Size From Rank " + std::to_string(source_rank), static_cast<unsigned long>(source_metric_data_[source_rank].data_size),
"B", 5, MetricMode::Accumulate);
302 metricMan->sendMetric(
"Total Receive Rate From Rank " + std::to_string(source_rank), source_metric_data_[source_rank].data_size / source_metric_data_[source_rank].delta_t,
"B/s", 5, MetricMode::Average);
304 metricMan->sendMetric(
"Header Receive Time From Rank " + std::to_string(source_rank), source_metric_data_[source_rank].hdr_delta_t,
"s", 5, MetricMode::Accumulate);
305 metricMan->sendMetric(
"Header Receive Size From Rank " + std::to_string(source_rank), static_cast<unsigned long>(source_metric_data_[source_rank].header_size),
"B", 5, MetricMode::Accumulate);
306 metricMan->sendMetric(
"Header Receive Rate From Rank " + std::to_string(source_rank), source_metric_data_[source_rank].header_size / source_metric_data_[source_rank].hdr_delta_t,
"B/s", 5, MetricMode::Average);
308 auto payloadSize = source_metric_data_[source_rank].data_size - source_metric_data_[source_rank].header_size;
309 metricMan->sendMetric(
"Data Receive Time From Rank " + std::to_string(source_rank), source_metric_data_[source_rank].data_delta_t,
"s", 5, MetricMode::Accumulate);
310 metricMan->sendMetric(
"Data Receive Size From Rank " + std::to_string(source_rank), static_cast<unsigned long>(payloadSize),
"B", 5, MetricMode::Accumulate);
311 metricMan->sendMetric(
"Data Receive Rate From Rank " + std::to_string(source_rank), payloadSize / source_metric_data_[source_rank].data_delta_t,
"B/s", 5, MetricMode::Average);
313 metricMan->sendMetric(
"Data Receive Count From Rank " + std::to_string(source_rank), recv_frag_count_.slotCount(source_rank),
"fragments", 3, MetricMode::LastPoint);
315 metricMan->sendMetric(
"Total Shared Memory Wait Time From Rank " + std::to_string(source_rank), source_metric_data_[source_rank].store_delta_t,
"s", 3, MetricMode::Accumulate);
316 metricMan->sendMetric(
"Avg Shared Memory Wait Time From Rank " + std::to_string(source_rank), source_metric_data_[source_rank].store_delta_t / source_metric_data_[source_rank].data_point_count,
"s", 3, MetricMode::Average);
317 metricMan->sendMetric(
"Avg Fragment Wait Time From Rank " + std::to_string(source_rank), source_metric_data_[source_rank].dead_t / source_metric_data_[source_rank].data_point_count,
"s", 3, MetricMode::Average);
319 TLOG(6) <<
"runReceiver_: Done sending receive stats for rank " << source_rank;
321 source_metric_send_time_[source_rank] = std::chrono::steady_clock::now();
322 source_metric_data_[source_rank] = source_metric_data();
325 end_time = std::chrono::steady_clock::now();
327 else if (header.type == Fragment::EndOfDataFragmentType || header.type == Fragment::InitFragmentType || header.type == Fragment::EndOfRunFragmentType || header.type == Fragment::EndOfSubrunFragmentType || header.type == Fragment::ShutdownFragmentType)
329 TLOG(TLVL_DEBUG) <<
"Received System Fragment from rank " << source_rank <<
" of type " << detail::RawFragmentHeader::SystemTypeToString(header.type) <<
".";
331 FragmentPtr frag(
new Fragment(header.word_count - header.num_words()));
332 memcpy(frag->headerAddress(), &header, header.num_words() *
sizeof(RawDataType));
333 auto ret3 = source_plugins_[source_rank]->receiveFragmentData(frag->headerAddress() + header.num_words(), header.word_count - header.num_words());
334 if (ret3 != source_rank)
336 TLOG(TLVL_ERROR) <<
"Unexpected return code from receiveFragmentData after receiveFragmentHeader while receiving System Fragment! (Expected: " << source_rank <<
", Got: " << ret3 <<
")";
337 throw cet::exception(
"DataReceiverManager") <<
"Unexpected return code from receiveFragmentData after receiveFragmentHeader while receiving System Fragment! (Expected: " << source_rank <<
", Got: " << ret3 <<
")";
342 case Fragment::EndOfDataFragmentType:
344 endOfDataCount = *(frag->dataBegin());
345 TLOG(TLVL_DEBUG) <<
"EndOfData Fragment indicates that " << endOfDataCount <<
" fragments are expected from rank " << source_rank
346 <<
" (recvd " << recv_frag_count_.slotCount(source_rank) <<
").";
348 case Fragment::InitFragmentType:
349 TLOG(TLVL_DEBUG) <<
"Received Init Fragment from rank " << source_rank <<
".";
351 shm_manager_->SetInitFragment(std::move(frag));
353 case Fragment::EndOfRunFragmentType:
357 case Fragment::EndOfSubrunFragmentType:
359 TLOG(TLVL_DEBUG) <<
"Received EndOfSubrun Fragment from rank " << source_rank
360 <<
" with sequence_id " << header.sequence_id <<
".";
361 if (header.sequence_id != Fragment::InvalidSequenceID) shm_manager_->rolloverSubrun(header.sequence_id);
362 else shm_manager_->rolloverSubrun(recv_seq_count_.slotCount(source_rank));
364 case Fragment::ShutdownFragmentType:
371 TLOG(TLVL_DEBUG) <<
"runReceiver_ " << source_rank <<
" receive loop exited";
372 running_sources_[source_rank] =
false;
std::set< int > running_sources() const
Get the list of sources which are still receiving data.
End of Run mode (Used to end request processing on receiver)
hostMap_t MakeHostMap(fhicl::ParameterSet pset, hostMap_t map=hostMap_t())
Make a hostMap_t from a HostMap::Config ParameterSet
This TransferInterface is a Receiver.
std::unique_ptr< artdaq::TransferInterface > MakeTransferPlugin(const fhicl::ParameterSet &pset, std::string plugin_label, TransferInterface::Role role)
Load a TransferInterface plugin.
DataReceiverManager(const fhicl::ParameterSet &ps, std::shared_ptr< SharedMemoryEventManager > shm)
DataReceiverManager Constructor.
std::set< int > enabled_sources() const
Get the list of enabled sources.
std::vector< fhicl::ParameterSet > MakeHostMapPset(std::map< int, std::string > input)
Create a list of HostMap::HostConfig ParameterSets from a hostMap_t map
Value that is to be returned when a Transfer plugin determines that no more data will be arriving...
virtual ~DataReceiverManager()
DataReceiverManager Destructor.
void start_threads()
Start receiver threads for all enabled sources.
std::map< int, std::string > hostMap_t
The host_map is a map associating ranks with artdaq::DestinationInfo objects.
void stop_threads()
Stop receiver threads.