artdaq  v3_03_02
DataReceiverManager.cc
1 #include <chrono>
2 
3 #define TRACE_NAME (app_name + "_DataReceiverManager").c_str()
4 #include "artdaq/DAQdata/Globals.hh"
5 #include "artdaq/DAQrate/DataReceiverManager.hh"
6 #include "artdaq/TransferPlugins/MakeTransferPlugin.hh"
7 #include "artdaq/TransferPlugins/detail/HostMap.hh"
8 #include "cetlib_except/exception.h"
9 #include <iomanip>
10 
11 artdaq::DataReceiverManager::DataReceiverManager(const fhicl::ParameterSet& pset, std::shared_ptr<SharedMemoryEventManager> shm)
12  : stop_requested_(false)
13  , stop_requested_time_(0)
14  , source_threads_()
15  , source_plugins_()
16  , source_metric_data_()
17  , source_metric_send_time_()
18  , enabled_sources_()
19  , running_sources_()
20  , recv_frag_count_()
21  , recv_frag_size_()
22  , recv_seq_count_()
23  , receive_timeout_(pset.get<size_t>("receive_timeout_usec", 100000))
24  , stop_timeout_ms_(pset.get<size_t>("stop_timeout_ms", 1500))
25  , shm_manager_(shm)
26  , non_reliable_mode_enabled_(pset.get<bool>("non_reliable_mode", false))
27  , non_reliable_mode_retry_count_(pset.get<size_t>("non_reliable_mode_retry_count", -1))
28 {
29  TLOG(TLVL_DEBUG) << "Constructor";
30  auto enabled_srcs = pset.get<std::vector<int>>("enabled_sources", std::vector<int>());
31  auto enabled_srcs_empty = enabled_srcs.size() == 0;
32 
33  if (non_reliable_mode_enabled_)
34  {
35  TLOG(TLVL_WARNING) << "DataReceiverManager is configured to drop data after " << non_reliable_mode_retry_count_
36  << " failed attempts to put data into the SharedMemoryEventManager! If this is unexpected, please check your configuration!";
37  }
38 
39  if (enabled_srcs_empty)
40  {
41  TLOG(TLVL_INFO) << "enabled_sources not specified, assuming all sources enabled.";
42  }
43  else
44  {
45  for (auto& s : enabled_srcs)
46  {
47  enabled_sources_[s] = true;
48  }
49  }
50 
51  hostMap_t host_map = MakeHostMap(pset);
52  size_t tcp_receive_buffer_size = pset.get<size_t>("tcp_receive_buffer_size", 0);
53  size_t max_fragment_size_words = pset.get<size_t>("max_fragment_size_words", 0);
54 
55  auto srcs = pset.get<fhicl::ParameterSet>("sources", fhicl::ParameterSet());
56  for (auto& s : srcs.get_pset_names())
57  {
58  auto src_pset = srcs.get<fhicl::ParameterSet>(s);
59  host_map = MakeHostMap(src_pset, host_map);
60  }
61  auto host_map_pset = MakeHostMapPset(host_map);
62  fhicl::ParameterSet srcs_mod;
63  for (auto& s : srcs.get_pset_names())
64  {
65  auto src_pset = srcs.get<fhicl::ParameterSet>(s);
66  src_pset.erase("host_map");
67  src_pset.put<std::vector<fhicl::ParameterSet>>("host_map", host_map_pset);
68 
69  if (tcp_receive_buffer_size != 0 && !src_pset.has_key("tcp_receive_buffer_size"))
70  {
71  src_pset.put<size_t>("tcp_receive_buffer_size", tcp_receive_buffer_size);
72  }
73  if (max_fragment_size_words != 0 && !src_pset.has_key("max_fragment_size_words"))
74  {
75  src_pset.put<size_t>("max_fragment_size_words", max_fragment_size_words);
76  }
77 
78  srcs_mod.put<fhicl::ParameterSet>(s, src_pset);
79  }
80 
81  for (auto& s : srcs_mod.get_pset_names())
82  {
83  try
84  {
85  auto transfer = std::unique_ptr<TransferInterface>(MakeTransferPlugin(srcs_mod, s,
87  auto source_rank = transfer->source_rank();
88  if (enabled_srcs_empty) enabled_sources_[source_rank] = true;
89  else if (!enabled_sources_.count(source_rank)) enabled_sources_[source_rank] = false;
90  running_sources_[source_rank] = false;
91  source_plugins_[source_rank] = std::move(transfer);
92  source_metric_send_time_[source_rank] = std::chrono::steady_clock::now();
93  source_metric_data_[source_rank] = source_metric_data();
94  }
95  catch (cet::exception ex)
96  {
97  TLOG(TLVL_WARNING) << "cet::exception caught while setting up source " << s << ": " << ex.what();
98  }
99  catch (std::exception ex)
100  {
101  TLOG(TLVL_WARNING) << "std::exception caught while setting up source " << s << ": " << ex.what();
102  }
103  catch (...)
104  {
105  TLOG(TLVL_WARNING) << "Non-cet exception caught while setting up source " << s << ".";
106  }
107  }
108  if (srcs.get_pset_names().size() == 0)
109  {
110  TLOG(TLVL_ERROR) << "No sources configured!";
111  }
112 }
113 
115 {
116  TLOG(TLVL_TRACE) << "~DataReceiverManager: BEGIN";
117  stop_threads();
118  shm_manager_.reset();
119  TLOG(TLVL_TRACE) << "Destructor END";
120 }
121 
122 
124 {
125  stop_requested_ = false;
126  if (shm_manager_) shm_manager_->setRequestMode(artdaq::detail::RequestMessageMode::Normal);
127  for (auto& source : source_plugins_)
128  {
129  auto& rank = source.first;
130  if (enabled_sources_.count(rank) && enabled_sources_[rank].load())
131  {
132  source_metric_data_[rank] = source_metric_data();
133  source_metric_send_time_[rank] = std::chrono::steady_clock::now();
134 
135  recv_frag_count_.setSlot(rank, 0);
136  recv_frag_size_.setSlot(rank,0);
137  recv_seq_count_.setSlot(rank,0);
138 
139  running_sources_[rank] = true;
140  boost::thread::attributes attrs;
141  attrs.set_stack_size(4096 * 2000); // 2000 KB
142  try {
143  source_threads_[rank] = boost::thread(attrs, boost::bind(&DataReceiverManager::runReceiver_, this, rank));
144  }
145  catch (const boost::exception& e)
146  {
147  TLOG(TLVL_ERROR) << "Caught boost::exception starting Receiver " << rank << " thread: " << boost::diagnostic_information(e) << ", errno=" << errno;
148  std::cerr << "Caught boost::exception starting Receiver " << rank << " thread: " << boost::diagnostic_information(e) << ", errno=" << errno << std::endl;
149  exit(5);
150  }
151  }
152  }
153 }
154 
156 {
157  TLOG(TLVL_TRACE) << "stop_threads: BEGIN: Setting stop_requested to true, frags=" << count() << ", bytes=" << byteCount();
158 
159  stop_requested_time_ = TimeUtils::gettimeofday_us();
160  stop_requested_ = true;
161 
162  TLOG(TLVL_TRACE) << "stop_threads: Joining all threads";
163  for (auto& s : source_threads_)
164  {
165  auto& thread = s.second;
166  if (thread.joinable()) thread.join();
167  }
168 }
169 
171 {
172  std::set<int> output;
173  for (auto& src : enabled_sources_)
174  {
175  if (src.second) output.insert(src.first);
176  }
177  return output;
178 }
179 
181 {
182  std::set<int> output;
183  for (auto& src : running_sources_)
184  {
185  if (src.second) output.insert(src.first);
186  }
187  return output;
188 }
189 
190 void artdaq::DataReceiverManager::runReceiver_(int source_rank)
191 {
192  std::chrono::steady_clock::time_point start_time, after_header, before_body,after_body, end_time = std::chrono::steady_clock::now();
193  int ret;
194  detail::RawFragmentHeader header;
195  size_t endOfDataCount = -1;
196  auto sleep_time = receive_timeout_ / 100 > 100000 ? 100000 : receive_timeout_ / 100;
197  if (sleep_time < 5000) sleep_time = 5000;
198  auto max_retries = non_reliable_mode_retry_count_ * ceil(receive_timeout_ / sleep_time);
199 
200  while (!(stop_requested_ && TimeUtils::gettimeofday_us() - stop_requested_time_ > stop_timeout_ms_ * 1000) && enabled_sources_.count(source_rank))
201  {
202  TLOG(16) << "runReceiver_: Begin loop";
203  std::this_thread::yield();
204 
205  // Don't stop receiving until we haven't received anything for 1 second
206  if (endOfDataCount <= recv_frag_count_.slotCount(source_rank) && !source_plugins_[source_rank]->isRunning())
207  {
208  TLOG(TLVL_DEBUG) << "runReceiver_: End of Data conditions met, ending runReceiver loop";
209  break;
210  }
211 
212  start_time = std::chrono::steady_clock::now();
213 
214  TLOG(16) << "runReceiver_: Calling receiveFragmentHeader tmo=" << receive_timeout_;
215  ret = source_plugins_[source_rank]->receiveFragmentHeader(header, receive_timeout_);
216  TLOG(16) << "runReceiver_: Done with receiveFragmentHeader, ret=" << ret << " (should be " << source_rank << ")";
217  if (ret != source_rank)
218  {
219  if (ret >= 0) {
220  TLOG(TLVL_WARNING) << "Received Fragment from rank " << ret << ", but was expecting one from rank " << source_rank << "!";
221  }
222  else if (ret == TransferInterface::DATA_END)
223  {
224  TLOG(TLVL_ERROR) << "Transfer Plugin returned DATA_END, ending receive loop!";
225  break;
226  }
227  if ((*running_sources_.begin()).first == source_rank) // Only do this for the first sender in the running_sources_ map
228  {
229  TLOG(TLVL_DEBUG) << "Calling SMEM::CheckPendingBuffers from DRM receiver thread for " << source_rank << " to make sure that things aren't stuck";
230  shm_manager_->CheckPendingBuffers();
231  }
232 
233  usleep(sleep_time);
234  continue; // Receive timeout or other oddness
235  }
236 
237  after_header = std::chrono::steady_clock::now();
238 
239  if (Fragment::isUserFragmentType(header.type) || header.type == Fragment::DataFragmentType || header.type == Fragment::EmptyFragmentType || header.type == Fragment::ContainerFragmentType) {
240  TLOG(TLVL_TRACE) << "Received Fragment Header from rank " << source_rank << ", sequence ID " << header.sequence_id << ", timestamp " << header.timestamp;
241  RawDataType* loc = nullptr;
242  size_t retries = 0;
243  while (loc == nullptr)//&& TimeUtils::GetElapsedTimeMicroseconds(after_header)) < receive_timeout_)
244  {
245  loc = shm_manager_->WriteFragmentHeader(header);
246  if (loc == nullptr && stop_requested_) return;
247  if (loc == nullptr) usleep(sleep_time);
248  retries++;
249  if (non_reliable_mode_enabled_ && retries > max_retries)
250  {
251  loc = shm_manager_->WriteFragmentHeader(header, true);
252  }
253  }
254  if (loc == nullptr)
255  {
256  // Could not enqueue event!
257  TLOG(TLVL_ERROR) << "runReceiver_: Could not get data location for event " << header.sequence_id;
258  continue;
259  }
260  before_body = std::chrono::steady_clock::now();
261 
262  auto hdrLoc = reinterpret_cast<artdaq::detail::RawFragmentHeader*>(loc - artdaq::detail::RawFragmentHeader::num_words());
263  TLOG(16) << "runReceiver_: Calling receiveFragmentData from rank " << source_rank << ", sequence ID " << header.sequence_id << ", timestamp " << header.timestamp;
264  auto ret2 = source_plugins_[source_rank]->receiveFragmentData(loc, header.word_count - header.num_words());
265  TLOG(16) << "runReceiver_: Done with receiveFragmentData, ret2=" << ret2 << " (should be " << source_rank << ")";
266 
267  if (ret != ret2) {
268  TLOG(TLVL_ERROR) << "Unexpected return code from receiveFragmentData after receiveFragmentHeader! (Expected: " << ret << ", Got: " << ret2 << ")";
269  TLOG(TLVL_ERROR) << "Error receiving data from rank " << source_rank << ", data has been lost! Event " << header.sequence_id << " will most likely be Incomplete!";
270 
271  // Mark the Fragment as invalid
272  /* \todo Make a RawFragmentHeader field that marks it as invalid while maintaining previous type! */
273  hdrLoc->type = Fragment::ErrorFragmentType;
274 
275  shm_manager_->DoneWritingFragment(header);
276  //throw cet::exception("DataReceiverManager") << "Unexpected return code from receiveFragmentData after receiveFragmentHeader! (Expected: " << ret << ", Got: " << ret2 << ")";
277  continue;
278  }
279 
280  shm_manager_->DoneWritingFragment(header);
281  TLOG(TLVL_TRACE) << "Done receiving fragment with sequence ID " << header.sequence_id << " from rank " << source_rank;
282 
283  recv_frag_count_.incSlot(source_rank);
284  recv_frag_size_.incSlot(source_rank, header.word_count * sizeof(RawDataType));
285  recv_seq_count_.setSlot(source_rank, header.sequence_id);
286  if (endOfDataCount != static_cast<size_t>(-1))
287  {
288  TLOG(TLVL_DEBUG) << "Received fragment " << header.sequence_id << " from rank " << source_rank
289  << " (" << recv_frag_count_.slotCount(source_rank) << "/" << endOfDataCount << ")";
290  }
291 
292  after_body = std::chrono::steady_clock::now();
293 
294  source_metric_data_[source_rank].hdr_delta_t += TimeUtils::GetElapsedTime(start_time, after_header);
295  source_metric_data_[source_rank].store_delta_t += TimeUtils::GetElapsedTime(after_header, before_body);
296  source_metric_data_[source_rank].data_delta_t += TimeUtils::GetElapsedTime(before_body, after_body);
297  source_metric_data_[source_rank].delta_t += TimeUtils::GetElapsedTime(start_time, after_body);
298  source_metric_data_[source_rank].dead_t += TimeUtils::GetElapsedTime(end_time, start_time);
299 
300  source_metric_data_[source_rank].data_size += header.word_count * sizeof(RawDataType);
301  source_metric_data_[source_rank].header_size += header.num_words() * sizeof(RawDataType);
302  source_metric_data_[source_rank].data_point_count++;
303 
304  if (metricMan && TimeUtils::GetElapsedTime(source_metric_send_time_[source_rank]) > 1)
305  {//&& recv_frag_count_.slotCount(source_rank) % 100 == 0) {
306  TLOG(6) << "runReceiver_: Sending receive stats for rank " << source_rank;
307  metricMan->sendMetric("Total Receive Time From Rank " + std::to_string(source_rank), source_metric_data_[source_rank].delta_t, "s", 5, MetricMode::Accumulate);
308  metricMan->sendMetric("Total Receive Size From Rank " + std::to_string(source_rank), static_cast<unsigned long>(source_metric_data_[source_rank].data_size), "B", 5, MetricMode::Accumulate);
309  metricMan->sendMetric("Total Receive Rate From Rank " + std::to_string(source_rank), source_metric_data_[source_rank].data_size / source_metric_data_[source_rank].delta_t, "B/s", 5, MetricMode::Average);
310 
311  metricMan->sendMetric("Header Receive Time From Rank " + std::to_string(source_rank), source_metric_data_[source_rank].hdr_delta_t, "s", 5, MetricMode::Accumulate);
312  metricMan->sendMetric("Header Receive Size From Rank " + std::to_string(source_rank), static_cast<unsigned long>(source_metric_data_[source_rank].header_size), "B", 5, MetricMode::Accumulate);
313  metricMan->sendMetric("Header Receive Rate From Rank " + std::to_string(source_rank), source_metric_data_[source_rank].header_size / source_metric_data_[source_rank].hdr_delta_t, "B/s", 5, MetricMode::Average);
314 
315  auto payloadSize = source_metric_data_[source_rank].data_size - source_metric_data_[source_rank].header_size;
316  metricMan->sendMetric("Data Receive Time From Rank " + std::to_string(source_rank), source_metric_data_[source_rank].data_delta_t, "s", 5, MetricMode::Accumulate);
317  metricMan->sendMetric("Data Receive Size From Rank " + std::to_string(source_rank), static_cast<unsigned long>(payloadSize), "B", 5, MetricMode::Accumulate);
318  metricMan->sendMetric("Data Receive Rate From Rank " + std::to_string(source_rank), payloadSize / source_metric_data_[source_rank].data_delta_t, "B/s", 5, MetricMode::Average);
319 
320  metricMan->sendMetric("Data Receive Count From Rank " + std::to_string(source_rank), recv_frag_count_.slotCount(source_rank), "fragments", 3, MetricMode::LastPoint);
321 
322  metricMan->sendMetric("Total Shared Memory Wait Time From Rank " + std::to_string(source_rank), source_metric_data_[source_rank].store_delta_t, "s", 3, MetricMode::Accumulate);
323  metricMan->sendMetric("Avg Shared Memory Wait Time From Rank " + std::to_string(source_rank), source_metric_data_[source_rank].store_delta_t / source_metric_data_[source_rank].data_point_count, "s", 3, MetricMode::Average);
324  metricMan->sendMetric("Avg Fragment Wait Time From Rank " + std::to_string(source_rank), source_metric_data_[source_rank].dead_t / source_metric_data_[source_rank].data_point_count, "s", 3, MetricMode::Average);
325 
326  TLOG(6) << "runReceiver_: Done sending receive stats for rank " << source_rank;
327 
328  source_metric_send_time_[source_rank] = std::chrono::steady_clock::now();
329  source_metric_data_[source_rank] = source_metric_data();
330  }
331 
332  end_time = std::chrono::steady_clock::now();
333  }
334  else if (header.type == Fragment::EndOfDataFragmentType || header.type == Fragment::InitFragmentType || header.type == Fragment::EndOfRunFragmentType || header.type == Fragment::EndOfSubrunFragmentType || header.type == Fragment::ShutdownFragmentType)
335  {
336  TLOG(TLVL_DEBUG) << "Received System Fragment from rank " << source_rank << " of type " << detail::RawFragmentHeader::SystemTypeToString(header.type) << ".";
337 
338  FragmentPtr frag(new Fragment(header.word_count - header.num_words()));
339  memcpy(frag->headerAddress(), &header, header.num_words() * sizeof(RawDataType));
340  auto ret3 = source_plugins_[source_rank]->receiveFragmentData(frag->headerAddress() + header.num_words(), header.word_count - header.num_words());
341  if (ret3 != source_rank)
342  {
343  TLOG(TLVL_ERROR) << "Unexpected return code from receiveFragmentData after receiveFragmentHeader while receiving System Fragment! (Expected: " << source_rank << ", Got: " << ret3 << ")";
344  throw cet::exception("DataReceiverManager") << "Unexpected return code from receiveFragmentData after receiveFragmentHeader while receiving System Fragment! (Expected: " << source_rank << ", Got: " << ret3 << ")";
345  }
346 
347  switch (header.type)
348  {
349  case Fragment::EndOfDataFragmentType:
350  shm_manager_->setRequestMode(detail::RequestMessageMode::EndOfRun);
351  if(endOfDataCount == static_cast<size_t>(-1) ) endOfDataCount = *(frag->dataBegin());
352  else endOfDataCount += *(frag->dataBegin());
353  TLOG(TLVL_DEBUG) << "EndOfData Fragment indicates that " << endOfDataCount << " fragments are expected from rank " << source_rank
354  << " (recvd " << recv_frag_count_.slotCount(source_rank) << ").";
355  break;
356  case Fragment::InitFragmentType:
357  TLOG(TLVL_DEBUG) << "Received Init Fragment from rank " << source_rank << ".";
358  shm_manager_->setRequestMode(detail::RequestMessageMode::Normal);
359  shm_manager_->SetInitFragment(std::move(frag));
360  break;
361  case Fragment::EndOfRunFragmentType:
362  shm_manager_->setRequestMode(detail::RequestMessageMode::EndOfRun);
363  //shm_manager_->endRun();
364  break;
365  case Fragment::EndOfSubrunFragmentType:
366  //shm_manager_->setRequestMode(detail::RequestMessageMode::EndOfRun);
367  TLOG(TLVL_DEBUG) << "Received EndOfSubrun Fragment from rank " << source_rank
368  << " with sequence_id " << header.sequence_id << ".";
369  if (header.sequence_id != Fragment::InvalidSequenceID) shm_manager_->rolloverSubrun(header.sequence_id);
370  else shm_manager_->rolloverSubrun(recv_seq_count_.slotCount(source_rank));
371  break;
372  case Fragment::ShutdownFragmentType:
373  shm_manager_->setRequestMode(detail::RequestMessageMode::EndOfRun);
374  break;
375  }
376  }
377  }
378 
379  TLOG(TLVL_DEBUG) << "runReceiver_ " << source_rank << " receive loop exited";
380  running_sources_[source_rank] = false;
381 }
std::set< int > running_sources() const
Get the list of sources which are still receiving data.
End of Run mode (Used to end request processing on receiver)
hostMap_t MakeHostMap(fhicl::ParameterSet pset, hostMap_t map=hostMap_t())
Make a hostMap_t from a HostMap::Config ParameterSet
Definition: HostMap.hh:66
This TransferInterface is a Receiver.
std::unique_ptr< artdaq::TransferInterface > MakeTransferPlugin(const fhicl::ParameterSet &pset, std::string plugin_label, TransferInterface::Role role)
Load a TransferInterface plugin.
DataReceiverManager(const fhicl::ParameterSet &ps, std::shared_ptr< SharedMemoryEventManager > shm)
DataReceiverManager Constructor.
std::set< int > enabled_sources() const
Get the list of enabled sources.
std::vector< fhicl::ParameterSet > MakeHostMapPset(std::map< int, std::string > input)
Create a list of HostMap::HostConfig ParameterSets from a hostMap_t map
Definition: HostMap.hh:46
Value that is to be returned when a Transfer plugin determines that no more data will be arriving...
virtual ~DataReceiverManager()
DataReceiverManager Destructor.
void start_threads()
Start receiver threads for all enabled sources.
std::map< int, std::string > hostMap_t
The host_map is a map associating ranks with artdaq::DestinationInfo objects.
Definition: HostMap.hh:39
void stop_threads()
Stop receiver threads.