artdaq  v2_03_02
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Pages
TransferTest.cc
1 #include "proto/TransferTest.hh"
2 
3 #include "artdaq-core/Data/Fragment.hh"
4 #include "artdaq/DAQrate/DataReceiverManager.hh"
5 #include "artdaq/DAQrate/DataSenderManager.hh"
6 
7 #include "artdaq/DAQdata/Globals.hh"
8 
9 #include "fhiclcpp/make_ParameterSet.h"
10 
11 #define TRACE_NAME "TransferTest"
12 
13 artdaq::TransferTest::TransferTest(fhicl::ParameterSet psi)
14  : senders_(psi.get<int>("num_senders"))
15  , receivers_(psi.get<int>("num_receivers"))
16  , sends_each_sender_(psi.get<int>("sends_per_sender"))
17  , receives_each_receiver_(senders_ * sends_each_sender_ / receivers_)
18  , buffer_count_(psi.get<int>("buffer_count", 10))
19  , max_payload_size_(psi.get<size_t>("fragment_size", 0x100000))
20  , ps_()
21  , validate_mode_(psi.get<bool>("validate_data_mode", false))
22 {
23  TRACE(10, "TransferTest CONSTRUCTOR");
24  metricMan = &metricMan_;
25 
26  fhicl::ParameterSet metric_pset;
27 
28  try
29  {
30  metric_pset = psi.get<fhicl::ParameterSet>("metrics");
31  }
32  catch (...) {} // OK if there's no metrics table defined in the FHiCL
33 
34  try
35  {
36  std::string name = "TransferTest" + std::to_string(my_rank);
37  metricMan_.initialize(metric_pset, name);
38  metricMan_.do_start();
39  }
40  catch (...) {}
41 
42  std::string type(psi.get<std::string>("transfer_plugin_type", "Shmem"));
43 
44  if (receivers_ > 0)
45  {
46  if (senders_ * sends_each_sender_ % receivers_ != 0)
47  {
48  std::cout << "Adding sends so that sends_each_sender * num_sending_ranks is a multiple of num_receiving_ranks" << std::endl;
49  while (senders_ * sends_each_sender_ % receivers_ != 0)
50  {
51  sends_each_sender_++;
52  }
53  receives_each_receiver_ = senders_ * sends_each_sender_ / receivers_;
54  std::cout << "sends_each_sender is now " << sends_each_sender_ << std::endl;
55  psi.put_or_replace("sends_per_sender", sends_each_sender_);
56  }
57  }
58 
59  std::string hostmap = "";
60  if (psi.has_key("hostmap"))
61  {
62  hostmap = " host_map: @local::hostmap";
63  }
64 
65  std::stringstream ss;
66  ss << psi.to_string();
67  ss << " sources: {";
68  for (int ii = 0; ii < senders_; ++ii)
69  {
70  ss << "s" << ii << ": { transferPluginType: " << type << " source_rank: " << ii << " max_fragment_size_words: " << max_payload_size_ << " buffer_count: " << buffer_count_ << hostmap << "}";
71  }
72  ss << "} destinations: {";
73  for (int jj = senders_; jj < senders_ + receivers_; ++jj)
74  {
75  ss << "d" << jj << ": { transferPluginType: " << type << " destination_rank: " << jj << " max_fragment_size_words: " << max_payload_size_ << " buffer_count: " << buffer_count_ << hostmap << "}";
76  }
77  ss << "}";
78 
79  make_ParameterSet(ss.str(), ps_);
80 
81 
82  std::cout << "Going to configure with ParameterSet: " << ps_.to_string() << std::endl;
83 }
84 
86 {
87  TRACE(11, "TransferTest::runTest BEGIN");
88  start_time_ = std::chrono::steady_clock::now();
89  std::pair<size_t, double> result;
90  if (my_rank < senders_)
91  {
92  result = do_sending();
93  }
94  else
95  {
96  result = do_receiving();
97  }
98  auto duration = std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1>>>(std::chrono::steady_clock::now() - start_time_).count();
99  std::cout << (my_rank < senders_ ? "Sent " : "Received ") << result.first << " bytes in " << duration << " seconds ( " << formatBytes(result.first / duration) << "/s )." << std::endl;
100  std::cout << "Rate of " << (my_rank < senders_ ? "sending" : "receiving") << ": " << formatBytes(result.first / result.second) << "/s." << std::endl;
101  metricMan_.do_stop();
102  metricMan_.shutdown();
103  TRACE(11, "TransferTest::runTest DONE");
104  return 0;
105 }
106 
107 std::pair<size_t, double> artdaq::TransferTest::do_sending()
108 {
109  TRACE(7, "do_sending entered RawFragmentHeader::num_words()=%lu"
110  , artdaq::detail::RawFragmentHeader::num_words());
111 
112  size_t totalSize = 0;
113  double totalTime = 0;
114  artdaq::DataSenderManager sender(ps_);
115 
116  unsigned data_size_wrds = max_payload_size_ / sizeof(artdaq::RawDataType) - artdaq::detail::RawFragmentHeader::num_words();
117  if (data_size_wrds < 8) data_size_wrds = 8; // min size
118  artdaq::Fragment frag(data_size_wrds);
119 
120  if (validate_mode_)
121  {
122  artdaq::RawDataType gen_seed = 0;
123 
124  std::generate_n(frag.dataBegin(), data_size_wrds, [&]() { return ++gen_seed; });
125  for (size_t ii = 0; ii < frag.dataSize(); ++ii)
126  {
127  if (*(frag.dataBegin() + ii) != ii + 1)
128  {
129  TLOG_ERROR("TransferTest") << "Data corruption detected! (" << std::to_string(*(frag.dataBegin() + ii)) << " != " << std::to_string(ii + 1) << ") Aborting!" << TLOG_ENDL;
130  exit(1);
131  }
132  }
133  }
134 
135  for (int ii = 0; ii < sends_each_sender_; ++ii)
136  {
137  auto loop_start = std::chrono::steady_clock::now();
138  TRACE(7, "sender rank %d #%u resized bytes=%ld", my_rank, ii, frag.sizeBytes());
139  totalSize += frag.sizeBytes();
140 
141  //unsigned sndDatSz = data_size_wrds;
142  frag.setSequenceID(ii);
143  frag.setFragmentID(my_rank);
144  frag.setSystemType(artdaq::Fragment::DataFragmentType);
145 /*
146  artdaq::Fragment::iterator it = frag.dataBegin();
147  *it = my_rank;
148  *++it = ii;
149  *++it = sndDatSz;*/
150 
151  auto send_start = std::chrono::steady_clock::now();
152  sender.sendFragment(std::move(frag));
153  auto after_send = std::chrono::steady_clock::now();
154  TRACE(1, "Sender %d sent fragment %d", my_rank, ii);
155  //usleep( (data_size_wrds*sizeof(artdaq::RawDataType))/233 );
156 
157  frag = artdaq::Fragment(data_size_wrds); // replace/renew
158  if (validate_mode_)
159  {
160  artdaq::RawDataType gen_seed = 0;
161 
162  std::generate_n(frag.dataBegin(), data_size_wrds, [&]() { return ++gen_seed; });
163  for (size_t ii = 0; ii < frag.dataSize(); ++ii)
164  {
165  if (*(frag.dataBegin() + ii) != ii + 1)
166  {
167  TLOG_ERROR("TransferTest") << "Data corruption detected! (" << std::to_string(*(frag.dataBegin() + ii)) << " != " << std::to_string(ii + 1) << ") Aborting!" << TLOG_ENDL;
168  exit(1);
169  }
170  }
171  }
172  TRACE(9, "sender rank %d frag replaced", my_rank);
173 
174  auto total_send_time = std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1>>>(after_send - send_start).count();
175  totalTime += total_send_time;
176  if (metricMan && ii % 100 == 0)
177  {
178  metricMan->sendMetric("send_init_time", std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1>>>(send_start - loop_start).count(), "seconds", 3);
179  metricMan->sendMetric("total_send_time", total_send_time, "seconds", 3);
180  metricMan->sendMetric("after_send_time", std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1>>>(std::chrono::steady_clock::now() - after_send).count(), "seconds", 3);
181  metricMan->sendMetric("send_rate", data_size_wrds * sizeof(artdaq::RawDataType) / total_send_time, "B/s", 3);
182  }
183  }
184 
185  return std::make_pair(totalSize, totalTime);
186 } // do_sending
187 
188 std::pair<size_t, double> artdaq::TransferTest::do_receiving()
189 {
190  TRACE(7, "do_receiving entered");
191  artdaq::DataReceiverManager receiver(ps_);
192  receiver.start_threads();
193  int counter = receives_each_receiver_;
194  size_t totalSize = 0;
195  double totalTime = 0;
196  bool first = true;
197  int activeSenders = senders_;
198  auto end_loop = std::chrono::steady_clock::now();
199 
200  while (activeSenders > 0)
201  {
202  auto start_loop = std::chrono::steady_clock::now();
203  TRACE(7, "TransferTest::do_receiving: Counter is %d, calling recvFragment", counter);
205  auto before_receive = std::chrono::steady_clock::now();
206  auto ignoreFragPtr = receiver.recvFragment(senderSlot);
207  auto after_receive = std::chrono::steady_clock::now();
208  size_t thisSize = 0;
209  if (senderSlot != artdaq::TransferInterface::RECV_TIMEOUT && ignoreFragPtr)
210  {
211  if (ignoreFragPtr->type() == artdaq::Fragment::EndOfDataFragmentType)
212  {
213  std::cout << "Receiver " << my_rank << " received EndOfData Fragment from Sender " << senderSlot << std::endl;
214  activeSenders--;
215  }
216  else
217  {
218  if (first)
219  {
220  start_time_ = std::chrono::steady_clock::now();
221  first = false;
222  }
223  counter--;
224  TRACE(1, "Receiver %d received fragment %d with seqID %lu from Sender %d (Expecting %d more)"
225  , my_rank, receives_each_receiver_ - counter, ignoreFragPtr->sequenceID(), senderSlot, counter);
226  thisSize = ignoreFragPtr->size() * sizeof(artdaq::RawDataType);
227  totalSize += thisSize;
228  if (validate_mode_)
229  {
230  for (size_t ii = 0; ii < ignoreFragPtr->dataSize(); ++ii)
231  {
232  if (*(ignoreFragPtr->dataBegin() + ii) != ii + 1)
233  {
234  TLOG_ERROR("TransferTest") << "Data corruption detected! (" << std::to_string(*(ignoreFragPtr->dataBegin() + ii)) << " != " << std::to_string(ii + 1) << ") Aborting!" << TLOG_ENDL;
235  exit(1);
236  }
237  }
238  }
239  }
240  if (metricMan)
241  {
242  metricMan->sendMetric("input_wait", std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1>>>(after_receive - end_loop).count(), "seconds", 3);
243  }
244 
245  }
246  TRACE(7, "TransferTest::do_receiving: Recv Loop end, counter is %d", counter);
247  auto total_recv_time = std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1>>>(after_receive - before_receive).count();
248  totalTime += total_recv_time;
249  if (metricMan && counter % 100 == 0)
250  {
251  metricMan->sendMetric("recv_init_time", std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1>>>(before_receive - start_loop).count(), "seconds", 3);
252  metricMan->sendMetric("total_recv_time", total_recv_time, "seconds", 3);
253  metricMan->sendMetric("recv_rate", thisSize / total_recv_time, "B/s", 3);
254  }
255  end_loop = std::chrono::steady_clock::now();
256  }
257 
258  return std::make_pair(totalSize, totalTime);
259 }
int runTest()
Run the test as configured.
Definition: TransferTest.cc:85
Sends Fragment objects using TransferInterface plugins. Uses Routing Tables if confgiured, otherwise will Round-Robin Fragments to the destinations.
static const int RECV_TIMEOUT
Value to be returned upon receive timeout. Because receivers otherwise return rank, this is also the limit on the number of ranks that artdaq currently supports.
Receives Fragment objects from one or more DataSenderManager instances using TransferInterface plugin...
TransferTest(fhicl::ParameterSet psi)
TransferTest Constructor.
Definition: TransferTest.cc:13