artdaq  v2_02_03
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Pages
TransferTest.cc
1 #include "test/DAQrate/TransferTest.hh"
2 
3 #include "artdaq-core/Data/Fragment.hh"
4 #include "artdaq/DAQrate/DataReceiverManager.hh"
5 #include "artdaq/DAQrate/DataSenderManager.hh"
6 
7 #include "artdaq/DAQdata/Globals.hh"
8 
9 #include <fhiclcpp/make_ParameterSet.h>
10 
11 #define TRACE_NAME "TransferTest"
12 
13 std::pair<size_t, double> artdaq::TransferTest::do_sending()
14 {
15  TRACE(7, "do_sending entered RawFragmentHeader::num_words()=%lu"
16  , artdaq::detail::RawFragmentHeader::num_words());
17 
18  size_t totalSize = 0;
19  double totalTime = 0;
20  artdaq::DataSenderManager sender(ps_);
21 
22  unsigned data_size_wrds = max_payload_size_ / sizeof(artdaq::RawDataType) - artdaq::detail::RawFragmentHeader::num_words();
23  if (data_size_wrds < 8) data_size_wrds = 8; // min size
24  artdaq::Fragment frag(data_size_wrds);
25 
26  for (int ii = 0; ii < sends_each_sender_; ++ii)
27  {
28  auto loop_start = std::chrono::steady_clock::now();
29  TRACE(7, "sender rank %d #%u resized bytes=%ld", my_rank, ii, frag.sizeBytes());
30  totalSize += frag.sizeBytes();
31 
32  unsigned sndDatSz = data_size_wrds;
33  frag.setSequenceID(ii);
34  frag.setFragmentID(my_rank);
35  frag.setSystemType(artdaq::Fragment::DataFragmentType);
36 
37  artdaq::Fragment::iterator it = frag.dataBegin();
38  *it = my_rank;
39  *++it = ii;
40  *++it = sndDatSz;
41 
42  auto send_start = std::chrono::steady_clock::now();
43  sender.sendFragment(std::move(frag));
44  auto after_send = std::chrono::steady_clock::now();
45  TRACE(1, "Sender %d sent fragment %d", my_rank, ii);
46  //usleep( (data_size_wrds*sizeof(artdaq::RawDataType))/233 );
47 
48  frag = artdaq::Fragment(data_size_wrds); // replace/renew
49  TRACE(9, "sender rank %d frag replaced", my_rank);
50 
51  auto total_send_time = std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1>>>(after_send - send_start).count();
52  totalTime += total_send_time;
53  if (metricMan && ii % 100 == 0)
54  {
55  metricMan->sendMetric("send_init_time", std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1>>>(send_start - loop_start).count(), "seconds", 3);
56  metricMan->sendMetric("total_send_time", total_send_time, "seconds", 3);
57  metricMan->sendMetric("after_send_time", std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1>>>(std::chrono::steady_clock::now() - after_send).count(), "seconds", 3);
58  metricMan->sendMetric("send_rate", data_size_wrds * sizeof(artdaq::RawDataType) / total_send_time, "B/s", 3);
59  }
60  }
61 
62  return std::make_pair(totalSize, totalTime);
63 } // do_sending
64 
65 std::pair<size_t, double> artdaq::TransferTest::do_receiving()
66 {
67  TRACE(7, "do_receiving entered");
68  artdaq::DataReceiverManager receiver(ps_);
69  receiver.start_threads();
70  int counter = receives_each_receiver_;
71  size_t totalSize = 0;
72  double totalTime = 0;
73  bool first = true;
74  int activeSenders = senders_;
75  auto last_receive = std::chrono::steady_clock::now();
76 
77  while (activeSenders > 0)
78  {
79  auto start_loop = std::chrono::steady_clock::now();
80  TRACE(7, "TransferTest::do_receiving: Counter is %d, calling recvFragment", counter);
82  auto before_receive = std::chrono::steady_clock::now();
83  auto ignoreFragPtr = receiver.recvFragment(senderSlot);
84  auto after_receive = std::chrono::steady_clock::now();
85  size_t thisSize = 0;
86  if (senderSlot != artdaq::TransferInterface::RECV_TIMEOUT && ignoreFragPtr)
87  {
88  if (ignoreFragPtr->type() == artdaq::Fragment::EndOfDataFragmentType)
89  {
90  std::cout << "Receiver " << my_rank << " received EndOfData Fragment from Sender " << senderSlot << std::endl;
91  activeSenders--;
92  }
93  else
94  {
95  if (first)
96  {
97  start_time_ = std::chrono::steady_clock::now();
98  first = false;
99  }
100  counter--;
101  TRACE(1, "Receiver %d received fragment %d with seqID %lu from Sender %d (Expecting %d more)"
102  , my_rank, receives_each_receiver_ - counter, ignoreFragPtr->sequenceID(), senderSlot, counter);
103  thisSize = ignoreFragPtr->size() * sizeof(artdaq::RawDataType);
104  totalSize += thisSize;
105  }
106  if (metricMan)
107  {
108  metricMan->sendMetric("input_wait", std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1>>>(after_receive - last_receive).count(), "seconds", 3);
109  }
110  last_receive = after_receive;
111  }
112  TRACE(7, "TransferTest::do_receiving: Recv Loop end, counter is %d", counter);
113  auto total_recv_time = std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1>>>(after_receive - before_receive).count();
114  totalTime += total_recv_time;
115  if (metricMan && counter % 100 == 0)
116  {
117  metricMan->sendMetric("recv_init_time", std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1>>>(before_receive - start_loop).count(), "seconds", 3);
118  metricMan->sendMetric("total_recv_time", total_recv_time, "seconds", 3);
119  metricMan->sendMetric("recv_rate", thisSize / total_recv_time, "B/s", 3);
120  }
121  }
122 
123  return std::make_pair(totalSize, totalTime);
124 }
125 
126 artdaq::TransferTest::TransferTest(fhicl::ParameterSet psi)
127  : senders_(psi.get<int>("num_senders"))
128  , receivers_(psi.get<int>("num_receivers"))
129  , sends_each_sender_(psi.get<int>("sends_per_sender"))
130  , receives_each_receiver_(senders_ * sends_each_sender_ / receivers_)
131  , buffer_count_(psi.get<int>("buffer_count", 10))
132  , max_payload_size_(psi.get<size_t>("fragment_size", 0x100000))
133  , ps_()
134 {
135  TRACE(10, "TransferTest CONSTRUCTOR");
136  metricMan = &metricMan_;
137 
138  fhicl::ParameterSet metric_pset;
139 
140  try
141  {
142  metric_pset = psi.get<fhicl::ParameterSet>("metrics");
143  }
144  catch (...) {} // OK if there's no metrics table defined in the FHiCL
145 
146  try
147  {
148  std::string name = "TransferTest" + std::to_string(my_rank);
149  metricMan_.initialize(metric_pset, name);
150  metricMan_.do_start();
151  }
152  catch (...) {}
153 
154  std::string type(psi.get<std::string>("transfer_plugin_type", "Shmem"));
155 
156  if (receivers_ > 0)
157  {
158  if (senders_ * sends_each_sender_ % receivers_ != 0)
159  {
160  std::cout << "Adding sends so that sends_each_sender * num_sending_ranks is a multiple of num_receiving_ranks" << std::endl;
161  while (senders_ * sends_each_sender_ % receivers_ != 0)
162  {
163  sends_each_sender_++;
164  }
165  receives_each_receiver_ = senders_ * sends_each_sender_ / receivers_;
166  std::cout << "sends_each_sender is now " << sends_each_sender_ << std::endl;
167  psi.put_or_replace("sends_per_sender", sends_each_sender_);
168  }
169  }
170 
171  std::string hostmap = "";
172  if (psi.has_key("hostmap"))
173  {
174  hostmap = " host_map: @local::hostmap";
175  }
176 
177  std::stringstream ss;
178  ss << psi.to_string();
179  ss << " sources: {";
180  for (int ii = 0; ii < senders_; ++ii)
181  {
182  ss << "s" << ii << ": { transferPluginType: " << type << " source_rank: " << ii << " max_fragment_size_words: " << max_payload_size_ << " buffer_count: " << buffer_count_ << hostmap << "}";
183  }
184  ss << "} destinations: {";
185  for (int jj = senders_; jj < senders_ + receivers_; ++jj)
186  {
187  ss << "d" << jj << ": { transferPluginType: " << type << " destination_rank: " << jj << " max_fragment_size_words: " << max_payload_size_ << " buffer_count: " << buffer_count_ << hostmap << "}";
188  }
189  ss << "}";
190 
191  make_ParameterSet(ss.str(), ps_);
192 
193 
194  std::cout << "Going to configure with ParameterSet: " << ps_.to_string() << std::endl;
195 }
196 
198 {
199  TRACE(11, "TransferTest::runTest BEGIN");
200  start_time_ = std::chrono::steady_clock::now();
201  std::pair<size_t, double> result;
202  if (my_rank < senders_)
203  {
204  result = do_sending();
205  }
206  else
207  {
208  result = do_receiving();
209  }
210  auto duration = std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1>>>(std::chrono::steady_clock::now() - start_time_).count();
211  std::cout << (my_rank < senders_ ? "Sent " : "Received ") << result.first << " bytes in " << duration << " seconds ( " << formatBytes(result.first / duration) << "/s )." << std::endl;
212  std::cout << "Rate of " << (my_rank < senders_ ? "sending" : "receiving") << ": " << formatBytes(result.first / result.second) << "/s." << std::endl;
213  metricMan_.do_stop();
214  metricMan_.shutdown();
215  TRACE(11, "TransferTest::runTest DONE");
216  return 0;
217 }
int runTest()
Run the test as configured.
Sends Fragment objects using TransferInterface plugins. Uses Routing Tables if confgiured, otherwise will Round-Robin Fragments to the destinations.
static const int RECV_TIMEOUT
Value to be returned upon receive timeout. Because receivers otherwise return rank, this is also the limit on the number of ranks that artdaq currently supports.
Receives Fragment objects from one or more DataSenderManager instances using TransferInterface plugin...
TransferTest(fhicl::ParameterSet psi)
TransferTest Constructor.