1 #include "test/DAQrate/TransferTest.hh"
3 #include "artdaq-core/Data/Fragment.hh"
4 #include "artdaq/DAQrate/DataReceiverManager.hh"
5 #include "artdaq/DAQrate/DataSenderManager.hh"
7 #include "artdaq/DAQdata/Globals.hh"
9 #include <fhiclcpp/make_ParameterSet.h>
11 #define TRACE_NAME "TransferTest"
13 std::pair<size_t, double> artdaq::TransferTest::do_sending()
15 TRACE(7,
"do_sending entered RawFragmentHeader::num_words()=%lu"
16 , artdaq::detail::RawFragmentHeader::num_words());
22 unsigned data_size_wrds = max_payload_size_ /
sizeof(artdaq::RawDataType) - artdaq::detail::RawFragmentHeader::num_words();
23 if (data_size_wrds < 8) data_size_wrds = 8;
24 artdaq::Fragment frag(data_size_wrds);
26 for (
int ii = 0; ii < sends_each_sender_; ++ii)
28 auto loop_start = std::chrono::steady_clock::now();
29 TRACE(7,
"sender rank %d #%u resized bytes=%ld", my_rank, ii, frag.sizeBytes());
30 totalSize += frag.sizeBytes();
32 unsigned sndDatSz = data_size_wrds;
33 frag.setSequenceID(ii);
34 frag.setFragmentID(my_rank);
35 frag.setSystemType(artdaq::Fragment::DataFragmentType);
37 artdaq::Fragment::iterator it = frag.dataBegin();
42 auto send_start = std::chrono::steady_clock::now();
43 sender.sendFragment(std::move(frag));
44 auto after_send = std::chrono::steady_clock::now();
45 TRACE(1,
"Sender %d sent fragment %d", my_rank, ii);
48 frag = artdaq::Fragment(data_size_wrds);
49 TRACE(9,
"sender rank %d frag replaced", my_rank);
51 auto total_send_time = std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1>>>(after_send - send_start).count();
52 totalTime += total_send_time;
53 if (metricMan && ii % 100 == 0)
55 metricMan->sendMetric(
"send_init_time", std::chrono::duration_cast<std::chrono::duration<
double, std::ratio<1>>>(send_start - loop_start).count(),
"seconds", 3);
56 metricMan->sendMetric(
"total_send_time", total_send_time,
"seconds", 3);
57 metricMan->sendMetric(
"after_send_time", std::chrono::duration_cast<std::chrono::duration<
double, std::ratio<1>>>(std::chrono::steady_clock::now() - after_send).count(),
"seconds", 3);
58 metricMan->sendMetric(
"send_rate", data_size_wrds *
sizeof(artdaq::RawDataType) / total_send_time,
"B/s", 3);
62 return std::make_pair(totalSize, totalTime);
65 std::pair<size_t, double> artdaq::TransferTest::do_receiving()
67 TRACE(7,
"do_receiving entered");
69 receiver.start_threads();
70 int counter = receives_each_receiver_;
74 int activeSenders = senders_;
75 auto last_receive = std::chrono::steady_clock::now();
77 while (activeSenders > 0)
79 auto start_loop = std::chrono::steady_clock::now();
80 TRACE(7,
"TransferTest::do_receiving: Counter is %d, calling recvFragment", counter);
82 auto before_receive = std::chrono::steady_clock::now();
83 auto ignoreFragPtr = receiver.recvFragment(senderSlot);
84 auto after_receive = std::chrono::steady_clock::now();
88 if (ignoreFragPtr->type() == artdaq::Fragment::EndOfDataFragmentType)
90 std::cout <<
"Receiver " << my_rank <<
" received EndOfData Fragment from Sender " << senderSlot << std::endl;
97 start_time_ = std::chrono::steady_clock::now();
101 TRACE(1,
"Receiver %d received fragment %d with seqID %lu from Sender %d (Expecting %d more)"
102 , my_rank, receives_each_receiver_ - counter, ignoreFragPtr->sequenceID(), senderSlot, counter);
103 thisSize = ignoreFragPtr->size() *
sizeof(artdaq::RawDataType);
104 totalSize += thisSize;
108 metricMan->sendMetric(
"input_wait", std::chrono::duration_cast<std::chrono::duration<
double, std::ratio<1>>>(after_receive - last_receive).count(),
"seconds", 3);
110 last_receive = after_receive;
112 TRACE(7,
"TransferTest::do_receiving: Recv Loop end, counter is %d", counter);
113 auto total_recv_time = std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1>>>(after_receive - before_receive).count();
114 totalTime += total_recv_time;
115 if (metricMan && counter % 100 == 0)
117 metricMan->sendMetric(
"recv_init_time", std::chrono::duration_cast<std::chrono::duration<
double, std::ratio<1>>>(before_receive - start_loop).count(),
"seconds", 3);
118 metricMan->sendMetric(
"total_recv_time", total_recv_time,
"seconds", 3);
119 metricMan->sendMetric(
"recv_rate", thisSize / total_recv_time,
"B/s", 3);
123 return std::make_pair(totalSize, totalTime);
127 : senders_(psi.get<int>(
"num_senders"))
128 , receivers_(psi.get<int>(
"num_receivers"))
129 , sends_each_sender_(psi.get<int>(
"sends_per_sender"))
130 , receives_each_receiver_(senders_ * sends_each_sender_ / receivers_)
131 , buffer_count_(psi.get<int>(
"buffer_count", 10))
132 , max_payload_size_(psi.get<size_t>(
"fragment_size", 0x100000))
135 TRACE(10,
"TransferTest CONSTRUCTOR");
136 metricMan = &metricMan_;
138 fhicl::ParameterSet metric_pset;
142 metric_pset = psi.get<fhicl::ParameterSet>(
"metrics");
148 std::string name =
"TransferTest" + std::to_string(my_rank);
149 metricMan_.initialize(metric_pset, name);
150 metricMan_.do_start();
154 std::string type(psi.get<std::string>(
"transfer_plugin_type",
"Shmem"));
158 if (senders_ * sends_each_sender_ % receivers_ != 0)
160 std::cout <<
"Adding sends so that sends_each_sender * num_sending_ranks is a multiple of num_receiving_ranks" << std::endl;
161 while (senders_ * sends_each_sender_ % receivers_ != 0)
163 sends_each_sender_++;
165 receives_each_receiver_ = senders_ * sends_each_sender_ / receivers_;
166 std::cout <<
"sends_each_sender is now " << sends_each_sender_ << std::endl;
167 psi.put_or_replace(
"sends_per_sender", sends_each_sender_);
171 std::string hostmap =
"";
172 if (psi.has_key(
"hostmap"))
174 hostmap =
" host_map: @local::hostmap";
177 std::stringstream ss;
178 ss << psi.to_string();
180 for (
int ii = 0; ii < senders_; ++ii)
182 ss <<
"s" << ii <<
": { transferPluginType: " << type <<
" source_rank: " << ii <<
" max_fragment_size_words: " << max_payload_size_ <<
" buffer_count: " << buffer_count_ << hostmap <<
"}";
184 ss <<
"} destinations: {";
185 for (
int jj = senders_; jj < senders_ + receivers_; ++jj)
187 ss <<
"d" << jj <<
": { transferPluginType: " << type <<
" destination_rank: " << jj <<
" max_fragment_size_words: " << max_payload_size_ <<
" buffer_count: " << buffer_count_ << hostmap <<
"}";
191 make_ParameterSet(ss.str(), ps_);
194 std::cout <<
"Going to configure with ParameterSet: " << ps_.to_string() << std::endl;
199 TRACE(11,
"TransferTest::runTest BEGIN");
200 start_time_ = std::chrono::steady_clock::now();
201 std::pair<size_t, double> result;
202 if (my_rank < senders_)
204 result = do_sending();
208 result = do_receiving();
210 auto duration = std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1>>>(std::chrono::steady_clock::now() - start_time_).count();
211 std::cout << (my_rank < senders_ ?
"Sent " :
"Received ") << result.first <<
" bytes in " << duration <<
" seconds ( " << formatBytes(result.first / duration) <<
"/s )." << std::endl;
212 std::cout <<
"Rate of " << (my_rank < senders_ ?
"sending" :
"receiving") <<
": " << formatBytes(result.first / result.second) <<
"/s." << std::endl;
213 metricMan_.do_stop();
214 metricMan_.shutdown();
215 TRACE(11,
"TransferTest::runTest DONE");
int runTest()
Run the test as configured.
Sends Fragment objects using TransferInterface plugins. Uses Routing Tables if confgiured, otherwise will Round-Robin Fragments to the destinations.
static const int RECV_TIMEOUT
Value to be returned upon receive timeout. Because receivers otherwise return rank, this is also the limit on the number of ranks that artdaq currently supports.
Receives Fragment objects from one or more DataSenderManager instances using TransferInterface plugin...
TransferTest(fhicl::ParameterSet psi)
TransferTest Constructor.