1 #include "proto/TransferTest.hh"
3 #include "artdaq-core/Data/Fragment.hh"
4 #include "artdaq/DAQrate/DataReceiverManager.hh"
5 #include "artdaq/DAQrate/DataSenderManager.hh"
7 #include "artdaq/DAQdata/Globals.hh"
9 #include "fhiclcpp/make_ParameterSet.h"
11 #define TRACE_NAME "TransferTest"
14 : senders_(psi.get<int>(
"num_senders"))
15 , receivers_(psi.get<int>(
"num_receivers"))
16 , sends_each_sender_(psi.get<int>(
"sends_per_sender"))
17 , receives_each_receiver_(senders_ * sends_each_sender_ / receivers_)
18 , buffer_count_(psi.get<int>(
"buffer_count", 10))
19 , max_payload_size_(psi.get<size_t>(
"fragment_size", 0x100000))
21 , validate_mode_(psi.get<bool>(
"validate_data_mode", false))
23 TRACE(10,
"TransferTest CONSTRUCTOR");
24 metricMan = &metricMan_;
26 fhicl::ParameterSet metric_pset;
30 metric_pset = psi.get<fhicl::ParameterSet>(
"metrics");
36 std::string name =
"TransferTest" + std::to_string(my_rank);
37 metricMan_.initialize(metric_pset, name);
38 metricMan_.do_start();
42 std::string type(psi.get<std::string>(
"transfer_plugin_type",
"Shmem"));
46 if (senders_ * sends_each_sender_ % receivers_ != 0)
48 std::cout <<
"Adding sends so that sends_each_sender * num_sending_ranks is a multiple of num_receiving_ranks" << std::endl;
49 while (senders_ * sends_each_sender_ % receivers_ != 0)
53 receives_each_receiver_ = senders_ * sends_each_sender_ / receivers_;
54 std::cout <<
"sends_each_sender is now " << sends_each_sender_ << std::endl;
55 psi.put_or_replace(
"sends_per_sender", sends_each_sender_);
59 std::string hostmap =
"";
60 if (psi.has_key(
"hostmap"))
62 hostmap =
" host_map: @local::hostmap";
66 ss << psi.to_string();
68 for (
int ii = 0; ii < senders_; ++ii)
70 ss <<
"s" << ii <<
": { transferPluginType: " << type <<
" source_rank: " << ii <<
" max_fragment_size_words: " << max_payload_size_ <<
" buffer_count: " << buffer_count_ << hostmap <<
"}";
72 ss <<
"} destinations: {";
73 for (
int jj = senders_; jj < senders_ + receivers_; ++jj)
75 ss <<
"d" << jj <<
": { transferPluginType: " << type <<
" destination_rank: " << jj <<
" max_fragment_size_words: " << max_payload_size_ <<
" buffer_count: " << buffer_count_ << hostmap <<
"}";
79 make_ParameterSet(ss.str(), ps_);
82 std::cout <<
"Going to configure with ParameterSet: " << ps_.to_string() << std::endl;
87 TRACE(11,
"TransferTest::runTest BEGIN");
88 start_time_ = std::chrono::steady_clock::now();
89 std::pair<size_t, double> result;
90 if (my_rank < senders_)
92 result = do_sending();
96 result = do_receiving();
98 auto duration = std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1>>>(std::chrono::steady_clock::now() - start_time_).count();
99 std::cout << (my_rank < senders_ ?
"Sent " :
"Received ") << result.first <<
" bytes in " << duration <<
" seconds ( " << formatBytes(result.first / duration) <<
"/s )." << std::endl;
100 std::cout <<
"Rate of " << (my_rank < senders_ ?
"sending" :
"receiving") <<
": " << formatBytes(result.first / result.second) <<
"/s." << std::endl;
101 metricMan_.do_stop();
102 metricMan_.shutdown();
103 TRACE(11,
"TransferTest::runTest DONE");
107 std::pair<size_t, double> artdaq::TransferTest::do_sending()
109 TRACE(7,
"do_sending entered RawFragmentHeader::num_words()=%lu"
110 , artdaq::detail::RawFragmentHeader::num_words());
112 size_t totalSize = 0;
113 double totalTime = 0;
116 unsigned data_size_wrds = max_payload_size_ /
sizeof(artdaq::RawDataType) - artdaq::detail::RawFragmentHeader::num_words();
117 if (data_size_wrds < 8) data_size_wrds = 8;
118 artdaq::Fragment frag(data_size_wrds);
122 artdaq::RawDataType gen_seed = 0;
124 std::generate_n(frag.dataBegin(), data_size_wrds, [&]() {
return ++gen_seed; });
125 for (
size_t ii = 0; ii < frag.dataSize(); ++ii)
127 if (*(frag.dataBegin() + ii) != ii + 1)
129 TLOG_ERROR(
"TransferTest") <<
"Data corruption detected! (" << std::to_string(*(frag.dataBegin() + ii)) <<
" != " << std::to_string(ii + 1) <<
") Aborting!" << TLOG_ENDL;
135 for (
int ii = 0; ii < sends_each_sender_; ++ii)
137 auto loop_start = std::chrono::steady_clock::now();
138 TRACE(7,
"sender rank %d #%u resized bytes=%ld", my_rank, ii, frag.sizeBytes());
139 totalSize += frag.sizeBytes();
142 frag.setSequenceID(ii);
143 frag.setFragmentID(my_rank);
144 frag.setSystemType(artdaq::Fragment::DataFragmentType);
151 auto send_start = std::chrono::steady_clock::now();
152 sender.sendFragment(std::move(frag));
153 auto after_send = std::chrono::steady_clock::now();
154 TRACE(1,
"Sender %d sent fragment %d", my_rank, ii);
157 frag = artdaq::Fragment(data_size_wrds);
160 artdaq::RawDataType gen_seed = 0;
162 std::generate_n(frag.dataBegin(), data_size_wrds, [&]() {
return ++gen_seed; });
163 for (
size_t ii = 0; ii < frag.dataSize(); ++ii)
165 if (*(frag.dataBegin() + ii) != ii + 1)
167 TLOG_ERROR(
"TransferTest") <<
"Data corruption detected! (" << std::to_string(*(frag.dataBegin() + ii)) <<
" != " << std::to_string(ii + 1) <<
") Aborting!" << TLOG_ENDL;
172 TRACE(9,
"sender rank %d frag replaced", my_rank);
174 auto total_send_time = std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1>>>(after_send - send_start).count();
175 totalTime += total_send_time;
176 if (metricMan && ii % 100 == 0)
178 metricMan->sendMetric(
"send_init_time", std::chrono::duration_cast<std::chrono::duration<
double, std::ratio<1>>>(send_start - loop_start).count(),
"seconds", 3);
179 metricMan->sendMetric(
"total_send_time", total_send_time,
"seconds", 3);
180 metricMan->sendMetric(
"after_send_time", std::chrono::duration_cast<std::chrono::duration<
double, std::ratio<1>>>(std::chrono::steady_clock::now() - after_send).count(),
"seconds", 3);
181 metricMan->sendMetric(
"send_rate", data_size_wrds *
sizeof(artdaq::RawDataType) / total_send_time,
"B/s", 3);
185 return std::make_pair(totalSize, totalTime);
188 std::pair<size_t, double> artdaq::TransferTest::do_receiving()
190 TRACE(7,
"do_receiving entered");
192 receiver.start_threads();
193 int counter = receives_each_receiver_;
194 size_t totalSize = 0;
195 double totalTime = 0;
197 int activeSenders = senders_;
198 auto end_loop = std::chrono::steady_clock::now();
200 while (activeSenders > 0)
202 auto start_loop = std::chrono::steady_clock::now();
203 TRACE(7,
"TransferTest::do_receiving: Counter is %d, calling recvFragment", counter);
205 auto before_receive = std::chrono::steady_clock::now();
206 auto ignoreFragPtr = receiver.recvFragment(senderSlot);
207 auto after_receive = std::chrono::steady_clock::now();
211 if (ignoreFragPtr->type() == artdaq::Fragment::EndOfDataFragmentType)
213 std::cout <<
"Receiver " << my_rank <<
" received EndOfData Fragment from Sender " << senderSlot << std::endl;
220 start_time_ = std::chrono::steady_clock::now();
224 TRACE(1,
"Receiver %d received fragment %d with seqID %lu from Sender %d (Expecting %d more)"
225 , my_rank, receives_each_receiver_ - counter, ignoreFragPtr->sequenceID(), senderSlot, counter);
226 thisSize = ignoreFragPtr->size() *
sizeof(artdaq::RawDataType);
227 totalSize += thisSize;
230 for (
size_t ii = 0; ii < ignoreFragPtr->dataSize(); ++ii)
232 if (*(ignoreFragPtr->dataBegin() + ii) != ii + 1)
234 TLOG_ERROR(
"TransferTest") <<
"Data corruption detected! (" << std::to_string(*(ignoreFragPtr->dataBegin() + ii)) <<
" != " << std::to_string(ii + 1) <<
") Aborting!" << TLOG_ENDL;
242 metricMan->sendMetric(
"input_wait", std::chrono::duration_cast<std::chrono::duration<
double, std::ratio<1>>>(after_receive - end_loop).count(),
"seconds", 3);
246 TRACE(7,
"TransferTest::do_receiving: Recv Loop end, counter is %d", counter);
247 auto total_recv_time = std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1>>>(after_receive - before_receive).count();
248 totalTime += total_recv_time;
249 if (metricMan && counter % 100 == 0)
251 metricMan->sendMetric(
"recv_init_time", std::chrono::duration_cast<std::chrono::duration<
double, std::ratio<1>>>(before_receive - start_loop).count(),
"seconds", 3);
252 metricMan->sendMetric(
"total_recv_time", total_recv_time,
"seconds", 3);
253 metricMan->sendMetric(
"recv_rate", thisSize / total_recv_time,
"B/s", 3);
255 end_loop = std::chrono::steady_clock::now();
258 return std::make_pair(totalSize, totalTime);
int runTest()
Run the test as configured.
Sends Fragment objects using TransferInterface plugins. Uses Routing Tables if confgiured, otherwise will Round-Robin Fragments to the destinations.
static const int RECV_TIMEOUT
Value to be returned upon receive timeout. Because receivers otherwise return rank, this is also the limit on the number of ranks that artdaq currently supports.
Receives Fragment objects from one or more DataSenderManager instances using TransferInterface plugin...
TransferTest(fhicl::ParameterSet psi)
TransferTest Constructor.