00001 #include "test/DAQrate/TransferTest.hh"
00002
00003 #include "artdaq-core/Data/Fragment.hh"
00004 #include "artdaq/DAQrate/DataReceiverManager.hh"
00005 #include "artdaq/DAQrate/DataSenderManager.hh"
00006
00007 #include "artdaq/DAQdata/Globals.hh"
00008
00009 #include <fhiclcpp/make_ParameterSet.h>
00010
00011 #define TRACE_NAME "TransferTest"
00012
00013 std::pair<size_t, double> artdaq::TransferTest::do_sending()
00014 {
00015 TRACE(7, "do_sending entered RawFragmentHeader::num_words()=%lu"
00016 , artdaq::detail::RawFragmentHeader::num_words());
00017
00018 size_t totalSize = 0;
00019 double totalTime = 0;
00020 artdaq::DataSenderManager sender(ps_);
00021
00022 unsigned data_size_wrds = max_payload_size_ / sizeof(artdaq::RawDataType) - artdaq::detail::RawFragmentHeader::num_words();
00023 if (data_size_wrds < 8) data_size_wrds = 8;
00024 artdaq::Fragment frag(data_size_wrds);
00025
00026 for (int ii = 0; ii < sends_each_sender_; ++ii)
00027 {
00028 auto loop_start = std::chrono::steady_clock::now();
00029 TRACE(7, "sender rank %d #%u resized bytes=%ld", my_rank, ii, frag.sizeBytes());
00030 totalSize += frag.sizeBytes();
00031
00032 unsigned sndDatSz = data_size_wrds;
00033 frag.setSequenceID(ii);
00034 frag.setFragmentID(my_rank);
00035 frag.setSystemType(artdaq::Fragment::DataFragmentType);
00036
00037 artdaq::Fragment::iterator it = frag.dataBegin();
00038 *it = my_rank;
00039 *++it = ii;
00040 *++it = sndDatSz;
00041
00042 auto send_start = std::chrono::steady_clock::now();
00043 sender.sendFragment(std::move(frag));
00044 auto after_send = std::chrono::steady_clock::now();
00045 TRACE(1, "Sender %d sent fragment %d", my_rank, ii);
00046
00047
00048 frag = artdaq::Fragment(data_size_wrds);
00049 TRACE(9, "sender rank %d frag replaced", my_rank);
00050
00051 auto total_send_time = std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1>>>(after_send - send_start).count();
00052 totalTime += total_send_time;
00053 if (metricMan && ii % 100 == 0)
00054 {
00055 metricMan->sendMetric("send_init_time", std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1>>>(send_start - loop_start).count(), "seconds", 3);
00056 metricMan->sendMetric("total_send_time", total_send_time, "seconds", 3);
00057 metricMan->sendMetric("after_send_time", std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1>>>(std::chrono::steady_clock::now() - after_send).count(), "seconds", 3);
00058 metricMan->sendMetric("send_rate", data_size_wrds * sizeof(artdaq::RawDataType) / total_send_time, "B/s", 3);
00059 }
00060 }
00061
00062 return std::make_pair(totalSize, totalTime);
00063 }
00064
00065 std::pair<size_t, double> artdaq::TransferTest::do_receiving()
00066 {
00067 TRACE(7, "do_receiving entered");
00068 artdaq::DataReceiverManager receiver(ps_);
00069 receiver.start_threads();
00070 int counter = receives_each_receiver_;
00071 size_t totalSize = 0;
00072 double totalTime = 0;
00073 bool first = true;
00074 int activeSenders = senders_;
00075 auto last_receive = std::chrono::steady_clock::now();
00076
00077 while (activeSenders > 0)
00078 {
00079 auto start_loop = std::chrono::steady_clock::now();
00080 TRACE(7, "TransferTest::do_receiving: Counter is %d, calling recvFragment", counter);
00081 int senderSlot = artdaq::TransferInterface::RECV_TIMEOUT;
00082 auto before_receive = std::chrono::steady_clock::now();
00083 auto ignoreFragPtr = receiver.recvFragment(senderSlot);
00084 auto after_receive = std::chrono::steady_clock::now();
00085 size_t thisSize = 0;
00086 if (senderSlot != artdaq::TransferInterface::RECV_TIMEOUT && ignoreFragPtr)
00087 {
00088 if (ignoreFragPtr->type() == artdaq::Fragment::EndOfDataFragmentType)
00089 {
00090 std::cout << "Receiver " << my_rank << " received EndOfData Fragment from Sender " << senderSlot << std::endl;
00091 activeSenders--;
00092 }
00093 else
00094 {
00095 if (first)
00096 {
00097 start_time_ = std::chrono::steady_clock::now();
00098 first = false;
00099 }
00100 counter--;
00101 TRACE(1, "Receiver %d received fragment %d with seqID %lu from Sender %d (Expecting %d more)"
00102 , my_rank, receives_each_receiver_ - counter, ignoreFragPtr->sequenceID(), senderSlot, counter);
00103 thisSize = ignoreFragPtr->size() * sizeof(artdaq::RawDataType);
00104 totalSize += thisSize;
00105 }
00106 if (metricMan)
00107 {
00108 metricMan->sendMetric("input_wait", std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1>>>(after_receive - last_receive).count(), "seconds", 3);
00109 }
00110 last_receive = after_receive;
00111 }
00112 TRACE(7, "TransferTest::do_receiving: Recv Loop end, counter is %d", counter);
00113 auto total_recv_time = std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1>>>(after_receive - before_receive).count();
00114 totalTime += total_recv_time;
00115 if (metricMan && counter % 100 == 0)
00116 {
00117 metricMan->sendMetric("recv_init_time", std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1>>>(before_receive - start_loop).count(), "seconds", 3);
00118 metricMan->sendMetric("total_recv_time", total_recv_time, "seconds", 3);
00119 metricMan->sendMetric("recv_rate", thisSize / total_recv_time, "B/s", 3);
00120 }
00121 }
00122
00123 return std::make_pair(totalSize, totalTime);
00124 }
00125
00126 artdaq::TransferTest::TransferTest(fhicl::ParameterSet psi)
00127 : senders_(psi.get<int>("num_senders"))
00128 , receivers_(psi.get<int>("num_receivers"))
00129 , sends_each_sender_(psi.get<int>("sends_per_sender"))
00130 , receives_each_receiver_(senders_ * sends_each_sender_ / receivers_)
00131 , buffer_count_(psi.get<int>("buffer_count", 10))
00132 , max_payload_size_(psi.get<size_t>("fragment_size", 0x100000))
00133 , ps_()
00134 {
00135 TRACE(10, "TransferTest CONSTRUCTOR");
00136 metricMan = &metricMan_;
00137
00138 fhicl::ParameterSet metric_pset;
00139
00140 try
00141 {
00142 metric_pset = psi.get<fhicl::ParameterSet>("metrics");
00143 }
00144 catch (...) {}
00145
00146 try
00147 {
00148 std::string name = "TransferTest" + std::to_string(my_rank);
00149 metricMan_.initialize(metric_pset, name);
00150 metricMan_.do_start();
00151 }
00152 catch (...) {}
00153
00154 std::string type(psi.get<std::string>("transfer_plugin_type", "Shmem"));
00155
00156 if (receivers_ > 0)
00157 {
00158 if (senders_ * sends_each_sender_ % receivers_ != 0)
00159 {
00160 std::cout << "Adding sends so that sends_each_sender * num_sending_ranks is a multiple of num_receiving_ranks" << std::endl;
00161 while (senders_ * sends_each_sender_ % receivers_ != 0)
00162 {
00163 sends_each_sender_++;
00164 }
00165 receives_each_receiver_ = senders_ * sends_each_sender_ / receivers_;
00166 std::cout << "sends_each_sender is now " << sends_each_sender_ << std::endl;
00167 psi.put_or_replace("sends_per_sender", sends_each_sender_);
00168 }
00169 }
00170
00171 std::string hostmap = "";
00172 if (psi.has_key("hostmap"))
00173 {
00174 hostmap = " host_map: @local::hostmap";
00175 }
00176
00177 std::stringstream ss;
00178 ss << psi.to_string();
00179 ss << " sources: {";
00180 for (int ii = 0; ii < senders_; ++ii)
00181 {
00182 ss << "s" << ii << ": { transferPluginType: " << type << " source_rank: " << ii << " max_fragment_size_words: " << max_payload_size_ << " buffer_count: " << buffer_count_ << hostmap << "}";
00183 }
00184 ss << "} destinations: {";
00185 for (int jj = senders_; jj < senders_ + receivers_; ++jj)
00186 {
00187 ss << "d" << jj << ": { transferPluginType: " << type << " destination_rank: " << jj << " max_fragment_size_words: " << max_payload_size_ << " buffer_count: " << buffer_count_ << hostmap << "}";
00188 }
00189 ss << "}";
00190
00191 make_ParameterSet(ss.str(), ps_);
00192
00193
00194 std::cout << "Going to configure with ParameterSet: " << ps_.to_string() << std::endl;
00195 }
00196
00197 int artdaq::TransferTest::runTest()
00198 {
00199 TRACE(11, "TransferTest::runTest BEGIN");
00200 start_time_ = std::chrono::steady_clock::now();
00201 std::pair<size_t, double> result;
00202 if (my_rank < senders_)
00203 {
00204 result = do_sending();
00205 }
00206 else
00207 {
00208 result = do_receiving();
00209 }
00210 auto duration = std::chrono::duration_cast<std::chrono::duration<double, std::ratio<1>>>(std::chrono::steady_clock::now() - start_time_).count();
00211 std::cout << (my_rank < senders_ ? "Sent " : "Received ") << result.first << " bytes in " << duration << " seconds ( " << formatBytes(result.first / duration) << "/s )." << std::endl;
00212 std::cout << "Rate of " << (my_rank < senders_ ? "sending" : "receiving") << ": " << formatBytes(result.first / result.second) << "/s." << std::endl;
00213 metricMan_.do_stop();
00214 metricMan_.shutdown();
00215 TRACE(11, "TransferTest::runTest DONE");
00216 return 0;
00217 }