00001 #define TRACE_NAME "ShmemTransfer"
00002 #include "artdaq/TransferPlugins/ShmemTransfer.hh"
00003 #include "cetlib_except/exception.h"
00004
00005 artdaq::ShmemTransfer::ShmemTransfer(fhicl::ParameterSet const& pset, Role role) :
00006 TransferInterface(pset, role)
00007 , role_(role)
00008 {
00009 TLOG(TLVL_DEBUG) << uniqueLabel() << " ShmemTransfer{} begin";
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027
00028
00029 if (buffer_count_ > 100)
00030 {
00031 throw cet::exception("ConfigurationException", "Buffer Count is too large for Shmem transfer!");
00032 }
00033
00034 auto shmKey = pset.get<uint32_t>("shm_key_offset", 0);
00035 shmKey += std::hash<std::string>()(uniqueLabel());
00036 if (role == Role::kReceive)
00037 {
00038 shm_manager_ = std::make_unique<SharedMemoryFragmentManager>(shmKey, buffer_count_, max_fragment_size_words_ * sizeof(artdaq::RawDataType));
00039 }
00040 else
00041 {
00042 shm_manager_ = std::make_unique<SharedMemoryFragmentManager>(shmKey, 0, 0);
00043 }
00044 }
00045
00046 artdaq::ShmemTransfer::~ShmemTransfer()
00047 {
00048 TLOG_ARB(5, "ShmemTransfer") << "~ShmemTransfer called - " << uniqueLabel() << TLOG_ENDL;
00049 shm_manager_.reset(nullptr);
00050 TLOG_ARB(5, "ShmemTransfer") << "~ShmemTransfer done - " << uniqueLabel() << TLOG_ENDL;
00051 }
00052
00053 int artdaq::ShmemTransfer::receiveFragment(artdaq::Fragment& fragment,
00054 size_t receiveTimeout)
00055 {
00056 auto waitStart = std::chrono::steady_clock::now();
00057 while (!shm_manager_->ReadyForRead() && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
00058 {
00059
00060 }
00061 if (!shm_manager_->ReadyForRead())
00062 {
00063 int64_t loopCount = 0;
00064 size_t sleepTime = 1000;
00065 int64_t nloops = (receiveTimeout - 1000) / sleepTime;
00066
00067 while (!shm_manager_->ReadyForRead() && loopCount < nloops)
00068 {
00069 usleep(sleepTime);
00070 ++loopCount;
00071 }
00072 }
00073
00074 TLOG(4) << uniqueLabel() << " ShmemTransfer::receiveFragment ReadyForRead=" << shm_manager_->ReadyForRead();
00075
00076 if (shm_manager_->ReadyForRead())
00077 {
00078 auto sts = shm_manager_->ReadFragment(fragment);
00079
00080 if (sts != 0) return RECV_TIMEOUT;
00081
00082 if (fragment.type() != artdaq::Fragment::DataFragmentType)
00083 {
00084 TLOG_ARB(8) << uniqueLabel() << " Recvd frag from shmem, type=" << fragment.typeString() << ", sequenceID=" << std::to_string(fragment.sequenceID()) << ", source_rank=" << source_rank() << TLOG_ENDL;
00085 }
00086
00087 return source_rank();
00088 }
00089
00090 return artdaq::TransferInterface::RECV_TIMEOUT;
00091 }
00092
00093 int artdaq::ShmemTransfer::receiveFragmentHeader(detail::RawFragmentHeader& header, size_t receiveTimeout)
00094 {
00095 auto waitStart = std::chrono::steady_clock::now();
00096 while (!shm_manager_->ReadyForRead() && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
00097 {
00098
00099 }
00100 if (!shm_manager_->ReadyForRead())
00101 {
00102 int64_t loopCount = 0;
00103 size_t sleepTime = 1000;
00104 int64_t nloops = (receiveTimeout - 1000) / sleepTime;
00105
00106 while (!shm_manager_->ReadyForRead() && loopCount < nloops)
00107 {
00108 usleep(sleepTime);
00109 ++loopCount;
00110 }
00111 }
00112
00113
00114
00115 if (shm_manager_->ReadyForRead())
00116 {
00117 auto sts = shm_manager_->ReadFragmentHeader(header);
00118
00119 if (sts != 0) return RECV_TIMEOUT;
00120
00121 if (header.type != artdaq::Fragment::DataFragmentType)
00122 {
00123 TLOG_ARB(8) << uniqueLabel() << " Recvd fragment header from shmem, type=" << (int)header.type << ", sequenceID=" << std::to_string(header.sequence_id) << ", source_rank=" << source_rank() << TLOG_ENDL;
00124 }
00125
00126 return source_rank();
00127 }
00128
00129 return artdaq::TransferInterface::RECV_TIMEOUT;
00130 }
00131
00132 int artdaq::ShmemTransfer::receiveFragmentData(RawDataType* destination, size_t word_count)
00133 {
00134 auto sts = shm_manager_->ReadFragmentData(destination, word_count);
00135
00136 TLOG(4) << uniqueLabel() << " Return status from ReadFragmentData is " << sts << TLOG_ENDL;
00137
00138 if (sts != 0) return RECV_TIMEOUT;
00139
00140 return source_rank();
00141
00142 return artdaq::TransferInterface::RECV_TIMEOUT;
00143 }
00144
00145 artdaq::TransferInterface::CopyStatus
00146 artdaq::ShmemTransfer::copyFragment(artdaq::Fragment& fragment, size_t send_timeout_usec)
00147 {
00148 return sendFragment(std::move(fragment), send_timeout_usec, false);
00149 }
00150
00151 artdaq::TransferInterface::CopyStatus
00152 artdaq::ShmemTransfer::moveFragment(artdaq::Fragment&& fragment,
00153 size_t send_timeout_usec)
00154 {
00155 return sendFragment(std::move(fragment), send_timeout_usec, true);
00156 }
00157
00158 artdaq::TransferInterface::CopyStatus
00159 artdaq::ShmemTransfer::sendFragment(artdaq::Fragment&& fragment, size_t send_timeout_usec, bool reliableMode)
00160 {
00161
00162 if (send_timeout_usec > 0 || reliableMode)
00163 {
00164 auto waitStart = std::chrono::steady_clock::now();
00165 while (!shm_manager_->ReadyForWrite(!reliableMode) && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
00166 {
00167
00168 }
00169 if (!shm_manager_->ReadyForWrite(!reliableMode))
00170 {
00171 int64_t loopCount = 0;
00172 size_t sleepTime = 1000;
00173 int64_t nloops = (send_timeout_usec - 1000) / sleepTime;
00174
00175 while (reliableMode && !shm_manager_->ReadyForWrite(!reliableMode) && (send_timeout_usec == 0 || loopCount < nloops))
00176 {
00177 usleep(sleepTime);
00178 ++loopCount;
00179 }
00180 }
00181 }
00182
00183 TLOG_ARB(5) << uniqueLabel() << " Either write has timed out or buffer ready" << TLOG_ENDL;
00184
00185
00186 if (shm_manager_->ReadyForWrite(!reliableMode))
00187 {
00188 TLOG_ARB(5) << uniqueLabel() << " Sending fragment with seqID=" << std::to_string(fragment.sequenceID()) << TLOG_ENDL;
00189 artdaq::RawDataType* fragAddr = fragment.headerAddress();
00190 size_t fragSize = fragment.size() * sizeof(artdaq::RawDataType);
00191
00192
00193
00194 if (fragment.type() != artdaq::Fragment::InvalidFragmentType && fragSize < (max_fragment_size_words_ * sizeof(artdaq::RawDataType)))
00195 {
00196 auto sts = shm_manager_->WriteFragment(std::move(fragment), !reliableMode);
00197 if (sts != 0) return CopyStatus::kErrorNotRequiringException;
00198
00199 TLOG_ARB(5) << uniqueLabel() << " Fragment send successfully" << TLOG_ENDL;
00200 return CopyStatus::kSuccess;
00201 }
00202 else
00203 {
00204 TLOG_WARNING("ShmemTransfer") << uniqueLabel() << " Fragment invalid for shared memory! "
00205 << "fragment address and size = "
00206 << fragAddr << " " << fragSize << " "
00207 << "sequence ID, fragment ID, and type = "
00208 << fragment.sequenceID() << " "
00209 << fragment.fragmentID() << " "
00210 << fragment.typeString() << TLOG_ENDL;
00211 return CopyStatus::kErrorNotRequiringException;
00212 }
00213 }
00214
00215 TLOG_ARB(5) << uniqueLabel() << " Fragment Send Timeout!" << TLOG_ENDL;
00216 return CopyStatus::kTimeout;
00217 }
00218
00219 DEFINE_ARTDAQ_TRANSFER(artdaq::ShmemTransfer)
00220
00221
00222
00223