00001 #include "artdaq/TransferPlugins/ShmemTransfer.hh"
00002 #include "cetlib_except/exception.h"
00003
00004 artdaq::ShmemTransfer::ShmemTransfer(fhicl::ParameterSet const& pset, Role role) :
00005 TransferInterface(pset, role)
00006 , role_(role)
00007 {
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027 if (buffer_count_ > 100)
00028 {
00029 throw cet::exception("ConfigurationException", "Buffer Count is too large for Shmem transfer!");
00030 }
00031
00032 auto shmKey = pset.get<uint32_t>("shm_key_offset", 0);
00033 shmKey += std::hash<std::string>()(uniqueLabel());
00034 if (role == Role::kReceive)
00035 {
00036 shm_manager_ = std::make_unique<SharedMemoryFragmentManager>(shmKey, buffer_count_, max_fragment_size_words_ * sizeof(artdaq::RawDataType));
00037 }
00038 else
00039 {
00040 shm_manager_ = std::make_unique<SharedMemoryFragmentManager>(shmKey, 0, 0);
00041 }
00042 }
00043
00044 artdaq::ShmemTransfer::~ShmemTransfer()
00045 {
00046 TLOG_ARB(5, uniqueLabel()) << "~ShmemTransfer called" << TLOG_ENDL;
00047 shm_manager_.reset(nullptr);
00048 TLOG_ARB(5, uniqueLabel()) << "~ShmemTransfer done" << TLOG_ENDL;
00049 }
00050
00051 int artdaq::ShmemTransfer::receiveFragment(artdaq::Fragment& fragment,
00052 size_t receiveTimeout)
00053 {
00054 auto waitStart = std::chrono::steady_clock::now();
00055 while (!shm_manager_->ReadyForRead() && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
00056 {
00057
00058 }
00059 if (!shm_manager_->ReadyForRead())
00060 {
00061 int64_t loopCount = 0;
00062 size_t sleepTime = 1000;
00063 int64_t nloops = (receiveTimeout - 1000) / sleepTime;
00064
00065 while (!shm_manager_->ReadyForRead() && loopCount < nloops)
00066 {
00067 usleep(sleepTime);
00068 ++loopCount;
00069 }
00070 }
00071
00072
00073
00074 if (shm_manager_->ReadyForRead())
00075 {
00076 auto sts = shm_manager_->ReadFragment(fragment);
00077
00078 if (sts != 0) return RECV_TIMEOUT;
00079
00080 if (fragment.type() != artdaq::Fragment::DataFragmentType)
00081 {
00082 TLOG_ARB(8, uniqueLabel()) << "Recvd frag from shmem, type=" << fragment.typeString() << ", sequenceID=" << std::to_string(fragment.sequenceID()) << ", source_rank=" << source_rank() << TLOG_ENDL;
00083 }
00084
00085 return source_rank();
00086 }
00087
00088 return artdaq::TransferInterface::RECV_TIMEOUT;
00089 }
00090
00091 int artdaq::ShmemTransfer::receiveFragmentHeader(detail::RawFragmentHeader& header, size_t receiveTimeout)
00092 {
00093 auto waitStart = std::chrono::steady_clock::now();
00094 while (!shm_manager_->ReadyForRead() && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
00095 {
00096
00097 }
00098 if (!shm_manager_->ReadyForRead())
00099 {
00100 int64_t loopCount = 0;
00101 size_t sleepTime = 1000;
00102 int64_t nloops = (receiveTimeout - 1000) / sleepTime;
00103
00104 while (!shm_manager_->ReadyForRead() && loopCount < nloops)
00105 {
00106 usleep(sleepTime);
00107 ++loopCount;
00108 }
00109 }
00110
00111
00112
00113 if (shm_manager_->ReadyForRead())
00114 {
00115 auto sts = shm_manager_->ReadFragmentHeader(header);
00116
00117 if (sts != 0) return RECV_TIMEOUT;
00118
00119 if (header.type != artdaq::Fragment::DataFragmentType)
00120 {
00121 TLOG_ARB(8, uniqueLabel()) << "Recvd fragment header from shmem, type=" << (int)header.type << ", sequenceID=" << std::to_string(header.sequence_id) << ", source_rank=" << source_rank() << TLOG_ENDL;
00122 }
00123
00124 return source_rank();
00125 }
00126
00127 return artdaq::TransferInterface::RECV_TIMEOUT;
00128 }
00129
00130 int artdaq::ShmemTransfer::receiveFragmentData(RawDataType* destination, size_t word_count)
00131 {
00132 auto sts = shm_manager_->ReadFragmentData(destination, word_count);
00133
00134 TLOG_TRACE(uniqueLabel()) << "Return status from ReadFragmentData is " << sts << TLOG_ENDL;
00135
00136 if (sts != 0) return RECV_TIMEOUT;
00137
00138 return source_rank();
00139
00140 return artdaq::TransferInterface::RECV_TIMEOUT;
00141 }
00142
00143 artdaq::TransferInterface::CopyStatus
00144 artdaq::ShmemTransfer::copyFragment(artdaq::Fragment& fragment, size_t send_timeout_usec)
00145 {
00146 return sendFragment(std::move(fragment), send_timeout_usec, false);
00147 }
00148
00149 artdaq::TransferInterface::CopyStatus
00150 artdaq::ShmemTransfer::moveFragment(artdaq::Fragment&& fragment,
00151 size_t send_timeout_usec)
00152 {
00153 return sendFragment(std::move(fragment), send_timeout_usec, true);
00154 }
00155
00156 artdaq::TransferInterface::CopyStatus
00157 artdaq::ShmemTransfer::sendFragment(artdaq::Fragment&& fragment, size_t send_timeout_usec, bool reliableMode)
00158 {
00159
00160 if (send_timeout_usec > 0 || reliableMode)
00161 {
00162 auto waitStart = std::chrono::steady_clock::now();
00163 while (!shm_manager_->ReadyForWrite(!reliableMode) && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
00164 {
00165
00166 }
00167 if (!shm_manager_->ReadyForWrite(!reliableMode))
00168 {
00169 int64_t loopCount = 0;
00170 size_t sleepTime = 1000;
00171 int64_t nloops = (send_timeout_usec - 1000) / sleepTime;
00172
00173 while (reliableMode && !shm_manager_->ReadyForWrite(!reliableMode) && (send_timeout_usec == 0 || loopCount < nloops))
00174 {
00175 usleep(sleepTime);
00176 ++loopCount;
00177 }
00178 }
00179 }
00180
00181 TLOG_ARB(5, uniqueLabel()) << "Either write has timed out or buffer ready" << TLOG_ENDL;
00182
00183
00184 if (shm_manager_->ReadyForWrite(!reliableMode))
00185 {
00186 TLOG_ARB(5, uniqueLabel()) << "Sending fragment with seqID=" << std::to_string(fragment.sequenceID()) << TLOG_ENDL;
00187 artdaq::RawDataType* fragAddr = fragment.headerAddress();
00188 size_t fragSize = fragment.size() * sizeof(artdaq::RawDataType);
00189
00190
00191
00192 if (fragment.type() != artdaq::Fragment::InvalidFragmentType && fragSize < (max_fragment_size_words_ * sizeof(artdaq::RawDataType)))
00193 {
00194 auto sts = shm_manager_->WriteFragment(std::move(fragment), !reliableMode);
00195 if (sts != 0) return CopyStatus::kErrorNotRequiringException;
00196
00197 TLOG_ARB(5, uniqueLabel()) << "Fragment send successfully" << TLOG_ENDL;
00198 return CopyStatus::kSuccess;
00199 }
00200 else
00201 {
00202 TLOG_WARNING(uniqueLabel()) << "Fragment invalid for shared memory! "
00203 << "fragment address and size = "
00204 << fragAddr << " " << fragSize << " "
00205 << "sequence ID, fragment ID, and type = "
00206 << fragment.sequenceID() << " "
00207 << fragment.fragmentID() << " "
00208 << fragment.typeString() << TLOG_ENDL;
00209 return CopyStatus::kErrorNotRequiringException;
00210 }
00211 }
00212
00213 TLOG_ARB(5, uniqueLabel()) << "Fragment Send Timeout!" << TLOG_ENDL;
00214 return CopyStatus::kTimeout;
00215 }
00216
00217 DEFINE_ARTDAQ_TRANSFER(artdaq::ShmemTransfer)
00218
00219
00220
00221