00001 #define TRACE_NAME (app_name + "_ShmemTransfer").c_str()
00002 #include "artdaq/DAQdata/Globals.hh"
00003
00004 #include "artdaq/TransferPlugins/ShmemTransfer.hh"
00005 #include "cetlib_except/exception.h"
00006 #include <signal.h>
00007
00008 artdaq::ShmemTransfer::ShmemTransfer(fhicl::ParameterSet const& pset, Role role) :
00009 TransferInterface(pset, role)
00010 {
00011 TLOG(TLVL_DEBUG) << GetTraceName() << ": Constructor BEGIN";
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027
00028
00029
00030
00031 if (buffer_count_ > 100)
00032 {
00033 throw cet::exception("ConfigurationException", "Buffer Count is too large for Shmem transfer!");
00034 }
00035
00036 auto partition = GetPartitionNumber() + 1;
00037
00038 auto shmKey = pset.get<uint32_t>("shm_key_offset", 0) + (partition << 24) + ((source_rank() & 0xFFF) << 12) + (destination_rank() & 0xFFF);
00039
00040
00041 if (pset.has_key("shm_key")) {
00042 shmKey = pset.get<uint32_t>("shm_key");
00043 }
00044
00045 if (role == Role::kReceive)
00046 {
00047 shm_manager_ = std::make_unique<SharedMemoryFragmentManager>(shmKey, buffer_count_, max_fragment_size_words_ * sizeof(artdaq::RawDataType));
00048 }
00049 else
00050 {
00051 shm_manager_ = std::make_unique<SharedMemoryFragmentManager>(shmKey, 0, 0);
00052 }
00053 TLOG(TLVL_DEBUG) << GetTraceName() << ": Constructor END";
00054 }
00055
00056 artdaq::ShmemTransfer::~ShmemTransfer() noexcept
00057 {
00058 TLOG(5) << GetTraceName() << " ~ShmemTransfer called - " << uniqueLabel() ;
00059 shm_manager_.reset(nullptr);
00060 TLOG(5) << GetTraceName() << " ~ShmemTransfer done - " << uniqueLabel() ;
00061 }
00062
00063 int artdaq::ShmemTransfer::receiveFragment(artdaq::Fragment& fragment,
00064 size_t receiveTimeout)
00065 {
00066 auto waitStart = std::chrono::steady_clock::now();
00067 while (!shm_manager_->ReadyForRead() && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
00068 {
00069
00070 }
00071 if (!shm_manager_->ReadyForRead())
00072 {
00073 int64_t loopCount = 0;
00074 size_t sleepTime = 1000;
00075 int64_t nloops = (receiveTimeout - 1000) / sleepTime;
00076
00077 while (!shm_manager_->ReadyForRead() && loopCount < nloops)
00078 {
00079 usleep(sleepTime);
00080 ++loopCount;
00081 }
00082 }
00083
00084 TLOG(TLVL_TRACE) << GetTraceName() << ": receiveFragment ReadyForRead=" << shm_manager_->ReadyForRead();
00085
00086 if (shm_manager_->ReadyForRead())
00087 {
00088 auto sts = shm_manager_->ReadFragment(fragment);
00089
00090 if (sts != 0) {
00091 TLOG(TLVL_TRACE) << "Non-zero status (" << sts << ") returned from ReadFragment, returning...";
00092 return RECV_TIMEOUT;
00093 }
00094
00095 if (fragment.type() != artdaq::Fragment::DataFragmentType)
00096 {
00097 TLOG(8) << GetTraceName() << ": Recvd frag from shmem, type=" << fragment.typeString() << ", sequenceID=" << fragment.sequenceID() << ", source_rank=" << source_rank() ;
00098 }
00099
00100 return source_rank();
00101 }
00102
00103 return artdaq::TransferInterface::RECV_TIMEOUT;
00104 }
00105
00106 int artdaq::ShmemTransfer::receiveFragmentHeader(detail::RawFragmentHeader& header, size_t receiveTimeout)
00107 {
00108 auto waitStart = std::chrono::steady_clock::now();
00109 while (!shm_manager_->ReadyForRead() && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
00110 {
00111
00112 }
00113 if (!shm_manager_->ReadyForRead())
00114 {
00115 int64_t loopCount = 0;
00116 size_t sleepTime = 1000;
00117 int64_t nloops = (receiveTimeout - 1000) / sleepTime;
00118
00119 while (!shm_manager_->ReadyForRead() && loopCount < nloops)
00120 {
00121 usleep(sleepTime);
00122 ++loopCount;
00123 }
00124 }
00125
00126 if (!shm_manager_->ReadyForRead() && shm_manager_->IsEndOfData())
00127 {
00128 return artdaq::TransferInterface::DATA_END;
00129 }
00130
00131
00132
00133 if (shm_manager_->ReadyForRead())
00134 {
00135 auto sts = shm_manager_->ReadFragmentHeader(header);
00136
00137 if (sts != 0) {
00138 TLOG(TLVL_TRACE) << "Non-zero status (" << sts << ") returned from ReadFragmentHeader, returning...";
00139 return RECV_TIMEOUT;
00140 }
00141
00142 if (header.type != artdaq::Fragment::DataFragmentType)
00143 {
00144 TLOG(8) << GetTraceName() << ": Recvd fragment header from shmem, type=" << (int)header.type << ", sequenceID=" << header.sequence_id << ", source_rank=" << source_rank() ;
00145 }
00146
00147 return source_rank();
00148 }
00149
00150 return artdaq::TransferInterface::RECV_TIMEOUT;
00151 }
00152
00153 int artdaq::ShmemTransfer::receiveFragmentData(RawDataType* destination, size_t word_count)
00154 {
00155 auto sts = shm_manager_->ReadFragmentData(destination, word_count);
00156
00157 TLOG(TLVL_TRACE) << GetTraceName() << ": Return status from ReadFragmentData is " << sts ;
00158
00159 if (sts != 0) {
00160 TLOG(TLVL_TRACE) << "Non-zero status (" << sts << ") returned from ReadFragmentData, returning...";
00161 return RECV_TIMEOUT;
00162 }
00163
00164 return source_rank();
00165
00166 return artdaq::TransferInterface::RECV_TIMEOUT;
00167 }
00168
00169 artdaq::TransferInterface::CopyStatus
00170 artdaq::ShmemTransfer::copyFragment(artdaq::Fragment& fragment, size_t send_timeout_usec)
00171 {
00172 return sendFragment(std::move(fragment), send_timeout_usec, false);
00173 }
00174
00175 artdaq::TransferInterface::CopyStatus
00176 artdaq::ShmemTransfer::moveFragment(artdaq::Fragment&& fragment)
00177 {
00178 return sendFragment(std::move(fragment), 0, true);
00179 }
00180
00181 artdaq::TransferInterface::CopyStatus
00182 artdaq::ShmemTransfer::sendFragment(artdaq::Fragment&& fragment, size_t send_timeout_usec, bool reliableMode)
00183 {
00184 if (!shm_manager_->IsValid()) {
00185 shm_manager_->Attach();
00186 if (!shm_manager_->IsValid()) {
00187 TLOG(TLVL_ERROR) << GetTraceName() << ": Attempted to send Fragment when not attached to Shared Memory! Returning kSuccess, and dropping data!";
00188 return CopyStatus::kSuccess;
00189 }
00190 }
00191 shm_manager_->SetRank(my_rank);
00192
00193
00194 TLOG(5) << GetTraceName() << ": Sending fragment with seqID=" << fragment.sequenceID() ;
00195 artdaq::RawDataType* fragAddr = fragment.headerAddress();
00196 size_t fragSize = fragment.size() * sizeof(artdaq::RawDataType);
00197
00198
00199
00200 if (fragment.type() != artdaq::Fragment::InvalidFragmentType && fragSize < (max_fragment_size_words_ * sizeof(artdaq::RawDataType)))
00201 {
00202 TLOG(5) << GetTraceName() << ": Writing fragment with seqID=" << fragment.sequenceID();
00203 auto sts = shm_manager_->WriteFragment(std::move(fragment), !reliableMode, send_timeout_usec);
00204 if (sts == -3)
00205 {
00206 TLOG(TLVL_WARNING) << GetTraceName() << ": Timeout writing fragment with seqID=" << fragment.sequenceID();
00207 return CopyStatus::kTimeout;
00208 }
00209 if (sts != 0)
00210 {
00211 TLOG(TLVL_WARNING) << GetTraceName() << ": Error writing fragment with seqID=" << fragment.sequenceID();
00212 return CopyStatus::kErrorNotRequiringException;
00213 }
00214
00215 TLOG(5) << GetTraceName() << ": Successfully sent Fragment with seqID=" << fragment.sequenceID();
00216 return CopyStatus::kSuccess;
00217 }
00218 else
00219 {
00220 TLOG(TLVL_WARNING) << GetTraceName() << ": Fragment invalid for shared memory! "
00221 << "fragment address and size = "
00222 << fragAddr << " " << fragSize << " "
00223 << "sequence ID, fragment ID, and type = "
00224 << fragment.sequenceID() << " "
00225 << fragment.fragmentID() << " "
00226 << fragment.typeString() ;
00227 return CopyStatus::kErrorNotRequiringException;
00228 }
00229
00230 TLOG(TLVL_WARNING) << GetTraceName() << ": Unreachable code reached!" ;
00231 return CopyStatus::kErrorNotRequiringException;
00232 }
00233
00234 bool artdaq::ShmemTransfer::isRunning()
00235 {
00236 bool ret = false;
00237 switch (role())
00238 {
00239 case TransferInterface::Role::kSend:
00240 ret = shm_manager_->IsValid();
00241 break;
00242 case TransferInterface::Role::kReceive:
00243 ret = shm_manager_->GetAttachedCount() > 1;
00244 break;
00245 }
00246 return ret;
00247
00248 }
00249
00250 DEFINE_ARTDAQ_TRANSFER(artdaq::ShmemTransfer)
00251
00252
00253
00254