00001 #define TRACE_NAME "ShmemTransfer"
00002
00003 #include "artdaq/TransferPlugins/ShmemTransfer.hh"
00004 #include "cetlib_except/exception.h"
00005 #include <signal.h>
00006
00007 artdaq::ShmemTransfer::ShmemTransfer(fhicl::ParameterSet const& pset, Role role) :
00008 TransferInterface(pset, role)
00009 {
00010 TLOG(TLVL_DEBUG) << GetTraceName() << ": Constructor BEGIN";
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027
00028
00029
00030 if (buffer_count_ > 100)
00031 {
00032 throw cet::exception("ConfigurationException", "Buffer Count is too large for Shmem transfer!");
00033 }
00034
00035 auto partition = partition_number_ + 1;
00036 if (partition > 127) partition = 127;
00037
00038 auto shmKey = pset.get<uint32_t>("shm_key_offset", 0) + (partition << 24) + ((source_rank() & 0xFFF) << 12) + (destination_rank() & 0xFFF);
00039
00040
00041 if (pset.has_key("shm_key")) {
00042 shmKey = pset.get<uint32_t>("shm_key");
00043 }
00044
00045 if (role == Role::kReceive)
00046 {
00047 shm_manager_ = std::make_unique<SharedMemoryFragmentManager>(shmKey, buffer_count_, max_fragment_size_words_ * sizeof(artdaq::RawDataType));
00048 }
00049 else
00050 {
00051 shm_manager_ = std::make_unique<SharedMemoryFragmentManager>(shmKey, 0, 0);
00052 }
00053 TLOG(TLVL_DEBUG) << GetTraceName() << ": Constructor END";
00054 }
00055
00056 artdaq::ShmemTransfer::~ShmemTransfer() noexcept
00057 {
00058 TLOG(5) << GetTraceName() << " ~ShmemTransfer called - " << uniqueLabel() ;
00059 shm_manager_.reset(nullptr);
00060 TLOG(5) << GetTraceName() << " ~ShmemTransfer done - " << uniqueLabel() ;
00061 }
00062
00063 int artdaq::ShmemTransfer::receiveFragment(artdaq::Fragment& fragment,
00064 size_t receiveTimeout)
00065 {
00066 auto waitStart = std::chrono::steady_clock::now();
00067 while (!shm_manager_->ReadyForRead() && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
00068 {
00069
00070 }
00071 if (!shm_manager_->ReadyForRead())
00072 {
00073 int64_t loopCount = 0;
00074 size_t sleepTime = 1000;
00075 int64_t nloops = (receiveTimeout - 1000) / sleepTime;
00076
00077 while (!shm_manager_->ReadyForRead() && loopCount < nloops)
00078 {
00079 usleep(sleepTime);
00080 ++loopCount;
00081 }
00082 }
00083
00084 TLOG(TLVL_TRACE) << GetTraceName() << ": receiveFragment ReadyForRead=" << shm_manager_->ReadyForRead();
00085
00086 if (shm_manager_->ReadyForRead())
00087 {
00088 auto sts = shm_manager_->ReadFragment(fragment);
00089
00090 if (sts != 0) return RECV_TIMEOUT;
00091
00092 if (fragment.type() != artdaq::Fragment::DataFragmentType)
00093 {
00094 TLOG(8) << GetTraceName() << ": Recvd frag from shmem, type=" << fragment.typeString() << ", sequenceID=" << fragment.sequenceID() << ", source_rank=" << source_rank() ;
00095 }
00096
00097 return source_rank();
00098 }
00099
00100 return artdaq::TransferInterface::RECV_TIMEOUT;
00101 }
00102
00103 int artdaq::ShmemTransfer::receiveFragmentHeader(detail::RawFragmentHeader& header, size_t receiveTimeout)
00104 {
00105 auto waitStart = std::chrono::steady_clock::now();
00106 while (!shm_manager_->ReadyForRead() && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
00107 {
00108
00109 }
00110 if (!shm_manager_->ReadyForRead())
00111 {
00112 int64_t loopCount = 0;
00113 size_t sleepTime = 1000;
00114 int64_t nloops = (receiveTimeout - 1000) / sleepTime;
00115
00116 while (!shm_manager_->ReadyForRead() && loopCount < nloops)
00117 {
00118 usleep(sleepTime);
00119 ++loopCount;
00120 }
00121 }
00122
00123 if (!shm_manager_->ReadyForRead() && shm_manager_->IsEndOfData())
00124 {
00125 return artdaq::TransferInterface::DATA_END;
00126 }
00127
00128
00129
00130 if (shm_manager_->ReadyForRead())
00131 {
00132 auto sts = shm_manager_->ReadFragmentHeader(header);
00133
00134 if (sts != 0) return RECV_TIMEOUT;
00135
00136 if (header.type != artdaq::Fragment::DataFragmentType)
00137 {
00138 TLOG(8) << GetTraceName() << ": Recvd fragment header from shmem, type=" << (int)header.type << ", sequenceID=" << header.sequence_id << ", source_rank=" << source_rank() ;
00139 }
00140
00141 return source_rank();
00142 }
00143
00144 return artdaq::TransferInterface::RECV_TIMEOUT;
00145 }
00146
00147 int artdaq::ShmemTransfer::receiveFragmentData(RawDataType* destination, size_t word_count)
00148 {
00149 auto sts = shm_manager_->ReadFragmentData(destination, word_count);
00150
00151 TLOG(TLVL_TRACE) << GetTraceName() << ": Return status from ReadFragmentData is " << sts ;
00152
00153 if (sts != 0) return RECV_TIMEOUT;
00154
00155 return source_rank();
00156
00157 return artdaq::TransferInterface::RECV_TIMEOUT;
00158 }
00159
00160 artdaq::TransferInterface::CopyStatus
00161 artdaq::ShmemTransfer::copyFragment(artdaq::Fragment& fragment, size_t send_timeout_usec)
00162 {
00163 return sendFragment(std::move(fragment), send_timeout_usec, false);
00164 }
00165
00166 artdaq::TransferInterface::CopyStatus
00167 artdaq::ShmemTransfer::moveFragment(artdaq::Fragment&& fragment)
00168 {
00169 return sendFragment(std::move(fragment), 0, true);
00170 }
00171
00172 artdaq::TransferInterface::CopyStatus
00173 artdaq::ShmemTransfer::sendFragment(artdaq::Fragment&& fragment, size_t send_timeout_usec, bool reliableMode)
00174 {
00175 if (!shm_manager_->IsValid()) {
00176 shm_manager_->Attach();
00177 if (!shm_manager_->IsValid()) {
00178 TLOG(TLVL_ERROR) << GetTraceName() << ": Attempted to send Fragment when not attached to Shared Memory! Returning kSuccess, and dropping data!";
00179 return CopyStatus::kSuccess;
00180 }
00181 }
00182 shm_manager_->SetRank(my_rank);
00183
00184
00185 TLOG(5) << GetTraceName() << ": Sending fragment with seqID=" << fragment.sequenceID() ;
00186 artdaq::RawDataType* fragAddr = fragment.headerAddress();
00187 size_t fragSize = fragment.size() * sizeof(artdaq::RawDataType);
00188
00189
00190
00191 if (fragment.type() != artdaq::Fragment::InvalidFragmentType && fragSize < (max_fragment_size_words_ * sizeof(artdaq::RawDataType)))
00192 {
00193 TLOG(5) << GetTraceName() << ": Writing fragment with seqID=" << fragment.sequenceID();
00194 auto sts = shm_manager_->WriteFragment(std::move(fragment), !reliableMode, send_timeout_usec);
00195 if (sts == -3)
00196 {
00197 TLOG(TLVL_WARNING) << GetTraceName() << ": Timeout writing fragment with seqID=" << fragment.sequenceID();
00198 return CopyStatus::kTimeout;
00199 }
00200 if (sts != 0)
00201 {
00202 TLOG(TLVL_WARNING) << GetTraceName() << ": Error writing fragment with seqID=" << fragment.sequenceID();
00203 return CopyStatus::kErrorNotRequiringException;
00204 }
00205
00206 TLOG(5) << GetTraceName() << ": Successfully sent Fragment with seqID=" << fragment.sequenceID();
00207 return CopyStatus::kSuccess;
00208 }
00209 else
00210 {
00211 TLOG(TLVL_WARNING) << GetTraceName() << ": Fragment invalid for shared memory! "
00212 << "fragment address and size = "
00213 << fragAddr << " " << fragSize << " "
00214 << "sequence ID, fragment ID, and type = "
00215 << fragment.sequenceID() << " "
00216 << fragment.fragmentID() << " "
00217 << fragment.typeString() ;
00218 return CopyStatus::kErrorNotRequiringException;
00219 }
00220
00221 TLOG(TLVL_WARNING) << GetTraceName() << ": Unreachable code reached!" ;
00222 return CopyStatus::kErrorNotRequiringException;
00223 }
00224
00225 bool artdaq::ShmemTransfer::isRunning()
00226 {
00227 bool ret = false;
00228 switch (role())
00229 {
00230 case TransferInterface::Role::kSend:
00231 ret = shm_manager_->IsValid();
00232 break;
00233 case TransferInterface::Role::kReceive:
00234 ret = shm_manager_->GetAttachedCount() > 1;
00235 break;
00236 }
00237 return ret;
00238
00239 }
00240
00241 DEFINE_ARTDAQ_TRANSFER(artdaq::ShmemTransfer)
00242
00243
00244
00245