$treeview $search $mathjax $extrastylesheet
artdaq
v3_04_00
$projectbrief
|
$projectbrief
|
$searchbox |
00001 #define TRACE_NAME (app_name + "_ShmemTransfer").c_str() 00002 #include "artdaq/DAQdata/Globals.hh" 00003 00004 #include "artdaq/TransferPlugins/ShmemTransfer.hh" 00005 #include "cetlib_except/exception.h" 00006 #include <signal.h> 00007 00008 artdaq::ShmemTransfer::ShmemTransfer(fhicl::ParameterSet const& pset, Role role) : 00009 TransferInterface(pset, role) 00010 { 00011 TLOG(TLVL_DEBUG) << GetTraceName() << ": Constructor BEGIN"; 00012 // char* keyChars = getenv("ARTDAQ_SHM_KEY"); 00013 // if (keyChars != NULL && shm_key_ == static_cast<int>(std::hash<std::string>()(unique_label_))) { 00014 // std::string keyString(keyChars); 00015 // try { 00016 // shm_key_ = boost::lexical_cast<int>(keyString); 00017 // } 00018 // catch (...) { 00019 // std::stringstream errmsg; 00020 // errmsg << uniqueLabel() << ": Problem performing lexical cast on " << keyString; 00021 // ExceptionHandler(ExceptionHandlerRethrow::yes, errmsg.str()); 00022 // } 00023 // } 00024 00025 // JCF, Aug-16-2016 00026 00027 // Note that there's a small but nonzero chance of a race condition 00028 // here where another process creates the shared memory buffer 00029 // between the first and second calls to shmget 00030 00031 if (buffer_count_ > 100) 00032 { 00033 throw cet::exception("ConfigurationException", "Buffer Count is too large for Shmem transfer!"); 00034 } 00035 00036 auto partition = GetPartitionNumber() + 1; // Can't be 0 00037 00038 auto shmKey = pset.get<uint32_t>("shm_key_offset", 0) + (partition << 24) + ((source_rank() & 0xFFF) << 12) + (destination_rank() & 0xFFF); 00039 00040 // Configured Shared Memory key overrides everything! Needed for Online Monitor connections! 00041 if (pset.has_key("shm_key")) { 00042 shmKey = pset.get<uint32_t>("shm_key"); 00043 } 00044 00045 if (role == Role::kReceive) 00046 { 00047 shm_manager_ = std::make_unique<SharedMemoryFragmentManager>(shmKey, buffer_count_, max_fragment_size_words_ * sizeof(artdaq::RawDataType)); 00048 } 00049 else 00050 { 00051 shm_manager_ = std::make_unique<SharedMemoryFragmentManager>(shmKey, 0, 0); 00052 } 00053 TLOG(TLVL_DEBUG) << GetTraceName() << ": Constructor END"; 00054 } 00055 00056 artdaq::ShmemTransfer::~ShmemTransfer() noexcept 00057 { 00058 TLOG(5) << GetTraceName() << " ~ShmemTransfer called - " << uniqueLabel() ; 00059 shm_manager_.reset(nullptr); 00060 TLOG(5) << GetTraceName() << " ~ShmemTransfer done - " << uniqueLabel() ; 00061 } 00062 00063 int artdaq::ShmemTransfer::receiveFragment(artdaq::Fragment& fragment, 00064 size_t receiveTimeout) 00065 { 00066 auto waitStart = std::chrono::steady_clock::now(); 00067 while (!shm_manager_->ReadyForRead() && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000) 00068 { 00069 // BURN THAT CPU! 00070 } 00071 if (!shm_manager_->ReadyForRead()) 00072 { 00073 int64_t loopCount = 0; 00074 size_t sleepTime = 1000; // microseconds 00075 int64_t nloops = (receiveTimeout - 1000) / sleepTime; 00076 00077 while (!shm_manager_->ReadyForRead() && loopCount < nloops) 00078 { 00079 usleep(sleepTime); 00080 ++loopCount; 00081 } 00082 } 00083 00084 TLOG(TLVL_TRACE) << GetTraceName() << ": receiveFragment ReadyForRead=" << shm_manager_->ReadyForRead(); 00085 00086 if (shm_manager_->ReadyForRead()) 00087 { 00088 auto sts = shm_manager_->ReadFragment(fragment); 00089 00090 if (sts != 0) { 00091 TLOG(TLVL_TRACE) << "Non-zero status (" << sts << ") returned from ReadFragment, returning..."; 00092 return RECV_TIMEOUT; 00093 } 00094 00095 if (fragment.type() != artdaq::Fragment::DataFragmentType) 00096 { 00097 TLOG(8) << GetTraceName() << ": Recvd frag from shmem, type=" << fragment.typeString() << ", sequenceID=" << fragment.sequenceID() << ", source_rank=" << source_rank() ; 00098 } 00099 00100 return source_rank(); 00101 } 00102 00103 return artdaq::TransferInterface::RECV_TIMEOUT; 00104 } 00105 00106 int artdaq::ShmemTransfer::receiveFragmentHeader(detail::RawFragmentHeader& header, size_t receiveTimeout) 00107 { 00108 auto waitStart = std::chrono::steady_clock::now(); 00109 while (!shm_manager_->ReadyForRead() && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000) 00110 { 00111 // BURN THAT CPU! 00112 } 00113 if (!shm_manager_->ReadyForRead()) 00114 { 00115 int64_t loopCount = 0; 00116 size_t sleepTime = 1000; // microseconds 00117 int64_t nloops = (receiveTimeout - 1000) / sleepTime; 00118 00119 while (!shm_manager_->ReadyForRead() && loopCount < nloops) 00120 { 00121 usleep(sleepTime); 00122 ++loopCount; 00123 } 00124 } 00125 00126 if (!shm_manager_->ReadyForRead() && shm_manager_->IsEndOfData()) 00127 { 00128 return artdaq::TransferInterface::DATA_END; 00129 } 00130 00131 //TLOG(TLVL_TRACE) << GetTraceName() << ": delta_=" << delta_() << ", rp=" << (int)shm_ptr_->read_pos << ", wp=" << (int)shm_ptr_->write_pos << ", loopCount=" << loopCount << ", nloops=" << nloops ; 00132 00133 if (shm_manager_->ReadyForRead()) 00134 { 00135 auto sts = shm_manager_->ReadFragmentHeader(header); 00136 00137 if (sts != 0) { 00138 TLOG(TLVL_TRACE) << "Non-zero status (" << sts << ") returned from ReadFragmentHeader, returning..."; 00139 return RECV_TIMEOUT; 00140 } 00141 00142 if (header.type != artdaq::Fragment::DataFragmentType) 00143 { 00144 TLOG(8) << GetTraceName() << ": Recvd fragment header from shmem, type=" << (int)header.type << ", sequenceID=" << header.sequence_id << ", source_rank=" << source_rank() ; 00145 } 00146 00147 return source_rank(); 00148 } 00149 00150 return artdaq::TransferInterface::RECV_TIMEOUT; 00151 } 00152 00153 int artdaq::ShmemTransfer::receiveFragmentData(RawDataType* destination, size_t word_count) 00154 { 00155 auto sts = shm_manager_->ReadFragmentData(destination, word_count); 00156 00157 TLOG(TLVL_TRACE) << GetTraceName() << ": Return status from ReadFragmentData is " << sts ; 00158 00159 if (sts != 0) { 00160 TLOG(TLVL_TRACE) << "Non-zero status (" << sts << ") returned from ReadFragmentData, returning..."; 00161 return RECV_TIMEOUT; 00162 } 00163 00164 return source_rank(); 00165 00166 return artdaq::TransferInterface::RECV_TIMEOUT; 00167 } 00168 00169 artdaq::TransferInterface::CopyStatus 00170 artdaq::ShmemTransfer::transfer_fragment_min_blocking_mode(artdaq::Fragment const& fragment, size_t send_timeout_usec) 00171 { 00172 return sendFragment(Fragment(fragment), send_timeout_usec, false); 00173 } 00174 00175 artdaq::TransferInterface::CopyStatus 00176 artdaq::ShmemTransfer::transfer_fragment_reliable_mode(artdaq::Fragment&& fragment) 00177 { 00178 return sendFragment(std::move(fragment), 0, true); 00179 } 00180 00181 artdaq::TransferInterface::CopyStatus 00182 artdaq::ShmemTransfer::sendFragment(artdaq::Fragment&& fragment, size_t send_timeout_usec, bool reliableMode) 00183 { 00184 if (!shm_manager_->IsValid()) { 00185 shm_manager_->Attach(); 00186 if (!shm_manager_->IsValid()) { 00187 TLOG(TLVL_ERROR) << GetTraceName() << ": Attempted to send Fragment when not attached to Shared Memory! Returning kSuccess, and dropping data!"; 00188 return CopyStatus::kSuccess; 00189 } 00190 } 00191 shm_manager_->SetRank(my_rank); 00192 // wait for the shm to become free, if requested 00193 00194 TLOG(5) << GetTraceName() << ": Sending fragment with seqID=" << fragment.sequenceID() ; 00195 artdaq::RawDataType* fragAddr = fragment.headerAddress(); 00196 size_t fragSize = fragment.size() * sizeof(artdaq::RawDataType); 00197 00198 // 10-Sep-2013, KAB - protect against large events and 00199 // invalid events (and large, invalid events) 00200 if (fragment.type() != artdaq::Fragment::InvalidFragmentType && fragSize < (max_fragment_size_words_ * sizeof(artdaq::RawDataType))) 00201 { 00202 TLOG(5) << GetTraceName() << ": Writing fragment with seqID=" << fragment.sequenceID(); 00203 auto sts = shm_manager_->WriteFragment(std::move(fragment), !reliableMode, send_timeout_usec); 00204 if (sts == -3) 00205 { 00206 TLOG(TLVL_WARNING) << GetTraceName() << ": Timeout writing fragment with seqID=" << fragment.sequenceID(); 00207 return CopyStatus::kTimeout; 00208 } 00209 if (sts != 0) 00210 { 00211 TLOG(TLVL_WARNING) << GetTraceName() << ": Error writing fragment with seqID=" << fragment.sequenceID(); 00212 return CopyStatus::kErrorNotRequiringException; 00213 } 00214 00215 TLOG(5) << GetTraceName() << ": Successfully sent Fragment with seqID=" << fragment.sequenceID(); 00216 return CopyStatus::kSuccess; 00217 } 00218 else 00219 { 00220 TLOG(TLVL_WARNING) << GetTraceName() << ": Fragment invalid for shared memory! " 00221 << "fragment address and size = " 00222 << fragAddr << " " << fragSize << " " 00223 << "sequence ID, fragment ID, and type = " 00224 << fragment.sequenceID() << " " 00225 << fragment.fragmentID() << " " 00226 << fragment.typeString() ; 00227 return CopyStatus::kErrorNotRequiringException; 00228 } 00229 00230 TLOG(TLVL_WARNING) << GetTraceName() << ": Unreachable code reached!" ; 00231 return CopyStatus::kErrorNotRequiringException; 00232 } 00233 00234 bool artdaq::ShmemTransfer::isRunning() 00235 { 00236 bool ret = false; 00237 switch (role()) 00238 { 00239 case TransferInterface::Role::kSend: 00240 ret = shm_manager_->IsValid(); 00241 break; 00242 case TransferInterface::Role::kReceive: 00243 ret = shm_manager_->GetAttachedCount() > 1; 00244 break; 00245 } 00246 return ret; 00247 00248 } 00249 00250 void artdaq::ShmemTransfer::flush_buffers() 00251 { 00252 for (size_t ii = 0; ii < shm_manager_->size(); ++ii) 00253 { 00254 shm_manager_->MarkBufferEmpty(ii, true); 00255 } 00256 } 00257 00258 DEFINE_ARTDAQ_TRANSFER(artdaq::ShmemTransfer) 00259 00260 // Local Variables: 00261 // mode: c++ 00262 // End: