1 #define TRACE_NAME "ShmemTransfer"
3 #include "artdaq/TransferPlugins/ShmemTransfer.hh"
4 #include "cetlib_except/exception.h"
10 TLOG(TLVL_DEBUG) <<
GetTraceName() <<
": Constructor BEGIN";
32 throw cet::exception(
"ConfigurationException",
"Buffer Count is too large for Shmem transfer!");
36 if (partition > 127) partition = 127;
38 auto shmKey = pset.get<uint32_t>(
"shm_key_offset", 0) + (partition << 24) + ((
source_rank() & 0xFFF) << 12) + (
destination_rank() & 0xFFF);
41 if (pset.has_key(
"shm_key")) {
42 shmKey = pset.get<uint32_t>(
"shm_key");
51 shm_manager_ = std::make_unique<SharedMemoryFragmentManager>(shmKey, 0, 0);
53 TLOG(TLVL_DEBUG) <<
GetTraceName() <<
": Constructor END";
58 TLOG(5) << GetTraceName() <<
" ~ShmemTransfer called - " << uniqueLabel() ;
59 shm_manager_.reset(
nullptr);
60 TLOG(5) << GetTraceName() <<
" ~ShmemTransfer done - " << uniqueLabel() ;
64 size_t receiveTimeout)
66 auto waitStart = std::chrono::steady_clock::now();
67 while (!shm_manager_->ReadyForRead() && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
71 if (!shm_manager_->ReadyForRead())
73 int64_t loopCount = 0;
74 size_t sleepTime = 1000;
75 int64_t nloops = (receiveTimeout - 1000) / sleepTime;
77 while (!shm_manager_->ReadyForRead() && loopCount < nloops)
84 TLOG(TLVL_TRACE) << GetTraceName() <<
": receiveFragment ReadyForRead=" << shm_manager_->ReadyForRead();
86 if (shm_manager_->ReadyForRead())
88 auto sts = shm_manager_->ReadFragment(fragment);
90 if (sts != 0)
return RECV_TIMEOUT;
92 if (fragment.type() != artdaq::Fragment::DataFragmentType)
94 TLOG(8) << GetTraceName() <<
": Recvd frag from shmem, type=" << fragment.typeString() <<
", sequenceID=" << std::to_string(fragment.sequenceID()) <<
", source_rank=" << source_rank() ;
105 auto waitStart = std::chrono::steady_clock::now();
106 while (!shm_manager_->ReadyForRead() && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
110 if (!shm_manager_->ReadyForRead())
112 int64_t loopCount = 0;
113 size_t sleepTime = 1000;
114 int64_t nloops = (receiveTimeout - 1000) / sleepTime;
116 while (!shm_manager_->ReadyForRead() && loopCount < nloops)
123 if (!shm_manager_->ReadyForRead() && shm_manager_->IsEndOfData())
130 if (shm_manager_->ReadyForRead())
132 auto sts = shm_manager_->ReadFragmentHeader(header);
134 if (sts != 0)
return RECV_TIMEOUT;
136 if (header.type != artdaq::Fragment::DataFragmentType)
138 TLOG(8) << GetTraceName() <<
": Recvd fragment header from shmem, type=" << (int)header.type <<
", sequenceID=" << std::to_string(header.sequence_id) <<
", source_rank=" << source_rank() ;
141 return source_rank();
149 auto sts = shm_manager_->ReadFragmentData(destination, word_count);
151 TLOG(TLVL_TRACE) << GetTraceName() <<
": Return status from ReadFragmentData is " << sts ;
153 if (sts != 0)
return RECV_TIMEOUT;
155 return source_rank();
163 return sendFragment(std::move(fragment), send_timeout_usec,
false);
169 return sendFragment(std::move(fragment), 0,
true);
173 artdaq::ShmemTransfer::sendFragment(artdaq::Fragment&& fragment,
size_t send_timeout_usec,
bool reliableMode)
175 if (!shm_manager_->IsValid()) {
176 shm_manager_->Attach();
177 if (!shm_manager_->IsValid()) {
178 TLOG(TLVL_ERROR) << GetTraceName() <<
": Attempted to send Fragment when not attached to Shared Memory! Returning kSuccess, and dropping data!";
179 return CopyStatus::kSuccess;
182 shm_manager_->SetRank(my_rank);
185 TLOG(5) << GetTraceName() <<
": Sending fragment with seqID=" << std::to_string(fragment.sequenceID()) ;
186 artdaq::RawDataType* fragAddr = fragment.headerAddress();
187 size_t fragSize = fragment.size() *
sizeof(artdaq::RawDataType);
191 if (fragment.type() != artdaq::Fragment::InvalidFragmentType && fragSize < (max_fragment_size_words_ *
sizeof(artdaq::RawDataType)))
193 TLOG(5) << GetTraceName() <<
": Writing fragment with seqID=" << std::to_string(fragment.sequenceID());
194 auto sts = shm_manager_->WriteFragment(std::move(fragment), !reliableMode, send_timeout_usec);
197 TLOG(TLVL_WARNING) << GetTraceName() <<
": Timeout writing fragment with seqID=" << std::to_string(fragment.sequenceID());
198 return CopyStatus::kTimeout;
202 TLOG(TLVL_WARNING) << GetTraceName() <<
": Error writing fragment with seqID=" << std::to_string(fragment.sequenceID());
203 return CopyStatus::kErrorNotRequiringException;
206 TLOG(5) << GetTraceName() <<
": Successfully sent Fragment with seqID=" << std::to_string(fragment.sequenceID());
207 return CopyStatus::kSuccess;
211 TLOG(TLVL_WARNING) << GetTraceName() <<
": Fragment invalid for shared memory! "
212 <<
"fragment address and size = "
213 << fragAddr <<
" " << fragSize <<
" "
214 <<
"sequence ID, fragment ID, and type = "
215 << fragment.sequenceID() <<
" "
216 << fragment.fragmentID() <<
" "
217 << fragment.typeString() ;
218 return CopyStatus::kErrorNotRequiringException;
221 TLOG(TLVL_WARNING) << GetTraceName() <<
": Unreachable code reached!" ;
222 return CopyStatus::kErrorNotRequiringException;
231 ret = shm_manager_->IsValid();
234 ret = shm_manager_->GetAttachedCount() > 1;
size_t buffer_count_
The number of Fragment transfers the TransferInterface can handle simultaneously. ...
virtual int source_rank() const
Get the source rank for this TransferInterface instance.
Value to be returned upon receive timeout.
std::string GetTraceName() const
Constructs a name suitable for TRACE messages.
CopyStatus copyFragment(Fragment &fragment, size_t send_timeout_usec) override
Copy a Fragment to the destination. May be unreliable.
This TransferInterface is a Receiver.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
const short partition_number_
The partition number of the DAQ.
Value that is to be returned when a Transfer plugin determines that no more data will be arriving...
ShmemTransfer(fhicl::ParameterSet const &pset, Role role)
ShmemTransfer Constructor.
CopyStatus moveFragment(Fragment &&fragment) override
Move a Fragment to the destination.
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
This TransferInterface is a Sender.
virtual ~ShmemTransfer() noexcept
ShmemTransfer Destructor.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
A TransferInterface implementation plugin that transfers data using Shared Memory.
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
This interface defines the functions used to transfer data between artdaq applications.
virtual int destination_rank() const
Get the destination rank for this TransferInterface instance.
int receiveFragment(Fragment &fragment, size_t receiveTimeout) override
Receive a Fragment from Shared Memory.
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.
const size_t max_fragment_size_words_
The maximum size of the transferred Fragment objects, in artdaq::Fragment::RawDataType words...