1 #include "artdaq/DAQdata/Globals.hh"
2 #define TRACE_NAME (app_name + "_ShmemTransfer").c_str()
3 #include "TRACE/tracemf.h"
5 #include "artdaq/TransferPlugins/ShmemTransfer.hh"
7 #include <boost/lexical_cast.hpp>
15 TLOG(TLVL_DEBUG + 32) << GetTraceName() <<
"Constructor BEGIN";
29 auto partition = GetPartitionNumber() + 1;
31 auto shmKey = pset.get<uint32_t>(
"shm_key_offset", 0) + (partition << 24) + ((
source_rank() & 0xFFF) << 12) + (
destination_rank() & 0xFFF);
34 if (pset.has_key(
"shm_key"))
36 shmKey = pset.get<uint32_t>(
"shm_key");
41 shm_manager_ = std::make_unique<SharedMemoryFragmentManager>(shmKey,
buffer_count_,
max_fragment_size_words_ *
sizeof(artdaq::RawDataType), pset.get<
size_t>(
"stale_buffer_timeout_usec", 100 * 1000000));
45 shm_manager_ = std::make_unique<SharedMemoryFragmentManager>(shmKey);
47 TLOG(TLVL_DEBUG + 32) << GetTraceName() <<
"Constructor END";
52 TLOG(TLVL_DEBUG + 34) << GetTraceName() <<
" ~ShmemTransfer called - " << uniqueLabel();
53 shm_manager_.reset(
nullptr);
54 TLOG(TLVL_DEBUG + 34) << GetTraceName() <<
" ~ShmemTransfer done - " << uniqueLabel();
58 size_t receiveTimeout)
60 auto waitStart = std::chrono::steady_clock::now();
61 while (!shm_manager_->ReadyForRead() && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
65 if (!shm_manager_->ReadyForRead())
67 int64_t loopCount = 0;
68 size_t sleepTime = 1000;
69 int64_t nloops = (receiveTimeout - 1000) / sleepTime;
71 while (!shm_manager_->ReadyForRead() && loopCount < nloops)
77 if (!shm_manager_->ReadyForRead() && shm_manager_->IsEndOfData())
82 TLOG(TLVL_DEBUG + 33) << GetTraceName() <<
"receiveFragment ReadyForRead=" << shm_manager_->ReadyForRead();
84 if (shm_manager_->ReadyForRead())
86 auto sts = shm_manager_->ReadFragment(fragment);
90 TLOG(TLVL_DEBUG + 33) <<
"Non-zero status (" << sts <<
") returned from ReadFragment, returning...";
94 if (fragment.type() != artdaq::Fragment::DataFragmentType)
96 TLOG(TLVL_DEBUG + 38) << GetTraceName() <<
"Recvd frag from shmem, type=" << fragment.typeString() <<
", sequenceID=" << fragment.sequenceID() <<
", source_rank=" << source_rank();
107 auto waitStart = std::chrono::steady_clock::now();
108 while (!shm_manager_->ReadyForRead() && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
112 if (!shm_manager_->ReadyForRead())
114 int64_t loopCount = 0;
115 size_t sleepTime = 1000;
116 int64_t nloops = (receiveTimeout - 1000) / sleepTime;
118 while (!shm_manager_->ReadyForRead() && loopCount < nloops)
125 if (!shm_manager_->ReadyForRead() && shm_manager_->IsEndOfData())
132 if (shm_manager_->ReadyForRead())
134 auto sts = shm_manager_->ReadFragmentHeader(header);
138 TLOG(TLVL_DEBUG + 33) <<
"Non-zero status (" << sts <<
") returned from ReadFragmentHeader, returning...";
142 if (header.type != artdaq::Fragment::DataFragmentType)
144 TLOG(TLVL_DEBUG + 38) << GetTraceName() <<
"Recvd fragment header from shmem, type=" <<
static_cast<int>(header.type)
145 <<
", sequenceID=" << header.sequence_id <<
", source_rank=" << source_rank();
148 return source_rank();
156 auto sts = shm_manager_->ReadFragmentData(destination, word_count);
158 TLOG(TLVL_DEBUG + 33) << GetTraceName() <<
"Return status from ReadFragmentData is " << sts;
162 TLOG(TLVL_DEBUG + 33) <<
"Non-zero status (" << sts <<
") returned from ReadFragmentData, returning...";
166 return source_rank();
174 return sendFragment(Fragment(fragment), send_timeout_usec,
false);
180 return sendFragment(std::move(fragment), 0,
true);
184 artdaq::ShmemTransfer::sendFragment(artdaq::Fragment&& fragment,
size_t send_timeout_usec,
bool reliableMode)
188 shm_manager_->Attach();
191 TLOG(TLVL_ERROR) << GetTraceName() <<
"Attempted to send Fragment when not attached to Shared Memory! Returning kErrorNotRequiringException, and dropping data!";
192 return CopyStatus::kErrorNotRequiringException;
195 shm_manager_->SetRank(my_rank);
198 TLOG(TLVL_DEBUG + 34) << GetTraceName() <<
"Sending fragment with seqID=" << fragment.sequenceID();
199 artdaq::RawDataType* fragAddr = fragment.headerAddress();
200 size_t fragSize = fragment.size() *
sizeof(artdaq::RawDataType);
204 if (fragment.type() != artdaq::Fragment::InvalidFragmentType && fragSize < (max_fragment_size_words_ *
sizeof(artdaq::RawDataType)))
206 auto seq = fragment.sequenceID();
207 TLOG(TLVL_DEBUG + 34) << GetTraceName() <<
"Writing fragment with seqID=" << seq;
208 auto sts = shm_manager_->WriteFragment(std::move(fragment), !reliableMode, send_timeout_usec);
211 TLOG(TLVL_WARNING) << GetTraceName() <<
"Timeout writing fragment with seqID=" << seq;
212 return CopyStatus::kTimeout;
216 TLOG(TLVL_WARNING) << GetTraceName() <<
"Error writing fragment with seqID=" << seq;
217 return CopyStatus::kErrorNotRequiringException;
220 TLOG(TLVL_DEBUG + 34) << GetTraceName() <<
"Successfully sent Fragment with seqID=" << seq;
221 return CopyStatus::kSuccess;
224 TLOG(TLVL_WARNING) << GetTraceName() <<
"Fragment invalid for shared memory! "
225 <<
"fragment address and size = "
226 << fragAddr <<
" " << fragSize <<
" "
227 <<
"sequence ID, fragment ID, and type = "
228 << fragment.sequenceID() <<
" "
229 << fragment.fragmentID() <<
" "
230 << fragment.typeString();
231 return CopyStatus::kErrorNotRequiringException;
233 TLOG(TLVL_WARNING) << GetTraceName() <<
"Unreachable code reached!";
234 return CopyStatus::kErrorNotRequiringException;
243 ret = shm_manager_->IsValid() && !shm_manager_->IsEndOfData();
246 ret = shm_manager_->GetAttachedCount() > 1;
254 for (
size_t ii = 0; ii < shm_manager_->size(); ++ii)
256 shm_manager_->MarkBufferEmpty(ii,
true);
size_t buffer_count_
The number of Fragment transfers the TransferInterface can handle simultaneously. ...
virtual int source_rank() const
Get the source rank for this TransferInterface instance.
CopyStatus transfer_fragment_reliable_mode(Fragment &&fragment) override
Transfer a Fragment to the destination. This should be reliable, if the underlying transport mechanis...
CopyStatus transfer_fragment_min_blocking_mode(Fragment const &fragment, size_t send_timeout_usec) override
Transfer a Fragment to the destination. May not necessarily be reliable, but will not block longer th...
This TransferInterface is a Receiver.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
void flush_buffers() override
Flush any in-flight data. This should be used by the receiver after the receive loop has ended...
ShmemTransfer(fhicl::ParameterSet const &pset, Role role)
ShmemTransfer Constructor.
Value that is to be returned when a Transfer plugin determines that no more data will be arriving...
This TransferInterface is a Sender.
virtual ~ShmemTransfer() noexcept
ShmemTransfer Destructor.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
This interface defines the functions used to transfer data between artdaq applications.
virtual int destination_rank() const
Get the destination rank for this TransferInterface instance.
int receiveFragment(Fragment &fragment, size_t receiveTimeout) override
Receive a Fragment from Shared Memory.
Value to be returned upon receive timeout.
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.
int receiveFragmentData(RawDataType *destination, size_t word_count) override
Receive the body of a Fragment to the given destination pointer.
const size_t max_fragment_size_words_
The maximum size of the transferred Fragment objects, in artdaq::Fragment::RawDataType words...