1 #define TRACE_NAME "ShmemTransfer"
2 #include "artdaq/TransferPlugins/ShmemTransfer.hh"
3 #include "cetlib_except/exception.h"
9 TLOG(TLVL_DEBUG) <<
uniqueLabel() <<
" ShmemTransfer{} begin";
31 throw cet::exception(
"ConfigurationException",
"Buffer Count is too large for Shmem transfer!");
34 auto shmKey = pset.get<uint32_t>(
"shm_key_offset", 0);
42 shm_manager_ = std::make_unique<SharedMemoryFragmentManager>(shmKey, 0, 0);
48 TLOG_ARB(5,
"ShmemTransfer") <<
"~ShmemTransfer called - " << uniqueLabel() << TLOG_ENDL;
49 shm_manager_.reset(
nullptr);
50 TLOG_ARB(5,
"ShmemTransfer") <<
"~ShmemTransfer done - " << uniqueLabel() << TLOG_ENDL;
54 size_t receiveTimeout)
56 auto waitStart = std::chrono::steady_clock::now();
57 while (!shm_manager_->ReadyForRead() && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
61 if (!shm_manager_->ReadyForRead())
63 int64_t loopCount = 0;
64 size_t sleepTime = 1000;
65 int64_t nloops = (receiveTimeout - 1000) / sleepTime;
67 while (!shm_manager_->ReadyForRead() && loopCount < nloops)
74 TLOG(4) << uniqueLabel() <<
" ShmemTransfer::receiveFragment ReadyForRead=" << shm_manager_->ReadyForRead();
76 if (shm_manager_->ReadyForRead())
78 auto sts = shm_manager_->ReadFragment(fragment);
80 if (sts != 0)
return RECV_TIMEOUT;
82 if (fragment.type() != artdaq::Fragment::DataFragmentType)
84 TLOG_ARB(8) << uniqueLabel() <<
" Recvd frag from shmem, type=" << fragment.typeString() <<
", sequenceID=" << std::to_string(fragment.sequenceID()) <<
", source_rank=" << source_rank() << TLOG_ENDL;
95 auto waitStart = std::chrono::steady_clock::now();
96 while (!shm_manager_->ReadyForRead() && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
100 if (!shm_manager_->ReadyForRead())
102 int64_t loopCount = 0;
103 size_t sleepTime = 1000;
104 int64_t nloops = (receiveTimeout - 1000) / sleepTime;
106 while (!shm_manager_->ReadyForRead() && loopCount < nloops)
115 if (shm_manager_->ReadyForRead())
117 auto sts = shm_manager_->ReadFragmentHeader(header);
119 if (sts != 0)
return RECV_TIMEOUT;
121 if (header.type != artdaq::Fragment::DataFragmentType)
123 TLOG_ARB(8) << uniqueLabel() <<
" Recvd fragment header from shmem, type=" << (int)header.type <<
", sequenceID=" << std::to_string(header.sequence_id) <<
", source_rank=" << source_rank() << TLOG_ENDL;
126 return source_rank();
134 auto sts = shm_manager_->ReadFragmentData(destination, word_count);
136 TLOG(4) << uniqueLabel() <<
" Return status from ReadFragmentData is " << sts << TLOG_ENDL;
138 if (sts != 0)
return RECV_TIMEOUT;
140 return source_rank();
148 return sendFragment(std::move(fragment), send_timeout_usec,
false);
153 size_t send_timeout_usec)
155 return sendFragment(std::move(fragment), send_timeout_usec,
true);
159 artdaq::ShmemTransfer::sendFragment(artdaq::Fragment&& fragment,
size_t send_timeout_usec,
bool reliableMode)
162 if (send_timeout_usec > 0 || reliableMode)
164 auto waitStart = std::chrono::steady_clock::now();
165 while (!shm_manager_->ReadyForWrite(!reliableMode) && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
169 if (!shm_manager_->ReadyForWrite(!reliableMode))
171 int64_t loopCount = 0;
172 size_t sleepTime = 1000;
173 int64_t nloops = (send_timeout_usec - 1000) / sleepTime;
175 while (reliableMode && !shm_manager_->ReadyForWrite(!reliableMode) && (send_timeout_usec == 0 || loopCount < nloops))
183 TLOG_ARB(5) << uniqueLabel() <<
" Either write has timed out or buffer ready" << TLOG_ENDL;
186 if (shm_manager_->ReadyForWrite(!reliableMode))
188 TLOG_ARB(5) << uniqueLabel() <<
" Sending fragment with seqID=" << std::to_string(fragment.sequenceID()) << TLOG_ENDL;
189 artdaq::RawDataType* fragAddr = fragment.headerAddress();
190 size_t fragSize = fragment.size() *
sizeof(artdaq::RawDataType);
194 if (fragment.type() != artdaq::Fragment::InvalidFragmentType && fragSize < (max_fragment_size_words_ *
sizeof(artdaq::RawDataType)))
196 auto sts = shm_manager_->WriteFragment(std::move(fragment), !reliableMode);
197 if (sts != 0)
return CopyStatus::kErrorNotRequiringException;
199 TLOG_ARB(5) << uniqueLabel() <<
" Fragment send successfully" << TLOG_ENDL;
200 return CopyStatus::kSuccess;
204 TLOG_WARNING(
"ShmemTransfer") << uniqueLabel() <<
" Fragment invalid for shared memory! "
205 <<
"fragment address and size = "
206 << fragAddr <<
" " << fragSize <<
" "
207 <<
"sequence ID, fragment ID, and type = "
208 << fragment.sequenceID() <<
" "
209 << fragment.fragmentID() <<
" "
210 << fragment.typeString() << TLOG_ENDL;
211 return CopyStatus::kErrorNotRequiringException;
215 TLOG_ARB(5) << uniqueLabel() <<
" Fragment Send Timeout!" << TLOG_ENDL;
216 return CopyStatus::kTimeout;
size_t buffer_count_
The number of Fragment transfers the TransferInterface can handle simultaneously. ...
CopyStatus moveFragment(Fragment &&fragment, size_t send_timeout_usec=std::numeric_limits< size_t >::max()) override
Move a Fragment to the destination.
static const int RECV_TIMEOUT
Value to be returned upon receive timeout. Because receivers otherwise return rank, this is also the limit on the number of ranks that artdaq currently supports.
This TransferInterface is a Receiver.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
CopyStatus copyFragment(Fragment &fragment, size_t send_timeout_usec=std::numeric_limits< size_t >::max()) override
Copy a Fragment to the destination. May be unreliable.
ShmemTransfer(fhicl::ParameterSet const &pset, Role role)
ShmemTransfer Constructor.
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
virtual ~ShmemTransfer() noexcept
ShmemTransfer Destructor.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
std::string uniqueLabel() const
Get the unique label of this TransferInterface instance.
A TransferInterface implementation plugin that transfers data using Shared Memory.
This interface defines the functions used to transfer data between artdaq applications.
int receiveFragment(Fragment &fragment, size_t receiveTimeout) override
Receive a Fragment from Shared Memory.
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.
const size_t max_fragment_size_words_
The maximum size of the transferred Fragment objects, in artdaq::Fragment::RawDataType words...