1 #include "artdaq/TransferPlugins/ShmemTransfer.hh"
2 #include "cetlib_except/exception.h"
29 throw cet::exception(
"ConfigurationException",
"Buffer Count is too large for Shmem transfer!");
32 auto shmKey = pset.get<uint32_t>(
"shm_key_offset", 0);
40 shm_manager_ = std::make_unique<SharedMemoryFragmentManager>(shmKey, 0, 0);
46 TLOG_ARB(5, uniqueLabel()) <<
"~ShmemTransfer called" << TLOG_ENDL;
47 shm_manager_.reset(
nullptr);
48 TLOG_ARB(5, uniqueLabel()) <<
"~ShmemTransfer done" << TLOG_ENDL;
52 size_t receiveTimeout)
54 auto waitStart = std::chrono::steady_clock::now();
55 while (!shm_manager_->ReadyForRead() && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
59 if (!shm_manager_->ReadyForRead())
61 int64_t loopCount = 0;
62 size_t sleepTime = 1000;
63 int64_t nloops = (receiveTimeout - 1000) / sleepTime;
65 while (!shm_manager_->ReadyForRead() && loopCount < nloops)
74 if (shm_manager_->ReadyForRead())
76 auto sts = shm_manager_->ReadFragment(fragment);
78 if (sts != 0)
return RECV_TIMEOUT;
80 if (fragment.type() != artdaq::Fragment::DataFragmentType)
82 TLOG_ARB(8, uniqueLabel()) <<
"Recvd frag from shmem, type=" << fragment.typeString() <<
", sequenceID=" << std::to_string(fragment.sequenceID()) <<
", source_rank=" << source_rank() << TLOG_ENDL;
93 auto waitStart = std::chrono::steady_clock::now();
94 while (!shm_manager_->ReadyForRead() && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
98 if (!shm_manager_->ReadyForRead())
100 int64_t loopCount = 0;
101 size_t sleepTime = 1000;
102 int64_t nloops = (receiveTimeout - 1000) / sleepTime;
104 while (!shm_manager_->ReadyForRead() && loopCount < nloops)
113 if (shm_manager_->ReadyForRead())
115 auto sts = shm_manager_->ReadFragmentHeader(header);
117 if (sts != 0)
return RECV_TIMEOUT;
119 if (header.type != artdaq::Fragment::DataFragmentType)
121 TLOG_ARB(8, uniqueLabel()) <<
"Recvd fragment header from shmem, type=" << (int)header.type <<
", sequenceID=" << std::to_string(header.sequence_id) <<
", source_rank=" << source_rank() << TLOG_ENDL;
124 return source_rank();
132 auto sts = shm_manager_->ReadFragmentData(destination, word_count);
134 TLOG_TRACE(uniqueLabel()) <<
"Return status from ReadFragmentData is " << sts << TLOG_ENDL;
136 if (sts != 0)
return RECV_TIMEOUT;
138 return source_rank();
146 return sendFragment(std::move(fragment), send_timeout_usec,
false);
151 size_t send_timeout_usec)
153 return sendFragment(std::move(fragment), send_timeout_usec,
true);
157 artdaq::ShmemTransfer::sendFragment(artdaq::Fragment&& fragment,
size_t send_timeout_usec,
bool reliableMode)
160 if (send_timeout_usec > 0 || reliableMode)
162 auto waitStart = std::chrono::steady_clock::now();
163 while (!shm_manager_->ReadyForWrite(!reliableMode) && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
167 if (!shm_manager_->ReadyForWrite(!reliableMode))
169 int64_t loopCount = 0;
170 size_t sleepTime = 1000;
171 int64_t nloops = (send_timeout_usec - 1000) / sleepTime;
173 while (reliableMode && !shm_manager_->ReadyForWrite(!reliableMode) && (send_timeout_usec == 0 || loopCount < nloops))
181 TLOG_ARB(5, uniqueLabel()) <<
"Either write has timed out or buffer ready" << TLOG_ENDL;
184 if (shm_manager_->ReadyForWrite(!reliableMode))
186 TLOG_ARB(5, uniqueLabel()) <<
"Sending fragment with seqID=" << std::to_string(fragment.sequenceID()) << TLOG_ENDL;
187 artdaq::RawDataType* fragAddr = fragment.headerAddress();
188 size_t fragSize = fragment.size() *
sizeof(artdaq::RawDataType);
192 if (fragment.type() != artdaq::Fragment::InvalidFragmentType && fragSize < (max_fragment_size_words_ *
sizeof(artdaq::RawDataType)))
194 auto sts = shm_manager_->WriteFragment(std::move(fragment), !reliableMode);
195 if (sts != 0)
return CopyStatus::kErrorNotRequiringException;
197 TLOG_ARB(5, uniqueLabel()) <<
"Fragment send successfully" << TLOG_ENDL;
198 return CopyStatus::kSuccess;
202 TLOG_WARNING(uniqueLabel()) <<
"Fragment invalid for shared memory! "
203 <<
"fragment address and size = "
204 << fragAddr <<
" " << fragSize <<
" "
205 <<
"sequence ID, fragment ID, and type = "
206 << fragment.sequenceID() <<
" "
207 << fragment.fragmentID() <<
" "
208 << fragment.typeString() << TLOG_ENDL;
209 return CopyStatus::kErrorNotRequiringException;
213 TLOG_ARB(5, uniqueLabel()) <<
"Fragment Send Timeout!" << TLOG_ENDL;
214 return CopyStatus::kTimeout;
size_t buffer_count_
The number of Fragment transfers the TransferInterface can handle simultaneously. ...
CopyStatus moveFragment(Fragment &&fragment, size_t send_timeout_usec=std::numeric_limits< size_t >::max()) override
Move a Fragment to the destination.
static const int RECV_TIMEOUT
Value to be returned upon receive timeout. Because receivers otherwise return rank, this is also the limit on the number of ranks that artdaq currently supports.
This TransferInterface is a Receiver.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
CopyStatus copyFragment(Fragment &fragment, size_t send_timeout_usec=std::numeric_limits< size_t >::max()) override
Copy a Fragment to the destination. May be unreliable.
ShmemTransfer(fhicl::ParameterSet const &pset, Role role)
ShmemTransfer Constructor.
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
virtual ~ShmemTransfer() noexcept
ShmemTransfer Destructor.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
std::string uniqueLabel() const
Get the unique label of this TransferInterface instance.
A TransferInterface implementation plugin that transfers data using Shared Memory.
This interface defines the functions used to transfer data between artdaq applications.
int receiveFragment(Fragment &fragment, size_t receiveTimeout) override
Receive a Fragment from Shared Memory.
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.
const size_t max_fragment_size_words_
The maximum size of the transferred Fragment objects, in artdaq::Fragment::RawDataType words...