2 #include "artdaq/TransferPlugins/ShmemTransfer.hh"
12 , shm_key_(pset.get<int>(
"shm_key", std::hash<std::string>()(uniqueLabel())))
36 throw cet::exception(
"ConfigurationException",
"Buffer Count is too large for Shmem transfer!");
41 shm_segment_id_ = shmget(shm_key_, shmSize, 0666);
43 if (shm_segment_id_ == -1)
45 shm_segment_id_ = shmget(shm_key_, shmSize, IPC_CREAT | 0666);
48 TLOG_DEBUG(
uniqueLabel()) <<
"shm_key == " << shm_key_ <<
", shm_segment_id == " << shm_segment_id_ << TLOG_ENDL;
50 if (shm_segment_id_ > -1)
53 <<
"Created/fetched shared memory segment with ID = " << shm_segment_id_
54 <<
" and size " << shmSize
55 <<
" bytes" << TLOG_ENDL;
56 shm_ptr_ = (ShmStruct*)shmat(shm_segment_id_, 0, 0);
58 <<
"Attached to shared memory segment at address "
59 << std::hex << shm_ptr_ << std::dec << TLOG_ENDL;
60 if (shm_ptr_ && shm_ptr_ != (
void *)-1)
64 shm_ptr_->read_pos = 0;
65 shm_ptr_->write_pos = 0;
66 size_t offset =
sizeof(ShmStruct);
69 TLOG_DEBUG(
uniqueLabel()) <<
"Buffer " << ii <<
" is at 0x" << std::hex << offset << std::dec << TLOG_ENDL;
70 shm_ptr_->buffers[ii].fragmentSizeWords = 0;
71 shm_ptr_->buffers[ii].offset = offset;
72 shm_ptr_->buffers[ii].sem = BUFFER_EMPTY;
73 offset += max_fragment_size_words_ *
sizeof(artdaq::RawDataType);
79 TLOG_ERROR(
uniqueLabel()) <<
"Failed to attach to shared memory segment "
80 << shm_segment_id_ << TLOG_ENDL;
85 TLOG_ERROR(
uniqueLabel()) <<
"Failed to connect to shared memory segment"
86 <<
", errno = " << errno <<
". Please check "
87 <<
"if a stale shared memory segment needs to "
88 <<
"be cleaned up. (ipcs, ipcrm -m <segId>)" << TLOG_ENDL;
94 TRACE(5,
"ShmemTransfer::~ShmemTransfer called");
101 if (role_ == Role::kReceive && shm_segment_id_ > -1)
103 shmctl(shm_segment_id_, IPC_RMID, NULL);
105 TRACE(5,
"ShmemTransfer::~ShmemTransfer done");
108 bool artdaq::ShmemTransfer::readyForRead_()
110 return shm_ptr_->buffers[shm_ptr_->read_pos].sem == FRAGMENT_READY;
113 bool artdaq::ShmemTransfer::readyForWrite_()
115 return shm_ptr_->buffers[shm_ptr_->write_pos].sem == BUFFER_EMPTY;
119 size_t receiveTimeout)
123 auto waitStart = std::chrono::steady_clock::now();
124 while (!readyForRead_() && std::chrono::duration_cast<std::chrono::duration<
size_t, std::ratio<1, 1000000>>>(std::chrono::steady_clock::now() - waitStart).count() < 1000)
128 if (!readyForRead_())
130 int64_t loopCount = 0;
131 size_t sleepTime = 1000;
132 int64_t nloops = (receiveTimeout - 1000) / sleepTime;
134 while (!readyForRead_() && loopCount < nloops)
152 auto buf = &shm_ptr_->buffers[shm_ptr_->read_pos];
153 auto initCount = buf->writeCount;
155 TRACE(TRANSFER_RECEIVE1,
"Setting semaphore on buf %u", shm_ptr_->read_pos.load());
156 buf->sem = READING_FRAGMENT;
157 RawDataType* bufPtr = offsetToPtr(buf->offset);
159 fragment.resize(buf->fragmentSizeWords);
161 artdaq::RawDataType* fragAddr = fragment.headerAddress();
162 size_t fragSize = fragment.size() *
sizeof(artdaq::RawDataType);
164 memcpy(fragAddr, bufPtr, fragSize);
167 auto wordsOfHeaderAndMetadata = &*fragment.dataBegin() - &*fragment.headerBegin();
168 fragment.resize(buf->fragmentSizeWords - wordsOfHeaderAndMetadata);
170 if (buf->sem != READING_FRAGMENT || buf->writeCount != initCount)
176 shm_ptr_->read_pos++;
177 if (shm_ptr_->read_pos >= buffer_count_) shm_ptr_->read_pos = 0;
178 buf->sem = BUFFER_EMPTY;
180 if (fragment.type() != artdaq::Fragment::DataFragmentType)
182 TRACE(TRANSFER_RECEIVE2,
"Recvd frag from shmem, type=%d, sequenceID=%zu, source_rank=%d", (
int)fragment.type(), fragment.sequenceID(), source_rank());
185 return source_rank();
192 TLOG_ERROR(uniqueLabel()) <<
"Error in shared memory transfer plugin: pointer to shared memory segment is null, will sleep for "
193 << receiveTimeout / 1.0e6 <<
" seconds and then return a timeout" << TLOG_ENDL;
194 usleep(receiveTimeout);
202 return sendFragment(std::move(fragment), send_timeout_usec,
false);
207 size_t send_timeout_usec)
209 return sendFragment(std::move(fragment), send_timeout_usec,
true);
213 artdaq::ShmemTransfer::sendFragment(artdaq::Fragment&& fragment,
size_t send_timeout_usec,
bool reliableMode)
215 if (!shm_ptr_) {
return CopyStatus::kErrorNotRequiringException; }
219 if (send_timeout_usec > 0)
221 auto waitStart = std::chrono::steady_clock::now();
222 while (!readyForWrite_() && std::chrono::duration_cast<std::chrono::duration<
size_t, std::ratio<1, 1000000>>>(std::chrono::steady_clock::now() - waitStart).count() < 1000)
226 if (!readyForWrite_())
228 int64_t loopCount = 0;
229 size_t sleepTime = 1000;
230 int64_t nloops = (send_timeout_usec - 1000) / sleepTime;
232 while (reliableMode && !readyForWrite_() && loopCount < nloops)
243 if ((reliableMode && readyForWrite_()) || !reliableMode)
245 TRACE(TRANSFER_SEND2,
"Sending fragment with seqID=%zu", fragment.sequenceID());
246 artdaq::RawDataType* fragAddr = fragment.headerAddress();
247 size_t fragSize = fragment.size() *
sizeof(artdaq::RawDataType);
251 if (fragment.type() != artdaq::Fragment::InvalidFragmentType && fragSize < (max_fragment_size_words_ *
sizeof(artdaq::RawDataType)))
253 auto buf = &shm_ptr_->buffers[shm_ptr_->write_pos];
255 buf->sem = WRITING_FRAGMENT;
257 memcpy(offsetToPtr(buf->offset), fragAddr, fragSize);
259 buf->fragmentSizeWords = fragment.size();
261 shm_ptr_->write_pos++;
262 if (shm_ptr_->write_pos >= buffer_count_) shm_ptr_->write_pos = 0;
263 if (shm_ptr_->write_pos == shm_ptr_->read_pos && !reliableMode)
265 shm_ptr_->read_pos++;
266 if (shm_ptr_->read_pos >= buffer_count_) shm_ptr_->read_pos = 0;
268 buf->sem = FRAGMENT_READY;
270 return CopyStatus::kSuccess;
274 TLOG_WARNING(uniqueLabel()) <<
"Fragment invalid for shared memory! "
275 <<
"fragment address and size = "
276 << fragAddr <<
" " << fragSize <<
" "
277 <<
"sequence ID, fragment ID, and type = "
278 << fragment.sequenceID() <<
" "
279 << fragment.fragmentID() <<
" "
280 << ((int)fragment.type()) << TLOG_ENDL;
281 return CopyStatus::kErrorNotRequiringException;
285 return CopyStatus::kTimeout;
288 artdaq::RawDataType* artdaq::ShmemTransfer::offsetToPtr(
size_t offset)
290 auto res =
reinterpret_cast<RawDataType*
>(
reinterpret_cast<uint8_t*
>(shm_ptr_) + offset);
size_t buffer_count_
The number of Fragment transfers the TransferInterface can handle simultaneously. ...
CopyStatus moveFragment(Fragment &&fragment, size_t send_timeout_usec=std::numeric_limits< size_t >::max()) override
Move a Fragment to the destination.
static const int RECV_TIMEOUT
Value to be returned upon receive timeout. Because receivers otherwise return rank, this is also the limit on the number of ranks that artdaq currently supports.
This TransferInterface is a Receiver.
CopyStatus copyFragment(Fragment &fragment, size_t send_timeout_usec=std::numeric_limits< size_t >::max()) override
Copy a Fragment to the destination. May be unreliable.
ShmemTransfer(fhicl::ParameterSet const &pset, Role role)
ShmemTransfer Constructor.
virtual ~ShmemTransfer() noexcept
ShmemTransfer Destructor.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
std::string uniqueLabel() const
Get the unique label of this TransferInterface instance.
A TransferInterface implementation plugin that transfers data using Shared Memory.
This interface defines the functions used to transfer data between artdaq applications.
int receiveFragment(Fragment &fragment, size_t receiveTimeout) override
Receive a Fragment from Shared Memory.
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.
const size_t max_fragment_size_words_
The maximum size of the transferred Fragment objects, in artdaq::Fragment::RawDataType words...