artdaq  v2_02_03
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Pages
Shmem_transfer.cc
1 #include <sys/shm.h>
2 #include "artdaq/TransferPlugins/ShmemTransfer.hh"
3 
4 #define SHMEM_SLEEP 0
5 // ----------------------------------------------------------------------
6 
7 
8 artdaq::ShmemTransfer::ShmemTransfer(fhicl::ParameterSet const& pset, Role role) :
9  TransferInterface(pset, role)
10  , shm_segment_id_(-1)
11  , shm_ptr_(NULL)
12  , shm_key_(pset.get<int>("shm_key", std::hash<std::string>()(uniqueLabel())))
13  , role_(role)
14 {
15  // char* keyChars = getenv("ARTDAQ_SHM_KEY");
16  // if (keyChars != NULL && shm_key_ == static_cast<int>(std::hash<std::string>()(unique_label_))) {
17  // std::string keyString(keyChars);
18  // try {
19  // shm_key_ = boost::lexical_cast<int>(keyString);
20  // }
21  // catch (...) {
22  // std::stringstream errmsg;
23  // errmsg << uniqueLabel() << ": Problem performing lexical cast on " << keyString;
24  // ExceptionHandler(ExceptionHandlerRethrow::yes, errmsg.str());
25  // }
26  // }
27 
28  // JCF, Aug-16-2016
29 
30  // Note that there's a small but nonzero chance of a race condition
31  // here where another process creates the shared memory buffer
32  // between the first and second calls to shmget
33 
34  if (buffer_count_ > 100)
35  {
36  throw cet::exception("ConfigurationException", "Buffer Count is too large for Shmem transfer!");
37  }
38 
39  size_t shmSize = buffer_count_ * max_fragment_size_words_ * sizeof(artdaq::RawDataType) + sizeof(ShmStruct);
40 
41  shm_segment_id_ = shmget(shm_key_, shmSize, 0666);
42 
43  if (shm_segment_id_ == -1)
44  {
45  shm_segment_id_ = shmget(shm_key_, shmSize, IPC_CREAT | 0666);
46  }
47 
48  TLOG_DEBUG(uniqueLabel()) << "shm_key == " << shm_key_ << ", shm_segment_id == " << shm_segment_id_ << TLOG_ENDL;
49 
50  if (shm_segment_id_ > -1)
51  {
52  TLOG_DEBUG(uniqueLabel())
53  << "Created/fetched shared memory segment with ID = " << shm_segment_id_
54  << " and size " << shmSize
55  << " bytes" << TLOG_ENDL;
56  shm_ptr_ = (ShmStruct*)shmat(shm_segment_id_, 0, 0);
57  TLOG_DEBUG(uniqueLabel())
58  << "Attached to shared memory segment at address "
59  << std::hex << shm_ptr_ << std::dec << TLOG_ENDL;
60  if (shm_ptr_ && shm_ptr_ != (void *)-1)
61  {
62  if (role_ == Role::kReceive)
63  {
64  shm_ptr_->read_pos = 0;
65  shm_ptr_->write_pos = 0;
66  size_t offset = sizeof(ShmStruct);
67  for (size_t ii = 0; ii < buffer_count_; ++ii)
68  {
69  TLOG_DEBUG(uniqueLabel()) << "Buffer " << ii << " is at 0x" << std::hex << offset << std::dec << TLOG_ENDL;
70  shm_ptr_->buffers[ii].fragmentSizeWords = 0;
71  shm_ptr_->buffers[ii].offset = offset;
72  shm_ptr_->buffers[ii].sem = BUFFER_EMPTY;
73  offset += max_fragment_size_words_ * sizeof(artdaq::RawDataType);
74  }
75  }
76  }
77  else
78  {
79  TLOG_ERROR(uniqueLabel()) << "Failed to attach to shared memory segment "
80  << shm_segment_id_ << TLOG_ENDL;
81  }
82  }
83  else
84  {
85  TLOG_ERROR(uniqueLabel()) << "Failed to connect to shared memory segment"
86  << ", errno = " << errno << ". Please check "
87  << "if a stale shared memory segment needs to "
88  << "be cleaned up. (ipcs, ipcrm -m <segId>)" << TLOG_ENDL;
89  }
90 }
91 
93 {
94  TRACE(5, "ShmemTransfer::~ShmemTransfer called");
95  if (shm_ptr_)
96  {
97  shmdt(shm_ptr_);
98  shm_ptr_ = NULL;
99  }
100 
101  if (role_ == Role::kReceive && shm_segment_id_ > -1)
102  {
103  shmctl(shm_segment_id_, IPC_RMID, NULL);
104  }
105  TRACE(5, "ShmemTransfer::~ShmemTransfer done");
106 }
107 
108 bool artdaq::ShmemTransfer::readyForRead_()
109 {
110  return shm_ptr_->buffers[shm_ptr_->read_pos].sem == FRAGMENT_READY;
111 }
112 
113 bool artdaq::ShmemTransfer::readyForWrite_()
114 {
115  return shm_ptr_->buffers[shm_ptr_->write_pos].sem == BUFFER_EMPTY;
116 }
117 
118 int artdaq::ShmemTransfer::receiveFragment(artdaq::Fragment& fragment,
119  size_t receiveTimeout)
120 {
121  if (shm_ptr_)
122  {
123  auto waitStart = std::chrono::steady_clock::now();
124  while (!readyForRead_() && std::chrono::duration_cast<std::chrono::duration<size_t, std::ratio<1, 1000000>>>(std::chrono::steady_clock::now() - waitStart).count() < 1000)
125  {
126  // BURN THAT CPU!
127  }
128  if (!readyForRead_())
129  {
130  int64_t loopCount = 0;
131  size_t sleepTime = 1000; // microseconds
132  int64_t nloops = (receiveTimeout - 1000) / sleepTime;
133 
134  while (!readyForRead_() && loopCount < nloops)
135  {
136  usleep(sleepTime);
137  ++loopCount;
138  }
139  }
140 
141  //TLOG_DEBUG(uniqueLabel()) << "delta_=" << delta_() << ", rp=" << (int)shm_ptr_->read_pos << ", wp=" << (int)shm_ptr_->write_pos << ", loopCount=" << loopCount << ", nloops=" << nloops << TLOG_ENDL;
142 
143  if (readyForRead_())
144  {
145  // JCF, Jul-7-2016
146 
147  // Calling artdaq::Fragment::resize with the argument
148  // shm_ptr_->fragmentSizeWords actually allocates more memory
149  // for "fragment" than is needed as shm_ptr_->fragmentSizeWords
150  // is the FULL size of the received fragment, not just the size
151  // of its payload. We correct for this below.
152  auto buf = &shm_ptr_->buffers[shm_ptr_->read_pos];
153  auto initCount = buf->writeCount;
154 
155  TRACE(TRANSFER_RECEIVE1, "Setting semaphore on buf %u", shm_ptr_->read_pos.load());
156  buf->sem = READING_FRAGMENT;
157  RawDataType* bufPtr = offsetToPtr(buf->offset);
158  //TLOG_DEBUG(uniqueLabel()) << "Pointer is " << bufPtr << TLOG_ENDL;
159  fragment.resize(buf->fragmentSizeWords);
160 
161  artdaq::RawDataType* fragAddr = fragment.headerAddress();
162  size_t fragSize = fragment.size() * sizeof(artdaq::RawDataType);
163  //TLOG_DEBUG(uniqueLabel()) << "Copying fragment of size " << fragSize << " from buffer " << (int)shm_ptr_->read_pos << " at 0x" << std::hex << buf->offset << std::dec << TLOG_ENDL;
164  memcpy(fragAddr, bufPtr, fragSize);
165  //TLOG_DEBUG(uniqueLabel()) << "Done with copy" << TLOG_ENDL;
166 
167  auto wordsOfHeaderAndMetadata = &*fragment.dataBegin() - &*fragment.headerBegin();
168  fragment.resize(buf->fragmentSizeWords - wordsOfHeaderAndMetadata);
169 
170  if (buf->sem != READING_FRAGMENT || buf->writeCount != initCount)
171  {
172  //TLOG_WARNING(uniqueLabel()) << "Semaphore was unset! This buffer has been clobbered!" << TLOG_ENDL;
173  return RECV_TIMEOUT; // Buffer was clobbered by a non-reliable writer...
174  }
175 
176  shm_ptr_->read_pos++;
177  if (shm_ptr_->read_pos >= buffer_count_) shm_ptr_->read_pos = 0;
178  buf->sem = BUFFER_EMPTY;
179 
180  if (fragment.type() != artdaq::Fragment::DataFragmentType)
181  {
182  TRACE(TRANSFER_RECEIVE2, "Recvd frag from shmem, type=%d, sequenceID=%zu, source_rank=%d", (int)fragment.type(), fragment.sequenceID(), source_rank());
183  }
184 
185  return source_rank();
186  }
187 
189  }
190  else
191  {
192  TLOG_ERROR(uniqueLabel()) << "Error in shared memory transfer plugin: pointer to shared memory segment is null, will sleep for "
193  << receiveTimeout / 1.0e6 << " seconds and then return a timeout" << TLOG_ENDL;
194  usleep(receiveTimeout);
195  return artdaq::TransferInterface::RECV_TIMEOUT; // Should we EVER get shm_ptr_ == 0?
196  }
197 }
198 
200 artdaq::ShmemTransfer::copyFragment(artdaq::Fragment& fragment, size_t send_timeout_usec)
201 {
202  return sendFragment(std::move(fragment), send_timeout_usec, false);
203 }
204 
206 artdaq::ShmemTransfer::moveFragment(artdaq::Fragment&& fragment,
207  size_t send_timeout_usec)
208 {
209  return sendFragment(std::move(fragment), send_timeout_usec, true);
210 }
211 
213 artdaq::ShmemTransfer::sendFragment(artdaq::Fragment&& fragment, size_t send_timeout_usec, bool reliableMode)
214 {
215  if (!shm_ptr_) { return CopyStatus::kErrorNotRequiringException; }
216 
217 
218  // wait for the shm to become free, if requested
219  if (send_timeout_usec > 0)
220  {
221  auto waitStart = std::chrono::steady_clock::now();
222  while (!readyForWrite_() && std::chrono::duration_cast<std::chrono::duration<size_t, std::ratio<1, 1000000>>>(std::chrono::steady_clock::now() - waitStart).count() < 1000)
223  {
224  // BURN THAT CPU!
225  }
226  if (!readyForWrite_())
227  {
228  int64_t loopCount = 0;
229  size_t sleepTime = 1000; // microseconds
230  int64_t nloops = (send_timeout_usec - 1000) / sleepTime;
231 
232  while (reliableMode && !readyForWrite_() && loopCount < nloops)
233  {
234  usleep(sleepTime);
235  ++loopCount;
236  }
237  }
238  }
239 
240  //TLOG_DEBUG(uniqueLabel()) << "delta_=" << delta_() << ", rp=" << (int)shm_ptr_->read_pos << ", wp=" << (int)shm_ptr_->write_pos << TLOG_ENDL;
241 
242  // copy the fragment if the shm is available
243  if ((reliableMode && readyForWrite_()) || !reliableMode)
244  {
245  TRACE(TRANSFER_SEND2, "Sending fragment with seqID=%zu", fragment.sequenceID());
246  artdaq::RawDataType* fragAddr = fragment.headerAddress();
247  size_t fragSize = fragment.size() * sizeof(artdaq::RawDataType);
248 
249  // 10-Sep-2013, KAB - protect against large events and
250  // invalid events (and large, invalid events)
251  if (fragment.type() != artdaq::Fragment::InvalidFragmentType && fragSize < (max_fragment_size_words_ * sizeof(artdaq::RawDataType)))
252  {
253  auto buf = &shm_ptr_->buffers[shm_ptr_->write_pos];
254  //TLOG_DEBUG(uniqueLabel()) << "Waiting for buffer " << (int)shm_ptr_->write_pos << TLOG_ENDL;
255  buf->sem = WRITING_FRAGMENT;
256  //TLOG_DEBUG(uniqueLabel()) << "Copying fragment with size " << fragSize << " into buffer " << (int)shm_ptr_->write_pos << " at 0x" << std::hex << buf->offset << std::dec << TLOG_ENDL;
257  memcpy(offsetToPtr(buf->offset), fragAddr, fragSize);
258  //TLOG_DEBUG(uniqueLabel()) << "Done with copy" << TLOG_ENDL;
259  buf->fragmentSizeWords = fragment.size();
260 
261  shm_ptr_->write_pos++;
262  if (shm_ptr_->write_pos >= buffer_count_) shm_ptr_->write_pos = 0;
263  if (shm_ptr_->write_pos == shm_ptr_->read_pos && !reliableMode)
264  {
265  shm_ptr_->read_pos++;
266  if (shm_ptr_->read_pos >= buffer_count_) shm_ptr_->read_pos = 0;
267  }
268  buf->sem = FRAGMENT_READY;
269 
270  return CopyStatus::kSuccess;
271  }
272  else
273  {
274  TLOG_WARNING(uniqueLabel()) << "Fragment invalid for shared memory! "
275  << "fragment address and size = "
276  << fragAddr << " " << fragSize << " "
277  << "sequence ID, fragment ID, and type = "
278  << fragment.sequenceID() << " "
279  << fragment.fragmentID() << " "
280  << ((int)fragment.type()) << TLOG_ENDL;
281  return CopyStatus::kErrorNotRequiringException;
282  }
283  }
284 
285  return CopyStatus::kTimeout;
286 }
287 
288 artdaq::RawDataType* artdaq::ShmemTransfer::offsetToPtr(size_t offset)
289 {
290  auto res = reinterpret_cast<RawDataType*>(reinterpret_cast<uint8_t*>(shm_ptr_) + offset);
291  //TLOG_DEBUG(uniqueLabel()) << std::hex << "base=" << shm_ptr_ << ", offset=0x" << offset << ", res=" << res << std::dec << TLOG_ENDL;
292  return res;
293 }
294 
295 DEFINE_ARTDAQ_TRANSFER(artdaq::ShmemTransfer)
296 
297 // Local Variables:
298 // mode: c++
299 // End:
size_t buffer_count_
The number of Fragment transfers the TransferInterface can handle simultaneously. ...
CopyStatus moveFragment(Fragment &&fragment, size_t send_timeout_usec=std::numeric_limits< size_t >::max()) override
Move a Fragment to the destination.
static const int RECV_TIMEOUT
Value to be returned upon receive timeout. Because receivers otherwise return rank, this is also the limit on the number of ranks that artdaq currently supports.
This TransferInterface is a Receiver.
CopyStatus copyFragment(Fragment &fragment, size_t send_timeout_usec=std::numeric_limits< size_t >::max()) override
Copy a Fragment to the destination. May be unreliable.
ShmemTransfer(fhicl::ParameterSet const &pset, Role role)
ShmemTransfer Constructor.
virtual ~ShmemTransfer() noexcept
ShmemTransfer Destructor.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
std::string uniqueLabel() const
Get the unique label of this TransferInterface instance.
A TransferInterface implementation plugin that transfers data using Shared Memory.
This interface defines the functions used to transfer data between artdaq applications.
int receiveFragment(Fragment &fragment, size_t receiveTimeout) override
Receive a Fragment from Shared Memory.
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.
const size_t max_fragment_size_words_
The maximum size of the transferred Fragment objects, in artdaq::Fragment::RawDataType words...