artdaq_core  v3_00_03
SharedMemoryManager.cc
1 #include <sys/shm.h>
2 #include <thread>
3 #include "tracemf.h"
4 #include "cetlib_except/exception.h"
5 #include "artdaq-core/Core/SharedMemoryManager.hh"
6 #include "artdaq-core/Utilities/TraceLock.hh"
7 #include "SharedMemoryManager.hh"
8 
9 artdaq::SharedMemoryManager::SharedMemoryManager(uint32_t shm_key, size_t buffer_count, size_t buffer_size, uint64_t buffer_timeout_us, bool destructive_read_mode)
10  : shm_segment_id_(-1)
11  , shm_ptr_(NULL)
12  , shm_key_(shm_key)
13  , manager_id_(-1)
14  , buffer_mutexes_()
15  , last_seen_id_(0)
16 {
17  requested_shm_parameters_.buffer_count = buffer_count;
18  requested_shm_parameters_.buffer_size = buffer_size;
19  requested_shm_parameters_.buffer_timeout_us = buffer_timeout_us;
20  requested_shm_parameters_.destructive_read_mode = destructive_read_mode;
21 
22  Attach();
23 }
24 
26 {
27  TLOG_ARB(5, "SharedMemoryManager") << "~SharedMemoryManager called" << TLOG_ENDL;
28  Detach();
29  TLOG_ARB(5, "SharedMemoryManager") << "~SharedMemoryManager done" << TLOG_ENDL;
30 }
31 
33 {
34  if (IsValid())
35  {
36  if (manager_id_ == 0) return;
37  Detach();
38  }
39  auto start_time = std::chrono::steady_clock::now();
40  last_seen_id_ = 0;
41  size_t shmSize = requested_shm_parameters_.buffer_count * (requested_shm_parameters_.buffer_size + sizeof(ShmBuffer)) + sizeof(ShmStruct);
42 
43  shm_segment_id_ = shmget(shm_key_, shmSize, 0666);
44  if (shm_segment_id_ == -1 && requested_shm_parameters_.buffer_count > 0 && manager_id_ <= 0)
45  {
46  TLOG_DEBUG("SharedMemoryManager") << "Creating shared memory segment with key 0x" << std::hex << shm_key_ << TLOG_ENDL;
47  shm_segment_id_ = shmget(shm_key_, shmSize, IPC_CREAT | 0666);
48  manager_id_ = 0;
49  }
50 
51  while (shm_segment_id_ == -1 && TimeUtils::GetElapsedTimeMilliseconds(start_time) < 1000)
52  {
53  shm_segment_id_ = shmget(shm_key_, shmSize, 0666);
54 
55  }
56  TLOG_DEBUG("SharedMemoryManager") << "shm_key == 0x" << std::hex << shm_key_ << ", shm_segment_id == " << shm_segment_id_ << TLOG_ENDL;
57 
58  if (shm_segment_id_ > -1)
59  {
60  TLOG_DEBUG("SharedMemoryManager")
61  << "Attached to shared memory segment with ID = " << shm_segment_id_
62  << " and size " << shmSize
63  << " bytes" << TLOG_ENDL;
64  shm_ptr_ = (ShmStruct*)shmat(shm_segment_id_, 0, 0);
65  TLOG_DEBUG("SharedMemoryManager")
66  << "Attached to shared memory segment at address "
67  << std::hex << (void*)shm_ptr_ << std::dec << TLOG_ENDL;
68  if (shm_ptr_ && shm_ptr_ != (void *)-1)
69  {
70  if (manager_id_ == 0)
71  {
72  if (shm_ptr_->ready_magic == 0xCAFE1111)
73  {
74  TLOG_ERROR("SharedMemoryManager") << "Owner encountered already-initialized Shared Memory!" << TLOG_ENDL;
75  exit(-2);
76  }
77  TLOG_DEBUG("SharedMemoryManager") << "Owner initializing Shared Memory" << TLOG_ENDL;
78  shm_ptr_->next_id = 1;
79  shm_ptr_->next_sequence_id = 0;
80  shm_ptr_->reader_pos = 0;
81  shm_ptr_->writer_pos = 0;
82  shm_ptr_->buffer_size = requested_shm_parameters_.buffer_size;
83  shm_ptr_->buffer_count = requested_shm_parameters_.buffer_count;
84  shm_ptr_->buffer_timeout_us = requested_shm_parameters_.buffer_timeout_us;
85  shm_ptr_->destructive_read_mode = requested_shm_parameters_.destructive_read_mode;
86 
87  for (int ii = 0; ii < static_cast<int>(requested_shm_parameters_.buffer_count); ++ii)
88  {
89  getBufferInfo_(ii)->writePos = 0;
90  getBufferInfo_(ii)->readPos = 0;
91  getBufferInfo_(ii)->sem = BufferSemaphoreFlags::Empty;
92  getBufferInfo_(ii)->sem_id = -1;
93  }
94 
95  shm_ptr_->ready_magic = 0xCAFE1111;
96  }
97  else
98  {
99  TLOG_DEBUG("SharedMemoryManager") << "Waiting for owner to initalize Shared Memory" << TLOG_ENDL;
100  while (shm_ptr_->ready_magic != 0xCAFE1111) { usleep(1000); }
101  TLOG_DEBUG("SharedMemoryManager") << "Getting ID from Shared Memory" << TLOG_ENDL;
102  GetNewId();
103  shm_ptr_->lowest_seq_id_read = 0;
104  TLOG_DEBUG("SharedMemoryManager") << "Getting Shared Memory Size parameters" << TLOG_ENDL;
105  }
106  //last_seen_id_ = shm_ptr_->next_sequence_id;
107  TLOG_DEBUG("SharedMemoryManager") << "Initialization Complete: "
108  << "key: 0x" << std::hex << shm_key_
109  << ", manager ID: " << manager_id_
110  << ", Buffer size: " << std::to_string(shm_ptr_->buffer_size)
111  << ", Buffer count: " << std::to_string(shm_ptr_->buffer_count) << TLOG_ENDL;
112  return;
113  }
114  else
115  {
116  TLOG_ERROR("SharedMemoryManager") << "Failed to attach to shared memory segment "
117  << shm_segment_id_ << TLOG_ENDL;
118  }
119  }
120  else
121  {
122  TLOG_ERROR("SharedMemoryManager") << "Failed to connect to shared memory segment"
123  << ", errno = " << errno << ". Please check "
124  << "if a stale shared memory segment needs to "
125  << "be cleaned up. (ipcs, ipcrm -m <segId>)" << TLOG_ENDL;
126  }
127  return;
128 }
129 
131 {
132  TLOG_ARB(13, "SharedMemoryManager") << "GetBufferForReading BEGIN" << TLOG_ENDL;
133 
134  //std::unique_lock<std::mutex> lk(search_mutex_);
135  TraceLock lk(search_mutex_, 11, "GetBufferForReadingSearch");
136  auto rp = shm_ptr_->reader_pos.load();
137 
138  TLOG_ARB(13, "SharedMemoryManager") << "GetBufferForReading lock acquired, scanning buffers" << TLOG_ENDL;
139  bool retry = true;
140  int buffer_num = -1;
141  while (retry)
142  {
143  ShmBuffer* buffer_ptr = nullptr;
144  uint64_t seqID = -1;
145  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
146  {
147  auto buffer = (ii + rp) % shm_ptr_->buffer_count;
148 
149 
150  TLOG_ARB(14, "SharedMemoryManager") << "GetBufferForReading Checking if buffer " << buffer << " is stale" << TLOG_ENDL;
151  ResetBuffer(buffer);
152 
153  auto buf = getBufferInfo_(buffer);
154  TLOG_ARB(14, "SharedMemoryManager") << "GetBufferForReading: Buffer " << buffer << ": sem=" << FlagToString(buf->sem) << " (expected " << FlagToString(BufferSemaphoreFlags::Full) << "), sem_id=" << buf->sem_id << " (" << manager_id_ << " or -1), seqid=" << buf->sequence_id << " (> " << last_seen_id_ << ")" << TLOG_ENDL;
155  if (buf->sem == BufferSemaphoreFlags::Full && (buf->sem_id == -1 || buf->sem_id == manager_id_) && buf->sequence_id > last_seen_id_)
156  {
157  if (buf->sequence_id < seqID)
158  {
159  buffer_ptr = buf;
160  seqID = buf->sequence_id;
161  buffer_num = buffer;
162  }
163  }
164  }
165 
166  if (!buffer_ptr || (buffer_ptr && buffer_ptr->sem_id != -1 && buffer_ptr->sem_id != manager_id_))
167  {
168  continue;
169  }
170 
171  if (buffer_num >= 0)
172  {
173  TLOG_ARB(13, "SharedMemoryManager") << "GetBufferForReading Found buffer " << buffer_num << TLOG_ENDL;
174  buffer_ptr->sem_id = manager_id_;
175  buffer_ptr->sem = BufferSemaphoreFlags::Reading;
176  if (buffer_ptr->sem_id != manager_id_) { continue; } // Failed to acquire buffer
177  buffer_ptr->readPos = 0;
178  touchBuffer_(buffer_ptr);
179  if (buffer_ptr->sem_id != manager_id_) { continue; } // Failed to acquire buffer
180  if (shm_ptr_->destructive_read_mode && shm_ptr_->lowest_seq_id_read == last_seen_id_)
181  {
182  shm_ptr_->lowest_seq_id_read = seqID;
183  }
184  last_seen_id_ = seqID;
185  if (shm_ptr_->destructive_read_mode) shm_ptr_->reader_pos = (buffer_num + 1) % shm_ptr_->buffer_count;
186  }
187  retry = false;
188  }
189 
190  TLOG_ARB(13, "SharedMemoryManager") << "GetBufferForReading returning -1 because no buffers are ready" << TLOG_ENDL;
191  return buffer_num;
192 }
193 
195 {
196  TLOG_ARB(13, "SharedMemoryManager") << "GetBufferForWriting BEGIN" << TLOG_ENDL;
197  //std::unique_lock<std::mutex> lk(search_mutex_);
198  TraceLock lk(search_mutex_, 12, "GetBufferForWritingSearch");
199  auto wp = shm_ptr_->writer_pos.load();
200 
201  // First, only look for "Empty" buffers
202  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
203  {
204  auto buffer = (ii + wp) % shm_ptr_->buffer_count;
205 
206  ResetBuffer(buffer);
207 
208  auto buf = getBufferInfo_(buffer);
209  if (buf->sem == BufferSemaphoreFlags::Empty)
210  {
211  buf->sem_id = manager_id_;
212  buf->sem = BufferSemaphoreFlags::Writing;
213  if (buf->sem_id != manager_id_) continue;
214  buf->sequence_id = ++shm_ptr_->next_sequence_id;
215  buf->writePos = 0;
216  shm_ptr_->writer_pos = (buffer + 1) % shm_ptr_->buffer_count;
217  touchBuffer_(buf);
218  TLOG_ARB(13, "SharedMemoryManager") << "GetBufferForWriting returning " << buffer << TLOG_ENDL;
219  return buffer;
220  }
221  }
222 
223  if (overwrite)
224  {
225  // Then, look for "Full" buffers
226  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
227  {
228  auto buffer = (ii + wp) % shm_ptr_->buffer_count;
229 
230  ResetBuffer(buffer);
231 
232  auto buf = getBufferInfo_(buffer);
233  if (buf->sem == BufferSemaphoreFlags::Full)
234  {
235  buf->sem_id = manager_id_;
236  buf->sem = BufferSemaphoreFlags::Writing;
237  if (buf->sem_id != manager_id_) continue;
238  buf->sequence_id = ++shm_ptr_->next_sequence_id;
239  buf->writePos = 0;
240  shm_ptr_->writer_pos = (buffer + 1) % shm_ptr_->buffer_count;
241  touchBuffer_(buf);
242  TLOG_ARB(13, "SharedMemoryManager") << "GetBufferForWriting returning " << buffer << TLOG_ENDL;
243  return buffer;
244  }
245  }
246 
247  // Finally, if we still haven't found a buffer, we have to clobber a reader...
248  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
249  {
250  auto buffer = (ii + wp) % shm_ptr_->buffer_count;
251 
252  ResetBuffer(buffer);
253 
254  auto buf = getBufferInfo_(buffer);
255  if (buf->sem == BufferSemaphoreFlags::Reading)
256  {
257  buf->sem_id = manager_id_;
258  buf->sem = BufferSemaphoreFlags::Writing;
259  if (buf->sem_id != manager_id_) continue;
260  buf->sequence_id = ++shm_ptr_->next_sequence_id;
261  buf->writePos = 0;
262  shm_ptr_->writer_pos = (buffer + 1) % shm_ptr_->buffer_count;
263  TLOG_ARB(13, "SharedMemoryManager") << "GetBufferForWriting returning " << buffer << TLOG_ENDL;
264  touchBuffer_(buf);
265  return buffer;
266  }
267  }
268  }
269 
270  TLOG_ARB(13, "SharedMemoryManager") << "GetBufferForWriting Returning -1 because no buffers are ready" << TLOG_ENDL;
271  return -1;
272 }
273 
275 {
276  if (!IsValid()) return 0;
277  TLOG_ARB(23, "SharedMemoryManager") << "ReadReadyCount BEGIN" << TLOG_ENDL;
278  //std::unique_lock<std::mutex> lk(search_mutex_);
279  TraceLock lk(search_mutex_, 14, "ReadReadyCountSearch");
280  size_t count = 0;
281  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
282  {
283  TLOG_ARB(24, "SharedMemoryManager") << "ReadReadyCount: Checking if buffer " << ii << " is stale." << TLOG_ENDL;
284  ResetBuffer(ii);
285  auto buf = getBufferInfo_(ii);
286  TLOG_ARB(24, "SharedMemoryManager") << "ReadReadyCount: Buffer " << ii << ": sem=" << FlagToString(buf->sem) << " (expected " << FlagToString(BufferSemaphoreFlags::Full) << "), sem_id=" << buf->sem_id << " (" << manager_id_ << " or -1), seqid=" << buf->sequence_id << " (> " << last_seen_id_ << ")" << TLOG_ENDL;
287  if (buf->sem == BufferSemaphoreFlags::Full && (buf->sem_id == -1 || buf->sem_id == manager_id_) && buf->sequence_id > last_seen_id_)
288  {
289  TLOG_ARB(24, "SharedMemoryManager") << "ReadReadyCount: Buffer " << ii << " is either unowned or owned by this manager, and is marked full." << TLOG_ENDL;
290  ++count;
291  }
292  }
293 
294  TLOG_ARB(23, "SharedMemoryManager") << "ReadReadyCount returning " << std::to_string(count) << TLOG_ENDL;
295  return count;
296 }
297 
299 {
300  if (!IsValid()) return 0;
301  //std::unique_lock<std::mutex> lk(search_mutex_);
302  TraceLock lk(search_mutex_, 15, "WriteReadyCountSearch");
303  size_t count = 0;
304  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
305  {
306  ResetBuffer(ii);
307  auto buf = getBufferInfo_(ii);
308  if ((buf->sem == BufferSemaphoreFlags::Empty && buf->sem_id == -1)
309  || (overwrite && buf->sem != BufferSemaphoreFlags::Writing))
310  {
311  ++count;
312  }
313  }
314  return count;
315 }
316 
318 {
319  //std::unique_lock<std::mutex> lk(search_mutex_);
320  std::deque<int> output;
321  if (!IsValid()) return output;
322  TraceLock lk(search_mutex_, 16, "GetOwnedSearch");
323  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
324  {
325  auto buf = getBufferInfo_(ii);
326  if (buf->sem_id == manager_id_)
327  {
328  output.push_back(ii);
329  }
330  }
331 
332  return output;
333 }
334 
336 {
337  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
338  TraceLock lk(buffer_mutexes_[buffer], 17, "DataSizeBuffer" + std::to_string(buffer));
339  auto buf = getBufferInfo_(buffer);
340  touchBuffer_(buf);
341  return buf->writePos;
342 }
343 
344 
346 {
347  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
348  TraceLock lk(buffer_mutexes_[buffer], 18, "ResetReadPosBuffer" + std::to_string(buffer));
349  auto buf = getBufferInfo_(buffer);
350  touchBuffer_(buf);
351  buf->readPos = 0;
352 }
353 
355 {
356  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
357  TraceLock lk(buffer_mutexes_[buffer], 18, "ResetWritePosBuffer" + std::to_string(buffer));
358  auto buf = getBufferInfo_(buffer);
359  checkBuffer_(buf, BufferSemaphoreFlags::Writing);
360  touchBuffer_(buf);
361  buf->writePos = 0;
362 }
363 
365 {
366  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
367  TraceLock lk(buffer_mutexes_[buffer], 19, "IncReadPosBuffer" + std::to_string(buffer));
368  auto buf = getBufferInfo_(buffer);
369  touchBuffer_(buf);
370  TLOG_ARB(13, "SharedMemoryManager") << "IncrementReadPos: buffer= " << buffer << ", readPos=" << std::to_string(buf->readPos) << ", bytes read=" << std::to_string(read) << TLOG_ENDL;
371  buf->readPos = buf->readPos + read;
372  TLOG_ARB(13, "SharedMemoryManager") << "IncrementReadPos: buffer= " << buffer << ", New readPos is " << std::to_string(buf->readPos) << TLOG_ENDL;
373  if (read == 0)
374  Detach(true, "LogicError", "Cannot increment Read pos by 0! (buffer=" + std::to_string(buffer) + ", readPos=" + std::to_string(buf->readPos) + ", writePos=" + std::to_string(buf->writePos) + ")");
375 }
376 
377 void artdaq::SharedMemoryManager::IncrementWritePos(int buffer, size_t written)
378 {
379  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
380  TraceLock lk(buffer_mutexes_[buffer], 20, "IncWritePosBuffer" + std::to_string(buffer));
381  auto buf = getBufferInfo_(buffer);
382  touchBuffer_(buf);
383  TLOG_ARB(13, "SharedMemoryManager") << "IncrementWritePos: buffer= " << buffer << ", writePos=" << std::to_string(buf->writePos) << ", bytes written=" << std::to_string(written) << TLOG_ENDL;
384  buf->writePos += written;
385  TLOG_ARB(13, "SharedMemoryManager") << "IncrementWritePos: buffer= " << buffer << ", New writePos is " << std::to_string(buf->writePos) << TLOG_ENDL;
386  if (written == 0) Detach(true, "LogicError", "Cannot increment Write pos by 0!");
387 }
388 
390 {
391  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
392  TraceLock lk(buffer_mutexes_[buffer], 21, "MoreDataInBuffer" + std::to_string(buffer));
393  auto buf = getBufferInfo_(buffer);
394  TLOG_ARB(13, "SharedMemoryManager") << "MoreDataInBuffer: buffer= " << buffer << ", readPos=" << std::to_string(buf->readPos) << ", writePos=" << std::to_string(buf->writePos) << TLOG_ENDL;
395  return buf->readPos < buf->writePos;
396 }
397 
399 {
400  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
401  TraceLock lk(buffer_mutexes_[buffer], 22, "CheckBuffer" + std::to_string(buffer));
402  return checkBuffer_(getBufferInfo_(buffer), flags, false);
403 }
404 
405 void artdaq::SharedMemoryManager::MarkBufferFull(int buffer, int destination)
406 {
407  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
408  TraceLock lk(buffer_mutexes_[buffer], 23, "FillBuffer" + std::to_string(buffer));
409  auto shmBuf = getBufferInfo_(buffer);
410  touchBuffer_(shmBuf);
411  if (shmBuf->sem_id == manager_id_)
412  {
413  if (shmBuf->sem != BufferSemaphoreFlags::Full)
414  shmBuf->sem = BufferSemaphoreFlags::Full;
415 
416  shmBuf->sem_id = destination;
417  }
418 }
419 
421 {
422  TLOG_ARB(13, "SharedMemoryManager") << "MarkBufferEmpty BEGIN" << TLOG_ENDL;
423  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
424  TraceLock lk(buffer_mutexes_[buffer], 24, "EmptyBuffer" + std::to_string(buffer));
425  auto shmBuf = getBufferInfo_(buffer);
426  if (!force)
427  {
428  checkBuffer_(shmBuf, BufferSemaphoreFlags::Reading, true);
429  }
430  touchBuffer_(shmBuf);
431 
432  shmBuf->readPos = 0;
433  shmBuf->sem = BufferSemaphoreFlags::Full;
434 
435  if ((force && (manager_id_ == 0 || manager_id_ == shmBuf->sem_id)) || (!force && shm_ptr_->destructive_read_mode))
436  {
437  TLOG_ARB(13, "SharedMemoryManager") << "MarkBufferEmpty Resetting buffer to Empty state" << TLOG_ENDL;
438  shmBuf->writePos = 0;
439  shmBuf->sem = BufferSemaphoreFlags::Empty;
440  if (shm_ptr_->reader_pos == static_cast<unsigned>(buffer) && !shm_ptr_->destructive_read_mode)
441  {
442  TLOG_ARB(13, "SharedMemoryManager") << "MarkBufferEmpty Broadcast mode; incrementing reader_pos" << TLOG_ENDL;
443  shm_ptr_->reader_pos = (buffer + 1) % shm_ptr_->buffer_count;
444  }
445  }
446  shmBuf->sem_id = -1;
447  TLOG_ARB(13, "SharedMemoryManager") << "MarkBufferEmpty END" << TLOG_ENDL;
448 }
449 
451 {
452  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
453  TraceLock lk(buffer_mutexes_[buffer], 25, "ResetBuffer" + std::to_string(buffer));
454  auto shmBuf = getBufferInfo_(buffer);
455  /*
456  if (shmBuf->sequence_id < shm_ptr_->lowest_seq_id_read - size() && shmBuf->sem == BufferSemaphoreFlags::Full)
457  {
458  TLOG_DEBUG("SharedMemoryManager") << "Buffer " << buffer << " has been passed by all readers, marking Empty" << TLOG_ENDL;
459  shmBuf->writePos = 0;
460  shmBuf->sem = BufferSemaphoreFlags::Empty;
461  shmBuf->sem_id = -1;
462  return true;
463  }*/
464 
465  if (TimeUtils::gettimeofday_us() - shmBuf->last_touch_time <= shm_ptr_->buffer_timeout_us) return false;
466 
467  if (shmBuf->sem_id == manager_id_ && shmBuf->sem == BufferSemaphoreFlags::Writing)
468  {
469  return true;
470  }
471  if (!shm_ptr_->destructive_read_mode && shmBuf->sem == BufferSemaphoreFlags::Full)
472  {
473  TLOG_DEBUG("SharedMemoryManager") << "Resetting old broadcast mode buffer" << TLOG_ENDL;
474  shmBuf->writePos = 0;
475  shmBuf->sem = BufferSemaphoreFlags::Empty;
476  shmBuf->sem_id = -1;
477  if (shm_ptr_->reader_pos = buffer)shm_ptr_->reader_pos = (buffer + 1) % shm_ptr_->buffer_count;
478  return true;
479  }
480 
481  if (shmBuf->sem_id != manager_id_ && shmBuf->sem == BufferSemaphoreFlags::Reading)
482  {
483  TLOG_WARNING("SharedMemoryManager") << "Stale Read buffer ( " << std::to_string(TimeUtils::gettimeofday_us() - shmBuf->last_touch_time) << " / " << std::to_string(shm_ptr_->buffer_timeout_us) << " us ) detected! Resetting..." << TLOG_ENDL;
484  shmBuf->readPos = 0;
485  shmBuf->sem = BufferSemaphoreFlags::Full;
486  shmBuf->sem_id = -1;
487  return true;
488  }
489  return false;
490 }
491 
492 size_t artdaq::SharedMemoryManager::Write(int buffer, void* data, size_t size)
493 {
494  TLOG_ARB(13, "SharedMemoryManager") << "Write BEGIN" << TLOG_ENDL;
495  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
496  TraceLock lk(buffer_mutexes_[buffer], 26, "WriteBuffer" + std::to_string(buffer));
497  auto shmBuf = getBufferInfo_(buffer);
498  checkBuffer_(shmBuf, BufferSemaphoreFlags::Writing);
499  touchBuffer_(shmBuf);
500  TLOG_ARB(13, "SharedMemoryManager") << "Buffer Write Pos is " << std::to_string(shmBuf->writePos) << ", write size is " << std::to_string(size) << TLOG_ENDL;
501  if (shmBuf->writePos + size > shm_ptr_->buffer_size) Detach(true, "SharedMemoryWrite", "Attempted to write more data than fits into Shared Memory! \nRe-run with a larger buffer size!");
502 
503  auto pos = GetWritePos(buffer);
504  memcpy(pos, data, size);
505  shmBuf->writePos = shmBuf->writePos + size;
506  if (shmBuf->sequence_id > last_seen_id_)
507  {
508  last_seen_id_ = shmBuf->sequence_id;
509  }
510  TLOG_ARB(13, "SharedMemoryManager") << "Write END" << TLOG_ENDL;
511  return size;
512 }
513 
514 bool artdaq::SharedMemoryManager::Read(int buffer, void* data, size_t size)
515 {
516  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
517  TraceLock lk(buffer_mutexes_[buffer], 27, "ReadBuffer" + std::to_string(buffer));
518  auto shmBuf = getBufferInfo_(buffer);
519  checkBuffer_(shmBuf, BufferSemaphoreFlags::Reading);
520  touchBuffer_(shmBuf);
521  if (shmBuf->readPos + size > shm_ptr_->buffer_size) Detach(true, "SharedMemoryRead", "Attempted to read more data than exists in Shared Memory!");
522 
523  auto pos = GetReadPos(buffer);
524  memcpy(data, pos, size);
525  shmBuf->readPos += size;
526  return checkBuffer_(shmBuf, BufferSemaphoreFlags::Reading, false);
527 }
528 
530 {
531  std::ostringstream ostr;
532  ostr << "ShmStruct: " << std::endl
533  << "Reader Position: " << shm_ptr_->reader_pos << std::endl
534  << "Writer Position: " << shm_ptr_->writer_pos << std::endl
535  << "Next ID Number: " << shm_ptr_->next_id << std::endl
536  << "Buffer Count: " << shm_ptr_->buffer_count << std::endl
537  << "Buffer Size: " << std::to_string(shm_ptr_->buffer_size) << " bytes" << std::endl
538  << "Buffers Written: " << std::to_string(shm_ptr_->next_sequence_id) << std::endl
539  << "Rank of Writer: " << shm_ptr_->rank << std::endl
540  << "Ready Magic Bytes: 0x" << std::hex << shm_ptr_->ready_magic << std::endl << std::endl;
541 
542  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
543  {
544  auto buf = getBufferInfo_(ii);
545  ostr << "ShmBuffer " << std::dec << ii << std::endl
546  << "sequenceID: " << std::to_string(buf->sequence_id) << std::endl
547  << "writePos: " << std::to_string(buf->writePos) << std::endl
548  << "readPos: " << std::to_string(buf->readPos) << std::endl
549  << "sem: " << FlagToString(buf->sem) << std::endl
550  << "Owner: " << std::to_string(buf->sem_id.load()) << std::endl
551  << "Last Touch Time: " << std::to_string(buf->last_touch_time / 1000000.0) << std::endl << std::endl;
552  }
553 
554  return ostr.str();
555 }
556 
558 {
559  auto buf = getBufferInfo_(buffer);
560  return bufferStart_(buffer) + buf->readPos;
561 }
563 {
564  auto buf = getBufferInfo_(buffer);
565  return bufferStart_(buffer) + buf->writePos;
566 }
567 
569 {
570  return bufferStart_(buffer);
571 }
572 
573 uint8_t* artdaq::SharedMemoryManager::dataStart_() const
574 {
575  return reinterpret_cast<uint8_t*>(shm_ptr_ + 1) + shm_ptr_->buffer_count * sizeof(ShmBuffer);
576 }
577 
578 uint8_t* artdaq::SharedMemoryManager::bufferStart_(int buffer)
579 {
580  if (buffer >= shm_ptr_->buffer_count) Detach(true, "ArgumentOutOfRange", "The specified buffer does not exist!");
581  return dataStart_() + buffer * shm_ptr_->buffer_size;
582 }
583 
584 artdaq::SharedMemoryManager::ShmBuffer* artdaq::SharedMemoryManager::getBufferInfo_(int buffer)
585 {
586  if (buffer >= shm_ptr_->buffer_count) Detach(true, "ArgumentOutOfRange", "The specified buffer does not exist!");
587  return reinterpret_cast<ShmBuffer*>(reinterpret_cast<uint8_t*>(shm_ptr_ + 1) + buffer * sizeof(ShmBuffer));
588 }
589 
590 bool artdaq::SharedMemoryManager::checkBuffer_(ShmBuffer* buffer, BufferSemaphoreFlags flags, bool exceptions)
591 {
592  if (exceptions)
593  {
594  if (buffer->sem != flags) Detach(true, "StateAccessViolation", "Shared Memory buffer is not in the correct state! (expected " + FlagToString(flags) + ", actual " + FlagToString(buffer->sem) + ")");
595  if (buffer->sem_id != manager_id_) Detach(true, "OwnerAccessViolation", "Shared Memory buffer is not owned by this manager instance!");
596  }
597  bool ret = (buffer->sem_id == manager_id_ || (buffer->sem_id == -1 && (flags == BufferSemaphoreFlags::Full || flags == BufferSemaphoreFlags::Empty))) && buffer->sem == flags;
598 
599  if (!ret)
600  {
601  TLOG_WARNING("SharedMemoryManager") << "CheckBuffer detected issue with buffer " << std::to_string(buffer->sequence_id) << "!"
602  << " ID: " << buffer->sem_id << " (" << manager_id_ << "), Flag: " << FlagToString(buffer->sem) << " (" << FlagToString(flags) << "). "
603  << "ID -1 is okay if desired flag is \"Full\" or \"Empty\"." << TLOG_ENDL;
604  }
605 
606  return ret;
607 }
608 
609 void artdaq::SharedMemoryManager::touchBuffer_(ShmBuffer* buffer)
610 {
611  if (buffer->sem_id != manager_id_) return;
612  TLOG_TRACE("SharedMemoryManager") << "touchBuffer_: Touching buffer with sequence_id " << std::to_string(buffer->sequence_id) << TLOG_ENDL;
613  buffer->last_touch_time = TimeUtils::gettimeofday_us();
614 }
615 
616 void artdaq::SharedMemoryManager::Detach(bool throwException, std::string category, std::string message)
617 {
618  if (IsValid())
619  {
620  auto bufs = GetBuffersOwnedByManager();
621  for (auto buf : bufs)
622  {
623  auto shmBuf = getBufferInfo_(buf);
624  if (shmBuf->sem == BufferSemaphoreFlags::Writing)
625  {
626  shmBuf->sem = BufferSemaphoreFlags::Empty;
627  }
628  else if (shmBuf->sem == BufferSemaphoreFlags::Reading)
629  {
630  shmBuf->sem = BufferSemaphoreFlags::Full;
631  }
632  shmBuf->sem_id = -1;
633  }
634  }
635 
636  if (shm_ptr_)
637  {
638  shmdt(shm_ptr_);
639  shm_ptr_ = NULL;
640  }
641 
642  if (manager_id_ == 0 && shm_segment_id_ > -1)
643  {
644  shmctl(shm_segment_id_, IPC_RMID, NULL);
645  }
646 
647  if (category.size() > 0 && message.size() > 0)
648  {
649  TLOG_ERROR("SharedMemoryManager") << category << ": " << message << TLOG_ENDL;
650 
651  if (throwException)
652  {
653  throw cet::exception(category) << message;
654  }
655  }
656 }
657 
658 
659 
660 // Local Variables:
661 // mode: c++
662 // End:
bool Read(int buffer, void *data, size_t size)
Read size bytes of data from buffer into the given pointer.
void IncrementReadPos(int buffer, size_t read)
Increment the read position for a given buffer.
constexpr size_t GetElapsedTimeMilliseconds(std::chrono::steady_clock::time_point then, std::chrono::steady_clock::time_point now=std::chrono::steady_clock::now())
Gets the number of milliseconds in the given time interval
Definition: TimeUtils.hh:54
void ResetReadPos(int buffer)
Set the read position of the given buffer to the beginning of the buffer.
bool MoreDataInBuffer(int buffer)
Determine if more data is available to be read, based on the read position and data size...
virtual std::string toString()
Write information about the SharedMemory to a string.
BufferSemaphoreFlags
The BufferSemaphoreFlags enumeration represents the different possible &quot;states&quot; of a given shared mem...
size_t BufferDataSize(int buffer)
Get the current size of the buffer&#39;s data.
void Attach()
Reconnect to the shared memory segment.
virtual ~SharedMemoryManager() noexcept
SharedMemoryManager Destructor.
size_t ReadReadyCount()
Count the number of buffers that are ready for reading.
bool ResetBuffer(int buffer)
Resets the buffer from Reading to Full. This operation will only have an effect if performed by the o...
bool CheckBuffer(int buffer, BufferSemaphoreFlags flags)
Check both semaphore conditions (Mode flag and manager ID) for a given buffer.
void IncrementWritePos(int buffer, size_t written)
Increment the write position for a given buffer.
The TraceLock class allows a user to debug the acquisition and releasing of locks, by wrapping the unique_lock&lt;std::mutex&gt; API with TRACE calls.
Definition: TraceLock.hh:10
void MarkBufferFull(int buffer, int destination=-1)
Release a buffer from a writer, marking it Full and ready for a reader.
void MarkBufferEmpty(int buffer, bool force=false)
Release a buffer from a reader, marking it Empty and ready to accept more data.
void ResetWritePos(int buffer)
Set the write position of the given buffer to the beginning of the buffer.
std::deque< int > GetBuffersOwnedByManager()
Get the list of all buffers currently owned by this manager instance.
void * GetWritePos(int buffer)
Get a pointer to the current write position of the buffer.
size_t Write(int buffer, void *data, size_t size)
Write size bytes of data from the given pointer to a buffer.
void * GetReadPos(int buffer)
Get a pointer to the current read position of the buffer.
void * GetBufferStart(int buffer)
Get a pointer to the start position of the buffer.
void Detach(bool throwException=false, std::string category="", std::string message="")
Detach from the Shared Memory segment, optionally throwing a cet::exception with the specified proper...
SharedMemoryManager(uint32_t shm_key, size_t buffer_count=0, size_t buffer_size=0, uint64_t buffer_timeout_us=100 *1000000, bool destructive_read_mode=true)
SharedMemoryManager Constructor.
int GetBufferForWriting(bool overwrite)
Finds a buffer that is ready to be written to, and reserves it for the calling manager.
size_t WriteReadyCount(bool overwrite)
Count the number of buffers that are ready for writing.
uint64_t gettimeofday_us()
Get the current time of day in microseconds (from gettimeofday system call)
Definition: TimeUtils.cc:51
int GetBufferForReading()
Finds a buffer that is ready to be read, and reserves it for the calling manager. ...