artdaq_core  v3_02_04
SharedMemoryManager.cc
1 #define TRACE_NAME "SharedMemoryManager"
2 #include <cstring>
3 #include <vector>
4 #include <sys/ipc.h>
5 #include <sys/shm.h>
6 #ifndef SHM_DEST // Lynn reports that this is missing on Mac OS X?!?
7 #define SHM_DEST 01000
8 #endif
9 #include "tracemf.h"
10 #include <signal.h>
11 #include "cetlib_except/exception.h"
12 #include "artdaq-core/Core/SharedMemoryManager.hh"
13 #include "artdaq-core/Utilities/TraceLock.hh"
14 
15 #define TLVL_DETACH 11
16 
17 static std::vector<artdaq::SharedMemoryManager const*> instances = std::vector<artdaq::SharedMemoryManager const*>();
18 
19 static std::unordered_map<int, struct sigaction> old_actions = std::unordered_map<int, struct sigaction>();
20 static bool sighandler_init = false;
21 static void signal_handler(int signum)
22 {
23  // Messagefacility may already be gone at this point, TRACE ONLY!
24  TRACE_STREAMER(TLVL_ERROR, &("SharedMemoryManager")[0], 0, 0, 0) << "A signal of type " << signum << " (" << std::string(strsignal(signum)) << ") was caught by SharedMemoryManager. Detaching all Shared Memory segments, then proceeding with default handlers!";
25  for (auto ii : instances)
26  {
27 
28  if (ii)
29  {
30  const_cast<artdaq::SharedMemoryManager*>(ii)->Detach(false, "", "", true);
31  }
32  ii = nullptr;
33  }
34 
35  sigset_t set;
36  pthread_sigmask(SIG_UNBLOCK, NULL, &set);
37  pthread_sigmask(SIG_UNBLOCK, &set, NULL);
38 
39  TRACE_STREAMER(TLVL_ERROR, &("SharedMemoryManager")[0], 0, 0, 0) << "Calling default signal handler";
40  if (signum != SIGUSR2)
41  {
42  sigaction(signum, &old_actions[signum], NULL);
43  kill(getpid(), signum); // Only send signal to self
44  }
45  else
46  {
47  // Send Interrupt signal if parsing SIGUSR2 (i.e. user-defined exception that should tear down ARTDAQ)
48  sigaction(SIGINT, &old_actions[SIGINT], NULL);
49  kill(getpid(), SIGINT); // Only send signal to self
50  }
51 }
52 
53 artdaq::SharedMemoryManager::SharedMemoryManager(uint32_t shm_key, size_t buffer_count, size_t buffer_size, uint64_t buffer_timeout_us, bool destructive_read_mode)
54  : shm_segment_id_(-1)
55  , shm_ptr_(NULL)
56  , shm_key_(shm_key)
57  , manager_id_(-1)
58  , buffer_mutexes_()
59  , last_seen_id_(0)
60 {
61  requested_shm_parameters_.buffer_count = buffer_count;
62  requested_shm_parameters_.buffer_size = buffer_size;
63  requested_shm_parameters_.buffer_timeout_us = buffer_timeout_us;
64  requested_shm_parameters_.destructive_read_mode = destructive_read_mode;
65 
66  instances.push_back(this);
67  Attach();
68 
69  static std::mutex sighandler_mutex;
70  std::unique_lock<std::mutex> lk(sighandler_mutex);
71 
72  if (!sighandler_init)//&& manager_id_ == 0) // ELF 3/22/18: Taking out manager_id_==0 requirement as I think kill(getpid()) is enough protection
73  {
74  sighandler_init = true;
75  std::vector<int> signals = { SIGINT, SIGQUIT, SIGILL, SIGABRT, SIGFPE, SIGSEGV, SIGPIPE, SIGALRM, SIGTERM, SIGUSR2 };
76  for (auto signal : signals)
77  {
78  struct sigaction old_action;
79  sigaction(signal, NULL, &old_action);
80 
81  //If the old handler wasn't SIG_IGN (it's a handler that just
82  // "ignore" the signal)
83  if (old_action.sa_handler != SIG_IGN)
84  {
85  struct sigaction action;
86  action.sa_handler = signal_handler;
87  sigemptyset(&action.sa_mask);
88  for (auto sigblk : signals)
89  {
90  sigaddset(&action.sa_mask, sigblk);
91  }
92  action.sa_flags = 0;
93 
94  //Replace the signal handler of SIGINT with the one described by new_action
95  sigaction(signal, &action, NULL);
96  old_actions[signal] = old_action;
97  }
98  }
99  }
100 }
101 
103 {
104  TLOG(TLVL_DEBUG) << "~SharedMemoryManager called";
105  Detach();
106  TLOG(TLVL_DEBUG) << "~SharedMemoryManager done";
107 }
108 
110 {
111  if (IsValid())
112  {
113  if (manager_id_ == 0) return;
114  Detach();
115  }
116  auto start_time = std::chrono::steady_clock::now();
117  last_seen_id_ = 0;
118  size_t shmSize = requested_shm_parameters_.buffer_count * (requested_shm_parameters_.buffer_size + sizeof(ShmBuffer)) + sizeof(ShmStruct);
119 
120  shm_segment_id_ = shmget(shm_key_, shmSize, 0666);
121  if (shm_segment_id_ == -1 && requested_shm_parameters_.buffer_count > 0 && manager_id_ <= 0)
122  {
123  TLOG(TLVL_DEBUG) << "Creating shared memory segment with key 0x" << std::hex << shm_key_ << " and size " << shmSize;
124  shm_segment_id_ = shmget(shm_key_, shmSize, IPC_CREAT | 0666);
125  manager_id_ = 0;
126 
127  if (shm_segment_id_ == -1)
128  {
129  TLOG(TLVL_ERROR) << "Error creating shared memory segment with key 0x" << std::hex << shm_key_ << ", errno=" << errno << " (" << strerror(errno) << ")";
130  }
131  }
132  else
133  {
134  while (shm_segment_id_ == -1 && TimeUtils::GetElapsedTimeMilliseconds(start_time) < 1000)
135  {
136  shm_segment_id_ = shmget(shm_key_, shmSize, 0666);
137 
138  }
139  }
140  TLOG(TLVL_DEBUG) << "shm_key == 0x" << std::hex << shm_key_ << ", shm_segment_id == " << shm_segment_id_;
141 
142  if (shm_segment_id_ > -1)
143  {
144  TLOG(TLVL_DEBUG)
145  << "Attached to shared memory segment with ID = " << shm_segment_id_
146  << " and size " << shmSize
147  << " bytes";
148  shm_ptr_ = (ShmStruct*)shmat(shm_segment_id_, 0, 0);
149  TLOG(TLVL_DEBUG)
150  << "Attached to shared memory segment at address "
151  << std::hex << (void*)shm_ptr_ << std::dec;
152  if (shm_ptr_ && shm_ptr_ != (void *)-1)
153  {
154  if (manager_id_ == 0)
155  {
156  if (shm_ptr_->ready_magic == 0xCAFE1111)
157  {
158  TLOG(TLVL_ERROR) << "Owner encountered already-initialized Shared Memory!";
159  exit(-2);
160  }
161  TLOG(TLVL_DEBUG) << "Owner initializing Shared Memory";
162  shm_ptr_->next_id = 1;
163  shm_ptr_->next_sequence_id = 0;
164  shm_ptr_->reader_pos = 0;
165  shm_ptr_->writer_pos = 0;
166  shm_ptr_->buffer_size = requested_shm_parameters_.buffer_size;
167  shm_ptr_->buffer_count = requested_shm_parameters_.buffer_count;
168  shm_ptr_->buffer_timeout_us = requested_shm_parameters_.buffer_timeout_us;
169  shm_ptr_->destructive_read_mode = requested_shm_parameters_.destructive_read_mode;
170 
171  for (int ii = 0; ii < static_cast<int>(requested_shm_parameters_.buffer_count); ++ii)
172  {
173  getBufferInfo_(ii)->writePos = 0;
174  getBufferInfo_(ii)->readPos = 0;
175  getBufferInfo_(ii)->sem = BufferSemaphoreFlags::Empty;
176  getBufferInfo_(ii)->sem_id = -1;
177  getBufferInfo_(ii)->last_touch_time = TimeUtils::gettimeofday_us();
178  }
179 
180  shm_ptr_->ready_magic = 0xCAFE1111;
181  }
182  else
183  {
184  TLOG(TLVL_DEBUG) << "Waiting for owner to initalize Shared Memory";
185  while (shm_ptr_->ready_magic != 0xCAFE1111) { usleep(1000); }
186  TLOG(TLVL_DEBUG) << "Getting ID from Shared Memory";
187  GetNewId();
188  shm_ptr_->lowest_seq_id_read = 0;
189  TLOG(TLVL_DEBUG) << "Getting Shared Memory Size parameters";
190  }
191  //last_seen_id_ = shm_ptr_->next_sequence_id;
192  TLOG(TLVL_DEBUG) << "Initialization Complete: "
193  << "key: 0x" << std::hex << shm_key_
194  << ", manager ID: " << manager_id_
195  << ", Buffer size: " << std::to_string(shm_ptr_->buffer_size)
196  << ", Buffer count: " << std::to_string(shm_ptr_->buffer_count);
197  return;
198  }
199  else
200  {
201  TLOG(TLVL_ERROR) << "Failed to attach to shared memory segment "
202  << shm_segment_id_;
203  }
204  }
205  else
206  {
207  TLOG(TLVL_ERROR) << "Failed to connect to shared memory segment with key 0x" << std::hex << shm_key_
208  << ", errno = " << strerror(errno) << ". Please check "
209  << "if a stale shared memory segment needs to "
210  << "be cleaned up. (ipcs, ipcrm -m <segId>)";
211  }
212  return;
213 }
214 
216 {
217  TLOG(13) << "GetBufferForReading BEGIN";
218 
219  //std::unique_lock<std::mutex> lk(search_mutex_);
220  TraceLock lk(search_mutex_, 11, "GetBufferForReadingSearch");
221  auto rp = shm_ptr_->reader_pos.load();
222 
223  TLOG(13) << "GetBufferForReading lock acquired, scanning buffers";
224  bool retry = true;
225  int buffer_num = -1;
226  while (retry)
227  {
228  ShmBuffer* buffer_ptr = nullptr;
229  uint64_t seqID = -1;
230  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
231  {
232  auto buffer = (ii + rp) % shm_ptr_->buffer_count;
233 
234 
235  TLOG(14) << "GetBufferForReading Checking if buffer " << buffer << " is stale";
236  ResetBuffer(buffer);
237 
238  auto buf = getBufferInfo_(buffer);
239  TLOG(14) << "GetBufferForReading: Buffer " << buffer << ": sem=" << FlagToString(buf->sem) << " (expected " << FlagToString(BufferSemaphoreFlags::Full) << "), sem_id=" << buf->sem_id << " )";
240  if (buf->sem == BufferSemaphoreFlags::Full && (buf->sem_id == -1 || buf->sem_id == manager_id_) && buf->sequence_id > last_seen_id_)
241  {
242  if (buf->sequence_id < seqID)
243  {
244  buffer_ptr = buf;
245  seqID = buf->sequence_id;
246  buffer_num = buffer;
247  }
248  }
249  }
250 
251  if (!buffer_ptr || (buffer_ptr && buffer_ptr->sem_id != -1 && buffer_ptr->sem_id != manager_id_))
252  {
253  continue;
254  }
255 
256  if (buffer_num >= 0)
257  {
258  TLOG(13) << "GetBufferForReading Found buffer " << buffer_num;
259  buffer_ptr->sem_id = manager_id_;
260  buffer_ptr->sem = BufferSemaphoreFlags::Reading;
261  if (buffer_ptr->sem_id != manager_id_) { continue; } // Failed to acquire buffer
262  buffer_ptr->readPos = 0;
263  touchBuffer_(buffer_ptr);
264  if (buffer_ptr->sem_id != manager_id_) { continue; } // Failed to acquire buffer
265  if (shm_ptr_->destructive_read_mode && shm_ptr_->lowest_seq_id_read == last_seen_id_)
266  {
267  shm_ptr_->lowest_seq_id_read = seqID;
268  }
269  last_seen_id_ = seqID;
270  if (shm_ptr_->destructive_read_mode) shm_ptr_->reader_pos = (buffer_num + 1) % shm_ptr_->buffer_count;
271  }
272  retry = false;
273  }
274 
275  TLOG(13) << "GetBufferForReading returning -1 because no buffers are ready";
276  return buffer_num;
277 }
278 
280 {
281  TLOG(14) << "GetBufferForWriting BEGIN";
282  //std::unique_lock<std::mutex> lk(search_mutex_);
283  TraceLock lk(search_mutex_, 12, "GetBufferForWritingSearch");
284  auto wp = shm_ptr_->writer_pos.load();
285 
286  // First, only look for "Empty" buffers
287  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
288  {
289  auto buffer = (ii + wp) % shm_ptr_->buffer_count;
290 
291  ResetBuffer(buffer);
292 
293  auto buf = getBufferInfo_(buffer);
294  if (buf->sem == BufferSemaphoreFlags::Empty && buf->sem_id == -1)
295  {
296  buf->sem_id = manager_id_;
297  buf->sem = BufferSemaphoreFlags::Writing;
298  touchBuffer_(buf);
299  shm_ptr_->writer_pos = (buffer + 1) % shm_ptr_->buffer_count;
300  if (buf->sem_id != manager_id_) continue;
301  buf->sequence_id = ++shm_ptr_->next_sequence_id;
302  buf->writePos = 0;
303  TLOG(14) << "GetBufferForWriting returning " << buffer;
304  return buffer;
305  }
306  }
307 
308  if (overwrite)
309  {
310  // Then, look for "Full" buffers
311  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
312  {
313  auto buffer = (ii + wp) % shm_ptr_->buffer_count;
314 
315  ResetBuffer(buffer);
316 
317  auto buf = getBufferInfo_(buffer);
318  if (buf->sem == BufferSemaphoreFlags::Full)
319  {
320  buf->sem_id = manager_id_;
321  buf->sem = BufferSemaphoreFlags::Writing;
322  if (buf->sem_id != manager_id_) continue;
323  buf->sequence_id = ++shm_ptr_->next_sequence_id;
324  buf->writePos = 0;
325  shm_ptr_->writer_pos = (buffer + 1) % shm_ptr_->buffer_count;
326  touchBuffer_(buf);
327  TLOG(14) << "GetBufferForWriting returning " << buffer;
328  return buffer;
329  }
330  }
331 
332  // Finally, if we still haven't found a buffer, we have to clobber a reader...
333  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
334  {
335  auto buffer = (ii + wp) % shm_ptr_->buffer_count;
336 
337  ResetBuffer(buffer);
338 
339  auto buf = getBufferInfo_(buffer);
340  if (buf->sem == BufferSemaphoreFlags::Reading)
341  {
342  buf->sem_id = manager_id_;
343  buf->sem = BufferSemaphoreFlags::Writing;
344  if (buf->sem_id != manager_id_) continue;
345  buf->sequence_id = ++shm_ptr_->next_sequence_id;
346  buf->writePos = 0;
347  shm_ptr_->writer_pos = (buffer + 1) % shm_ptr_->buffer_count;
348  TLOG(14) << "GetBufferForWriting returning " << buffer;
349  touchBuffer_(buf);
350  return buffer;
351  }
352  }
353  }
354 
355  TLOG(14) << "GetBufferForWriting Returning -1 because no buffers are ready";
356  return -1;
357 }
358 
360 {
361  if (!IsValid()) return 0;
362  TLOG(23) << "0x" << std::hex << shm_key_ << " ReadReadyCount BEGIN";
363  //std::unique_lock<std::mutex> lk(search_mutex_);
364  TraceLock lk(search_mutex_, 14, "ReadReadyCountSearch");
365  size_t count = 0;
366  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
367  {
368  TLOG(24) << "0x" << std::hex << shm_key_ << std::dec << " ReadReadyCount: Checking if buffer " << ii << " is stale.";
369  ResetBuffer(ii);
370  auto buf = getBufferInfo_(ii);
371  TLOG(25) << "0x" << std::hex << shm_key_ << std::dec << " ReadReadyCount: Buffer " << ii << ": sem=" << FlagToString(buf->sem) << " (expected " << FlagToString(BufferSemaphoreFlags::Full) << "), sem_id=" << buf->sem_id << " )";
372  if (buf->sem == BufferSemaphoreFlags::Full && (buf->sem_id == -1 || buf->sem_id == manager_id_) && buf->sequence_id > last_seen_id_)
373  {
374  TLOG(26) << "0x" << std::hex << shm_key_ << std::dec << " ReadReadyCount: Buffer " << ii << " is either unowned or owned by this manager, and is marked full.";
375  ++count;
376  }
377  }
378  return count;
379 }
380 
382 {
383  if (!IsValid()) return 0;
384  //std::unique_lock<std::mutex> lk(search_mutex_);
385  TraceLock lk(search_mutex_, 15, "WriteReadyCountSearch");
386  size_t count = 0;
387  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
388  {
389  ResetBuffer(ii);
390  auto buf = getBufferInfo_(ii);
391  if ((buf->sem == BufferSemaphoreFlags::Empty && buf->sem_id == -1)
392  || (overwrite && buf->sem != BufferSemaphoreFlags::Writing))
393  {
394  ++count;
395  }
396  }
397  return count;
398 }
399 
401 {
402  //std::unique_lock<std::mutex> lk(search_mutex_);
403  std::deque<int> output;
404  if (!IsValid()) return output;
405  if (locked)
406  {
407  TraceLock lk(search_mutex_, 16, "GetOwnedSearch");
408  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
409  {
410  auto buf = getBufferInfo_(ii);
411  if (buf->sem_id == manager_id_)
412  {
413  output.push_back(ii);
414  }
415  }
416  }
417  else
418  {
419  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
420  {
421  auto buf = getBufferInfo_(ii);
422  if (buf->sem_id == manager_id_)
423  {
424  output.push_back(ii);
425  }
426  }
427  }
428 
429  return output;
430 }
431 
433 {
434  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
435  TraceLock lk(buffer_mutexes_[buffer], 17, "DataSizeBuffer" + std::to_string(buffer));
436  auto buf = getBufferInfo_(buffer);
437  touchBuffer_(buf);
438  return buf->writePos;
439 }
440 
441 
443 {
444  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
445  TraceLock lk(buffer_mutexes_[buffer], 18, "ResetReadPosBuffer" + std::to_string(buffer));
446  auto buf = getBufferInfo_(buffer);
447  touchBuffer_(buf);
448  buf->readPos = 0;
449 }
450 
452 {
453  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
454  TraceLock lk(buffer_mutexes_[buffer], 18, "ResetWritePosBuffer" + std::to_string(buffer));
455  auto buf = getBufferInfo_(buffer);
456  checkBuffer_(buf, BufferSemaphoreFlags::Writing);
457  touchBuffer_(buf);
458  buf->writePos = 0;
459 }
460 
462 {
463  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
464  TraceLock lk(buffer_mutexes_[buffer], 19, "IncReadPosBuffer" + std::to_string(buffer));
465  auto buf = getBufferInfo_(buffer);
466  touchBuffer_(buf);
467  TLOG(15) << "IncrementReadPos: buffer= " << buffer << ", readPos=" << std::to_string(buf->readPos) << ", bytes read=" << std::to_string(read);
468  buf->readPos = buf->readPos + read;
469  TLOG(15) << "IncrementReadPos: buffer= " << buffer << ", New readPos is " << std::to_string(buf->readPos);
470  if (read == 0) Detach(true, "LogicError", "Cannot increment Read pos by 0! (buffer=" + std::to_string(buffer) + ", readPos=" + std::to_string(buf->readPos) + ", writePos=" + std::to_string(buf->writePos) + ")");
471 }
472 
473 void artdaq::SharedMemoryManager::IncrementWritePos(int buffer, size_t written)
474 {
475  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
476  TraceLock lk(buffer_mutexes_[buffer], 20, "IncWritePosBuffer" + std::to_string(buffer));
477  auto buf = getBufferInfo_(buffer);
478  touchBuffer_(buf);
479  TLOG(16) << "IncrementWritePos: buffer= " << buffer << ", writePos=" << std::to_string(buf->writePos) << ", bytes written=" << std::to_string(written);
480  buf->writePos += written;
481  TLOG(16) << "IncrementWritePos: buffer= " << buffer << ", New writePos is " << std::to_string(buf->writePos);
482  if (written == 0) Detach(true, "LogicError", "Cannot increment Write pos by 0!");
483 }
484 
486 {
487  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
488  TraceLock lk(buffer_mutexes_[buffer], 21, "MoreDataInBuffer" + std::to_string(buffer));
489  auto buf = getBufferInfo_(buffer);
490  TLOG(17) << "MoreDataInBuffer: buffer= " << buffer << ", readPos=" << std::to_string(buf->readPos) << ", writePos=" << std::to_string(buf->writePos);
491  return buf->readPos < buf->writePos;
492 }
493 
495 {
496  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
497  TraceLock lk(buffer_mutexes_[buffer], 22, "CheckBuffer" + std::to_string(buffer));
498  return checkBuffer_(getBufferInfo_(buffer), flags, false);
499 }
500 
501 void artdaq::SharedMemoryManager::MarkBufferFull(int buffer, int destination)
502 {
503  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
504  TraceLock lk(buffer_mutexes_[buffer], 23, "FillBuffer" + std::to_string(buffer));
505  auto shmBuf = getBufferInfo_(buffer);
506  touchBuffer_(shmBuf);
507  if (shmBuf->sem_id == manager_id_)
508  {
509  if (shmBuf->sem != BufferSemaphoreFlags::Full)
510  shmBuf->sem = BufferSemaphoreFlags::Full;
511 
512  shmBuf->sem_id = destination;
513  }
514 }
515 
517 {
518  TLOG(18) << "MarkBufferEmpty BEGIN";
519  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
520  TraceLock lk(buffer_mutexes_[buffer], 24, "EmptyBuffer" + std::to_string(buffer));
521  auto shmBuf = getBufferInfo_(buffer);
522  if (!force)
523  {
524  checkBuffer_(shmBuf, BufferSemaphoreFlags::Reading, true);
525  }
526  touchBuffer_(shmBuf);
527 
528  shmBuf->readPos = 0;
529  shmBuf->sem = BufferSemaphoreFlags::Full;
530 
531  if ((force && (manager_id_ == 0 || manager_id_ == shmBuf->sem_id)) || (!force && shm_ptr_->destructive_read_mode))
532  {
533  TLOG(18) << "MarkBufferEmpty Resetting buffer to Empty state";
534  shmBuf->writePos = 0;
535  shmBuf->sem = BufferSemaphoreFlags::Empty;
536  if (shm_ptr_->reader_pos == static_cast<unsigned>(buffer) && !shm_ptr_->destructive_read_mode)
537  {
538  TLOG(18) << "MarkBufferEmpty Broadcast mode; incrementing reader_pos";
539  shm_ptr_->reader_pos = (buffer + 1) % shm_ptr_->buffer_count;
540  }
541  }
542  shmBuf->sem_id = -1;
543  TLOG(18) << "MarkBufferEmpty END";
544 }
545 
547 {
548  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
549  TraceLock lk(buffer_mutexes_[buffer], 25, "ResetBuffer" + std::to_string(buffer));
550  auto shmBuf = getBufferInfo_(buffer);
551  /*
552  if (shmBuf->sequence_id < shm_ptr_->lowest_seq_id_read - size() && shmBuf->sem == BufferSemaphoreFlags::Full)
553  {
554  TLOG(TLVL_DEBUG) << "Buffer " << buffer << " has been passed by all readers, marking Empty" ;
555  shmBuf->writePos = 0;
556  shmBuf->sem = BufferSemaphoreFlags::Empty;
557  shmBuf->sem_id = -1;
558  return true;
559  }*/
560 
561  size_t delta = TimeUtils::gettimeofday_us() - shmBuf->last_touch_time;
562  if (delta > 0xFFFFFFFF)
563  {
564  TLOG(TLVL_TRACE) << "Buffer has touch time in the future, setting it to current time and ignoring...";
565  shmBuf->last_touch_time = TimeUtils::gettimeofday_us();
566  return false;
567  }
568  if (shm_ptr_->buffer_timeout_us == 0 || delta <= shm_ptr_->buffer_timeout_us || shmBuf->sem == BufferSemaphoreFlags::Empty) return false;
569  TLOG(27) << "Buffer " << buffer << " is stale, time=" << TimeUtils::gettimeofday_us() << ", last touch=" << shmBuf->last_touch_time << ", d=" << delta << ", timeout=" << shm_ptr_->buffer_timeout_us;
570 
571  if (shmBuf->sem_id == manager_id_ && shmBuf->sem == BufferSemaphoreFlags::Writing)
572  {
573  return true;
574  }
575  if (!shm_ptr_->destructive_read_mode && shmBuf->sem == BufferSemaphoreFlags::Full)
576  {
577  TLOG(TLVL_DEBUG) << "Resetting old broadcast mode buffer";
578  shmBuf->writePos = 0;
579  shmBuf->sem = BufferSemaphoreFlags::Empty;
580  shmBuf->sem_id = -1;
581  if (shm_ptr_->reader_pos == static_cast<unsigned>(buffer)) shm_ptr_->reader_pos = (buffer + 1) % shm_ptr_->buffer_count;
582  return true;
583  }
584 
585  if (shmBuf->sem_id != manager_id_ && shmBuf->sem == BufferSemaphoreFlags::Reading)
586  {
587  TLOG(TLVL_WARNING) << "Stale Read buffer ( " << delta << " / " << shm_ptr_->buffer_timeout_us << " us ) detected! Resetting...";
588  shmBuf->readPos = 0;
589  shmBuf->sem = BufferSemaphoreFlags::Full;
590  shmBuf->sem_id = -1;
591  return true;
592  }
593  return false;
594 }
595 
597 {
598  if (!IsValid()) return true;
599 
600  struct shmid_ds info;
601  auto sts = shmctl(shm_segment_id_, IPC_STAT, &info);
602  if (sts < 0)
603  {
604  TLOG(TLVL_TRACE) << "Error accessing Shared Memory info: " << errno << " (" << strerror(errno) << ").";
605  return true;
606  }
607 
608  if (info.shm_perm.mode & SHM_DEST)
609  {
610  TLOG(TLVL_INFO) << "Shared Memory marked for destruction. Probably an end-of-data condition!";
611  return true;
612  }
613 
614  return false;
615 }
616 
618 {
619  if (!IsValid()) return 0;
620 
621  struct shmid_ds info;
622  auto sts = shmctl(shm_segment_id_, IPC_STAT, &info);
623  if (sts < 0)
624  {
625  TLOG(TLVL_TRACE) << "Error accessing Shared Memory info: " << errno << " (" << strerror(errno) << ").";
626  return 0;
627  }
628 
629  return info.shm_nattch;
630 }
631 
632 size_t artdaq::SharedMemoryManager::Write(int buffer, void* data, size_t size)
633 {
634  TLOG(19) << "Write BEGIN";
635  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
636  TraceLock lk(buffer_mutexes_[buffer], 26, "WriteBuffer" + std::to_string(buffer));
637  auto shmBuf = getBufferInfo_(buffer);
638  checkBuffer_(shmBuf, BufferSemaphoreFlags::Writing);
639  touchBuffer_(shmBuf);
640  TLOG(19) << "Buffer Write Pos is " << std::to_string(shmBuf->writePos) << ", write size is " << std::to_string(size);
641  if (shmBuf->writePos + size > shm_ptr_->buffer_size) Detach(true, "SharedMemoryWrite", "Attempted to write more data than fits into Shared Memory! \nRe-run with a larger buffer size!");
642 
643  auto pos = GetWritePos(buffer);
644  memcpy(pos, data, size);
645  shmBuf->writePos = shmBuf->writePos + size;
646  if (shmBuf->sequence_id > last_seen_id_)
647  {
648  last_seen_id_ = shmBuf->sequence_id;
649  }
650  TLOG(19) << "Write END";
651  return size;
652 }
653 
654 bool artdaq::SharedMemoryManager::Read(int buffer, void* data, size_t size)
655 {
656  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
657  TraceLock lk(buffer_mutexes_[buffer], 27, "ReadBuffer" + std::to_string(buffer));
658  auto shmBuf = getBufferInfo_(buffer);
659  checkBuffer_(shmBuf, BufferSemaphoreFlags::Reading);
660  touchBuffer_(shmBuf);
661  if (shmBuf->readPos + size > shm_ptr_->buffer_size) Detach(true, "SharedMemoryRead", "Attempted to read more data than exists in Shared Memory!");
662 
663  auto pos = GetReadPos(buffer);
664  memcpy(data, pos, size);
665  shmBuf->readPos += size;
666  touchBuffer_(shmBuf);
667  return checkBuffer_(shmBuf, BufferSemaphoreFlags::Reading, false);
668 }
669 
671 {
672  std::ostringstream ostr;
673  ostr << "ShmStruct: " << std::endl
674  << "Reader Position: " << shm_ptr_->reader_pos << std::endl
675  << "Writer Position: " << shm_ptr_->writer_pos << std::endl
676  << "Next ID Number: " << shm_ptr_->next_id << std::endl
677  << "Buffer Count: " << shm_ptr_->buffer_count << std::endl
678  << "Buffer Size: " << std::to_string(shm_ptr_->buffer_size) << " bytes" << std::endl
679  << "Buffers Written: " << std::to_string(shm_ptr_->next_sequence_id) << std::endl
680  << "Rank of Writer: " << shm_ptr_->rank << std::endl
681  << "Ready Magic Bytes: 0x" << std::hex << shm_ptr_->ready_magic << std::endl << std::endl;
682 
683  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
684  {
685  auto buf = getBufferInfo_(ii);
686  ostr << "ShmBuffer " << std::dec << ii << std::endl
687  << "sequenceID: " << std::to_string(buf->sequence_id) << std::endl
688  << "writePos: " << std::to_string(buf->writePos) << std::endl
689  << "readPos: " << std::to_string(buf->readPos) << std::endl
690  << "sem: " << FlagToString(buf->sem) << std::endl
691  << "Owner: " << std::to_string(buf->sem_id.load()) << std::endl
692  << "Last Touch Time: " << std::to_string(buf->last_touch_time / 1000000.0) << std::endl << std::endl;
693  }
694 
695  return ostr.str();
696 }
697 
699 {
700  auto buf = getBufferInfo_(buffer);
701  return bufferStart_(buffer) + buf->readPos;
702 }
704 {
705  auto buf = getBufferInfo_(buffer);
706  return bufferStart_(buffer) + buf->writePos;
707 }
708 
710 {
711  return bufferStart_(buffer);
712 }
713 
714 uint8_t* artdaq::SharedMemoryManager::dataStart_() const
715 {
716  return reinterpret_cast<uint8_t*>(shm_ptr_ + 1) + shm_ptr_->buffer_count * sizeof(ShmBuffer);
717 }
718 
719 uint8_t* artdaq::SharedMemoryManager::bufferStart_(int buffer)
720 {
721  if (buffer >= shm_ptr_->buffer_count) Detach(true, "ArgumentOutOfRange", "The specified buffer does not exist!");
722  return dataStart_() + buffer * shm_ptr_->buffer_size;
723 }
724 
725 artdaq::SharedMemoryManager::ShmBuffer* artdaq::SharedMemoryManager::getBufferInfo_(int buffer)
726 {
727  if (buffer >= shm_ptr_->buffer_count) Detach(true, "ArgumentOutOfRange", "The specified buffer does not exist!");
728  return reinterpret_cast<ShmBuffer*>(reinterpret_cast<uint8_t*>(shm_ptr_ + 1) + buffer * sizeof(ShmBuffer));
729 }
730 
731 bool artdaq::SharedMemoryManager::checkBuffer_(ShmBuffer* buffer, BufferSemaphoreFlags flags, bool exceptions)
732 {
733  TLOG(TLVL_TRACE) << "checkBuffer_: Checking that buffer " << buffer->sequence_id << " has sem_id " << manager_id_ << " (Current: " << buffer->sem_id << ") and is in state " << FlagToString(flags) << " (current: " << FlagToString(buffer->sem) << ")";
734  if (exceptions)
735  {
736  if (buffer->sem != flags) Detach(true, "StateAccessViolation", "Shared Memory buffer is not in the correct state! (expected " + FlagToString(flags) + ", actual " + FlagToString(buffer->sem) + ")");
737  if (buffer->sem_id != manager_id_) Detach(true, "OwnerAccessViolation", "Shared Memory buffer is not owned by this manager instance! (Expected: " + std::to_string(manager_id_) + ", Actual: " + std::to_string(buffer->sem_id) + ")");
738  }
739  bool ret = (buffer->sem_id == manager_id_ || (buffer->sem_id == -1 && (flags == BufferSemaphoreFlags::Full || flags == BufferSemaphoreFlags::Empty))) && buffer->sem == flags;
740 
741  if (!ret)
742  {
743  TLOG(TLVL_WARNING) << "CheckBuffer detected issue with buffer " << std::to_string(buffer->sequence_id) << "!"
744  << " ID: " << buffer->sem_id << " (" << manager_id_ << "), Flag: " << FlagToString(buffer->sem) << " (" << FlagToString(flags) << "). "
745  << "ID -1 is okay if desired flag is \"Full\" or \"Empty\".";
746  }
747 
748  return ret;
749 }
750 
751 void artdaq::SharedMemoryManager::touchBuffer_(ShmBuffer* buffer)
752 {
753  if (buffer->sem_id != manager_id_) return;
754  TLOG(TLVL_TRACE) << "touchBuffer_: Touching buffer with sequence_id " << std::to_string(buffer->sequence_id);
755  buffer->last_touch_time = TimeUtils::gettimeofday_us();
756 }
757 
758 void artdaq::SharedMemoryManager::Detach(bool throwException, std::string category, std::string message, bool force)
759 {
760  TLOG(TLVL_DETACH) << "Detach BEGIN: throwException: " << std::boolalpha << throwException << ", force: " << force;
761  if (IsValid())
762  {
763  TLOG(TLVL_DETACH) << "Detach: Resetting owned buffers";
764  auto bufs = GetBuffersOwnedByManager(false);
765  for (auto buf : bufs)
766  {
767  auto shmBuf = getBufferInfo_(buf);
768  if (shmBuf->sem == BufferSemaphoreFlags::Writing)
769  {
770  shmBuf->sem = BufferSemaphoreFlags::Empty;
771  }
772  else if (shmBuf->sem == BufferSemaphoreFlags::Reading)
773  {
774  shmBuf->sem = BufferSemaphoreFlags::Full;
775  }
776  shmBuf->sem_id = -1;
777  }
778  }
779 
780  if (shm_ptr_)
781  {
782  TLOG(TLVL_DETACH) << "Detach: Detaching shared memory";
783  shmdt(shm_ptr_);
784  shm_ptr_ = NULL;
785  }
786 
787  if ((force || manager_id_ == 0) && shm_segment_id_ > -1)
788  {
789  TLOG(TLVL_DETACH) << "Detach: Marking Shared memory for removal";
790  shmctl(shm_segment_id_, IPC_RMID, NULL);
791  shm_segment_id_ = -1;
792  }
793 
794  if (category.size() > 0 && message.size() > 0)
795  {
796  TLOG(TLVL_ERROR) << category << ": " << message;
797 
798  if (throwException)
799  {
800  throw cet::exception(category) << message;
801  }
802  }
803 }
804 
805 
806 
807 // Local Variables:
808 // mode: c++
809 // End:
bool Read(int buffer, void *data, size_t size)
Read size bytes of data from buffer into the given pointer.
void IncrementReadPos(int buffer, size_t read)
Increment the read position for a given buffer.
constexpr size_t GetElapsedTimeMilliseconds(std::chrono::steady_clock::time_point then, std::chrono::steady_clock::time_point now=std::chrono::steady_clock::now())
Gets the number of milliseconds in the given time interval
Definition: TimeUtils.hh:54
std::deque< int > GetBuffersOwnedByManager(bool locked=true)
Get the list of all buffers currently owned by this manager instance.
void ResetReadPos(int buffer)
Set the read position of the given buffer to the beginning of the buffer.
bool MoreDataInBuffer(int buffer)
Determine if more data is available to be read, based on the read position and data size...
virtual std::string toString()
Write information about the SharedMemory to a string.
BufferSemaphoreFlags
The BufferSemaphoreFlags enumeration represents the different possible &quot;states&quot; of a given shared mem...
size_t BufferDataSize(int buffer)
Get the current size of the buffer&#39;s data.
void Attach()
Reconnect to the shared memory segment.
void Detach(bool throwException=false, std::string category="", std::string message="", bool force=false)
Detach from the Shared Memory segment, optionally throwing a cet::exception with the specified proper...
virtual ~SharedMemoryManager() noexcept
SharedMemoryManager Destructor.
The SharedMemoryManager creates a Shared Memory area which is divided into a number of fixed-size buf...
size_t ReadReadyCount()
Count the number of buffers that are ready for reading.
bool ResetBuffer(int buffer)
Resets the buffer from Reading to Full. This operation will only have an effect if performed by the o...
bool IsEndOfData() const
Determine whether the Shared Memory is marked for destruction (End of Data)
bool CheckBuffer(int buffer, BufferSemaphoreFlags flags)
Check both semaphore conditions (Mode flag and manager ID) for a given buffer.
void IncrementWritePos(int buffer, size_t written)
Increment the write position for a given buffer.
The TraceLock class allows a user to debug the acquisition and releasing of locks, by wrapping the unique_lock&lt;std::mutex&gt; API with TRACE calls.
Definition: TraceLock.hh:10
void MarkBufferFull(int buffer, int destination=-1)
Release a buffer from a writer, marking it Full and ready for a reader.
void MarkBufferEmpty(int buffer, bool force=false)
Release a buffer from a reader, marking it Empty and ready to accept more data.
void ResetWritePos(int buffer)
Set the write position of the given buffer to the beginning of the buffer.
void * GetWritePos(int buffer)
Get a pointer to the current write position of the buffer.
size_t Write(int buffer, void *data, size_t size)
Write size bytes of data from the given pointer to a buffer.
void * GetReadPos(int buffer)
Get a pointer to the current read position of the buffer.
void * GetBufferStart(int buffer)
Get a pointer to the start position of the buffer.
SharedMemoryManager(uint32_t shm_key, size_t buffer_count=0, size_t buffer_size=0, uint64_t buffer_timeout_us=100 *1000000, bool destructive_read_mode=true)
SharedMemoryManager Constructor.
int GetBufferForWriting(bool overwrite)
Finds a buffer that is ready to be written to, and reserves it for the calling manager.
uint16_t GetAttachedCount() const
Get the number of attached SharedMemoryManagers.
size_t WriteReadyCount(bool overwrite)
Count the number of buffers that are ready for writing.
uint64_t gettimeofday_us()
Get the current time of day in microseconds (from gettimeofday system call)
Definition: TimeUtils.cc:51
int GetBufferForReading()
Finds a buffer that is ready to be read, and reserves it for the calling manager. ...