artdaq_core  v3_02_00
SharedMemoryManager.cc
1 #define TRACE_NAME "SharedMemoryManager"
2 #include <cstring>
3 #include <vector>
4 #include <sys/shm.h>
5 #include "tracemf.h"
6 #include <signal.h>
7 #include "cetlib_except/exception.h"
8 #include "artdaq-core/Core/SharedMemoryManager.hh"
9 #include "artdaq-core/Utilities/TraceLock.hh"
10 
11 #define TLVL_DETACH 11
12 
13 static std::vector<artdaq::SharedMemoryManager*> instances = std::vector<artdaq::SharedMemoryManager*>();
14 
15 static std::unordered_map<int, struct sigaction> old_actions = std::unordered_map<int, struct sigaction>();
16 static bool sighandler_init = false;
17 static void signal_handler(int signum)
18 {
19  // Messagefacility may already be gone at this point, TRACE ONLY!
20  TRACE_STREAMER(TLVL_ERROR, &("SharedMemoryManager")[0], 0, 0, 0) << "A signal of type " << signum << " (" << std::string(strsignal(signum)) << ") was caught by SharedMemoryManager. Detaching all Shared Memory segments, then proceeding with default handlers!";
21  for (auto ii : instances)
22  {
23 
24  if (ii) {
25  ii->Detach(false, "", "", true);
26  }
27  ii = nullptr;
28  }
29 
30  sigset_t set;
31  pthread_sigmask(SIG_UNBLOCK, NULL, &set);
32  pthread_sigmask(SIG_UNBLOCK, &set, NULL);
33 
34  TRACE_STREAMER(TLVL_ERROR, &("SharedMemoryManager")[0], 0, 0, 0) << "Calling default signal handler";
35  if (signum != SIGUSR2) {
36  sigaction(signum, &old_actions[signum], NULL);
37  kill(getpid(), signum); // Only send signal to self
38  }
39  else {
40  // Send Interrupt signal if parsing SIGUSR2 (i.e. user-defined exception that should tear down ARTDAQ)
41  sigaction(SIGINT, &old_actions[SIGINT], NULL);
42  kill(getpid(), SIGINT); // Only send signal to self
43  }
44 }
45 
46 artdaq::SharedMemoryManager::SharedMemoryManager(uint32_t shm_key, size_t buffer_count, size_t buffer_size, uint64_t buffer_timeout_us, bool destructive_read_mode)
47  : shm_segment_id_(-1)
48  , shm_ptr_(NULL)
49  , shm_key_(shm_key)
50  , manager_id_(-1)
51  , buffer_mutexes_()
52  , last_seen_id_(0)
53 {
54  requested_shm_parameters_.buffer_count = buffer_count;
55  requested_shm_parameters_.buffer_size = buffer_size;
56  requested_shm_parameters_.buffer_timeout_us = buffer_timeout_us;
57  requested_shm_parameters_.destructive_read_mode = destructive_read_mode;
58 
59  instances.push_back(this);
60  Attach();
61 
62  static std::mutex sighandler_mutex;
63  std::unique_lock<std::mutex> lk(sighandler_mutex);
64 
65  if (!sighandler_init)//&& manager_id_ == 0) // ELF 3/22/18: Taking out manager_id_==0 requirement as I think kill(getpid()) is enough protection
66  {
67  sighandler_init = true;
68  std::vector<int> signals = { SIGINT, SIGQUIT, SIGILL, SIGABRT, SIGFPE, SIGSEGV, SIGPIPE, SIGALRM, SIGTERM, SIGUSR2 };
69  for (auto signal : signals)
70  {
71  struct sigaction old_action;
72  sigaction(signal, NULL, &old_action);
73 
74  //If the old handler wasn't SIG_IGN (it's a handler that just
75  // "ignore" the signal)
76  if (old_action.sa_handler != SIG_IGN)
77  {
78  struct sigaction action;
79  action.sa_handler = signal_handler;
80  sigemptyset(&action.sa_mask);
81  for (auto sigblk : signals)
82  {
83  sigaddset(&action.sa_mask, sigblk);
84  }
85  action.sa_flags = 0;
86 
87  //Replace the signal handler of SIGINT with the one described by new_action
88  sigaction(signal, &action, NULL);
89  old_actions[signal] = old_action;
90  }
91  }
92  }
93 }
94 
96 {
97  TLOG(TLVL_DEBUG) << "~SharedMemoryManager called" ;
98  Detach();
99  TLOG(TLVL_DEBUG) << "~SharedMemoryManager done" ;
100 }
101 
103 {
104  if (IsValid())
105  {
106  if (manager_id_ == 0) return;
107  Detach();
108  }
109  auto start_time = std::chrono::steady_clock::now();
110  last_seen_id_ = 0;
111  size_t shmSize = requested_shm_parameters_.buffer_count * (requested_shm_parameters_.buffer_size + sizeof(ShmBuffer)) + sizeof(ShmStruct);
112 
113  shm_segment_id_ = shmget(shm_key_, shmSize, 0666);
114  if (shm_segment_id_ == -1 && requested_shm_parameters_.buffer_count > 0 && manager_id_ <= 0)
115  {
116  TLOG(TLVL_DEBUG) << "Creating shared memory segment with key 0x" << std::hex << shm_key_ << " and size " << shmSize;
117  shm_segment_id_ = shmget(shm_key_, shmSize, IPC_CREAT | 0666);
118  manager_id_ = 0;
119 
120  if (shm_segment_id_ == -1)
121  {
122  TLOG(TLVL_ERROR) << "Error creating shared memory segment, errno=" << errno << " (" << strerror(errno) << ")";
123  }
124  }
125  else
126  {
127  while (shm_segment_id_ == -1 && TimeUtils::GetElapsedTimeMilliseconds(start_time) < 1000)
128  {
129  shm_segment_id_ = shmget(shm_key_, shmSize, 0666);
130 
131  }
132  }
133  TLOG(TLVL_DEBUG) << "shm_key == 0x" << std::hex << shm_key_ << ", shm_segment_id == " << shm_segment_id_ ;
134 
135  if (shm_segment_id_ > -1)
136  {
137  TLOG(TLVL_DEBUG)
138  << "Attached to shared memory segment with ID = " << shm_segment_id_
139  << " and size " << shmSize
140  << " bytes" ;
141  shm_ptr_ = (ShmStruct*)shmat(shm_segment_id_, 0, 0);
142  TLOG(TLVL_DEBUG)
143  << "Attached to shared memory segment at address "
144  << std::hex << (void*)shm_ptr_ << std::dec ;
145  if (shm_ptr_ && shm_ptr_ != (void *)-1)
146  {
147  if (manager_id_ == 0)
148  {
149  if (shm_ptr_->ready_magic == 0xCAFE1111)
150  {
151  TLOG(TLVL_ERROR) << "Owner encountered already-initialized Shared Memory!" ;
152  exit(-2);
153  }
154  TLOG(TLVL_DEBUG) << "Owner initializing Shared Memory" ;
155  shm_ptr_->next_id = 1;
156  shm_ptr_->next_sequence_id = 0;
157  shm_ptr_->reader_pos = 0;
158  shm_ptr_->writer_pos = 0;
159  shm_ptr_->buffer_size = requested_shm_parameters_.buffer_size;
160  shm_ptr_->buffer_count = requested_shm_parameters_.buffer_count;
161  shm_ptr_->buffer_timeout_us = requested_shm_parameters_.buffer_timeout_us;
162  shm_ptr_->destructive_read_mode = requested_shm_parameters_.destructive_read_mode;
163 
164  for (int ii = 0; ii < static_cast<int>(requested_shm_parameters_.buffer_count); ++ii)
165  {
166  getBufferInfo_(ii)->writePos = 0;
167  getBufferInfo_(ii)->readPos = 0;
168  getBufferInfo_(ii)->sem = BufferSemaphoreFlags::Empty;
169  getBufferInfo_(ii)->sem_id = -1;
170  getBufferInfo_(ii)->last_touch_time = TimeUtils::gettimeofday_us();
171  }
172 
173  shm_ptr_->ready_magic = 0xCAFE1111;
174  }
175  else
176  {
177  TLOG(TLVL_DEBUG) << "Waiting for owner to initalize Shared Memory" ;
178  while (shm_ptr_->ready_magic != 0xCAFE1111) { usleep(1000); }
179  TLOG(TLVL_DEBUG) << "Getting ID from Shared Memory" ;
180  GetNewId();
181  shm_ptr_->lowest_seq_id_read = 0;
182  TLOG(TLVL_DEBUG) << "Getting Shared Memory Size parameters" ;
183  }
184  //last_seen_id_ = shm_ptr_->next_sequence_id;
185  TLOG(TLVL_DEBUG) << "Initialization Complete: "
186  << "key: 0x" << std::hex << shm_key_
187  << ", manager ID: " << manager_id_
188  << ", Buffer size: " << std::to_string(shm_ptr_->buffer_size)
189  << ", Buffer count: " << std::to_string(shm_ptr_->buffer_count) ;
190  return;
191  }
192  else
193  {
194  TLOG(TLVL_ERROR) << "Failed to attach to shared memory segment "
195  << shm_segment_id_ ;
196  }
197  }
198  else
199  {
200  TLOG(TLVL_ERROR) << "Failed to connect to shared memory segment"
201  << ", errno = " << strerror(errno) << ". Please check "
202  << "if a stale shared memory segment needs to "
203  << "be cleaned up. (ipcs, ipcrm -m <segId>)" ;
204  }
205  return;
206 }
207 
209 {
210  TLOG(13) << "GetBufferForReading BEGIN" ;
211 
212  //std::unique_lock<std::mutex> lk(search_mutex_);
213  TraceLock lk(search_mutex_, 11, "GetBufferForReadingSearch");
214  auto rp = shm_ptr_->reader_pos.load();
215 
216  TLOG(13) << "GetBufferForReading lock acquired, scanning buffers" ;
217  bool retry = true;
218  int buffer_num = -1;
219  while (retry)
220  {
221  ShmBuffer* buffer_ptr = nullptr;
222  uint64_t seqID = -1;
223  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
224  {
225  auto buffer = (ii + rp) % shm_ptr_->buffer_count;
226 
227 
228  TLOG(14) << "GetBufferForReading Checking if buffer " << buffer << " is stale" ;
229  ResetBuffer(buffer);
230 
231  auto buf = getBufferInfo_(buffer);
232  TLOG(14) << "GetBufferForReading: Buffer " << buffer << ": sem=" << FlagToString(buf->sem) << " (expected " << FlagToString(BufferSemaphoreFlags::Full) << "), sem_id=" << buf->sem_id << " )" ;
233  if (buf->sem == BufferSemaphoreFlags::Full && (buf->sem_id == -1 || buf->sem_id == manager_id_) && buf->sequence_id > last_seen_id_)
234  {
235  if (buf->sequence_id < seqID)
236  {
237  buffer_ptr = buf;
238  seqID = buf->sequence_id;
239  buffer_num = buffer;
240  }
241  }
242  }
243 
244  if (!buffer_ptr || (buffer_ptr && buffer_ptr->sem_id != -1 && buffer_ptr->sem_id != manager_id_))
245  {
246  continue;
247  }
248 
249  if (buffer_num >= 0)
250  {
251  TLOG(13) << "GetBufferForReading Found buffer " << buffer_num ;
252  buffer_ptr->sem_id = manager_id_;
253  buffer_ptr->sem = BufferSemaphoreFlags::Reading;
254  if (buffer_ptr->sem_id != manager_id_) { continue; } // Failed to acquire buffer
255  buffer_ptr->readPos = 0;
256  touchBuffer_(buffer_ptr);
257  if (buffer_ptr->sem_id != manager_id_) { continue; } // Failed to acquire buffer
258  if (shm_ptr_->destructive_read_mode && shm_ptr_->lowest_seq_id_read == last_seen_id_)
259  {
260  shm_ptr_->lowest_seq_id_read = seqID;
261  }
262  last_seen_id_ = seqID;
263  if (shm_ptr_->destructive_read_mode) shm_ptr_->reader_pos = (buffer_num + 1) % shm_ptr_->buffer_count;
264  }
265  retry = false;
266  }
267 
268  TLOG(13) << "GetBufferForReading returning -1 because no buffers are ready" ;
269  return buffer_num;
270 }
271 
273 {
274  TLOG(14) << "GetBufferForWriting BEGIN" ;
275  //std::unique_lock<std::mutex> lk(search_mutex_);
276  TraceLock lk(search_mutex_, 12, "GetBufferForWritingSearch");
277  auto wp = shm_ptr_->writer_pos.load();
278 
279  // First, only look for "Empty" buffers
280  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
281  {
282  auto buffer = (ii + wp) % shm_ptr_->buffer_count;
283 
284  ResetBuffer(buffer);
285 
286  auto buf = getBufferInfo_(buffer);
287  if (buf->sem == BufferSemaphoreFlags::Empty && buf->sem_id == -1)
288  {
289  buf->sem_id = manager_id_;
290  buf->sem = BufferSemaphoreFlags::Writing;
291  touchBuffer_(buf);
292  shm_ptr_->writer_pos = (buffer + 1) % shm_ptr_->buffer_count;
293  if (buf->sem_id != manager_id_) continue;
294  buf->sequence_id = ++shm_ptr_->next_sequence_id;
295  buf->writePos = 0;
296  TLOG(14) << "GetBufferForWriting returning " << buffer ;
297  return buffer;
298  }
299  }
300 
301  if (overwrite)
302  {
303  // Then, look for "Full" buffers
304  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
305  {
306  auto buffer = (ii + wp) % shm_ptr_->buffer_count;
307 
308  ResetBuffer(buffer);
309 
310  auto buf = getBufferInfo_(buffer);
311  if (buf->sem == BufferSemaphoreFlags::Full)
312  {
313  buf->sem_id = manager_id_;
314  buf->sem = BufferSemaphoreFlags::Writing;
315  if (buf->sem_id != manager_id_) continue;
316  buf->sequence_id = ++shm_ptr_->next_sequence_id;
317  buf->writePos = 0;
318  shm_ptr_->writer_pos = (buffer + 1) % shm_ptr_->buffer_count;
319  touchBuffer_(buf);
320  TLOG(14) << "GetBufferForWriting returning " << buffer ;
321  return buffer;
322  }
323  }
324 
325  // Finally, if we still haven't found a buffer, we have to clobber a reader...
326  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
327  {
328  auto buffer = (ii + wp) % shm_ptr_->buffer_count;
329 
330  ResetBuffer(buffer);
331 
332  auto buf = getBufferInfo_(buffer);
333  if (buf->sem == BufferSemaphoreFlags::Reading)
334  {
335  buf->sem_id = manager_id_;
336  buf->sem = BufferSemaphoreFlags::Writing;
337  if (buf->sem_id != manager_id_) continue;
338  buf->sequence_id = ++shm_ptr_->next_sequence_id;
339  buf->writePos = 0;
340  shm_ptr_->writer_pos = (buffer + 1) % shm_ptr_->buffer_count;
341  TLOG(14) << "GetBufferForWriting returning " << buffer ;
342  touchBuffer_(buf);
343  return buffer;
344  }
345  }
346  }
347 
348  TLOG(14) << "GetBufferForWriting Returning -1 because no buffers are ready" ;
349  return -1;
350 }
351 
353 {
354  if (!IsValid()) return 0;
355  TLOG(23) << "0x" << std::hex << shm_key_ << " ReadReadyCount BEGIN" ;
356  //std::unique_lock<std::mutex> lk(search_mutex_);
357  TraceLock lk(search_mutex_, 14, "ReadReadyCountSearch");
358  size_t count = 0;
359  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
360  {
361  TLOG(24) << "0x" << std::hex << shm_key_ << std::dec << " ReadReadyCount: Checking if buffer " << ii << " is stale." ;
362  ResetBuffer(ii);
363  auto buf = getBufferInfo_(ii);
364  TLOG(25) << "0x" << std::hex << shm_key_ << std::dec << " ReadReadyCount: Buffer " << ii << ": sem=" << FlagToString(buf->sem) << " (expected " << FlagToString(BufferSemaphoreFlags::Full) << "), sem_id=" << buf->sem_id << " )" ;
365  if (buf->sem == BufferSemaphoreFlags::Full && (buf->sem_id == -1 || buf->sem_id == manager_id_) && buf->sequence_id > last_seen_id_)
366  {
367  TLOG(26) << "0x" << std::hex << shm_key_ << std::dec << " ReadReadyCount: Buffer " << ii << " is either unowned or owned by this manager, and is marked full." ;
368  ++count;
369  }
370  }
371  return count;
372 }
373 
375 {
376  if (!IsValid()) return 0;
377  //std::unique_lock<std::mutex> lk(search_mutex_);
378  TraceLock lk(search_mutex_, 15, "WriteReadyCountSearch");
379  size_t count = 0;
380  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
381  {
382  ResetBuffer(ii);
383  auto buf = getBufferInfo_(ii);
384  if ((buf->sem == BufferSemaphoreFlags::Empty && buf->sem_id == -1)
385  || (overwrite && buf->sem != BufferSemaphoreFlags::Writing))
386  {
387  ++count;
388  }
389  }
390  return count;
391 }
392 
394 {
395  //std::unique_lock<std::mutex> lk(search_mutex_);
396  std::deque<int> output;
397  if (!IsValid()) return output;
398  if(locked) {
399  TraceLock lk(search_mutex_, 16, "GetOwnedSearch");
400  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
401  {
402  auto buf = getBufferInfo_(ii);
403  if (buf->sem_id == manager_id_)
404  {
405  output.push_back(ii);
406  }
407  }
408  }
409  else
410  {
411  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
412  {
413  auto buf = getBufferInfo_(ii);
414  if (buf->sem_id == manager_id_)
415  {
416  output.push_back(ii);
417  }
418  }
419  }
420 
421  return output;
422 }
423 
425 {
426  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
427  TraceLock lk(buffer_mutexes_[buffer], 17, "DataSizeBuffer" + std::to_string(buffer));
428  auto buf = getBufferInfo_(buffer);
429  touchBuffer_(buf);
430  return buf->writePos;
431 }
432 
433 
435 {
436  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
437  TraceLock lk(buffer_mutexes_[buffer], 18, "ResetReadPosBuffer" + std::to_string(buffer));
438  auto buf = getBufferInfo_(buffer);
439  touchBuffer_(buf);
440  buf->readPos = 0;
441 }
442 
444 {
445  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
446  TraceLock lk(buffer_mutexes_[buffer], 18, "ResetWritePosBuffer" + std::to_string(buffer));
447  auto buf = getBufferInfo_(buffer);
448  checkBuffer_(buf, BufferSemaphoreFlags::Writing);
449  touchBuffer_(buf);
450  buf->writePos = 0;
451 }
452 
454 {
455  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
456  TraceLock lk(buffer_mutexes_[buffer], 19, "IncReadPosBuffer" + std::to_string(buffer));
457  auto buf = getBufferInfo_(buffer);
458  touchBuffer_(buf);
459  TLOG(15) << "IncrementReadPos: buffer= " << buffer << ", readPos=" << std::to_string(buf->readPos) << ", bytes read=" << std::to_string(read) ;
460  buf->readPos = buf->readPos + read;
461  TLOG(15) << "IncrementReadPos: buffer= " << buffer << ", New readPos is " << std::to_string(buf->readPos) ;
462  if (read == 0) Detach(true, "LogicError", "Cannot increment Read pos by 0! (buffer=" + std::to_string(buffer) + ", readPos=" + std::to_string(buf->readPos) + ", writePos=" + std::to_string(buf->writePos) + ")");
463 }
464 
465 void artdaq::SharedMemoryManager::IncrementWritePos(int buffer, size_t written)
466 {
467  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
468  TraceLock lk(buffer_mutexes_[buffer], 20, "IncWritePosBuffer" + std::to_string(buffer));
469  auto buf = getBufferInfo_(buffer);
470  touchBuffer_(buf);
471  TLOG(16) << "IncrementWritePos: buffer= " << buffer << ", writePos=" << std::to_string(buf->writePos) << ", bytes written=" << std::to_string(written) ;
472  buf->writePos += written;
473  TLOG(16) << "IncrementWritePos: buffer= " << buffer << ", New writePos is " << std::to_string(buf->writePos) ;
474  if (written == 0) Detach(true, "LogicError", "Cannot increment Write pos by 0!");
475 }
476 
478 {
479  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
480  TraceLock lk(buffer_mutexes_[buffer], 21, "MoreDataInBuffer" + std::to_string(buffer));
481  auto buf = getBufferInfo_(buffer);
482  TLOG(17) << "MoreDataInBuffer: buffer= " << buffer << ", readPos=" << std::to_string(buf->readPos) << ", writePos=" << std::to_string(buf->writePos) ;
483  return buf->readPos < buf->writePos;
484 }
485 
487 {
488  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
489  TraceLock lk(buffer_mutexes_[buffer], 22, "CheckBuffer" + std::to_string(buffer));
490  return checkBuffer_(getBufferInfo_(buffer), flags, false);
491 }
492 
493 void artdaq::SharedMemoryManager::MarkBufferFull(int buffer, int destination)
494 {
495  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
496  TraceLock lk(buffer_mutexes_[buffer], 23, "FillBuffer" + std::to_string(buffer));
497  auto shmBuf = getBufferInfo_(buffer);
498  touchBuffer_(shmBuf);
499  if (shmBuf->sem_id == manager_id_)
500  {
501  if (shmBuf->sem != BufferSemaphoreFlags::Full)
502  shmBuf->sem = BufferSemaphoreFlags::Full;
503 
504  shmBuf->sem_id = destination;
505  }
506 }
507 
509 {
510  TLOG(18) << "MarkBufferEmpty BEGIN" ;
511  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
512  TraceLock lk(buffer_mutexes_[buffer], 24, "EmptyBuffer" + std::to_string(buffer));
513  auto shmBuf = getBufferInfo_(buffer);
514  if (!force)
515  {
516  checkBuffer_(shmBuf, BufferSemaphoreFlags::Reading, true);
517  }
518  touchBuffer_(shmBuf);
519 
520  shmBuf->readPos = 0;
521  shmBuf->sem = BufferSemaphoreFlags::Full;
522 
523  if ((force && (manager_id_ == 0 || manager_id_ == shmBuf->sem_id)) || (!force && shm_ptr_->destructive_read_mode))
524  {
525  TLOG(18) << "MarkBufferEmpty Resetting buffer to Empty state" ;
526  shmBuf->writePos = 0;
527  shmBuf->sem = BufferSemaphoreFlags::Empty;
528  if (shm_ptr_->reader_pos == static_cast<unsigned>(buffer) && !shm_ptr_->destructive_read_mode)
529  {
530  TLOG(18) << "MarkBufferEmpty Broadcast mode; incrementing reader_pos" ;
531  shm_ptr_->reader_pos = (buffer + 1) % shm_ptr_->buffer_count;
532  }
533  }
534  shmBuf->sem_id = -1;
535  TLOG(18) << "MarkBufferEmpty END" ;
536 }
537 
539 {
540  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
541  TraceLock lk(buffer_mutexes_[buffer], 25, "ResetBuffer" + std::to_string(buffer));
542  auto shmBuf = getBufferInfo_(buffer);
543  /*
544  if (shmBuf->sequence_id < shm_ptr_->lowest_seq_id_read - size() && shmBuf->sem == BufferSemaphoreFlags::Full)
545  {
546  TLOG(TLVL_DEBUG) << "Buffer " << buffer << " has been passed by all readers, marking Empty" ;
547  shmBuf->writePos = 0;
548  shmBuf->sem = BufferSemaphoreFlags::Empty;
549  shmBuf->sem_id = -1;
550  return true;
551  }*/
552 
553  size_t delta = TimeUtils::gettimeofday_us() - shmBuf->last_touch_time;
554  if (delta > 0xFFFFFFFF) {
555  TLOG(TLVL_TRACE) << "Buffer has touch time in the future, setting it to current time and ignoring..." ;
556  shmBuf->last_touch_time = TimeUtils::gettimeofday_us();
557  return false;
558  }
559  if (shm_ptr_->buffer_timeout_us == 0 || delta <= shm_ptr_->buffer_timeout_us || shmBuf->sem == BufferSemaphoreFlags::Empty) return false;
560  TLOG(27) << "Buffer " << buffer << " is stale, time=" << TimeUtils::gettimeofday_us() << ", last touch=" << shmBuf->last_touch_time << ", d=" << delta << ", timeout=" << shm_ptr_->buffer_timeout_us ;
561 
562  if (shmBuf->sem_id == manager_id_ && shmBuf->sem == BufferSemaphoreFlags::Writing)
563  {
564  return true;
565  }
566  if (!shm_ptr_->destructive_read_mode && shmBuf->sem == BufferSemaphoreFlags::Full)
567  {
568  TLOG(TLVL_DEBUG) << "Resetting old broadcast mode buffer" ;
569  shmBuf->writePos = 0;
570  shmBuf->sem = BufferSemaphoreFlags::Empty;
571  shmBuf->sem_id = -1;
572  if (shm_ptr_->reader_pos == static_cast<unsigned>(buffer)) shm_ptr_->reader_pos = (buffer + 1) % shm_ptr_->buffer_count;
573  return true;
574  }
575 
576  if (shmBuf->sem_id != manager_id_ && shmBuf->sem == BufferSemaphoreFlags::Reading)
577  {
578  TLOG(TLVL_WARNING) << "Stale Read buffer ( " << delta << " / " << shm_ptr_->buffer_timeout_us << " us ) detected! Resetting..." ;
579  shmBuf->readPos = 0;
580  shmBuf->sem = BufferSemaphoreFlags::Full;
581  shmBuf->sem_id = -1;
582  return true;
583  }
584  return false;
585 }
586 
588 {
589  if (!IsValid()) return true;
590 
591  struct shmid_ds info;
592  auto sts = shmctl(shm_segment_id_, IPC_STAT, &info);
593  if (sts < 0) {
594  TLOG(TLVL_TRACE) << "Error accessing Shared Memory info: " << errno << " (" << strerror(errno) << ").";
595  return true;
596  }
597 
598  if (info.shm_perm.mode & SHM_DEST) {
599  TLOG(TLVL_INFO) << "Shared Memory marked for destruction. Probably an end-of-data condition!";
600  return true;
601  }
602 
603  return false;
604 }
605 
606 size_t artdaq::SharedMemoryManager::Write(int buffer, void* data, size_t size)
607 {
608  TLOG(19) << "Write BEGIN" ;
609  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
610  TraceLock lk(buffer_mutexes_[buffer], 26, "WriteBuffer" + std::to_string(buffer));
611  auto shmBuf = getBufferInfo_(buffer);
612  checkBuffer_(shmBuf, BufferSemaphoreFlags::Writing);
613  touchBuffer_(shmBuf);
614  TLOG(19) << "Buffer Write Pos is " << std::to_string(shmBuf->writePos) << ", write size is " << std::to_string(size) ;
615  if (shmBuf->writePos + size > shm_ptr_->buffer_size) Detach(true, "SharedMemoryWrite", "Attempted to write more data than fits into Shared Memory! \nRe-run with a larger buffer size!");
616 
617  auto pos = GetWritePos(buffer);
618  memcpy(pos, data, size);
619  shmBuf->writePos = shmBuf->writePos + size;
620  if (shmBuf->sequence_id > last_seen_id_)
621  {
622  last_seen_id_ = shmBuf->sequence_id;
623  }
624  TLOG(19) << "Write END" ;
625  return size;
626 }
627 
628 bool artdaq::SharedMemoryManager::Read(int buffer, void* data, size_t size)
629 {
630  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
631  TraceLock lk(buffer_mutexes_[buffer], 27, "ReadBuffer" + std::to_string(buffer));
632  auto shmBuf = getBufferInfo_(buffer);
633  checkBuffer_(shmBuf, BufferSemaphoreFlags::Reading);
634  touchBuffer_(shmBuf);
635  if (shmBuf->readPos + size > shm_ptr_->buffer_size) Detach(true, "SharedMemoryRead", "Attempted to read more data than exists in Shared Memory!");
636 
637  auto pos = GetReadPos(buffer);
638  memcpy(data, pos, size);
639  shmBuf->readPos += size;
640  touchBuffer_(shmBuf);
641  return checkBuffer_(shmBuf, BufferSemaphoreFlags::Reading, false);
642 }
643 
645 {
646  std::ostringstream ostr;
647  ostr << "ShmStruct: " << std::endl
648  << "Reader Position: " << shm_ptr_->reader_pos << std::endl
649  << "Writer Position: " << shm_ptr_->writer_pos << std::endl
650  << "Next ID Number: " << shm_ptr_->next_id << std::endl
651  << "Buffer Count: " << shm_ptr_->buffer_count << std::endl
652  << "Buffer Size: " << std::to_string(shm_ptr_->buffer_size) << " bytes" << std::endl
653  << "Buffers Written: " << std::to_string(shm_ptr_->next_sequence_id) << std::endl
654  << "Rank of Writer: " << shm_ptr_->rank << std::endl
655  << "Ready Magic Bytes: 0x" << std::hex << shm_ptr_->ready_magic << std::endl << std::endl;
656 
657  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
658  {
659  auto buf = getBufferInfo_(ii);
660  ostr << "ShmBuffer " << std::dec << ii << std::endl
661  << "sequenceID: " << std::to_string(buf->sequence_id) << std::endl
662  << "writePos: " << std::to_string(buf->writePos) << std::endl
663  << "readPos: " << std::to_string(buf->readPos) << std::endl
664  << "sem: " << FlagToString(buf->sem) << std::endl
665  << "Owner: " << std::to_string(buf->sem_id.load()) << std::endl
666  << "Last Touch Time: " << std::to_string(buf->last_touch_time / 1000000.0) << std::endl << std::endl;
667  }
668 
669  return ostr.str();
670 }
671 
673 {
674  auto buf = getBufferInfo_(buffer);
675  return bufferStart_(buffer) + buf->readPos;
676 }
678 {
679  auto buf = getBufferInfo_(buffer);
680  return bufferStart_(buffer) + buf->writePos;
681 }
682 
684 {
685  return bufferStart_(buffer);
686 }
687 
688 uint8_t* artdaq::SharedMemoryManager::dataStart_() const
689 {
690  return reinterpret_cast<uint8_t*>(shm_ptr_ + 1) + shm_ptr_->buffer_count * sizeof(ShmBuffer);
691 }
692 
693 uint8_t* artdaq::SharedMemoryManager::bufferStart_(int buffer)
694 {
695  if (buffer >= shm_ptr_->buffer_count) Detach(true, "ArgumentOutOfRange", "The specified buffer does not exist!");
696  return dataStart_() + buffer * shm_ptr_->buffer_size;
697 }
698 
699 artdaq::SharedMemoryManager::ShmBuffer* artdaq::SharedMemoryManager::getBufferInfo_(int buffer)
700 {
701  if (buffer >= shm_ptr_->buffer_count) Detach(true, "ArgumentOutOfRange", "The specified buffer does not exist!");
702  return reinterpret_cast<ShmBuffer*>(reinterpret_cast<uint8_t*>(shm_ptr_ + 1) + buffer * sizeof(ShmBuffer));
703 }
704 
705 bool artdaq::SharedMemoryManager::checkBuffer_(ShmBuffer* buffer, BufferSemaphoreFlags flags, bool exceptions)
706 {
707  TLOG(TLVL_TRACE) << "checkBuffer_: Checking that buffer " << buffer->sequence_id << " has sem_id " << manager_id_ << " (Current: " << buffer->sem_id << ") and is in state " << FlagToString(flags) << " (current: " << FlagToString(buffer->sem) << ")";
708  if (exceptions)
709  {
710  if (buffer->sem != flags) Detach(true, "StateAccessViolation", "Shared Memory buffer is not in the correct state! (expected " + FlagToString(flags) + ", actual " + FlagToString(buffer->sem) + ")");
711  if (buffer->sem_id != manager_id_) Detach(true, "OwnerAccessViolation", "Shared Memory buffer is not owned by this manager instance! (Expected: " + std::to_string(manager_id_) + ", Actual: " + std::to_string(buffer->sem_id) + ")");
712  }
713  bool ret = (buffer->sem_id == manager_id_ || (buffer->sem_id == -1 && (flags == BufferSemaphoreFlags::Full || flags == BufferSemaphoreFlags::Empty))) && buffer->sem == flags;
714 
715  if (!ret)
716  {
717  TLOG(TLVL_WARNING) << "CheckBuffer detected issue with buffer " << std::to_string(buffer->sequence_id) << "!"
718  << " ID: " << buffer->sem_id << " (" << manager_id_ << "), Flag: " << FlagToString(buffer->sem) << " (" << FlagToString(flags) << "). "
719  << "ID -1 is okay if desired flag is \"Full\" or \"Empty\"." ;
720  }
721 
722  return ret;
723 }
724 
725 void artdaq::SharedMemoryManager::touchBuffer_(ShmBuffer* buffer)
726 {
727  if (buffer->sem_id != manager_id_) return;
728  TLOG(TLVL_TRACE) << "touchBuffer_: Touching buffer with sequence_id " << std::to_string(buffer->sequence_id) ;
729  buffer->last_touch_time = TimeUtils::gettimeofday_us();
730 }
731 
732 void artdaq::SharedMemoryManager::Detach(bool throwException, std::string category, std::string message, bool force)
733 {
734  TLOG(TLVL_DETACH) << "Detach BEGIN: throwException: " << std::boolalpha << throwException << ", force: " << force;
735  if (IsValid())
736  {
737  TLOG(TLVL_DETACH) << "Detach: Resetting owned buffers";
738  auto bufs = GetBuffersOwnedByManager(false);
739  for (auto buf : bufs)
740  {
741  auto shmBuf = getBufferInfo_(buf);
742  if (shmBuf->sem == BufferSemaphoreFlags::Writing)
743  {
744  shmBuf->sem = BufferSemaphoreFlags::Empty;
745  }
746  else if (shmBuf->sem == BufferSemaphoreFlags::Reading)
747  {
748  shmBuf->sem = BufferSemaphoreFlags::Full;
749  }
750  shmBuf->sem_id = -1;
751  }
752  }
753 
754  if (shm_ptr_)
755  {
756  TLOG(TLVL_DETACH) << "Detach: Detaching shared memory";
757  shmdt(shm_ptr_);
758  shm_ptr_ = NULL;
759  }
760 
761  if ((force || manager_id_ == 0) && shm_segment_id_ > -1)
762  {
763  TLOG(TLVL_DETACH) << "Detach: Marking Shared memory for removal";
764  shmctl(shm_segment_id_, IPC_RMID, NULL);
765  shm_segment_id_ = -1;
766  }
767 
768  if (category.size() > 0 && message.size() > 0)
769  {
770  TLOG(TLVL_ERROR) << category << ": " << message ;
771 
772  if (throwException)
773  {
774  throw cet::exception(category) << message;
775  }
776  }
777 }
778 
779 
780 
781 // Local Variables:
782 // mode: c++
783 // End:
bool Read(int buffer, void *data, size_t size)
Read size bytes of data from buffer into the given pointer.
void IncrementReadPos(int buffer, size_t read)
Increment the read position for a given buffer.
constexpr size_t GetElapsedTimeMilliseconds(std::chrono::steady_clock::time_point then, std::chrono::steady_clock::time_point now=std::chrono::steady_clock::now())
Gets the number of milliseconds in the given time interval
Definition: TimeUtils.hh:54
std::deque< int > GetBuffersOwnedByManager(bool locked=true)
Get the list of all buffers currently owned by this manager instance.
void ResetReadPos(int buffer)
Set the read position of the given buffer to the beginning of the buffer.
bool MoreDataInBuffer(int buffer)
Determine if more data is available to be read, based on the read position and data size...
virtual std::string toString()
Write information about the SharedMemory to a string.
BufferSemaphoreFlags
The BufferSemaphoreFlags enumeration represents the different possible &quot;states&quot; of a given shared mem...
size_t BufferDataSize(int buffer)
Get the current size of the buffer&#39;s data.
void Attach()
Reconnect to the shared memory segment.
void Detach(bool throwException=false, std::string category="", std::string message="", bool force=false)
Detach from the Shared Memory segment, optionally throwing a cet::exception with the specified proper...
virtual ~SharedMemoryManager() noexcept
SharedMemoryManager Destructor.
size_t ReadReadyCount()
Count the number of buffers that are ready for reading.
bool ResetBuffer(int buffer)
Resets the buffer from Reading to Full. This operation will only have an effect if performed by the o...
bool IsEndOfData() const
Determine whether the Shared Memory is marked for destruction (End of Data)
bool CheckBuffer(int buffer, BufferSemaphoreFlags flags)
Check both semaphore conditions (Mode flag and manager ID) for a given buffer.
void IncrementWritePos(int buffer, size_t written)
Increment the write position for a given buffer.
The TraceLock class allows a user to debug the acquisition and releasing of locks, by wrapping the unique_lock&lt;std::mutex&gt; API with TRACE calls.
Definition: TraceLock.hh:10
void MarkBufferFull(int buffer, int destination=-1)
Release a buffer from a writer, marking it Full and ready for a reader.
void MarkBufferEmpty(int buffer, bool force=false)
Release a buffer from a reader, marking it Empty and ready to accept more data.
void ResetWritePos(int buffer)
Set the write position of the given buffer to the beginning of the buffer.
void * GetWritePos(int buffer)
Get a pointer to the current write position of the buffer.
size_t Write(int buffer, void *data, size_t size)
Write size bytes of data from the given pointer to a buffer.
void * GetReadPos(int buffer)
Get a pointer to the current read position of the buffer.
void * GetBufferStart(int buffer)
Get a pointer to the start position of the buffer.
SharedMemoryManager(uint32_t shm_key, size_t buffer_count=0, size_t buffer_size=0, uint64_t buffer_timeout_us=100 *1000000, bool destructive_read_mode=true)
SharedMemoryManager Constructor.
int GetBufferForWriting(bool overwrite)
Finds a buffer that is ready to be written to, and reserves it for the calling manager.
size_t WriteReadyCount(bool overwrite)
Count the number of buffers that are ready for writing.
uint64_t gettimeofday_us()
Get the current time of day in microseconds (from gettimeofday system call)
Definition: TimeUtils.cc:51
int GetBufferForReading()
Finds a buffer that is ready to be read, and reserves it for the calling manager. ...