artdaq_core  v3_02_01
SharedMemoryManager.cc
1 #define TRACE_NAME "SharedMemoryManager"
2 #include <cstring>
3 #include <vector>
4 #include <sys/shm.h>
5 #include "tracemf.h"
6 #include <signal.h>
7 #include "cetlib_except/exception.h"
8 #include "artdaq-core/Core/SharedMemoryManager.hh"
9 #include "artdaq-core/Utilities/TraceLock.hh"
10 
11 #define TLVL_DETACH 11
12 
13 static std::vector<artdaq::SharedMemoryManager const*> instances = std::vector<artdaq::SharedMemoryManager const*>();
14 
15 static std::unordered_map<int, struct sigaction> old_actions = std::unordered_map<int, struct sigaction>();
16 static bool sighandler_init = false;
17 static void signal_handler(int signum)
18 {
19  // Messagefacility may already be gone at this point, TRACE ONLY!
20  TRACE_STREAMER(TLVL_ERROR, &("SharedMemoryManager")[0], 0, 0, 0) << "A signal of type " << signum << " (" << std::string(strsignal(signum)) << ") was caught by SharedMemoryManager. Detaching all Shared Memory segments, then proceeding with default handlers!";
21  for (auto ii : instances)
22  {
23 
24  if (ii)
25  {
26  const_cast<artdaq::SharedMemoryManager*>(ii)->Detach(false, "", "", true);
27  }
28  ii = nullptr;
29  }
30 
31  sigset_t set;
32  pthread_sigmask(SIG_UNBLOCK, NULL, &set);
33  pthread_sigmask(SIG_UNBLOCK, &set, NULL);
34 
35  TRACE_STREAMER(TLVL_ERROR, &("SharedMemoryManager")[0], 0, 0, 0) << "Calling default signal handler";
36  if (signum != SIGUSR2)
37  {
38  sigaction(signum, &old_actions[signum], NULL);
39  kill(getpid(), signum); // Only send signal to self
40  }
41  else
42  {
43  // Send Interrupt signal if parsing SIGUSR2 (i.e. user-defined exception that should tear down ARTDAQ)
44  sigaction(SIGINT, &old_actions[SIGINT], NULL);
45  kill(getpid(), SIGINT); // Only send signal to self
46  }
47 }
48 
49 artdaq::SharedMemoryManager::SharedMemoryManager(uint32_t shm_key, size_t buffer_count, size_t buffer_size, uint64_t buffer_timeout_us, bool destructive_read_mode)
50  : shm_segment_id_(-1)
51  , shm_ptr_(NULL)
52  , shm_key_(shm_key)
53  , manager_id_(-1)
54  , buffer_mutexes_()
55  , last_seen_id_(0)
56 {
57  requested_shm_parameters_.buffer_count = buffer_count;
58  requested_shm_parameters_.buffer_size = buffer_size;
59  requested_shm_parameters_.buffer_timeout_us = buffer_timeout_us;
60  requested_shm_parameters_.destructive_read_mode = destructive_read_mode;
61 
62  instances.push_back(this);
63  Attach();
64 
65  static std::mutex sighandler_mutex;
66  std::unique_lock<std::mutex> lk(sighandler_mutex);
67 
68  if (!sighandler_init)//&& manager_id_ == 0) // ELF 3/22/18: Taking out manager_id_==0 requirement as I think kill(getpid()) is enough protection
69  {
70  sighandler_init = true;
71  std::vector<int> signals = { SIGINT, SIGQUIT, SIGILL, SIGABRT, SIGFPE, SIGSEGV, SIGPIPE, SIGALRM, SIGTERM, SIGUSR2 };
72  for (auto signal : signals)
73  {
74  struct sigaction old_action;
75  sigaction(signal, NULL, &old_action);
76 
77  //If the old handler wasn't SIG_IGN (it's a handler that just
78  // "ignore" the signal)
79  if (old_action.sa_handler != SIG_IGN)
80  {
81  struct sigaction action;
82  action.sa_handler = signal_handler;
83  sigemptyset(&action.sa_mask);
84  for (auto sigblk : signals)
85  {
86  sigaddset(&action.sa_mask, sigblk);
87  }
88  action.sa_flags = 0;
89 
90  //Replace the signal handler of SIGINT with the one described by new_action
91  sigaction(signal, &action, NULL);
92  old_actions[signal] = old_action;
93  }
94  }
95  }
96 }
97 
99 {
100  TLOG(TLVL_DEBUG) << "~SharedMemoryManager called";
101  Detach();
102  TLOG(TLVL_DEBUG) << "~SharedMemoryManager done";
103 }
104 
106 {
107  if (IsValid())
108  {
109  if (manager_id_ == 0) return;
110  Detach();
111  }
112  auto start_time = std::chrono::steady_clock::now();
113  last_seen_id_ = 0;
114  size_t shmSize = requested_shm_parameters_.buffer_count * (requested_shm_parameters_.buffer_size + sizeof(ShmBuffer)) + sizeof(ShmStruct);
115 
116  shm_segment_id_ = shmget(shm_key_, shmSize, 0666);
117  if (shm_segment_id_ == -1 && requested_shm_parameters_.buffer_count > 0 && manager_id_ <= 0)
118  {
119  TLOG(TLVL_DEBUG) << "Creating shared memory segment with key 0x" << std::hex << shm_key_ << " and size " << shmSize;
120  shm_segment_id_ = shmget(shm_key_, shmSize, IPC_CREAT | 0666);
121  manager_id_ = 0;
122 
123  if (shm_segment_id_ == -1)
124  {
125  TLOG(TLVL_ERROR) << "Error creating shared memory segment with key 0x" << std::hex << shm_key_ << ", errno=" << errno << " (" << strerror(errno) << ")";
126  }
127  }
128  else
129  {
130  while (shm_segment_id_ == -1 && TimeUtils::GetElapsedTimeMilliseconds(start_time) < 1000)
131  {
132  shm_segment_id_ = shmget(shm_key_, shmSize, 0666);
133 
134  }
135  }
136  TLOG(TLVL_DEBUG) << "shm_key == 0x" << std::hex << shm_key_ << ", shm_segment_id == " << shm_segment_id_;
137 
138  if (shm_segment_id_ > -1)
139  {
140  TLOG(TLVL_DEBUG)
141  << "Attached to shared memory segment with ID = " << shm_segment_id_
142  << " and size " << shmSize
143  << " bytes";
144  shm_ptr_ = (ShmStruct*)shmat(shm_segment_id_, 0, 0);
145  TLOG(TLVL_DEBUG)
146  << "Attached to shared memory segment at address "
147  << std::hex << (void*)shm_ptr_ << std::dec;
148  if (shm_ptr_ && shm_ptr_ != (void *)-1)
149  {
150  if (manager_id_ == 0)
151  {
152  if (shm_ptr_->ready_magic == 0xCAFE1111)
153  {
154  TLOG(TLVL_ERROR) << "Owner encountered already-initialized Shared Memory!";
155  exit(-2);
156  }
157  TLOG(TLVL_DEBUG) << "Owner initializing Shared Memory";
158  shm_ptr_->next_id = 1;
159  shm_ptr_->next_sequence_id = 0;
160  shm_ptr_->reader_pos = 0;
161  shm_ptr_->writer_pos = 0;
162  shm_ptr_->buffer_size = requested_shm_parameters_.buffer_size;
163  shm_ptr_->buffer_count = requested_shm_parameters_.buffer_count;
164  shm_ptr_->buffer_timeout_us = requested_shm_parameters_.buffer_timeout_us;
165  shm_ptr_->destructive_read_mode = requested_shm_parameters_.destructive_read_mode;
166 
167  for (int ii = 0; ii < static_cast<int>(requested_shm_parameters_.buffer_count); ++ii)
168  {
169  getBufferInfo_(ii)->writePos = 0;
170  getBufferInfo_(ii)->readPos = 0;
171  getBufferInfo_(ii)->sem = BufferSemaphoreFlags::Empty;
172  getBufferInfo_(ii)->sem_id = -1;
173  getBufferInfo_(ii)->last_touch_time = TimeUtils::gettimeofday_us();
174  }
175 
176  shm_ptr_->ready_magic = 0xCAFE1111;
177  }
178  else
179  {
180  TLOG(TLVL_DEBUG) << "Waiting for owner to initalize Shared Memory";
181  while (shm_ptr_->ready_magic != 0xCAFE1111) { usleep(1000); }
182  TLOG(TLVL_DEBUG) << "Getting ID from Shared Memory";
183  GetNewId();
184  shm_ptr_->lowest_seq_id_read = 0;
185  TLOG(TLVL_DEBUG) << "Getting Shared Memory Size parameters";
186  }
187  //last_seen_id_ = shm_ptr_->next_sequence_id;
188  TLOG(TLVL_DEBUG) << "Initialization Complete: "
189  << "key: 0x" << std::hex << shm_key_
190  << ", manager ID: " << manager_id_
191  << ", Buffer size: " << std::to_string(shm_ptr_->buffer_size)
192  << ", Buffer count: " << std::to_string(shm_ptr_->buffer_count);
193  return;
194  }
195  else
196  {
197  TLOG(TLVL_ERROR) << "Failed to attach to shared memory segment "
198  << shm_segment_id_;
199  }
200  }
201  else
202  {
203  TLOG(TLVL_ERROR) << "Failed to connect to shared memory segment with key 0x" << std::hex << shm_key_
204  << ", errno = " << strerror(errno) << ". Please check "
205  << "if a stale shared memory segment needs to "
206  << "be cleaned up. (ipcs, ipcrm -m <segId>)";
207  }
208  return;
209 }
210 
212 {
213  TLOG(13) << "GetBufferForReading BEGIN";
214 
215  //std::unique_lock<std::mutex> lk(search_mutex_);
216  TraceLock lk(search_mutex_, 11, "GetBufferForReadingSearch");
217  auto rp = shm_ptr_->reader_pos.load();
218 
219  TLOG(13) << "GetBufferForReading lock acquired, scanning buffers";
220  bool retry = true;
221  int buffer_num = -1;
222  while (retry)
223  {
224  ShmBuffer* buffer_ptr = nullptr;
225  uint64_t seqID = -1;
226  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
227  {
228  auto buffer = (ii + rp) % shm_ptr_->buffer_count;
229 
230 
231  TLOG(14) << "GetBufferForReading Checking if buffer " << buffer << " is stale";
232  ResetBuffer(buffer);
233 
234  auto buf = getBufferInfo_(buffer);
235  TLOG(14) << "GetBufferForReading: Buffer " << buffer << ": sem=" << FlagToString(buf->sem) << " (expected " << FlagToString(BufferSemaphoreFlags::Full) << "), sem_id=" << buf->sem_id << " )";
236  if (buf->sem == BufferSemaphoreFlags::Full && (buf->sem_id == -1 || buf->sem_id == manager_id_) && buf->sequence_id > last_seen_id_)
237  {
238  if (buf->sequence_id < seqID)
239  {
240  buffer_ptr = buf;
241  seqID = buf->sequence_id;
242  buffer_num = buffer;
243  }
244  }
245  }
246 
247  if (!buffer_ptr || (buffer_ptr && buffer_ptr->sem_id != -1 && buffer_ptr->sem_id != manager_id_))
248  {
249  continue;
250  }
251 
252  if (buffer_num >= 0)
253  {
254  TLOG(13) << "GetBufferForReading Found buffer " << buffer_num;
255  buffer_ptr->sem_id = manager_id_;
256  buffer_ptr->sem = BufferSemaphoreFlags::Reading;
257  if (buffer_ptr->sem_id != manager_id_) { continue; } // Failed to acquire buffer
258  buffer_ptr->readPos = 0;
259  touchBuffer_(buffer_ptr);
260  if (buffer_ptr->sem_id != manager_id_) { continue; } // Failed to acquire buffer
261  if (shm_ptr_->destructive_read_mode && shm_ptr_->lowest_seq_id_read == last_seen_id_)
262  {
263  shm_ptr_->lowest_seq_id_read = seqID;
264  }
265  last_seen_id_ = seqID;
266  if (shm_ptr_->destructive_read_mode) shm_ptr_->reader_pos = (buffer_num + 1) % shm_ptr_->buffer_count;
267  }
268  retry = false;
269  }
270 
271  TLOG(13) << "GetBufferForReading returning -1 because no buffers are ready";
272  return buffer_num;
273 }
274 
276 {
277  TLOG(14) << "GetBufferForWriting BEGIN";
278  //std::unique_lock<std::mutex> lk(search_mutex_);
279  TraceLock lk(search_mutex_, 12, "GetBufferForWritingSearch");
280  auto wp = shm_ptr_->writer_pos.load();
281 
282  // First, only look for "Empty" buffers
283  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
284  {
285  auto buffer = (ii + wp) % shm_ptr_->buffer_count;
286 
287  ResetBuffer(buffer);
288 
289  auto buf = getBufferInfo_(buffer);
290  if (buf->sem == BufferSemaphoreFlags::Empty && buf->sem_id == -1)
291  {
292  buf->sem_id = manager_id_;
293  buf->sem = BufferSemaphoreFlags::Writing;
294  touchBuffer_(buf);
295  shm_ptr_->writer_pos = (buffer + 1) % shm_ptr_->buffer_count;
296  if (buf->sem_id != manager_id_) continue;
297  buf->sequence_id = ++shm_ptr_->next_sequence_id;
298  buf->writePos = 0;
299  TLOG(14) << "GetBufferForWriting returning " << buffer;
300  return buffer;
301  }
302  }
303 
304  if (overwrite)
305  {
306  // Then, look for "Full" buffers
307  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
308  {
309  auto buffer = (ii + wp) % shm_ptr_->buffer_count;
310 
311  ResetBuffer(buffer);
312 
313  auto buf = getBufferInfo_(buffer);
314  if (buf->sem == BufferSemaphoreFlags::Full)
315  {
316  buf->sem_id = manager_id_;
317  buf->sem = BufferSemaphoreFlags::Writing;
318  if (buf->sem_id != manager_id_) continue;
319  buf->sequence_id = ++shm_ptr_->next_sequence_id;
320  buf->writePos = 0;
321  shm_ptr_->writer_pos = (buffer + 1) % shm_ptr_->buffer_count;
322  touchBuffer_(buf);
323  TLOG(14) << "GetBufferForWriting returning " << buffer;
324  return buffer;
325  }
326  }
327 
328  // Finally, if we still haven't found a buffer, we have to clobber a reader...
329  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
330  {
331  auto buffer = (ii + wp) % shm_ptr_->buffer_count;
332 
333  ResetBuffer(buffer);
334 
335  auto buf = getBufferInfo_(buffer);
336  if (buf->sem == BufferSemaphoreFlags::Reading)
337  {
338  buf->sem_id = manager_id_;
339  buf->sem = BufferSemaphoreFlags::Writing;
340  if (buf->sem_id != manager_id_) continue;
341  buf->sequence_id = ++shm_ptr_->next_sequence_id;
342  buf->writePos = 0;
343  shm_ptr_->writer_pos = (buffer + 1) % shm_ptr_->buffer_count;
344  TLOG(14) << "GetBufferForWriting returning " << buffer;
345  touchBuffer_(buf);
346  return buffer;
347  }
348  }
349  }
350 
351  TLOG(14) << "GetBufferForWriting Returning -1 because no buffers are ready";
352  return -1;
353 }
354 
356 {
357  if (!IsValid()) return 0;
358  TLOG(23) << "0x" << std::hex << shm_key_ << " ReadReadyCount BEGIN";
359  //std::unique_lock<std::mutex> lk(search_mutex_);
360  TraceLock lk(search_mutex_, 14, "ReadReadyCountSearch");
361  size_t count = 0;
362  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
363  {
364  TLOG(24) << "0x" << std::hex << shm_key_ << std::dec << " ReadReadyCount: Checking if buffer " << ii << " is stale.";
365  ResetBuffer(ii);
366  auto buf = getBufferInfo_(ii);
367  TLOG(25) << "0x" << std::hex << shm_key_ << std::dec << " ReadReadyCount: Buffer " << ii << ": sem=" << FlagToString(buf->sem) << " (expected " << FlagToString(BufferSemaphoreFlags::Full) << "), sem_id=" << buf->sem_id << " )";
368  if (buf->sem == BufferSemaphoreFlags::Full && (buf->sem_id == -1 || buf->sem_id == manager_id_) && buf->sequence_id > last_seen_id_)
369  {
370  TLOG(26) << "0x" << std::hex << shm_key_ << std::dec << " ReadReadyCount: Buffer " << ii << " is either unowned or owned by this manager, and is marked full.";
371  ++count;
372  }
373  }
374  return count;
375 }
376 
378 {
379  if (!IsValid()) return 0;
380  //std::unique_lock<std::mutex> lk(search_mutex_);
381  TraceLock lk(search_mutex_, 15, "WriteReadyCountSearch");
382  size_t count = 0;
383  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
384  {
385  ResetBuffer(ii);
386  auto buf = getBufferInfo_(ii);
387  if ((buf->sem == BufferSemaphoreFlags::Empty && buf->sem_id == -1)
388  || (overwrite && buf->sem != BufferSemaphoreFlags::Writing))
389  {
390  ++count;
391  }
392  }
393  return count;
394 }
395 
397 {
398  //std::unique_lock<std::mutex> lk(search_mutex_);
399  std::deque<int> output;
400  if (!IsValid()) return output;
401  if (locked)
402  {
403  TraceLock lk(search_mutex_, 16, "GetOwnedSearch");
404  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
405  {
406  auto buf = getBufferInfo_(ii);
407  if (buf->sem_id == manager_id_)
408  {
409  output.push_back(ii);
410  }
411  }
412  }
413  else
414  {
415  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
416  {
417  auto buf = getBufferInfo_(ii);
418  if (buf->sem_id == manager_id_)
419  {
420  output.push_back(ii);
421  }
422  }
423  }
424 
425  return output;
426 }
427 
429 {
430  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
431  TraceLock lk(buffer_mutexes_[buffer], 17, "DataSizeBuffer" + std::to_string(buffer));
432  auto buf = getBufferInfo_(buffer);
433  touchBuffer_(buf);
434  return buf->writePos;
435 }
436 
437 
439 {
440  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
441  TraceLock lk(buffer_mutexes_[buffer], 18, "ResetReadPosBuffer" + std::to_string(buffer));
442  auto buf = getBufferInfo_(buffer);
443  touchBuffer_(buf);
444  buf->readPos = 0;
445 }
446 
448 {
449  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
450  TraceLock lk(buffer_mutexes_[buffer], 18, "ResetWritePosBuffer" + std::to_string(buffer));
451  auto buf = getBufferInfo_(buffer);
452  checkBuffer_(buf, BufferSemaphoreFlags::Writing);
453  touchBuffer_(buf);
454  buf->writePos = 0;
455 }
456 
458 {
459  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
460  TraceLock lk(buffer_mutexes_[buffer], 19, "IncReadPosBuffer" + std::to_string(buffer));
461  auto buf = getBufferInfo_(buffer);
462  touchBuffer_(buf);
463  TLOG(15) << "IncrementReadPos: buffer= " << buffer << ", readPos=" << std::to_string(buf->readPos) << ", bytes read=" << std::to_string(read);
464  buf->readPos = buf->readPos + read;
465  TLOG(15) << "IncrementReadPos: buffer= " << buffer << ", New readPos is " << std::to_string(buf->readPos);
466  if (read == 0) Detach(true, "LogicError", "Cannot increment Read pos by 0! (buffer=" + std::to_string(buffer) + ", readPos=" + std::to_string(buf->readPos) + ", writePos=" + std::to_string(buf->writePos) + ")");
467 }
468 
469 void artdaq::SharedMemoryManager::IncrementWritePos(int buffer, size_t written)
470 {
471  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
472  TraceLock lk(buffer_mutexes_[buffer], 20, "IncWritePosBuffer" + std::to_string(buffer));
473  auto buf = getBufferInfo_(buffer);
474  touchBuffer_(buf);
475  TLOG(16) << "IncrementWritePos: buffer= " << buffer << ", writePos=" << std::to_string(buf->writePos) << ", bytes written=" << std::to_string(written);
476  buf->writePos += written;
477  TLOG(16) << "IncrementWritePos: buffer= " << buffer << ", New writePos is " << std::to_string(buf->writePos);
478  if (written == 0) Detach(true, "LogicError", "Cannot increment Write pos by 0!");
479 }
480 
482 {
483  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
484  TraceLock lk(buffer_mutexes_[buffer], 21, "MoreDataInBuffer" + std::to_string(buffer));
485  auto buf = getBufferInfo_(buffer);
486  TLOG(17) << "MoreDataInBuffer: buffer= " << buffer << ", readPos=" << std::to_string(buf->readPos) << ", writePos=" << std::to_string(buf->writePos);
487  return buf->readPos < buf->writePos;
488 }
489 
491 {
492  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
493  TraceLock lk(buffer_mutexes_[buffer], 22, "CheckBuffer" + std::to_string(buffer));
494  return checkBuffer_(getBufferInfo_(buffer), flags, false);
495 }
496 
497 void artdaq::SharedMemoryManager::MarkBufferFull(int buffer, int destination)
498 {
499  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
500  TraceLock lk(buffer_mutexes_[buffer], 23, "FillBuffer" + std::to_string(buffer));
501  auto shmBuf = getBufferInfo_(buffer);
502  touchBuffer_(shmBuf);
503  if (shmBuf->sem_id == manager_id_)
504  {
505  if (shmBuf->sem != BufferSemaphoreFlags::Full)
506  shmBuf->sem = BufferSemaphoreFlags::Full;
507 
508  shmBuf->sem_id = destination;
509  }
510 }
511 
513 {
514  TLOG(18) << "MarkBufferEmpty BEGIN";
515  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
516  TraceLock lk(buffer_mutexes_[buffer], 24, "EmptyBuffer" + std::to_string(buffer));
517  auto shmBuf = getBufferInfo_(buffer);
518  if (!force)
519  {
520  checkBuffer_(shmBuf, BufferSemaphoreFlags::Reading, true);
521  }
522  touchBuffer_(shmBuf);
523 
524  shmBuf->readPos = 0;
525  shmBuf->sem = BufferSemaphoreFlags::Full;
526 
527  if ((force && (manager_id_ == 0 || manager_id_ == shmBuf->sem_id)) || (!force && shm_ptr_->destructive_read_mode))
528  {
529  TLOG(18) << "MarkBufferEmpty Resetting buffer to Empty state";
530  shmBuf->writePos = 0;
531  shmBuf->sem = BufferSemaphoreFlags::Empty;
532  if (shm_ptr_->reader_pos == static_cast<unsigned>(buffer) && !shm_ptr_->destructive_read_mode)
533  {
534  TLOG(18) << "MarkBufferEmpty Broadcast mode; incrementing reader_pos";
535  shm_ptr_->reader_pos = (buffer + 1) % shm_ptr_->buffer_count;
536  }
537  }
538  shmBuf->sem_id = -1;
539  TLOG(18) << "MarkBufferEmpty END";
540 }
541 
543 {
544  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
545  TraceLock lk(buffer_mutexes_[buffer], 25, "ResetBuffer" + std::to_string(buffer));
546  auto shmBuf = getBufferInfo_(buffer);
547  /*
548  if (shmBuf->sequence_id < shm_ptr_->lowest_seq_id_read - size() && shmBuf->sem == BufferSemaphoreFlags::Full)
549  {
550  TLOG(TLVL_DEBUG) << "Buffer " << buffer << " has been passed by all readers, marking Empty" ;
551  shmBuf->writePos = 0;
552  shmBuf->sem = BufferSemaphoreFlags::Empty;
553  shmBuf->sem_id = -1;
554  return true;
555  }*/
556 
557  size_t delta = TimeUtils::gettimeofday_us() - shmBuf->last_touch_time;
558  if (delta > 0xFFFFFFFF)
559  {
560  TLOG(TLVL_TRACE) << "Buffer has touch time in the future, setting it to current time and ignoring...";
561  shmBuf->last_touch_time = TimeUtils::gettimeofday_us();
562  return false;
563  }
564  if (shm_ptr_->buffer_timeout_us == 0 || delta <= shm_ptr_->buffer_timeout_us || shmBuf->sem == BufferSemaphoreFlags::Empty) return false;
565  TLOG(27) << "Buffer " << buffer << " is stale, time=" << TimeUtils::gettimeofday_us() << ", last touch=" << shmBuf->last_touch_time << ", d=" << delta << ", timeout=" << shm_ptr_->buffer_timeout_us;
566 
567  if (shmBuf->sem_id == manager_id_ && shmBuf->sem == BufferSemaphoreFlags::Writing)
568  {
569  return true;
570  }
571  if (!shm_ptr_->destructive_read_mode && shmBuf->sem == BufferSemaphoreFlags::Full)
572  {
573  TLOG(TLVL_DEBUG) << "Resetting old broadcast mode buffer";
574  shmBuf->writePos = 0;
575  shmBuf->sem = BufferSemaphoreFlags::Empty;
576  shmBuf->sem_id = -1;
577  if (shm_ptr_->reader_pos == static_cast<unsigned>(buffer)) shm_ptr_->reader_pos = (buffer + 1) % shm_ptr_->buffer_count;
578  return true;
579  }
580 
581  if (shmBuf->sem_id != manager_id_ && shmBuf->sem == BufferSemaphoreFlags::Reading)
582  {
583  TLOG(TLVL_WARNING) << "Stale Read buffer ( " << delta << " / " << shm_ptr_->buffer_timeout_us << " us ) detected! Resetting...";
584  shmBuf->readPos = 0;
585  shmBuf->sem = BufferSemaphoreFlags::Full;
586  shmBuf->sem_id = -1;
587  return true;
588  }
589  return false;
590 }
591 
593 {
594  if (!IsValid()) return true;
595 
596  struct shmid_ds info;
597  auto sts = shmctl(shm_segment_id_, IPC_STAT, &info);
598  if (sts < 0)
599  {
600  TLOG(TLVL_TRACE) << "Error accessing Shared Memory info: " << errno << " (" << strerror(errno) << ").";
601  return true;
602  }
603 
604  if (info.shm_perm.mode & SHM_DEST)
605  {
606  TLOG(TLVL_INFO) << "Shared Memory marked for destruction. Probably an end-of-data condition!";
607  return true;
608  }
609 
610  return false;
611 }
612 
614 {
615  if (!IsValid()) return 0;
616 
617  struct shmid_ds info;
618  auto sts = shmctl(shm_segment_id_, IPC_STAT, &info);
619  if (sts < 0)
620  {
621  TLOG(TLVL_TRACE) << "Error accessing Shared Memory info: " << errno << " (" << strerror(errno) << ").";
622  return 0;
623  }
624 
625  return info.shm_nattch;
626 }
627 
628 size_t artdaq::SharedMemoryManager::Write(int buffer, void* data, size_t size)
629 {
630  TLOG(19) << "Write BEGIN";
631  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
632  TraceLock lk(buffer_mutexes_[buffer], 26, "WriteBuffer" + std::to_string(buffer));
633  auto shmBuf = getBufferInfo_(buffer);
634  checkBuffer_(shmBuf, BufferSemaphoreFlags::Writing);
635  touchBuffer_(shmBuf);
636  TLOG(19) << "Buffer Write Pos is " << std::to_string(shmBuf->writePos) << ", write size is " << std::to_string(size);
637  if (shmBuf->writePos + size > shm_ptr_->buffer_size) Detach(true, "SharedMemoryWrite", "Attempted to write more data than fits into Shared Memory! \nRe-run with a larger buffer size!");
638 
639  auto pos = GetWritePos(buffer);
640  memcpy(pos, data, size);
641  shmBuf->writePos = shmBuf->writePos + size;
642  if (shmBuf->sequence_id > last_seen_id_)
643  {
644  last_seen_id_ = shmBuf->sequence_id;
645  }
646  TLOG(19) << "Write END";
647  return size;
648 }
649 
650 bool artdaq::SharedMemoryManager::Read(int buffer, void* data, size_t size)
651 {
652  //std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
653  TraceLock lk(buffer_mutexes_[buffer], 27, "ReadBuffer" + std::to_string(buffer));
654  auto shmBuf = getBufferInfo_(buffer);
655  checkBuffer_(shmBuf, BufferSemaphoreFlags::Reading);
656  touchBuffer_(shmBuf);
657  if (shmBuf->readPos + size > shm_ptr_->buffer_size) Detach(true, "SharedMemoryRead", "Attempted to read more data than exists in Shared Memory!");
658 
659  auto pos = GetReadPos(buffer);
660  memcpy(data, pos, size);
661  shmBuf->readPos += size;
662  touchBuffer_(shmBuf);
663  return checkBuffer_(shmBuf, BufferSemaphoreFlags::Reading, false);
664 }
665 
667 {
668  std::ostringstream ostr;
669  ostr << "ShmStruct: " << std::endl
670  << "Reader Position: " << shm_ptr_->reader_pos << std::endl
671  << "Writer Position: " << shm_ptr_->writer_pos << std::endl
672  << "Next ID Number: " << shm_ptr_->next_id << std::endl
673  << "Buffer Count: " << shm_ptr_->buffer_count << std::endl
674  << "Buffer Size: " << std::to_string(shm_ptr_->buffer_size) << " bytes" << std::endl
675  << "Buffers Written: " << std::to_string(shm_ptr_->next_sequence_id) << std::endl
676  << "Rank of Writer: " << shm_ptr_->rank << std::endl
677  << "Ready Magic Bytes: 0x" << std::hex << shm_ptr_->ready_magic << std::endl << std::endl;
678 
679  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
680  {
681  auto buf = getBufferInfo_(ii);
682  ostr << "ShmBuffer " << std::dec << ii << std::endl
683  << "sequenceID: " << std::to_string(buf->sequence_id) << std::endl
684  << "writePos: " << std::to_string(buf->writePos) << std::endl
685  << "readPos: " << std::to_string(buf->readPos) << std::endl
686  << "sem: " << FlagToString(buf->sem) << std::endl
687  << "Owner: " << std::to_string(buf->sem_id.load()) << std::endl
688  << "Last Touch Time: " << std::to_string(buf->last_touch_time / 1000000.0) << std::endl << std::endl;
689  }
690 
691  return ostr.str();
692 }
693 
695 {
696  auto buf = getBufferInfo_(buffer);
697  return bufferStart_(buffer) + buf->readPos;
698 }
700 {
701  auto buf = getBufferInfo_(buffer);
702  return bufferStart_(buffer) + buf->writePos;
703 }
704 
706 {
707  return bufferStart_(buffer);
708 }
709 
710 uint8_t* artdaq::SharedMemoryManager::dataStart_() const
711 {
712  return reinterpret_cast<uint8_t*>(shm_ptr_ + 1) + shm_ptr_->buffer_count * sizeof(ShmBuffer);
713 }
714 
715 uint8_t* artdaq::SharedMemoryManager::bufferStart_(int buffer)
716 {
717  if (buffer >= shm_ptr_->buffer_count) Detach(true, "ArgumentOutOfRange", "The specified buffer does not exist!");
718  return dataStart_() + buffer * shm_ptr_->buffer_size;
719 }
720 
721 artdaq::SharedMemoryManager::ShmBuffer* artdaq::SharedMemoryManager::getBufferInfo_(int buffer)
722 {
723  if (buffer >= shm_ptr_->buffer_count) Detach(true, "ArgumentOutOfRange", "The specified buffer does not exist!");
724  return reinterpret_cast<ShmBuffer*>(reinterpret_cast<uint8_t*>(shm_ptr_ + 1) + buffer * sizeof(ShmBuffer));
725 }
726 
727 bool artdaq::SharedMemoryManager::checkBuffer_(ShmBuffer* buffer, BufferSemaphoreFlags flags, bool exceptions)
728 {
729  TLOG(TLVL_TRACE) << "checkBuffer_: Checking that buffer " << buffer->sequence_id << " has sem_id " << manager_id_ << " (Current: " << buffer->sem_id << ") and is in state " << FlagToString(flags) << " (current: " << FlagToString(buffer->sem) << ")";
730  if (exceptions)
731  {
732  if (buffer->sem != flags) Detach(true, "StateAccessViolation", "Shared Memory buffer is not in the correct state! (expected " + FlagToString(flags) + ", actual " + FlagToString(buffer->sem) + ")");
733  if (buffer->sem_id != manager_id_) Detach(true, "OwnerAccessViolation", "Shared Memory buffer is not owned by this manager instance! (Expected: " + std::to_string(manager_id_) + ", Actual: " + std::to_string(buffer->sem_id) + ")");
734  }
735  bool ret = (buffer->sem_id == manager_id_ || (buffer->sem_id == -1 && (flags == BufferSemaphoreFlags::Full || flags == BufferSemaphoreFlags::Empty))) && buffer->sem == flags;
736 
737  if (!ret)
738  {
739  TLOG(TLVL_WARNING) << "CheckBuffer detected issue with buffer " << std::to_string(buffer->sequence_id) << "!"
740  << " ID: " << buffer->sem_id << " (" << manager_id_ << "), Flag: " << FlagToString(buffer->sem) << " (" << FlagToString(flags) << "). "
741  << "ID -1 is okay if desired flag is \"Full\" or \"Empty\".";
742  }
743 
744  return ret;
745 }
746 
747 void artdaq::SharedMemoryManager::touchBuffer_(ShmBuffer* buffer)
748 {
749  if (buffer->sem_id != manager_id_) return;
750  TLOG(TLVL_TRACE) << "touchBuffer_: Touching buffer with sequence_id " << std::to_string(buffer->sequence_id);
751  buffer->last_touch_time = TimeUtils::gettimeofday_us();
752 }
753 
754 void artdaq::SharedMemoryManager::Detach(bool throwException, std::string category, std::string message, bool force)
755 {
756  TLOG(TLVL_DETACH) << "Detach BEGIN: throwException: " << std::boolalpha << throwException << ", force: " << force;
757  if (IsValid())
758  {
759  TLOG(TLVL_DETACH) << "Detach: Resetting owned buffers";
760  auto bufs = GetBuffersOwnedByManager(false);
761  for (auto buf : bufs)
762  {
763  auto shmBuf = getBufferInfo_(buf);
764  if (shmBuf->sem == BufferSemaphoreFlags::Writing)
765  {
766  shmBuf->sem = BufferSemaphoreFlags::Empty;
767  }
768  else if (shmBuf->sem == BufferSemaphoreFlags::Reading)
769  {
770  shmBuf->sem = BufferSemaphoreFlags::Full;
771  }
772  shmBuf->sem_id = -1;
773  }
774  }
775 
776  if (shm_ptr_)
777  {
778  TLOG(TLVL_DETACH) << "Detach: Detaching shared memory";
779  shmdt(shm_ptr_);
780  shm_ptr_ = NULL;
781  }
782 
783  if ((force || manager_id_ == 0) && shm_segment_id_ > -1)
784  {
785  TLOG(TLVL_DETACH) << "Detach: Marking Shared memory for removal";
786  shmctl(shm_segment_id_, IPC_RMID, NULL);
787  shm_segment_id_ = -1;
788  }
789 
790  if (category.size() > 0 && message.size() > 0)
791  {
792  TLOG(TLVL_ERROR) << category << ": " << message;
793 
794  if (throwException)
795  {
796  throw cet::exception(category) << message;
797  }
798  }
799 }
800 
801 
802 
803 // Local Variables:
804 // mode: c++
805 // End:
bool Read(int buffer, void *data, size_t size)
Read size bytes of data from buffer into the given pointer.
void IncrementReadPos(int buffer, size_t read)
Increment the read position for a given buffer.
constexpr size_t GetElapsedTimeMilliseconds(std::chrono::steady_clock::time_point then, std::chrono::steady_clock::time_point now=std::chrono::steady_clock::now())
Gets the number of milliseconds in the given time interval
Definition: TimeUtils.hh:54
std::deque< int > GetBuffersOwnedByManager(bool locked=true)
Get the list of all buffers currently owned by this manager instance.
void ResetReadPos(int buffer)
Set the read position of the given buffer to the beginning of the buffer.
bool MoreDataInBuffer(int buffer)
Determine if more data is available to be read, based on the read position and data size...
virtual std::string toString()
Write information about the SharedMemory to a string.
BufferSemaphoreFlags
The BufferSemaphoreFlags enumeration represents the different possible &quot;states&quot; of a given shared mem...
size_t BufferDataSize(int buffer)
Get the current size of the buffer&#39;s data.
void Attach()
Reconnect to the shared memory segment.
void Detach(bool throwException=false, std::string category="", std::string message="", bool force=false)
Detach from the Shared Memory segment, optionally throwing a cet::exception with the specified proper...
virtual ~SharedMemoryManager() noexcept
SharedMemoryManager Destructor.
The SharedMemoryManager creates a Shared Memory area which is divided into a number of fixed-size buf...
size_t ReadReadyCount()
Count the number of buffers that are ready for reading.
bool ResetBuffer(int buffer)
Resets the buffer from Reading to Full. This operation will only have an effect if performed by the o...
bool IsEndOfData() const
Determine whether the Shared Memory is marked for destruction (End of Data)
bool CheckBuffer(int buffer, BufferSemaphoreFlags flags)
Check both semaphore conditions (Mode flag and manager ID) for a given buffer.
void IncrementWritePos(int buffer, size_t written)
Increment the write position for a given buffer.
The TraceLock class allows a user to debug the acquisition and releasing of locks, by wrapping the unique_lock&lt;std::mutex&gt; API with TRACE calls.
Definition: TraceLock.hh:10
void MarkBufferFull(int buffer, int destination=-1)
Release a buffer from a writer, marking it Full and ready for a reader.
void MarkBufferEmpty(int buffer, bool force=false)
Release a buffer from a reader, marking it Empty and ready to accept more data.
void ResetWritePos(int buffer)
Set the write position of the given buffer to the beginning of the buffer.
void * GetWritePos(int buffer)
Get a pointer to the current write position of the buffer.
size_t Write(int buffer, void *data, size_t size)
Write size bytes of data from the given pointer to a buffer.
void * GetReadPos(int buffer)
Get a pointer to the current read position of the buffer.
void * GetBufferStart(int buffer)
Get a pointer to the start position of the buffer.
SharedMemoryManager(uint32_t shm_key, size_t buffer_count=0, size_t buffer_size=0, uint64_t buffer_timeout_us=100 *1000000, bool destructive_read_mode=true)
SharedMemoryManager Constructor.
int GetBufferForWriting(bool overwrite)
Finds a buffer that is ready to be written to, and reserves it for the calling manager.
uint16_t GetAttachedCount() const
Get the number of attached SharedMemoryManagers.
size_t WriteReadyCount(bool overwrite)
Count the number of buffers that are ready for writing.
uint64_t gettimeofday_us()
Get the current time of day in microseconds (from gettimeofday system call)
Definition: TimeUtils.cc:51
int GetBufferForReading()
Finds a buffer that is ready to be read, and reserves it for the calling manager. ...