artdaq_core  v3_04_09
SharedMemoryManager.cc
1 #define TRACE_NAME "SharedMemoryManager"
2 #include <cstring>
3 #include <unordered_map>
4 #include <set>
5 #include <sys/ipc.h>
6 #include <sys/shm.h>
7 #ifndef SHM_DEST // Lynn reports that this is missing on Mac OS X?!?
8 #define SHM_DEST 01000
9 #endif
10 #include "tracemf.h"
11 #include <signal.h>
12 #include "cetlib_except/exception.h"
13 #include "artdaq-core/Core/SharedMemoryManager.hh"
14 #include "artdaq-core/Utilities/TraceLock.hh"
15 
16 #define TLVL_DETACH 11
17 #define TLVL_BUFFER 40
18 #define TLVL_BUFLCK 41
19 
20 static std::set<artdaq::SharedMemoryManager const*> instances = std::set<artdaq::SharedMemoryManager const*>();
21 
22 static std::unordered_map<int, struct sigaction> old_actions = std::unordered_map<int, struct sigaction>();
23 static bool sighandler_init = false;
24 static void signal_handler(int signum)
25 {
26  // Messagefacility may already be gone at this point, TRACE ONLY!
27  TRACE_STREAMER(TLVL_ERROR, &("SharedMemoryManager")[0], 0, 0, 0) << "A signal of type " << signum << " was caught by SharedMemoryManager. Detaching all Shared Memory segments, then proceeding with default handlers!";
28  for (auto ii : instances)
29  {
30  if (ii)
31  {
32  const_cast<artdaq::SharedMemoryManager*>(ii)->Detach(false, "", ""
33  , false /* don't force destruct segment, allows reconnection (applicable for
34  restart and/or multiple art processes (i.e. dispatcher)) */
35  );
36  }
37  ii = nullptr;
38  }
39 
40  sigset_t set;
41  pthread_sigmask(SIG_UNBLOCK, NULL, &set);
42  pthread_sigmask(SIG_UNBLOCK, &set, NULL);
43 
44  TRACE_STREAMER(TLVL_ERROR, &("SharedMemoryManager")[0], 0, 0, 0) << "Calling default signal handler";
45  if (signum != SIGUSR2)
46  {
47  sigaction(signum, &old_actions[signum], NULL);
48  kill(getpid(), signum); // Only send signal to self
49  }
50  else
51  {
52  // Send Interrupt signal if parsing SIGUSR2 (i.e. user-defined exception that should tear down ARTDAQ)
53  sigaction(SIGINT, &old_actions[SIGINT], NULL);
54  kill(getpid(), SIGINT); // Only send signal to self
55  }
56 }
57 
58 artdaq::SharedMemoryManager::SharedMemoryManager(uint32_t shm_key, size_t buffer_count, size_t buffer_size, uint64_t buffer_timeout_us, bool destructive_read_mode)
59  : shm_segment_id_(-1)
60  , shm_ptr_(NULL)
61  , shm_key_(shm_key)
62  , manager_id_(-1)
63  , buffer_mutexes_()
64  , last_seen_id_(0)
65 {
66  requested_shm_parameters_.buffer_count = buffer_count;
67  requested_shm_parameters_.buffer_size = buffer_size;
68  requested_shm_parameters_.buffer_timeout_us = buffer_timeout_us;
69  requested_shm_parameters_.destructive_read_mode = destructive_read_mode;
70 
71  instances.insert(this);
72  Attach();
73 
74  static std::mutex sighandler_mutex;
75  std::unique_lock<std::mutex> lk(sighandler_mutex);
76 
77  if (!sighandler_init)//&& manager_id_ == 0) // ELF 3/22/18: Taking out manager_id_==0 requirement as I think kill(getpid()) is enough protection
78  {
79  sighandler_init = true;
80  std::vector<int> signals = { SIGINT, SIGILL, SIGABRT, SIGFPE, SIGSEGV, SIGPIPE, SIGALRM, SIGTERM, SIGUSR2 }; // SIGQUIT is used by art in normal operation
81  for (auto signal : signals)
82  {
83  struct sigaction old_action;
84  sigaction(signal, NULL, &old_action);
85 
86  //If the old handler wasn't SIG_IGN (it's a handler that just
87  // "ignore" the signal)
88  if (old_action.sa_handler != SIG_IGN)
89  {
90  struct sigaction action;
91  action.sa_handler = signal_handler;
92  sigemptyset(&action.sa_mask);
93  for (auto sigblk : signals)
94  {
95  sigaddset(&action.sa_mask, sigblk);
96  }
97  action.sa_flags = 0;
98 
99  //Replace the signal handler of SIGINT with the one described by new_action
100  sigaction(signal, &action, NULL);
101  old_actions[signal] = old_action;
102  }
103  }
104  }
105 }
106 
108 {
109  instances.erase(this);
110  TLOG(TLVL_DEBUG) << "~SharedMemoryManager called";
111  Detach();
112  TLOG(TLVL_DEBUG) << "~SharedMemoryManager done";
113 }
114 
116 {
117  if (IsValid())
118  {
119  if (manager_id_ == 0) return;
120  Detach();
121  }
122  auto start_time = std::chrono::steady_clock::now();
123  last_seen_id_ = 0;
124  size_t shmSize = requested_shm_parameters_.buffer_count * (requested_shm_parameters_.buffer_size + sizeof(ShmBuffer)) + sizeof(ShmStruct);
125 
126  shm_segment_id_ = shmget(shm_key_, shmSize, 0666);
127  if (shm_segment_id_ == -1 && requested_shm_parameters_.buffer_count > 0 && manager_id_ <= 0)
128  {
129  TLOG(TLVL_DEBUG) << "Creating shared memory segment with key 0x" << std::hex << shm_key_ << " and size " << std::dec << shmSize;
130  shm_segment_id_ = shmget(shm_key_, shmSize, IPC_CREAT | 0666);
131  manager_id_ = 0;
132 
133  if (shm_segment_id_ == -1)
134  {
135  TLOG(TLVL_ERROR) << "Error creating shared memory segment with key 0x" << std::hex << shm_key_ << ", errno=" << errno << " (" << strerror(errno) << ")";
136  }
137  }
138  else
139  {
140  while (shm_segment_id_ == -1 && TimeUtils::GetElapsedTimeMilliseconds(start_time) < 1000)
141  {
142  shm_segment_id_ = shmget(shm_key_, shmSize, 0666);
143 
144  }
145  }
146  TLOG(TLVL_DEBUG) << "shm_key == 0x" << std::hex << shm_key_ << ", shm_segment_id == " << shm_segment_id_;
147 
148  if (shm_segment_id_ > -1)
149  {
150  TLOG(TLVL_DEBUG)
151  << "Attached to shared memory segment with ID = " << shm_segment_id_
152  << " and size " << shmSize
153  << " bytes";
154  shm_ptr_ = (ShmStruct*)shmat(shm_segment_id_, 0, 0);
155  TLOG(TLVL_DEBUG)
156  << "Attached to shared memory segment at address "
157  << std::hex << (void*)shm_ptr_ << std::dec;
158  if (shm_ptr_ && shm_ptr_ != (void *)-1)
159  {
160  if (manager_id_ == 0)
161  {
162  if (shm_ptr_->ready_magic == 0xCAFE1111)
163  {
164  TLOG(TLVL_ERROR) << "Owner encountered already-initialized Shared Memory!";
165  exit(-2);
166  }
167  TLOG(TLVL_DEBUG) << "Owner initializing Shared Memory";
168  shm_ptr_->next_id = 1;
169  shm_ptr_->next_sequence_id = 0;
170  shm_ptr_->reader_pos = 0;
171  shm_ptr_->writer_pos = 0;
172  shm_ptr_->buffer_size = requested_shm_parameters_.buffer_size;
173  shm_ptr_->buffer_count = requested_shm_parameters_.buffer_count;
174  shm_ptr_->buffer_timeout_us = requested_shm_parameters_.buffer_timeout_us;
175  shm_ptr_->destructive_read_mode = requested_shm_parameters_.destructive_read_mode;
176 
177  for (int ii = 0; ii < static_cast<int>(requested_shm_parameters_.buffer_count); ++ii)
178  {
179  if (!getBufferInfo_(ii)) return;
180  getBufferInfo_(ii)->writePos = 0;
181  getBufferInfo_(ii)->readPos = 0;
182  getBufferInfo_(ii)->sem = BufferSemaphoreFlags::Empty;
183  getBufferInfo_(ii)->sem_id = -1;
184  getBufferInfo_(ii)->last_touch_time = TimeUtils::gettimeofday_us();
185  }
186 
187  shm_ptr_->ready_magic = 0xCAFE1111;
188  }
189  else
190  {
191  TLOG(TLVL_DEBUG) << "Waiting for owner to initalize Shared Memory";
192  while (shm_ptr_->ready_magic != 0xCAFE1111) { usleep(1000); }
193  TLOG(TLVL_DEBUG) << "Getting ID from Shared Memory";
194  GetNewId();
195  shm_ptr_->lowest_seq_id_read = 0;
196  TLOG(TLVL_DEBUG) << "Getting Shared Memory Size parameters";
197  }
198  //last_seen_id_ = shm_ptr_->next_sequence_id;
199  buffer_mutexes_ = std::vector<std::mutex>(shm_ptr_->buffer_count);
200  TLOG(TLVL_DEBUG) << "Initialization Complete: "
201  << "key: 0x" << std::hex << shm_key_
202  << ", manager ID: " << std::dec << manager_id_
203  << ", Buffer size: " << shm_ptr_->buffer_size
204  << ", Buffer count: " << shm_ptr_->buffer_count;
205  return;
206  }
207  else
208  {
209  TLOG(TLVL_ERROR) << "Failed to attach to shared memory segment "
210  << shm_segment_id_;
211  }
212  }
213  else
214  {
215  TLOG(TLVL_ERROR) << "Failed to connect to shared memory segment with key 0x" << std::hex << shm_key_
216  << ", errno=" << errno << " (" << strerror(errno) << ")" << ". Please check "
217  << "if a stale shared memory segment needs to "
218  << "be cleaned up. (ipcs, ipcrm -m <segId>)";
219  }
220  return;
221 }
222 
224 {
225  TLOG(13) << "GetBufferForReading BEGIN";
226 
227  std::unique_lock<std::mutex> lk(search_mutex_);
228  //TraceLock lk(search_mutex_, 11, "GetBufferForReadingSearch");
229  auto rp = shm_ptr_->reader_pos.load();
230 
231  TLOG(13) << "GetBufferForReading lock acquired, scanning " << shm_ptr_->buffer_count << " buffers";
232 
233  for (int retry = 0; retry < 5; retry++)
234  {
236  short sem_id;
237  int buffer_num = -1;
238  ShmBuffer* buffer_ptr = nullptr;
239  uint64_t seqID = -1;
240 
241  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
242  {
243  auto buffer = (ii + rp) % shm_ptr_->buffer_count;
244 
245 
246  TLOG(14) << "GetBufferForReading Checking if buffer " << buffer << " is stale. Shm destructive_read_mode=" << shm_ptr_->destructive_read_mode;
247  ResetBuffer(buffer);
248 
249  auto buf = getBufferInfo_(buffer);
250  if (!buf) continue;
251 
252  sem = buf->sem.load();
253  sem_id = buf->sem_id.load();
254 
255  TLOG(14) << "GetBufferForReading: Buffer " << buffer << ": sem=" << FlagToString(sem)
256  << " (expected " << FlagToString(BufferSemaphoreFlags::Full) << "), sem_id=" << sem_id << ", seq_id=" << buf->sequence_id << " )";
257  if (sem == BufferSemaphoreFlags::Full && (sem_id == -1 || sem_id == manager_id_)
258  && (shm_ptr_->destructive_read_mode || buf->sequence_id > last_seen_id_))
259  {
260  if (buf->sequence_id < seqID)
261  {
262  buffer_ptr = buf;
263  seqID = buf->sequence_id;
264  buffer_num = buffer;
265  touchBuffer_(buf);
266  if (shm_ptr_->destructive_read_mode || seqID == last_seen_id_ + 1) break;
267  }
268  }
269  }
270 
271  if (buffer_ptr)
272  {
273  sem = buffer_ptr->sem.load();
274  sem_id = buffer_ptr->sem_id.load();
275  }
276 
277  if (!buffer_ptr || (sem_id != -1 && sem_id != manager_id_) || sem != BufferSemaphoreFlags::Full)
278  {
279  continue;
280  }
281 
282  if (buffer_num >= 0)
283  {
284  TLOG(13) << "GetBufferForReading Found buffer " << buffer_num;
285  touchBuffer_(buffer_ptr);
286  if (!buffer_ptr->sem_id.compare_exchange_strong(sem_id, manager_id_)) continue;
287  if (!buffer_ptr->sem.compare_exchange_strong(sem, BufferSemaphoreFlags::Reading)) continue;
288  if (!checkBuffer_(buffer_ptr, BufferSemaphoreFlags::Reading, false)) {
289  TLOG(13) << "GetBufferForReading: Failed to acquire buffer " << buffer_num << " (someone else changed manager ID while I was changing sem)";
290  continue;
291  }
292  buffer_ptr->readPos = 0;
293  touchBuffer_(buffer_ptr);
294  if (!checkBuffer_(buffer_ptr, BufferSemaphoreFlags::Reading, false)) {
295  TLOG(13) << "GetBufferForReading: Failed to acquire buffer " << buffer_num << " (someone else changed manager ID while I was touching buffer SHOULD NOT HAPPEN!)";
296  continue;
297  }
298  if (shm_ptr_->destructive_read_mode && shm_ptr_->lowest_seq_id_read == last_seen_id_)
299  {
300  shm_ptr_->lowest_seq_id_read = seqID;
301  }
302  last_seen_id_ = seqID;
303  if (shm_ptr_->destructive_read_mode) shm_ptr_->reader_pos = (buffer_num + 1) % shm_ptr_->buffer_count;
304 
305  TLOG(13) << "GetBufferForReading returning " << buffer_num;
306  return buffer_num;
307  }
308  retry = 5;
309  }
310 
311  TLOG(13) << "GetBufferForReading returning -1 because no buffers are ready";
312  return -1;
313 }
314 
316 {
317  TLOG(14) << "GetBufferForWriting BEGIN, overwrite=" << (overwrite ? "true" : "false");
318 
319  std::unique_lock<std::mutex> lk(search_mutex_);
320  //TraceLock lk(search_mutex_, 12, "GetBufferForWritingSearch");
321  auto wp = shm_ptr_->writer_pos.load();
322 
323  TLOG(13) << "GetBufferForWriting lock acquired, scanning " << shm_ptr_->buffer_count << " buffers";
324 
325  // First, only look for "Empty" buffers
326  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
327  {
328  auto buffer = (ii + wp) % shm_ptr_->buffer_count;
329 
330  ResetBuffer(buffer);
331 
332  auto buf = getBufferInfo_(buffer);
333  if (!buf) continue;
334 
335  auto sem = buf->sem.load();
336  auto sem_id = buf->sem_id.load();
337 
338  if (sem == BufferSemaphoreFlags::Empty && sem_id == -1)
339  {
340  touchBuffer_(buf);
341  if (!buf->sem_id.compare_exchange_strong(sem_id, manager_id_)) continue;
342  if (!buf->sem.compare_exchange_strong(sem, BufferSemaphoreFlags::Writing)) continue;
343  if (!checkBuffer_(buf, BufferSemaphoreFlags::Writing, false)) continue;
344  shm_ptr_->writer_pos = (buffer + 1) % shm_ptr_->buffer_count;
345  buf->sequence_id = ++shm_ptr_->next_sequence_id;
346  buf->writePos = 0;
347  if (!checkBuffer_(buf, BufferSemaphoreFlags::Writing, false)) continue;
348  touchBuffer_(buf);
349  TLOG(14) << "GetBufferForWriting returning " << buffer;
350  return buffer;
351  }
352  }
353 
354  if (overwrite)
355  {
356  // Then, look for "Full" buffers
357  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
358  {
359  auto buffer = (ii + wp) % shm_ptr_->buffer_count;
360 
361  ResetBuffer(buffer);
362 
363  auto buf = getBufferInfo_(buffer);
364  if (!buf) continue;
365 
366  auto sem = buf->sem.load();
367  auto sem_id = buf->sem_id.load();
368 
369  if (sem == BufferSemaphoreFlags::Full)
370  {
371  touchBuffer_(buf);
372  if (!buf->sem_id.compare_exchange_strong(sem_id, manager_id_)) continue;
373  if (!buf->sem.compare_exchange_strong(sem, BufferSemaphoreFlags::Writing)) continue;
374  if (!checkBuffer_(buf, BufferSemaphoreFlags::Writing, false)) continue;
375  shm_ptr_->writer_pos = (buffer + 1) % shm_ptr_->buffer_count;
376  buf->sequence_id = ++shm_ptr_->next_sequence_id;
377  buf->writePos = 0;
378  if (!checkBuffer_(buf, BufferSemaphoreFlags::Writing, false)) continue;
379  touchBuffer_(buf);
380  TLOG(14) << "GetBufferForWriting returning " << buffer;
381  return buffer;
382  }
383  }
384 
385  // Finally, if we still haven't found a buffer, we have to clobber a reader...
386  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
387  {
388  auto buffer = (ii + wp) % shm_ptr_->buffer_count;
389 
390  ResetBuffer(buffer);
391 
392  auto buf = getBufferInfo_(buffer);
393  if (!buf) continue;
394 
395  auto sem = buf->sem.load();
396  auto sem_id = buf->sem_id.load();
397 
398  if (sem == BufferSemaphoreFlags::Reading)
399  {
400  touchBuffer_(buf);
401  if (!buf->sem_id.compare_exchange_strong(sem_id, manager_id_)) continue;
402  if (!buf->sem.compare_exchange_strong(sem, BufferSemaphoreFlags::Writing)) continue;
403  if (!checkBuffer_(buf, BufferSemaphoreFlags::Writing, false)) continue;
404  shm_ptr_->writer_pos = (buffer + 1) % shm_ptr_->buffer_count;
405  buf->sequence_id = ++shm_ptr_->next_sequence_id;
406  buf->writePos = 0;
407  if (!checkBuffer_(buf, BufferSemaphoreFlags::Writing, false)) continue;
408  touchBuffer_(buf);
409  TLOG(14) << "GetBufferForWriting returning " << buffer;
410  return buffer;
411  }
412  }
413  }
414  TLOG(14) << "GetBufferForWriting Returning -1 because no buffers are ready";
415  return -1;
416 }
417 
419 {
420  if (!IsValid()) return 0;
421  TLOG(23) << "0x" << std::hex << shm_key_ << " ReadReadyCount BEGIN";
422  std::unique_lock<std::mutex> lk(search_mutex_);
423  TLOG(23) << "ReadReadyCount lock acquired, scanning " << shm_ptr_->buffer_count << " buffers";
424  //TraceLock lk(search_mutex_, 14, "ReadReadyCountSearch");
425  size_t count = 0;
426  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
427  {
428  TLOG(24) << "0x" << std::hex << shm_key_ << std::dec << " ReadReadyCount: Checking if buffer " << ii << " is stale.";
429  ResetBuffer(ii);
430  auto buf = getBufferInfo_(ii);
431  if (!buf) continue;
432  TLOG(25) << "0x" << std::hex << shm_key_ << std::dec << " ReadReadyCount: Buffer " << ii << ": sem=" << FlagToString(buf->sem) << " (expected " << FlagToString(BufferSemaphoreFlags::Full) << "), sem_id=" << buf->sem_id << " )";
433  if (buf->sem == BufferSemaphoreFlags::Full && (buf->sem_id == -1 || buf->sem_id == manager_id_) && (shm_ptr_->destructive_read_mode || buf->sequence_id > last_seen_id_))
434  {
435  TLOG(26) << "0x" << std::hex << shm_key_ << std::dec << " ReadReadyCount: Buffer " << ii << " is either unowned or owned by this manager, and is marked full.";
436  touchBuffer_(buf);
437  ++count;
438  }
439  }
440  return count;
441 }
442 
444 {
445  if (!IsValid()) return 0;
446  TLOG(28) << "0x" << std::hex << shm_key_ << " ReadReadyCount BEGIN";
447  std::unique_lock<std::mutex> lk(search_mutex_);
448  //TraceLock lk(search_mutex_, 15, "WriteReadyCountSearch");
449  TLOG(28) << "WriteReadyCount(" << overwrite << ") lock acquired, scanning " << shm_ptr_->buffer_count << " buffers";
450  size_t count = 0;
451  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
452  {
453  TLOG(29) << "0x" << std::hex << shm_key_ << std::dec << " WriteReadyCount: Checking if buffer " << ii << " is stale.";
454  ResetBuffer(ii);
455  auto buf = getBufferInfo_(ii);
456  if (!buf) continue;
457  if ((buf->sem == BufferSemaphoreFlags::Empty && buf->sem_id == -1)
458  || (overwrite && buf->sem != BufferSemaphoreFlags::Writing))
459  {
460  TLOG(29) << "0x" << std::hex << shm_key_ << std::dec << " WriteReadyCount: Buffer " << ii << " is either empty or is available for overwrite.";
461  ++count;
462  }
463  }
464  return count;
465 }
466 
468 {
469  if (!IsValid()) return false;
470  TLOG(23) << "0x" << std::hex << shm_key_ << " ReadyForRead BEGIN";
471  std::unique_lock<std::mutex> lk(search_mutex_);
472  //TraceLock lk(search_mutex_, 14, "ReadyForReadSearch");
473 
474  auto rp = shm_ptr_->reader_pos.load();
475 
476  TLOG(23) << "ReadyForRead lock acquired, scanning " << shm_ptr_->buffer_count << " buffers";
477 
478  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
479  {
480  auto buffer = (rp + ii) % shm_ptr_->buffer_count;
481  TLOG(24) << "0x" << std::hex << shm_key_ << std::dec << " ReadyForRead: Checking if buffer " << buffer << " is stale.";
482  ResetBuffer(buffer);
483  auto buf = getBufferInfo_(buffer);
484  if (!buf) continue;
485  TLOG(25) << "0x" << std::hex << shm_key_ << std::dec << " ReadyForRead: Buffer " << buffer << ": sem=" << FlagToString(buf->sem) << " (expected " << FlagToString(BufferSemaphoreFlags::Full) << "), sem_id=" << buf->sem_id << " )" << " seq_id=" << buf->sequence_id << " >? " << last_seen_id_;
486  if (buf->sem == BufferSemaphoreFlags::Full && (buf->sem_id == -1 || buf->sem_id == manager_id_) && (shm_ptr_->destructive_read_mode || buf->sequence_id > last_seen_id_))
487  {
488  TLOG(26) << "0x" << std::hex << shm_key_ << std::dec << " ReadyForRead: Buffer " << buffer << " is either unowned or owned by this manager, and is marked full.";
489  touchBuffer_(buf);
490  return true;
491  }
492  }
493  return false;
494 }
495 
497 {
498  if (!IsValid()) return false;
499  TLOG(28) << "0x" << std::hex << shm_key_ << " ReadyForWrite BEGIN";
500 
501  std::unique_lock<std::mutex> lk(search_mutex_);
502  //TraceLock lk(search_mutex_, 15, "ReadyForWriteSearch");
503 
504  auto wp = shm_ptr_->writer_pos.load();
505 
506  TLOG(28) << "ReadyForWrite lock acquired, scanning " << shm_ptr_->buffer_count << " buffers";
507 
508  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
509  {
510  auto buffer = (wp + ii) % shm_ptr_->buffer_count;
511  TLOG(29) << "0x" << std::hex << shm_key_ << std::dec << " ReadyForWrite: Checking if buffer " << buffer << " is stale.";
512  ResetBuffer(buffer);
513  auto buf = getBufferInfo_(buffer);
514  if (!buf) continue;
515  if ((buf->sem == BufferSemaphoreFlags::Empty && buf->sem_id == -1)
516  || (overwrite && buf->sem != BufferSemaphoreFlags::Writing))
517  {
518  TLOG(29) << "0x" << std::hex << shm_key_
519  << std::dec
520  << " WriteReadyCount: Buffer " << ii << " is either empty or available for overwrite.";
521  return true;
522  }
523  }
524  return false;
525 }
526 
528 {
529  std::deque<int> output;
530  if (!IsValid()) return output;
531  TLOG(TLVL_BUFFER) << "GetBuffersOwnedByManager BEGIN. Locked? " << locked;
532  if (locked)
533  {
534  TLOG(TLVL_BUFLCK) << "GetBuffersOwnedByManager obtaining search_mutex";
535  std::unique_lock<std::mutex> lk(search_mutex_);
536  TLOG(TLVL_BUFLCK) << "GetBuffersOwnedByManager obtained search_mutex";
537  //TraceLock lk(search_mutex_, 16, "GetOwnedSearch");
538  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
539  {
540  auto buf = getBufferInfo_(ii);
541  if (!buf) continue;
542  if (buf->sem_id == manager_id_)
543  {
544  output.push_back(ii);
545  }
546  }
547  }
548  else
549  {
550  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
551  {
552  auto buf = getBufferInfo_(ii);
553  if (!buf) continue;
554  if (buf->sem_id == manager_id_)
555  {
556  output.push_back(ii);
557  }
558  }
559  }
560 
561  TLOG(TLVL_BUFFER) << "GetBuffersOwnedByManager: own " << output.size() << " / " << shm_ptr_->buffer_count << " buffers.";
562  return output;
563 }
564 
566 {
567  TLOG(TLVL_BUFFER) << "BufferDataSize(" << buffer << ") called.";
568 
569  if (buffer >= shm_ptr_->buffer_count) Detach(true, "ArgumentOutOfRange", "The specified buffer does not exist!");
570 
571  TLOG(TLVL_BUFLCK) << "BufferDataSize obtaining buffer_mutex for buffer " << buffer;
572  std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
573  TLOG(TLVL_BUFLCK) << "BufferDataSize obtained buffer_mutex for buffer " << buffer;
574  //TraceLock lk(buffer_mutexes_[buffer], 17, "DataSizeBuffer" + std::to_string(buffer));
575 
576  auto buf = getBufferInfo_(buffer);
577  if (!buf) return 0;
578  touchBuffer_(buf);
579 
580  TLOG(TLVL_BUFFER) << "BufferDataSize: buffer " << buffer << ", size=" << buf->writePos;
581  return buf->writePos;
582 }
583 
585 {
586  TLOG(15) << "ResetReadPos(" << buffer << ") called.";
587 
588  if (buffer >= shm_ptr_->buffer_count) Detach(true, "ArgumentOutOfRange", "The specified buffer does not exist!");
589 
590  TLOG(TLVL_BUFLCK) << "ResetReadPos obtaining buffer_mutex for buffer " << buffer;
591  std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
592  TLOG(TLVL_BUFLCK) << "ResetReadPos obtained buffer_mutex for buffer " << buffer;
593 
594  //TraceLock lk(buffer_mutexes_[buffer], 18, "ResetReadPosBuffer" + std::to_string(buffer));
595  auto buf = getBufferInfo_(buffer);
596  if (!buf || buf->sem_id != manager_id_) return;
597  touchBuffer_(buf);
598  buf->readPos = 0;
599 
600  TLOG(15) << "ResetReadPos(" << buffer << ") ended.";
601 }
602 
604 {
605  TLOG(16) << "ResetWritePos(" << buffer << ") called.";
606 
607  if (buffer >= shm_ptr_->buffer_count) Detach(true, "ArgumentOutOfRange", "The specified buffer does not exist!");
608 
609  TLOG(TLVL_BUFLCK) << "ResetWritePos obtaining buffer_mutex for buffer " << buffer;
610  std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
611  TLOG(TLVL_BUFLCK) << "ResetWritePos obtained buffer_mutex for buffer " << buffer;
612 
613  //TraceLock lk(buffer_mutexes_[buffer], 18, "ResetWritePosBuffer" + std::to_string(buffer));
614  auto buf = getBufferInfo_(buffer);
615  if (!buf) return;
616  checkBuffer_(buf, BufferSemaphoreFlags::Writing);
617  touchBuffer_(buf);
618  buf->writePos = 0;
619 
620  TLOG(16) << "ResetWritePos(" << buffer << ") ended.";
621 }
622 
624 {
625  TLOG(15) << "IncrementReadPos called: buffer= " << buffer << ", bytes to read=" << read;
626 
627  if (buffer >= shm_ptr_->buffer_count) Detach(true, "ArgumentOutOfRange", "The specified buffer does not exist!");
628 
629  TLOG(TLVL_BUFLCK) << "IncrementReadPos obtaining buffer_mutex for buffer " << buffer;
630  std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
631  TLOG(TLVL_BUFLCK) << "IncrementReadPos obtained buffer_mutex for buffer " << buffer;
632  //TraceLock lk(buffer_mutexes_[buffer], 19, "IncReadPosBuffer" + std::to_string(buffer));
633  auto buf = getBufferInfo_(buffer);
634  if (!buf || buf->sem_id != manager_id_) return;
635  touchBuffer_(buf);
636  TLOG(15) << "IncrementReadPos: buffer= " << buffer << ", readPos=" << buf->readPos << ", bytes read=" << read;
637  buf->readPos = buf->readPos + read;
638  TLOG(15) << "IncrementReadPos: buffer= " << buffer << ", New readPos is " << buf->readPos;
639  if (read == 0) Detach(true, "LogicError", "Cannot increment Read pos by 0! (buffer=" + std::to_string(buffer) + ", readPos=" + std::to_string(buf->readPos) + ", writePos=" + std::to_string(buf->writePos) + ")");
640 }
641 
642 bool artdaq::SharedMemoryManager::IncrementWritePos(int buffer, size_t written)
643 {
644  TLOG(16) << "IncrementWritePos called: buffer= " << buffer << ", bytes written=" << written;
645 
646  if (buffer >= shm_ptr_->buffer_count) Detach(true, "ArgumentOutOfRange", "The specified buffer does not exist!");
647 
648  TLOG(TLVL_BUFLCK) << "IncrementWritePos obtaining buffer_mutex for buffer " << buffer;
649  std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
650  TLOG(TLVL_BUFLCK) << "IncrementWritePos obtained buffer_mutex for buffer " << buffer;
651  //TraceLock lk(buffer_mutexes_[buffer], 20, "IncWritePosBuffer" + std::to_string(buffer));
652  auto buf = getBufferInfo_(buffer);
653  if (!buf) return false;
654  checkBuffer_(buf, BufferSemaphoreFlags::Writing);
655  touchBuffer_(buf);
656  if (buf->writePos + written > shm_ptr_->buffer_size)
657  {
658  TLOG(TLVL_ERROR) << "Requested write size is larger than the buffer size! (sz=" << std::hex << shm_ptr_->buffer_size << ", cur + req=" << buf->writePos + written << ")";
659  return false;
660  }
661  TLOG(16) << "IncrementWritePos: buffer= " << buffer << ", writePos=" << buf->writePos << ", bytes written=" << written;
662  buf->writePos += written;
663  TLOG(16) << "IncrementWritePos: buffer= " << buffer << ", New writePos is " << buf->writePos;
664  if (written == 0) Detach(true, "LogicError", "Cannot increment Write pos by 0!");
665 
666  return true;
667 }
668 
670 {
671  TLOG(17) << "MoreDataInBuffer(" << buffer << ") called.";
672 
673  if (buffer >= shm_ptr_->buffer_count) Detach(true, "ArgumentOutOfRange", "The specified buffer does not exist!");
674 
675  TLOG(TLVL_BUFLCK) << "MoreDataInBuffer obtaining buffer_mutex for buffer " << buffer;
676  std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
677  TLOG(TLVL_BUFLCK) << "MoreDataInBuffer obtained buffer_mutex for buffer " << buffer;
678  //TraceLock lk(buffer_mutexes_[buffer], 21, "MoreDataInBuffer" + std::to_string(buffer));
679  auto buf = getBufferInfo_(buffer);
680  if (!buf) return false;
681  TLOG(17) << "MoreDataInBuffer: buffer= " << buffer << ", readPos=" << std::to_string(buf->readPos) << ", writePos=" << buf->writePos;
682  return buf->readPos < buf->writePos;
683 }
684 
686 {
687  if (buffer >= shm_ptr_->buffer_count) Detach(true, "ArgumentOutOfRange", "The specified buffer does not exist!");
688 
689  TLOG(TLVL_BUFLCK) << "CheckBuffer obtaining buffer_mutex for buffer " << buffer;
690  std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
691  TLOG(TLVL_BUFLCK) << "CheckBuffer obtained buffer_mutex for buffer " << buffer;
692  //TraceLock lk(buffer_mutexes_[buffer], 22, "CheckBuffer" + std::to_string(buffer));
693  return checkBuffer_(getBufferInfo_(buffer), flags, false);
694 }
695 
696 void artdaq::SharedMemoryManager::MarkBufferFull(int buffer, int destination)
697 {
698  if (buffer >= shm_ptr_->buffer_count) Detach(true, "ArgumentOutOfRange", "The specified buffer does not exist!");
699 
700  TLOG(TLVL_BUFLCK) << "MarkBufferFull obtaining buffer_mutex for buffer " << buffer;
701  std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
702  TLOG(TLVL_BUFLCK) << "MarkBufferFull obtained buffer_mutex for buffer " << buffer;
703 
704  //TraceLock lk(buffer_mutexes_[buffer], 23, "FillBuffer" + std::to_string(buffer));
705  auto shmBuf = getBufferInfo_(buffer);
706  if (!shmBuf) return;
707  touchBuffer_(shmBuf);
708  if (shmBuf->sem_id == manager_id_)
709  {
710  if (shmBuf->sem != BufferSemaphoreFlags::Full)
711  shmBuf->sem = BufferSemaphoreFlags::Full;
712 
713  shmBuf->sem_id = destination;
714  }
715 }
716 
718 {
719  TLOG(18) << "MarkBufferEmpty BEGIN";
720  if (buffer >= shm_ptr_->buffer_count) Detach(true, "ArgumentOutOfRange", "The specified buffer does not exist!");
721  std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
722  //TraceLock lk(buffer_mutexes_[buffer], 24, "EmptyBuffer" + std::to_string(buffer));
723  auto shmBuf = getBufferInfo_(buffer);
724  if (!shmBuf) return;
725  if (!force)
726  {
727  checkBuffer_(shmBuf, BufferSemaphoreFlags::Reading, true);
728  }
729  touchBuffer_(shmBuf);
730 
731  shmBuf->readPos = 0;
732  shmBuf->sem = BufferSemaphoreFlags::Full;
733 
734  if ((force && (manager_id_ == 0 || manager_id_ == shmBuf->sem_id)) || (!force && shm_ptr_->destructive_read_mode))
735  {
736  TLOG(18) << "MarkBufferEmpty Resetting buffer " << buffer << " to Empty state";
737  shmBuf->writePos = 0;
738  shmBuf->sem = BufferSemaphoreFlags::Empty;
739  if (shm_ptr_->reader_pos == static_cast<unsigned>(buffer) && !shm_ptr_->destructive_read_mode)
740  {
741  TLOG(18) << "MarkBufferEmpty Broadcast mode; incrementing reader_pos from " << shm_ptr_->reader_pos << " to " << (buffer + 1) % shm_ptr_->buffer_count;
742  shm_ptr_->reader_pos = (buffer + 1) % shm_ptr_->buffer_count;
743  }
744  }
745  shmBuf->sem_id = -1;
746  TLOG(18) << "MarkBufferEmpty END";
747 }
748 
750 {
751  if (buffer >= shm_ptr_->buffer_count) Detach(true, "ArgumentOutOfRange", "The specified buffer does not exist!");
752 
753  TLOG(TLVL_BUFLCK) << "ResetBuffer: obtaining buffer_mutex lock for buffer " << buffer;
754  std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
755  TLOG(TLVL_BUFLCK) << "ResetBuffer: obtained buffer_mutex lock for buffer " << buffer;
756 
757  //TraceLock lk(buffer_mutexes_[buffer], 25, "ResetBuffer" + std::to_string(buffer));
758  auto shmBuf = getBufferInfo_(buffer);
759  if (!shmBuf) return false;
760  /*
761  if (shmBuf->sequence_id < shm_ptr_->lowest_seq_id_read - size() && shmBuf->sem == BufferSemaphoreFlags::Full)
762  {
763  TLOG(TLVL_DEBUG) << "Buffer " << buffer << " has been passed by all readers, marking Empty" ;
764  shmBuf->writePos = 0;
765  shmBuf->sem = BufferSemaphoreFlags::Empty;
766  shmBuf->sem_id = -1;
767  return true;
768  }*/
769 
770  size_t delta = TimeUtils::gettimeofday_us() - shmBuf->last_touch_time;
771  if (delta > 0xFFFFFFFF)
772  {
773  TLOG(TLVL_TRACE) << "Buffer has touch time in the future, setting it to current time and ignoring...";
774  shmBuf->last_touch_time = TimeUtils::gettimeofday_us();
775  return false;
776  }
777  if (shm_ptr_->buffer_timeout_us == 0 || delta <= shm_ptr_->buffer_timeout_us || shmBuf->sem == BufferSemaphoreFlags::Empty) return false;
778  TLOG(27) << "Buffer " << buffer << " at " << (void*)shmBuf << " is stale, time=" << TimeUtils::gettimeofday_us() << ", last touch=" << shmBuf->last_touch_time << ", d=" << delta << ", timeout=" << shm_ptr_->buffer_timeout_us;
779 
780  if (shmBuf->sem_id == manager_id_ && shmBuf->sem == BufferSemaphoreFlags::Writing)
781  {
782  return true;
783  }
784 
785  if (!shm_ptr_->destructive_read_mode && shmBuf->sem == BufferSemaphoreFlags::Full && (shmBuf->sequence_id < last_seen_id_ || manager_id_ == 0))
786  {
787  TLOG(TLVL_DEBUG) << "Resetting old broadcast mode buffer " << buffer << " (seqid=" << shmBuf->sequence_id << "). State: Full-->Empty";
788  shmBuf->writePos = 0;
789  shmBuf->sem = BufferSemaphoreFlags::Empty;
790  shmBuf->sem_id = -1;
791  if (shm_ptr_->reader_pos == static_cast<unsigned>(buffer)) shm_ptr_->reader_pos = (buffer + 1) % shm_ptr_->buffer_count;
792  return true;
793  }
794 
795  if (shmBuf->sem_id != manager_id_ && shmBuf->sem == BufferSemaphoreFlags::Reading)
796  {
797  // Ron wants to re-check for potential interleave of buffer state updates
798  size_t delta = TimeUtils::gettimeofday_us() - shmBuf->last_touch_time;
799  if (delta <= shm_ptr_->buffer_timeout_us) return false;
800  TLOG(TLVL_WARNING) << "Stale Read buffer " << buffer << " at " << (void*)shmBuf
801  << " ( " << delta << " / " << shm_ptr_->buffer_timeout_us << " us ) detected! (seqid="
802  << shmBuf->sequence_id << ") Resetting... Reading-->Full";
803  shmBuf->readPos = 0;
804  shmBuf->sem = BufferSemaphoreFlags::Full;
805  shmBuf->sem_id = -1;
806  return true;
807  }
808  return false;
809 }
810 
812 {
813  if (!IsValid()) return true;
814 
815  struct shmid_ds info;
816  auto sts = shmctl(shm_segment_id_, IPC_STAT, &info);
817  if (sts < 0)
818  {
819  TLOG(TLVL_TRACE) << "Error accessing Shared Memory info: " << errno << " (" << strerror(errno) << ").";
820  return true;
821  }
822 
823  if (info.shm_perm.mode & SHM_DEST)
824  {
825  TLOG(TLVL_INFO) << "Shared Memory marked for destruction. Probably an end-of-data condition!";
826  return true;
827  }
828 
829  return false;
830 }
831 
833 {
834  if (!IsValid()) return 0;
835 
836  struct shmid_ds info;
837  auto sts = shmctl(shm_segment_id_, IPC_STAT, &info);
838  if (sts < 0)
839  {
840  TLOG(TLVL_TRACE) << "Error accessing Shared Memory info: " << errno << " (" << strerror(errno) << ").";
841  return 0;
842  }
843 
844  return info.shm_nattch;
845 }
846 
847 size_t artdaq::SharedMemoryManager::Write(int buffer, void* data, size_t size)
848 {
849  TLOG(19) << "Write BEGIN";
850  if (buffer >= shm_ptr_->buffer_count) Detach(true, "ArgumentOutOfRange", "The specified buffer does not exist!");
851  std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
852  //TraceLock lk(buffer_mutexes_[buffer], 26, "WriteBuffer" + std::to_string(buffer));
853  auto shmBuf = getBufferInfo_(buffer);
854  if (!shmBuf) return -1;
855  checkBuffer_(shmBuf, BufferSemaphoreFlags::Writing);
856  touchBuffer_(shmBuf);
857  TLOG(19) << "Buffer Write Pos is " << shmBuf->writePos << ", write size is " << size;
858  if (shmBuf->writePos + size > shm_ptr_->buffer_size)
859  {
860  TLOG(TLVL_ERROR) << "Attempted to write more data than fits into Shared Memory, bufferSize=" << shm_ptr_->buffer_size
861  << ",writePos=" << shmBuf->writePos << ",writeSize=" << size;
862  Detach(true, "SharedMemoryWrite", "Attempted to write more data than fits into Shared Memory! \nRe-run with a larger buffer size!");
863  }
864 
865  auto pos = GetWritePos(buffer);
866  memcpy(pos, data, size);
867  touchBuffer_(shmBuf);
868  shmBuf->writePos = shmBuf->writePos + size;
869 
870  auto last_seen = last_seen_id_.load();
871  while (last_seen < shmBuf->sequence_id && !last_seen_id_.compare_exchange_weak(last_seen, shmBuf->sequence_id)) {}
872 
873  TLOG(19) << "Write END";
874  return size;
875 }
876 
877 bool artdaq::SharedMemoryManager::Read(int buffer, void* data, size_t size)
878 {
879  if (buffer >= shm_ptr_->buffer_count) Detach(true, "ArgumentOutOfRange", "The specified buffer does not exist!");
880  std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
881  //TraceLock lk(buffer_mutexes_[buffer], 27, "ReadBuffer" + std::to_string(buffer));
882  auto shmBuf = getBufferInfo_(buffer);
883  if (!shmBuf) return false;
884  checkBuffer_(shmBuf, BufferSemaphoreFlags::Reading);
885  touchBuffer_(shmBuf);
886  if (shmBuf->readPos + size > shm_ptr_->buffer_size)
887  {
888  TLOG(TLVL_ERROR) << "Attempted to read more data than fits into Shared Memory, bufferSize=" << shm_ptr_->buffer_size
889  << ",readPos=" << shmBuf->readPos << ",readSize=" << size;
890  Detach(true, "SharedMemoryRead", "Attempted to read more data than exists in Shared Memory!");
891  }
892 
893  auto pos = GetReadPos(buffer);
894  memcpy(data, pos, size);
895  auto sts = checkBuffer_(shmBuf, BufferSemaphoreFlags::Reading, false);
896  if (sts)
897  {
898  shmBuf->readPos += size;
899  touchBuffer_(shmBuf);
900  return true;
901  }
902  return false;
903 }
904 
906 {
907  std::ostringstream ostr;
908  ostr << "ShmStruct: " << std::endl
909  << "Reader Position: " << shm_ptr_->reader_pos << std::endl
910  << "Writer Position: " << shm_ptr_->writer_pos << std::endl
911  << "Next ID Number: " << shm_ptr_->next_id << std::endl
912  << "Buffer Count: " << shm_ptr_->buffer_count << std::endl
913  << "Buffer Size: " << std::to_string(shm_ptr_->buffer_size) << " bytes" << std::endl
914  << "Buffers Written: " << std::to_string(shm_ptr_->next_sequence_id) << std::endl
915  << "Rank of Writer: " << shm_ptr_->rank << std::endl
916  << "Ready Magic Bytes: 0x" << std::hex << shm_ptr_->ready_magic << std::endl << std::endl;
917 
918  for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
919  {
920  auto buf = getBufferInfo_(ii);
921  if (!buf) continue;
922 
923  ostr << "ShmBuffer " << std::dec << ii << std::endl
924  << "sequenceID: " << std::to_string(buf->sequence_id) << std::endl
925  << "writePos: " << std::to_string(buf->writePos) << std::endl
926  << "readPos: " << std::to_string(buf->readPos) << std::endl
927  << "sem: " << FlagToString(buf->sem) << std::endl
928  << "Owner: " << std::to_string(buf->sem_id.load()) << std::endl
929  << "Last Touch Time: " << std::to_string(buf->last_touch_time / 1000000.0) << std::endl << std::endl;
930  }
931 
932  return ostr.str();
933 }
934 
936 {
937  auto buf = getBufferInfo_(buffer);
938  if (!buf) return nullptr;
939  return bufferStart_(buffer) + buf->readPos;
940 }
942 {
943  auto buf = getBufferInfo_(buffer);
944  if (!buf) return nullptr;
945  return bufferStart_(buffer) + buf->writePos;
946 }
947 
949 {
950  return bufferStart_(buffer);
951 }
952 
953 std::list<std::pair<int, artdaq::SharedMemoryManager::BufferSemaphoreFlags>> artdaq::SharedMemoryManager::GetBufferReport()
954 {
955  auto output = std::list<std::pair<int, BufferSemaphoreFlags>>();
956  for (size_t ii = 0; ii < size(); ++ii)
957  {
958  auto buf = getBufferInfo_(ii);
959  output.emplace_back(std::make_pair(buf->sem_id.load(), buf->sem.load()));
960  }
961  return output;
962 }
963 
964 uint8_t* artdaq::SharedMemoryManager::dataStart_() const
965 {
966  if (shm_ptr_ == nullptr) return nullptr;
967  return reinterpret_cast<uint8_t*>(shm_ptr_ + 1) + shm_ptr_->buffer_count * sizeof(ShmBuffer);
968 }
969 
970 uint8_t* artdaq::SharedMemoryManager::bufferStart_(int buffer)
971 {
972  if (shm_ptr_ == nullptr) return nullptr;
973  if (buffer >= shm_ptr_->buffer_count) Detach(true, "ArgumentOutOfRange", "The specified buffer does not exist!");
974  return dataStart_() + buffer * shm_ptr_->buffer_size;
975 }
976 
977 artdaq::SharedMemoryManager::ShmBuffer* artdaq::SharedMemoryManager::getBufferInfo_(int buffer)
978 {
979  if (shm_ptr_ == nullptr) return nullptr;
980  if (buffer >= shm_ptr_->buffer_count) Detach(true, "ArgumentOutOfRange", "The specified buffer does not exist!");
981  return reinterpret_cast<ShmBuffer*>(reinterpret_cast<uint8_t*>(shm_ptr_ + 1) + buffer * sizeof(ShmBuffer));
982 }
983 
984 bool artdaq::SharedMemoryManager::checkBuffer_(ShmBuffer* buffer, BufferSemaphoreFlags flags, bool exceptions)
985 {
986  if (!buffer)
987  {
988  if (exceptions)
989  {
990  Detach(true, "BufferNotThereException", "Request to check buffer that does not exist!");
991  }
992  return false;
993  }
994  TLOG(TLVL_TRACE) << "checkBuffer_: Checking that buffer " << buffer->sequence_id << " has sem_id " << manager_id_ << " (Current: " << buffer->sem_id << ") and is in state " << FlagToString(flags) << " (current: " << FlagToString(buffer->sem) << ")";
995  if (exceptions)
996  {
997  if (buffer->sem != flags) Detach(true, "StateAccessViolation", "Shared Memory buffer is not in the correct state! (expected " + FlagToString(flags) + ", actual " + FlagToString(buffer->sem) + ")");
998  if (buffer->sem_id != manager_id_) Detach(true, "OwnerAccessViolation", "Shared Memory buffer is not owned by this manager instance! (Expected: " + std::to_string(manager_id_) + ", Actual: " + std::to_string(buffer->sem_id) + ")");
999  }
1000  bool ret = (buffer->sem_id == manager_id_ || (buffer->sem_id == -1 && (flags == BufferSemaphoreFlags::Full || flags == BufferSemaphoreFlags::Empty))) && buffer->sem == flags;
1001 
1002  if (!ret)
1003  {
1004  TLOG(TLVL_WARNING) << "CheckBuffer detected issue with buffer " << buffer->sequence_id << "!"
1005  << " ID: " << buffer->sem_id << " (Expected " << manager_id_ << "), Flag: " << FlagToString(buffer->sem) << " (Expected " << FlagToString(flags) << "). "
1006  << "ID -1 is okay if expected flag is \"Full\" or \"Empty\".";
1007  }
1008 
1009  return ret;
1010 }
1011 
1012 void artdaq::SharedMemoryManager::touchBuffer_(ShmBuffer* buffer)
1013 {
1014  if (!buffer || (buffer->sem_id != -1 && buffer->sem_id != manager_id_)) return;
1015  TLOG(TLVL_TRACE) << "touchBuffer_: Touching buffer at " << (void*)buffer << " with sequence_id " << buffer->sequence_id;
1016  buffer->last_touch_time = TimeUtils::gettimeofday_us();
1017 }
1018 
1019 void artdaq::SharedMemoryManager::Detach(bool throwException, std::string category, std::string message, bool force)
1020 {
1021  TLOG(TLVL_DETACH) << "Detach BEGIN: throwException: " << std::boolalpha << throwException << ", force: " << force;
1022  if (IsValid())
1023  {
1024  TLOG(TLVL_DETACH) << "Detach: Resetting owned buffers";
1025  auto bufs = GetBuffersOwnedByManager(false);
1026  for (auto buf : bufs)
1027  {
1028  auto shmBuf = getBufferInfo_(buf);
1029  if (!shmBuf) continue;
1030  if (shmBuf->sem == BufferSemaphoreFlags::Writing)
1031  {
1032  shmBuf->sem = BufferSemaphoreFlags::Empty;
1033  }
1034  else if (shmBuf->sem == BufferSemaphoreFlags::Reading)
1035  {
1036  shmBuf->sem = BufferSemaphoreFlags::Full;
1037  }
1038  shmBuf->sem_id = -1;
1039  }
1040  }
1041 
1042  if (shm_ptr_)
1043  {
1044  TLOG(TLVL_DETACH) << "Detach: Detaching shared memory";
1045  shmdt(shm_ptr_);
1046  shm_ptr_ = NULL;
1047  }
1048 
1049  if ((force || manager_id_ == 0) && shm_segment_id_ > -1)
1050  {
1051  TLOG(TLVL_DETACH) << "Detach: Marking Shared memory for removal";
1052  shmctl(shm_segment_id_, IPC_RMID, NULL);
1053  shm_segment_id_ = -1;
1054  }
1055 
1056  if (category.size() > 0 && message.size() > 0)
1057  {
1058  TLOG(TLVL_ERROR) << category << ": " << message;
1059 
1060  if (throwException)
1061  {
1062  throw cet::exception(category) << message;
1063  }
1064  }
1065 }
1066 
1067 
1068 
1069 // Local Variables:
1070 // mode: c++
1071 // End:
bool Read(int buffer, void *data, size_t size)
Read size bytes of data from buffer into the given pointer.
void IncrementReadPos(int buffer, size_t read)
Increment the read position for a given buffer.
constexpr size_t GetElapsedTimeMilliseconds(std::chrono::steady_clock::time_point then, std::chrono::steady_clock::time_point now=std::chrono::steady_clock::now())
Gets the number of milliseconds in the given time interval
Definition: TimeUtils.hh:54
std::deque< int > GetBuffersOwnedByManager(bool locked=true)
Get the list of all buffers currently owned by this manager instance.
void ResetReadPos(int buffer)
Set the read position of the given buffer to the beginning of the buffer.
bool MoreDataInBuffer(int buffer)
Determine if more data is available to be read, based on the read position and data size...
virtual std::string toString()
Write information about the SharedMemory to a string.
std::list< std::pair< int, BufferSemaphoreFlags > > GetBufferReport()
Get a report on the status of each buffer.
BufferSemaphoreFlags
The BufferSemaphoreFlags enumeration represents the different possible &quot;states&quot; of a given shared mem...
size_t BufferDataSize(int buffer)
Get the current size of the buffer&#39;s data.
void Attach()
Reconnect to the shared memory segment.
void Detach(bool throwException=false, std::string category="", std::string message="", bool force=false)
Detach from the Shared Memory segment, optionally throwing a cet::exception with the specified proper...
virtual ~SharedMemoryManager() noexcept
SharedMemoryManager Destructor.
The SharedMemoryManager creates a Shared Memory area which is divided into a number of fixed-size buf...
size_t ReadReadyCount()
Count the number of buffers that are ready for reading.
bool ResetBuffer(int buffer)
Resets the buffer from Reading to Full. This operation will only have an effect if performed by the o...
bool IncrementWritePos(int buffer, size_t written)
Increment the write position for a given buffer.
bool IsEndOfData() const
Determine whether the Shared Memory is marked for destruction (End of Data)
bool CheckBuffer(int buffer, BufferSemaphoreFlags flags)
Check both semaphore conditions (Mode flag and manager ID) for a given buffer.
void MarkBufferFull(int buffer, int destination=-1)
Release a buffer from a writer, marking it Full and ready for a reader.
void MarkBufferEmpty(int buffer, bool force=false)
Release a buffer from a reader, marking it Empty and ready to accept more data.
void ResetWritePos(int buffer)
Set the write position of the given buffer to the beginning of the buffer.
void * GetWritePos(int buffer)
Get a pointer to the current write position of the buffer.
size_t Write(int buffer, void *data, size_t size)
Write size bytes of data from the given pointer to a buffer.
void * GetReadPos(int buffer)
Get a pointer to the current read position of the buffer.
void * GetBufferStart(int buffer)
Get a pointer to the start position of the buffer.
SharedMemoryManager(uint32_t shm_key, size_t buffer_count=0, size_t buffer_size=0, uint64_t buffer_timeout_us=100 *1000000, bool destructive_read_mode=true)
SharedMemoryManager Constructor.
bool ReadyForRead()
Whether any buffer is ready for read.
int GetBufferForWriting(bool overwrite)
Finds a buffer that is ready to be written to, and reserves it for the calling manager.
uint16_t GetAttachedCount() const
Get the number of attached SharedMemoryManagers.
size_t WriteReadyCount(bool overwrite)
Count the number of buffers that are ready for writing.
uint64_t gettimeofday_us()
Get the current time of day in microseconds (from gettimeofday system call)
Definition: TimeUtils.cc:51
int GetBufferForReading()
Finds a buffer that is ready to be read, and reserves it for the calling manager. ...
virtual bool ReadyForWrite(bool overwrite)
Whether any buffer is available for write.