1 #define TRACE_NAME "SharedMemoryManager"
6 #ifndef SHM_DEST // Lynn reports that this is missing on Mac OS X?!?
11 #include "cetlib_except/exception.h"
12 #include "artdaq-core/Core/SharedMemoryManager.hh"
13 #include "artdaq-core/Utilities/TraceLock.hh"
15 #define TLVL_DETACH 11
17 static std::vector<artdaq::SharedMemoryManager const*> instances = std::vector<artdaq::SharedMemoryManager const*>();
19 static std::unordered_map<int, struct sigaction> old_actions = std::unordered_map<int, struct sigaction>();
20 static bool sighandler_init =
false;
21 static void signal_handler(
int signum)
24 TRACE_STREAMER(TLVL_ERROR, &(
"SharedMemoryManager")[0], 0, 0, 0) <<
"A signal of type " << signum <<
" (" << std::string(strsignal(signum)) <<
") was caught by SharedMemoryManager. Detaching all Shared Memory segments, then proceeding with default handlers!";
25 for (
auto ii : instances)
36 pthread_sigmask(SIG_UNBLOCK, NULL, &set);
37 pthread_sigmask(SIG_UNBLOCK, &set, NULL);
39 TRACE_STREAMER(TLVL_ERROR, &(
"SharedMemoryManager")[0], 0, 0, 0) <<
"Calling default signal handler";
40 if (signum != SIGUSR2)
42 sigaction(signum, &old_actions[signum], NULL);
43 kill(getpid(), signum);
48 sigaction(SIGINT, &old_actions[SIGINT], NULL);
49 kill(getpid(), SIGINT);
61 requested_shm_parameters_.buffer_count = buffer_count;
62 requested_shm_parameters_.buffer_size = buffer_size;
63 requested_shm_parameters_.buffer_timeout_us = buffer_timeout_us;
64 requested_shm_parameters_.destructive_read_mode = destructive_read_mode;
66 instances.push_back(
this);
69 static std::mutex sighandler_mutex;
70 std::unique_lock<std::mutex> lk(sighandler_mutex);
74 sighandler_init =
true;
75 std::vector<int> signals = { SIGINT, SIGQUIT, SIGILL, SIGABRT, SIGFPE, SIGSEGV, SIGPIPE, SIGALRM, SIGTERM, SIGUSR2 };
76 for (
auto signal : signals)
78 struct sigaction old_action;
79 sigaction(signal, NULL, &old_action);
83 if (old_action.sa_handler != SIG_IGN)
85 struct sigaction action;
86 action.sa_handler = signal_handler;
87 sigemptyset(&action.sa_mask);
88 for (
auto sigblk : signals)
90 sigaddset(&action.sa_mask, sigblk);
95 sigaction(signal, &action, NULL);
96 old_actions[signal] = old_action;
104 TLOG(TLVL_DEBUG) <<
"~SharedMemoryManager called";
106 TLOG(TLVL_DEBUG) <<
"~SharedMemoryManager done";
113 if (manager_id_ == 0)
return;
116 auto start_time = std::chrono::steady_clock::now();
118 size_t shmSize = requested_shm_parameters_.buffer_count * (requested_shm_parameters_.buffer_size +
sizeof(ShmBuffer)) +
sizeof(ShmStruct);
120 shm_segment_id_ = shmget(shm_key_, shmSize, 0666);
121 if (shm_segment_id_ == -1 && requested_shm_parameters_.buffer_count > 0 && manager_id_ <= 0)
123 TLOG(TLVL_DEBUG) <<
"Creating shared memory segment with key 0x" << std::hex << shm_key_ <<
" and size " << shmSize;
124 shm_segment_id_ = shmget(shm_key_, shmSize, IPC_CREAT | 0666);
127 if (shm_segment_id_ == -1)
129 TLOG(TLVL_ERROR) <<
"Error creating shared memory segment with key 0x" << std::hex << shm_key_ <<
", errno=" << errno <<
" (" << strerror(errno) <<
")";
136 shm_segment_id_ = shmget(shm_key_, shmSize, 0666);
140 TLOG(TLVL_DEBUG) <<
"shm_key == 0x" << std::hex << shm_key_ <<
", shm_segment_id == " << shm_segment_id_;
142 if (shm_segment_id_ > -1)
145 <<
"Attached to shared memory segment with ID = " << shm_segment_id_
146 <<
" and size " << shmSize
148 shm_ptr_ = (ShmStruct*)shmat(shm_segment_id_, 0, 0);
150 <<
"Attached to shared memory segment at address "
151 << std::hex << (
void*)shm_ptr_ << std::dec;
152 if (shm_ptr_ && shm_ptr_ != (
void *)-1)
154 if (manager_id_ == 0)
156 if (shm_ptr_->ready_magic == 0xCAFE1111)
158 TLOG(TLVL_ERROR) <<
"Owner encountered already-initialized Shared Memory!";
161 TLOG(TLVL_DEBUG) <<
"Owner initializing Shared Memory";
162 shm_ptr_->next_id = 1;
163 shm_ptr_->next_sequence_id = 0;
164 shm_ptr_->reader_pos = 0;
165 shm_ptr_->writer_pos = 0;
166 shm_ptr_->buffer_size = requested_shm_parameters_.buffer_size;
167 shm_ptr_->buffer_count = requested_shm_parameters_.buffer_count;
168 shm_ptr_->buffer_timeout_us = requested_shm_parameters_.buffer_timeout_us;
169 shm_ptr_->destructive_read_mode = requested_shm_parameters_.destructive_read_mode;
171 for (
int ii = 0; ii < static_cast<int>(requested_shm_parameters_.buffer_count); ++ii)
173 getBufferInfo_(ii)->writePos = 0;
174 getBufferInfo_(ii)->readPos = 0;
175 getBufferInfo_(ii)->sem = BufferSemaphoreFlags::Empty;
176 getBufferInfo_(ii)->sem_id = -1;
180 shm_ptr_->ready_magic = 0xCAFE1111;
184 TLOG(TLVL_DEBUG) <<
"Waiting for owner to initalize Shared Memory";
185 while (shm_ptr_->ready_magic != 0xCAFE1111) { usleep(1000); }
186 TLOG(TLVL_DEBUG) <<
"Getting ID from Shared Memory";
188 shm_ptr_->lowest_seq_id_read = 0;
189 TLOG(TLVL_DEBUG) <<
"Getting Shared Memory Size parameters";
192 TLOG(TLVL_DEBUG) <<
"Initialization Complete: "
193 <<
"key: 0x" << std::hex << shm_key_
194 <<
", manager ID: " << manager_id_
195 <<
", Buffer size: " << std::to_string(shm_ptr_->buffer_size)
196 <<
", Buffer count: " << std::to_string(shm_ptr_->buffer_count);
201 TLOG(TLVL_ERROR) <<
"Failed to attach to shared memory segment "
207 TLOG(TLVL_ERROR) <<
"Failed to connect to shared memory segment with key 0x" << std::hex << shm_key_
208 <<
", errno = " << strerror(errno) <<
". Please check "
209 <<
"if a stale shared memory segment needs to "
210 <<
"be cleaned up. (ipcs, ipcrm -m <segId>)";
217 TLOG(13) <<
"GetBufferForReading BEGIN";
220 TraceLock lk(search_mutex_, 11,
"GetBufferForReadingSearch");
221 auto rp = shm_ptr_->reader_pos.load();
223 TLOG(13) <<
"GetBufferForReading lock acquired, scanning buffers";
228 ShmBuffer* buffer_ptr =
nullptr;
230 for (
auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
232 auto buffer = (ii + rp) % shm_ptr_->buffer_count;
235 TLOG(14) <<
"GetBufferForReading Checking if buffer " << buffer <<
" is stale";
238 auto buf = getBufferInfo_(buffer);
239 TLOG(14) <<
"GetBufferForReading: Buffer " << buffer <<
": sem=" << FlagToString(buf->sem) <<
" (expected " << FlagToString(BufferSemaphoreFlags::Full) <<
"), sem_id=" << buf->sem_id <<
" )";
240 if (buf->sem == BufferSemaphoreFlags::Full && (buf->sem_id == -1 || buf->sem_id == manager_id_) && buf->sequence_id > last_seen_id_)
242 if (buf->sequence_id < seqID)
245 seqID = buf->sequence_id;
251 if (!buffer_ptr || (buffer_ptr && buffer_ptr->sem_id != -1 && buffer_ptr->sem_id != manager_id_))
258 TLOG(13) <<
"GetBufferForReading Found buffer " << buffer_num;
259 buffer_ptr->sem_id = manager_id_;
260 buffer_ptr->sem = BufferSemaphoreFlags::Reading;
261 if (buffer_ptr->sem_id != manager_id_) {
continue; }
262 buffer_ptr->readPos = 0;
263 touchBuffer_(buffer_ptr);
264 if (buffer_ptr->sem_id != manager_id_) {
continue; }
265 if (shm_ptr_->destructive_read_mode && shm_ptr_->lowest_seq_id_read == last_seen_id_)
267 shm_ptr_->lowest_seq_id_read = seqID;
269 last_seen_id_ = seqID;
270 if (shm_ptr_->destructive_read_mode) shm_ptr_->reader_pos = (buffer_num + 1) % shm_ptr_->buffer_count;
275 TLOG(13) <<
"GetBufferForReading returning -1 because no buffers are ready";
281 TLOG(14) <<
"GetBufferForWriting BEGIN";
283 TraceLock lk(search_mutex_, 12,
"GetBufferForWritingSearch");
284 auto wp = shm_ptr_->writer_pos.load();
287 for (
auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
289 auto buffer = (ii + wp) % shm_ptr_->buffer_count;
293 auto buf = getBufferInfo_(buffer);
294 if (buf->sem == BufferSemaphoreFlags::Empty && buf->sem_id == -1)
296 buf->sem_id = manager_id_;
297 buf->sem = BufferSemaphoreFlags::Writing;
299 shm_ptr_->writer_pos = (buffer + 1) % shm_ptr_->buffer_count;
300 if (buf->sem_id != manager_id_)
continue;
301 buf->sequence_id = ++shm_ptr_->next_sequence_id;
303 TLOG(14) <<
"GetBufferForWriting returning " << buffer;
311 for (
auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
313 auto buffer = (ii + wp) % shm_ptr_->buffer_count;
317 auto buf = getBufferInfo_(buffer);
318 if (buf->sem == BufferSemaphoreFlags::Full)
320 buf->sem_id = manager_id_;
321 buf->sem = BufferSemaphoreFlags::Writing;
322 if (buf->sem_id != manager_id_)
continue;
323 buf->sequence_id = ++shm_ptr_->next_sequence_id;
325 shm_ptr_->writer_pos = (buffer + 1) % shm_ptr_->buffer_count;
327 TLOG(14) <<
"GetBufferForWriting returning " << buffer;
333 for (
auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
335 auto buffer = (ii + wp) % shm_ptr_->buffer_count;
339 auto buf = getBufferInfo_(buffer);
340 if (buf->sem == BufferSemaphoreFlags::Reading)
342 buf->sem_id = manager_id_;
343 buf->sem = BufferSemaphoreFlags::Writing;
344 if (buf->sem_id != manager_id_)
continue;
345 buf->sequence_id = ++shm_ptr_->next_sequence_id;
347 shm_ptr_->writer_pos = (buffer + 1) % shm_ptr_->buffer_count;
348 TLOG(14) <<
"GetBufferForWriting returning " << buffer;
355 TLOG(14) <<
"GetBufferForWriting Returning -1 because no buffers are ready";
361 if (!IsValid())
return 0;
362 TLOG(23) <<
"0x" << std::hex << shm_key_ <<
" ReadReadyCount BEGIN";
364 TraceLock lk(search_mutex_, 14,
"ReadReadyCountSearch");
366 for (
auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
368 TLOG(24) <<
"0x" << std::hex << shm_key_ << std::dec <<
" ReadReadyCount: Checking if buffer " << ii <<
" is stale.";
370 auto buf = getBufferInfo_(ii);
371 TLOG(25) <<
"0x" << std::hex << shm_key_ << std::dec <<
" ReadReadyCount: Buffer " << ii <<
": sem=" << FlagToString(buf->sem) <<
" (expected " << FlagToString(BufferSemaphoreFlags::Full) <<
"), sem_id=" << buf->sem_id <<
" )";
372 if (buf->sem == BufferSemaphoreFlags::Full && (buf->sem_id == -1 || buf->sem_id == manager_id_) && buf->sequence_id > last_seen_id_)
374 TLOG(26) <<
"0x" << std::hex << shm_key_ << std::dec <<
" ReadReadyCount: Buffer " << ii <<
" is either unowned or owned by this manager, and is marked full.";
383 if (!IsValid())
return 0;
385 TraceLock lk(search_mutex_, 15,
"WriteReadyCountSearch");
387 for (
auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
390 auto buf = getBufferInfo_(ii);
391 if ((buf->sem == BufferSemaphoreFlags::Empty && buf->sem_id == -1)
392 || (overwrite && buf->sem != BufferSemaphoreFlags::Writing))
403 std::deque<int> output;
404 if (!IsValid())
return output;
407 TraceLock lk(search_mutex_, 16,
"GetOwnedSearch");
408 for (
auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
410 auto buf = getBufferInfo_(ii);
411 if (buf->sem_id == manager_id_)
413 output.push_back(ii);
419 for (
auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
421 auto buf = getBufferInfo_(ii);
422 if (buf->sem_id == manager_id_)
424 output.push_back(ii);
435 TraceLock lk(buffer_mutexes_[buffer], 17,
"DataSizeBuffer" + std::to_string(buffer));
436 auto buf = getBufferInfo_(buffer);
438 return buf->writePos;
445 TraceLock lk(buffer_mutexes_[buffer], 18,
"ResetReadPosBuffer" + std::to_string(buffer));
446 auto buf = getBufferInfo_(buffer);
454 TraceLock lk(buffer_mutexes_[buffer], 18,
"ResetWritePosBuffer" + std::to_string(buffer));
455 auto buf = getBufferInfo_(buffer);
456 checkBuffer_(buf, BufferSemaphoreFlags::Writing);
464 TraceLock lk(buffer_mutexes_[buffer], 19,
"IncReadPosBuffer" + std::to_string(buffer));
465 auto buf = getBufferInfo_(buffer);
467 TLOG(15) <<
"IncrementReadPos: buffer= " << buffer <<
", readPos=" << std::to_string(buf->readPos) <<
", bytes read=" << std::to_string(read);
468 buf->readPos = buf->readPos + read;
469 TLOG(15) <<
"IncrementReadPos: buffer= " << buffer <<
", New readPos is " << std::to_string(buf->readPos);
470 if (read == 0) Detach(
true,
"LogicError",
"Cannot increment Read pos by 0! (buffer=" + std::to_string(buffer) +
", readPos=" + std::to_string(buf->readPos) +
", writePos=" + std::to_string(buf->writePos) +
")");
476 TraceLock lk(buffer_mutexes_[buffer], 20,
"IncWritePosBuffer" + std::to_string(buffer));
477 auto buf = getBufferInfo_(buffer);
479 TLOG(16) <<
"IncrementWritePos: buffer= " << buffer <<
", writePos=" << std::to_string(buf->writePos) <<
", bytes written=" << std::to_string(written);
480 buf->writePos += written;
481 TLOG(16) <<
"IncrementWritePos: buffer= " << buffer <<
", New writePos is " << std::to_string(buf->writePos);
482 if (written == 0) Detach(
true,
"LogicError",
"Cannot increment Write pos by 0!");
488 TraceLock lk(buffer_mutexes_[buffer], 21,
"MoreDataInBuffer" + std::to_string(buffer));
489 auto buf = getBufferInfo_(buffer);
490 TLOG(17) <<
"MoreDataInBuffer: buffer= " << buffer <<
", readPos=" << std::to_string(buf->readPos) <<
", writePos=" << std::to_string(buf->writePos);
491 return buf->readPos < buf->writePos;
497 TraceLock lk(buffer_mutexes_[buffer], 22,
"CheckBuffer" + std::to_string(buffer));
498 return checkBuffer_(getBufferInfo_(buffer), flags,
false);
504 TraceLock lk(buffer_mutexes_[buffer], 23,
"FillBuffer" + std::to_string(buffer));
505 auto shmBuf = getBufferInfo_(buffer);
506 touchBuffer_(shmBuf);
507 if (shmBuf->sem_id == manager_id_)
509 if (shmBuf->sem != BufferSemaphoreFlags::Full)
510 shmBuf->sem = BufferSemaphoreFlags::Full;
512 shmBuf->sem_id = destination;
518 TLOG(18) <<
"MarkBufferEmpty BEGIN";
520 TraceLock lk(buffer_mutexes_[buffer], 24,
"EmptyBuffer" + std::to_string(buffer));
521 auto shmBuf = getBufferInfo_(buffer);
524 checkBuffer_(shmBuf, BufferSemaphoreFlags::Reading,
true);
526 touchBuffer_(shmBuf);
529 shmBuf->sem = BufferSemaphoreFlags::Full;
531 if ((force && (manager_id_ == 0 || manager_id_ == shmBuf->sem_id)) || (!force && shm_ptr_->destructive_read_mode))
533 TLOG(18) <<
"MarkBufferEmpty Resetting buffer to Empty state";
534 shmBuf->writePos = 0;
535 shmBuf->sem = BufferSemaphoreFlags::Empty;
536 if (shm_ptr_->reader_pos == static_cast<unsigned>(buffer) && !shm_ptr_->destructive_read_mode)
538 TLOG(18) <<
"MarkBufferEmpty Broadcast mode; incrementing reader_pos";
539 shm_ptr_->reader_pos = (buffer + 1) % shm_ptr_->buffer_count;
543 TLOG(18) <<
"MarkBufferEmpty END";
549 TraceLock lk(buffer_mutexes_[buffer], 25,
"ResetBuffer" + std::to_string(buffer));
550 auto shmBuf = getBufferInfo_(buffer);
562 if (delta > 0xFFFFFFFF)
564 TLOG(TLVL_TRACE) <<
"Buffer has touch time in the future, setting it to current time and ignoring...";
568 if (shm_ptr_->buffer_timeout_us == 0 || delta <= shm_ptr_->buffer_timeout_us || shmBuf->sem == BufferSemaphoreFlags::Empty)
return false;
569 TLOG(27) <<
"Buffer " << buffer <<
" is stale, time=" <<
TimeUtils::gettimeofday_us() <<
", last touch=" << shmBuf->last_touch_time <<
", d=" << delta <<
", timeout=" << shm_ptr_->buffer_timeout_us;
571 if (shmBuf->sem_id == manager_id_ && shmBuf->sem == BufferSemaphoreFlags::Writing)
575 if (!shm_ptr_->destructive_read_mode && shmBuf->sem == BufferSemaphoreFlags::Full)
577 TLOG(TLVL_DEBUG) <<
"Resetting old broadcast mode buffer";
578 shmBuf->writePos = 0;
579 shmBuf->sem = BufferSemaphoreFlags::Empty;
581 if (shm_ptr_->reader_pos == static_cast<unsigned>(buffer)) shm_ptr_->reader_pos = (buffer + 1) % shm_ptr_->buffer_count;
585 if (shmBuf->sem_id != manager_id_ && shmBuf->sem == BufferSemaphoreFlags::Reading)
587 TLOG(TLVL_WARNING) <<
"Stale Read buffer ( " << delta <<
" / " << shm_ptr_->buffer_timeout_us <<
" us ) detected! Resetting...";
589 shmBuf->sem = BufferSemaphoreFlags::Full;
598 if (!IsValid())
return true;
600 struct shmid_ds info;
601 auto sts = shmctl(shm_segment_id_, IPC_STAT, &info);
604 TLOG(TLVL_TRACE) <<
"Error accessing Shared Memory info: " << errno <<
" (" << strerror(errno) <<
").";
608 if (info.shm_perm.mode & SHM_DEST)
610 TLOG(TLVL_INFO) <<
"Shared Memory marked for destruction. Probably an end-of-data condition!";
619 if (!IsValid())
return 0;
621 struct shmid_ds info;
622 auto sts = shmctl(shm_segment_id_, IPC_STAT, &info);
625 TLOG(TLVL_TRACE) <<
"Error accessing Shared Memory info: " << errno <<
" (" << strerror(errno) <<
").";
629 return info.shm_nattch;
634 TLOG(19) <<
"Write BEGIN";
636 TraceLock lk(buffer_mutexes_[buffer], 26,
"WriteBuffer" + std::to_string(buffer));
637 auto shmBuf = getBufferInfo_(buffer);
638 checkBuffer_(shmBuf, BufferSemaphoreFlags::Writing);
639 touchBuffer_(shmBuf);
640 TLOG(19) <<
"Buffer Write Pos is " << std::to_string(shmBuf->writePos) <<
", write size is " << std::to_string(size);
641 if (shmBuf->writePos + size > shm_ptr_->buffer_size) Detach(
true,
"SharedMemoryWrite",
"Attempted to write more data than fits into Shared Memory! \nRe-run with a larger buffer size!");
643 auto pos = GetWritePos(buffer);
644 memcpy(pos, data, size);
645 shmBuf->writePos = shmBuf->writePos + size;
646 if (shmBuf->sequence_id > last_seen_id_)
648 last_seen_id_ = shmBuf->sequence_id;
650 TLOG(19) <<
"Write END";
657 TraceLock lk(buffer_mutexes_[buffer], 27,
"ReadBuffer" + std::to_string(buffer));
658 auto shmBuf = getBufferInfo_(buffer);
659 checkBuffer_(shmBuf, BufferSemaphoreFlags::Reading);
660 touchBuffer_(shmBuf);
661 if (shmBuf->readPos + size > shm_ptr_->buffer_size) Detach(
true,
"SharedMemoryRead",
"Attempted to read more data than exists in Shared Memory!");
663 auto pos = GetReadPos(buffer);
664 memcpy(data, pos, size);
665 shmBuf->readPos += size;
666 touchBuffer_(shmBuf);
667 return checkBuffer_(shmBuf, BufferSemaphoreFlags::Reading,
false);
672 std::ostringstream ostr;
673 ostr <<
"ShmStruct: " << std::endl
674 <<
"Reader Position: " << shm_ptr_->reader_pos << std::endl
675 <<
"Writer Position: " << shm_ptr_->writer_pos << std::endl
676 <<
"Next ID Number: " << shm_ptr_->next_id << std::endl
677 <<
"Buffer Count: " << shm_ptr_->buffer_count << std::endl
678 <<
"Buffer Size: " << std::to_string(shm_ptr_->buffer_size) <<
" bytes" << std::endl
679 <<
"Buffers Written: " << std::to_string(shm_ptr_->next_sequence_id) << std::endl
680 <<
"Rank of Writer: " << shm_ptr_->rank << std::endl
681 <<
"Ready Magic Bytes: 0x" << std::hex << shm_ptr_->ready_magic << std::endl << std::endl;
683 for (
auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
685 auto buf = getBufferInfo_(ii);
686 ostr <<
"ShmBuffer " << std::dec << ii << std::endl
687 <<
"sequenceID: " << std::to_string(buf->sequence_id) << std::endl
688 <<
"writePos: " << std::to_string(buf->writePos) << std::endl
689 <<
"readPos: " << std::to_string(buf->readPos) << std::endl
690 <<
"sem: " << FlagToString(buf->sem) << std::endl
691 <<
"Owner: " << std::to_string(buf->sem_id.load()) << std::endl
692 <<
"Last Touch Time: " << std::to_string(buf->last_touch_time / 1000000.0) << std::endl << std::endl;
700 auto buf = getBufferInfo_(buffer);
701 return bufferStart_(buffer) + buf->readPos;
705 auto buf = getBufferInfo_(buffer);
706 return bufferStart_(buffer) + buf->writePos;
711 return bufferStart_(buffer);
714 uint8_t* artdaq::SharedMemoryManager::dataStart_()
const
716 return reinterpret_cast<uint8_t*
>(shm_ptr_ + 1) + shm_ptr_->buffer_count *
sizeof(ShmBuffer);
719 uint8_t* artdaq::SharedMemoryManager::bufferStart_(
int buffer)
721 if (buffer >= shm_ptr_->buffer_count) Detach(
true,
"ArgumentOutOfRange",
"The specified buffer does not exist!");
722 return dataStart_() + buffer * shm_ptr_->buffer_size;
725 artdaq::SharedMemoryManager::ShmBuffer* artdaq::SharedMemoryManager::getBufferInfo_(
int buffer)
727 if (buffer >= shm_ptr_->buffer_count) Detach(
true,
"ArgumentOutOfRange",
"The specified buffer does not exist!");
728 return reinterpret_cast<ShmBuffer*
>(
reinterpret_cast<uint8_t*
>(shm_ptr_ + 1) + buffer *
sizeof(ShmBuffer));
731 bool artdaq::SharedMemoryManager::checkBuffer_(ShmBuffer* buffer, BufferSemaphoreFlags flags,
bool exceptions)
733 TLOG(TLVL_TRACE) <<
"checkBuffer_: Checking that buffer " << buffer->sequence_id <<
" has sem_id " << manager_id_ <<
" (Current: " << buffer->sem_id <<
") and is in state " << FlagToString(flags) <<
" (current: " << FlagToString(buffer->sem) <<
")";
736 if (buffer->sem != flags) Detach(
true,
"StateAccessViolation",
"Shared Memory buffer is not in the correct state! (expected " + FlagToString(flags) +
", actual " + FlagToString(buffer->sem) +
")");
737 if (buffer->sem_id != manager_id_) Detach(
true,
"OwnerAccessViolation",
"Shared Memory buffer is not owned by this manager instance! (Expected: " + std::to_string(manager_id_) +
", Actual: " + std::to_string(buffer->sem_id) +
")");
739 bool ret = (buffer->sem_id == manager_id_ || (buffer->sem_id == -1 && (flags == BufferSemaphoreFlags::Full || flags == BufferSemaphoreFlags::Empty))) && buffer->sem == flags;
743 TLOG(TLVL_WARNING) <<
"CheckBuffer detected issue with buffer " << std::to_string(buffer->sequence_id) <<
"!"
744 <<
" ID: " << buffer->sem_id <<
" (" << manager_id_ <<
"), Flag: " << FlagToString(buffer->sem) <<
" (" << FlagToString(flags) <<
"). "
745 <<
"ID -1 is okay if desired flag is \"Full\" or \"Empty\".";
751 void artdaq::SharedMemoryManager::touchBuffer_(ShmBuffer* buffer)
753 if (buffer->sem_id != manager_id_)
return;
754 TLOG(TLVL_TRACE) <<
"touchBuffer_: Touching buffer with sequence_id " << std::to_string(buffer->sequence_id);
760 TLOG(TLVL_DETACH) <<
"Detach BEGIN: throwException: " << std::boolalpha << throwException <<
", force: " << force;
763 TLOG(TLVL_DETACH) <<
"Detach: Resetting owned buffers";
764 auto bufs = GetBuffersOwnedByManager(
false);
765 for (
auto buf : bufs)
767 auto shmBuf = getBufferInfo_(buf);
768 if (shmBuf->sem == BufferSemaphoreFlags::Writing)
770 shmBuf->sem = BufferSemaphoreFlags::Empty;
772 else if (shmBuf->sem == BufferSemaphoreFlags::Reading)
774 shmBuf->sem = BufferSemaphoreFlags::Full;
782 TLOG(TLVL_DETACH) <<
"Detach: Detaching shared memory";
787 if ((force || manager_id_ == 0) && shm_segment_id_ > -1)
789 TLOG(TLVL_DETACH) <<
"Detach: Marking Shared memory for removal";
790 shmctl(shm_segment_id_, IPC_RMID, NULL);
791 shm_segment_id_ = -1;
794 if (category.size() > 0 && message.size() > 0)
796 TLOG(TLVL_ERROR) << category <<
": " << message;
800 throw cet::exception(category) << message;
bool Read(int buffer, void *data, size_t size)
Read size bytes of data from buffer into the given pointer.
void IncrementReadPos(int buffer, size_t read)
Increment the read position for a given buffer.
constexpr size_t GetElapsedTimeMilliseconds(std::chrono::steady_clock::time_point then, std::chrono::steady_clock::time_point now=std::chrono::steady_clock::now())
Gets the number of milliseconds in the given time interval
std::deque< int > GetBuffersOwnedByManager(bool locked=true)
Get the list of all buffers currently owned by this manager instance.
void ResetReadPos(int buffer)
Set the read position of the given buffer to the beginning of the buffer.
bool MoreDataInBuffer(int buffer)
Determine if more data is available to be read, based on the read position and data size...
virtual std::string toString()
Write information about the SharedMemory to a string.
BufferSemaphoreFlags
The BufferSemaphoreFlags enumeration represents the different possible "states" of a given shared mem...
size_t BufferDataSize(int buffer)
Get the current size of the buffer's data.
void Attach()
Reconnect to the shared memory segment.
void Detach(bool throwException=false, std::string category="", std::string message="", bool force=false)
Detach from the Shared Memory segment, optionally throwing a cet::exception with the specified proper...
virtual ~SharedMemoryManager() noexcept
SharedMemoryManager Destructor.
The SharedMemoryManager creates a Shared Memory area which is divided into a number of fixed-size buf...
size_t ReadReadyCount()
Count the number of buffers that are ready for reading.
bool ResetBuffer(int buffer)
Resets the buffer from Reading to Full. This operation will only have an effect if performed by the o...
bool IsEndOfData() const
Determine whether the Shared Memory is marked for destruction (End of Data)
bool CheckBuffer(int buffer, BufferSemaphoreFlags flags)
Check both semaphore conditions (Mode flag and manager ID) for a given buffer.
void IncrementWritePos(int buffer, size_t written)
Increment the write position for a given buffer.
The TraceLock class allows a user to debug the acquisition and releasing of locks, by wrapping the unique_lock<std::mutex> API with TRACE calls.
void MarkBufferFull(int buffer, int destination=-1)
Release a buffer from a writer, marking it Full and ready for a reader.
void MarkBufferEmpty(int buffer, bool force=false)
Release a buffer from a reader, marking it Empty and ready to accept more data.
void ResetWritePos(int buffer)
Set the write position of the given buffer to the beginning of the buffer.
void * GetWritePos(int buffer)
Get a pointer to the current write position of the buffer.
size_t Write(int buffer, void *data, size_t size)
Write size bytes of data from the given pointer to a buffer.
void * GetReadPos(int buffer)
Get a pointer to the current read position of the buffer.
void * GetBufferStart(int buffer)
Get a pointer to the start position of the buffer.
SharedMemoryManager(uint32_t shm_key, size_t buffer_count=0, size_t buffer_size=0, uint64_t buffer_timeout_us=100 *1000000, bool destructive_read_mode=true)
SharedMemoryManager Constructor.
int GetBufferForWriting(bool overwrite)
Finds a buffer that is ready to be written to, and reserves it for the calling manager.
uint16_t GetAttachedCount() const
Get the number of attached SharedMemoryManagers.
size_t WriteReadyCount(bool overwrite)
Count the number of buffers that are ready for writing.
uint64_t gettimeofday_us()
Get the current time of day in microseconds (from gettimeofday system call)
int GetBufferForReading()
Finds a buffer that is ready to be read, and reserves it for the calling manager. ...