1 #define TRACE_NAME "SharedMemoryManager"
7 #include "cetlib_except/exception.h"
8 #include "artdaq-core/Core/SharedMemoryManager.hh"
9 #include "artdaq-core/Utilities/TraceLock.hh"
11 #define TLVL_DETACH 11
13 static std::vector<artdaq::SharedMemoryManager*> instances = std::vector<artdaq::SharedMemoryManager*>();
15 static std::unordered_map<int, struct sigaction> old_actions = std::unordered_map<int, struct sigaction>();
16 static bool sighandler_init =
false;
17 static void signal_handler(
int signum)
20 TRACE_STREAMER(TLVL_ERROR, &(
"SharedMemoryManager")[0], 0, 0, 0) <<
"A signal of type " << signum <<
" (" << std::string(strsignal(signum)) <<
") was caught by SharedMemoryManager. Detaching all Shared Memory segments, then proceeding with default handlers!";
21 for (
auto ii : instances)
25 ii->Detach(
false,
"",
"",
true);
31 pthread_sigmask(SIG_UNBLOCK, NULL, &set);
32 pthread_sigmask(SIG_UNBLOCK, &set, NULL);
34 TRACE_STREAMER(TLVL_ERROR, &(
"SharedMemoryManager")[0], 0, 0, 0) <<
"Calling default signal handler";
35 if (signum != SIGUSR2) {
36 sigaction(signum, &old_actions[signum], NULL);
37 kill(getpid(), signum);
41 sigaction(SIGINT, &old_actions[SIGINT], NULL);
42 kill(getpid(), SIGINT);
54 requested_shm_parameters_.buffer_count = buffer_count;
55 requested_shm_parameters_.buffer_size = buffer_size;
56 requested_shm_parameters_.buffer_timeout_us = buffer_timeout_us;
57 requested_shm_parameters_.destructive_read_mode = destructive_read_mode;
59 instances.push_back(
this);
62 static std::mutex sighandler_mutex;
63 std::unique_lock<std::mutex> lk(sighandler_mutex);
67 sighandler_init =
true;
68 std::vector<int> signals = { SIGINT, SIGQUIT, SIGILL, SIGABRT, SIGFPE, SIGSEGV, SIGPIPE, SIGALRM, SIGTERM, SIGUSR2 };
69 for (
auto signal : signals)
71 struct sigaction old_action;
72 sigaction(signal, NULL, &old_action);
76 if (old_action.sa_handler != SIG_IGN)
78 struct sigaction action;
79 action.sa_handler = signal_handler;
80 sigemptyset(&action.sa_mask);
81 for (
auto sigblk : signals)
83 sigaddset(&action.sa_mask, sigblk);
88 sigaction(signal, &action, NULL);
89 old_actions[signal] = old_action;
97 TLOG(TLVL_DEBUG) <<
"~SharedMemoryManager called" ;
99 TLOG(TLVL_DEBUG) <<
"~SharedMemoryManager done" ;
106 if (manager_id_ == 0)
return;
109 auto start_time = std::chrono::steady_clock::now();
111 size_t shmSize = requested_shm_parameters_.buffer_count * (requested_shm_parameters_.buffer_size +
sizeof(ShmBuffer)) +
sizeof(ShmStruct);
113 shm_segment_id_ = shmget(shm_key_, shmSize, 0666);
114 if (shm_segment_id_ == -1 && requested_shm_parameters_.buffer_count > 0 && manager_id_ <= 0)
116 TLOG(TLVL_DEBUG) <<
"Creating shared memory segment with key 0x" << std::hex << shm_key_ <<
" and size " << shmSize;
117 shm_segment_id_ = shmget(shm_key_, shmSize, IPC_CREAT | 0666);
120 if (shm_segment_id_ == -1)
122 TLOG(TLVL_ERROR) <<
"Error creating shared memory segment, errno=" << errno <<
" (" << strerror(errno) <<
")";
129 shm_segment_id_ = shmget(shm_key_, shmSize, 0666);
133 TLOG(TLVL_DEBUG) <<
"shm_key == 0x" << std::hex << shm_key_ <<
", shm_segment_id == " << shm_segment_id_ ;
135 if (shm_segment_id_ > -1)
138 <<
"Attached to shared memory segment with ID = " << shm_segment_id_
139 <<
" and size " << shmSize
141 shm_ptr_ = (ShmStruct*)shmat(shm_segment_id_, 0, 0);
143 <<
"Attached to shared memory segment at address "
144 << std::hex << (
void*)shm_ptr_ << std::dec ;
145 if (shm_ptr_ && shm_ptr_ != (
void *)-1)
147 if (manager_id_ == 0)
149 if (shm_ptr_->ready_magic == 0xCAFE1111)
151 TLOG(TLVL_ERROR) <<
"Owner encountered already-initialized Shared Memory!" ;
154 TLOG(TLVL_DEBUG) <<
"Owner initializing Shared Memory" ;
155 shm_ptr_->next_id = 1;
156 shm_ptr_->next_sequence_id = 0;
157 shm_ptr_->reader_pos = 0;
158 shm_ptr_->writer_pos = 0;
159 shm_ptr_->buffer_size = requested_shm_parameters_.buffer_size;
160 shm_ptr_->buffer_count = requested_shm_parameters_.buffer_count;
161 shm_ptr_->buffer_timeout_us = requested_shm_parameters_.buffer_timeout_us;
162 shm_ptr_->destructive_read_mode = requested_shm_parameters_.destructive_read_mode;
164 for (
int ii = 0; ii < static_cast<int>(requested_shm_parameters_.buffer_count); ++ii)
166 getBufferInfo_(ii)->writePos = 0;
167 getBufferInfo_(ii)->readPos = 0;
168 getBufferInfo_(ii)->sem = BufferSemaphoreFlags::Empty;
169 getBufferInfo_(ii)->sem_id = -1;
173 shm_ptr_->ready_magic = 0xCAFE1111;
177 TLOG(TLVL_DEBUG) <<
"Waiting for owner to initalize Shared Memory" ;
178 while (shm_ptr_->ready_magic != 0xCAFE1111) { usleep(1000); }
179 TLOG(TLVL_DEBUG) <<
"Getting ID from Shared Memory" ;
181 shm_ptr_->lowest_seq_id_read = 0;
182 TLOG(TLVL_DEBUG) <<
"Getting Shared Memory Size parameters" ;
185 TLOG(TLVL_DEBUG) <<
"Initialization Complete: "
186 <<
"key: 0x" << std::hex << shm_key_
187 <<
", manager ID: " << manager_id_
188 <<
", Buffer size: " << std::to_string(shm_ptr_->buffer_size)
189 <<
", Buffer count: " << std::to_string(shm_ptr_->buffer_count) ;
194 TLOG(TLVL_ERROR) <<
"Failed to attach to shared memory segment "
200 TLOG(TLVL_ERROR) <<
"Failed to connect to shared memory segment"
201 <<
", errno = " << strerror(errno) <<
". Please check "
202 <<
"if a stale shared memory segment needs to "
203 <<
"be cleaned up. (ipcs, ipcrm -m <segId>)" ;
210 TLOG(13) <<
"GetBufferForReading BEGIN" ;
213 TraceLock lk(search_mutex_, 11,
"GetBufferForReadingSearch");
214 auto rp = shm_ptr_->reader_pos.load();
216 TLOG(13) <<
"GetBufferForReading lock acquired, scanning buffers" ;
221 ShmBuffer* buffer_ptr =
nullptr;
223 for (
auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
225 auto buffer = (ii + rp) % shm_ptr_->buffer_count;
228 TLOG(14) <<
"GetBufferForReading Checking if buffer " << buffer <<
" is stale" ;
231 auto buf = getBufferInfo_(buffer);
232 TLOG(14) <<
"GetBufferForReading: Buffer " << buffer <<
": sem=" << FlagToString(buf->sem) <<
" (expected " << FlagToString(BufferSemaphoreFlags::Full) <<
"), sem_id=" << buf->sem_id <<
" )" ;
233 if (buf->sem == BufferSemaphoreFlags::Full && (buf->sem_id == -1 || buf->sem_id == manager_id_) && buf->sequence_id > last_seen_id_)
235 if (buf->sequence_id < seqID)
238 seqID = buf->sequence_id;
244 if (!buffer_ptr || (buffer_ptr && buffer_ptr->sem_id != -1 && buffer_ptr->sem_id != manager_id_))
251 TLOG(13) <<
"GetBufferForReading Found buffer " << buffer_num ;
252 buffer_ptr->sem_id = manager_id_;
253 buffer_ptr->sem = BufferSemaphoreFlags::Reading;
254 if (buffer_ptr->sem_id != manager_id_) {
continue; }
255 buffer_ptr->readPos = 0;
256 touchBuffer_(buffer_ptr);
257 if (buffer_ptr->sem_id != manager_id_) {
continue; }
258 if (shm_ptr_->destructive_read_mode && shm_ptr_->lowest_seq_id_read == last_seen_id_)
260 shm_ptr_->lowest_seq_id_read = seqID;
262 last_seen_id_ = seqID;
263 if (shm_ptr_->destructive_read_mode) shm_ptr_->reader_pos = (buffer_num + 1) % shm_ptr_->buffer_count;
268 TLOG(13) <<
"GetBufferForReading returning -1 because no buffers are ready" ;
274 TLOG(14) <<
"GetBufferForWriting BEGIN" ;
276 TraceLock lk(search_mutex_, 12,
"GetBufferForWritingSearch");
277 auto wp = shm_ptr_->writer_pos.load();
280 for (
auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
282 auto buffer = (ii + wp) % shm_ptr_->buffer_count;
286 auto buf = getBufferInfo_(buffer);
287 if (buf->sem == BufferSemaphoreFlags::Empty && buf->sem_id == -1)
289 buf->sem_id = manager_id_;
290 buf->sem = BufferSemaphoreFlags::Writing;
292 shm_ptr_->writer_pos = (buffer + 1) % shm_ptr_->buffer_count;
293 if (buf->sem_id != manager_id_)
continue;
294 buf->sequence_id = ++shm_ptr_->next_sequence_id;
296 TLOG(14) <<
"GetBufferForWriting returning " << buffer ;
304 for (
auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
306 auto buffer = (ii + wp) % shm_ptr_->buffer_count;
310 auto buf = getBufferInfo_(buffer);
311 if (buf->sem == BufferSemaphoreFlags::Full)
313 buf->sem_id = manager_id_;
314 buf->sem = BufferSemaphoreFlags::Writing;
315 if (buf->sem_id != manager_id_)
continue;
316 buf->sequence_id = ++shm_ptr_->next_sequence_id;
318 shm_ptr_->writer_pos = (buffer + 1) % shm_ptr_->buffer_count;
320 TLOG(14) <<
"GetBufferForWriting returning " << buffer ;
326 for (
auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
328 auto buffer = (ii + wp) % shm_ptr_->buffer_count;
332 auto buf = getBufferInfo_(buffer);
333 if (buf->sem == BufferSemaphoreFlags::Reading)
335 buf->sem_id = manager_id_;
336 buf->sem = BufferSemaphoreFlags::Writing;
337 if (buf->sem_id != manager_id_)
continue;
338 buf->sequence_id = ++shm_ptr_->next_sequence_id;
340 shm_ptr_->writer_pos = (buffer + 1) % shm_ptr_->buffer_count;
341 TLOG(14) <<
"GetBufferForWriting returning " << buffer ;
348 TLOG(14) <<
"GetBufferForWriting Returning -1 because no buffers are ready" ;
354 if (!IsValid())
return 0;
355 TLOG(23) <<
"0x" << std::hex << shm_key_ <<
" ReadReadyCount BEGIN" ;
357 TraceLock lk(search_mutex_, 14,
"ReadReadyCountSearch");
359 for (
auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
361 TLOG(24) <<
"0x" << std::hex << shm_key_ << std::dec <<
" ReadReadyCount: Checking if buffer " << ii <<
" is stale." ;
363 auto buf = getBufferInfo_(ii);
364 TLOG(25) <<
"0x" << std::hex << shm_key_ << std::dec <<
" ReadReadyCount: Buffer " << ii <<
": sem=" << FlagToString(buf->sem) <<
" (expected " << FlagToString(BufferSemaphoreFlags::Full) <<
"), sem_id=" << buf->sem_id <<
" )" ;
365 if (buf->sem == BufferSemaphoreFlags::Full && (buf->sem_id == -1 || buf->sem_id == manager_id_) && buf->sequence_id > last_seen_id_)
367 TLOG(26) <<
"0x" << std::hex << shm_key_ << std::dec <<
" ReadReadyCount: Buffer " << ii <<
" is either unowned or owned by this manager, and is marked full." ;
376 if (!IsValid())
return 0;
378 TraceLock lk(search_mutex_, 15,
"WriteReadyCountSearch");
380 for (
auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
383 auto buf = getBufferInfo_(ii);
384 if ((buf->sem == BufferSemaphoreFlags::Empty && buf->sem_id == -1)
385 || (overwrite && buf->sem != BufferSemaphoreFlags::Writing))
396 std::deque<int> output;
397 if (!IsValid())
return output;
399 TraceLock lk(search_mutex_, 16,
"GetOwnedSearch");
400 for (
auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
402 auto buf = getBufferInfo_(ii);
403 if (buf->sem_id == manager_id_)
405 output.push_back(ii);
411 for (
auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
413 auto buf = getBufferInfo_(ii);
414 if (buf->sem_id == manager_id_)
416 output.push_back(ii);
427 TraceLock lk(buffer_mutexes_[buffer], 17,
"DataSizeBuffer" + std::to_string(buffer));
428 auto buf = getBufferInfo_(buffer);
430 return buf->writePos;
437 TraceLock lk(buffer_mutexes_[buffer], 18,
"ResetReadPosBuffer" + std::to_string(buffer));
438 auto buf = getBufferInfo_(buffer);
446 TraceLock lk(buffer_mutexes_[buffer], 18,
"ResetWritePosBuffer" + std::to_string(buffer));
447 auto buf = getBufferInfo_(buffer);
448 checkBuffer_(buf, BufferSemaphoreFlags::Writing);
456 TraceLock lk(buffer_mutexes_[buffer], 19,
"IncReadPosBuffer" + std::to_string(buffer));
457 auto buf = getBufferInfo_(buffer);
459 TLOG(15) <<
"IncrementReadPos: buffer= " << buffer <<
", readPos=" << std::to_string(buf->readPos) <<
", bytes read=" << std::to_string(read) ;
460 buf->readPos = buf->readPos + read;
461 TLOG(15) <<
"IncrementReadPos: buffer= " << buffer <<
", New readPos is " << std::to_string(buf->readPos) ;
462 if (read == 0) Detach(
true,
"LogicError",
"Cannot increment Read pos by 0! (buffer=" + std::to_string(buffer) +
", readPos=" + std::to_string(buf->readPos) +
", writePos=" + std::to_string(buf->writePos) +
")");
468 TraceLock lk(buffer_mutexes_[buffer], 20,
"IncWritePosBuffer" + std::to_string(buffer));
469 auto buf = getBufferInfo_(buffer);
471 TLOG(16) <<
"IncrementWritePos: buffer= " << buffer <<
", writePos=" << std::to_string(buf->writePos) <<
", bytes written=" << std::to_string(written) ;
472 buf->writePos += written;
473 TLOG(16) <<
"IncrementWritePos: buffer= " << buffer <<
", New writePos is " << std::to_string(buf->writePos) ;
474 if (written == 0) Detach(
true,
"LogicError",
"Cannot increment Write pos by 0!");
480 TraceLock lk(buffer_mutexes_[buffer], 21,
"MoreDataInBuffer" + std::to_string(buffer));
481 auto buf = getBufferInfo_(buffer);
482 TLOG(17) <<
"MoreDataInBuffer: buffer= " << buffer <<
", readPos=" << std::to_string(buf->readPos) <<
", writePos=" << std::to_string(buf->writePos) ;
483 return buf->readPos < buf->writePos;
489 TraceLock lk(buffer_mutexes_[buffer], 22,
"CheckBuffer" + std::to_string(buffer));
490 return checkBuffer_(getBufferInfo_(buffer), flags,
false);
496 TraceLock lk(buffer_mutexes_[buffer], 23,
"FillBuffer" + std::to_string(buffer));
497 auto shmBuf = getBufferInfo_(buffer);
498 touchBuffer_(shmBuf);
499 if (shmBuf->sem_id == manager_id_)
501 if (shmBuf->sem != BufferSemaphoreFlags::Full)
502 shmBuf->sem = BufferSemaphoreFlags::Full;
504 shmBuf->sem_id = destination;
510 TLOG(18) <<
"MarkBufferEmpty BEGIN" ;
512 TraceLock lk(buffer_mutexes_[buffer], 24,
"EmptyBuffer" + std::to_string(buffer));
513 auto shmBuf = getBufferInfo_(buffer);
516 checkBuffer_(shmBuf, BufferSemaphoreFlags::Reading,
true);
518 touchBuffer_(shmBuf);
521 shmBuf->sem = BufferSemaphoreFlags::Full;
523 if ((force && (manager_id_ == 0 || manager_id_ == shmBuf->sem_id)) || (!force && shm_ptr_->destructive_read_mode))
525 TLOG(18) <<
"MarkBufferEmpty Resetting buffer to Empty state" ;
526 shmBuf->writePos = 0;
527 shmBuf->sem = BufferSemaphoreFlags::Empty;
528 if (shm_ptr_->reader_pos == static_cast<unsigned>(buffer) && !shm_ptr_->destructive_read_mode)
530 TLOG(18) <<
"MarkBufferEmpty Broadcast mode; incrementing reader_pos" ;
531 shm_ptr_->reader_pos = (buffer + 1) % shm_ptr_->buffer_count;
535 TLOG(18) <<
"MarkBufferEmpty END" ;
541 TraceLock lk(buffer_mutexes_[buffer], 25,
"ResetBuffer" + std::to_string(buffer));
542 auto shmBuf = getBufferInfo_(buffer);
554 if (delta > 0xFFFFFFFF) {
555 TLOG(TLVL_TRACE) <<
"Buffer has touch time in the future, setting it to current time and ignoring..." ;
559 if (shm_ptr_->buffer_timeout_us == 0 || delta <= shm_ptr_->buffer_timeout_us || shmBuf->sem == BufferSemaphoreFlags::Empty)
return false;
560 TLOG(27) <<
"Buffer " << buffer <<
" is stale, time=" <<
TimeUtils::gettimeofday_us() <<
", last touch=" << shmBuf->last_touch_time <<
", d=" << delta <<
", timeout=" << shm_ptr_->buffer_timeout_us ;
562 if (shmBuf->sem_id == manager_id_ && shmBuf->sem == BufferSemaphoreFlags::Writing)
566 if (!shm_ptr_->destructive_read_mode && shmBuf->sem == BufferSemaphoreFlags::Full)
568 TLOG(TLVL_DEBUG) <<
"Resetting old broadcast mode buffer" ;
569 shmBuf->writePos = 0;
570 shmBuf->sem = BufferSemaphoreFlags::Empty;
572 if (shm_ptr_->reader_pos == static_cast<unsigned>(buffer)) shm_ptr_->reader_pos = (buffer + 1) % shm_ptr_->buffer_count;
576 if (shmBuf->sem_id != manager_id_ && shmBuf->sem == BufferSemaphoreFlags::Reading)
578 TLOG(TLVL_WARNING) <<
"Stale Read buffer ( " << delta <<
" / " << shm_ptr_->buffer_timeout_us <<
" us ) detected! Resetting..." ;
580 shmBuf->sem = BufferSemaphoreFlags::Full;
589 if (!IsValid())
return true;
591 struct shmid_ds info;
592 auto sts = shmctl(shm_segment_id_, IPC_STAT, &info);
594 TLOG(TLVL_TRACE) <<
"Error accessing Shared Memory info: " << errno <<
" (" << strerror(errno) <<
").";
598 if (info.shm_perm.mode & SHM_DEST) {
599 TLOG(TLVL_INFO) <<
"Shared Memory marked for destruction. Probably an end-of-data condition!";
608 TLOG(19) <<
"Write BEGIN" ;
610 TraceLock lk(buffer_mutexes_[buffer], 26,
"WriteBuffer" + std::to_string(buffer));
611 auto shmBuf = getBufferInfo_(buffer);
612 checkBuffer_(shmBuf, BufferSemaphoreFlags::Writing);
613 touchBuffer_(shmBuf);
614 TLOG(19) <<
"Buffer Write Pos is " << std::to_string(shmBuf->writePos) <<
", write size is " << std::to_string(size) ;
615 if (shmBuf->writePos + size > shm_ptr_->buffer_size) Detach(
true,
"SharedMemoryWrite",
"Attempted to write more data than fits into Shared Memory! \nRe-run with a larger buffer size!");
617 auto pos = GetWritePos(buffer);
618 memcpy(pos, data, size);
619 shmBuf->writePos = shmBuf->writePos + size;
620 if (shmBuf->sequence_id > last_seen_id_)
622 last_seen_id_ = shmBuf->sequence_id;
624 TLOG(19) <<
"Write END" ;
631 TraceLock lk(buffer_mutexes_[buffer], 27,
"ReadBuffer" + std::to_string(buffer));
632 auto shmBuf = getBufferInfo_(buffer);
633 checkBuffer_(shmBuf, BufferSemaphoreFlags::Reading);
634 touchBuffer_(shmBuf);
635 if (shmBuf->readPos + size > shm_ptr_->buffer_size) Detach(
true,
"SharedMemoryRead",
"Attempted to read more data than exists in Shared Memory!");
637 auto pos = GetReadPos(buffer);
638 memcpy(data, pos, size);
639 shmBuf->readPos += size;
640 touchBuffer_(shmBuf);
641 return checkBuffer_(shmBuf, BufferSemaphoreFlags::Reading,
false);
646 std::ostringstream ostr;
647 ostr <<
"ShmStruct: " << std::endl
648 <<
"Reader Position: " << shm_ptr_->reader_pos << std::endl
649 <<
"Writer Position: " << shm_ptr_->writer_pos << std::endl
650 <<
"Next ID Number: " << shm_ptr_->next_id << std::endl
651 <<
"Buffer Count: " << shm_ptr_->buffer_count << std::endl
652 <<
"Buffer Size: " << std::to_string(shm_ptr_->buffer_size) <<
" bytes" << std::endl
653 <<
"Buffers Written: " << std::to_string(shm_ptr_->next_sequence_id) << std::endl
654 <<
"Rank of Writer: " << shm_ptr_->rank << std::endl
655 <<
"Ready Magic Bytes: 0x" << std::hex << shm_ptr_->ready_magic << std::endl << std::endl;
657 for (
auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
659 auto buf = getBufferInfo_(ii);
660 ostr <<
"ShmBuffer " << std::dec << ii << std::endl
661 <<
"sequenceID: " << std::to_string(buf->sequence_id) << std::endl
662 <<
"writePos: " << std::to_string(buf->writePos) << std::endl
663 <<
"readPos: " << std::to_string(buf->readPos) << std::endl
664 <<
"sem: " << FlagToString(buf->sem) << std::endl
665 <<
"Owner: " << std::to_string(buf->sem_id.load()) << std::endl
666 <<
"Last Touch Time: " << std::to_string(buf->last_touch_time / 1000000.0) << std::endl << std::endl;
674 auto buf = getBufferInfo_(buffer);
675 return bufferStart_(buffer) + buf->readPos;
679 auto buf = getBufferInfo_(buffer);
680 return bufferStart_(buffer) + buf->writePos;
685 return bufferStart_(buffer);
688 uint8_t* artdaq::SharedMemoryManager::dataStart_()
const
690 return reinterpret_cast<uint8_t*
>(shm_ptr_ + 1) + shm_ptr_->buffer_count *
sizeof(ShmBuffer);
693 uint8_t* artdaq::SharedMemoryManager::bufferStart_(
int buffer)
695 if (buffer >= shm_ptr_->buffer_count) Detach(
true,
"ArgumentOutOfRange",
"The specified buffer does not exist!");
696 return dataStart_() + buffer * shm_ptr_->buffer_size;
699 artdaq::SharedMemoryManager::ShmBuffer* artdaq::SharedMemoryManager::getBufferInfo_(
int buffer)
701 if (buffer >= shm_ptr_->buffer_count) Detach(
true,
"ArgumentOutOfRange",
"The specified buffer does not exist!");
702 return reinterpret_cast<ShmBuffer*
>(
reinterpret_cast<uint8_t*
>(shm_ptr_ + 1) + buffer *
sizeof(ShmBuffer));
705 bool artdaq::SharedMemoryManager::checkBuffer_(ShmBuffer* buffer, BufferSemaphoreFlags flags,
bool exceptions)
707 TLOG(TLVL_TRACE) <<
"checkBuffer_: Checking that buffer " << buffer->sequence_id <<
" has sem_id " << manager_id_ <<
" (Current: " << buffer->sem_id <<
") and is in state " << FlagToString(flags) <<
" (current: " << FlagToString(buffer->sem) <<
")";
710 if (buffer->sem != flags) Detach(
true,
"StateAccessViolation",
"Shared Memory buffer is not in the correct state! (expected " + FlagToString(flags) +
", actual " + FlagToString(buffer->sem) +
")");
711 if (buffer->sem_id != manager_id_) Detach(
true,
"OwnerAccessViolation",
"Shared Memory buffer is not owned by this manager instance! (Expected: " + std::to_string(manager_id_) +
", Actual: " + std::to_string(buffer->sem_id) +
")");
713 bool ret = (buffer->sem_id == manager_id_ || (buffer->sem_id == -1 && (flags == BufferSemaphoreFlags::Full || flags == BufferSemaphoreFlags::Empty))) && buffer->sem == flags;
717 TLOG(TLVL_WARNING) <<
"CheckBuffer detected issue with buffer " << std::to_string(buffer->sequence_id) <<
"!"
718 <<
" ID: " << buffer->sem_id <<
" (" << manager_id_ <<
"), Flag: " << FlagToString(buffer->sem) <<
" (" << FlagToString(flags) <<
"). "
719 <<
"ID -1 is okay if desired flag is \"Full\" or \"Empty\"." ;
725 void artdaq::SharedMemoryManager::touchBuffer_(ShmBuffer* buffer)
727 if (buffer->sem_id != manager_id_)
return;
728 TLOG(TLVL_TRACE) <<
"touchBuffer_: Touching buffer with sequence_id " << std::to_string(buffer->sequence_id) ;
734 TLOG(TLVL_DETACH) <<
"Detach BEGIN: throwException: " << std::boolalpha << throwException <<
", force: " << force;
737 TLOG(TLVL_DETACH) <<
"Detach: Resetting owned buffers";
738 auto bufs = GetBuffersOwnedByManager(
false);
739 for (
auto buf : bufs)
741 auto shmBuf = getBufferInfo_(buf);
742 if (shmBuf->sem == BufferSemaphoreFlags::Writing)
744 shmBuf->sem = BufferSemaphoreFlags::Empty;
746 else if (shmBuf->sem == BufferSemaphoreFlags::Reading)
748 shmBuf->sem = BufferSemaphoreFlags::Full;
756 TLOG(TLVL_DETACH) <<
"Detach: Detaching shared memory";
761 if ((force || manager_id_ == 0) && shm_segment_id_ > -1)
763 TLOG(TLVL_DETACH) <<
"Detach: Marking Shared memory for removal";
764 shmctl(shm_segment_id_, IPC_RMID, NULL);
765 shm_segment_id_ = -1;
768 if (category.size() > 0 && message.size() > 0)
770 TLOG(TLVL_ERROR) << category <<
": " << message ;
774 throw cet::exception(category) << message;
bool Read(int buffer, void *data, size_t size)
Read size bytes of data from buffer into the given pointer.
void IncrementReadPos(int buffer, size_t read)
Increment the read position for a given buffer.
constexpr size_t GetElapsedTimeMilliseconds(std::chrono::steady_clock::time_point then, std::chrono::steady_clock::time_point now=std::chrono::steady_clock::now())
Gets the number of milliseconds in the given time interval
std::deque< int > GetBuffersOwnedByManager(bool locked=true)
Get the list of all buffers currently owned by this manager instance.
void ResetReadPos(int buffer)
Set the read position of the given buffer to the beginning of the buffer.
bool MoreDataInBuffer(int buffer)
Determine if more data is available to be read, based on the read position and data size...
virtual std::string toString()
Write information about the SharedMemory to a string.
BufferSemaphoreFlags
The BufferSemaphoreFlags enumeration represents the different possible "states" of a given shared mem...
size_t BufferDataSize(int buffer)
Get the current size of the buffer's data.
void Attach()
Reconnect to the shared memory segment.
void Detach(bool throwException=false, std::string category="", std::string message="", bool force=false)
Detach from the Shared Memory segment, optionally throwing a cet::exception with the specified proper...
virtual ~SharedMemoryManager() noexcept
SharedMemoryManager Destructor.
size_t ReadReadyCount()
Count the number of buffers that are ready for reading.
bool ResetBuffer(int buffer)
Resets the buffer from Reading to Full. This operation will only have an effect if performed by the o...
bool IsEndOfData() const
Determine whether the Shared Memory is marked for destruction (End of Data)
bool CheckBuffer(int buffer, BufferSemaphoreFlags flags)
Check both semaphore conditions (Mode flag and manager ID) for a given buffer.
void IncrementWritePos(int buffer, size_t written)
Increment the write position for a given buffer.
The TraceLock class allows a user to debug the acquisition and releasing of locks, by wrapping the unique_lock<std::mutex> API with TRACE calls.
void MarkBufferFull(int buffer, int destination=-1)
Release a buffer from a writer, marking it Full and ready for a reader.
void MarkBufferEmpty(int buffer, bool force=false)
Release a buffer from a reader, marking it Empty and ready to accept more data.
void ResetWritePos(int buffer)
Set the write position of the given buffer to the beginning of the buffer.
void * GetWritePos(int buffer)
Get a pointer to the current write position of the buffer.
size_t Write(int buffer, void *data, size_t size)
Write size bytes of data from the given pointer to a buffer.
void * GetReadPos(int buffer)
Get a pointer to the current read position of the buffer.
void * GetBufferStart(int buffer)
Get a pointer to the start position of the buffer.
SharedMemoryManager(uint32_t shm_key, size_t buffer_count=0, size_t buffer_size=0, uint64_t buffer_timeout_us=100 *1000000, bool destructive_read_mode=true)
SharedMemoryManager Constructor.
int GetBufferForWriting(bool overwrite)
Finds a buffer that is ready to be written to, and reserves it for the calling manager.
size_t WriteReadyCount(bool overwrite)
Count the number of buffers that are ready for writing.
uint64_t gettimeofday_us()
Get the current time of day in microseconds (from gettimeofday system call)
int GetBufferForReading()
Finds a buffer that is ready to be read, and reserves it for the calling manager. ...