1 #define TRACE_NAME "SharedMemoryManager"
7 #include "cetlib_except/exception.h"
8 #include "artdaq-core/Core/SharedMemoryManager.hh"
9 #include "artdaq-core/Utilities/TraceLock.hh"
11 #define TLVL_DETACH 11
13 static std::vector<artdaq::SharedMemoryManager const*> instances = std::vector<artdaq::SharedMemoryManager const*>();
15 static std::unordered_map<int, struct sigaction> old_actions = std::unordered_map<int, struct sigaction>();
16 static bool sighandler_init =
false;
17 static void signal_handler(
int signum)
20 TRACE_STREAMER(TLVL_ERROR, &(
"SharedMemoryManager")[0], 0, 0, 0) <<
"A signal of type " << signum <<
" (" << std::string(strsignal(signum)) <<
") was caught by SharedMemoryManager. Detaching all Shared Memory segments, then proceeding with default handlers!";
21 for (
auto ii : instances)
32 pthread_sigmask(SIG_UNBLOCK, NULL, &set);
33 pthread_sigmask(SIG_UNBLOCK, &set, NULL);
35 TRACE_STREAMER(TLVL_ERROR, &(
"SharedMemoryManager")[0], 0, 0, 0) <<
"Calling default signal handler";
36 if (signum != SIGUSR2)
38 sigaction(signum, &old_actions[signum], NULL);
39 kill(getpid(), signum);
44 sigaction(SIGINT, &old_actions[SIGINT], NULL);
45 kill(getpid(), SIGINT);
57 requested_shm_parameters_.buffer_count = buffer_count;
58 requested_shm_parameters_.buffer_size = buffer_size;
59 requested_shm_parameters_.buffer_timeout_us = buffer_timeout_us;
60 requested_shm_parameters_.destructive_read_mode = destructive_read_mode;
62 instances.push_back(
this);
65 static std::mutex sighandler_mutex;
66 std::unique_lock<std::mutex> lk(sighandler_mutex);
70 sighandler_init =
true;
71 std::vector<int> signals = { SIGINT, SIGQUIT, SIGILL, SIGABRT, SIGFPE, SIGSEGV, SIGPIPE, SIGALRM, SIGTERM, SIGUSR2 };
72 for (
auto signal : signals)
74 struct sigaction old_action;
75 sigaction(signal, NULL, &old_action);
79 if (old_action.sa_handler != SIG_IGN)
81 struct sigaction action;
82 action.sa_handler = signal_handler;
83 sigemptyset(&action.sa_mask);
84 for (
auto sigblk : signals)
86 sigaddset(&action.sa_mask, sigblk);
91 sigaction(signal, &action, NULL);
92 old_actions[signal] = old_action;
100 TLOG(TLVL_DEBUG) <<
"~SharedMemoryManager called";
102 TLOG(TLVL_DEBUG) <<
"~SharedMemoryManager done";
109 if (manager_id_ == 0)
return;
112 auto start_time = std::chrono::steady_clock::now();
114 size_t shmSize = requested_shm_parameters_.buffer_count * (requested_shm_parameters_.buffer_size +
sizeof(ShmBuffer)) +
sizeof(ShmStruct);
116 shm_segment_id_ = shmget(shm_key_, shmSize, 0666);
117 if (shm_segment_id_ == -1 && requested_shm_parameters_.buffer_count > 0 && manager_id_ <= 0)
119 TLOG(TLVL_DEBUG) <<
"Creating shared memory segment with key 0x" << std::hex << shm_key_ <<
" and size " << shmSize;
120 shm_segment_id_ = shmget(shm_key_, shmSize, IPC_CREAT | 0666);
123 if (shm_segment_id_ == -1)
125 TLOG(TLVL_ERROR) <<
"Error creating shared memory segment with key 0x" << std::hex << shm_key_ <<
", errno=" << errno <<
" (" << strerror(errno) <<
")";
132 shm_segment_id_ = shmget(shm_key_, shmSize, 0666);
136 TLOG(TLVL_DEBUG) <<
"shm_key == 0x" << std::hex << shm_key_ <<
", shm_segment_id == " << shm_segment_id_;
138 if (shm_segment_id_ > -1)
141 <<
"Attached to shared memory segment with ID = " << shm_segment_id_
142 <<
" and size " << shmSize
144 shm_ptr_ = (ShmStruct*)shmat(shm_segment_id_, 0, 0);
146 <<
"Attached to shared memory segment at address "
147 << std::hex << (
void*)shm_ptr_ << std::dec;
148 if (shm_ptr_ && shm_ptr_ != (
void *)-1)
150 if (manager_id_ == 0)
152 if (shm_ptr_->ready_magic == 0xCAFE1111)
154 TLOG(TLVL_ERROR) <<
"Owner encountered already-initialized Shared Memory!";
157 TLOG(TLVL_DEBUG) <<
"Owner initializing Shared Memory";
158 shm_ptr_->next_id = 1;
159 shm_ptr_->next_sequence_id = 0;
160 shm_ptr_->reader_pos = 0;
161 shm_ptr_->writer_pos = 0;
162 shm_ptr_->buffer_size = requested_shm_parameters_.buffer_size;
163 shm_ptr_->buffer_count = requested_shm_parameters_.buffer_count;
164 shm_ptr_->buffer_timeout_us = requested_shm_parameters_.buffer_timeout_us;
165 shm_ptr_->destructive_read_mode = requested_shm_parameters_.destructive_read_mode;
167 for (
int ii = 0; ii < static_cast<int>(requested_shm_parameters_.buffer_count); ++ii)
169 getBufferInfo_(ii)->writePos = 0;
170 getBufferInfo_(ii)->readPos = 0;
171 getBufferInfo_(ii)->sem = BufferSemaphoreFlags::Empty;
172 getBufferInfo_(ii)->sem_id = -1;
176 shm_ptr_->ready_magic = 0xCAFE1111;
180 TLOG(TLVL_DEBUG) <<
"Waiting for owner to initalize Shared Memory";
181 while (shm_ptr_->ready_magic != 0xCAFE1111) { usleep(1000); }
182 TLOG(TLVL_DEBUG) <<
"Getting ID from Shared Memory";
184 shm_ptr_->lowest_seq_id_read = 0;
185 TLOG(TLVL_DEBUG) <<
"Getting Shared Memory Size parameters";
188 TLOG(TLVL_DEBUG) <<
"Initialization Complete: "
189 <<
"key: 0x" << std::hex << shm_key_
190 <<
", manager ID: " << manager_id_
191 <<
", Buffer size: " << std::to_string(shm_ptr_->buffer_size)
192 <<
", Buffer count: " << std::to_string(shm_ptr_->buffer_count);
197 TLOG(TLVL_ERROR) <<
"Failed to attach to shared memory segment "
203 TLOG(TLVL_ERROR) <<
"Failed to connect to shared memory segment with key 0x" << std::hex << shm_key_
204 <<
", errno = " << strerror(errno) <<
". Please check "
205 <<
"if a stale shared memory segment needs to "
206 <<
"be cleaned up. (ipcs, ipcrm -m <segId>)";
213 TLOG(13) <<
"GetBufferForReading BEGIN";
216 TraceLock lk(search_mutex_, 11,
"GetBufferForReadingSearch");
217 auto rp = shm_ptr_->reader_pos.load();
219 TLOG(13) <<
"GetBufferForReading lock acquired, scanning buffers";
224 ShmBuffer* buffer_ptr =
nullptr;
226 for (
auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
228 auto buffer = (ii + rp) % shm_ptr_->buffer_count;
231 TLOG(14) <<
"GetBufferForReading Checking if buffer " << buffer <<
" is stale";
234 auto buf = getBufferInfo_(buffer);
235 TLOG(14) <<
"GetBufferForReading: Buffer " << buffer <<
": sem=" << FlagToString(buf->sem) <<
" (expected " << FlagToString(BufferSemaphoreFlags::Full) <<
"), sem_id=" << buf->sem_id <<
" )";
236 if (buf->sem == BufferSemaphoreFlags::Full && (buf->sem_id == -1 || buf->sem_id == manager_id_) && buf->sequence_id > last_seen_id_)
238 if (buf->sequence_id < seqID)
241 seqID = buf->sequence_id;
247 if (!buffer_ptr || (buffer_ptr && buffer_ptr->sem_id != -1 && buffer_ptr->sem_id != manager_id_))
254 TLOG(13) <<
"GetBufferForReading Found buffer " << buffer_num;
255 buffer_ptr->sem_id = manager_id_;
256 buffer_ptr->sem = BufferSemaphoreFlags::Reading;
257 if (buffer_ptr->sem_id != manager_id_) {
continue; }
258 buffer_ptr->readPos = 0;
259 touchBuffer_(buffer_ptr);
260 if (buffer_ptr->sem_id != manager_id_) {
continue; }
261 if (shm_ptr_->destructive_read_mode && shm_ptr_->lowest_seq_id_read == last_seen_id_)
263 shm_ptr_->lowest_seq_id_read = seqID;
265 last_seen_id_ = seqID;
266 if (shm_ptr_->destructive_read_mode) shm_ptr_->reader_pos = (buffer_num + 1) % shm_ptr_->buffer_count;
271 TLOG(13) <<
"GetBufferForReading returning -1 because no buffers are ready";
277 TLOG(14) <<
"GetBufferForWriting BEGIN";
279 TraceLock lk(search_mutex_, 12,
"GetBufferForWritingSearch");
280 auto wp = shm_ptr_->writer_pos.load();
283 for (
auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
285 auto buffer = (ii + wp) % shm_ptr_->buffer_count;
289 auto buf = getBufferInfo_(buffer);
290 if (buf->sem == BufferSemaphoreFlags::Empty && buf->sem_id == -1)
292 buf->sem_id = manager_id_;
293 buf->sem = BufferSemaphoreFlags::Writing;
295 shm_ptr_->writer_pos = (buffer + 1) % shm_ptr_->buffer_count;
296 if (buf->sem_id != manager_id_)
continue;
297 buf->sequence_id = ++shm_ptr_->next_sequence_id;
299 TLOG(14) <<
"GetBufferForWriting returning " << buffer;
307 for (
auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
309 auto buffer = (ii + wp) % shm_ptr_->buffer_count;
313 auto buf = getBufferInfo_(buffer);
314 if (buf->sem == BufferSemaphoreFlags::Full)
316 buf->sem_id = manager_id_;
317 buf->sem = BufferSemaphoreFlags::Writing;
318 if (buf->sem_id != manager_id_)
continue;
319 buf->sequence_id = ++shm_ptr_->next_sequence_id;
321 shm_ptr_->writer_pos = (buffer + 1) % shm_ptr_->buffer_count;
323 TLOG(14) <<
"GetBufferForWriting returning " << buffer;
329 for (
auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
331 auto buffer = (ii + wp) % shm_ptr_->buffer_count;
335 auto buf = getBufferInfo_(buffer);
336 if (buf->sem == BufferSemaphoreFlags::Reading)
338 buf->sem_id = manager_id_;
339 buf->sem = BufferSemaphoreFlags::Writing;
340 if (buf->sem_id != manager_id_)
continue;
341 buf->sequence_id = ++shm_ptr_->next_sequence_id;
343 shm_ptr_->writer_pos = (buffer + 1) % shm_ptr_->buffer_count;
344 TLOG(14) <<
"GetBufferForWriting returning " << buffer;
351 TLOG(14) <<
"GetBufferForWriting Returning -1 because no buffers are ready";
357 if (!IsValid())
return 0;
358 TLOG(23) <<
"0x" << std::hex << shm_key_ <<
" ReadReadyCount BEGIN";
360 TraceLock lk(search_mutex_, 14,
"ReadReadyCountSearch");
362 for (
auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
364 TLOG(24) <<
"0x" << std::hex << shm_key_ << std::dec <<
" ReadReadyCount: Checking if buffer " << ii <<
" is stale.";
366 auto buf = getBufferInfo_(ii);
367 TLOG(25) <<
"0x" << std::hex << shm_key_ << std::dec <<
" ReadReadyCount: Buffer " << ii <<
": sem=" << FlagToString(buf->sem) <<
" (expected " << FlagToString(BufferSemaphoreFlags::Full) <<
"), sem_id=" << buf->sem_id <<
" )";
368 if (buf->sem == BufferSemaphoreFlags::Full && (buf->sem_id == -1 || buf->sem_id == manager_id_) && buf->sequence_id > last_seen_id_)
370 TLOG(26) <<
"0x" << std::hex << shm_key_ << std::dec <<
" ReadReadyCount: Buffer " << ii <<
" is either unowned or owned by this manager, and is marked full.";
379 if (!IsValid())
return 0;
381 TraceLock lk(search_mutex_, 15,
"WriteReadyCountSearch");
383 for (
auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
386 auto buf = getBufferInfo_(ii);
387 if ((buf->sem == BufferSemaphoreFlags::Empty && buf->sem_id == -1)
388 || (overwrite && buf->sem != BufferSemaphoreFlags::Writing))
399 std::deque<int> output;
400 if (!IsValid())
return output;
403 TraceLock lk(search_mutex_, 16,
"GetOwnedSearch");
404 for (
auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
406 auto buf = getBufferInfo_(ii);
407 if (buf->sem_id == manager_id_)
409 output.push_back(ii);
415 for (
auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
417 auto buf = getBufferInfo_(ii);
418 if (buf->sem_id == manager_id_)
420 output.push_back(ii);
431 TraceLock lk(buffer_mutexes_[buffer], 17,
"DataSizeBuffer" + std::to_string(buffer));
432 auto buf = getBufferInfo_(buffer);
434 return buf->writePos;
441 TraceLock lk(buffer_mutexes_[buffer], 18,
"ResetReadPosBuffer" + std::to_string(buffer));
442 auto buf = getBufferInfo_(buffer);
450 TraceLock lk(buffer_mutexes_[buffer], 18,
"ResetWritePosBuffer" + std::to_string(buffer));
451 auto buf = getBufferInfo_(buffer);
452 checkBuffer_(buf, BufferSemaphoreFlags::Writing);
460 TraceLock lk(buffer_mutexes_[buffer], 19,
"IncReadPosBuffer" + std::to_string(buffer));
461 auto buf = getBufferInfo_(buffer);
463 TLOG(15) <<
"IncrementReadPos: buffer= " << buffer <<
", readPos=" << std::to_string(buf->readPos) <<
", bytes read=" << std::to_string(read);
464 buf->readPos = buf->readPos + read;
465 TLOG(15) <<
"IncrementReadPos: buffer= " << buffer <<
", New readPos is " << std::to_string(buf->readPos);
466 if (read == 0) Detach(
true,
"LogicError",
"Cannot increment Read pos by 0! (buffer=" + std::to_string(buffer) +
", readPos=" + std::to_string(buf->readPos) +
", writePos=" + std::to_string(buf->writePos) +
")");
472 TraceLock lk(buffer_mutexes_[buffer], 20,
"IncWritePosBuffer" + std::to_string(buffer));
473 auto buf = getBufferInfo_(buffer);
475 TLOG(16) <<
"IncrementWritePos: buffer= " << buffer <<
", writePos=" << std::to_string(buf->writePos) <<
", bytes written=" << std::to_string(written);
476 buf->writePos += written;
477 TLOG(16) <<
"IncrementWritePos: buffer= " << buffer <<
", New writePos is " << std::to_string(buf->writePos);
478 if (written == 0) Detach(
true,
"LogicError",
"Cannot increment Write pos by 0!");
484 TraceLock lk(buffer_mutexes_[buffer], 21,
"MoreDataInBuffer" + std::to_string(buffer));
485 auto buf = getBufferInfo_(buffer);
486 TLOG(17) <<
"MoreDataInBuffer: buffer= " << buffer <<
", readPos=" << std::to_string(buf->readPos) <<
", writePos=" << std::to_string(buf->writePos);
487 return buf->readPos < buf->writePos;
493 TraceLock lk(buffer_mutexes_[buffer], 22,
"CheckBuffer" + std::to_string(buffer));
494 return checkBuffer_(getBufferInfo_(buffer), flags,
false);
500 TraceLock lk(buffer_mutexes_[buffer], 23,
"FillBuffer" + std::to_string(buffer));
501 auto shmBuf = getBufferInfo_(buffer);
502 touchBuffer_(shmBuf);
503 if (shmBuf->sem_id == manager_id_)
505 if (shmBuf->sem != BufferSemaphoreFlags::Full)
506 shmBuf->sem = BufferSemaphoreFlags::Full;
508 shmBuf->sem_id = destination;
514 TLOG(18) <<
"MarkBufferEmpty BEGIN";
516 TraceLock lk(buffer_mutexes_[buffer], 24,
"EmptyBuffer" + std::to_string(buffer));
517 auto shmBuf = getBufferInfo_(buffer);
520 checkBuffer_(shmBuf, BufferSemaphoreFlags::Reading,
true);
522 touchBuffer_(shmBuf);
525 shmBuf->sem = BufferSemaphoreFlags::Full;
527 if ((force && (manager_id_ == 0 || manager_id_ == shmBuf->sem_id)) || (!force && shm_ptr_->destructive_read_mode))
529 TLOG(18) <<
"MarkBufferEmpty Resetting buffer to Empty state";
530 shmBuf->writePos = 0;
531 shmBuf->sem = BufferSemaphoreFlags::Empty;
532 if (shm_ptr_->reader_pos == static_cast<unsigned>(buffer) && !shm_ptr_->destructive_read_mode)
534 TLOG(18) <<
"MarkBufferEmpty Broadcast mode; incrementing reader_pos";
535 shm_ptr_->reader_pos = (buffer + 1) % shm_ptr_->buffer_count;
539 TLOG(18) <<
"MarkBufferEmpty END";
545 TraceLock lk(buffer_mutexes_[buffer], 25,
"ResetBuffer" + std::to_string(buffer));
546 auto shmBuf = getBufferInfo_(buffer);
558 if (delta > 0xFFFFFFFF)
560 TLOG(TLVL_TRACE) <<
"Buffer has touch time in the future, setting it to current time and ignoring...";
564 if (shm_ptr_->buffer_timeout_us == 0 || delta <= shm_ptr_->buffer_timeout_us || shmBuf->sem == BufferSemaphoreFlags::Empty)
return false;
565 TLOG(27) <<
"Buffer " << buffer <<
" is stale, time=" <<
TimeUtils::gettimeofday_us() <<
", last touch=" << shmBuf->last_touch_time <<
", d=" << delta <<
", timeout=" << shm_ptr_->buffer_timeout_us;
567 if (shmBuf->sem_id == manager_id_ && shmBuf->sem == BufferSemaphoreFlags::Writing)
571 if (!shm_ptr_->destructive_read_mode && shmBuf->sem == BufferSemaphoreFlags::Full)
573 TLOG(TLVL_DEBUG) <<
"Resetting old broadcast mode buffer";
574 shmBuf->writePos = 0;
575 shmBuf->sem = BufferSemaphoreFlags::Empty;
577 if (shm_ptr_->reader_pos == static_cast<unsigned>(buffer)) shm_ptr_->reader_pos = (buffer + 1) % shm_ptr_->buffer_count;
581 if (shmBuf->sem_id != manager_id_ && shmBuf->sem == BufferSemaphoreFlags::Reading)
583 TLOG(TLVL_WARNING) <<
"Stale Read buffer ( " << delta <<
" / " << shm_ptr_->buffer_timeout_us <<
" us ) detected! Resetting...";
585 shmBuf->sem = BufferSemaphoreFlags::Full;
594 if (!IsValid())
return true;
596 struct shmid_ds info;
597 auto sts = shmctl(shm_segment_id_, IPC_STAT, &info);
600 TLOG(TLVL_TRACE) <<
"Error accessing Shared Memory info: " << errno <<
" (" << strerror(errno) <<
").";
604 if (info.shm_perm.mode & SHM_DEST)
606 TLOG(TLVL_INFO) <<
"Shared Memory marked for destruction. Probably an end-of-data condition!";
615 if (!IsValid())
return 0;
617 struct shmid_ds info;
618 auto sts = shmctl(shm_segment_id_, IPC_STAT, &info);
621 TLOG(TLVL_TRACE) <<
"Error accessing Shared Memory info: " << errno <<
" (" << strerror(errno) <<
").";
625 return info.shm_nattch;
630 TLOG(19) <<
"Write BEGIN";
632 TraceLock lk(buffer_mutexes_[buffer], 26,
"WriteBuffer" + std::to_string(buffer));
633 auto shmBuf = getBufferInfo_(buffer);
634 checkBuffer_(shmBuf, BufferSemaphoreFlags::Writing);
635 touchBuffer_(shmBuf);
636 TLOG(19) <<
"Buffer Write Pos is " << std::to_string(shmBuf->writePos) <<
", write size is " << std::to_string(size);
637 if (shmBuf->writePos + size > shm_ptr_->buffer_size) Detach(
true,
"SharedMemoryWrite",
"Attempted to write more data than fits into Shared Memory! \nRe-run with a larger buffer size!");
639 auto pos = GetWritePos(buffer);
640 memcpy(pos, data, size);
641 shmBuf->writePos = shmBuf->writePos + size;
642 if (shmBuf->sequence_id > last_seen_id_)
644 last_seen_id_ = shmBuf->sequence_id;
646 TLOG(19) <<
"Write END";
653 TraceLock lk(buffer_mutexes_[buffer], 27,
"ReadBuffer" + std::to_string(buffer));
654 auto shmBuf = getBufferInfo_(buffer);
655 checkBuffer_(shmBuf, BufferSemaphoreFlags::Reading);
656 touchBuffer_(shmBuf);
657 if (shmBuf->readPos + size > shm_ptr_->buffer_size) Detach(
true,
"SharedMemoryRead",
"Attempted to read more data than exists in Shared Memory!");
659 auto pos = GetReadPos(buffer);
660 memcpy(data, pos, size);
661 shmBuf->readPos += size;
662 touchBuffer_(shmBuf);
663 return checkBuffer_(shmBuf, BufferSemaphoreFlags::Reading,
false);
668 std::ostringstream ostr;
669 ostr <<
"ShmStruct: " << std::endl
670 <<
"Reader Position: " << shm_ptr_->reader_pos << std::endl
671 <<
"Writer Position: " << shm_ptr_->writer_pos << std::endl
672 <<
"Next ID Number: " << shm_ptr_->next_id << std::endl
673 <<
"Buffer Count: " << shm_ptr_->buffer_count << std::endl
674 <<
"Buffer Size: " << std::to_string(shm_ptr_->buffer_size) <<
" bytes" << std::endl
675 <<
"Buffers Written: " << std::to_string(shm_ptr_->next_sequence_id) << std::endl
676 <<
"Rank of Writer: " << shm_ptr_->rank << std::endl
677 <<
"Ready Magic Bytes: 0x" << std::hex << shm_ptr_->ready_magic << std::endl << std::endl;
679 for (
auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
681 auto buf = getBufferInfo_(ii);
682 ostr <<
"ShmBuffer " << std::dec << ii << std::endl
683 <<
"sequenceID: " << std::to_string(buf->sequence_id) << std::endl
684 <<
"writePos: " << std::to_string(buf->writePos) << std::endl
685 <<
"readPos: " << std::to_string(buf->readPos) << std::endl
686 <<
"sem: " << FlagToString(buf->sem) << std::endl
687 <<
"Owner: " << std::to_string(buf->sem_id.load()) << std::endl
688 <<
"Last Touch Time: " << std::to_string(buf->last_touch_time / 1000000.0) << std::endl << std::endl;
696 auto buf = getBufferInfo_(buffer);
697 return bufferStart_(buffer) + buf->readPos;
701 auto buf = getBufferInfo_(buffer);
702 return bufferStart_(buffer) + buf->writePos;
707 return bufferStart_(buffer);
710 uint8_t* artdaq::SharedMemoryManager::dataStart_()
const
712 return reinterpret_cast<uint8_t*
>(shm_ptr_ + 1) + shm_ptr_->buffer_count *
sizeof(ShmBuffer);
715 uint8_t* artdaq::SharedMemoryManager::bufferStart_(
int buffer)
717 if (buffer >= shm_ptr_->buffer_count) Detach(
true,
"ArgumentOutOfRange",
"The specified buffer does not exist!");
718 return dataStart_() + buffer * shm_ptr_->buffer_size;
721 artdaq::SharedMemoryManager::ShmBuffer* artdaq::SharedMemoryManager::getBufferInfo_(
int buffer)
723 if (buffer >= shm_ptr_->buffer_count) Detach(
true,
"ArgumentOutOfRange",
"The specified buffer does not exist!");
724 return reinterpret_cast<ShmBuffer*
>(
reinterpret_cast<uint8_t*
>(shm_ptr_ + 1) + buffer *
sizeof(ShmBuffer));
727 bool artdaq::SharedMemoryManager::checkBuffer_(ShmBuffer* buffer, BufferSemaphoreFlags flags,
bool exceptions)
729 TLOG(TLVL_TRACE) <<
"checkBuffer_: Checking that buffer " << buffer->sequence_id <<
" has sem_id " << manager_id_ <<
" (Current: " << buffer->sem_id <<
") and is in state " << FlagToString(flags) <<
" (current: " << FlagToString(buffer->sem) <<
")";
732 if (buffer->sem != flags) Detach(
true,
"StateAccessViolation",
"Shared Memory buffer is not in the correct state! (expected " + FlagToString(flags) +
", actual " + FlagToString(buffer->sem) +
")");
733 if (buffer->sem_id != manager_id_) Detach(
true,
"OwnerAccessViolation",
"Shared Memory buffer is not owned by this manager instance! (Expected: " + std::to_string(manager_id_) +
", Actual: " + std::to_string(buffer->sem_id) +
")");
735 bool ret = (buffer->sem_id == manager_id_ || (buffer->sem_id == -1 && (flags == BufferSemaphoreFlags::Full || flags == BufferSemaphoreFlags::Empty))) && buffer->sem == flags;
739 TLOG(TLVL_WARNING) <<
"CheckBuffer detected issue with buffer " << std::to_string(buffer->sequence_id) <<
"!"
740 <<
" ID: " << buffer->sem_id <<
" (" << manager_id_ <<
"), Flag: " << FlagToString(buffer->sem) <<
" (" << FlagToString(flags) <<
"). "
741 <<
"ID -1 is okay if desired flag is \"Full\" or \"Empty\".";
747 void artdaq::SharedMemoryManager::touchBuffer_(ShmBuffer* buffer)
749 if (buffer->sem_id != manager_id_)
return;
750 TLOG(TLVL_TRACE) <<
"touchBuffer_: Touching buffer with sequence_id " << std::to_string(buffer->sequence_id);
756 TLOG(TLVL_DETACH) <<
"Detach BEGIN: throwException: " << std::boolalpha << throwException <<
", force: " << force;
759 TLOG(TLVL_DETACH) <<
"Detach: Resetting owned buffers";
760 auto bufs = GetBuffersOwnedByManager(
false);
761 for (
auto buf : bufs)
763 auto shmBuf = getBufferInfo_(buf);
764 if (shmBuf->sem == BufferSemaphoreFlags::Writing)
766 shmBuf->sem = BufferSemaphoreFlags::Empty;
768 else if (shmBuf->sem == BufferSemaphoreFlags::Reading)
770 shmBuf->sem = BufferSemaphoreFlags::Full;
778 TLOG(TLVL_DETACH) <<
"Detach: Detaching shared memory";
783 if ((force || manager_id_ == 0) && shm_segment_id_ > -1)
785 TLOG(TLVL_DETACH) <<
"Detach: Marking Shared memory for removal";
786 shmctl(shm_segment_id_, IPC_RMID, NULL);
787 shm_segment_id_ = -1;
790 if (category.size() > 0 && message.size() > 0)
792 TLOG(TLVL_ERROR) << category <<
": " << message;
796 throw cet::exception(category) << message;
bool Read(int buffer, void *data, size_t size)
Read size bytes of data from buffer into the given pointer.
void IncrementReadPos(int buffer, size_t read)
Increment the read position for a given buffer.
constexpr size_t GetElapsedTimeMilliseconds(std::chrono::steady_clock::time_point then, std::chrono::steady_clock::time_point now=std::chrono::steady_clock::now())
Gets the number of milliseconds in the given time interval
std::deque< int > GetBuffersOwnedByManager(bool locked=true)
Get the list of all buffers currently owned by this manager instance.
void ResetReadPos(int buffer)
Set the read position of the given buffer to the beginning of the buffer.
bool MoreDataInBuffer(int buffer)
Determine if more data is available to be read, based on the read position and data size...
virtual std::string toString()
Write information about the SharedMemory to a string.
BufferSemaphoreFlags
The BufferSemaphoreFlags enumeration represents the different possible "states" of a given shared mem...
size_t BufferDataSize(int buffer)
Get the current size of the buffer's data.
void Attach()
Reconnect to the shared memory segment.
void Detach(bool throwException=false, std::string category="", std::string message="", bool force=false)
Detach from the Shared Memory segment, optionally throwing a cet::exception with the specified proper...
virtual ~SharedMemoryManager() noexcept
SharedMemoryManager Destructor.
The SharedMemoryManager creates a Shared Memory area which is divided into a number of fixed-size buf...
size_t ReadReadyCount()
Count the number of buffers that are ready for reading.
bool ResetBuffer(int buffer)
Resets the buffer from Reading to Full. This operation will only have an effect if performed by the o...
bool IsEndOfData() const
Determine whether the Shared Memory is marked for destruction (End of Data)
bool CheckBuffer(int buffer, BufferSemaphoreFlags flags)
Check both semaphore conditions (Mode flag and manager ID) for a given buffer.
void IncrementWritePos(int buffer, size_t written)
Increment the write position for a given buffer.
The TraceLock class allows a user to debug the acquisition and releasing of locks, by wrapping the unique_lock<std::mutex> API with TRACE calls.
void MarkBufferFull(int buffer, int destination=-1)
Release a buffer from a writer, marking it Full and ready for a reader.
void MarkBufferEmpty(int buffer, bool force=false)
Release a buffer from a reader, marking it Empty and ready to accept more data.
void ResetWritePos(int buffer)
Set the write position of the given buffer to the beginning of the buffer.
void * GetWritePos(int buffer)
Get a pointer to the current write position of the buffer.
size_t Write(int buffer, void *data, size_t size)
Write size bytes of data from the given pointer to a buffer.
void * GetReadPos(int buffer)
Get a pointer to the current read position of the buffer.
void * GetBufferStart(int buffer)
Get a pointer to the start position of the buffer.
SharedMemoryManager(uint32_t shm_key, size_t buffer_count=0, size_t buffer_size=0, uint64_t buffer_timeout_us=100 *1000000, bool destructive_read_mode=true)
SharedMemoryManager Constructor.
int GetBufferForWriting(bool overwrite)
Finds a buffer that is ready to be written to, and reserves it for the calling manager.
uint16_t GetAttachedCount() const
Get the number of attached SharedMemoryManagers.
size_t WriteReadyCount(bool overwrite)
Count the number of buffers that are ready for writing.
uint64_t gettimeofday_us()
Get the current time of day in microseconds (from gettimeofday system call)
int GetBufferForReading()
Finds a buffer that is ready to be read, and reserves it for the calling manager. ...