1 #define TRACE_NAME "SharedMemoryManager"
6 #ifndef SHM_DEST // Lynn reports that this is missing on Mac OS X?!?
11 #include "cetlib_except/exception.h"
12 #include "artdaq-core/Core/SharedMemoryManager.hh"
13 #include "artdaq-core/Utilities/TraceLock.hh"
15 #define TLVL_DETACH 11
17 static std::vector<artdaq::SharedMemoryManager const*> instances = std::vector<artdaq::SharedMemoryManager const*>();
19 static std::unordered_map<int, struct sigaction> old_actions = std::unordered_map<int, struct sigaction>();
20 static bool sighandler_init =
false;
21 static void signal_handler(
int signum)
24 TRACE_STREAMER(TLVL_ERROR, &(
"SharedMemoryManager")[0], 0, 0, 0) <<
"A signal of type " << signum <<
" (" << std::string(strsignal(signum)) <<
") was caught by SharedMemoryManager. Detaching all Shared Memory segments, then proceeding with default handlers!";
25 for (
auto ii : instances)
36 pthread_sigmask(SIG_UNBLOCK, NULL, &set);
37 pthread_sigmask(SIG_UNBLOCK, &set, NULL);
39 TRACE_STREAMER(TLVL_ERROR, &(
"SharedMemoryManager")[0], 0, 0, 0) <<
"Calling default signal handler";
40 if (signum != SIGUSR2)
42 sigaction(signum, &old_actions[signum], NULL);
43 kill(getpid(), signum);
48 sigaction(SIGINT, &old_actions[SIGINT], NULL);
49 kill(getpid(), SIGINT);
61 requested_shm_parameters_.buffer_count = buffer_count;
62 requested_shm_parameters_.buffer_size = buffer_size;
63 requested_shm_parameters_.buffer_timeout_us = buffer_timeout_us;
64 requested_shm_parameters_.destructive_read_mode = destructive_read_mode;
66 instances.push_back(
this);
69 static std::mutex sighandler_mutex;
70 std::unique_lock<std::mutex> lk(sighandler_mutex);
74 sighandler_init =
true;
75 std::vector<int> signals = { SIGINT, SIGQUIT, SIGILL, SIGABRT, SIGFPE, SIGSEGV, SIGPIPE, SIGALRM, SIGTERM, SIGUSR2 };
76 for (
auto signal : signals)
78 struct sigaction old_action;
79 sigaction(signal, NULL, &old_action);
83 if (old_action.sa_handler != SIG_IGN)
85 struct sigaction action;
86 action.sa_handler = signal_handler;
87 sigemptyset(&action.sa_mask);
88 for (
auto sigblk : signals)
90 sigaddset(&action.sa_mask, sigblk);
95 sigaction(signal, &action, NULL);
96 old_actions[signal] = old_action;
104 TLOG(TLVL_DEBUG) <<
"~SharedMemoryManager called";
106 TLOG(TLVL_DEBUG) <<
"~SharedMemoryManager done";
113 if (manager_id_ == 0)
return;
116 auto start_time = std::chrono::steady_clock::now();
118 size_t shmSize = requested_shm_parameters_.buffer_count * (requested_shm_parameters_.buffer_size +
sizeof(ShmBuffer)) +
sizeof(ShmStruct);
120 shm_segment_id_ = shmget(shm_key_, shmSize, 0666);
121 if (shm_segment_id_ == -1 && requested_shm_parameters_.buffer_count > 0 && manager_id_ <= 0)
123 TLOG(TLVL_DEBUG) <<
"Creating shared memory segment with key 0x" << std::hex << shm_key_ <<
" and size " << shmSize;
124 shm_segment_id_ = shmget(shm_key_, shmSize, IPC_CREAT | 0666);
127 if (shm_segment_id_ == -1)
129 TLOG(TLVL_ERROR) <<
"Error creating shared memory segment with key 0x" << std::hex << shm_key_ <<
", errno=" << errno <<
" (" << strerror(errno) <<
")";
136 shm_segment_id_ = shmget(shm_key_, shmSize, 0666);
140 TLOG(TLVL_DEBUG) <<
"shm_key == 0x" << std::hex << shm_key_ <<
", shm_segment_id == " << shm_segment_id_;
142 if (shm_segment_id_ > -1)
145 <<
"Attached to shared memory segment with ID = " << shm_segment_id_
146 <<
" and size " << shmSize
148 shm_ptr_ = (ShmStruct*)shmat(shm_segment_id_, 0, 0);
150 <<
"Attached to shared memory segment at address "
151 << std::hex << (
void*)shm_ptr_ << std::dec;
152 if (shm_ptr_ && shm_ptr_ != (
void *)-1)
154 if (manager_id_ == 0)
156 if (shm_ptr_->ready_magic == 0xCAFE1111)
158 TLOG(TLVL_ERROR) <<
"Owner encountered already-initialized Shared Memory!";
161 TLOG(TLVL_DEBUG) <<
"Owner initializing Shared Memory";
162 shm_ptr_->next_id = 1;
163 shm_ptr_->next_sequence_id = 0;
164 shm_ptr_->reader_pos = 0;
165 shm_ptr_->writer_pos = 0;
166 shm_ptr_->buffer_size = requested_shm_parameters_.buffer_size;
167 shm_ptr_->buffer_count = requested_shm_parameters_.buffer_count;
168 shm_ptr_->buffer_timeout_us = requested_shm_parameters_.buffer_timeout_us;
169 shm_ptr_->destructive_read_mode = requested_shm_parameters_.destructive_read_mode;
171 for (
int ii = 0; ii < static_cast<int>(requested_shm_parameters_.buffer_count); ++ii)
173 getBufferInfo_(ii)->writePos = 0;
174 getBufferInfo_(ii)->readPos = 0;
175 getBufferInfo_(ii)->sem = BufferSemaphoreFlags::Empty;
176 getBufferInfo_(ii)->sem_id = -1;
180 shm_ptr_->ready_magic = 0xCAFE1111;
184 TLOG(TLVL_DEBUG) <<
"Waiting for owner to initalize Shared Memory";
185 while (shm_ptr_->ready_magic != 0xCAFE1111) { usleep(1000); }
186 TLOG(TLVL_DEBUG) <<
"Getting ID from Shared Memory";
188 shm_ptr_->lowest_seq_id_read = 0;
189 TLOG(TLVL_DEBUG) <<
"Getting Shared Memory Size parameters";
192 TLOG(TLVL_DEBUG) <<
"Initialization Complete: "
193 <<
"key: 0x" << std::hex << shm_key_
194 <<
", manager ID: " << manager_id_
195 <<
", Buffer size: " << shm_ptr_->buffer_size
196 <<
", Buffer count: " << shm_ptr_->buffer_count;
201 TLOG(TLVL_ERROR) <<
"Failed to attach to shared memory segment "
207 TLOG(TLVL_ERROR) <<
"Failed to connect to shared memory segment with key 0x" << std::hex << shm_key_
208 <<
", errno = " << strerror(errno) <<
". Please check "
209 <<
"if a stale shared memory segment needs to "
210 <<
"be cleaned up. (ipcs, ipcrm -m <segId>)";
217 TLOG(13) <<
"GetBufferForReading BEGIN";
219 std::unique_lock<std::mutex> lk(search_mutex_);
221 auto rp = shm_ptr_->reader_pos.load();
223 TLOG(13) <<
"GetBufferForReading lock acquired, scanning buffers";
228 ShmBuffer* buffer_ptr =
nullptr;
230 for (
auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
232 auto buffer = (ii + rp) % shm_ptr_->buffer_count;
235 TLOG(14) <<
"GetBufferForReading Checking if buffer " << buffer <<
" is stale";
238 auto buf = getBufferInfo_(buffer);
239 TLOG(14) <<
"GetBufferForReading: Buffer " << buffer <<
": sem=" << FlagToString(buf->sem) <<
" (expected " << FlagToString(BufferSemaphoreFlags::Full) <<
"), sem_id=" << buf->sem_id <<
" )";
240 if (buf->sem == BufferSemaphoreFlags::Full && (buf->sem_id == -1 || buf->sem_id == manager_id_) && buf->sequence_id > last_seen_id_)
242 if (buf->sequence_id < seqID)
245 seqID = buf->sequence_id;
251 if (!buffer_ptr || (buffer_ptr && buffer_ptr->sem_id != -1 && buffer_ptr->sem_id != manager_id_))
258 TLOG(13) <<
"GetBufferForReading Found buffer " << buffer_num;
259 buffer_ptr->sem_id = manager_id_;
260 buffer_ptr->sem = BufferSemaphoreFlags::Reading;
261 if (buffer_ptr->sem_id != manager_id_) {
continue; }
262 buffer_ptr->readPos = 0;
263 touchBuffer_(buffer_ptr);
264 if (buffer_ptr->sem_id != manager_id_) {
continue; }
265 if (shm_ptr_->destructive_read_mode && shm_ptr_->lowest_seq_id_read == last_seen_id_)
267 shm_ptr_->lowest_seq_id_read = seqID;
269 last_seen_id_ = seqID;
270 if (shm_ptr_->destructive_read_mode) shm_ptr_->reader_pos = (buffer_num + 1) % shm_ptr_->buffer_count;
275 TLOG(13) <<
"GetBufferForReading returning -1 because no buffers are ready";
281 TLOG(14) <<
"GetBufferForWriting BEGIN";
282 std::unique_lock<std::mutex> lk(search_mutex_);
284 auto wp = shm_ptr_->writer_pos.load();
287 for (
auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
289 auto buffer = (ii + wp) % shm_ptr_->buffer_count;
293 auto buf = getBufferInfo_(buffer);
294 if (buf->sem == BufferSemaphoreFlags::Empty && buf->sem_id == -1)
296 buf->sem_id = manager_id_;
297 buf->sem = BufferSemaphoreFlags::Writing;
299 shm_ptr_->writer_pos = (buffer + 1) % shm_ptr_->buffer_count;
300 if (buf->sem_id != manager_id_)
continue;
301 buf->sequence_id = ++shm_ptr_->next_sequence_id;
303 TLOG(14) <<
"GetBufferForWriting returning " << buffer;
311 for (
auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
313 auto buffer = (ii + wp) % shm_ptr_->buffer_count;
317 auto buf = getBufferInfo_(buffer);
318 if (buf->sem == BufferSemaphoreFlags::Full)
320 buf->sem_id = manager_id_;
321 buf->sem = BufferSemaphoreFlags::Writing;
322 if (buf->sem_id != manager_id_)
continue;
323 buf->sequence_id = ++shm_ptr_->next_sequence_id;
325 shm_ptr_->writer_pos = (buffer + 1) % shm_ptr_->buffer_count;
327 TLOG(14) <<
"GetBufferForWriting returning " << buffer;
333 for (
auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
335 auto buffer = (ii + wp) % shm_ptr_->buffer_count;
339 auto buf = getBufferInfo_(buffer);
340 if (buf->sem == BufferSemaphoreFlags::Reading)
342 buf->sem_id = manager_id_;
343 buf->sem = BufferSemaphoreFlags::Writing;
344 if (buf->sem_id != manager_id_)
continue;
345 buf->sequence_id = ++shm_ptr_->next_sequence_id;
347 shm_ptr_->writer_pos = (buffer + 1) % shm_ptr_->buffer_count;
348 TLOG(14) <<
"GetBufferForWriting returning " << buffer;
355 TLOG(14) <<
"GetBufferForWriting Returning -1 because no buffers are ready";
361 if (!IsValid())
return 0;
362 TLOG(23) <<
"0x" << std::hex << shm_key_ <<
" ReadReadyCount BEGIN";
363 std::unique_lock<std::mutex> lk(search_mutex_);
366 for (
auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
368 TLOG(24) <<
"0x" << std::hex << shm_key_ << std::dec <<
" ReadReadyCount: Checking if buffer " << ii <<
" is stale.";
370 auto buf = getBufferInfo_(ii);
371 TLOG(25) <<
"0x" << std::hex << shm_key_ << std::dec <<
" ReadReadyCount: Buffer " << ii <<
": sem=" << FlagToString(buf->sem) <<
" (expected " << FlagToString(BufferSemaphoreFlags::Full) <<
"), sem_id=" << buf->sem_id <<
" )";
372 if (buf->sem == BufferSemaphoreFlags::Full && (buf->sem_id == -1 || buf->sem_id == manager_id_) && buf->sequence_id > last_seen_id_)
374 TLOG(26) <<
"0x" << std::hex << shm_key_ << std::dec <<
" ReadReadyCount: Buffer " << ii <<
" is either unowned or owned by this manager, and is marked full.";
383 if (!IsValid())
return 0;
384 std::unique_lock<std::mutex> lk(search_mutex_);
387 for (
auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
390 auto buf = getBufferInfo_(ii);
391 if ((buf->sem == BufferSemaphoreFlags::Empty && buf->sem_id == -1)
392 || (overwrite && buf->sem != BufferSemaphoreFlags::Writing))
402 if (!IsValid())
return false;
403 TLOG(23) <<
"0x" << std::hex << shm_key_ <<
" ReadyForRead BEGIN";
404 std::unique_lock<std::mutex> lk(search_mutex_);
407 auto rp = shm_ptr_->reader_pos.load();
409 for (
auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
411 auto buffer = (rp + ii) % shm_ptr_->buffer_count;
412 TLOG(24) <<
"0x" << std::hex << shm_key_ << std::dec <<
" ReadyForRead: Checking if buffer " << buffer <<
" is stale.";
414 auto buf = getBufferInfo_(buffer);
415 TLOG(25) <<
"0x" << std::hex << shm_key_ << std::dec <<
" ReadyForRead: Buffer " << buffer <<
": sem=" << FlagToString(buf->sem) <<
" (expected " << FlagToString(BufferSemaphoreFlags::Full) <<
"), sem_id=" << buf->sem_id <<
" )";
416 if (buf->sem == BufferSemaphoreFlags::Full && (buf->sem_id == -1 || buf->sem_id == manager_id_) && buf->sequence_id > last_seen_id_)
418 TLOG(26) <<
"0x" << std::hex << shm_key_ << std::dec <<
" ReadyForRead: Buffer " << buffer <<
" is either unowned or owned by this manager, and is marked full.";
427 if (!IsValid())
return false;
428 std::unique_lock<std::mutex> lk(search_mutex_);
431 auto wp = shm_ptr_->writer_pos.load();
433 for (
auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
435 auto buffer = (wp + ii) % shm_ptr_->buffer_count;
437 auto buf = getBufferInfo_(buffer);
438 if ((buf->sem == BufferSemaphoreFlags::Empty && buf->sem_id == -1)
439 || (overwrite && buf->sem != BufferSemaphoreFlags::Writing))
449 std::deque<int> output;
450 if (!IsValid())
return output;
453 std::unique_lock<std::mutex> lk(search_mutex_);
455 for (
auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
457 auto buf = getBufferInfo_(ii);
458 if (buf->sem_id == manager_id_)
460 output.push_back(ii);
466 for (
auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
468 auto buf = getBufferInfo_(ii);
469 if (buf->sem_id == manager_id_)
471 output.push_back(ii);
481 std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
483 auto buf = getBufferInfo_(buffer);
485 return buf->writePos;
491 std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
493 auto buf = getBufferInfo_(buffer);
500 std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
502 auto buf = getBufferInfo_(buffer);
503 checkBuffer_(buf, BufferSemaphoreFlags::Writing);
510 std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
512 auto buf = getBufferInfo_(buffer);
514 TLOG(15) <<
"IncrementReadPos: buffer= " << buffer <<
", readPos=" << buf->readPos <<
", bytes read=" << read;
515 buf->readPos = buf->readPos + read;
516 TLOG(15) <<
"IncrementReadPos: buffer= " << buffer <<
", New readPos is " << buf->readPos;
517 if (read == 0) Detach(
true,
"LogicError",
"Cannot increment Read pos by 0! (buffer=" + std::to_string(buffer) +
", readPos=" + std::to_string(buf->readPos) +
", writePos=" + std::to_string(buf->writePos) +
")");
522 std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
524 auto buf = getBufferInfo_(buffer);
526 if (buf->writePos + written > shm_ptr_->buffer_size)
528 TLOG(TLVL_ERROR) <<
"Requested write size is larger than the buffer size! (sz=" << std::hex << shm_ptr_->buffer_size <<
", cur + req=" << buf->writePos + written <<
")";
531 TLOG(16) <<
"IncrementWritePos: buffer= " << buffer <<
", writePos=" << buf->writePos <<
", bytes written=" << written;
532 buf->writePos += written;
533 TLOG(16) <<
"IncrementWritePos: buffer= " << buffer <<
", New writePos is " << buf->writePos;
534 if (written == 0) Detach(
true,
"LogicError",
"Cannot increment Write pos by 0!");
541 std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
543 auto buf = getBufferInfo_(buffer);
544 TLOG(17) <<
"MoreDataInBuffer: buffer= " << buffer <<
", readPos=" << std::to_string(buf->readPos) <<
", writePos=" << buf->writePos;
545 return buf->readPos < buf->writePos;
550 std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
552 return checkBuffer_(getBufferInfo_(buffer), flags,
false);
557 std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
559 auto shmBuf = getBufferInfo_(buffer);
560 touchBuffer_(shmBuf);
561 if (shmBuf->sem_id == manager_id_)
563 if (shmBuf->sem != BufferSemaphoreFlags::Full)
564 shmBuf->sem = BufferSemaphoreFlags::Full;
566 shmBuf->sem_id = destination;
572 TLOG(18) <<
"MarkBufferEmpty BEGIN";
573 std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
575 auto shmBuf = getBufferInfo_(buffer);
578 checkBuffer_(shmBuf, BufferSemaphoreFlags::Reading,
true);
580 touchBuffer_(shmBuf);
583 shmBuf->sem = BufferSemaphoreFlags::Full;
585 if ((force && (manager_id_ == 0 || manager_id_ == shmBuf->sem_id)) || (!force && shm_ptr_->destructive_read_mode))
587 TLOG(18) <<
"MarkBufferEmpty Resetting buffer to Empty state";
588 shmBuf->writePos = 0;
589 shmBuf->sem = BufferSemaphoreFlags::Empty;
590 if (shm_ptr_->reader_pos == static_cast<unsigned>(buffer) && !shm_ptr_->destructive_read_mode)
592 TLOG(18) <<
"MarkBufferEmpty Broadcast mode; incrementing reader_pos";
593 shm_ptr_->reader_pos = (buffer + 1) % shm_ptr_->buffer_count;
597 TLOG(18) <<
"MarkBufferEmpty END";
602 std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
604 auto shmBuf = getBufferInfo_(buffer);
616 if (delta > 0xFFFFFFFF)
618 TLOG(TLVL_TRACE) <<
"Buffer has touch time in the future, setting it to current time and ignoring...";
622 if (shm_ptr_->buffer_timeout_us == 0 || delta <= shm_ptr_->buffer_timeout_us || shmBuf->sem == BufferSemaphoreFlags::Empty)
return false;
623 TLOG(27) <<
"Buffer " << buffer <<
" is stale, time=" <<
TimeUtils::gettimeofday_us() <<
", last touch=" << shmBuf->last_touch_time <<
", d=" << delta <<
", timeout=" << shm_ptr_->buffer_timeout_us;
625 if (shmBuf->sem_id == manager_id_ && shmBuf->sem == BufferSemaphoreFlags::Writing)
629 if (!shm_ptr_->destructive_read_mode && shmBuf->sem == BufferSemaphoreFlags::Full)
631 TLOG(TLVL_DEBUG) <<
"Resetting old broadcast mode buffer";
632 shmBuf->writePos = 0;
633 shmBuf->sem = BufferSemaphoreFlags::Empty;
635 if (shm_ptr_->reader_pos == static_cast<unsigned>(buffer)) shm_ptr_->reader_pos = (buffer + 1) % shm_ptr_->buffer_count;
639 if (shmBuf->sem_id != manager_id_ && shmBuf->sem == BufferSemaphoreFlags::Reading)
641 TLOG(TLVL_WARNING) <<
"Stale Read buffer ( " << delta <<
" / " << shm_ptr_->buffer_timeout_us <<
" us ) detected! Resetting...";
643 shmBuf->sem = BufferSemaphoreFlags::Full;
652 if (!IsValid())
return true;
654 struct shmid_ds info;
655 auto sts = shmctl(shm_segment_id_, IPC_STAT, &info);
658 TLOG(TLVL_TRACE) <<
"Error accessing Shared Memory info: " << errno <<
" (" << strerror(errno) <<
").";
662 if (info.shm_perm.mode & SHM_DEST)
664 TLOG(TLVL_INFO) <<
"Shared Memory marked for destruction. Probably an end-of-data condition!";
673 if (!IsValid())
return 0;
675 struct shmid_ds info;
676 auto sts = shmctl(shm_segment_id_, IPC_STAT, &info);
679 TLOG(TLVL_TRACE) <<
"Error accessing Shared Memory info: " << errno <<
" (" << strerror(errno) <<
").";
683 return info.shm_nattch;
688 TLOG(19) <<
"Write BEGIN";
689 std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
691 auto shmBuf = getBufferInfo_(buffer);
692 checkBuffer_(shmBuf, BufferSemaphoreFlags::Writing);
693 touchBuffer_(shmBuf);
694 TLOG(19) <<
"Buffer Write Pos is " << shmBuf->writePos <<
", write size is " << size;
695 if (shmBuf->writePos + size > shm_ptr_->buffer_size) Detach(
true,
"SharedMemoryWrite",
"Attempted to write more data than fits into Shared Memory! \nRe-run with a larger buffer size!");
697 auto pos = GetWritePos(buffer);
698 memcpy(pos, data, size);
699 shmBuf->writePos = shmBuf->writePos + size;
700 if (shmBuf->sequence_id > last_seen_id_)
702 last_seen_id_ = shmBuf->sequence_id;
704 TLOG(19) <<
"Write END";
710 std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
712 auto shmBuf = getBufferInfo_(buffer);
713 checkBuffer_(shmBuf, BufferSemaphoreFlags::Reading);
714 touchBuffer_(shmBuf);
715 if (shmBuf->readPos + size > shm_ptr_->buffer_size) Detach(
true,
"SharedMemoryRead",
"Attempted to read more data than exists in Shared Memory!");
717 auto pos = GetReadPos(buffer);
718 memcpy(data, pos, size);
719 shmBuf->readPos += size;
720 touchBuffer_(shmBuf);
721 return checkBuffer_(shmBuf, BufferSemaphoreFlags::Reading,
false);
726 std::ostringstream ostr;
727 ostr <<
"ShmStruct: " << std::endl
728 <<
"Reader Position: " << shm_ptr_->reader_pos << std::endl
729 <<
"Writer Position: " << shm_ptr_->writer_pos << std::endl
730 <<
"Next ID Number: " << shm_ptr_->next_id << std::endl
731 <<
"Buffer Count: " << shm_ptr_->buffer_count << std::endl
732 <<
"Buffer Size: " << std::to_string(shm_ptr_->buffer_size) <<
" bytes" << std::endl
733 <<
"Buffers Written: " << std::to_string(shm_ptr_->next_sequence_id) << std::endl
734 <<
"Rank of Writer: " << shm_ptr_->rank << std::endl
735 <<
"Ready Magic Bytes: 0x" << std::hex << shm_ptr_->ready_magic << std::endl << std::endl;
737 for (
auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
739 auto buf = getBufferInfo_(ii);
740 ostr <<
"ShmBuffer " << std::dec << ii << std::endl
741 <<
"sequenceID: " << std::to_string(buf->sequence_id) << std::endl
742 <<
"writePos: " << std::to_string(buf->writePos) << std::endl
743 <<
"readPos: " << std::to_string(buf->readPos) << std::endl
744 <<
"sem: " << FlagToString(buf->sem) << std::endl
745 <<
"Owner: " << std::to_string(buf->sem_id.load()) << std::endl
746 <<
"Last Touch Time: " << std::to_string(buf->last_touch_time / 1000000.0) << std::endl << std::endl;
754 auto buf = getBufferInfo_(buffer);
755 return bufferStart_(buffer) + buf->readPos;
759 auto buf = getBufferInfo_(buffer);
760 return bufferStart_(buffer) + buf->writePos;
765 return bufferStart_(buffer);
768 uint8_t* artdaq::SharedMemoryManager::dataStart_()
const
770 return reinterpret_cast<uint8_t*
>(shm_ptr_ + 1) + shm_ptr_->buffer_count *
sizeof(ShmBuffer);
773 uint8_t* artdaq::SharedMemoryManager::bufferStart_(
int buffer)
775 if (buffer >= shm_ptr_->buffer_count) Detach(
true,
"ArgumentOutOfRange",
"The specified buffer does not exist!");
776 return dataStart_() + buffer * shm_ptr_->buffer_size;
779 artdaq::SharedMemoryManager::ShmBuffer* artdaq::SharedMemoryManager::getBufferInfo_(
int buffer)
781 if (buffer >= shm_ptr_->buffer_count) Detach(
true,
"ArgumentOutOfRange",
"The specified buffer does not exist!");
782 return reinterpret_cast<ShmBuffer*
>(
reinterpret_cast<uint8_t*
>(shm_ptr_ + 1) + buffer *
sizeof(ShmBuffer));
785 bool artdaq::SharedMemoryManager::checkBuffer_(ShmBuffer* buffer, BufferSemaphoreFlags flags,
bool exceptions)
787 TLOG(TLVL_TRACE) <<
"checkBuffer_: Checking that buffer " << buffer->sequence_id <<
" has sem_id " << manager_id_ <<
" (Current: " << buffer->sem_id <<
") and is in state " << FlagToString(flags) <<
" (current: " << FlagToString(buffer->sem) <<
")";
790 if (buffer->sem != flags) Detach(
true,
"StateAccessViolation",
"Shared Memory buffer is not in the correct state! (expected " + FlagToString(flags) +
", actual " + FlagToString(buffer->sem) +
")");
791 if (buffer->sem_id != manager_id_) Detach(
true,
"OwnerAccessViolation",
"Shared Memory buffer is not owned by this manager instance! (Expected: " + std::to_string(manager_id_) +
", Actual: " + std::to_string(buffer->sem_id) +
")");
793 bool ret = (buffer->sem_id == manager_id_ || (buffer->sem_id == -1 && (flags == BufferSemaphoreFlags::Full || flags == BufferSemaphoreFlags::Empty))) && buffer->sem == flags;
797 TLOG(TLVL_WARNING) <<
"CheckBuffer detected issue with buffer " << buffer->sequence_id <<
"!"
798 <<
" ID: " << buffer->sem_id <<
" (" << manager_id_ <<
"), Flag: " << FlagToString(buffer->sem) <<
" (" << FlagToString(flags) <<
"). "
799 <<
"ID -1 is okay if desired flag is \"Full\" or \"Empty\".";
805 void artdaq::SharedMemoryManager::touchBuffer_(ShmBuffer* buffer)
807 if (buffer->sem_id != manager_id_)
return;
808 TLOG(TLVL_TRACE) <<
"touchBuffer_: Touching buffer with sequence_id " << buffer->sequence_id;
814 TLOG(TLVL_DETACH) <<
"Detach BEGIN: throwException: " << std::boolalpha << throwException <<
", force: " << force;
817 TLOG(TLVL_DETACH) <<
"Detach: Resetting owned buffers";
818 auto bufs = GetBuffersOwnedByManager(
false);
819 for (
auto buf : bufs)
821 auto shmBuf = getBufferInfo_(buf);
822 if (shmBuf->sem == BufferSemaphoreFlags::Writing)
824 shmBuf->sem = BufferSemaphoreFlags::Empty;
826 else if (shmBuf->sem == BufferSemaphoreFlags::Reading)
828 shmBuf->sem = BufferSemaphoreFlags::Full;
836 TLOG(TLVL_DETACH) <<
"Detach: Detaching shared memory";
841 if ((force || manager_id_ == 0) && shm_segment_id_ > -1)
843 TLOG(TLVL_DETACH) <<
"Detach: Marking Shared memory for removal";
844 shmctl(shm_segment_id_, IPC_RMID, NULL);
845 shm_segment_id_ = -1;
848 if (category.size() > 0 && message.size() > 0)
850 TLOG(TLVL_ERROR) << category <<
": " << message;
854 throw cet::exception(category) << message;
bool Read(int buffer, void *data, size_t size)
Read size bytes of data from buffer into the given pointer.
void IncrementReadPos(int buffer, size_t read)
Increment the read position for a given buffer.
constexpr size_t GetElapsedTimeMilliseconds(std::chrono::steady_clock::time_point then, std::chrono::steady_clock::time_point now=std::chrono::steady_clock::now())
Gets the number of milliseconds in the given time interval
std::deque< int > GetBuffersOwnedByManager(bool locked=true)
Get the list of all buffers currently owned by this manager instance.
void ResetReadPos(int buffer)
Set the read position of the given buffer to the beginning of the buffer.
bool MoreDataInBuffer(int buffer)
Determine if more data is available to be read, based on the read position and data size...
virtual std::string toString()
Write information about the SharedMemory to a string.
BufferSemaphoreFlags
The BufferSemaphoreFlags enumeration represents the different possible "states" of a given shared mem...
size_t BufferDataSize(int buffer)
Get the current size of the buffer's data.
void Attach()
Reconnect to the shared memory segment.
void Detach(bool throwException=false, std::string category="", std::string message="", bool force=false)
Detach from the Shared Memory segment, optionally throwing a cet::exception with the specified proper...
virtual ~SharedMemoryManager() noexcept
SharedMemoryManager Destructor.
The SharedMemoryManager creates a Shared Memory area which is divided into a number of fixed-size buf...
size_t ReadReadyCount()
Count the number of buffers that are ready for reading.
bool ResetBuffer(int buffer)
Resets the buffer from Reading to Full. This operation will only have an effect if performed by the o...
bool IncrementWritePos(int buffer, size_t written)
Increment the write position for a given buffer.
bool IsEndOfData() const
Determine whether the Shared Memory is marked for destruction (End of Data)
bool CheckBuffer(int buffer, BufferSemaphoreFlags flags)
Check both semaphore conditions (Mode flag and manager ID) for a given buffer.
void MarkBufferFull(int buffer, int destination=-1)
Release a buffer from a writer, marking it Full and ready for a reader.
void MarkBufferEmpty(int buffer, bool force=false)
Release a buffer from a reader, marking it Empty and ready to accept more data.
void ResetWritePos(int buffer)
Set the write position of the given buffer to the beginning of the buffer.
void * GetWritePos(int buffer)
Get a pointer to the current write position of the buffer.
size_t Write(int buffer, void *data, size_t size)
Write size bytes of data from the given pointer to a buffer.
void * GetReadPos(int buffer)
Get a pointer to the current read position of the buffer.
void * GetBufferStart(int buffer)
Get a pointer to the start position of the buffer.
SharedMemoryManager(uint32_t shm_key, size_t buffer_count=0, size_t buffer_size=0, uint64_t buffer_timeout_us=100 *1000000, bool destructive_read_mode=true)
SharedMemoryManager Constructor.
bool ReadyForRead()
Whether any buffer is ready for read.
int GetBufferForWriting(bool overwrite)
Finds a buffer that is ready to be written to, and reserves it for the calling manager.
uint16_t GetAttachedCount() const
Get the number of attached SharedMemoryManagers.
size_t WriteReadyCount(bool overwrite)
Count the number of buffers that are ready for writing.
uint64_t gettimeofday_us()
Get the current time of day in microseconds (from gettimeofday system call)
int GetBufferForReading()
Finds a buffer that is ready to be read, and reserves it for the calling manager. ...
virtual bool ReadyForWrite(bool overwrite)
Whether any buffer is available for write.