00001 #define TRACE_NAME "SharedMemoryManager"
00002 #include <cstring>
00003 #include <unordered_map>
00004 #include <set>
00005 #include <sys/ipc.h>
00006 #include <sys/shm.h>
00007 #ifndef SHM_DEST // Lynn reports that this is missing on Mac OS X?!?
00008 #define SHM_DEST 01000
00009 #endif
00010 #include "tracemf.h"
00011 #include <signal.h>
00012 #include "cetlib_except/exception.h"
00013 #include "artdaq-core/Core/SharedMemoryManager.hh"
00014 #include "artdaq-core/Utilities/TraceLock.hh"
00015
00016 #define TLVL_DETACH 11
00017 #define TLVL_BUFFER 40
00018 #define TLVL_BUFLCK 41
00019
00020 static std::set<artdaq::SharedMemoryManager const*> instances = std::set<artdaq::SharedMemoryManager const*>();
00021
00022 static std::unordered_map<int, struct sigaction> old_actions = std::unordered_map<int, struct sigaction>();
00023 static bool sighandler_init = false;
00024 static void signal_handler(int signum)
00025 {
00026
00027 TRACE_STREAMER(TLVL_ERROR, &("SharedMemoryManager")[0], 0, 0, 0) << "A signal of type " << signum << " was caught by SharedMemoryManager. Detaching all Shared Memory segments, then proceeding with default handlers!";
00028 for (auto ii : instances)
00029 {
00030 if (ii)
00031 {
00032 const_cast<artdaq::SharedMemoryManager*>(ii)->Detach(false, "", ""
00033 , false
00034
00035 );
00036 }
00037 ii = nullptr;
00038 }
00039
00040 sigset_t set;
00041 pthread_sigmask(SIG_UNBLOCK, NULL, &set);
00042 pthread_sigmask(SIG_UNBLOCK, &set, NULL);
00043
00044 TRACE_STREAMER(TLVL_ERROR, &("SharedMemoryManager")[0], 0, 0, 0) << "Calling default signal handler";
00045 if (signum != SIGUSR2)
00046 {
00047 sigaction(signum, &old_actions[signum], NULL);
00048 kill(getpid(), signum);
00049 }
00050 else
00051 {
00052
00053 sigaction(SIGINT, &old_actions[SIGINT], NULL);
00054 kill(getpid(), SIGINT);
00055 }
00056 }
00057
00058 artdaq::SharedMemoryManager::SharedMemoryManager(uint32_t shm_key, size_t buffer_count, size_t buffer_size, uint64_t buffer_timeout_us, bool destructive_read_mode)
00059 : shm_segment_id_(-1)
00060 , shm_ptr_(NULL)
00061 , shm_key_(shm_key)
00062 , manager_id_(-1)
00063 , buffer_mutexes_()
00064 , last_seen_id_(0)
00065 {
00066 requested_shm_parameters_.buffer_count = buffer_count;
00067 requested_shm_parameters_.buffer_size = buffer_size;
00068 requested_shm_parameters_.buffer_timeout_us = buffer_timeout_us;
00069 requested_shm_parameters_.destructive_read_mode = destructive_read_mode;
00070
00071 instances.insert(this);
00072 Attach();
00073
00074 static std::mutex sighandler_mutex;
00075 std::unique_lock<std::mutex> lk(sighandler_mutex);
00076
00077 if (!sighandler_init)
00078 {
00079 sighandler_init = true;
00080 std::vector<int> signals = { SIGINT, SIGILL, SIGABRT, SIGFPE, SIGSEGV, SIGPIPE, SIGALRM, SIGTERM, SIGUSR2 };
00081 for (auto signal : signals)
00082 {
00083 struct sigaction old_action;
00084 sigaction(signal, NULL, &old_action);
00085
00086
00087
00088 if (old_action.sa_handler != SIG_IGN)
00089 {
00090 struct sigaction action;
00091 action.sa_handler = signal_handler;
00092 sigemptyset(&action.sa_mask);
00093 for (auto sigblk : signals)
00094 {
00095 sigaddset(&action.sa_mask, sigblk);
00096 }
00097 action.sa_flags = 0;
00098
00099
00100 sigaction(signal, &action, NULL);
00101 old_actions[signal] = old_action;
00102 }
00103 }
00104 }
00105 }
00106
00107 artdaq::SharedMemoryManager::~SharedMemoryManager() noexcept
00108 {
00109 instances.erase(this);
00110 TLOG(TLVL_DEBUG) << "~SharedMemoryManager called";
00111 Detach();
00112 TLOG(TLVL_DEBUG) << "~SharedMemoryManager done";
00113 }
00114
00115 void artdaq::SharedMemoryManager::Attach()
00116 {
00117 if (IsValid())
00118 {
00119 if (manager_id_ == 0) return;
00120 Detach();
00121 }
00122 auto start_time = std::chrono::steady_clock::now();
00123 last_seen_id_ = 0;
00124 size_t shmSize = requested_shm_parameters_.buffer_count * (requested_shm_parameters_.buffer_size + sizeof(ShmBuffer)) + sizeof(ShmStruct);
00125
00126 shm_segment_id_ = shmget(shm_key_, shmSize, 0666);
00127 if (shm_segment_id_ == -1 && requested_shm_parameters_.buffer_count > 0 && manager_id_ <= 0)
00128 {
00129 TLOG(TLVL_DEBUG) << "Creating shared memory segment with key 0x" << std::hex << shm_key_ << " and size " << std::dec << shmSize;
00130 shm_segment_id_ = shmget(shm_key_, shmSize, IPC_CREAT | 0666);
00131 manager_id_ = 0;
00132
00133 if (shm_segment_id_ == -1)
00134 {
00135 TLOG(TLVL_ERROR) << "Error creating shared memory segment with key 0x" << std::hex << shm_key_ << ", errno=" << errno << " (" << strerror(errno) << ")";
00136 }
00137 }
00138 else
00139 {
00140 while (shm_segment_id_ == -1 && TimeUtils::GetElapsedTimeMilliseconds(start_time) < 1000)
00141 {
00142 shm_segment_id_ = shmget(shm_key_, shmSize, 0666);
00143
00144 }
00145 }
00146 TLOG(TLVL_DEBUG) << "shm_key == 0x" << std::hex << shm_key_ << ", shm_segment_id == " << shm_segment_id_;
00147
00148 if (shm_segment_id_ > -1)
00149 {
00150 TLOG(TLVL_DEBUG)
00151 << "Attached to shared memory segment with ID = " << shm_segment_id_
00152 << " and size " << shmSize
00153 << " bytes";
00154 shm_ptr_ = (ShmStruct*)shmat(shm_segment_id_, 0, 0);
00155 TLOG(TLVL_DEBUG)
00156 << "Attached to shared memory segment at address "
00157 << std::hex << (void*)shm_ptr_ << std::dec;
00158 if (shm_ptr_ && shm_ptr_ != (void *)-1)
00159 {
00160 if (manager_id_ == 0)
00161 {
00162 if (shm_ptr_->ready_magic == 0xCAFE1111)
00163 {
00164 TLOG(TLVL_ERROR) << "Owner encountered already-initialized Shared Memory!";
00165 exit(-2);
00166 }
00167 TLOG(TLVL_DEBUG) << "Owner initializing Shared Memory";
00168 shm_ptr_->next_id = 1;
00169 shm_ptr_->next_sequence_id = 0;
00170 shm_ptr_->reader_pos = 0;
00171 shm_ptr_->writer_pos = 0;
00172 shm_ptr_->buffer_size = requested_shm_parameters_.buffer_size;
00173 shm_ptr_->buffer_count = requested_shm_parameters_.buffer_count;
00174 shm_ptr_->buffer_timeout_us = requested_shm_parameters_.buffer_timeout_us;
00175 shm_ptr_->destructive_read_mode = requested_shm_parameters_.destructive_read_mode;
00176
00177 for (int ii = 0; ii < static_cast<int>(requested_shm_parameters_.buffer_count); ++ii)
00178 {
00179 if (!getBufferInfo_(ii)) return;
00180 getBufferInfo_(ii)->writePos = 0;
00181 getBufferInfo_(ii)->readPos = 0;
00182 getBufferInfo_(ii)->sem = BufferSemaphoreFlags::Empty;
00183 getBufferInfo_(ii)->sem_id = -1;
00184 getBufferInfo_(ii)->last_touch_time = TimeUtils::gettimeofday_us();
00185 }
00186
00187 shm_ptr_->ready_magic = 0xCAFE1111;
00188 }
00189 else
00190 {
00191 TLOG(TLVL_DEBUG) << "Waiting for owner to initalize Shared Memory";
00192 while (shm_ptr_->ready_magic != 0xCAFE1111) { usleep(1000); }
00193 TLOG(TLVL_DEBUG) << "Getting ID from Shared Memory";
00194 GetNewId();
00195 shm_ptr_->lowest_seq_id_read = 0;
00196 TLOG(TLVL_DEBUG) << "Getting Shared Memory Size parameters";
00197 }
00198
00199 buffer_mutexes_ = std::vector<std::mutex>(shm_ptr_->buffer_count);
00200 TLOG(TLVL_DEBUG) << "Initialization Complete: "
00201 << "key: 0x" << std::hex << shm_key_
00202 << ", manager ID: " << std::dec << manager_id_
00203 << ", Buffer size: " << shm_ptr_->buffer_size
00204 << ", Buffer count: " << shm_ptr_->buffer_count;
00205 return;
00206 }
00207 else
00208 {
00209 TLOG(TLVL_ERROR) << "Failed to attach to shared memory segment "
00210 << shm_segment_id_;
00211 }
00212 }
00213 else
00214 {
00215 TLOG(TLVL_ERROR) << "Failed to connect to shared memory segment with key 0x" << std::hex << shm_key_
00216 << ", errno=" << errno << " (" << strerror(errno) << ")" << ". Please check "
00217 << "if a stale shared memory segment needs to "
00218 << "be cleaned up. (ipcs, ipcrm -m <segId>)";
00219 }
00220 return;
00221 }
00222
00223 int artdaq::SharedMemoryManager::GetBufferForReading()
00224 {
00225 TLOG(13) << "GetBufferForReading BEGIN";
00226
00227 std::unique_lock<std::mutex> lk(search_mutex_);
00228
00229 auto rp = shm_ptr_->reader_pos.load();
00230
00231 TLOG(13) << "GetBufferForReading lock acquired, scanning " << shm_ptr_->buffer_count << " buffers";
00232
00233 for (int retry = 0; retry < 5; retry++)
00234 {
00235 BufferSemaphoreFlags sem;
00236 short sem_id;
00237 int buffer_num = -1;
00238 ShmBuffer* buffer_ptr = nullptr;
00239 uint64_t seqID = -1;
00240
00241 for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
00242 {
00243 auto buffer = (ii + rp) % shm_ptr_->buffer_count;
00244
00245
00246 TLOG(14) << "GetBufferForReading Checking if buffer " << buffer << " is stale. Shm destructive_read_mode=" << shm_ptr_->destructive_read_mode;
00247 ResetBuffer(buffer);
00248
00249 auto buf = getBufferInfo_(buffer);
00250 if (!buf) continue;
00251
00252 sem = buf->sem.load();
00253 sem_id = buf->sem_id.load();
00254
00255 TLOG(14) << "GetBufferForReading: Buffer " << buffer << ": sem=" << FlagToString(sem)
00256 << " (expected " << FlagToString(BufferSemaphoreFlags::Full) << "), sem_id=" << sem_id << ", seq_id=" << buf->sequence_id << " )";
00257 if (sem == BufferSemaphoreFlags::Full && (sem_id == -1 || sem_id == manager_id_)
00258 && (shm_ptr_->destructive_read_mode || buf->sequence_id > last_seen_id_))
00259 {
00260 if (buf->sequence_id < seqID)
00261 {
00262 buffer_ptr = buf;
00263 seqID = buf->sequence_id;
00264 buffer_num = buffer;
00265 touchBuffer_(buf);
00266 if (shm_ptr_->destructive_read_mode || seqID == last_seen_id_ + 1) break;
00267 }
00268 }
00269 }
00270
00271 if (buffer_ptr)
00272 {
00273 sem = buffer_ptr->sem.load();
00274 sem_id = buffer_ptr->sem_id.load();
00275 }
00276
00277 if (!buffer_ptr || (sem_id != -1 && sem_id != manager_id_) || sem != BufferSemaphoreFlags::Full)
00278 {
00279 continue;
00280 }
00281
00282 if (buffer_num >= 0)
00283 {
00284 TLOG(13) << "GetBufferForReading Found buffer " << buffer_num;
00285 touchBuffer_(buffer_ptr);
00286 if (!buffer_ptr->sem_id.compare_exchange_strong(sem_id, manager_id_)) continue;
00287 if (!buffer_ptr->sem.compare_exchange_strong(sem, BufferSemaphoreFlags::Reading)) continue;
00288 if (!checkBuffer_(buffer_ptr, BufferSemaphoreFlags::Reading, false)) {
00289 TLOG(13) << "GetBufferForReading: Failed to acquire buffer " << buffer_num << " (someone else changed manager ID while I was changing sem)";
00290 continue;
00291 }
00292 buffer_ptr->readPos = 0;
00293 touchBuffer_(buffer_ptr);
00294 if (!checkBuffer_(buffer_ptr, BufferSemaphoreFlags::Reading, false)) {
00295 TLOG(13) << "GetBufferForReading: Failed to acquire buffer " << buffer_num << " (someone else changed manager ID while I was touching buffer SHOULD NOT HAPPEN!)";
00296 continue;
00297 }
00298 if (shm_ptr_->destructive_read_mode && shm_ptr_->lowest_seq_id_read == last_seen_id_)
00299 {
00300 shm_ptr_->lowest_seq_id_read = seqID;
00301 }
00302 last_seen_id_ = seqID;
00303 if (shm_ptr_->destructive_read_mode) shm_ptr_->reader_pos = (buffer_num + 1) % shm_ptr_->buffer_count;
00304
00305 TLOG(13) << "GetBufferForReading returning " << buffer_num;
00306 return buffer_num;
00307 }
00308 retry = 5;
00309 }
00310
00311 TLOG(13) << "GetBufferForReading returning -1 because no buffers are ready";
00312 return -1;
00313 }
00314
00315 int artdaq::SharedMemoryManager::GetBufferForWriting(bool overwrite)
00316 {
00317 TLOG(14) << "GetBufferForWriting BEGIN, overwrite=" << (overwrite ? "true" : "false");
00318
00319 std::unique_lock<std::mutex> lk(search_mutex_);
00320
00321 auto wp = shm_ptr_->writer_pos.load();
00322
00323 TLOG(13) << "GetBufferForWriting lock acquired, scanning " << shm_ptr_->buffer_count << " buffers";
00324
00325
00326 for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
00327 {
00328 auto buffer = (ii + wp) % shm_ptr_->buffer_count;
00329
00330 ResetBuffer(buffer);
00331
00332 auto buf = getBufferInfo_(buffer);
00333 if (!buf) continue;
00334
00335 auto sem = buf->sem.load();
00336 auto sem_id = buf->sem_id.load();
00337
00338 if (sem == BufferSemaphoreFlags::Empty && sem_id == -1)
00339 {
00340 touchBuffer_(buf);
00341 if (!buf->sem_id.compare_exchange_strong(sem_id, manager_id_)) continue;
00342 if (!buf->sem.compare_exchange_strong(sem, BufferSemaphoreFlags::Writing)) continue;
00343 if (!checkBuffer_(buf, BufferSemaphoreFlags::Writing, false)) continue;
00344 shm_ptr_->writer_pos = (buffer + 1) % shm_ptr_->buffer_count;
00345 buf->sequence_id = ++shm_ptr_->next_sequence_id;
00346 buf->writePos = 0;
00347 if (!checkBuffer_(buf, BufferSemaphoreFlags::Writing, false)) continue;
00348 touchBuffer_(buf);
00349 TLOG(14) << "GetBufferForWriting returning " << buffer;
00350 return buffer;
00351 }
00352 }
00353
00354 if (overwrite)
00355 {
00356
00357 for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
00358 {
00359 auto buffer = (ii + wp) % shm_ptr_->buffer_count;
00360
00361 ResetBuffer(buffer);
00362
00363 auto buf = getBufferInfo_(buffer);
00364 if (!buf) continue;
00365
00366 auto sem = buf->sem.load();
00367 auto sem_id = buf->sem_id.load();
00368
00369 if (sem == BufferSemaphoreFlags::Full)
00370 {
00371 touchBuffer_(buf);
00372 if (!buf->sem_id.compare_exchange_strong(sem_id, manager_id_)) continue;
00373 if (!buf->sem.compare_exchange_strong(sem, BufferSemaphoreFlags::Writing)) continue;
00374 if (!checkBuffer_(buf, BufferSemaphoreFlags::Writing, false)) continue;
00375 shm_ptr_->writer_pos = (buffer + 1) % shm_ptr_->buffer_count;
00376 buf->sequence_id = ++shm_ptr_->next_sequence_id;
00377 buf->writePos = 0;
00378 if (!checkBuffer_(buf, BufferSemaphoreFlags::Writing, false)) continue;
00379 touchBuffer_(buf);
00380 TLOG(14) << "GetBufferForWriting returning " << buffer;
00381 return buffer;
00382 }
00383 }
00384
00385
00386 for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
00387 {
00388 auto buffer = (ii + wp) % shm_ptr_->buffer_count;
00389
00390 ResetBuffer(buffer);
00391
00392 auto buf = getBufferInfo_(buffer);
00393 if (!buf) continue;
00394
00395 auto sem = buf->sem.load();
00396 auto sem_id = buf->sem_id.load();
00397
00398 if (sem == BufferSemaphoreFlags::Reading)
00399 {
00400 touchBuffer_(buf);
00401 if (!buf->sem_id.compare_exchange_strong(sem_id, manager_id_)) continue;
00402 if (!buf->sem.compare_exchange_strong(sem, BufferSemaphoreFlags::Writing)) continue;
00403 if (!checkBuffer_(buf, BufferSemaphoreFlags::Writing, false)) continue;
00404 shm_ptr_->writer_pos = (buffer + 1) % shm_ptr_->buffer_count;
00405 buf->sequence_id = ++shm_ptr_->next_sequence_id;
00406 buf->writePos = 0;
00407 if (!checkBuffer_(buf, BufferSemaphoreFlags::Writing, false)) continue;
00408 touchBuffer_(buf);
00409 TLOG(14) << "GetBufferForWriting returning " << buffer;
00410 return buffer;
00411 }
00412 }
00413 }
00414 TLOG(14) << "GetBufferForWriting Returning -1 because no buffers are ready";
00415 return -1;
00416 }
00417
00418 size_t artdaq::SharedMemoryManager::ReadReadyCount()
00419 {
00420 if (!IsValid()) return 0;
00421 TLOG(23) << "0x" << std::hex << shm_key_ << " ReadReadyCount BEGIN";
00422 std::unique_lock<std::mutex> lk(search_mutex_);
00423 TLOG(23) << "ReadReadyCount lock acquired, scanning " << shm_ptr_->buffer_count << " buffers";
00424
00425 size_t count = 0;
00426 for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
00427 {
00428 TLOG(24) << "0x" << std::hex << shm_key_ << std::dec << " ReadReadyCount: Checking if buffer " << ii << " is stale.";
00429 ResetBuffer(ii);
00430 auto buf = getBufferInfo_(ii);
00431 if (!buf) continue;
00432 TLOG(25) << "0x" << std::hex << shm_key_ << std::dec << " ReadReadyCount: Buffer " << ii << ": sem=" << FlagToString(buf->sem) << " (expected " << FlagToString(BufferSemaphoreFlags::Full) << "), sem_id=" << buf->sem_id << " )";
00433 if (buf->sem == BufferSemaphoreFlags::Full && (buf->sem_id == -1 || buf->sem_id == manager_id_) && (shm_ptr_->destructive_read_mode || buf->sequence_id > last_seen_id_))
00434 {
00435 TLOG(26) << "0x" << std::hex << shm_key_ << std::dec << " ReadReadyCount: Buffer " << ii << " is either unowned or owned by this manager, and is marked full.";
00436 touchBuffer_(buf);
00437 ++count;
00438 }
00439 }
00440 return count;
00441 }
00442
00443 size_t artdaq::SharedMemoryManager::WriteReadyCount(bool overwrite)
00444 {
00445 if (!IsValid()) return 0;
00446 TLOG(28) << "0x" << std::hex << shm_key_ << " ReadReadyCount BEGIN";
00447 std::unique_lock<std::mutex> lk(search_mutex_);
00448
00449 TLOG(28) << "WriteReadyCount(" << overwrite << ") lock acquired, scanning " << shm_ptr_->buffer_count << " buffers";
00450 size_t count = 0;
00451 for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
00452 {
00453 TLOG(29) << "0x" << std::hex << shm_key_ << std::dec << " WriteReadyCount: Checking if buffer " << ii << " is stale.";
00454 ResetBuffer(ii);
00455 auto buf = getBufferInfo_(ii);
00456 if (!buf) continue;
00457 if ((buf->sem == BufferSemaphoreFlags::Empty && buf->sem_id == -1)
00458 || (overwrite && buf->sem != BufferSemaphoreFlags::Writing))
00459 {
00460 TLOG(29) << "0x" << std::hex << shm_key_ << std::dec << " WriteReadyCount: Buffer " << ii << " is either empty or is available for overwrite.";
00461 ++count;
00462 }
00463 }
00464 return count;
00465 }
00466
00467 bool artdaq::SharedMemoryManager::ReadyForRead()
00468 {
00469 if (!IsValid()) return false;
00470 TLOG(23) << "0x" << std::hex << shm_key_ << " ReadyForRead BEGIN";
00471 std::unique_lock<std::mutex> lk(search_mutex_);
00472
00473
00474 auto rp = shm_ptr_->reader_pos.load();
00475
00476 TLOG(23) << "ReadyForRead lock acquired, scanning " << shm_ptr_->buffer_count << " buffers";
00477
00478 for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
00479 {
00480 auto buffer = (rp + ii) % shm_ptr_->buffer_count;
00481 TLOG(24) << "0x" << std::hex << shm_key_ << std::dec << " ReadyForRead: Checking if buffer " << buffer << " is stale.";
00482 ResetBuffer(buffer);
00483 auto buf = getBufferInfo_(buffer);
00484 if (!buf) continue;
00485 TLOG(25) << "0x" << std::hex << shm_key_ << std::dec << " ReadyForRead: Buffer " << buffer << ": sem=" << FlagToString(buf->sem) << " (expected " << FlagToString(BufferSemaphoreFlags::Full) << "), sem_id=" << buf->sem_id << " )" << " seq_id=" << buf->sequence_id << " >? " << last_seen_id_;
00486 if (buf->sem == BufferSemaphoreFlags::Full && (buf->sem_id == -1 || buf->sem_id == manager_id_) && (shm_ptr_->destructive_read_mode || buf->sequence_id > last_seen_id_))
00487 {
00488 TLOG(26) << "0x" << std::hex << shm_key_ << std::dec << " ReadyForRead: Buffer " << buffer << " is either unowned or owned by this manager, and is marked full.";
00489 touchBuffer_(buf);
00490 return true;
00491 }
00492 }
00493 return false;
00494 }
00495
00496 bool artdaq::SharedMemoryManager::ReadyForWrite(bool overwrite)
00497 {
00498 if (!IsValid()) return false;
00499 TLOG(28) << "0x" << std::hex << shm_key_ << " ReadyForWrite BEGIN";
00500
00501 std::unique_lock<std::mutex> lk(search_mutex_);
00502
00503
00504 auto wp = shm_ptr_->writer_pos.load();
00505
00506 TLOG(28) << "ReadyForWrite lock acquired, scanning " << shm_ptr_->buffer_count << " buffers";
00507
00508 for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
00509 {
00510 auto buffer = (wp + ii) % shm_ptr_->buffer_count;
00511 TLOG(29) << "0x" << std::hex << shm_key_ << std::dec << " ReadyForWrite: Checking if buffer " << buffer << " is stale.";
00512 ResetBuffer(buffer);
00513 auto buf = getBufferInfo_(buffer);
00514 if (!buf) continue;
00515 if ((buf->sem == BufferSemaphoreFlags::Empty && buf->sem_id == -1)
00516 || (overwrite && buf->sem != BufferSemaphoreFlags::Writing))
00517 {
00518 TLOG(29) << "0x" << std::hex << shm_key_
00519 << std::dec
00520 << " WriteReadyCount: Buffer " << ii << " is either empty or available for overwrite.";
00521 return true;
00522 }
00523 }
00524 return false;
00525 }
00526
00527 std::deque<int> artdaq::SharedMemoryManager::GetBuffersOwnedByManager(bool locked)
00528 {
00529 std::deque<int> output;
00530 if (!IsValid()) return output;
00531 TLOG(TLVL_BUFFER) << "GetBuffersOwnedByManager BEGIN. Locked? " << locked;
00532 if (locked)
00533 {
00534 TLOG(TLVL_BUFLCK) << "GetBuffersOwnedByManager obtaining search_mutex";
00535 std::unique_lock<std::mutex> lk(search_mutex_);
00536 TLOG(TLVL_BUFLCK) << "GetBuffersOwnedByManager obtained search_mutex";
00537
00538 for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
00539 {
00540 auto buf = getBufferInfo_(ii);
00541 if (!buf) continue;
00542 if (buf->sem_id == manager_id_)
00543 {
00544 output.push_back(ii);
00545 }
00546 }
00547 }
00548 else
00549 {
00550 for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
00551 {
00552 auto buf = getBufferInfo_(ii);
00553 if (!buf) continue;
00554 if (buf->sem_id == manager_id_)
00555 {
00556 output.push_back(ii);
00557 }
00558 }
00559 }
00560
00561 TLOG(TLVL_BUFFER) << "GetBuffersOwnedByManager: own " << output.size() << " / " << shm_ptr_->buffer_count << " buffers.";
00562 return output;
00563 }
00564
00565 size_t artdaq::SharedMemoryManager::BufferDataSize(int buffer)
00566 {
00567 TLOG(TLVL_BUFFER) << "BufferDataSize(" << buffer << ") called.";
00568
00569 if (buffer >= shm_ptr_->buffer_count) Detach(true, "ArgumentOutOfRange", "The specified buffer does not exist!");
00570
00571 TLOG(TLVL_BUFLCK) << "BufferDataSize obtaining buffer_mutex for buffer " << buffer;
00572 std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
00573 TLOG(TLVL_BUFLCK) << "BufferDataSize obtained buffer_mutex for buffer " << buffer;
00574
00575
00576 auto buf = getBufferInfo_(buffer);
00577 if (!buf) return 0;
00578 touchBuffer_(buf);
00579
00580 TLOG(TLVL_BUFFER) << "BufferDataSize: buffer " << buffer << ", size=" << buf->writePos;
00581 return buf->writePos;
00582 }
00583
00584 void artdaq::SharedMemoryManager::ResetReadPos(int buffer)
00585 {
00586 TLOG(15) << "ResetReadPos(" << buffer << ") called.";
00587
00588 if (buffer >= shm_ptr_->buffer_count) Detach(true, "ArgumentOutOfRange", "The specified buffer does not exist!");
00589
00590 TLOG(TLVL_BUFLCK) << "ResetReadPos obtaining buffer_mutex for buffer " << buffer;
00591 std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
00592 TLOG(TLVL_BUFLCK) << "ResetReadPos obtained buffer_mutex for buffer " << buffer;
00593
00594
00595 auto buf = getBufferInfo_(buffer);
00596 if (!buf || buf->sem_id != manager_id_) return;
00597 touchBuffer_(buf);
00598 buf->readPos = 0;
00599
00600 TLOG(15) << "ResetReadPos(" << buffer << ") ended.";
00601 }
00602
00603 void artdaq::SharedMemoryManager::ResetWritePos(int buffer)
00604 {
00605 TLOG(16) << "ResetWritePos(" << buffer << ") called.";
00606
00607 if (buffer >= shm_ptr_->buffer_count) Detach(true, "ArgumentOutOfRange", "The specified buffer does not exist!");
00608
00609 TLOG(TLVL_BUFLCK) << "ResetWritePos obtaining buffer_mutex for buffer " << buffer;
00610 std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
00611 TLOG(TLVL_BUFLCK) << "ResetWritePos obtained buffer_mutex for buffer " << buffer;
00612
00613
00614 auto buf = getBufferInfo_(buffer);
00615 if (!buf) return;
00616 checkBuffer_(buf, BufferSemaphoreFlags::Writing);
00617 touchBuffer_(buf);
00618 buf->writePos = 0;
00619
00620 TLOG(16) << "ResetWritePos(" << buffer << ") ended.";
00621 }
00622
00623 void artdaq::SharedMemoryManager::IncrementReadPos(int buffer, size_t read)
00624 {
00625 TLOG(15) << "IncrementReadPos called: buffer= " << buffer << ", bytes to read=" << read;
00626
00627 if (buffer >= shm_ptr_->buffer_count) Detach(true, "ArgumentOutOfRange", "The specified buffer does not exist!");
00628
00629 TLOG(TLVL_BUFLCK) << "IncrementReadPos obtaining buffer_mutex for buffer " << buffer;
00630 std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
00631 TLOG(TLVL_BUFLCK) << "IncrementReadPos obtained buffer_mutex for buffer " << buffer;
00632
00633 auto buf = getBufferInfo_(buffer);
00634 if (!buf || buf->sem_id != manager_id_) return;
00635 touchBuffer_(buf);
00636 TLOG(15) << "IncrementReadPos: buffer= " << buffer << ", readPos=" << buf->readPos << ", bytes read=" << read;
00637 buf->readPos = buf->readPos + read;
00638 TLOG(15) << "IncrementReadPos: buffer= " << buffer << ", New readPos is " << buf->readPos;
00639 if (read == 0) Detach(true, "LogicError", "Cannot increment Read pos by 0! (buffer=" + std::to_string(buffer) + ", readPos=" + std::to_string(buf->readPos) + ", writePos=" + std::to_string(buf->writePos) + ")");
00640 }
00641
00642 bool artdaq::SharedMemoryManager::IncrementWritePos(int buffer, size_t written)
00643 {
00644 TLOG(16) << "IncrementWritePos called: buffer= " << buffer << ", bytes written=" << written;
00645
00646 if (buffer >= shm_ptr_->buffer_count) Detach(true, "ArgumentOutOfRange", "The specified buffer does not exist!");
00647
00648 TLOG(TLVL_BUFLCK) << "IncrementWritePos obtaining buffer_mutex for buffer " << buffer;
00649 std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
00650 TLOG(TLVL_BUFLCK) << "IncrementWritePos obtained buffer_mutex for buffer " << buffer;
00651
00652 auto buf = getBufferInfo_(buffer);
00653 if (!buf) return false;
00654 checkBuffer_(buf, BufferSemaphoreFlags::Writing);
00655 touchBuffer_(buf);
00656 if (buf->writePos + written > shm_ptr_->buffer_size)
00657 {
00658 TLOG(TLVL_ERROR) << "Requested write size is larger than the buffer size! (sz=" << std::hex << shm_ptr_->buffer_size << ", cur + req=" << buf->writePos + written << ")";
00659 return false;
00660 }
00661 TLOG(16) << "IncrementWritePos: buffer= " << buffer << ", writePos=" << buf->writePos << ", bytes written=" << written;
00662 buf->writePos += written;
00663 TLOG(16) << "IncrementWritePos: buffer= " << buffer << ", New writePos is " << buf->writePos;
00664 if (written == 0) Detach(true, "LogicError", "Cannot increment Write pos by 0!");
00665
00666 return true;
00667 }
00668
00669 bool artdaq::SharedMemoryManager::MoreDataInBuffer(int buffer)
00670 {
00671 TLOG(17) << "MoreDataInBuffer(" << buffer << ") called.";
00672
00673 if (buffer >= shm_ptr_->buffer_count) Detach(true, "ArgumentOutOfRange", "The specified buffer does not exist!");
00674
00675 TLOG(TLVL_BUFLCK) << "MoreDataInBuffer obtaining buffer_mutex for buffer " << buffer;
00676 std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
00677 TLOG(TLVL_BUFLCK) << "MoreDataInBuffer obtained buffer_mutex for buffer " << buffer;
00678
00679 auto buf = getBufferInfo_(buffer);
00680 if (!buf) return false;
00681 TLOG(17) << "MoreDataInBuffer: buffer= " << buffer << ", readPos=" << std::to_string(buf->readPos) << ", writePos=" << buf->writePos;
00682 return buf->readPos < buf->writePos;
00683 }
00684
00685 bool artdaq::SharedMemoryManager::CheckBuffer(int buffer, BufferSemaphoreFlags flags)
00686 {
00687 if (buffer >= shm_ptr_->buffer_count) Detach(true, "ArgumentOutOfRange", "The specified buffer does not exist!");
00688
00689 TLOG(TLVL_BUFLCK) << "CheckBuffer obtaining buffer_mutex for buffer " << buffer;
00690 std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
00691 TLOG(TLVL_BUFLCK) << "CheckBuffer obtained buffer_mutex for buffer " << buffer;
00692
00693 return checkBuffer_(getBufferInfo_(buffer), flags, false);
00694 }
00695
00696 void artdaq::SharedMemoryManager::MarkBufferFull(int buffer, int destination)
00697 {
00698 if (buffer >= shm_ptr_->buffer_count) Detach(true, "ArgumentOutOfRange", "The specified buffer does not exist!");
00699
00700 TLOG(TLVL_BUFLCK) << "MarkBufferFull obtaining buffer_mutex for buffer " << buffer;
00701 std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
00702 TLOG(TLVL_BUFLCK) << "MarkBufferFull obtained buffer_mutex for buffer " << buffer;
00703
00704
00705 auto shmBuf = getBufferInfo_(buffer);
00706 if (!shmBuf) return;
00707 touchBuffer_(shmBuf);
00708 if (shmBuf->sem_id == manager_id_)
00709 {
00710 if (shmBuf->sem != BufferSemaphoreFlags::Full)
00711 shmBuf->sem = BufferSemaphoreFlags::Full;
00712
00713 shmBuf->sem_id = destination;
00714 }
00715 }
00716
00717 void artdaq::SharedMemoryManager::MarkBufferEmpty(int buffer, bool force)
00718 {
00719 TLOG(18) << "MarkBufferEmpty BEGIN";
00720 if (buffer >= shm_ptr_->buffer_count) Detach(true, "ArgumentOutOfRange", "The specified buffer does not exist!");
00721 std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
00722
00723 auto shmBuf = getBufferInfo_(buffer);
00724 if (!shmBuf) return;
00725 if (!force)
00726 {
00727 checkBuffer_(shmBuf, BufferSemaphoreFlags::Reading, true);
00728 }
00729 touchBuffer_(shmBuf);
00730
00731 shmBuf->readPos = 0;
00732 shmBuf->sem = BufferSemaphoreFlags::Full;
00733
00734 if ((force && (manager_id_ == 0 || manager_id_ == shmBuf->sem_id)) || (!force && shm_ptr_->destructive_read_mode))
00735 {
00736 TLOG(18) << "MarkBufferEmpty Resetting buffer " << buffer << " to Empty state";
00737 shmBuf->writePos = 0;
00738 shmBuf->sem = BufferSemaphoreFlags::Empty;
00739 if (shm_ptr_->reader_pos == static_cast<unsigned>(buffer) && !shm_ptr_->destructive_read_mode)
00740 {
00741 TLOG(18) << "MarkBufferEmpty Broadcast mode; incrementing reader_pos from " << shm_ptr_->reader_pos << " to " << (buffer + 1) % shm_ptr_->buffer_count;
00742 shm_ptr_->reader_pos = (buffer + 1) % shm_ptr_->buffer_count;
00743 }
00744 }
00745 shmBuf->sem_id = -1;
00746 TLOG(18) << "MarkBufferEmpty END";
00747 }
00748
00749 bool artdaq::SharedMemoryManager::ResetBuffer(int buffer)
00750 {
00751 if (buffer >= shm_ptr_->buffer_count) Detach(true, "ArgumentOutOfRange", "The specified buffer does not exist!");
00752
00753 TLOG(TLVL_BUFLCK) << "ResetBuffer: obtaining buffer_mutex lock for buffer " << buffer;
00754 std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
00755 TLOG(TLVL_BUFLCK) << "ResetBuffer: obtained buffer_mutex lock for buffer " << buffer;
00756
00757
00758 auto shmBuf = getBufferInfo_(buffer);
00759 if (!shmBuf) return false;
00760
00761
00762
00763
00764
00765
00766
00767
00768
00769
00770 size_t delta = TimeUtils::gettimeofday_us() - shmBuf->last_touch_time;
00771 if (delta > 0xFFFFFFFF)
00772 {
00773 TLOG(TLVL_TRACE) << "Buffer has touch time in the future, setting it to current time and ignoring...";
00774 shmBuf->last_touch_time = TimeUtils::gettimeofday_us();
00775 return false;
00776 }
00777 if (shm_ptr_->buffer_timeout_us == 0 || delta <= shm_ptr_->buffer_timeout_us || shmBuf->sem == BufferSemaphoreFlags::Empty) return false;
00778 TLOG(27) << "Buffer " << buffer << " at " << (void*)shmBuf << " is stale, time=" << TimeUtils::gettimeofday_us() << ", last touch=" << shmBuf->last_touch_time << ", d=" << delta << ", timeout=" << shm_ptr_->buffer_timeout_us;
00779
00780 if (shmBuf->sem_id == manager_id_ && shmBuf->sem == BufferSemaphoreFlags::Writing)
00781 {
00782 return true;
00783 }
00784
00785 if (!shm_ptr_->destructive_read_mode && shmBuf->sem == BufferSemaphoreFlags::Full && (shmBuf->sequence_id < last_seen_id_ || manager_id_ == 0))
00786 {
00787 TLOG(TLVL_DEBUG) << "Resetting old broadcast mode buffer " << buffer << " (seqid=" << shmBuf->sequence_id << "). State: Full-->Empty";
00788 shmBuf->writePos = 0;
00789 shmBuf->sem = BufferSemaphoreFlags::Empty;
00790 shmBuf->sem_id = -1;
00791 if (shm_ptr_->reader_pos == static_cast<unsigned>(buffer)) shm_ptr_->reader_pos = (buffer + 1) % shm_ptr_->buffer_count;
00792 return true;
00793 }
00794
00795 if (shmBuf->sem_id != manager_id_ && shmBuf->sem == BufferSemaphoreFlags::Reading)
00796 {
00797
00798 size_t delta = TimeUtils::gettimeofday_us() - shmBuf->last_touch_time;
00799 if (delta <= shm_ptr_->buffer_timeout_us) return false;
00800 TLOG(TLVL_WARNING) << "Stale Read buffer " << buffer << " at " << (void*)shmBuf
00801 << " ( " << delta << " / " << shm_ptr_->buffer_timeout_us << " us ) detected! (seqid="
00802 << shmBuf->sequence_id << ") Resetting... Reading-->Full";
00803 shmBuf->readPos = 0;
00804 shmBuf->sem = BufferSemaphoreFlags::Full;
00805 shmBuf->sem_id = -1;
00806 return true;
00807 }
00808 return false;
00809 }
00810
00811 bool artdaq::SharedMemoryManager::IsEndOfData() const
00812 {
00813 if (!IsValid()) return true;
00814
00815 struct shmid_ds info;
00816 auto sts = shmctl(shm_segment_id_, IPC_STAT, &info);
00817 if (sts < 0)
00818 {
00819 TLOG(TLVL_TRACE) << "Error accessing Shared Memory info: " << errno << " (" << strerror(errno) << ").";
00820 return true;
00821 }
00822
00823 if (info.shm_perm.mode & SHM_DEST)
00824 {
00825 TLOG(TLVL_INFO) << "Shared Memory marked for destruction. Probably an end-of-data condition!";
00826 return true;
00827 }
00828
00829 return false;
00830 }
00831
00832 uint16_t artdaq::SharedMemoryManager::GetAttachedCount() const
00833 {
00834 if (!IsValid()) return 0;
00835
00836 struct shmid_ds info;
00837 auto sts = shmctl(shm_segment_id_, IPC_STAT, &info);
00838 if (sts < 0)
00839 {
00840 TLOG(TLVL_TRACE) << "Error accessing Shared Memory info: " << errno << " (" << strerror(errno) << ").";
00841 return 0;
00842 }
00843
00844 return info.shm_nattch;
00845 }
00846
00847 size_t artdaq::SharedMemoryManager::Write(int buffer, void* data, size_t size)
00848 {
00849 TLOG(19) << "Write BEGIN";
00850 if (buffer >= shm_ptr_->buffer_count) Detach(true, "ArgumentOutOfRange", "The specified buffer does not exist!");
00851 std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
00852
00853 auto shmBuf = getBufferInfo_(buffer);
00854 if (!shmBuf) return -1;
00855 checkBuffer_(shmBuf, BufferSemaphoreFlags::Writing);
00856 touchBuffer_(shmBuf);
00857 TLOG(19) << "Buffer Write Pos is " << shmBuf->writePos << ", write size is " << size;
00858 if (shmBuf->writePos + size > shm_ptr_->buffer_size)
00859 {
00860 TLOG(TLVL_ERROR) << "Attempted to write more data than fits into Shared Memory, bufferSize=" << shm_ptr_->buffer_size
00861 << ",writePos=" << shmBuf->writePos << ",writeSize=" << size;
00862 Detach(true, "SharedMemoryWrite", "Attempted to write more data than fits into Shared Memory! \nRe-run with a larger buffer size!");
00863 }
00864
00865 auto pos = GetWritePos(buffer);
00866 memcpy(pos, data, size);
00867 touchBuffer_(shmBuf);
00868 shmBuf->writePos = shmBuf->writePos + size;
00869
00870 auto last_seen = last_seen_id_.load();
00871 while (last_seen < shmBuf->sequence_id && !last_seen_id_.compare_exchange_weak(last_seen, shmBuf->sequence_id)) {}
00872
00873 TLOG(19) << "Write END";
00874 return size;
00875 }
00876
00877 bool artdaq::SharedMemoryManager::Read(int buffer, void* data, size_t size)
00878 {
00879 if (buffer >= shm_ptr_->buffer_count) Detach(true, "ArgumentOutOfRange", "The specified buffer does not exist!");
00880 std::unique_lock<std::mutex> lk(buffer_mutexes_[buffer]);
00881
00882 auto shmBuf = getBufferInfo_(buffer);
00883 if (!shmBuf) return false;
00884 checkBuffer_(shmBuf, BufferSemaphoreFlags::Reading);
00885 touchBuffer_(shmBuf);
00886 if (shmBuf->readPos + size > shm_ptr_->buffer_size)
00887 {
00888 TLOG(TLVL_ERROR) << "Attempted to read more data than fits into Shared Memory, bufferSize=" << shm_ptr_->buffer_size
00889 << ",readPos=" << shmBuf->readPos << ",readSize=" << size;
00890 Detach(true, "SharedMemoryRead", "Attempted to read more data than exists in Shared Memory!");
00891 }
00892
00893 auto pos = GetReadPos(buffer);
00894 memcpy(data, pos, size);
00895 auto sts = checkBuffer_(shmBuf, BufferSemaphoreFlags::Reading, false);
00896 if (sts)
00897 {
00898 shmBuf->readPos += size;
00899 touchBuffer_(shmBuf);
00900 return true;
00901 }
00902 return false;
00903 }
00904
00905 std::string artdaq::SharedMemoryManager::toString()
00906 {
00907 std::ostringstream ostr;
00908 ostr << "ShmStruct: " << std::endl
00909 << "Reader Position: " << shm_ptr_->reader_pos << std::endl
00910 << "Writer Position: " << shm_ptr_->writer_pos << std::endl
00911 << "Next ID Number: " << shm_ptr_->next_id << std::endl
00912 << "Buffer Count: " << shm_ptr_->buffer_count << std::endl
00913 << "Buffer Size: " << std::to_string(shm_ptr_->buffer_size) << " bytes" << std::endl
00914 << "Buffers Written: " << std::to_string(shm_ptr_->next_sequence_id) << std::endl
00915 << "Rank of Writer: " << shm_ptr_->rank << std::endl
00916 << "Ready Magic Bytes: 0x" << std::hex << shm_ptr_->ready_magic << std::endl << std::endl;
00917
00918 for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
00919 {
00920 auto buf = getBufferInfo_(ii);
00921 if (!buf) continue;
00922
00923 ostr << "ShmBuffer " << std::dec << ii << std::endl
00924 << "sequenceID: " << std::to_string(buf->sequence_id) << std::endl
00925 << "writePos: " << std::to_string(buf->writePos) << std::endl
00926 << "readPos: " << std::to_string(buf->readPos) << std::endl
00927 << "sem: " << FlagToString(buf->sem) << std::endl
00928 << "Owner: " << std::to_string(buf->sem_id.load()) << std::endl
00929 << "Last Touch Time: " << std::to_string(buf->last_touch_time / 1000000.0) << std::endl << std::endl;
00930 }
00931
00932 return ostr.str();
00933 }
00934
00935 void* artdaq::SharedMemoryManager::GetReadPos(int buffer)
00936 {
00937 auto buf = getBufferInfo_(buffer);
00938 if (!buf) return nullptr;
00939 return bufferStart_(buffer) + buf->readPos;
00940 }
00941 void* artdaq::SharedMemoryManager::GetWritePos(int buffer)
00942 {
00943 auto buf = getBufferInfo_(buffer);
00944 if (!buf) return nullptr;
00945 return bufferStart_(buffer) + buf->writePos;
00946 }
00947
00948 void* artdaq::SharedMemoryManager::GetBufferStart(int buffer)
00949 {
00950 return bufferStart_(buffer);
00951 }
00952
00953 std::list<std::pair<int, artdaq::SharedMemoryManager::BufferSemaphoreFlags>> artdaq::SharedMemoryManager::GetBufferReport()
00954 {
00955 auto output = std::list<std::pair<int, BufferSemaphoreFlags>>();
00956 for (size_t ii = 0; ii < size(); ++ii)
00957 {
00958 auto buf = getBufferInfo_(ii);
00959 output.emplace_back(std::make_pair(buf->sem_id.load(), buf->sem.load()));
00960 }
00961 return output;
00962 }
00963
00964 uint8_t* artdaq::SharedMemoryManager::dataStart_() const
00965 {
00966 if (shm_ptr_ == nullptr) return nullptr;
00967 return reinterpret_cast<uint8_t*>(shm_ptr_ + 1) + shm_ptr_->buffer_count * sizeof(ShmBuffer);
00968 }
00969
00970 uint8_t* artdaq::SharedMemoryManager::bufferStart_(int buffer)
00971 {
00972 if (shm_ptr_ == nullptr) return nullptr;
00973 if (buffer >= shm_ptr_->buffer_count) Detach(true, "ArgumentOutOfRange", "The specified buffer does not exist!");
00974 return dataStart_() + buffer * shm_ptr_->buffer_size;
00975 }
00976
00977 artdaq::SharedMemoryManager::ShmBuffer* artdaq::SharedMemoryManager::getBufferInfo_(int buffer)
00978 {
00979 if (shm_ptr_ == nullptr) return nullptr;
00980 if (buffer >= shm_ptr_->buffer_count) Detach(true, "ArgumentOutOfRange", "The specified buffer does not exist!");
00981 return reinterpret_cast<ShmBuffer*>(reinterpret_cast<uint8_t*>(shm_ptr_ + 1) + buffer * sizeof(ShmBuffer));
00982 }
00983
00984 bool artdaq::SharedMemoryManager::checkBuffer_(ShmBuffer* buffer, BufferSemaphoreFlags flags, bool exceptions)
00985 {
00986 if (!buffer)
00987 {
00988 if (exceptions)
00989 {
00990 Detach(true, "BufferNotThereException", "Request to check buffer that does not exist!");
00991 }
00992 return false;
00993 }
00994 TLOG(TLVL_TRACE) << "checkBuffer_: Checking that buffer " << buffer->sequence_id << " has sem_id " << manager_id_ << " (Current: " << buffer->sem_id << ") and is in state " << FlagToString(flags) << " (current: " << FlagToString(buffer->sem) << ")";
00995 if (exceptions)
00996 {
00997 if (buffer->sem != flags) Detach(true, "StateAccessViolation", "Shared Memory buffer is not in the correct state! (expected " + FlagToString(flags) + ", actual " + FlagToString(buffer->sem) + ")");
00998 if (buffer->sem_id != manager_id_) Detach(true, "OwnerAccessViolation", "Shared Memory buffer is not owned by this manager instance! (Expected: " + std::to_string(manager_id_) + ", Actual: " + std::to_string(buffer->sem_id) + ")");
00999 }
01000 bool ret = (buffer->sem_id == manager_id_ || (buffer->sem_id == -1 && (flags == BufferSemaphoreFlags::Full || flags == BufferSemaphoreFlags::Empty))) && buffer->sem == flags;
01001
01002 if (!ret)
01003 {
01004 TLOG(TLVL_WARNING) << "CheckBuffer detected issue with buffer " << buffer->sequence_id << "!"
01005 << " ID: " << buffer->sem_id << " (Expected " << manager_id_ << "), Flag: " << FlagToString(buffer->sem) << " (Expected " << FlagToString(flags) << "). "
01006 << "ID -1 is okay if expected flag is \"Full\" or \"Empty\".";
01007 }
01008
01009 return ret;
01010 }
01011
01012 void artdaq::SharedMemoryManager::touchBuffer_(ShmBuffer* buffer)
01013 {
01014 if (!buffer || (buffer->sem_id != -1 && buffer->sem_id != manager_id_)) return;
01015 TLOG(TLVL_TRACE) << "touchBuffer_: Touching buffer at " << (void*)buffer << " with sequence_id " << buffer->sequence_id;
01016 buffer->last_touch_time = TimeUtils::gettimeofday_us();
01017 }
01018
01019 void artdaq::SharedMemoryManager::Detach(bool throwException, std::string category, std::string message, bool force)
01020 {
01021 TLOG(TLVL_DETACH) << "Detach BEGIN: throwException: " << std::boolalpha << throwException << ", force: " << force;
01022 if (IsValid())
01023 {
01024 TLOG(TLVL_DETACH) << "Detach: Resetting owned buffers";
01025 auto bufs = GetBuffersOwnedByManager(false);
01026 for (auto buf : bufs)
01027 {
01028 auto shmBuf = getBufferInfo_(buf);
01029 if (!shmBuf) continue;
01030 if (shmBuf->sem == BufferSemaphoreFlags::Writing)
01031 {
01032 shmBuf->sem = BufferSemaphoreFlags::Empty;
01033 }
01034 else if (shmBuf->sem == BufferSemaphoreFlags::Reading)
01035 {
01036 shmBuf->sem = BufferSemaphoreFlags::Full;
01037 }
01038 shmBuf->sem_id = -1;
01039 }
01040 }
01041
01042 if (shm_ptr_)
01043 {
01044 TLOG(TLVL_DETACH) << "Detach: Detaching shared memory";
01045 shmdt(shm_ptr_);
01046 shm_ptr_ = NULL;
01047 }
01048
01049 if ((force || manager_id_ == 0) && shm_segment_id_ > -1)
01050 {
01051 TLOG(TLVL_DETACH) << "Detach: Marking Shared memory for removal";
01052 shmctl(shm_segment_id_, IPC_RMID, NULL);
01053 shm_segment_id_ = -1;
01054 }
01055
01056 if (category.size() > 0 && message.size() > 0)
01057 {
01058 TLOG(TLVL_ERROR) << category << ": " << message;
01059
01060 if (throwException)
01061 {
01062 throw cet::exception(category) << message;
01063 }
01064 }
01065 }
01066
01067
01068
01069
01070
01071