00001 #include <cstring>
00002 #include <sys/shm.h>
00003 #include "tracemf.h"
00004 #include "cetlib_except/exception.h"
00005 #include "artdaq-core/Core/SharedMemoryManager.hh"
00006 #include "artdaq-core/Utilities/TraceLock.hh"
00007
00008 #undef TRACE_NAME
00009 #define TRACE_NAME "SharedMemoryManager"
00010
00011
00012 artdaq::SharedMemoryManager::SharedMemoryManager(uint32_t shm_key, size_t buffer_count, size_t buffer_size, uint64_t buffer_timeout_us, bool destructive_read_mode)
00013 : shm_segment_id_(-1)
00014 , shm_ptr_(NULL)
00015 , shm_key_(shm_key)
00016 , manager_id_(-1)
00017 , buffer_mutexes_()
00018 , last_seen_id_(0)
00019 {
00020 requested_shm_parameters_.buffer_count = buffer_count;
00021 requested_shm_parameters_.buffer_size = buffer_size;
00022 requested_shm_parameters_.buffer_timeout_us = buffer_timeout_us;
00023 requested_shm_parameters_.destructive_read_mode = destructive_read_mode;
00024
00025 Attach();
00026 }
00027
00028 artdaq::SharedMemoryManager::~SharedMemoryManager() noexcept
00029 {
00030 TLOG(5) << "~SharedMemoryManager called" << TLOG_ENDL;
00031 Detach();
00032 TLOG(5) << "~SharedMemoryManager done" << TLOG_ENDL;
00033 }
00034
00035 void artdaq::SharedMemoryManager::Attach()
00036 {
00037 if (IsValid())
00038 {
00039 if (manager_id_ == 0) return;
00040 Detach();
00041 }
00042 auto start_time = std::chrono::steady_clock::now();
00043 last_seen_id_ = 0;
00044 size_t shmSize = requested_shm_parameters_.buffer_count * (requested_shm_parameters_.buffer_size + sizeof(ShmBuffer)) + sizeof(ShmStruct);
00045
00046 shm_segment_id_ = shmget(shm_key_, shmSize, 0666);
00047 if (shm_segment_id_ == -1 && requested_shm_parameters_.buffer_count > 0 && manager_id_ <= 0)
00048 {
00049 TLOG(TLVL_DEBUG) << "Creating shared memory segment with key 0x" << std::hex << shm_key_ << TLOG_ENDL;
00050 shm_segment_id_ = shmget(shm_key_, shmSize, IPC_CREAT | 0666);
00051 manager_id_ = 0;
00052 }
00053
00054 while (shm_segment_id_ == -1 && TimeUtils::GetElapsedTimeMilliseconds(start_time) < 1000)
00055 {
00056 shm_segment_id_ = shmget(shm_key_, shmSize, 0666);
00057
00058 }
00059 TLOG(TLVL_DEBUG) << "shm_key == 0x" << std::hex << shm_key_ << ", shm_segment_id == " << shm_segment_id_ << TLOG_ENDL;
00060
00061 if (shm_segment_id_ > -1)
00062 {
00063 TLOG(TLVL_DEBUG)
00064 << "Attached to shared memory segment with ID = " << shm_segment_id_
00065 << " and size " << shmSize
00066 << " bytes" << TLOG_ENDL;
00067 shm_ptr_ = (ShmStruct*)shmat(shm_segment_id_, 0, 0);
00068 TLOG(TLVL_DEBUG)
00069 << "Attached to shared memory segment at address "
00070 << std::hex << (void*)shm_ptr_ << std::dec << TLOG_ENDL;
00071 if (shm_ptr_ && shm_ptr_ != (void *)-1)
00072 {
00073 if (manager_id_ == 0)
00074 {
00075 if (shm_ptr_->ready_magic == 0xCAFE1111)
00076 {
00077 TLOG(TLVL_ERROR) << "Owner encountered already-initialized Shared Memory!" << TLOG_ENDL;
00078 exit(-2);
00079 }
00080 TLOG(TLVL_DEBUG) << "Owner initializing Shared Memory" << TLOG_ENDL;
00081 shm_ptr_->next_id = 1;
00082 shm_ptr_->next_sequence_id = 0;
00083 shm_ptr_->reader_pos = 0;
00084 shm_ptr_->writer_pos = 0;
00085 shm_ptr_->buffer_size = requested_shm_parameters_.buffer_size;
00086 shm_ptr_->buffer_count = requested_shm_parameters_.buffer_count;
00087 shm_ptr_->buffer_timeout_us = requested_shm_parameters_.buffer_timeout_us;
00088 shm_ptr_->destructive_read_mode = requested_shm_parameters_.destructive_read_mode;
00089
00090 for (int ii = 0; ii < static_cast<int>(requested_shm_parameters_.buffer_count); ++ii)
00091 {
00092 getBufferInfo_(ii)->writePos = 0;
00093 getBufferInfo_(ii)->readPos = 0;
00094 getBufferInfo_(ii)->sem = BufferSemaphoreFlags::Empty;
00095 getBufferInfo_(ii)->sem_id = -1;
00096 getBufferInfo_(ii)->last_touch_time = TimeUtils::gettimeofday_us();
00097 }
00098
00099 shm_ptr_->ready_magic = 0xCAFE1111;
00100 }
00101 else
00102 {
00103 TLOG(TLVL_DEBUG) << "Waiting for owner to initalize Shared Memory" << TLOG_ENDL;
00104 while (shm_ptr_->ready_magic != 0xCAFE1111) { usleep(1000); }
00105 TLOG(TLVL_DEBUG) << "Getting ID from Shared Memory" << TLOG_ENDL;
00106 GetNewId();
00107 shm_ptr_->lowest_seq_id_read = 0;
00108 TLOG(TLVL_DEBUG) << "Getting Shared Memory Size parameters" << TLOG_ENDL;
00109 }
00110
00111 TLOG(TLVL_DEBUG) << "Initialization Complete: "
00112 << "key: 0x" << std::hex << shm_key_
00113 << ", manager ID: " << manager_id_
00114 << ", Buffer size: " << std::to_string(shm_ptr_->buffer_size)
00115 << ", Buffer count: " << std::to_string(shm_ptr_->buffer_count) << TLOG_ENDL;
00116 return;
00117 }
00118 else
00119 {
00120 TLOG(TLVL_ERROR) << "Failed to attach to shared memory segment "
00121 << shm_segment_id_ << TLOG_ENDL;
00122 }
00123 }
00124 else
00125 {
00126 TLOG(TLVL_ERROR) << "Failed to connect to shared memory segment"
00127 << ", errno = " << strerror(errno) << ". Please check "
00128 << "if a stale shared memory segment needs to "
00129 << "be cleaned up. (ipcs, ipcrm -m <segId>)" << TLOG_ENDL;
00130 }
00131 return;
00132 }
00133
00134 int artdaq::SharedMemoryManager::GetBufferForReading()
00135 {
00136 TLOG(13) << "GetBufferForReading BEGIN" << TLOG_ENDL;
00137
00138
00139 TraceLock lk(search_mutex_, 11, "GetBufferForReadingSearch");
00140 auto rp = shm_ptr_->reader_pos.load();
00141
00142 TLOG(13) << "GetBufferForReading lock acquired, scanning buffers" << TLOG_ENDL;
00143 bool retry = true;
00144 int buffer_num = -1;
00145 while (retry)
00146 {
00147 ShmBuffer* buffer_ptr = nullptr;
00148 uint64_t seqID = -1;
00149 for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
00150 {
00151 auto buffer = (ii + rp) % shm_ptr_->buffer_count;
00152
00153
00154 TLOG(14) << "GetBufferForReading Checking if buffer " << buffer << " is stale" << TLOG_ENDL;
00155 ResetBuffer(buffer);
00156
00157 auto buf = getBufferInfo_(buffer);
00158 TLOG(14) << "GetBufferForReading: Buffer " << buffer << ": sem=" << FlagToString(buf->sem) << " (expected " << FlagToString(BufferSemaphoreFlags::Full) << "), sem_id=" << buf->sem_id << " )" << TLOG_ENDL;
00159 if (buf->sem == BufferSemaphoreFlags::Full && (buf->sem_id == -1 || buf->sem_id == manager_id_) && buf->sequence_id > last_seen_id_)
00160 {
00161 if (buf->sequence_id < seqID)
00162 {
00163 buffer_ptr = buf;
00164 seqID = buf->sequence_id;
00165 buffer_num = buffer;
00166 }
00167 }
00168 }
00169
00170 if (!buffer_ptr || (buffer_ptr && buffer_ptr->sem_id != -1 && buffer_ptr->sem_id != manager_id_))
00171 {
00172 continue;
00173 }
00174
00175 if (buffer_num >= 0)
00176 {
00177 TLOG(13) << "GetBufferForReading Found buffer " << buffer_num << TLOG_ENDL;
00178 buffer_ptr->sem_id = manager_id_;
00179 buffer_ptr->sem = BufferSemaphoreFlags::Reading;
00180 if (buffer_ptr->sem_id != manager_id_) { continue; }
00181 buffer_ptr->readPos = 0;
00182 touchBuffer_(buffer_ptr);
00183 if (buffer_ptr->sem_id != manager_id_) { continue; }
00184 if (shm_ptr_->destructive_read_mode && shm_ptr_->lowest_seq_id_read == last_seen_id_)
00185 {
00186 shm_ptr_->lowest_seq_id_read = seqID;
00187 }
00188 last_seen_id_ = seqID;
00189 if (shm_ptr_->destructive_read_mode) shm_ptr_->reader_pos = (buffer_num + 1) % shm_ptr_->buffer_count;
00190 }
00191 retry = false;
00192 }
00193
00194 TLOG(13) << "GetBufferForReading returning -1 because no buffers are ready" << TLOG_ENDL;
00195 return buffer_num;
00196 }
00197
00198 int artdaq::SharedMemoryManager::GetBufferForWriting(bool overwrite)
00199 {
00200 TLOG(14) << "GetBufferForWriting BEGIN" << TLOG_ENDL;
00201
00202 TraceLock lk(search_mutex_, 12, "GetBufferForWritingSearch");
00203 auto wp = shm_ptr_->writer_pos.load();
00204
00205
00206 for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
00207 {
00208 auto buffer = (ii + wp) % shm_ptr_->buffer_count;
00209
00210 ResetBuffer(buffer);
00211
00212 auto buf = getBufferInfo_(buffer);
00213 if (buf->sem == BufferSemaphoreFlags::Empty)
00214 {
00215 buf->sem_id = manager_id_;
00216 buf->sem = BufferSemaphoreFlags::Writing;
00217 if (buf->sem_id != manager_id_) continue;
00218 buf->sequence_id = ++shm_ptr_->next_sequence_id;
00219 buf->writePos = 0;
00220 shm_ptr_->writer_pos = (buffer + 1) % shm_ptr_->buffer_count;
00221 touchBuffer_(buf);
00222 TLOG(14) << "GetBufferForWriting returning " << buffer << TLOG_ENDL;
00223 return buffer;
00224 }
00225 }
00226
00227 if (overwrite)
00228 {
00229
00230 for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
00231 {
00232 auto buffer = (ii + wp) % shm_ptr_->buffer_count;
00233
00234 ResetBuffer(buffer);
00235
00236 auto buf = getBufferInfo_(buffer);
00237 if (buf->sem == BufferSemaphoreFlags::Full)
00238 {
00239 buf->sem_id = manager_id_;
00240 buf->sem = BufferSemaphoreFlags::Writing;
00241 if (buf->sem_id != manager_id_) continue;
00242 buf->sequence_id = ++shm_ptr_->next_sequence_id;
00243 buf->writePos = 0;
00244 shm_ptr_->writer_pos = (buffer + 1) % shm_ptr_->buffer_count;
00245 touchBuffer_(buf);
00246 TLOG(14) << "GetBufferForWriting returning " << buffer << TLOG_ENDL;
00247 return buffer;
00248 }
00249 }
00250
00251
00252 for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
00253 {
00254 auto buffer = (ii + wp) % shm_ptr_->buffer_count;
00255
00256 ResetBuffer(buffer);
00257
00258 auto buf = getBufferInfo_(buffer);
00259 if (buf->sem == BufferSemaphoreFlags::Reading)
00260 {
00261 buf->sem_id = manager_id_;
00262 buf->sem = BufferSemaphoreFlags::Writing;
00263 if (buf->sem_id != manager_id_) continue;
00264 buf->sequence_id = ++shm_ptr_->next_sequence_id;
00265 buf->writePos = 0;
00266 shm_ptr_->writer_pos = (buffer + 1) % shm_ptr_->buffer_count;
00267 TLOG(14) << "GetBufferForWriting returning " << buffer << TLOG_ENDL;
00268 touchBuffer_(buf);
00269 return buffer;
00270 }
00271 }
00272 }
00273
00274 TLOG(14) << "GetBufferForWriting Returning -1 because no buffers are ready" << TLOG_ENDL;
00275 return -1;
00276 }
00277
00278 size_t artdaq::SharedMemoryManager::ReadReadyCount()
00279 {
00280 if (!IsValid()) return 0;
00281 TLOG(23) << "ReadReadyCount BEGIN" << TLOG_ENDL;
00282
00283 TraceLock lk(search_mutex_, 14, "ReadReadyCountSearch");
00284 size_t count = 0;
00285 for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
00286 {
00287 TLOG(24) << "ReadReadyCount: Checking if buffer " << ii << " is stale." << TLOG_ENDL;
00288 ResetBuffer(ii);
00289 auto buf = getBufferInfo_(ii);
00290 TLOG(24) << "ReadReadyCount: Buffer " << ii << ": sem=" << FlagToString(buf->sem) << " (expected " << FlagToString(BufferSemaphoreFlags::Full) << "), sem_id=" << buf->sem_id << " )" << TLOG_ENDL;
00291 if (buf->sem == BufferSemaphoreFlags::Full && (buf->sem_id == -1 || buf->sem_id == manager_id_) && buf->sequence_id > last_seen_id_)
00292 {
00293 TLOG(24) << "ReadReadyCount: Buffer " << ii << " is either unowned or owned by this manager, and is marked full." << TLOG_ENDL;
00294 ++count;
00295 }
00296 }
00297
00298 TLOG(23) << TLOG_ENDL;
00299 return count;
00300 }
00301
00302 size_t artdaq::SharedMemoryManager::WriteReadyCount(bool overwrite)
00303 {
00304 if (!IsValid()) return 0;
00305
00306 TraceLock lk(search_mutex_, 15, "WriteReadyCountSearch");
00307 size_t count = 0;
00308 for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
00309 {
00310 ResetBuffer(ii);
00311 auto buf = getBufferInfo_(ii);
00312 if ((buf->sem == BufferSemaphoreFlags::Empty && buf->sem_id == -1)
00313 || (overwrite && buf->sem != BufferSemaphoreFlags::Writing))
00314 {
00315 ++count;
00316 }
00317 }
00318 return count;
00319 }
00320
00321 std::deque<int> artdaq::SharedMemoryManager::GetBuffersOwnedByManager()
00322 {
00323
00324 std::deque<int> output;
00325 if (!IsValid()) return output;
00326 TraceLock lk(search_mutex_, 16, "GetOwnedSearch");
00327 for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
00328 {
00329 auto buf = getBufferInfo_(ii);
00330 if (buf->sem_id == manager_id_)
00331 {
00332 output.push_back(ii);
00333 }
00334 }
00335
00336 return output;
00337 }
00338
00339 size_t artdaq::SharedMemoryManager::BufferDataSize(int buffer)
00340 {
00341
00342 TraceLock lk(buffer_mutexes_[buffer], 17, "DataSizeBuffer" + std::to_string(buffer));
00343 auto buf = getBufferInfo_(buffer);
00344 touchBuffer_(buf);
00345 return buf->writePos;
00346 }
00347
00348
00349 void artdaq::SharedMemoryManager::ResetReadPos(int buffer)
00350 {
00351
00352 TraceLock lk(buffer_mutexes_[buffer], 18, "ResetReadPosBuffer" + std::to_string(buffer));
00353 auto buf = getBufferInfo_(buffer);
00354 touchBuffer_(buf);
00355 buf->readPos = 0;
00356 }
00357
00358 void artdaq::SharedMemoryManager::ResetWritePos(int buffer)
00359 {
00360
00361 TraceLock lk(buffer_mutexes_[buffer], 18, "ResetWritePosBuffer" + std::to_string(buffer));
00362 auto buf = getBufferInfo_(buffer);
00363 checkBuffer_(buf, BufferSemaphoreFlags::Writing);
00364 touchBuffer_(buf);
00365 buf->writePos = 0;
00366 }
00367
00368 void artdaq::SharedMemoryManager::IncrementReadPos(int buffer, size_t read)
00369 {
00370
00371 TraceLock lk(buffer_mutexes_[buffer], 19, "IncReadPosBuffer" + std::to_string(buffer));
00372 auto buf = getBufferInfo_(buffer);
00373 touchBuffer_(buf);
00374 TLOG(15) << TLOG_ENDL;
00375 buf->readPos = buf->readPos + read;
00376 TLOG(15) << TLOG_ENDL;
00377 if (read == 0) Detach(true, "LogicError", "Cannot increment Read pos by 0! (buffer=" + std::to_string(buffer) + ", readPos=" + std::to_string(buf->readPos) + ", writePos=" + std::to_string(buf->writePos) + ")");
00378 }
00379
00380 void artdaq::SharedMemoryManager::IncrementWritePos(int buffer, size_t written)
00381 {
00382
00383 TraceLock lk(buffer_mutexes_[buffer], 20, "IncWritePosBuffer" + std::to_string(buffer));
00384 auto buf = getBufferInfo_(buffer);
00385 touchBuffer_(buf);
00386 TLOG(16) << TLOG_ENDL;
00387 buf->writePos += written;
00388 TLOG(16) << TLOG_ENDL;
00389 if (written == 0) Detach(true, "LogicError", "Cannot increment Write pos by 0!");
00390 }
00391
00392 bool artdaq::SharedMemoryManager::MoreDataInBuffer(int buffer)
00393 {
00394
00395 TraceLock lk(buffer_mutexes_[buffer], 21, "MoreDataInBuffer" + std::to_string(buffer));
00396 auto buf = getBufferInfo_(buffer);
00397 TLOG(17) << TLOG_ENDL;
00398 return buf->readPos < buf->writePos;
00399 }
00400
00401 bool artdaq::SharedMemoryManager::CheckBuffer(int buffer, BufferSemaphoreFlags flags)
00402 {
00403
00404 TraceLock lk(buffer_mutexes_[buffer], 22, "CheckBuffer" + std::to_string(buffer));
00405 return checkBuffer_(getBufferInfo_(buffer), flags, false);
00406 }
00407
00408 void artdaq::SharedMemoryManager::MarkBufferFull(int buffer, int destination)
00409 {
00410
00411 TraceLock lk(buffer_mutexes_[buffer], 23, "FillBuffer" + std::to_string(buffer));
00412 auto shmBuf = getBufferInfo_(buffer);
00413 touchBuffer_(shmBuf);
00414 if (shmBuf->sem_id == manager_id_)
00415 {
00416 if (shmBuf->sem != BufferSemaphoreFlags::Full)
00417 shmBuf->sem = BufferSemaphoreFlags::Full;
00418
00419 shmBuf->sem_id = destination;
00420 }
00421 }
00422
00423 void artdaq::SharedMemoryManager::MarkBufferEmpty(int buffer, bool force)
00424 {
00425 TLOG(18) << "MarkBufferEmpty BEGIN" << TLOG_ENDL;
00426
00427 TraceLock lk(buffer_mutexes_[buffer], 24, "EmptyBuffer" + std::to_string(buffer));
00428 auto shmBuf = getBufferInfo_(buffer);
00429 if (!force)
00430 {
00431 checkBuffer_(shmBuf, BufferSemaphoreFlags::Reading, true);
00432 }
00433 touchBuffer_(shmBuf);
00434
00435 shmBuf->readPos = 0;
00436 shmBuf->sem = BufferSemaphoreFlags::Full;
00437
00438 if ((force && (manager_id_ == 0 || manager_id_ == shmBuf->sem_id)) || (!force && shm_ptr_->destructive_read_mode))
00439 {
00440 TLOG(18) << "MarkBufferEmpty Resetting buffer to Empty state" << TLOG_ENDL;
00441 shmBuf->writePos = 0;
00442 shmBuf->sem = BufferSemaphoreFlags::Empty;
00443 if (shm_ptr_->reader_pos == static_cast<unsigned>(buffer) && !shm_ptr_->destructive_read_mode)
00444 {
00445 TLOG(18) << "MarkBufferEmpty Broadcast mode; incrementing reader_pos" << TLOG_ENDL;
00446 shm_ptr_->reader_pos = (buffer + 1) % shm_ptr_->buffer_count;
00447 }
00448 }
00449 shmBuf->sem_id = -1;
00450 TLOG(18) << "MarkBufferEmpty END" << TLOG_ENDL;
00451 }
00452
00453 bool artdaq::SharedMemoryManager::ResetBuffer(int buffer)
00454 {
00455
00456 TraceLock lk(buffer_mutexes_[buffer], 25, "ResetBuffer" + std::to_string(buffer));
00457 auto shmBuf = getBufferInfo_(buffer);
00458
00459
00460
00461
00462
00463
00464
00465
00466
00467
00468 size_t delta = TimeUtils::gettimeofday_us() - shmBuf->last_touch_time;
00469 if (delta > 0xFFFFFFFF) {
00470 TLOG(TLVL_TRACE) << "Buffer has touch time in the future, setting it to current time and ignoring..." << TLOG_ENDL;
00471 shmBuf->last_touch_time = TimeUtils::gettimeofday_us();
00472 return false;
00473 }
00474 if (delta <= shm_ptr_->buffer_timeout_us || shmBuf->sem == BufferSemaphoreFlags::Empty) return false;
00475 TLOG(TLVL_TRACE) << "Buffer " << buffer << " is stale, time=" << TimeUtils::gettimeofday_us() << ", last touch=" << shmBuf->last_touch_time << ", d=" << delta << ", timeout=" << shm_ptr_->buffer_timeout_us << TLOG_ENDL;
00476
00477 if (shmBuf->sem_id == manager_id_ && shmBuf->sem == BufferSemaphoreFlags::Writing)
00478 {
00479 return true;
00480 }
00481 if (!shm_ptr_->destructive_read_mode && shmBuf->sem == BufferSemaphoreFlags::Full)
00482 {
00483 TLOG(TLVL_DEBUG) << "Resetting old broadcast mode buffer" << TLOG_ENDL;
00484 shmBuf->writePos = 0;
00485 shmBuf->sem = BufferSemaphoreFlags::Empty;
00486 shmBuf->sem_id = -1;
00487 if (shm_ptr_->reader_pos == static_cast<unsigned>(buffer)) shm_ptr_->reader_pos = (buffer + 1) % shm_ptr_->buffer_count;
00488 return true;
00489 }
00490
00491 if (shmBuf->sem_id != manager_id_ && shmBuf->sem == BufferSemaphoreFlags::Reading)
00492 {
00493 TLOG(TLVL_WARNING) << "Stale Read buffer ( " << delta << " / " << shm_ptr_->buffer_timeout_us << " us ) detected! Resetting..." << TLOG_ENDL;
00494 shmBuf->readPos = 0;
00495 shmBuf->sem = BufferSemaphoreFlags::Full;
00496 shmBuf->sem_id = -1;
00497 return true;
00498 }
00499 return false;
00500 }
00501
00502 size_t artdaq::SharedMemoryManager::Write(int buffer, void* data, size_t size)
00503 {
00504 TLOG(19) << "Write BEGIN" << TLOG_ENDL;
00505
00506 TraceLock lk(buffer_mutexes_[buffer], 26, "WriteBuffer" + std::to_string(buffer));
00507 auto shmBuf = getBufferInfo_(buffer);
00508 checkBuffer_(shmBuf, BufferSemaphoreFlags::Writing);
00509 touchBuffer_(shmBuf);
00510 TLOG(19) << TLOG_ENDL;
00511 if (shmBuf->writePos + size > shm_ptr_->buffer_size) Detach(true, "SharedMemoryWrite", "Attempted to write more data than fits into Shared Memory! \nRe-run with a larger buffer size!");
00512
00513 auto pos = GetWritePos(buffer);
00514 memcpy(pos, data, size);
00515 shmBuf->writePos = shmBuf->writePos + size;
00516 if (shmBuf->sequence_id > last_seen_id_)
00517 {
00518 last_seen_id_ = shmBuf->sequence_id;
00519 }
00520 TLOG(19) << "Write END" << TLOG_ENDL;
00521 return size;
00522 }
00523
00524 bool artdaq::SharedMemoryManager::Read(int buffer, void* data, size_t size)
00525 {
00526
00527 TraceLock lk(buffer_mutexes_[buffer], 27, "ReadBuffer" + std::to_string(buffer));
00528 auto shmBuf = getBufferInfo_(buffer);
00529 checkBuffer_(shmBuf, BufferSemaphoreFlags::Reading);
00530 touchBuffer_(shmBuf);
00531 if (shmBuf->readPos + size > shm_ptr_->buffer_size) Detach(true, "SharedMemoryRead", "Attempted to read more data than exists in Shared Memory!");
00532
00533 auto pos = GetReadPos(buffer);
00534 memcpy(data, pos, size);
00535 shmBuf->readPos += size;
00536 touchBuffer_(shmBuf);
00537 return checkBuffer_(shmBuf, BufferSemaphoreFlags::Reading, false);
00538 }
00539
00540 std::string artdaq::SharedMemoryManager::toString()
00541 {
00542 std::ostringstream ostr;
00543 ostr << "ShmStruct: " << std::endl
00544 << "Reader Position: " << shm_ptr_->reader_pos << std::endl
00545 << "Writer Position: " << shm_ptr_->writer_pos << std::endl
00546 << "Next ID Number: " << shm_ptr_->next_id << std::endl
00547 << "Buffer Count: " << shm_ptr_->buffer_count << std::endl
00548 << "Buffer Size: " << std::to_string(shm_ptr_->buffer_size) << " bytes" << std::endl
00549 << "Buffers Written: " << std::to_string(shm_ptr_->next_sequence_id) << std::endl
00550 << "Rank of Writer: " << shm_ptr_->rank << std::endl
00551 << "Ready Magic Bytes: 0x" << std::hex << shm_ptr_->ready_magic << std::endl << std::endl;
00552
00553 for (auto ii = 0; ii < shm_ptr_->buffer_count; ++ii)
00554 {
00555 auto buf = getBufferInfo_(ii);
00556 ostr << "ShmBuffer " << std::dec << ii << std::endl
00557 << "sequenceID: " << std::to_string(buf->sequence_id) << std::endl
00558 << "writePos: " << std::to_string(buf->writePos) << std::endl
00559 << "readPos: " << std::to_string(buf->readPos) << std::endl
00560 << "sem: " << FlagToString(buf->sem) << std::endl
00561 << "Owner: " << std::to_string(buf->sem_id.load()) << std::endl
00562 << "Last Touch Time: " << std::to_string(buf->last_touch_time / 1000000.0) << std::endl << std::endl;
00563 }
00564
00565 return ostr.str();
00566 }
00567
00568 void* artdaq::SharedMemoryManager::GetReadPos(int buffer)
00569 {
00570 auto buf = getBufferInfo_(buffer);
00571 return bufferStart_(buffer) + buf->readPos;
00572 }
00573 void* artdaq::SharedMemoryManager::GetWritePos(int buffer)
00574 {
00575 auto buf = getBufferInfo_(buffer);
00576 return bufferStart_(buffer) + buf->writePos;
00577 }
00578
00579 void* artdaq::SharedMemoryManager::GetBufferStart(int buffer)
00580 {
00581 return bufferStart_(buffer);
00582 }
00583
00584 uint8_t* artdaq::SharedMemoryManager::dataStart_() const
00585 {
00586 return reinterpret_cast<uint8_t*>(shm_ptr_ + 1) + shm_ptr_->buffer_count * sizeof(ShmBuffer);
00587 }
00588
00589 uint8_t* artdaq::SharedMemoryManager::bufferStart_(int buffer)
00590 {
00591 if (buffer >= shm_ptr_->buffer_count) Detach(true, "ArgumentOutOfRange", "The specified buffer does not exist!");
00592 return dataStart_() + buffer * shm_ptr_->buffer_size;
00593 }
00594
00595 artdaq::SharedMemoryManager::ShmBuffer* artdaq::SharedMemoryManager::getBufferInfo_(int buffer)
00596 {
00597 if (buffer >= shm_ptr_->buffer_count) Detach(true, "ArgumentOutOfRange", "The specified buffer does not exist!");
00598 return reinterpret_cast<ShmBuffer*>(reinterpret_cast<uint8_t*>(shm_ptr_ + 1) + buffer * sizeof(ShmBuffer));
00599 }
00600
00601 bool artdaq::SharedMemoryManager::checkBuffer_(ShmBuffer* buffer, BufferSemaphoreFlags flags, bool exceptions)
00602 {
00603 if (exceptions)
00604 {
00605 if (buffer->sem != flags) Detach(true, "StateAccessViolation", "Shared Memory buffer is not in the correct state! (expected " + FlagToString(flags) + ", actual " + FlagToString(buffer->sem) + ")");
00606 if (buffer->sem_id != manager_id_) Detach(true, "OwnerAccessViolation", "Shared Memory buffer is not owned by this manager instance!");
00607 }
00608 bool ret = (buffer->sem_id == manager_id_ || (buffer->sem_id == -1 && (flags == BufferSemaphoreFlags::Full || flags == BufferSemaphoreFlags::Empty))) && buffer->sem == flags;
00609
00610 if (!ret)
00611 {
00612 TLOG(TLVL_WARNING) << "CheckBuffer detected issue with buffer " << std::to_string(buffer->sequence_id) << "!"
00613 << " ID: " << buffer->sem_id << " (" << manager_id_ << "), Flag: " << FlagToString(buffer->sem) << " (" << FlagToString(flags) << "). "
00614 << "ID -1 is okay if desired flag is \"Full\" or \"Empty\"." << TLOG_ENDL;
00615 }
00616
00617 return ret;
00618 }
00619
00620 void artdaq::SharedMemoryManager::touchBuffer_(ShmBuffer* buffer)
00621 {
00622 if (buffer->sem_id != manager_id_) return;
00623 TLOG(TLVL_TRACE) << "touchBuffer_: Touching buffer with sequence_id " << std::to_string(buffer->sequence_id) << TLOG_ENDL;
00624 buffer->last_touch_time = TimeUtils::gettimeofday_us();
00625 }
00626
00627 void artdaq::SharedMemoryManager::Detach(bool throwException, std::string category, std::string message)
00628 {
00629 if (IsValid())
00630 {
00631 auto bufs = GetBuffersOwnedByManager();
00632 for (auto buf : bufs)
00633 {
00634 auto shmBuf = getBufferInfo_(buf);
00635 if (shmBuf->sem == BufferSemaphoreFlags::Writing)
00636 {
00637 shmBuf->sem = BufferSemaphoreFlags::Empty;
00638 }
00639 else if (shmBuf->sem == BufferSemaphoreFlags::Reading)
00640 {
00641 shmBuf->sem = BufferSemaphoreFlags::Full;
00642 }
00643 shmBuf->sem_id = -1;
00644 }
00645 }
00646
00647 if (shm_ptr_)
00648 {
00649 shmdt(shm_ptr_);
00650 shm_ptr_ = NULL;
00651 }
00652
00653 if (manager_id_ == 0 && shm_segment_id_ > -1)
00654 {
00655 shmctl(shm_segment_id_, IPC_RMID, NULL);
00656 }
00657
00658 if (category.size() > 0 && message.size() > 0)
00659 {
00660 TLOG(TLVL_ERROR) << category << ": " << message << TLOG_ENDL;
00661
00662 if (throwException)
00663 {
00664 throw cet::exception(category) << message;
00665 }
00666 }
00667 }
00668
00669
00670
00671
00672
00673