artdaq  v3_00_01
Shmem_transfer.cc
1 #include "artdaq/TransferPlugins/ShmemTransfer.hh"
2 #include "cetlib_except/exception.h"
3 
4 artdaq::ShmemTransfer::ShmemTransfer(fhicl::ParameterSet const& pset, Role role) :
5  TransferInterface(pset, role)
6  , role_(role)
7 {
8  // char* keyChars = getenv("ARTDAQ_SHM_KEY");
9  // if (keyChars != NULL && shm_key_ == static_cast<int>(std::hash<std::string>()(unique_label_))) {
10  // std::string keyString(keyChars);
11  // try {
12  // shm_key_ = boost::lexical_cast<int>(keyString);
13  // }
14  // catch (...) {
15  // std::stringstream errmsg;
16  // errmsg << uniqueLabel() << ": Problem performing lexical cast on " << keyString;
17  // ExceptionHandler(ExceptionHandlerRethrow::yes, errmsg.str());
18  // }
19  // }
20 
21  // JCF, Aug-16-2016
22 
23  // Note that there's a small but nonzero chance of a race condition
24  // here where another process creates the shared memory buffer
25  // between the first and second calls to shmget
26 
27  if (buffer_count_ > 100)
28  {
29  throw cet::exception("ConfigurationException", "Buffer Count is too large for Shmem transfer!");
30  }
31 
32  auto shmKey = pset.get<uint32_t>("shm_key_offset", 0);
33  shmKey += std::hash<std::string>()(uniqueLabel());
34  if (role == Role::kReceive)
35  {
36  shm_manager_ = std::make_unique<SharedMemoryFragmentManager>(shmKey, buffer_count_, max_fragment_size_words_ * sizeof(artdaq::RawDataType));
37  }
38  else
39  {
40  shm_manager_ = std::make_unique<SharedMemoryFragmentManager>(shmKey, 0, 0);
41  }
42 }
43 
45 {
46  TLOG_ARB(5, uniqueLabel()) << "~ShmemTransfer called" << TLOG_ENDL;
47  shm_manager_.reset(nullptr);
48  TLOG_ARB(5, uniqueLabel()) << "~ShmemTransfer done" << TLOG_ENDL;
49 }
50 
51 int artdaq::ShmemTransfer::receiveFragment(artdaq::Fragment& fragment,
52  size_t receiveTimeout)
53 {
54  auto waitStart = std::chrono::steady_clock::now();
55  while (!shm_manager_->ReadyForRead() && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
56  {
57  // BURN THAT CPU!
58  }
59  if (!shm_manager_->ReadyForRead())
60  {
61  int64_t loopCount = 0;
62  size_t sleepTime = 1000; // microseconds
63  int64_t nloops = (receiveTimeout - 1000) / sleepTime;
64 
65  while (!shm_manager_->ReadyForRead() && loopCount < nloops)
66  {
67  usleep(sleepTime);
68  ++loopCount;
69  }
70  }
71 
72  //TLOG_DEBUG(uniqueLabel()) << "delta_=" << delta_() << ", rp=" << (int)shm_ptr_->read_pos << ", wp=" << (int)shm_ptr_->write_pos << ", loopCount=" << loopCount << ", nloops=" << nloops << TLOG_ENDL;
73 
74  if (shm_manager_->ReadyForRead())
75  {
76  auto sts = shm_manager_->ReadFragment(fragment);
77 
78  if (sts != 0) return RECV_TIMEOUT;
79 
80  if (fragment.type() != artdaq::Fragment::DataFragmentType)
81  {
82  TLOG_ARB(8, uniqueLabel()) << "Recvd frag from shmem, type=" << fragment.typeString() << ", sequenceID=" << std::to_string(fragment.sequenceID()) << ", source_rank=" << source_rank() << TLOG_ENDL;
83  }
84 
85  return source_rank();
86  }
87 
89 }
90 
91 int artdaq::ShmemTransfer::receiveFragmentHeader(detail::RawFragmentHeader& header, size_t receiveTimeout)
92 {
93  auto waitStart = std::chrono::steady_clock::now();
94  while (!shm_manager_->ReadyForRead() && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
95  {
96  // BURN THAT CPU!
97  }
98  if (!shm_manager_->ReadyForRead())
99  {
100  int64_t loopCount = 0;
101  size_t sleepTime = 1000; // microseconds
102  int64_t nloops = (receiveTimeout - 1000) / sleepTime;
103 
104  while (!shm_manager_->ReadyForRead() && loopCount < nloops)
105  {
106  usleep(sleepTime);
107  ++loopCount;
108  }
109  }
110 
111  //TLOG_DEBUG(uniqueLabel()) << "delta_=" << delta_() << ", rp=" << (int)shm_ptr_->read_pos << ", wp=" << (int)shm_ptr_->write_pos << ", loopCount=" << loopCount << ", nloops=" << nloops << TLOG_ENDL;
112 
113  if (shm_manager_->ReadyForRead())
114  {
115  auto sts = shm_manager_->ReadFragmentHeader(header);
116 
117  if (sts != 0) return RECV_TIMEOUT;
118 
119  if (header.type != artdaq::Fragment::DataFragmentType)
120  {
121  TLOG_ARB(8, uniqueLabel()) << "Recvd fragment header from shmem, type=" << (int)header.type << ", sequenceID=" << std::to_string(header.sequence_id) << ", source_rank=" << source_rank() << TLOG_ENDL;
122  }
123 
124  return source_rank();
125  }
126 
128 }
129 
130 int artdaq::ShmemTransfer::receiveFragmentData(RawDataType* destination, size_t word_count)
131 {
132  auto sts = shm_manager_->ReadFragmentData(destination, word_count);
133 
134  TLOG_TRACE(uniqueLabel()) << "Return status from ReadFragmentData is " << sts << TLOG_ENDL;
135 
136  if (sts != 0) return RECV_TIMEOUT;
137 
138  return source_rank();
139 
141 }
142 
144 artdaq::ShmemTransfer::copyFragment(artdaq::Fragment& fragment, size_t send_timeout_usec)
145 {
146  return sendFragment(std::move(fragment), send_timeout_usec, false);
147 }
148 
150 artdaq::ShmemTransfer::moveFragment(artdaq::Fragment&& fragment,
151  size_t send_timeout_usec)
152 {
153  return sendFragment(std::move(fragment), send_timeout_usec, true);
154 }
155 
157 artdaq::ShmemTransfer::sendFragment(artdaq::Fragment&& fragment, size_t send_timeout_usec, bool reliableMode)
158 {
159  // wait for the shm to become free, if requested
160  if (send_timeout_usec > 0 || reliableMode)
161  {
162  auto waitStart = std::chrono::steady_clock::now();
163  while (!shm_manager_->ReadyForWrite(!reliableMode) && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
164  {
165  // BURN THAT CPU!
166  }
167  if (!shm_manager_->ReadyForWrite(!reliableMode))
168  {
169  int64_t loopCount = 0;
170  size_t sleepTime = 1000; // microseconds
171  int64_t nloops = (send_timeout_usec - 1000) / sleepTime;
172 
173  while (reliableMode && !shm_manager_->ReadyForWrite(!reliableMode) && (send_timeout_usec == 0 || loopCount < nloops))
174  {
175  usleep(sleepTime);
176  ++loopCount;
177  }
178  }
179  }
180 
181  TLOG_ARB(5, uniqueLabel()) << "Either write has timed out or buffer ready" << TLOG_ENDL;
182 
183  // copy the fragment if the shm is available
184  if (shm_manager_->ReadyForWrite(!reliableMode))
185  {
186  TLOG_ARB(5, uniqueLabel()) << "Sending fragment with seqID=" << std::to_string(fragment.sequenceID()) << TLOG_ENDL;
187  artdaq::RawDataType* fragAddr = fragment.headerAddress();
188  size_t fragSize = fragment.size() * sizeof(artdaq::RawDataType);
189 
190  // 10-Sep-2013, KAB - protect against large events and
191  // invalid events (and large, invalid events)
192  if (fragment.type() != artdaq::Fragment::InvalidFragmentType && fragSize < (max_fragment_size_words_ * sizeof(artdaq::RawDataType)))
193  {
194  auto sts = shm_manager_->WriteFragment(std::move(fragment), !reliableMode);
195  if (sts != 0) return CopyStatus::kErrorNotRequiringException;
196 
197  TLOG_ARB(5, uniqueLabel()) << "Fragment send successfully" << TLOG_ENDL;
198  return CopyStatus::kSuccess;
199  }
200  else
201  {
202  TLOG_WARNING(uniqueLabel()) << "Fragment invalid for shared memory! "
203  << "fragment address and size = "
204  << fragAddr << " " << fragSize << " "
205  << "sequence ID, fragment ID, and type = "
206  << fragment.sequenceID() << " "
207  << fragment.fragmentID() << " "
208  << fragment.typeString() << TLOG_ENDL;
209  return CopyStatus::kErrorNotRequiringException;
210  }
211  }
212 
213  TLOG_ARB(5, uniqueLabel()) << "Fragment Send Timeout!" << TLOG_ENDL;
214  return CopyStatus::kTimeout;
215 }
216 
217 DEFINE_ARTDAQ_TRANSFER(artdaq::ShmemTransfer)
218 
219 // Local Variables:
220 // mode: c++
221 // End:
size_t buffer_count_
The number of Fragment transfers the TransferInterface can handle simultaneously. ...
CopyStatus moveFragment(Fragment &&fragment, size_t send_timeout_usec=std::numeric_limits< size_t >::max()) override
Move a Fragment to the destination.
static const int RECV_TIMEOUT
Value to be returned upon receive timeout. Because receivers otherwise return rank, this is also the limit on the number of ranks that artdaq currently supports.
This TransferInterface is a Receiver.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
CopyStatus copyFragment(Fragment &fragment, size_t send_timeout_usec=std::numeric_limits< size_t >::max()) override
Copy a Fragment to the destination. May be unreliable.
ShmemTransfer(fhicl::ParameterSet const &pset, Role role)
ShmemTransfer Constructor.
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
virtual ~ShmemTransfer() noexcept
ShmemTransfer Destructor.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
std::string uniqueLabel() const
Get the unique label of this TransferInterface instance.
A TransferInterface implementation plugin that transfers data using Shared Memory.
This interface defines the functions used to transfer data between artdaq applications.
int receiveFragment(Fragment &fragment, size_t receiveTimeout) override
Receive a Fragment from Shared Memory.
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.
const size_t max_fragment_size_words_
The maximum size of the transferred Fragment objects, in artdaq::Fragment::RawDataType words...