artdaq  v3_06_02
Shmem_transfer.cc
1 #include "artdaq/DAQdata/Globals.hh"
2 #define TRACE_NAME (app_name + "_ShmemTransfer").c_str()
3 
4 #include <signal.h>
5 #include "artdaq/TransferPlugins/ShmemTransfer.hh"
6 #include "cetlib_except/exception.h"
7 
8 artdaq::ShmemTransfer::ShmemTransfer(fhicl::ParameterSet const& pset, Role role)
9  : TransferInterface(pset, role)
10 {
11  TLOG(TLVL_DEBUG) << GetTraceName() << ": Constructor BEGIN";
12  // char* keyChars = getenv("ARTDAQ_SHM_KEY");
13  // if (keyChars != NULL && shm_key_ == static_cast<int>(std::hash<std::string>()(unique_label_))) {
14  // std::string keyString(keyChars);
15  // try {
16  // shm_key_ = boost::lexical_cast<int>(keyString);
17  // }
18  // catch (...) {
19  // std::stringstream errmsg;
20  // errmsg << uniqueLabel() << ": Problem performing lexical cast on " << keyString;
21  // ExceptionHandler(ExceptionHandlerRethrow::yes, errmsg.str());
22  // }
23  // }
24 
25  // JCF, Aug-16-2016
26 
27  // Note that there's a small but nonzero chance of a race condition
28  // here where another process creates the shared memory buffer
29  // between the first and second calls to shmget
30 
31  if (buffer_count_ > 100)
32  {
33  throw cet::exception("ConfigurationException", "Buffer Count is too large for Shmem transfer!");
34  }
35 
36  auto partition = GetPartitionNumber() + 1; // Can't be 0
37 
38  auto shmKey = pset.get<uint32_t>("shm_key_offset", 0) + (partition << 24) + ((source_rank() & 0xFFF) << 12) + (destination_rank() & 0xFFF);
39 
40  // Configured Shared Memory key overrides everything! Needed for Online Monitor connections!
41  if (pset.has_key("shm_key"))
42  {
43  shmKey = pset.get<uint32_t>("shm_key");
44  }
45 
46  if (role == Role::kReceive)
47  {
48  shm_manager_ = std::make_unique<SharedMemoryFragmentManager>(shmKey, buffer_count_, max_fragment_size_words_ * sizeof(artdaq::RawDataType));
49  }
50  else
51  {
52  shm_manager_ = std::make_unique<SharedMemoryFragmentManager>(shmKey);
53  }
54  TLOG(TLVL_DEBUG) << GetTraceName() << ": Constructor END";
55 }
56 
58 {
59  TLOG(5) << GetTraceName() << " ~ShmemTransfer called - " << uniqueLabel();
60  shm_manager_.reset(nullptr);
61  TLOG(5) << GetTraceName() << " ~ShmemTransfer done - " << uniqueLabel();
62 }
63 
64 int artdaq::ShmemTransfer::receiveFragment(artdaq::Fragment& fragment,
65  size_t receiveTimeout)
66 {
67  auto waitStart = std::chrono::steady_clock::now();
68  while (!shm_manager_->ReadyForRead() && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
69  {
70  // BURN THAT CPU!
71  }
72  if (!shm_manager_->ReadyForRead())
73  {
74  int64_t loopCount = 0;
75  size_t sleepTime = 1000; // microseconds
76  int64_t nloops = (receiveTimeout - 1000) / sleepTime;
77 
78  while (!shm_manager_->ReadyForRead() && loopCount < nloops)
79  {
80  usleep(sleepTime);
81  ++loopCount;
82  }
83  }
84 
85  TLOG(TLVL_TRACE) << GetTraceName() << ": receiveFragment ReadyForRead=" << shm_manager_->ReadyForRead();
86 
87  if (shm_manager_->ReadyForRead())
88  {
89  auto sts = shm_manager_->ReadFragment(fragment);
90 
91  if (sts != 0)
92  {
93  TLOG(TLVL_TRACE) << "Non-zero status (" << sts << ") returned from ReadFragment, returning...";
94  return RECV_TIMEOUT;
95  }
96 
97  if (fragment.type() != artdaq::Fragment::DataFragmentType)
98  {
99  TLOG(8) << GetTraceName() << ": Recvd frag from shmem, type=" << fragment.typeString() << ", sequenceID=" << fragment.sequenceID() << ", source_rank=" << source_rank();
100  }
101 
102  return source_rank();
103  }
104 
106 }
107 
108 int artdaq::ShmemTransfer::receiveFragmentHeader(detail::RawFragmentHeader& header, size_t receiveTimeout)
109 {
110  auto waitStart = std::chrono::steady_clock::now();
111  while (!shm_manager_->ReadyForRead() && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
112  {
113  // BURN THAT CPU!
114  }
115  if (!shm_manager_->ReadyForRead())
116  {
117  int64_t loopCount = 0;
118  size_t sleepTime = 1000; // microseconds
119  int64_t nloops = (receiveTimeout - 1000) / sleepTime;
120 
121  while (!shm_manager_->ReadyForRead() && loopCount < nloops)
122  {
123  usleep(sleepTime);
124  ++loopCount;
125  }
126  }
127 
128  if (!shm_manager_->ReadyForRead() && shm_manager_->IsEndOfData())
129  {
131  }
132 
133  //TLOG(TLVL_TRACE) << GetTraceName() << ": delta_=" << delta_() << ", rp=" << (int)shm_ptr_->read_pos << ", wp=" << (int)shm_ptr_->write_pos << ", loopCount=" << loopCount << ", nloops=" << nloops ;
134 
135  if (shm_manager_->ReadyForRead())
136  {
137  auto sts = shm_manager_->ReadFragmentHeader(header);
138 
139  if (sts != 0)
140  {
141  TLOG(TLVL_TRACE) << "Non-zero status (" << sts << ") returned from ReadFragmentHeader, returning...";
142  return RECV_TIMEOUT;
143  }
144 
145  if (header.type != artdaq::Fragment::DataFragmentType)
146  {
147  TLOG(8) << GetTraceName() << ": Recvd fragment header from shmem, type=" << (int)header.type << ", sequenceID=" << header.sequence_id << ", source_rank=" << source_rank();
148  }
149 
150  return source_rank();
151  }
152 
154 }
155 
156 int artdaq::ShmemTransfer::receiveFragmentData(RawDataType* destination, size_t word_count)
157 {
158  auto sts = shm_manager_->ReadFragmentData(destination, word_count);
159 
160  TLOG(TLVL_TRACE) << GetTraceName() << ": Return status from ReadFragmentData is " << sts;
161 
162  if (sts != 0)
163  {
164  TLOG(TLVL_TRACE) << "Non-zero status (" << sts << ") returned from ReadFragmentData, returning...";
165  return RECV_TIMEOUT;
166  }
167 
168  return source_rank();
169 
171 }
172 
174 artdaq::ShmemTransfer::transfer_fragment_min_blocking_mode(artdaq::Fragment const& fragment, size_t send_timeout_usec)
175 {
176  return sendFragment(Fragment(fragment), send_timeout_usec, false);
177 }
178 
181 {
182  return sendFragment(std::move(fragment), 0, true);
183 }
184 
186 artdaq::ShmemTransfer::sendFragment(artdaq::Fragment&& fragment, size_t send_timeout_usec, bool reliableMode)
187 {
188  if (!isRunning())
189  {
190  shm_manager_->Attach();
191  if (!isRunning())
192  {
193  TLOG(TLVL_ERROR) << GetTraceName() << ": Attempted to send Fragment when not attached to Shared Memory! Returning kSuccess, and dropping data!";
194  return CopyStatus::kSuccess;
195  }
196  }
197  shm_manager_->SetRank(my_rank);
198  // wait for the shm to become free, if requested
199 
200  TLOG(5) << GetTraceName() << ": Sending fragment with seqID=" << fragment.sequenceID();
201  artdaq::RawDataType* fragAddr = fragment.headerAddress();
202  size_t fragSize = fragment.size() * sizeof(artdaq::RawDataType);
203 
204  // 10-Sep-2013, KAB - protect against large events and
205  // invalid events (and large, invalid events)
206  if (fragment.type() != artdaq::Fragment::InvalidFragmentType && fragSize < (max_fragment_size_words_ * sizeof(artdaq::RawDataType)))
207  {
208  TLOG(5) << GetTraceName() << ": Writing fragment with seqID=" << fragment.sequenceID();
209  auto sts = shm_manager_->WriteFragment(std::move(fragment), !reliableMode, send_timeout_usec);
210  if (sts == -3)
211  {
212  TLOG(TLVL_WARNING) << GetTraceName() << ": Timeout writing fragment with seqID=" << fragment.sequenceID();
213  return CopyStatus::kTimeout;
214  }
215  if (sts != 0)
216  {
217  TLOG(TLVL_WARNING) << GetTraceName() << ": Error writing fragment with seqID=" << fragment.sequenceID();
218  return CopyStatus::kErrorNotRequiringException;
219  }
220 
221  TLOG(5) << GetTraceName() << ": Successfully sent Fragment with seqID=" << fragment.sequenceID();
222  return CopyStatus::kSuccess;
223  }
224  else
225  {
226  TLOG(TLVL_WARNING) << GetTraceName() << ": Fragment invalid for shared memory! "
227  << "fragment address and size = "
228  << fragAddr << " " << fragSize << " "
229  << "sequence ID, fragment ID, and type = "
230  << fragment.sequenceID() << " "
231  << fragment.fragmentID() << " "
232  << fragment.typeString();
233  return CopyStatus::kErrorNotRequiringException;
234  }
235 
236  TLOG(TLVL_WARNING) << GetTraceName() << ": Unreachable code reached!";
237  return CopyStatus::kErrorNotRequiringException;
238 }
239 
241 {
242  bool ret = false;
243  switch (role())
244  {
246  ret = shm_manager_->IsValid() && !shm_manager_->IsEndOfData();
247  break;
249  ret = shm_manager_->GetAttachedCount() > 1;
250  break;
251  }
252  return ret;
253 }
254 
256 {
257  for (size_t ii = 0; ii < shm_manager_->size(); ++ii)
258  {
259  shm_manager_->MarkBufferEmpty(ii, true);
260  }
261 }
262 
263 DEFINE_ARTDAQ_TRANSFER(artdaq::ShmemTransfer)
264 
265 // Local Variables:
266 // mode: c++
267 // End:
size_t buffer_count_
The number of Fragment transfers the TransferInterface can handle simultaneously. ...
virtual int source_rank() const
Get the source rank for this TransferInterface instance.
CopyStatus transfer_fragment_reliable_mode(Fragment &&fragment) override
Transfer a Fragment to the destination. This should be reliable, if the underlying transport mechanis...
CopyStatus transfer_fragment_min_blocking_mode(Fragment const &fragment, size_t send_timeout_usec) override
Transfer a Fragment to the destination. May not necessarily be reliable, but will not block longer th...
This TransferInterface is a Receiver.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
void flush_buffers() override
Flush any in-flight data. This should be used by the receiver after the receive loop has ended...
ShmemTransfer(fhicl::ParameterSet const &pset, Role role)
ShmemTransfer Constructor.
Value that is to be returned when a Transfer plugin determines that no more data will be arriving...
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
This TransferInterface is a Sender.
virtual ~ShmemTransfer() noexcept
ShmemTransfer Destructor.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
A TransferInterface implementation plugin that transfers data using Shared Memory.
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
This interface defines the functions used to transfer data between artdaq applications.
virtual int destination_rank() const
Get the destination rank for this TransferInterface instance.
int receiveFragment(Fragment &fragment, size_t receiveTimeout) override
Receive a Fragment from Shared Memory.
Value to be returned upon receive timeout.
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.
const size_t max_fragment_size_words_
The maximum size of the transferred Fragment objects, in artdaq::Fragment::RawDataType words...