artdaq  v3_04_01
Shmem_transfer.cc
1 #define TRACE_NAME (app_name + "_ShmemTransfer").c_str()
2 #include "artdaq/DAQdata/Globals.hh"
3 
4 #include "artdaq/TransferPlugins/ShmemTransfer.hh"
5 #include "cetlib_except/exception.h"
6 #include <signal.h>
7 
8 artdaq::ShmemTransfer::ShmemTransfer(fhicl::ParameterSet const& pset, Role role) :
9  TransferInterface(pset, role)
10 {
11  TLOG(TLVL_DEBUG) << GetTraceName() << ": Constructor BEGIN";
12  // char* keyChars = getenv("ARTDAQ_SHM_KEY");
13  // if (keyChars != NULL && shm_key_ == static_cast<int>(std::hash<std::string>()(unique_label_))) {
14  // std::string keyString(keyChars);
15  // try {
16  // shm_key_ = boost::lexical_cast<int>(keyString);
17  // }
18  // catch (...) {
19  // std::stringstream errmsg;
20  // errmsg << uniqueLabel() << ": Problem performing lexical cast on " << keyString;
21  // ExceptionHandler(ExceptionHandlerRethrow::yes, errmsg.str());
22  // }
23  // }
24 
25  // JCF, Aug-16-2016
26 
27  // Note that there's a small but nonzero chance of a race condition
28  // here where another process creates the shared memory buffer
29  // between the first and second calls to shmget
30 
31  if (buffer_count_ > 100)
32  {
33  throw cet::exception("ConfigurationException", "Buffer Count is too large for Shmem transfer!");
34  }
35 
36  auto partition = GetPartitionNumber() + 1; // Can't be 0
37 
38  auto shmKey = pset.get<uint32_t>("shm_key_offset", 0) + (partition << 24) + ((source_rank() & 0xFFF) << 12) + (destination_rank() & 0xFFF);
39 
40  // Configured Shared Memory key overrides everything! Needed for Online Monitor connections!
41  if (pset.has_key("shm_key")) {
42  shmKey = pset.get<uint32_t>("shm_key");
43  }
44 
45  if (role == Role::kReceive)
46  {
47  shm_manager_ = std::make_unique<SharedMemoryFragmentManager>(shmKey, buffer_count_, max_fragment_size_words_ * sizeof(artdaq::RawDataType));
48  }
49  else
50  {
51  shm_manager_ = std::make_unique<SharedMemoryFragmentManager>(shmKey, 0, 0);
52  }
53  TLOG(TLVL_DEBUG) << GetTraceName() << ": Constructor END";
54 }
55 
57 {
58  TLOG(5) << GetTraceName() << " ~ShmemTransfer called - " << uniqueLabel() ;
59  shm_manager_.reset(nullptr);
60  TLOG(5) << GetTraceName() << " ~ShmemTransfer done - " << uniqueLabel() ;
61 }
62 
63 int artdaq::ShmemTransfer::receiveFragment(artdaq::Fragment& fragment,
64  size_t receiveTimeout)
65 {
66  auto waitStart = std::chrono::steady_clock::now();
67  while (!shm_manager_->ReadyForRead() && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
68  {
69  // BURN THAT CPU!
70  }
71  if (!shm_manager_->ReadyForRead())
72  {
73  int64_t loopCount = 0;
74  size_t sleepTime = 1000; // microseconds
75  int64_t nloops = (receiveTimeout - 1000) / sleepTime;
76 
77  while (!shm_manager_->ReadyForRead() && loopCount < nloops)
78  {
79  usleep(sleepTime);
80  ++loopCount;
81  }
82  }
83 
84  TLOG(TLVL_TRACE) << GetTraceName() << ": receiveFragment ReadyForRead=" << shm_manager_->ReadyForRead();
85 
86  if (shm_manager_->ReadyForRead())
87  {
88  auto sts = shm_manager_->ReadFragment(fragment);
89 
90  if (sts != 0) {
91  TLOG(TLVL_TRACE) << "Non-zero status (" << sts << ") returned from ReadFragment, returning...";
92  return RECV_TIMEOUT;
93  }
94 
95  if (fragment.type() != artdaq::Fragment::DataFragmentType)
96  {
97  TLOG(8) << GetTraceName() << ": Recvd frag from shmem, type=" << fragment.typeString() << ", sequenceID=" << fragment.sequenceID() << ", source_rank=" << source_rank() ;
98  }
99 
100  return source_rank();
101  }
102 
104 }
105 
106 int artdaq::ShmemTransfer::receiveFragmentHeader(detail::RawFragmentHeader& header, size_t receiveTimeout)
107 {
108  auto waitStart = std::chrono::steady_clock::now();
109  while (!shm_manager_->ReadyForRead() && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
110  {
111  // BURN THAT CPU!
112  }
113  if (!shm_manager_->ReadyForRead())
114  {
115  int64_t loopCount = 0;
116  size_t sleepTime = 1000; // microseconds
117  int64_t nloops = (receiveTimeout - 1000) / sleepTime;
118 
119  while (!shm_manager_->ReadyForRead() && loopCount < nloops)
120  {
121  usleep(sleepTime);
122  ++loopCount;
123  }
124  }
125 
126  if (!shm_manager_->ReadyForRead() && shm_manager_->IsEndOfData())
127  {
129  }
130 
131  //TLOG(TLVL_TRACE) << GetTraceName() << ": delta_=" << delta_() << ", rp=" << (int)shm_ptr_->read_pos << ", wp=" << (int)shm_ptr_->write_pos << ", loopCount=" << loopCount << ", nloops=" << nloops ;
132 
133  if (shm_manager_->ReadyForRead())
134  {
135  auto sts = shm_manager_->ReadFragmentHeader(header);
136 
137  if (sts != 0) {
138  TLOG(TLVL_TRACE) << "Non-zero status (" << sts << ") returned from ReadFragmentHeader, returning...";
139  return RECV_TIMEOUT;
140  }
141 
142  if (header.type != artdaq::Fragment::DataFragmentType)
143  {
144  TLOG(8) << GetTraceName() << ": Recvd fragment header from shmem, type=" << (int)header.type << ", sequenceID=" << header.sequence_id << ", source_rank=" << source_rank() ;
145  }
146 
147  return source_rank();
148  }
149 
151 }
152 
153 int artdaq::ShmemTransfer::receiveFragmentData(RawDataType* destination, size_t word_count)
154 {
155  auto sts = shm_manager_->ReadFragmentData(destination, word_count);
156 
157  TLOG(TLVL_TRACE) << GetTraceName() << ": Return status from ReadFragmentData is " << sts ;
158 
159  if (sts != 0) {
160  TLOG(TLVL_TRACE) << "Non-zero status (" << sts << ") returned from ReadFragmentData, returning...";
161  return RECV_TIMEOUT;
162  }
163 
164  return source_rank();
165 
167 }
168 
170 artdaq::ShmemTransfer::transfer_fragment_min_blocking_mode(artdaq::Fragment const& fragment, size_t send_timeout_usec)
171 {
172  return sendFragment(Fragment(fragment), send_timeout_usec, false);
173 }
174 
177 {
178  return sendFragment(std::move(fragment), 0, true);
179 }
180 
182 artdaq::ShmemTransfer::sendFragment(artdaq::Fragment&& fragment, size_t send_timeout_usec, bool reliableMode)
183 {
184  if (!shm_manager_->IsValid()) {
185  shm_manager_->Attach();
186  if (!shm_manager_->IsValid()) {
187  TLOG(TLVL_ERROR) << GetTraceName() << ": Attempted to send Fragment when not attached to Shared Memory! Returning kSuccess, and dropping data!";
188  return CopyStatus::kSuccess;
189  }
190  }
191  shm_manager_->SetRank(my_rank);
192  // wait for the shm to become free, if requested
193 
194  TLOG(5) << GetTraceName() << ": Sending fragment with seqID=" << fragment.sequenceID() ;
195  artdaq::RawDataType* fragAddr = fragment.headerAddress();
196  size_t fragSize = fragment.size() * sizeof(artdaq::RawDataType);
197 
198  // 10-Sep-2013, KAB - protect against large events and
199  // invalid events (and large, invalid events)
200  if (fragment.type() != artdaq::Fragment::InvalidFragmentType && fragSize < (max_fragment_size_words_ * sizeof(artdaq::RawDataType)))
201  {
202  TLOG(5) << GetTraceName() << ": Writing fragment with seqID=" << fragment.sequenceID();
203  auto sts = shm_manager_->WriteFragment(std::move(fragment), !reliableMode, send_timeout_usec);
204  if (sts == -3)
205  {
206  TLOG(TLVL_WARNING) << GetTraceName() << ": Timeout writing fragment with seqID=" << fragment.sequenceID();
207  return CopyStatus::kTimeout;
208  }
209  if (sts != 0)
210  {
211  TLOG(TLVL_WARNING) << GetTraceName() << ": Error writing fragment with seqID=" << fragment.sequenceID();
212  return CopyStatus::kErrorNotRequiringException;
213  }
214 
215  TLOG(5) << GetTraceName() << ": Successfully sent Fragment with seqID=" << fragment.sequenceID();
216  return CopyStatus::kSuccess;
217  }
218  else
219  {
220  TLOG(TLVL_WARNING) << GetTraceName() << ": Fragment invalid for shared memory! "
221  << "fragment address and size = "
222  << fragAddr << " " << fragSize << " "
223  << "sequence ID, fragment ID, and type = "
224  << fragment.sequenceID() << " "
225  << fragment.fragmentID() << " "
226  << fragment.typeString() ;
227  return CopyStatus::kErrorNotRequiringException;
228  }
229 
230  TLOG(TLVL_WARNING) << GetTraceName() << ": Unreachable code reached!" ;
231  return CopyStatus::kErrorNotRequiringException;
232 }
233 
235 {
236  bool ret = false;
237  switch (role())
238  {
240  ret = shm_manager_->IsValid();
241  break;
243  ret = shm_manager_->GetAttachedCount() > 1;
244  break;
245  }
246  return ret;
247 
248 }
249 
251 {
252  for (size_t ii = 0; ii < shm_manager_->size(); ++ii)
253  {
254  shm_manager_->MarkBufferEmpty(ii, true);
255  }
256 }
257 
258 DEFINE_ARTDAQ_TRANSFER(artdaq::ShmemTransfer)
259 
260 // Local Variables:
261 // mode: c++
262 // End:
size_t buffer_count_
The number of Fragment transfers the TransferInterface can handle simultaneously. ...
virtual int source_rank() const
Get the source rank for this TransferInterface instance.
CopyStatus transfer_fragment_reliable_mode(Fragment &&fragment) override
Transfer a Fragment to the destination. This should be reliable, if the underlying transport mechanis...
CopyStatus transfer_fragment_min_blocking_mode(Fragment const &fragment, size_t send_timeout_usec) override
Transfer a Fragment to the destination. May not necessarily be reliable, but will not block longer th...
This TransferInterface is a Receiver.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
void flush_buffers() override
Flush any in-flight data. This should be used by the receiver after the receive loop has ended...
ShmemTransfer(fhicl::ParameterSet const &pset, Role role)
ShmemTransfer Constructor.
Value that is to be returned when a Transfer plugin determines that no more data will be arriving...
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
This TransferInterface is a Sender.
virtual ~ShmemTransfer() noexcept
ShmemTransfer Destructor.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
A TransferInterface implementation plugin that transfers data using Shared Memory.
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
This interface defines the functions used to transfer data between artdaq applications.
virtual int destination_rank() const
Get the destination rank for this TransferInterface instance.
int receiveFragment(Fragment &fragment, size_t receiveTimeout) override
Receive a Fragment from Shared Memory.
Value to be returned upon receive timeout.
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.
const size_t max_fragment_size_words_
The maximum size of the transferred Fragment objects, in artdaq::Fragment::RawDataType words...