artdaq  v3_12_02
ShmemTransfer.cc
1 #include "artdaq/DAQdata/Globals.hh"
2 #define TRACE_NAME (app_name + "_ShmemTransfer").c_str()
3 #include "TRACE/tracemf.h"
4 
5 #include "artdaq/TransferPlugins/ShmemTransfer.hh"
6 
7 #include <boost/lexical_cast.hpp>
8 
9 #include <csignal>
10 #include <memory>
11 
12 artdaq::ShmemTransfer::ShmemTransfer(fhicl::ParameterSet const& pset, Role role)
13  : TransferInterface(pset, role)
14 {
15  TLOG(TLVL_DEBUG + 32) << GetTraceName() << "Constructor BEGIN";
16  // char* keyChars = getenv("ARTDAQ_SHM_KEY");
17  // if (keyChars != NULL && shm_key_ == static_cast<int>(std::hash<std::string>()(unique_label_))) {
18  // std::string keyString(keyChars);
19  // try {
20  // shm_key_ = boost::lexical_cast<int>(keyString);
21  // }
22  // catch (...) {
23  // std::stringstream errmsg;
24  // errmsg << uniqueLabel() << ": Problem performing lexical cast on " << keyString;
25  // ExceptionHandler(ExceptionHandlerRethrow::yes, errmsg.str());
26  // }
27  // }
28 
29  auto partition = GetPartitionNumber() + 1; // Can't be 0
30 
31  auto shmKey = pset.get<uint32_t>("shm_key_offset", 0) + (partition << 24) + ((source_rank() & 0xFFF) << 12) + (destination_rank() & 0xFFF);
32 
33  // Configured Shared Memory key overrides everything! Needed for Online Monitor connections!
34  if (pset.has_key("shm_key"))
35  {
36  shmKey = pset.get<uint32_t>("shm_key");
37  }
38 
39  if (role == Role::kReceive)
40  {
41  shm_manager_ = std::make_unique<SharedMemoryFragmentManager>(shmKey, buffer_count_, max_fragment_size_words_ * sizeof(artdaq::RawDataType), pset.get<size_t>("stale_buffer_timeout_usec", 100 * 1000000));
42  }
43  else
44  {
45  shm_manager_ = std::make_unique<SharedMemoryFragmentManager>(shmKey);
46  }
47  TLOG(TLVL_DEBUG + 32) << GetTraceName() << "Constructor END";
48 }
49 
51 {
52  TLOG(TLVL_DEBUG + 34) << GetTraceName() << " ~ShmemTransfer called - " << uniqueLabel();
53  shm_manager_.reset(nullptr);
54  TLOG(TLVL_DEBUG + 34) << GetTraceName() << " ~ShmemTransfer done - " << uniqueLabel();
55 }
56 
57 int artdaq::ShmemTransfer::receiveFragment(artdaq::Fragment& fragment,
58  size_t receiveTimeout)
59 {
60  auto waitStart = std::chrono::steady_clock::now();
61  while (!shm_manager_->ReadyForRead() && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
62  {
63  // BURN THAT CPU!
64  }
65  if (!shm_manager_->ReadyForRead())
66  {
67  int64_t loopCount = 0;
68  size_t sleepTime = 1000; // microseconds
69  int64_t nloops = (receiveTimeout - 1000) / sleepTime;
70 
71  while (!shm_manager_->ReadyForRead() && loopCount < nloops)
72  {
73  usleep(sleepTime);
74  ++loopCount;
75  }
76  }
77  if (!shm_manager_->ReadyForRead() && shm_manager_->IsEndOfData())
78  {
80  }
81 
82  TLOG(TLVL_DEBUG + 33) << GetTraceName() << "receiveFragment ReadyForRead=" << shm_manager_->ReadyForRead();
83 
84  if (shm_manager_->ReadyForRead())
85  {
86  auto sts = shm_manager_->ReadFragment(fragment);
87 
88  if (sts != 0)
89  {
90  TLOG(TLVL_DEBUG + 33) << "Non-zero status (" << sts << ") returned from ReadFragment, returning...";
91  return RECV_TIMEOUT;
92  }
93 
94  if (fragment.type() != artdaq::Fragment::DataFragmentType)
95  {
96  TLOG(TLVL_DEBUG + 38) << GetTraceName() << "Recvd frag from shmem, type=" << fragment.typeString() << ", sequenceID=" << fragment.sequenceID() << ", source_rank=" << source_rank();
97  }
98 
99  return source_rank();
100  }
101 
103 }
104 
105 int artdaq::ShmemTransfer::receiveFragmentHeader(detail::RawFragmentHeader& header, size_t receiveTimeout)
106 {
107  auto waitStart = std::chrono::steady_clock::now();
108  while (!shm_manager_->ReadyForRead() && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
109  {
110  // BURN THAT CPU!
111  }
112  if (!shm_manager_->ReadyForRead())
113  {
114  int64_t loopCount = 0;
115  size_t sleepTime = 1000; // microseconds
116  int64_t nloops = (receiveTimeout - 1000) / sleepTime;
117 
118  while (!shm_manager_->ReadyForRead() && loopCount < nloops)
119  {
120  usleep(sleepTime);
121  ++loopCount;
122  }
123  }
124 
125  if (!shm_manager_->ReadyForRead() && shm_manager_->IsEndOfData())
126  {
128  }
129 
130  // TLOG(TLVL_DEBUG + 33) << GetTraceName() << "delta_=" << delta_() << ", rp=" << (int)shm_ptr_->read_pos << ", wp=" << (int)shm_ptr_->write_pos << ", loopCount=" << loopCount << ", nloops=" << nloops ;
131 
132  if (shm_manager_->ReadyForRead())
133  {
134  auto sts = shm_manager_->ReadFragmentHeader(header);
135 
136  if (sts != 0)
137  {
138  TLOG(TLVL_DEBUG + 33) << "Non-zero status (" << sts << ") returned from ReadFragmentHeader, returning...";
139  return RECV_TIMEOUT;
140  }
141 
142  if (header.type != artdaq::Fragment::DataFragmentType)
143  {
144  TLOG(TLVL_DEBUG + 38) << GetTraceName() << "Recvd fragment header from shmem, type=" << static_cast<int>(header.type)
145  << ", sequenceID=" << header.sequence_id << ", source_rank=" << source_rank();
146  }
147 
148  return source_rank();
149  }
150 
152 }
153 
154 int artdaq::ShmemTransfer::receiveFragmentData(RawDataType* destination, size_t word_count)
155 {
156  auto sts = shm_manager_->ReadFragmentData(destination, word_count);
157 
158  TLOG(TLVL_DEBUG + 33) << GetTraceName() << "Return status from ReadFragmentData is " << sts;
159 
160  if (sts != 0)
161  {
162  TLOG(TLVL_DEBUG + 33) << "Non-zero status (" << sts << ") returned from ReadFragmentData, returning...";
163  return RECV_TIMEOUT;
164  }
165 
166  return source_rank();
167 
169 }
170 
172 artdaq::ShmemTransfer::transfer_fragment_min_blocking_mode(artdaq::Fragment const& fragment, size_t send_timeout_usec)
173 {
174  return sendFragment(Fragment(fragment), send_timeout_usec, false);
175 }
176 
179 {
180  return sendFragment(std::move(fragment), 0, true);
181 }
182 
184 artdaq::ShmemTransfer::sendFragment(artdaq::Fragment&& fragment, size_t send_timeout_usec, bool reliableMode)
185 {
186  if (!isRunning())
187  {
188  shm_manager_->Attach();
189  if (!isRunning())
190  {
191  TLOG(TLVL_ERROR) << GetTraceName() << "Attempted to send Fragment when not attached to Shared Memory! Returning kErrorNotRequiringException, and dropping data!";
192  return CopyStatus::kErrorNotRequiringException;
193  }
194  }
195  shm_manager_->SetRank(my_rank);
196  // wait for the shm to become free, if requested
197 
198  TLOG(TLVL_DEBUG + 34) << GetTraceName() << "Sending fragment with seqID=" << fragment.sequenceID();
199  artdaq::RawDataType* fragAddr = fragment.headerAddress();
200  size_t fragSize = fragment.size() * sizeof(artdaq::RawDataType);
201 
202  // 10-Sep-2013, KAB - protect against large events and
203  // invalid events (and large, invalid events)
204  if (fragment.type() != artdaq::Fragment::InvalidFragmentType && fragSize < (max_fragment_size_words_ * sizeof(artdaq::RawDataType)))
205  {
206  auto seq = fragment.sequenceID();
207  TLOG(TLVL_DEBUG + 34) << GetTraceName() << "Writing fragment with seqID=" << seq;
208  auto sts = shm_manager_->WriteFragment(std::move(fragment), !reliableMode, send_timeout_usec);
209  if (sts == -3)
210  {
211  TLOG(TLVL_WARNING) << GetTraceName() << "Timeout writing fragment with seqID=" << seq;
212  return CopyStatus::kTimeout;
213  }
214  if (sts != 0)
215  {
216  TLOG(TLVL_WARNING) << GetTraceName() << "Error writing fragment with seqID=" << seq;
217  return CopyStatus::kErrorNotRequiringException;
218  }
219 
220  TLOG(TLVL_DEBUG + 34) << GetTraceName() << "Successfully sent Fragment with seqID=" << seq;
221  return CopyStatus::kSuccess;
222  }
223 
224  TLOG(TLVL_WARNING) << GetTraceName() << "Fragment invalid for shared memory! "
225  << "fragment address and size = "
226  << fragAddr << " " << fragSize << " "
227  << "sequence ID, fragment ID, and type = "
228  << fragment.sequenceID() << " "
229  << fragment.fragmentID() << " "
230  << fragment.typeString();
231  return CopyStatus::kErrorNotRequiringException;
232 
233  TLOG(TLVL_WARNING) << GetTraceName() << "Unreachable code reached!";
234  return CopyStatus::kErrorNotRequiringException;
235 }
236 
238 {
239  bool ret = false;
240  switch (role())
241  {
243  ret = shm_manager_->IsValid() && !shm_manager_->IsEndOfData();
244  break;
246  ret = shm_manager_->GetAttachedCount() > 1;
247  break;
248  }
249  return ret;
250 }
251 
253 {
254  for (size_t ii = 0; ii < shm_manager_->size(); ++ii)
255  {
256  shm_manager_->MarkBufferEmpty(ii, true);
257  }
258 }
259 
260 // Local Variables:
261 // mode: c++
262 // End:
size_t buffer_count_
The number of Fragment transfers the TransferInterface can handle simultaneously. ...
virtual int source_rank() const
Get the source rank for this TransferInterface instance.
CopyStatus transfer_fragment_reliable_mode(Fragment &&fragment) override
Transfer a Fragment to the destination. This should be reliable, if the underlying transport mechanis...
CopyStatus transfer_fragment_min_blocking_mode(Fragment const &fragment, size_t send_timeout_usec) override
Transfer a Fragment to the destination. May not necessarily be reliable, but will not block longer th...
This TransferInterface is a Receiver.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
void flush_buffers() override
Flush any in-flight data. This should be used by the receiver after the receive loop has ended...
ShmemTransfer(fhicl::ParameterSet const &pset, Role role)
ShmemTransfer Constructor.
Value that is to be returned when a Transfer plugin determines that no more data will be arriving...
This TransferInterface is a Sender.
virtual ~ShmemTransfer() noexcept
ShmemTransfer Destructor.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
This interface defines the functions used to transfer data between artdaq applications.
virtual int destination_rank() const
Get the destination rank for this TransferInterface instance.
int receiveFragment(Fragment &fragment, size_t receiveTimeout) override
Receive a Fragment from Shared Memory.
Value to be returned upon receive timeout.
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.
int receiveFragmentData(RawDataType *destination, size_t word_count) override
Receive the body of a Fragment to the given destination pointer.
const size_t max_fragment_size_words_
The maximum size of the transferred Fragment objects, in artdaq::Fragment::RawDataType words...