artdaq  v3_11_01
Shmem_transfer.cc
1 #include "artdaq/DAQdata/Globals.hh"
2 #define TRACE_NAME (app_name + "_ShmemTransfer").c_str()
3 
4 #include <csignal>
5 #include "artdaq/TransferPlugins/ShmemTransfer.hh"
6 #include "cetlib_except/exception.h"
7 
8 artdaq::ShmemTransfer::ShmemTransfer(fhicl::ParameterSet const& pset, Role role)
9  : TransferInterface(pset, role)
10 {
11  TLOG(TLVL_DEBUG) << GetTraceName() << "Constructor BEGIN";
12  // char* keyChars = getenv("ARTDAQ_SHM_KEY");
13  // if (keyChars != NULL && shm_key_ == static_cast<int>(std::hash<std::string>()(unique_label_))) {
14  // std::string keyString(keyChars);
15  // try {
16  // shm_key_ = boost::lexical_cast<int>(keyString);
17  // }
18  // catch (...) {
19  // std::stringstream errmsg;
20  // errmsg << uniqueLabel() << ": Problem performing lexical cast on " << keyString;
21  // ExceptionHandler(ExceptionHandlerRethrow::yes, errmsg.str());
22  // }
23  // }
24 
25  auto partition = GetPartitionNumber() + 1; // Can't be 0
26 
27  auto shmKey = pset.get<uint32_t>("shm_key_offset", 0) + (partition << 24) + ((source_rank() & 0xFFF) << 12) + (destination_rank() & 0xFFF);
28 
29  // Configured Shared Memory key overrides everything! Needed for Online Monitor connections!
30  if (pset.has_key("shm_key"))
31  {
32  shmKey = pset.get<uint32_t>("shm_key");
33  }
34 
35  if (role == Role::kReceive)
36  {
37  shm_manager_ = std::make_unique<SharedMemoryFragmentManager>(shmKey, buffer_count_, max_fragment_size_words_ * sizeof(artdaq::RawDataType), pset.get<size_t>("stale_buffer_timeout_usec", 100 * 1000000));
38  }
39  else
40  {
41  shm_manager_ = std::make_unique<SharedMemoryFragmentManager>(shmKey);
42  }
43  TLOG(TLVL_DEBUG) << GetTraceName() << "Constructor END";
44 }
45 
47 {
48  TLOG(TLVL_DEBUG + 2) << GetTraceName() << " ~ShmemTransfer called - " << uniqueLabel();
49  shm_manager_.reset(nullptr);
50  TLOG(TLVL_DEBUG + 2) << GetTraceName() << " ~ShmemTransfer done - " << uniqueLabel();
51 }
52 
53 int artdaq::ShmemTransfer::receiveFragment(artdaq::Fragment& fragment,
54  size_t receiveTimeout)
55 {
56  auto waitStart = std::chrono::steady_clock::now();
57  while (!shm_manager_->ReadyForRead() && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
58  {
59  // BURN THAT CPU!
60  }
61  if (!shm_manager_->ReadyForRead())
62  {
63  int64_t loopCount = 0;
64  size_t sleepTime = 1000; // microseconds
65  int64_t nloops = (receiveTimeout - 1000) / sleepTime;
66 
67  while (!shm_manager_->ReadyForRead() && loopCount < nloops)
68  {
69  usleep(sleepTime);
70  ++loopCount;
71  }
72  }
73  if (!shm_manager_->ReadyForRead() && shm_manager_->IsEndOfData())
74  {
76  }
77 
78  TLOG(TLVL_TRACE) << GetTraceName() << "receiveFragment ReadyForRead=" << shm_manager_->ReadyForRead();
79 
80  if (shm_manager_->ReadyForRead())
81  {
82  auto sts = shm_manager_->ReadFragment(fragment);
83 
84  if (sts != 0)
85  {
86  TLOG(TLVL_TRACE) << "Non-zero status (" << sts << ") returned from ReadFragment, returning...";
87  return RECV_TIMEOUT;
88  }
89 
90  if (fragment.type() != artdaq::Fragment::DataFragmentType)
91  {
92  TLOG(8) << GetTraceName() << "Recvd frag from shmem, type=" << fragment.typeString() << ", sequenceID=" << fragment.sequenceID() << ", source_rank=" << source_rank();
93  }
94 
95  return source_rank();
96  }
97 
99 }
100 
101 int artdaq::ShmemTransfer::receiveFragmentHeader(detail::RawFragmentHeader& header, size_t receiveTimeout)
102 {
103  auto waitStart = std::chrono::steady_clock::now();
104  while (!shm_manager_->ReadyForRead() && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
105  {
106  // BURN THAT CPU!
107  }
108  if (!shm_manager_->ReadyForRead())
109  {
110  int64_t loopCount = 0;
111  size_t sleepTime = 1000; // microseconds
112  int64_t nloops = (receiveTimeout - 1000) / sleepTime;
113 
114  while (!shm_manager_->ReadyForRead() && loopCount < nloops)
115  {
116  usleep(sleepTime);
117  ++loopCount;
118  }
119  }
120 
121  if (!shm_manager_->ReadyForRead() && shm_manager_->IsEndOfData())
122  {
124  }
125 
126  //TLOG(TLVL_TRACE) << GetTraceName() << "delta_=" << delta_() << ", rp=" << (int)shm_ptr_->read_pos << ", wp=" << (int)shm_ptr_->write_pos << ", loopCount=" << loopCount << ", nloops=" << nloops ;
127 
128  if (shm_manager_->ReadyForRead())
129  {
130  auto sts = shm_manager_->ReadFragmentHeader(header);
131 
132  if (sts != 0)
133  {
134  TLOG(TLVL_TRACE) << "Non-zero status (" << sts << ") returned from ReadFragmentHeader, returning...";
135  return RECV_TIMEOUT;
136  }
137 
138  if (header.type != artdaq::Fragment::DataFragmentType)
139  {
140  TLOG(8) << GetTraceName() << "Recvd fragment header from shmem, type=" << static_cast<int>(header.type)
141  << ", sequenceID=" << header.sequence_id << ", source_rank=" << source_rank();
142  }
143 
144  return source_rank();
145  }
146 
148 }
149 
150 int artdaq::ShmemTransfer::receiveFragmentData(RawDataType* destination, size_t word_count)
151 {
152  auto sts = shm_manager_->ReadFragmentData(destination, word_count);
153 
154  TLOG(TLVL_TRACE) << GetTraceName() << "Return status from ReadFragmentData is " << sts;
155 
156  if (sts != 0)
157  {
158  TLOG(TLVL_TRACE) << "Non-zero status (" << sts << ") returned from ReadFragmentData, returning...";
159  return RECV_TIMEOUT;
160  }
161 
162  return source_rank();
163 
165 }
166 
168 artdaq::ShmemTransfer::transfer_fragment_min_blocking_mode(artdaq::Fragment const& fragment, size_t send_timeout_usec)
169 {
170  return sendFragment(Fragment(fragment), send_timeout_usec, false);
171 }
172 
175 {
176  return sendFragment(std::move(fragment), 0, true);
177 }
178 
180 artdaq::ShmemTransfer::sendFragment(artdaq::Fragment&& fragment, size_t send_timeout_usec, bool reliableMode)
181 {
182  if (!isRunning())
183  {
184  shm_manager_->Attach();
185  if (!isRunning())
186  {
187  TLOG(TLVL_ERROR) << GetTraceName() << "Attempted to send Fragment when not attached to Shared Memory! Returning kErrorNotRequiringException, and dropping data!";
188  return CopyStatus::kErrorNotRequiringException;
189  }
190  }
191  shm_manager_->SetRank(my_rank);
192  // wait for the shm to become free, if requested
193 
194  TLOG(TLVL_DEBUG + 2) << GetTraceName() << "Sending fragment with seqID=" << fragment.sequenceID();
195  artdaq::RawDataType* fragAddr = fragment.headerAddress();
196  size_t fragSize = fragment.size() * sizeof(artdaq::RawDataType);
197 
198  // 10-Sep-2013, KAB - protect against large events and
199  // invalid events (and large, invalid events)
200  if (fragment.type() != artdaq::Fragment::InvalidFragmentType && fragSize < (max_fragment_size_words_ * sizeof(artdaq::RawDataType)))
201  {
202  auto seq = fragment.sequenceID();
203  TLOG(TLVL_DEBUG + 2) << GetTraceName() << "Writing fragment with seqID=" << seq;
204  auto sts = shm_manager_->WriteFragment(std::move(fragment), !reliableMode, send_timeout_usec);
205  if (sts == -3)
206  {
207  TLOG(TLVL_WARNING) << GetTraceName() << "Timeout writing fragment with seqID=" << seq;
208  return CopyStatus::kTimeout;
209  }
210  if (sts != 0)
211  {
212  TLOG(TLVL_WARNING) << GetTraceName() << "Error writing fragment with seqID=" << seq;
213  return CopyStatus::kErrorNotRequiringException;
214  }
215 
216  TLOG(TLVL_DEBUG + 2) << GetTraceName() << "Successfully sent Fragment with seqID=" << seq;
217  return CopyStatus::kSuccess;
218  }
219 
220  TLOG(TLVL_WARNING) << GetTraceName() << "Fragment invalid for shared memory! "
221  << "fragment address and size = "
222  << fragAddr << " " << fragSize << " "
223  << "sequence ID, fragment ID, and type = "
224  << fragment.sequenceID() << " "
225  << fragment.fragmentID() << " "
226  << fragment.typeString();
227  return CopyStatus::kErrorNotRequiringException;
228 
229  TLOG(TLVL_WARNING) << GetTraceName() << "Unreachable code reached!";
230  return CopyStatus::kErrorNotRequiringException;
231 }
232 
234 {
235  bool ret = false;
236  switch (role())
237  {
239  ret = shm_manager_->IsValid() && !shm_manager_->IsEndOfData();
240  break;
242  ret = shm_manager_->GetAttachedCount() > 1;
243  break;
244  }
245  return ret;
246 }
247 
249 {
250  for (size_t ii = 0; ii < shm_manager_->size(); ++ii)
251  {
252  shm_manager_->MarkBufferEmpty(ii, true);
253  }
254 }
255 
256 DEFINE_ARTDAQ_TRANSFER(artdaq::ShmemTransfer)
257 
258 // Local Variables:
259 // mode: c++
260 // End:
size_t buffer_count_
The number of Fragment transfers the TransferInterface can handle simultaneously. ...
virtual int source_rank() const
Get the source rank for this TransferInterface instance.
CopyStatus transfer_fragment_reliable_mode(Fragment &&fragment) override
Transfer a Fragment to the destination. This should be reliable, if the underlying transport mechanis...
CopyStatus transfer_fragment_min_blocking_mode(Fragment const &fragment, size_t send_timeout_usec) override
Transfer a Fragment to the destination. May not necessarily be reliable, but will not block longer th...
This TransferInterface is a Receiver.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
void flush_buffers() override
Flush any in-flight data. This should be used by the receiver after the receive loop has ended...
ShmemTransfer(fhicl::ParameterSet const &pset, Role role)
ShmemTransfer Constructor.
Value that is to be returned when a Transfer plugin determines that no more data will be arriving...
This TransferInterface is a Sender.
virtual ~ShmemTransfer() noexcept
ShmemTransfer Destructor.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
A TransferInterface implementation plugin that transfers data using Shared Memory.
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
This interface defines the functions used to transfer data between artdaq applications.
virtual int destination_rank() const
Get the destination rank for this TransferInterface instance.
int receiveFragment(Fragment &fragment, size_t receiveTimeout) override
Receive a Fragment from Shared Memory.
Value to be returned upon receive timeout.
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.
int receiveFragmentData(RawDataType *destination, size_t word_count) override
Receive the body of a Fragment to the given destination pointer.
const size_t max_fragment_size_words_
The maximum size of the transferred Fragment objects, in artdaq::Fragment::RawDataType words...