artdaq  v3_09_00
Shmem_transfer.cc
1 #include "artdaq/DAQdata/Globals.hh"
2 #define TRACE_NAME (app_name + "_ShmemTransfer").c_str()
3 
4 #include <csignal>
5 #include "artdaq/TransferPlugins/ShmemTransfer.hh"
6 #include "cetlib_except/exception.h"
7 
8 artdaq::ShmemTransfer::ShmemTransfer(fhicl::ParameterSet const& pset, Role role)
9  : TransferInterface(pset, role)
10 {
11  TLOG(TLVL_DEBUG) << GetTraceName() << "Constructor BEGIN";
12  // char* keyChars = getenv("ARTDAQ_SHM_KEY");
13  // if (keyChars != NULL && shm_key_ == static_cast<int>(std::hash<std::string>()(unique_label_))) {
14  // std::string keyString(keyChars);
15  // try {
16  // shm_key_ = boost::lexical_cast<int>(keyString);
17  // }
18  // catch (...) {
19  // std::stringstream errmsg;
20  // errmsg << uniqueLabel() << ": Problem performing lexical cast on " << keyString;
21  // ExceptionHandler(ExceptionHandlerRethrow::yes, errmsg.str());
22  // }
23  // }
24 
25  auto partition = GetPartitionNumber() + 1; // Can't be 0
26 
27  auto shmKey = pset.get<uint32_t>("shm_key_offset", 0) + (partition << 24) + ((source_rank() & 0xFFF) << 12) + (destination_rank() & 0xFFF);
28 
29  // Configured Shared Memory key overrides everything! Needed for Online Monitor connections!
30  if (pset.has_key("shm_key"))
31  {
32  shmKey = pset.get<uint32_t>("shm_key");
33  }
34 
35  if (role == Role::kReceive)
36  {
37  shm_manager_ = std::make_unique<SharedMemoryFragmentManager>(shmKey, buffer_count_, max_fragment_size_words_ * sizeof(artdaq::RawDataType));
38  }
39  else
40  {
41  shm_manager_ = std::make_unique<SharedMemoryFragmentManager>(shmKey);
42  }
43  TLOG(TLVL_DEBUG) << GetTraceName() << "Constructor END";
44 }
45 
47 {
48  TLOG(5) << GetTraceName() << " ~ShmemTransfer called - " << uniqueLabel();
49  shm_manager_.reset(nullptr);
50  TLOG(5) << GetTraceName() << " ~ShmemTransfer done - " << uniqueLabel();
51 }
52 
53 int artdaq::ShmemTransfer::receiveFragment(artdaq::Fragment& fragment,
54  size_t receiveTimeout)
55 {
56  auto waitStart = std::chrono::steady_clock::now();
57  while (!shm_manager_->ReadyForRead() && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
58  {
59  // BURN THAT CPU!
60  }
61  if (!shm_manager_->ReadyForRead())
62  {
63  int64_t loopCount = 0;
64  size_t sleepTime = 1000; // microseconds
65  int64_t nloops = (receiveTimeout - 1000) / sleepTime;
66 
67  while (!shm_manager_->ReadyForRead() && loopCount < nloops)
68  {
69  usleep(sleepTime);
70  ++loopCount;
71  }
72  }
73 
74  TLOG(TLVL_TRACE) << GetTraceName() << "receiveFragment ReadyForRead=" << shm_manager_->ReadyForRead();
75 
76  if (shm_manager_->ReadyForRead())
77  {
78  auto sts = shm_manager_->ReadFragment(fragment);
79 
80  if (sts != 0)
81  {
82  TLOG(TLVL_TRACE) << "Non-zero status (" << sts << ") returned from ReadFragment, returning...";
83  return RECV_TIMEOUT;
84  }
85 
86  if (fragment.type() != artdaq::Fragment::DataFragmentType)
87  {
88  TLOG(8) << GetTraceName() << "Recvd frag from shmem, type=" << fragment.typeString() << ", sequenceID=" << fragment.sequenceID() << ", source_rank=" << source_rank();
89  }
90 
91  return source_rank();
92  }
93 
95 }
96 
97 int artdaq::ShmemTransfer::receiveFragmentHeader(detail::RawFragmentHeader& header, size_t receiveTimeout)
98 {
99  auto waitStart = std::chrono::steady_clock::now();
100  while (!shm_manager_->ReadyForRead() && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
101  {
102  // BURN THAT CPU!
103  }
104  if (!shm_manager_->ReadyForRead())
105  {
106  int64_t loopCount = 0;
107  size_t sleepTime = 1000; // microseconds
108  int64_t nloops = (receiveTimeout - 1000) / sleepTime;
109 
110  while (!shm_manager_->ReadyForRead() && loopCount < nloops)
111  {
112  usleep(sleepTime);
113  ++loopCount;
114  }
115  }
116 
117  if (!shm_manager_->ReadyForRead() && shm_manager_->IsEndOfData())
118  {
120  }
121 
122  //TLOG(TLVL_TRACE) << GetTraceName() << "delta_=" << delta_() << ", rp=" << (int)shm_ptr_->read_pos << ", wp=" << (int)shm_ptr_->write_pos << ", loopCount=" << loopCount << ", nloops=" << nloops ;
123 
124  if (shm_manager_->ReadyForRead())
125  {
126  auto sts = shm_manager_->ReadFragmentHeader(header);
127 
128  if (sts != 0)
129  {
130  TLOG(TLVL_TRACE) << "Non-zero status (" << sts << ") returned from ReadFragmentHeader, returning...";
131  return RECV_TIMEOUT;
132  }
133 
134  if (header.type != artdaq::Fragment::DataFragmentType)
135  {
136  TLOG(8) << GetTraceName() << "Recvd fragment header from shmem, type=" << static_cast<int>(header.type)
137  << ", sequenceID=" << header.sequence_id << ", source_rank=" << source_rank();
138  }
139 
140  return source_rank();
141  }
142 
144 }
145 
146 int artdaq::ShmemTransfer::receiveFragmentData(RawDataType* destination, size_t word_count)
147 {
148  auto sts = shm_manager_->ReadFragmentData(destination, word_count);
149 
150  TLOG(TLVL_TRACE) << GetTraceName() << "Return status from ReadFragmentData is " << sts;
151 
152  if (sts != 0)
153  {
154  TLOG(TLVL_TRACE) << "Non-zero status (" << sts << ") returned from ReadFragmentData, returning...";
155  return RECV_TIMEOUT;
156  }
157 
158  return source_rank();
159 
161 }
162 
164 artdaq::ShmemTransfer::transfer_fragment_min_blocking_mode(artdaq::Fragment const& fragment, size_t send_timeout_usec)
165 {
166  return sendFragment(Fragment(fragment), send_timeout_usec, false);
167 }
168 
171 {
172  return sendFragment(std::move(fragment), 0, true);
173 }
174 
176 artdaq::ShmemTransfer::sendFragment(artdaq::Fragment&& fragment, size_t send_timeout_usec, bool reliableMode)
177 {
178  if (!isRunning())
179  {
180  shm_manager_->Attach();
181  if (!isRunning())
182  {
183  TLOG(TLVL_ERROR) << GetTraceName() << "Attempted to send Fragment when not attached to Shared Memory! Returning kErrorNotRequiringException, and dropping data!";
184  return CopyStatus::kErrorNotRequiringException;
185  }
186  }
187  shm_manager_->SetRank(my_rank);
188  // wait for the shm to become free, if requested
189 
190  TLOG(5) << GetTraceName() << "Sending fragment with seqID=" << fragment.sequenceID();
191  artdaq::RawDataType* fragAddr = fragment.headerAddress();
192  size_t fragSize = fragment.size() * sizeof(artdaq::RawDataType);
193 
194  // 10-Sep-2013, KAB - protect against large events and
195  // invalid events (and large, invalid events)
196  if (fragment.type() != artdaq::Fragment::InvalidFragmentType && fragSize < (max_fragment_size_words_ * sizeof(artdaq::RawDataType)))
197  {
198  auto seq = fragment.sequenceID();
199  TLOG(5) << GetTraceName() << "Writing fragment with seqID=" << seq;
200  auto sts = shm_manager_->WriteFragment(std::move(fragment), !reliableMode, send_timeout_usec);
201  if (sts == -3)
202  {
203  TLOG(TLVL_WARNING) << GetTraceName() << "Timeout writing fragment with seqID=" << seq;
204  return CopyStatus::kTimeout;
205  }
206  if (sts != 0)
207  {
208  TLOG(TLVL_WARNING) << GetTraceName() << "Error writing fragment with seqID=" << seq;
209  return CopyStatus::kErrorNotRequiringException;
210  }
211 
212  TLOG(5) << GetTraceName() << "Successfully sent Fragment with seqID=" << seq;
213  return CopyStatus::kSuccess;
214  }
215 
216  TLOG(TLVL_WARNING) << GetTraceName() << "Fragment invalid for shared memory! "
217  << "fragment address and size = "
218  << fragAddr << " " << fragSize << " "
219  << "sequence ID, fragment ID, and type = "
220  << fragment.sequenceID() << " "
221  << fragment.fragmentID() << " "
222  << fragment.typeString();
223  return CopyStatus::kErrorNotRequiringException;
224 
225  TLOG(TLVL_WARNING) << GetTraceName() << "Unreachable code reached!";
226  return CopyStatus::kErrorNotRequiringException;
227 }
228 
230 {
231  bool ret = false;
232  switch (role())
233  {
235  ret = shm_manager_->IsValid() && !shm_manager_->IsEndOfData();
236  break;
238  ret = shm_manager_->GetAttachedCount() > 1;
239  break;
240  }
241  return ret;
242 }
243 
245 {
246  for (size_t ii = 0; ii < shm_manager_->size(); ++ii)
247  {
248  shm_manager_->MarkBufferEmpty(ii, true);
249  }
250 }
251 
252 DEFINE_ARTDAQ_TRANSFER(artdaq::ShmemTransfer)
253 
254 // Local Variables:
255 // mode: c++
256 // End:
size_t buffer_count_
The number of Fragment transfers the TransferInterface can handle simultaneously. ...
virtual int source_rank() const
Get the source rank for this TransferInterface instance.
CopyStatus transfer_fragment_reliable_mode(Fragment &&fragment) override
Transfer a Fragment to the destination. This should be reliable, if the underlying transport mechanis...
CopyStatus transfer_fragment_min_blocking_mode(Fragment const &fragment, size_t send_timeout_usec) override
Transfer a Fragment to the destination. May not necessarily be reliable, but will not block longer th...
This TransferInterface is a Receiver.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
void flush_buffers() override
Flush any in-flight data. This should be used by the receiver after the receive loop has ended...
ShmemTransfer(fhicl::ParameterSet const &pset, Role role)
ShmemTransfer Constructor.
Value that is to be returned when a Transfer plugin determines that no more data will be arriving...
This TransferInterface is a Sender.
virtual ~ShmemTransfer() noexcept
ShmemTransfer Destructor.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
A TransferInterface implementation plugin that transfers data using Shared Memory.
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
This interface defines the functions used to transfer data between artdaq applications.
virtual int destination_rank() const
Get the destination rank for this TransferInterface instance.
int receiveFragment(Fragment &fragment, size_t receiveTimeout) override
Receive a Fragment from Shared Memory.
Value to be returned upon receive timeout.
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.
int receiveFragmentData(RawDataType *destination, size_t word_count) override
Receive the body of a Fragment to the given destination pointer.
const size_t max_fragment_size_words_
The maximum size of the transferred Fragment objects, in artdaq::Fragment::RawDataType words...