artdaq  v3_07_01
Shmem_transfer.cc
1 #include "artdaq/DAQdata/Globals.hh"
2 #define TRACE_NAME (app_name + "_ShmemTransfer").c_str()
3 
4 #include <signal.h>
5 #include "artdaq/TransferPlugins/ShmemTransfer.hh"
6 #include "cetlib_except/exception.h"
7 
8 artdaq::ShmemTransfer::ShmemTransfer(fhicl::ParameterSet const& pset, Role role)
9  : TransferInterface(pset, role)
10 {
11  TLOG(TLVL_DEBUG) << GetTraceName() << ": Constructor BEGIN";
12  // char* keyChars = getenv("ARTDAQ_SHM_KEY");
13  // if (keyChars != NULL && shm_key_ == static_cast<int>(std::hash<std::string>()(unique_label_))) {
14  // std::string keyString(keyChars);
15  // try {
16  // shm_key_ = boost::lexical_cast<int>(keyString);
17  // }
18  // catch (...) {
19  // std::stringstream errmsg;
20  // errmsg << uniqueLabel() << ": Problem performing lexical cast on " << keyString;
21  // ExceptionHandler(ExceptionHandlerRethrow::yes, errmsg.str());
22  // }
23  // }
24 
25  auto partition = GetPartitionNumber() + 1; // Can't be 0
26 
27  auto shmKey = pset.get<uint32_t>("shm_key_offset", 0) + (partition << 24) + ((source_rank() & 0xFFF) << 12) + (destination_rank() & 0xFFF);
28 
29  // Configured Shared Memory key overrides everything! Needed for Online Monitor connections!
30  if (pset.has_key("shm_key"))
31  {
32  shmKey = pset.get<uint32_t>("shm_key");
33  }
34 
35  if (role == Role::kReceive)
36  {
37  shm_manager_ = std::make_unique<SharedMemoryFragmentManager>(shmKey, buffer_count_, max_fragment_size_words_ * sizeof(artdaq::RawDataType));
38  }
39  else
40  {
41  shm_manager_ = std::make_unique<SharedMemoryFragmentManager>(shmKey);
42  }
43  TLOG(TLVL_DEBUG) << GetTraceName() << ": Constructor END";
44 }
45 
47 {
48  TLOG(5) << GetTraceName() << " ~ShmemTransfer called - " << uniqueLabel();
49  shm_manager_.reset(nullptr);
50  TLOG(5) << GetTraceName() << " ~ShmemTransfer done - " << uniqueLabel();
51 }
52 
53 int artdaq::ShmemTransfer::receiveFragment(artdaq::Fragment& fragment,
54  size_t receiveTimeout)
55 {
56  auto waitStart = std::chrono::steady_clock::now();
57  while (!shm_manager_->ReadyForRead() && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
58  {
59  // BURN THAT CPU!
60  }
61  if (!shm_manager_->ReadyForRead())
62  {
63  int64_t loopCount = 0;
64  size_t sleepTime = 1000; // microseconds
65  int64_t nloops = (receiveTimeout - 1000) / sleepTime;
66 
67  while (!shm_manager_->ReadyForRead() && loopCount < nloops)
68  {
69  usleep(sleepTime);
70  ++loopCount;
71  }
72  }
73 
74  TLOG(TLVL_TRACE) << GetTraceName() << ": receiveFragment ReadyForRead=" << shm_manager_->ReadyForRead();
75 
76  if (shm_manager_->ReadyForRead())
77  {
78  auto sts = shm_manager_->ReadFragment(fragment);
79 
80  if (sts != 0)
81  {
82  TLOG(TLVL_TRACE) << "Non-zero status (" << sts << ") returned from ReadFragment, returning...";
83  return RECV_TIMEOUT;
84  }
85 
86  if (fragment.type() != artdaq::Fragment::DataFragmentType)
87  {
88  TLOG(8) << GetTraceName() << ": Recvd frag from shmem, type=" << fragment.typeString() << ", sequenceID=" << fragment.sequenceID() << ", source_rank=" << source_rank();
89  }
90 
91  return source_rank();
92  }
93 
95 }
96 
97 int artdaq::ShmemTransfer::receiveFragmentHeader(detail::RawFragmentHeader& header, size_t receiveTimeout)
98 {
99  auto waitStart = std::chrono::steady_clock::now();
100  while (!shm_manager_->ReadyForRead() && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
101  {
102  // BURN THAT CPU!
103  }
104  if (!shm_manager_->ReadyForRead())
105  {
106  int64_t loopCount = 0;
107  size_t sleepTime = 1000; // microseconds
108  int64_t nloops = (receiveTimeout - 1000) / sleepTime;
109 
110  while (!shm_manager_->ReadyForRead() && loopCount < nloops)
111  {
112  usleep(sleepTime);
113  ++loopCount;
114  }
115  }
116 
117  if (!shm_manager_->ReadyForRead() && shm_manager_->IsEndOfData())
118  {
120  }
121 
122  //TLOG(TLVL_TRACE) << GetTraceName() << ": delta_=" << delta_() << ", rp=" << (int)shm_ptr_->read_pos << ", wp=" << (int)shm_ptr_->write_pos << ", loopCount=" << loopCount << ", nloops=" << nloops ;
123 
124  if (shm_manager_->ReadyForRead())
125  {
126  auto sts = shm_manager_->ReadFragmentHeader(header);
127 
128  if (sts != 0)
129  {
130  TLOG(TLVL_TRACE) << "Non-zero status (" << sts << ") returned from ReadFragmentHeader, returning...";
131  return RECV_TIMEOUT;
132  }
133 
134  if (header.type != artdaq::Fragment::DataFragmentType)
135  {
136  TLOG(8) << GetTraceName() << ": Recvd fragment header from shmem, type=" << (int)header.type << ", sequenceID=" << header.sequence_id << ", source_rank=" << source_rank();
137  }
138 
139  return source_rank();
140  }
141 
143 }
144 
145 int artdaq::ShmemTransfer::receiveFragmentData(RawDataType* destination, size_t word_count)
146 {
147  auto sts = shm_manager_->ReadFragmentData(destination, word_count);
148 
149  TLOG(TLVL_TRACE) << GetTraceName() << ": Return status from ReadFragmentData is " << sts;
150 
151  if (sts != 0)
152  {
153  TLOG(TLVL_TRACE) << "Non-zero status (" << sts << ") returned from ReadFragmentData, returning...";
154  return RECV_TIMEOUT;
155  }
156 
157  return source_rank();
158 
160 }
161 
163 artdaq::ShmemTransfer::transfer_fragment_min_blocking_mode(artdaq::Fragment const& fragment, size_t send_timeout_usec)
164 {
165  return sendFragment(Fragment(fragment), send_timeout_usec, false);
166 }
167 
170 {
171  return sendFragment(std::move(fragment), 0, true);
172 }
173 
175 artdaq::ShmemTransfer::sendFragment(artdaq::Fragment&& fragment, size_t send_timeout_usec, bool reliableMode)
176 {
177  if (!isRunning())
178  {
179  shm_manager_->Attach();
180  if (!isRunning())
181  {
182  TLOG(TLVL_ERROR) << GetTraceName() << ": Attempted to send Fragment when not attached to Shared Memory! Returning kSuccess, and dropping data!";
183  return CopyStatus::kSuccess;
184  }
185  }
186  shm_manager_->SetRank(my_rank);
187  // wait for the shm to become free, if requested
188 
189  TLOG(5) << GetTraceName() << ": Sending fragment with seqID=" << fragment.sequenceID();
190  artdaq::RawDataType* fragAddr = fragment.headerAddress();
191  size_t fragSize = fragment.size() * sizeof(artdaq::RawDataType);
192 
193  // 10-Sep-2013, KAB - protect against large events and
194  // invalid events (and large, invalid events)
195  if (fragment.type() != artdaq::Fragment::InvalidFragmentType && fragSize < (max_fragment_size_words_ * sizeof(artdaq::RawDataType)))
196  {
197  TLOG(5) << GetTraceName() << ": Writing fragment with seqID=" << fragment.sequenceID();
198  auto sts = shm_manager_->WriteFragment(std::move(fragment), !reliableMode, send_timeout_usec);
199  if (sts == -3)
200  {
201  TLOG(TLVL_WARNING) << GetTraceName() << ": Timeout writing fragment with seqID=" << fragment.sequenceID();
202  return CopyStatus::kTimeout;
203  }
204  if (sts != 0)
205  {
206  TLOG(TLVL_WARNING) << GetTraceName() << ": Error writing fragment with seqID=" << fragment.sequenceID();
207  return CopyStatus::kErrorNotRequiringException;
208  }
209 
210  TLOG(5) << GetTraceName() << ": Successfully sent Fragment with seqID=" << fragment.sequenceID();
211  return CopyStatus::kSuccess;
212  }
213  else
214  {
215  TLOG(TLVL_WARNING) << GetTraceName() << ": Fragment invalid for shared memory! "
216  << "fragment address and size = "
217  << fragAddr << " " << fragSize << " "
218  << "sequence ID, fragment ID, and type = "
219  << fragment.sequenceID() << " "
220  << fragment.fragmentID() << " "
221  << fragment.typeString();
222  return CopyStatus::kErrorNotRequiringException;
223  }
224 
225  TLOG(TLVL_WARNING) << GetTraceName() << ": Unreachable code reached!";
226  return CopyStatus::kErrorNotRequiringException;
227 }
228 
230 {
231  bool ret = false;
232  switch (role())
233  {
235  ret = shm_manager_->IsValid() && !shm_manager_->IsEndOfData();
236  break;
238  ret = shm_manager_->GetAttachedCount() > 1;
239  break;
240  }
241  return ret;
242 }
243 
245 {
246  for (size_t ii = 0; ii < shm_manager_->size(); ++ii)
247  {
248  shm_manager_->MarkBufferEmpty(ii, true);
249  }
250 }
251 
252 DEFINE_ARTDAQ_TRANSFER(artdaq::ShmemTransfer)
253 
254 // Local Variables:
255 // mode: c++
256 // End:
size_t buffer_count_
The number of Fragment transfers the TransferInterface can handle simultaneously. ...
virtual int source_rank() const
Get the source rank for this TransferInterface instance.
CopyStatus transfer_fragment_reliable_mode(Fragment &&fragment) override
Transfer a Fragment to the destination. This should be reliable, if the underlying transport mechanis...
CopyStatus transfer_fragment_min_blocking_mode(Fragment const &fragment, size_t send_timeout_usec) override
Transfer a Fragment to the destination. May not necessarily be reliable, but will not block longer th...
This TransferInterface is a Receiver.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
void flush_buffers() override
Flush any in-flight data. This should be used by the receiver after the receive loop has ended...
ShmemTransfer(fhicl::ParameterSet const &pset, Role role)
ShmemTransfer Constructor.
Value that is to be returned when a Transfer plugin determines that no more data will be arriving...
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
This TransferInterface is a Sender.
virtual ~ShmemTransfer() noexcept
ShmemTransfer Destructor.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
A TransferInterface implementation plugin that transfers data using Shared Memory.
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
This interface defines the functions used to transfer data between artdaq applications.
virtual int destination_rank() const
Get the destination rank for this TransferInterface instance.
int receiveFragment(Fragment &fragment, size_t receiveTimeout) override
Receive a Fragment from Shared Memory.
Value to be returned upon receive timeout.
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.
const size_t max_fragment_size_words_
The maximum size of the transferred Fragment objects, in artdaq::Fragment::RawDataType words...