artdaq  v3_02_00
Shmem_transfer.cc
1 #define TRACE_NAME "ShmemTransfer"
2 
3 #include "artdaq/TransferPlugins/ShmemTransfer.hh"
4 #include "cetlib_except/exception.h"
5 #include <signal.h>
6 
7 artdaq::ShmemTransfer::ShmemTransfer(fhicl::ParameterSet const& pset, Role role) :
8  TransferInterface(pset, role)
9 {
10  TLOG(TLVL_DEBUG) << GetTraceName() << ": Constructor BEGIN";
11  // char* keyChars = getenv("ARTDAQ_SHM_KEY");
12  // if (keyChars != NULL && shm_key_ == static_cast<int>(std::hash<std::string>()(unique_label_))) {
13  // std::string keyString(keyChars);
14  // try {
15  // shm_key_ = boost::lexical_cast<int>(keyString);
16  // }
17  // catch (...) {
18  // std::stringstream errmsg;
19  // errmsg << uniqueLabel() << ": Problem performing lexical cast on " << keyString;
20  // ExceptionHandler(ExceptionHandlerRethrow::yes, errmsg.str());
21  // }
22  // }
23 
24  // JCF, Aug-16-2016
25 
26  // Note that there's a small but nonzero chance of a race condition
27  // here where another process creates the shared memory buffer
28  // between the first and second calls to shmget
29 
30  if (buffer_count_ > 100)
31  {
32  throw cet::exception("ConfigurationException", "Buffer Count is too large for Shmem transfer!");
33  }
34 
35  auto partition = partition_number_ + 1; // Can't be 0
36  if (partition > 127) partition = 127; // Can't be > 127
37 
38  auto shmKey = pset.get<uint32_t>("shm_key_offset", 0) + (partition << 24) + ((source_rank() & 0xFFF) << 12) + (destination_rank() & 0xFFF);
39 
40  // Configured Shared Memory key overrides everything! Needed for Online Monitor connections!
41  if (pset.has_key("shm_key")) {
42  shmKey = pset.get<uint32_t>("shm_key");
43  }
44 
45  if (role == Role::kReceive)
46  {
47  shm_manager_ = std::make_unique<SharedMemoryFragmentManager>(shmKey, buffer_count_, max_fragment_size_words_ * sizeof(artdaq::RawDataType));
48  }
49  else
50  {
51  shm_manager_ = std::make_unique<SharedMemoryFragmentManager>(shmKey, 0, 0);
52  }
53  TLOG(TLVL_DEBUG) << GetTraceName() << ": Constructor END";
54 }
55 
57 {
58  TLOG(5) << GetTraceName() << " ~ShmemTransfer called - " << uniqueLabel() ;
59  shm_manager_.reset(nullptr);
60  TLOG(5) << GetTraceName() << " ~ShmemTransfer done - " << uniqueLabel() ;
61 }
62 
63 int artdaq::ShmemTransfer::receiveFragment(artdaq::Fragment& fragment,
64  size_t receiveTimeout)
65 {
66  auto waitStart = std::chrono::steady_clock::now();
67  while (!shm_manager_->ReadyForRead() && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
68  {
69  // BURN THAT CPU!
70  }
71  if (!shm_manager_->ReadyForRead())
72  {
73  int64_t loopCount = 0;
74  size_t sleepTime = 1000; // microseconds
75  int64_t nloops = (receiveTimeout - 1000) / sleepTime;
76 
77  while (!shm_manager_->ReadyForRead() && loopCount < nloops)
78  {
79  usleep(sleepTime);
80  ++loopCount;
81  }
82  }
83 
84  TLOG(TLVL_TRACE) << GetTraceName() << ": receiveFragment ReadyForRead=" << shm_manager_->ReadyForRead();
85 
86  if (shm_manager_->ReadyForRead())
87  {
88  auto sts = shm_manager_->ReadFragment(fragment);
89 
90  if (sts != 0) return RECV_TIMEOUT;
91 
92  if (fragment.type() != artdaq::Fragment::DataFragmentType)
93  {
94  TLOG(8) << GetTraceName() << ": Recvd frag from shmem, type=" << fragment.typeString() << ", sequenceID=" << std::to_string(fragment.sequenceID()) << ", source_rank=" << source_rank() ;
95  }
96 
97  return source_rank();
98  }
99 
101 }
102 
103 int artdaq::ShmemTransfer::receiveFragmentHeader(detail::RawFragmentHeader& header, size_t receiveTimeout)
104 {
105  auto waitStart = std::chrono::steady_clock::now();
106  while (!shm_manager_->ReadyForRead() && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
107  {
108  // BURN THAT CPU!
109  }
110  if (!shm_manager_->ReadyForRead())
111  {
112  int64_t loopCount = 0;
113  size_t sleepTime = 1000; // microseconds
114  int64_t nloops = (receiveTimeout - 1000) / sleepTime;
115 
116  while (!shm_manager_->ReadyForRead() && loopCount < nloops)
117  {
118  usleep(sleepTime);
119  ++loopCount;
120  }
121  }
122 
123  if (!shm_manager_->ReadyForRead() && shm_manager_->IsEndOfData())
124  {
126  }
127 
128  //TLOG(TLVL_TRACE) << GetTraceName() << ": delta_=" << delta_() << ", rp=" << (int)shm_ptr_->read_pos << ", wp=" << (int)shm_ptr_->write_pos << ", loopCount=" << loopCount << ", nloops=" << nloops ;
129 
130  if (shm_manager_->ReadyForRead())
131  {
132  auto sts = shm_manager_->ReadFragmentHeader(header);
133 
134  if (sts != 0) return RECV_TIMEOUT;
135 
136  if (header.type != artdaq::Fragment::DataFragmentType)
137  {
138  TLOG(8) << GetTraceName() << ": Recvd fragment header from shmem, type=" << (int)header.type << ", sequenceID=" << std::to_string(header.sequence_id) << ", source_rank=" << source_rank() ;
139  }
140 
141  return source_rank();
142  }
143 
145 }
146 
147 int artdaq::ShmemTransfer::receiveFragmentData(RawDataType* destination, size_t word_count)
148 {
149  auto sts = shm_manager_->ReadFragmentData(destination, word_count);
150 
151  TLOG(TLVL_TRACE) << GetTraceName() << ": Return status from ReadFragmentData is " << sts ;
152 
153  if (sts != 0) return RECV_TIMEOUT;
154 
155  return source_rank();
156 
158 }
159 
161 artdaq::ShmemTransfer::copyFragment(artdaq::Fragment& fragment, size_t send_timeout_usec)
162 {
163  return sendFragment(std::move(fragment), send_timeout_usec, false);
164 }
165 
167 artdaq::ShmemTransfer::moveFragment(artdaq::Fragment&& fragment)
168 {
169  return sendFragment(std::move(fragment), 0, true);
170 }
171 
173 artdaq::ShmemTransfer::sendFragment(artdaq::Fragment&& fragment, size_t send_timeout_usec, bool reliableMode)
174 {
175  if (!shm_manager_->IsValid()) {
176  shm_manager_->Attach();
177  if (!shm_manager_->IsValid()) {
178  TLOG(TLVL_ERROR) << GetTraceName() << ": Attempted to send Fragment when not attached to Shared Memory! Returning kSuccess, and dropping data!";
179  return CopyStatus::kSuccess;
180  }
181  }
182  shm_manager_->SetRank(my_rank);
183  // wait for the shm to become free, if requested
184 
185  TLOG(5) << GetTraceName() << ": Sending fragment with seqID=" << std::to_string(fragment.sequenceID()) ;
186  artdaq::RawDataType* fragAddr = fragment.headerAddress();
187  size_t fragSize = fragment.size() * sizeof(artdaq::RawDataType);
188 
189  // 10-Sep-2013, KAB - protect against large events and
190  // invalid events (and large, invalid events)
191  if (fragment.type() != artdaq::Fragment::InvalidFragmentType && fragSize < (max_fragment_size_words_ * sizeof(artdaq::RawDataType)))
192  {
193  TLOG(5) << GetTraceName() << ": Writing fragment with seqID=" << std::to_string(fragment.sequenceID());
194  auto sts = shm_manager_->WriteFragment(std::move(fragment), !reliableMode, send_timeout_usec);
195  if (sts == -3)
196  {
197  TLOG(TLVL_WARNING) << GetTraceName() << ": Timeout writing fragment with seqID=" << std::to_string(fragment.sequenceID());
198  return CopyStatus::kTimeout;
199  }
200  if (sts != 0)
201  {
202  TLOG(TLVL_WARNING) << GetTraceName() << ": Error writing fragment with seqID=" << std::to_string(fragment.sequenceID());
203  return CopyStatus::kErrorNotRequiringException;
204  }
205 
206  TLOG(5) << GetTraceName() << ": Successfully sent Fragment with seqID=" << std::to_string(fragment.sequenceID());
207  return CopyStatus::kSuccess;
208  }
209  else
210  {
211  TLOG(TLVL_WARNING) << GetTraceName() << ": Fragment invalid for shared memory! "
212  << "fragment address and size = "
213  << fragAddr << " " << fragSize << " "
214  << "sequence ID, fragment ID, and type = "
215  << fragment.sequenceID() << " "
216  << fragment.fragmentID() << " "
217  << fragment.typeString() ;
218  return CopyStatus::kErrorNotRequiringException;
219  }
220 
221  TLOG(TLVL_WARNING) << GetTraceName() << ": Unreachable code reached!" ;
222  return CopyStatus::kErrorNotRequiringException;
223 }
224 
226 {
227  bool ret = false;
228  switch (role())
229  {
231  ret = shm_manager_->IsValid();
232  break;
234  ret = shm_manager_->GetAttachedCount() > 1;
235  break;
236  }
237  return ret;
238 
239 }
240 
241 DEFINE_ARTDAQ_TRANSFER(artdaq::ShmemTransfer)
242 
243 // Local Variables:
244 // mode: c++
245 // End:
size_t buffer_count_
The number of Fragment transfers the TransferInterface can handle simultaneously. ...
virtual int source_rank() const
Get the source rank for this TransferInterface instance.
Value to be returned upon receive timeout.
std::string GetTraceName() const
Constructs a name suitable for TRACE messages.
CopyStatus copyFragment(Fragment &fragment, size_t send_timeout_usec) override
Copy a Fragment to the destination. May be unreliable.
This TransferInterface is a Receiver.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
const short partition_number_
The partition number of the DAQ.
Value that is to be returned when a Transfer plugin determines that no more data will be arriving...
ShmemTransfer(fhicl::ParameterSet const &pset, Role role)
ShmemTransfer Constructor.
CopyStatus moveFragment(Fragment &&fragment) override
Move a Fragment to the destination.
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
This TransferInterface is a Sender.
virtual ~ShmemTransfer() noexcept
ShmemTransfer Destructor.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
A TransferInterface implementation plugin that transfers data using Shared Memory.
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
This interface defines the functions used to transfer data between artdaq applications.
virtual int destination_rank() const
Get the destination rank for this TransferInterface instance.
int receiveFragment(Fragment &fragment, size_t receiveTimeout) override
Receive a Fragment from Shared Memory.
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.
const size_t max_fragment_size_words_
The maximum size of the transferred Fragment objects, in artdaq::Fragment::RawDataType words...