artdaq  v3_00_03
Shmem_transfer.cc
1 #define TRACE_NAME "ShmemTransfer"
2 #include "artdaq/TransferPlugins/ShmemTransfer.hh"
3 #include "cetlib_except/exception.h"
4 
5 artdaq::ShmemTransfer::ShmemTransfer(fhicl::ParameterSet const& pset, Role role) :
6  TransferInterface(pset, role)
7  , role_(role)
8 {
9  TLOG(TLVL_DEBUG) << uniqueLabel() << " ShmemTransfer{} begin";
10  // char* keyChars = getenv("ARTDAQ_SHM_KEY");
11  // if (keyChars != NULL && shm_key_ == static_cast<int>(std::hash<std::string>()(unique_label_))) {
12  // std::string keyString(keyChars);
13  // try {
14  // shm_key_ = boost::lexical_cast<int>(keyString);
15  // }
16  // catch (...) {
17  // std::stringstream errmsg;
18  // errmsg << uniqueLabel() << ": Problem performing lexical cast on " << keyString;
19  // ExceptionHandler(ExceptionHandlerRethrow::yes, errmsg.str());
20  // }
21  // }
22 
23  // JCF, Aug-16-2016
24 
25  // Note that there's a small but nonzero chance of a race condition
26  // here where another process creates the shared memory buffer
27  // between the first and second calls to shmget
28 
29  if (buffer_count_ > 100)
30  {
31  throw cet::exception("ConfigurationException", "Buffer Count is too large for Shmem transfer!");
32  }
33 
34  auto shmKey = pset.get<uint32_t>("shm_key_offset", 0);
35  shmKey += std::hash<std::string>()(uniqueLabel());
36  if (role == Role::kReceive)
37  {
38  shm_manager_ = std::make_unique<SharedMemoryFragmentManager>(shmKey, buffer_count_, max_fragment_size_words_ * sizeof(artdaq::RawDataType));
39  }
40  else
41  {
42  shm_manager_ = std::make_unique<SharedMemoryFragmentManager>(shmKey, 0, 0);
43  }
44 }
45 
47 {
48  TLOG_ARB(5, "ShmemTransfer") << "~ShmemTransfer called - " << uniqueLabel() << TLOG_ENDL;
49  shm_manager_.reset(nullptr);
50  TLOG_ARB(5, "ShmemTransfer") << "~ShmemTransfer done - " << uniqueLabel() << TLOG_ENDL;
51 }
52 
53 int artdaq::ShmemTransfer::receiveFragment(artdaq::Fragment& fragment,
54  size_t receiveTimeout)
55 {
56  auto waitStart = std::chrono::steady_clock::now();
57  while (!shm_manager_->ReadyForRead() && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
58  {
59  // BURN THAT CPU!
60  }
61  if (!shm_manager_->ReadyForRead())
62  {
63  int64_t loopCount = 0;
64  size_t sleepTime = 1000; // microseconds
65  int64_t nloops = (receiveTimeout - 1000) / sleepTime;
66 
67  while (!shm_manager_->ReadyForRead() && loopCount < nloops)
68  {
69  usleep(sleepTime);
70  ++loopCount;
71  }
72  }
73 
74  TLOG(4) << uniqueLabel() << " ShmemTransfer::receiveFragment ReadyForRead=" << shm_manager_->ReadyForRead();
75 
76  if (shm_manager_->ReadyForRead())
77  {
78  auto sts = shm_manager_->ReadFragment(fragment);
79 
80  if (sts != 0) return RECV_TIMEOUT;
81 
82  if (fragment.type() != artdaq::Fragment::DataFragmentType)
83  {
84  TLOG_ARB(8) << uniqueLabel() << " Recvd frag from shmem, type=" << fragment.typeString() << ", sequenceID=" << std::to_string(fragment.sequenceID()) << ", source_rank=" << source_rank() << TLOG_ENDL;
85  }
86 
87  return source_rank();
88  }
89 
91 }
92 
93 int artdaq::ShmemTransfer::receiveFragmentHeader(detail::RawFragmentHeader& header, size_t receiveTimeout)
94 {
95  auto waitStart = std::chrono::steady_clock::now();
96  while (!shm_manager_->ReadyForRead() && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
97  {
98  // BURN THAT CPU!
99  }
100  if (!shm_manager_->ReadyForRead())
101  {
102  int64_t loopCount = 0;
103  size_t sleepTime = 1000; // microseconds
104  int64_t nloops = (receiveTimeout - 1000) / sleepTime;
105 
106  while (!shm_manager_->ReadyForRead() && loopCount < nloops)
107  {
108  usleep(sleepTime);
109  ++loopCount;
110  }
111  }
112 
113  //TLOG(4) << uniqueLabel() << " delta_=" << delta_() << ", rp=" << (int)shm_ptr_->read_pos << ", wp=" << (int)shm_ptr_->write_pos << ", loopCount=" << loopCount << ", nloops=" << nloops << TLOG_ENDL;
114 
115  if (shm_manager_->ReadyForRead())
116  {
117  auto sts = shm_manager_->ReadFragmentHeader(header);
118 
119  if (sts != 0) return RECV_TIMEOUT;
120 
121  if (header.type != artdaq::Fragment::DataFragmentType)
122  {
123  TLOG_ARB(8) << uniqueLabel() << " Recvd fragment header from shmem, type=" << (int)header.type << ", sequenceID=" << std::to_string(header.sequence_id) << ", source_rank=" << source_rank() << TLOG_ENDL;
124  }
125 
126  return source_rank();
127  }
128 
130 }
131 
132 int artdaq::ShmemTransfer::receiveFragmentData(RawDataType* destination, size_t word_count)
133 {
134  auto sts = shm_manager_->ReadFragmentData(destination, word_count);
135 
136  TLOG(4) << uniqueLabel() << " Return status from ReadFragmentData is " << sts << TLOG_ENDL;
137 
138  if (sts != 0) return RECV_TIMEOUT;
139 
140  return source_rank();
141 
143 }
144 
146 artdaq::ShmemTransfer::copyFragment(artdaq::Fragment& fragment, size_t send_timeout_usec)
147 {
148  return sendFragment(std::move(fragment), send_timeout_usec, false);
149 }
150 
152 artdaq::ShmemTransfer::moveFragment(artdaq::Fragment&& fragment,
153  size_t send_timeout_usec)
154 {
155  return sendFragment(std::move(fragment), send_timeout_usec, true);
156 }
157 
159 artdaq::ShmemTransfer::sendFragment(artdaq::Fragment&& fragment, size_t send_timeout_usec, bool reliableMode)
160 {
161  // wait for the shm to become free, if requested
162  if (send_timeout_usec > 0 || reliableMode)
163  {
164  auto waitStart = std::chrono::steady_clock::now();
165  while (!shm_manager_->ReadyForWrite(!reliableMode) && TimeUtils::GetElapsedTimeMicroseconds(waitStart) < 1000)
166  {
167  // BURN THAT CPU!
168  }
169  if (!shm_manager_->ReadyForWrite(!reliableMode))
170  {
171  int64_t loopCount = 0;
172  size_t sleepTime = 1000; // microseconds
173  int64_t nloops = (send_timeout_usec - 1000) / sleepTime;
174 
175  while (reliableMode && !shm_manager_->ReadyForWrite(!reliableMode) && (send_timeout_usec == 0 || loopCount < nloops))
176  {
177  usleep(sleepTime);
178  ++loopCount;
179  }
180  }
181  }
182 
183  TLOG_ARB(5) << uniqueLabel() << " Either write has timed out or buffer ready" << TLOG_ENDL;
184 
185  // copy the fragment if the shm is available
186  if (shm_manager_->ReadyForWrite(!reliableMode))
187  {
188  TLOG_ARB(5) << uniqueLabel() << " Sending fragment with seqID=" << std::to_string(fragment.sequenceID()) << TLOG_ENDL;
189  artdaq::RawDataType* fragAddr = fragment.headerAddress();
190  size_t fragSize = fragment.size() * sizeof(artdaq::RawDataType);
191 
192  // 10-Sep-2013, KAB - protect against large events and
193  // invalid events (and large, invalid events)
194  if (fragment.type() != artdaq::Fragment::InvalidFragmentType && fragSize < (max_fragment_size_words_ * sizeof(artdaq::RawDataType)))
195  {
196  auto sts = shm_manager_->WriteFragment(std::move(fragment), !reliableMode);
197  if (sts != 0) return CopyStatus::kErrorNotRequiringException;
198 
199  TLOG_ARB(5) << uniqueLabel() << " Fragment send successfully" << TLOG_ENDL;
200  return CopyStatus::kSuccess;
201  }
202  else
203  {
204  TLOG_WARNING("ShmemTransfer") << uniqueLabel() << " Fragment invalid for shared memory! "
205  << "fragment address and size = "
206  << fragAddr << " " << fragSize << " "
207  << "sequence ID, fragment ID, and type = "
208  << fragment.sequenceID() << " "
209  << fragment.fragmentID() << " "
210  << fragment.typeString() << TLOG_ENDL;
211  return CopyStatus::kErrorNotRequiringException;
212  }
213  }
214 
215  TLOG_ARB(5) << uniqueLabel() << " Fragment Send Timeout!" << TLOG_ENDL;
216  return CopyStatus::kTimeout;
217 }
218 
219 DEFINE_ARTDAQ_TRANSFER(artdaq::ShmemTransfer)
220 
221 // Local Variables:
222 // mode: c++
223 // End:
size_t buffer_count_
The number of Fragment transfers the TransferInterface can handle simultaneously. ...
CopyStatus moveFragment(Fragment &&fragment, size_t send_timeout_usec=std::numeric_limits< size_t >::max()) override
Move a Fragment to the destination.
static const int RECV_TIMEOUT
Value to be returned upon receive timeout. Because receivers otherwise return rank, this is also the limit on the number of ranks that artdaq currently supports.
This TransferInterface is a Receiver.
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
CopyStatus copyFragment(Fragment &fragment, size_t send_timeout_usec=std::numeric_limits< size_t >::max()) override
Copy a Fragment to the destination. May be unreliable.
ShmemTransfer(fhicl::ParameterSet const &pset, Role role)
ShmemTransfer Constructor.
int receiveFragmentData(RawDataType *destination, size_t wordCount) override
Receive the body of a Fragment to the given destination pointer.
virtual ~ShmemTransfer() noexcept
ShmemTransfer Destructor.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
std::string uniqueLabel() const
Get the unique label of this TransferInterface instance.
A TransferInterface implementation plugin that transfers data using Shared Memory.
This interface defines the functions used to transfer data between artdaq applications.
int receiveFragment(Fragment &fragment, size_t receiveTimeout) override
Receive a Fragment from Shared Memory.
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.
const size_t max_fragment_size_words_
The maximum size of the transferred Fragment objects, in artdaq::Fragment::RawDataType words...