3 #include "artdaq/DAQdata/Globals.hh"
4 #define TRACE_NAME (app_name + "_BundleTransfer").c_str()
6 #include "artdaq-core/Data/ContainerFragmentLoader.hh"
7 #include "artdaq/TransferPlugins/TCPSocketTransfer.hh"
8 #include "artdaq/TransferPlugins/TransferInterface.hh"
10 #include <boost/thread.hpp>
41 size_t receiveTimeout)
override
43 if (bundle_fragment_ ==
nullptr)
45 receive_bundle_fragment_(receiveTimeout);
49 ContainerFragment cf(*bundle_fragment_);
50 TLOG(TLVL_DEBUG + 32) << GetTraceName() <<
"Retrieving Fragment " << (current_block_index_ + 1) <<
" of " << cf.block_count();
51 fragment.resizeBytes(cf.fragSize(current_block_index_) -
sizeof(detail::RawFragmentHeader));
52 memcpy(fragment.headerAddress(),
static_cast<const uint8_t*
>(cf.dataBegin()) + cf.fragmentIndex(current_block_index_), cf.fragSize(current_block_index_));
53 current_block_index_++;
54 if (current_block_index_ >= cf.block_count())
56 bundle_fragment_.reset(
nullptr);
69 if (bundle_fragment_ ==
nullptr)
71 receive_bundle_fragment_(receiveTimeout);
74 ContainerFragment cf(*bundle_fragment_);
75 TLOG(TLVL_DEBUG + 32) << GetTraceName() <<
"Retrieving Fragment Header " << (current_block_index_ + 1) <<
" of " << cf.block_count();
76 memcpy(&header, static_cast<const uint8_t*>(cf.dataBegin()) + cf.fragmentIndex(current_block_index_),
sizeof(detail::RawFragmentHeader));
88 if (bundle_fragment_ ==
nullptr)
92 ContainerFragment cf(*bundle_fragment_);
93 TLOG(TLVL_DEBUG + 32) << GetTraceName() <<
"Retrieving Fragment Data " << (current_block_index_ + 1) <<
" of " << cf.block_count();
94 memcpy(destination, static_cast<const uint8_t*>(cf.dataBegin()) + cf.fragmentIndex(current_block_index_) +
sizeof(detail::RawFragmentHeader), cf.fragSize(current_block_index_) -
sizeof(detail::RawFragmentHeader));
95 current_block_index_++;
96 if (current_block_index_ >= cf.block_count())
98 bundle_fragment_.reset(
nullptr);
100 return current_rank_;
111 TLOG(TLVL_DEBUG + 35) << GetTraceName() <<
"transfer_fragment_min_blocking_mode START";
112 last_send_call_reliable_ =
false;
113 last_send_timeout_usec_ = send_timeout_usec;
115 std::unique_lock<std::mutex> lk(fragment_mutex_);
116 if (current_buffer_size_bytes_ > max_hold_size_bytes_)
118 fragment_cv_.wait_for(lk, std::chrono::microseconds(send_timeout_usec), [&] {
return current_buffer_size_bytes_ < max_hold_size_bytes_; });
121 if (current_buffer_size_bytes_ > max_hold_size_bytes_)
123 TLOG(TLVL_WARNING) << GetTraceName() <<
"Dropping data due to timeout in min_blocking_mode";
127 TLOG(TLVL_DEBUG + 35) << GetTraceName() <<
"transfer_fragment_min_blocking_mode after wait for buffer";
129 if (Fragment::isSystemFragmentType(fragment.type()))
131 system_fragment_cached_ =
true;
134 current_buffer_size_bytes_ += fragment.sizeBytes();
136 fragment_buffer_.emplace_back(fragment);
138 TLOG(TLVL_DEBUG + 35) << GetTraceName() <<
"transfer_fragment_min_blocking_mode END";
149 TLOG(TLVL_DEBUG + 36) << GetTraceName() <<
"transfer_fragment_reliable_mode START";
150 last_send_call_reliable_ =
true;
152 std::unique_lock<std::mutex> lk(fragment_mutex_);
153 if (current_buffer_size_bytes_ > max_hold_size_bytes_)
155 fragment_cv_.wait(lk, [&] {
return current_buffer_size_bytes_ < max_hold_size_bytes_; });
158 TLOG(TLVL_DEBUG + 36) << GetTraceName() <<
"transfer_fragment_reliable_mode after wait for buffer";
161 if (Fragment::isSystemFragmentType(fragment.type()))
163 system_fragment_cached_ =
true;
166 current_buffer_size_bytes_ += fragment.sizeBytes();
167 fragment_buffer_.emplace_back(std::move(fragment));
169 TLOG(TLVL_DEBUG + 36) << GetTraceName() <<
"transfer_fragment_reliable_mode END";
192 std::unique_ptr<TransferInterface> theTransfer_;
193 size_t send_threshold_bytes_;
194 size_t max_hold_size_bytes_;
195 int max_hold_time_us_;
196 FragmentPtr bundle_fragment_{
nullptr};
197 Fragments fragment_buffer_;
198 size_t current_block_index_{0};
199 int current_rank_ = 0;
201 std::chrono::steady_clock::time_point send_fragment_started_;
202 std::atomic<size_t> current_buffer_size_bytes_{0};
203 std::unique_ptr<boost::thread> send_timeout_thread_;
204 std::atomic<bool> system_fragment_cached_{
false};
205 std::atomic<bool> send_timeout_thread_running_{
false};
206 std::atomic<bool> last_send_call_reliable_{
true};
207 std::atomic<size_t> last_send_timeout_usec_{1000000};
208 std::atomic<bool> running_{
true};
209 std::mutex fragment_mutex_;
210 std::condition_variable fragment_cv_;
212 bool check_send_(
bool force);
213 void start_timeout_thread_();
214 void send_timeout_thread_proc_();
215 bool send_bundle_fragment_(
bool forceSend =
false);
216 void receive_bundle_fragment_(
size_t receiveTimeout);
222 , send_threshold_bytes_(pset.get<size_t>(
"send_threshold_bytes", 10 * 0x100000))
223 , max_hold_size_bytes_(pset.get<size_t>(
"max_hold_size_bytes", 1000 * 0x100000))
224 , max_hold_time_us_(pset.get<int>(
"max_hold_time_us", 100000))
226 TLOG(TLVL_INFO) << GetTraceName() <<
"Begin BundleTransfer constructor";
227 TLOG(TLVL_INFO) << GetTraceName() <<
"Constructing TCPSocketTransfer";
228 theTransfer_ = std::make_unique<TCPSocketTransfer>(pset,
role);
232 start_timeout_thread_();
238 if (role_ == Role::kSend)
240 send_timeout_thread_running_ =
false;
241 if (send_timeout_thread_ && send_timeout_thread_->joinable())
243 send_timeout_thread_->join();
245 send_bundle_fragment_(
true);
250 void artdaq::BundleTransfer::start_timeout_thread_()
252 if (send_timeout_thread_ && send_timeout_thread_->joinable())
254 send_timeout_thread_->join();
256 send_timeout_thread_running_ =
true;
257 TLOG(TLVL_INFO) << GetTraceName() <<
"Starting Send Timeout Thread";
261 send_timeout_thread_ = std::make_unique<boost::thread>(&BundleTransfer::send_timeout_thread_proc_,
this);
263 snprintf(tname,
sizeof(tname) - 1,
"%d-SNDTMO", my_rank);
264 tname[
sizeof(tname) - 1] =
'\0';
265 auto handle = send_timeout_thread_->native_handle();
266 pthread_setname_np(handle, tname);
268 catch (
const boost::exception& e)
270 TLOG(TLVL_ERROR) << GetTraceName() <<
"Caught boost::exception starting Send Timeout thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
271 std::cerr << GetTraceName() <<
"Caught boost::exception starting Send Timeout thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
276 void artdaq::BundleTransfer::send_timeout_thread_proc_()
278 while (send_timeout_thread_running_)
280 if (!send_bundle_fragment_())
287 bool artdaq::BundleTransfer::check_send_(
bool force)
291 TLOG(TLVL_DEBUG + 37) << GetTraceName() <<
"check_send_: Send is forced, returning true";
295 if (system_fragment_cached_.load())
297 TLOG(TLVL_DEBUG + 37) << GetTraceName() <<
"check_send_: System Fragment in cache, returning true";
301 if (std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() - send_fragment_started_).count() >= max_hold_time_us_)
303 TLOG(TLVL_DEBUG + 37) << GetTraceName() <<
"check_send_: Send timeout reached, returning true";
307 if (current_buffer_size_bytes_ >= send_threshold_bytes_)
309 TLOG(TLVL_DEBUG + 37) << GetTraceName() <<
"check_send_: Buffer is full, returning true";
313 TLOG(TLVL_DEBUG + 37) << GetTraceName() <<
"check_send_: returning false";
317 bool artdaq::BundleTransfer::send_bundle_fragment_(
bool forceSend)
320 std::unique_lock<std::mutex> lk(fragment_mutex_);
322 bool send_fragment = check_send_(forceSend);
324 if (send_fragment && fragment_buffer_.size() > 0)
326 TLOG(TLVL_DEBUG + 38) << GetTraceName() <<
"Swapping in new buffer";
327 Fragments temp_buffer;
328 size_t size = current_buffer_size_bytes_;
329 fragment_buffer_.swap(temp_buffer);
330 send_fragment_started_ = std::chrono::steady_clock::now();
331 current_buffer_size_bytes_ = 0;
333 TLOG(TLVL_DEBUG + 38) << GetTraceName() <<
"Notifying waiters";
334 fragment_cv_.notify_one();
336 TLOG(TLVL_DEBUG + 38) << GetTraceName() <<
"Setting up Bundle Fragment";
337 bundle_fragment_.reset(
new artdaq::Fragment(temp_buffer.front().sequenceID() + 1, temp_buffer.front().fragmentID()));
338 bundle_fragment_->setTimestamp(temp_buffer.front().timestamp());
339 bundle_fragment_->reserve(size /
sizeof(artdaq::RawDataType));
341 TLOG(TLVL_DEBUG + 38) << GetTraceName() <<
"Filling Bundle Fragment";
342 ContainerFragmentLoader container_fragment(*bundle_fragment_);
343 container_fragment.set_missing_data(
false);
344 container_fragment.addFragments(temp_buffer,
true);
347 TLOG(TLVL_DEBUG + 38) << GetTraceName() <<
"Sending Fragment, reliable mode " << last_send_call_reliable_.load();
348 CopyStatus sts = CopyStatus::kSuccess;
349 if (last_send_call_reliable_)
351 sts = theTransfer_->transfer_fragment_reliable_mode(std::move(*bundle_fragment_.get()));
352 bundle_fragment_.reset(
nullptr);
356 while (sts != CopyStatus::kSuccess && send_timeout_thread_running_)
358 sts = theTransfer_->transfer_fragment_min_blocking_mode(*bundle_fragment_.get(), last_send_timeout_usec_);
360 bundle_fragment_.reset(
nullptr);
363 if (sts != CopyStatus::kSuccess)
365 auto sts_string = sts == CopyStatus::kTimeout ?
"timeout" :
"other error";
366 TLOG(TLVL_WARNING) << GetTraceName() <<
"Transfer of Bundle fragment returned status " << sts_string;
369 TLOG(TLVL_DEBUG + 38) << GetTraceName() <<
"Done sending Bundle Fragment";
377 void artdaq::BundleTransfer::receive_bundle_fragment_(
size_t receiveTimeout)
379 std::lock_guard<std::mutex> lk(fragment_mutex_);
380 bundle_fragment_.reset(
new artdaq::Fragment(1));
382 TLOG(TLVL_DEBUG + 34) << GetTraceName() <<
"Going to receive next bundle fragment";
383 current_rank_ = theTransfer_->receiveFragment(*bundle_fragment_, receiveTimeout);
384 TLOG(TLVL_DEBUG + 34) << GetTraceName() <<
"Done with receiveFragment, current_rank_ = " << current_rank_;
386 if (current_rank_ < RECV_SUCCESS)
388 bundle_fragment_.reset(
nullptr);
390 current_block_index_ = 0;
The BundleTransfer TransferInterface plugin sets up a Shmem_transfer plugin or TCPSocket_transfer plu...
Role role() const
Get the TransferInterface::Role of this TransferInterface.
The send operation timed out.
void flush_buffers() override
Flush any in-flight data. This should be used by the receiver after the receive loop has ended...
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
BundleTransfer(const fhicl::ParameterSet &pset, Role role)
BundleTransfer Constructor.
This TransferInterface is a Sender.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
The send operation completed successfully.
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
This interface defines the functions used to transfer data between artdaq applications.
CopyStatus transfer_fragment_reliable_mode(artdaq::Fragment &&fragment) override
Send a Fragment in reliable mode, using the underlying transfer plugin.
int receiveFragmentData(RawDataType *destination, size_t) override
Receive the body of a Fragment to the given destination pointer.
CopyStatus transfer_fragment_min_blocking_mode(artdaq::Fragment const &fragment, size_t send_timeout_usec) override
Send a Fragment in non-reliable mode, using the underlying transfer plugin.
For code clarity, things checking for successful receive should check retval >= NO_RANK_INFO.
int receiveFragment(artdaq::Fragment &fragment, size_t receiveTimeout) override
Receive a Fragment, using the underlying transfer plugin.
Value to be returned upon receive timeout.
~BundleTransfer() override
BundleTransfer default Destructor.
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.