3 #include "artdaq/DAQdata/Globals.hh"
4 #define TRACE_NAME (app_name + "_BundleTransfer").c_str()
6 #include "artdaq-core/Data/ContainerFragmentLoader.hh"
7 #include "artdaq/TransferPlugins/TCPSocketTransfer.hh"
8 #include "artdaq/TransferPlugins/TransferInterface.hh"
10 #include <boost/thread.hpp>
41 size_t receiveTimeout)
override
43 if (bundle_fragment_ ==
nullptr)
45 receive_bundle_fragment_(receiveTimeout);
49 ContainerFragment cf(*bundle_fragment_);
50 TLOG(TLVL_DEBUG + 32) <<
"Retrieving Fragment " << (current_block_index_ + 1) <<
" of " << cf.block_count();
51 fragment.resizeBytes(cf.fragSize(current_block_index_) -
sizeof(detail::RawFragmentHeader));
52 memcpy(fragment.headerAddress(),
static_cast<const uint8_t*
>(cf.dataBegin()) + cf.fragmentIndex(current_block_index_), cf.fragSize(current_block_index_));
53 current_block_index_++;
54 if (current_block_index_ >= cf.block_count())
56 bundle_fragment_.reset(
nullptr);
69 if (bundle_fragment_ ==
nullptr)
71 receive_bundle_fragment_(receiveTimeout);
74 ContainerFragment cf(*bundle_fragment_);
75 TLOG(TLVL_DEBUG + 32) <<
"Retrieving Fragment Header " << (current_block_index_ + 1) <<
" of " << cf.block_count();
76 memcpy(&header, static_cast<const uint8_t*>(cf.dataBegin()) + cf.fragmentIndex(current_block_index_),
sizeof(detail::RawFragmentHeader));
88 if (bundle_fragment_ ==
nullptr)
92 ContainerFragment cf(*bundle_fragment_);
93 TLOG(TLVL_DEBUG + 32) <<
"Retrieving Fragment Data " << (current_block_index_ + 1) <<
" of " << cf.block_count();
94 memcpy(destination, static_cast<const uint8_t*>(cf.dataBegin()) + cf.fragmentIndex(current_block_index_) +
sizeof(detail::RawFragmentHeader), cf.fragSize(current_block_index_) -
sizeof(detail::RawFragmentHeader));
95 current_block_index_++;
96 if (current_block_index_ >= cf.block_count())
98 bundle_fragment_.reset(
nullptr);
100 return current_rank_;
111 last_send_call_reliable_ =
false;
113 std::lock_guard<std::mutex> lk(fragment_mutex_);
114 if (bundle_fragment_ ==
nullptr)
116 bundle_fragment_.reset(
new artdaq::Fragment(fragment.sequenceID() + 1, fragment.fragmentID()));
117 bundle_fragment_->setTimestamp(fragment.timestamp());
118 bundle_fragment_->reserve(max_hold_size_bytes_ /
sizeof(artdaq::RawDataType));
119 send_fragment_started_ = std::chrono::steady_clock::now();
121 container_fragment_.reset(
new ContainerFragmentLoader(*bundle_fragment_));
122 container_fragment_->set_missing_data(
false);
126 artdaq::Fragment frag(fragment);
128 container_fragment_->addFragment(frag,
true);
130 return send_bundle_fragment_(send_timeout_usec);
140 last_send_call_reliable_ =
true;
142 std::lock_guard<std::mutex> lk(fragment_mutex_);
143 if (bundle_fragment_ ==
nullptr)
145 bundle_fragment_.reset(
new artdaq::Fragment(fragment.sequenceID() + 1, fragment.fragmentID()));
146 bundle_fragment_->setTimestamp(fragment.timestamp());
147 bundle_fragment_->reserve(max_hold_size_bytes_ /
sizeof(artdaq::RawDataType));
148 send_fragment_started_ = std::chrono::steady_clock::now();
150 container_fragment_.reset(
new ContainerFragmentLoader(*bundle_fragment_));
151 container_fragment_->set_missing_data(
false);
154 container_fragment_->addFragment(fragment,
true);
156 return send_bundle_fragment_(0);
178 std::unique_ptr<TransferInterface> theTransfer_;
179 size_t max_hold_size_bytes_;
180 int max_hold_time_us_;
181 FragmentPtr bundle_fragment_{
nullptr};
182 std::unique_ptr<ContainerFragmentLoader> container_fragment_{
nullptr};
183 size_t current_block_index_{0};
184 int current_rank_ = 0;
186 std::chrono::steady_clock::time_point send_fragment_started_;
187 std::unique_ptr<boost::thread> send_timeout_thread_;
188 std::atomic<bool> send_timeout_thread_running_{
false};
189 std::atomic<bool> last_send_call_reliable_{
true};
190 std::atomic<bool> running_{
true};
191 std::mutex fragment_mutex_;
193 void start_timeout_thread_();
194 void send_timeout_thread_proc_();
195 CopyStatus send_bundle_fragment_(
size_t send_timeout_usec,
bool forceSend =
false);
196 void receive_bundle_fragment_(
size_t receiveTimeout);
202 , max_hold_size_bytes_(pset.get<size_t>(
"max_hold_size_bytes", 0x1000000))
203 , max_hold_time_us_(pset.get<int>(
"max_hold_time_us", 250000))
205 TLOG(TLVL_INFO) << GetTraceName() <<
"Begin BundleTransfer constructor";
206 TLOG(TLVL_INFO) << GetTraceName() <<
"Constructing TCPSocketTransfer";
207 theTransfer_ = std::make_unique<TCPSocketTransfer>(pset,
role);
211 start_timeout_thread_();
217 if (role_ == Role::kSend)
219 send_timeout_thread_running_ =
false;
220 if (send_timeout_thread_ && send_timeout_thread_->joinable())
222 send_timeout_thread_->join();
224 send_bundle_fragment_(1000000,
true);
229 void artdaq::BundleTransfer::start_timeout_thread_()
231 if (send_timeout_thread_ && send_timeout_thread_->joinable())
233 send_timeout_thread_->join();
235 send_timeout_thread_running_ =
true;
236 TLOG(TLVL_INFO) << GetTraceName() <<
"Starting Send Timeout Thread";
240 send_timeout_thread_ = std::make_unique<boost::thread>(&BundleTransfer::send_timeout_thread_proc_,
this);
242 snprintf(tname,
sizeof(tname) - 1,
"%d-SNDTMO", my_rank);
243 tname[
sizeof(tname) - 1] =
'\0';
244 auto handle = send_timeout_thread_->native_handle();
245 pthread_setname_np(handle, tname);
247 catch (
const boost::exception& e)
249 TLOG(TLVL_ERROR) <<
"Caught boost::exception starting Send Timeout thread: " << boost::diagnostic_information(e) <<
", errno=" << errno;
250 std::cerr <<
"Caught boost::exception starting Send Timeout thread: " << boost::diagnostic_information(e) <<
", errno=" << errno << std::endl;
255 void artdaq::BundleTransfer::send_timeout_thread_proc_()
257 while (send_timeout_thread_running_)
259 if (send_bundle_fragment_(1000000) != CopyStatus::kSuccess)
268 CopyStatus sts = CopyStatus::kErrorNotRequiringException;
269 std::lock_guard<std::mutex> lk(fragment_mutex_);
271 if (bundle_fragment_ ==
nullptr)
276 bool send_fragment = forceSend;
277 if (std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() - send_fragment_started_).count() >= max_hold_time_us_)
279 send_fragment =
true;
282 if (bundle_fragment_->dataSizeBytes() >= max_hold_size_bytes_)
284 send_fragment =
true;
289 if (last_send_call_reliable_)
291 sts = theTransfer_->transfer_fragment_reliable_mode(std::move(*bundle_fragment_.get()));
292 bundle_fragment_.reset(
nullptr);
296 while (sts != CopyStatus::kSuccess && send_timeout_thread_running_)
298 sts = theTransfer_->transfer_fragment_min_blocking_mode(*bundle_fragment_.get(), send_timeout_usec);
300 bundle_fragment_.reset(
nullptr);
305 return CopyStatus::kSuccess;
308 void artdaq::BundleTransfer::receive_bundle_fragment_(
size_t receiveTimeout)
310 std::lock_guard<std::mutex> lk(fragment_mutex_);
311 bundle_fragment_.reset(
new artdaq::Fragment(1));
313 TLOG(TLVL_DEBUG + 34) <<
"Going to receive next bundle fragment";
314 current_rank_ = theTransfer_->receiveFragment(*bundle_fragment_, receiveTimeout);
315 TLOG(TLVL_DEBUG + 34) <<
"Done with receiveFragment, current_rank_ = " << current_rank_;
317 if (current_rank_ < RECV_SUCCESS)
319 bundle_fragment_.reset(
nullptr);
321 current_block_index_ = 0;
The BundleTransfer TransferInterface plugin sets up a Shmem_transfer plugin or TCPSocket_transfer plu...
Role role() const
Get the TransferInterface::Role of this TransferInterface.
void flush_buffers() override
Flush any in-flight data. This should be used by the receiver after the receive loop has ended...
int receiveFragmentHeader(detail::RawFragmentHeader &header, size_t receiveTimeout) override
Receive a Fragment Header from the transport mechanism.
BundleTransfer(const fhicl::ParameterSet &pset, Role role)
BundleTransfer Constructor.
This TransferInterface is a Sender.
Role
Used to determine if a TransferInterface is a Sender or Receiver.
bool isRunning() override
Determine whether the TransferInterface plugin is able to send/receive data.
This interface defines the functions used to transfer data between artdaq applications.
CopyStatus transfer_fragment_reliable_mode(artdaq::Fragment &&fragment) override
Send a Fragment in reliable mode, using the underlying transfer plugin.
int receiveFragmentData(RawDataType *destination, size_t) override
Receive the body of a Fragment to the given destination pointer.
CopyStatus transfer_fragment_min_blocking_mode(artdaq::Fragment const &fragment, size_t send_timeout_usec) override
Send a Fragment in non-reliable mode, using the underlying transfer plugin.
For code clarity, things checking for successful receive should check retval >= NO_RANK_INFO.
int receiveFragment(artdaq::Fragment &fragment, size_t receiveTimeout) override
Receive a Fragment, using the underlying transfer plugin.
Value to be returned upon receive timeout.
~BundleTransfer() override
BundleTransfer default Destructor.
CopyStatus
Returned from the send functions, this enumeration describes the possible return codes. If an exception occurs, it will be thrown and should be handled normally.