artdaq  v3_12_02
RoutingManagerPolicy.cc
1 #define TRACE_NAME "RoutingManagerPolicy"
2 #include "TRACE/tracemf.h"
3 
4 #include "artdaq/DAQdata/Globals.hh"
5 #include "artdaq/RoutingPolicies/RoutingManagerPolicy.hh"
6 
7 #include "fhiclcpp/ParameterSet.h"
8 
10  : tokens_used_since_last_update_(0)
11  , next_sequence_id_(1)
12  , max_token_count_(0)
13 {
14  routing_mode_ = detail::RoutingManagerModeConverter::stringToRoutingManagerMode(ps.get<std::string>("routing_manager_mode", "EventBuilding"));
15  routing_cache_max_size_ = ps.get<size_t>("routing_cache_size", 1000);
16 }
17 
19 {
20  auto table = detail::RoutingPacket();
21 
22  if (routing_mode_ == detail::RoutingManagerMode::EventBuilding)
23  {
24  std::lock_guard<std::mutex> lk(tokens_mutex_);
25  CreateRoutingTable(table);
26  UpdateCache(table);
27  CreateRoutingTableFromCache(table);
28  }
30  {
31  CreateRoutingTableFromCache(table);
32  }
33 
34  return table;
35 }
36 
37 void artdaq::RoutingManagerPolicy::AddReceiverToken(int rank, unsigned new_slots_free)
38 {
39  if (receiver_ranks_.count(rank) == 0u)
40  {
41  TLOG(TLVL_INFO) << "Adding rank " << rank << " to receivers list (initial tokens=" << new_slots_free << ")";
42  receiver_ranks_.insert(rank);
43  }
44  TLOG(TLVL_DEBUG + 35) << "AddReceiverToken BEGIN";
45  std::lock_guard<std::mutex> lk(tokens_mutex_);
46  if (new_slots_free == 1)
47  {
48  tokens_.push_back(rank);
49  }
50  else
51  {
52  // Randomly distribute multitokens through the token list
53  // Only used at start run time, so we can take the performance hit
54  for (unsigned i = 0; i < new_slots_free; ++i)
55  {
56  auto it = tokens_.begin();
57  if (!tokens_.empty())
58  {
59  std::advance(it, rand() % tokens_.size()); // NOLINT(cert-msc50-cpp)
60  }
61  tokens_.insert(it, rank);
62  }
63  }
64  if (tokens_.size() > max_token_count_)
65  {
66  max_token_count_ = tokens_.size();
67  }
68  TLOG(TLVL_DEBUG + 35) << "AddReceiverToken END";
69 }
70 
72 {
73  next_sequence_id_ = 1;
74  std::unique_lock<std::mutex> lk(tokens_mutex_);
75  tokens_.clear();
76  receiver_ranks_.clear();
77 }
78 
79 artdaq::detail::RoutingPacketEntry artdaq::RoutingManagerPolicy::GetRouteForSequenceID(artdaq::Fragment::sequence_id_t seq, int requesting_rank)
80 {
81  if (routing_mode_ != detail::RoutingManagerMode::DataFlow)
82  {
83  std::lock_guard<std::mutex> lk(routing_cache_mutex_);
84  if (routing_cache_.count(seq))
85  {
86  return detail::RoutingPacketEntry(seq, routing_cache_[seq][0].destination_rank);
87  }
88  else
89  {
90  std::lock_guard<std::mutex> tlk(tokens_mutex_);
91  auto entry = CreateRouteForSequenceID(seq, requesting_rank);
92  if (entry.sequence_id == seq)
93  {
94  routing_cache_[seq].emplace_back(seq, entry.destination_rank, requesting_rank);
95  }
96  return entry;
97  }
98  }
99  else
100  {
101  std::lock_guard<std::mutex> lk(routing_cache_mutex_);
102  if (routing_cache_.count(seq))
103  {
104  for (auto& entry : routing_cache_[seq])
105  {
106  if (entry.requesting_rank == requesting_rank)
107  {
108  return detail::RoutingPacketEntry(seq, entry.destination_rank);
109  }
110  }
111  }
112 
113  std::lock_guard<std::mutex> tlk(tokens_mutex_);
114  auto entry = CreateRouteForSequenceID(seq, requesting_rank);
115  if (entry.sequence_id == seq)
116  {
117  routing_cache_[seq].emplace_back(seq, entry.destination_rank, requesting_rank);
118  }
119 
120  TrimRoutingCache();
121  return entry;
122  }
124 }
125 
126 void artdaq::RoutingManagerPolicy::TrimRoutingCache()
127 {
128  while (routing_cache_.size() > routing_cache_max_size_)
129  {
130  routing_cache_.erase(routing_cache_.begin());
131  }
132 }
133 
134 void artdaq::RoutingManagerPolicy::UpdateCache(detail::RoutingPacket& table)
135 {
136  std::lock_guard<std::mutex> lk(routing_cache_mutex_);
137 
138  for (auto& entry : table)
139  {
140  if (!routing_cache_.count(entry.sequence_id))
141  {
142  routing_cache_[entry.sequence_id].emplace_back(entry.sequence_id, entry.destination_rank, my_rank);
143  }
144  }
145 }
146 
147 void artdaq::RoutingManagerPolicy::CreateRoutingTableFromCache(detail::RoutingPacket& table)
148 {
149  std::lock_guard<std::mutex> lk(routing_cache_mutex_);
150 
152  {
153  for (auto& cache_entry : routing_cache_)
154  {
155  if (!cache_entry.second[0].included_in_table)
156  {
157  table.push_back(artdaq::detail::RoutingPacketEntry(cache_entry.second[0].sequence_id, cache_entry.second[0].destination_rank));
158  cache_entry.second[0].included_in_table = true;
159  }
160  }
161 
162  TrimRoutingCache();
163  }
164 }
void AddReceiverToken(int rank, unsigned new_slots_free)
Add a token to the token list.
A row of the Routing Table.
void Reset()
Reset the policy, setting the next sequence ID to be used to 1, and removing any tokens.
detail::RoutingPacketEntry GetRouteForSequenceID(artdaq::Fragment::sequence_id_t seq, int requesting_rank)
Get an artdaq::detail::RoutingPacketEntry for a given sequence ID and rank. Used by RequestBasedEvent...
Multiple sources sending to a single destination. RoutingManager pushes table updates to all senders...
std::vector< RoutingPacketEntry > RoutingPacket
A RoutingPacket is simply a vector of RoutingPacketEntry objects. It is not suitable for network tran...
static RoutingManagerMode stringToRoutingManagerMode(std::string const &modeString)
Convert String to RoutingManagerMode.
RoutingManagerPolicy(const fhicl::ParameterSet &ps)
RoutingManagerPolicy Constructor.
Multiple sources sending to a single destination. Table updates are triggered by senders requesting r...
detail::RoutingManagerMode routing_mode_
Current routing mode.
detail::RoutingPacket GetCurrentTable()
Create a RoutingPacket from currently-owned tokens. Used by EventBuilder and RequestBasedEventBuilder...
One source sending to one destination (i.e. moving around completed events). Uses request-based routi...