artdaq  v3_11_00
RoutingManagerPolicy.cc
1 #define TRACE_NAME "RoutingManagerPolicy"
2 
3 #include "artdaq/RoutingPolicies/RoutingManagerPolicy.hh"
4 #include "fhiclcpp/ParameterSet.h"
5 
7  : tokens_used_since_last_update_(0)
8  , next_sequence_id_(1)
9  , max_token_count_(0)
10 {
11  routing_mode_ = detail::RoutingManagerModeConverter::stringToRoutingManagerMode(ps.get<std::string>("routing_manager_mode", "EventBuilding"));
12  routing_cache_max_size_ = ps.get<size_t>("routing_cache_size", 1000);
13 }
14 
16 {
17  auto table = detail::RoutingPacket();
18 
19  if (routing_mode_ == detail::RoutingManagerMode::EventBuilding)
20  {
21  std::lock_guard<std::mutex> lk(tokens_mutex_);
22  CreateRoutingTable(table);
23  UpdateCache(table);
24  CreateRoutingTableFromCache(table);
25  }
27  {
28  CreateRoutingTableFromCache(table);
29  }
30 
31  return table;
32 }
33 
34 void artdaq::RoutingManagerPolicy::AddReceiverToken(int rank, unsigned new_slots_free)
35 {
36  if (receiver_ranks_.count(rank) == 0u)
37  {
38  TLOG(TLVL_INFO) << "Adding rank " << rank << " to receivers list (initial tokens=" << new_slots_free << ")";
39  receiver_ranks_.insert(rank);
40  }
41  TLOG(10) << "AddReceiverToken BEGIN";
42  std::lock_guard<std::mutex> lk(tokens_mutex_);
43  if (new_slots_free == 1)
44  {
45  tokens_.push_back(rank);
46  }
47  else
48  {
49  // Randomly distribute multitokens through the token list
50  // Only used at start run time, so we can take the performance hit
51  for (unsigned i = 0; i < new_slots_free; ++i)
52  {
53  auto it = tokens_.begin();
54  if (!tokens_.empty())
55  {
56  std::advance(it, rand() % tokens_.size()); // NOLINT(cert-msc50-cpp)
57  }
58  tokens_.insert(it, rank);
59  }
60  }
61  if (tokens_.size() > max_token_count_)
62  {
63  max_token_count_ = tokens_.size();
64  }
65  TLOG(10) << "AddReceiverToken END";
66 }
67 
69 {
70  next_sequence_id_ = 1;
71  std::unique_lock<std::mutex> lk(tokens_mutex_);
72  tokens_.clear();
73  receiver_ranks_.clear();
74 }
75 
76 artdaq::detail::RoutingPacketEntry artdaq::RoutingManagerPolicy::GetRouteForSequenceID(artdaq::Fragment::sequence_id_t seq, int requesting_rank)
77 {
78  if (routing_mode_ != detail::RoutingManagerMode::DataFlow)
79  {
80  std::lock_guard<std::mutex> lk(routing_cache_mutex_);
81  if (routing_cache_.count(seq))
82  {
83  return detail::RoutingPacketEntry(seq, routing_cache_[seq][0].destination_rank);
84  }
85  else
86  {
87  std::lock_guard<std::mutex> tlk(tokens_mutex_);
88  auto entry = CreateRouteForSequenceID(seq, requesting_rank);
89  if (entry.sequence_id == seq)
90  {
91  routing_cache_[seq].emplace_back(seq, entry.destination_rank, requesting_rank);
92  }
93  return entry;
94  }
95  }
96  else
97  {
98  std::lock_guard<std::mutex> lk(routing_cache_mutex_);
99  if (routing_cache_.count(seq))
100  {
101  for (auto& entry : routing_cache_[seq])
102  {
103  if (entry.requesting_rank == requesting_rank)
104  {
105  return detail::RoutingPacketEntry(seq, entry.destination_rank);
106  }
107  }
108  }
109 
110  std::lock_guard<std::mutex> tlk(tokens_mutex_);
111  auto entry = CreateRouteForSequenceID(seq, requesting_rank);
112  if (entry.sequence_id == seq)
113  {
114  routing_cache_[seq].emplace_back(seq, entry.destination_rank, requesting_rank);
115  }
116 
117  TrimRoutingCache();
118  return entry;
119  }
121 }
122 
123 void artdaq::RoutingManagerPolicy::TrimRoutingCache()
124 {
125  while (routing_cache_.size() > routing_cache_max_size_)
126  {
127  routing_cache_.erase(routing_cache_.begin());
128  }
129 
130 }
131 
132 void artdaq::RoutingManagerPolicy::UpdateCache(detail::RoutingPacket& table)
133 {
134  std::lock_guard<std::mutex> lk(routing_cache_mutex_);
135 
136  for(auto& entry : table)
137  {
138  if (!routing_cache_.count(entry.sequence_id)) {
139 
140  routing_cache_[entry.sequence_id].emplace_back(entry.sequence_id, entry.destination_rank, my_rank);
141  }
142  }
143 
144 }
145 
146 void artdaq::RoutingManagerPolicy::CreateRoutingTableFromCache(detail::RoutingPacket& table)
147 {
148  std::lock_guard<std::mutex> lk(routing_cache_mutex_);
149 
151  {
152  for (auto& cache_entry : routing_cache_)
153  {
154  if (!cache_entry.second[0].included_in_table) {
155  table.push_back(artdaq::detail::RoutingPacketEntry(cache_entry.second[0].sequence_id, cache_entry.second[0].destination_rank));
156  cache_entry.second[0].included_in_table = true;
157  }
158  }
159 
160  TrimRoutingCache();
161  }
162 }
void AddReceiverToken(int rank, unsigned new_slots_free)
Add a token to the token list.
A row of the Routing Table.
void Reset()
Reset the policy, setting the next sequence ID to be used to 1, and removing any tokens.
detail::RoutingPacketEntry GetRouteForSequenceID(artdaq::Fragment::sequence_id_t seq, int requesting_rank)
Get an artdaq::detail::RoutingPacketEntry for a given sequence ID and rank. Used by RequestBasedEvent...
Multiple sources sending to a single destination. RoutingManager pushes table updates to all senders...
std::vector< RoutingPacketEntry > RoutingPacket
A RoutingPacket is simply a vector of RoutingPacketEntry objects. It is not suitable for network tran...
static RoutingManagerMode stringToRoutingManagerMode(std::string const &modeString)
Convert String to RoutingManagerMode.
RoutingManagerPolicy(const fhicl::ParameterSet &ps)
RoutingManagerPolicy Constructor.
Multiple sources sending to a single destination. Table updates are triggered by senders requesting r...
detail::RoutingManagerMode routing_mode_
Current routing mode.
detail::RoutingPacket GetCurrentTable()
Create a RoutingPacket from currently-owned tokens. Used by EventBuilder and RequestBasedEventBuilder...
One source sending to one destination (i.e. moving around completed events). Uses request-based routi...