artdaq  v3_12_02
DispatcherCore.cc
1 #include "TRACE/tracemf.h"
2 #include "artdaq/DAQdata/Globals.hh" // include these 2 first -
3 #define TRACE_NAME (app_name + "_DispatcherCore").c_str()
4 
5 #include "artdaq/Application/DispatcherCore.hh"
6 
7 #include "fhiclcpp/ParameterSet.h"
8 
9 #include <boost/algorithm/string.hpp>
10 #include <boost/exception/all.hpp>
11 #include <boost/throw_exception.hpp>
12 #include <boost/tokenizer.hpp>
13 
14 #include <bitset>
15 #include <cerrno>
16 #include <csignal>
17 #include <iomanip>
18 #include <set>
19 #include <sstream>
20 #include <string>
21 #include <unordered_map>
22 #include <utility>
23 #include <vector>
24 
25 bool artdaq::DispatcherCore::initialize(fhicl::ParameterSet const& pset)
26 {
27  TLOG(TLVL_DEBUG + 32) << "initialize method called with DAQ "
28  << "ParameterSet = \"" << pset.to_string() << "\".";
29 
30  pset_ = pset;
31  // 04-Apr-2019, KAB: added support for art config params to be in an "art" block
32  if (pset_.has_key("art"))
33  {
34  pset_ = pset_.get<fhicl::ParameterSet>("art");
35  }
36  pset_.erase("outputs");
37  pset_.erase("physics");
38  pset_.erase("daq");
39 
40  TLOG(TLVL_DEBUG + 32) << "Pieces of the input pset that are saved for later: \"" << pset_.to_string() << "\".";
41 
42  // pull out the relevant parts of the ParameterSet
43  fhicl::ParameterSet daq_pset;
44  try
45  {
46  daq_pset = pset.get<fhicl::ParameterSet>("daq");
47  }
48  catch (...)
49  {
50  TLOG(TLVL_ERROR)
51  << "Unable to find the DAQ parameters in the initialization "
52  << "ParameterSet: \"" + pset.to_string() + "\".";
53  return false;
54  }
55  fhicl::ParameterSet agg_pset;
56  try
57  {
58  agg_pset = daq_pset.get<fhicl::ParameterSet>("dispatcher", daq_pset.get<fhicl::ParameterSet>("aggregator"));
59  }
60  catch (...)
61  {
62  TLOG(TLVL_ERROR)
63  << "Unable to find the Dispatcher parameters in the DAQ "
64  << "initialization ParameterSet: \"" + daq_pset.to_string() + "\".";
65  return false;
66  }
67 
68  broadcast_mode_ = agg_pset.get<bool>("broadcast_mode", true);
69  if (broadcast_mode_ && !agg_pset.has_key("broadcast_mode"))
70  {
71  agg_pset.put<bool>("broadcast_mode", true);
72  if (!agg_pset.has_key("non_reliable_mode"))
73  {
74  agg_pset.put<bool>("non_reliable_mode", true);
75  }
76  if (!agg_pset.has_key("non_reliable_mode_retry_count"))
77  {
78  agg_pset.put<int>("non_reliable_mode_retry_count", 2);
79  }
80  }
81 
82  agg_pset.erase("restart_crashed_art_processes");
83  agg_pset.put<bool>("restart_crashed_art_processes", false);
84 
85  agg_pset.erase("art_analyzer_count");
86  agg_pset.put<int>("art_analyzer_count", 0);
87 
88  // initialize the MetricManager and the names of our metrics
89  fhicl::ParameterSet metric_pset = daq_pset.get<fhicl::ParameterSet>("metrics", fhicl::ParameterSet());
90 
91  return initializeDataReceiver(pset, agg_pset, metric_pset);
92 }
93 
94 std::string artdaq::DispatcherCore::register_monitor(fhicl::ParameterSet const& pset)
95 {
96  TLOG(TLVL_DEBUG + 32) << "DispatcherCore::register_monitor called with argument \"" << pset.to_string() << "\"";
97  check_filters_();
98 
99  if (stop_requested_.load())
100  {
101  TLOG(TLVL_WARNING) << "DispatcherCore::register_monitor called after stop. NOT registering new monitor!";
102  return "Stopped";
103  }
104 
105  try
106  {
107  TLOG(TLVL_DEBUG + 32) << "Getting unique_label from input ParameterSet";
108  auto const& label = pset.get<std::string>("unique_label");
109  TLOG(TLVL_DEBUG + 32) << "Unique label is " << label;
110  {
111  std::lock_guard<std::mutex> lock(dispatcher_transfers_mutex_);
112  if (registered_monitors_.count(label) != 0u)
113  {
114  throw cet::exception("DispatcherCore") << "Unique label already exists!"; // NOLINT(cert-err60-cpp)
115  }
116 
117  registered_monitors_[label] = pset;
118  }
119 
120  // ELF, Jul 21, 2020: This can take a long time, and we don't want to block the XMLRPC thread
121  boost::thread thread([=] { start_art_process_(label); });
122  thread.detach();
123  }
124  catch (const cet::exception& e)
125  {
126  std::stringstream errmsg;
127  errmsg << "Unable to create a Transfer plugin with the FHiCL code \"" << pset.to_string() << "\", a new monitor has not been registered" << std::endl;
128  errmsg << "Exception: " << e.what();
129  TLOG(TLVL_ERROR) << errmsg.str();
130  return errmsg.str();
131  }
132  catch (const boost::exception& e)
133  {
134  std::stringstream errmsg;
135  errmsg << "Unable to create a Transfer plugin with the FHiCL code \"" << pset.to_string() << "\", a new monitor has not been registered" << std::endl;
136  errmsg << "Exception: " << boost::diagnostic_information(e);
137  TLOG(TLVL_ERROR) << errmsg.str();
138  return errmsg.str();
139  }
140  catch (const std::exception& e)
141  {
142  std::stringstream errmsg;
143  errmsg << "Unable to create a Transfer plugin with the FHiCL code \"" << pset.to_string() << "\", a new monitor has not been registered" << std::endl;
144  errmsg << "Exception: " << e.what();
145  TLOG(TLVL_ERROR) << errmsg.str();
146  return errmsg.str();
147  }
148  catch (...)
149  {
150  std::stringstream errmsg;
151  errmsg << "Unable to create a Transfer plugin with the FHiCL code \"" << pset.to_string() << "\", a new monitor has not been registered";
152  TLOG(TLVL_ERROR) << errmsg.str();
153  return errmsg.str();
154  }
155 
156  TLOG(TLVL_DEBUG + 32) << "Successfully registered monitor";
157  return "Success";
158 }
159 
160 std::string artdaq::DispatcherCore::unregister_monitor(std::string const& label)
161 {
162  TLOG(TLVL_DEBUG + 32) << "DispatcherCore::unregister_monitor called with argument \"" << label << "\"";
163  check_filters_();
164 
165  try
166  {
167  {
168  std::lock_guard<std::mutex> lock(dispatcher_transfers_mutex_);
169  if (registered_monitors_.count(label) == 0)
170  {
171  std::stringstream errmsg;
172  errmsg << "Warning in DispatcherCore::unregister_monitor: unable to find requested transfer plugin with "
173  << "label \"" << label << "\"";
174  TLOG(TLVL_WARNING) << errmsg.str();
175  return errmsg.str();
176  }
177 
178  registered_monitors_.erase(label);
179  }
180 
181  // ELF, Jul 21, 2020: This can take a long time, and we don't want to block the XMLRPC thread
182  boost::thread thread([=] { stop_art_process_(label); });
183  thread.detach();
184  }
185  catch (...)
186  {
187  std::stringstream errmsg;
188  errmsg << "Unable to unregister transfer plugin with label \"" << label << "\"";
189  TLOG(TLVL_ERROR) << errmsg.str();
190  return errmsg.str();
191  }
192 
193  TLOG(TLVL_DEBUG + 32) << "unregister_monitor completed successfully";
194  return "Success";
195 }
196 
197 fhicl::ParameterSet artdaq::DispatcherCore::merge_parameter_sets_(fhicl::ParameterSet const& skel, const std::string& label, const fhicl::ParameterSet& pset)
198 {
199  fhicl::ParameterSet generated_pset(skel);
200  fhicl::ParameterSet generated_outputs;
201  fhicl::ParameterSet generated_physics;
202  fhicl::ParameterSet generated_physics_analyzers;
203  fhicl::ParameterSet generated_physics_producers;
204  fhicl::ParameterSet generated_physics_filters;
205  std::unordered_map<std::string, std::vector<std::string>> generated_physics_filter_paths;
206 
207  TLOG(TLVL_DEBUG + 32) << "merge_parameter_sets_: Generating fhicl for monitor " << label;
208 
209  try
210  {
211  auto path = pset.get<std::vector<std::string>>("path");
212 
213  auto filters = pset.get<std::vector<fhicl::ParameterSet>>("filter_paths", std::vector<fhicl::ParameterSet>());
214  for (auto& filter : filters)
215  {
216  try
217  {
218  auto name = filter.get<std::string>("name");
219  auto path = filter.get<std::vector<std::string>>("path");
220  if (generated_physics_filter_paths.count(name) != 0u)
221  {
222  bool matched = generated_physics_filter_paths[name].size() == path.size();
223  for (size_t ii = 0; matched && ii < generated_physics_filter_paths[name].size(); ++ii)
224  {
225  matched = matched && path[ii] == generated_physics_filter_paths[name][ii];
226  }
227 
228  if (matched)
229  {
230  // Path is already configured
231  continue;
232  }
233 
234  auto newname = label + name;
235  generated_physics_filter_paths[newname] = path;
236  }
237  else
238  {
239  generated_physics_filter_paths[name] = path;
240  }
241  }
242  catch (...)
243  {
244  }
245  }
246 
247  // outputs section
248  auto outputs = pset.get<fhicl::ParameterSet>("outputs");
249  if (outputs.get_pset_names().size() > 1 || outputs.get_pset_names().empty())
250  {
251  // Only one output allowed per monitor
252  }
253  auto output_name = outputs.get_pset_names()[0];
254  auto output_pset = outputs.get<fhicl::ParameterSet>(output_name);
255  generated_outputs.put(label + output_name, output_pset);
256  bool outputInPath = false;
257  for (auto& ii : path)
258  {
259  if (ii == output_name)
260  {
261  ii = label + output_name;
262  outputInPath = true;
263  }
264  }
265  if (!outputInPath)
266  {
267  path.push_back(label + output_name);
268  }
269 
270  // physics section
271  auto physics_pset = pset.get<fhicl::ParameterSet>("physics");
272 
273  if (physics_pset.has_key("analyzers"))
274  {
275  auto analyzers = physics_pset.get<fhicl::ParameterSet>("analyzers");
276  for (const auto& key : analyzers.get_pset_names())
277  {
278  if (generated_physics_analyzers.has_key(key) && analyzers.get<fhicl::ParameterSet>(key) == generated_physics_analyzers.get<fhicl::ParameterSet>(key))
279  {
280  // Module is already configured
281  continue;
282  }
283  if (generated_physics_analyzers.has_key(key))
284  {
285  // Module already exists with name, rename
286  auto newkey = label + key;
287  generated_physics_analyzers.put<fhicl::ParameterSet>(newkey, analyzers.get<fhicl::ParameterSet>(key));
288  for (auto& ii : path)
289  {
290  if (ii == key)
291  {
292  ii = newkey;
293  }
294  }
295  }
296  else
297  {
298  generated_physics_analyzers.put<fhicl::ParameterSet>(key, analyzers.get<fhicl::ParameterSet>(key));
299  }
300  }
301  }
302  if (physics_pset.has_key("producers"))
303  {
304  auto producers = physics_pset.get<fhicl::ParameterSet>("producers");
305  for (const auto& key : producers.get_pset_names())
306  {
307  if (generated_physics_producers.has_key(key) && producers.get<fhicl::ParameterSet>(key) == generated_physics_producers.get<fhicl::ParameterSet>(key))
308  {
309  // Module is already configured
310  continue;
311  }
312  if (generated_physics_producers.has_key(key))
313  {
314  // Module already exists with name, rename
315  auto newkey = label + key;
316  generated_physics_producers.put<fhicl::ParameterSet>(newkey, producers.get<fhicl::ParameterSet>(key));
317  for (auto& ii : path)
318  {
319  if (ii == key)
320  {
321  ii = newkey;
322  }
323  }
324  }
325  else
326  {
327  generated_physics_producers.put<fhicl::ParameterSet>(key, producers.get<fhicl::ParameterSet>(key));
328  }
329  }
330  }
331  if (physics_pset.has_key("filters"))
332  {
333  auto filters = physics_pset.get<fhicl::ParameterSet>("filters");
334  for (const auto& key : filters.get_pset_names())
335  {
336  if (generated_physics_filters.has_key(key) && filters.get<fhicl::ParameterSet>(key) == generated_physics_filters.get<fhicl::ParameterSet>(key))
337  {
338  // Module is already configured
339  continue;
340  }
341  if (generated_physics_filters.has_key(key))
342  {
343  // Module already exists with name, rename
344  auto newkey = label + key;
345  generated_physics_filters.put<fhicl::ParameterSet>(newkey, filters.get<fhicl::ParameterSet>(key));
346  for (auto& ii : path)
347  {
348  if (ii == key)
349  {
350  ii = newkey;
351  }
352  }
353  }
354  else
355  {
356  generated_physics_filters.put<fhicl::ParameterSet>(key, filters.get<fhicl::ParameterSet>(key));
357  }
358  }
359  }
360  generated_physics.put<std::vector<std::string>>(label, path);
361  }
362  catch (cet::exception& e)
363  {
364  // Error in parsing input fhicl
365  TLOG(TLVL_ERROR) << "merge_parameter_sets_: Error processing input fhicl: " << e.what();
366  }
367 
368  TLOG(TLVL_DEBUG + 32) << "merge_parameter_sets_: Building final ParameterSet";
369  generated_pset.put("outputs", generated_outputs);
370 
371  generated_physics.put("analyzers", generated_physics_analyzers);
372  generated_physics.put("producers", generated_physics_producers);
373  generated_physics.put("filters", generated_physics_filters);
374 
375  for (auto& path : generated_physics_filter_paths)
376  {
377  generated_physics.put(path.first, path.second);
378  }
379 
380  generated_pset.put("physics", generated_physics);
381 
382  return generated_pset;
383 }
384 
385 fhicl::ParameterSet artdaq::DispatcherCore::generate_filter_fhicl_()
386 {
387  TLOG(TLVL_DEBUG + 32) << "generate_filter_fhicl_ BEGIN";
388  std::lock_guard<std::mutex> lock(dispatcher_transfers_mutex_);
389  fhicl::ParameterSet generated_pset = pset_;
390 
391  for (auto& monitor : registered_monitors_)
392  {
393  auto label = monitor.first;
394  auto pset = monitor.second;
395  generated_pset = merge_parameter_sets_(generated_pset, label, pset);
396  }
397 
398  TLOG(TLVL_DEBUG + 32) << "generate_filter_fhicl_ returning ParameterSet: " << generated_pset.to_string();
399  return generated_pset;
400 }
401 
402 void artdaq::DispatcherCore::check_filters_()
403 {
404  std::lock_guard<std::mutex> lock(dispatcher_transfers_mutex_);
405  auto it = registered_monitors_.begin();
406  while (it != registered_monitors_.end())
407  {
408  if (registered_monitor_pids_.count(it->first) != 0u)
409  {
410  if (!event_store_ptr_)
411  {
412  registered_monitor_pids_.erase(it->first);
413  }
414  else
415  {
416  auto pid = registered_monitor_pids_[it->first];
417  auto sts = kill(pid, 0);
418  if (sts < 0)
419  {
420  registered_monitor_pids_.erase(it->first);
421  }
422  }
423  }
424  ++it;
425  }
426 }
427 
428 void artdaq::DispatcherCore::start_art_process_(std::string const& label)
429 {
430  if (event_store_ptr_ != nullptr)
431  {
432  if (broadcast_mode_)
433  {
434  fhicl::ParameterSet pset;
435  {
436  std::lock_guard<std::mutex> lock(dispatcher_transfers_mutex_);
437  pset = registered_monitors_[label];
438  }
439  fhicl::ParameterSet ps = merge_parameter_sets_(pset_, label, pset);
440  TLOG(TLVL_DEBUG + 32) << "Starting art process with received fhicl";
441  auto pid = event_store_ptr_->StartArtProcess(ps, registered_monitors_.size() - 1);
442  {
443  std::lock_guard<std::mutex> lock(dispatcher_transfers_mutex_);
444  registered_monitor_pids_[label] = pid;
445  }
446  }
447  else
448  {
449  TLOG(TLVL_DEBUG + 32) << "Calling ReconfigureArt";
450  fhicl::ParameterSet generated_fhicl;
451  {
452  std::lock_guard<std::mutex> lock(dispatcher_transfers_mutex_);
453  generated_fhicl = generate_filter_fhicl_();
454  }
455  event_store_ptr_->ReconfigureArt(generated_fhicl);
456  TLOG(TLVL_DEBUG + 32) << "Done with ReconfigureArt";
457  }
458  }
459  else
460  {
461  TLOG(TLVL_ERROR) << "Unable to add monitor as there is no SharedMemoryEventManager instance!";
462  }
463 }
464 
465 void artdaq::DispatcherCore::stop_art_process_(std::string const& label)
466 {
467  if (event_store_ptr_ != nullptr)
468  {
469  if (broadcast_mode_)
470  {
471  if (registered_monitor_pids_.count(label) != 0u)
472  {
473  std::set<pid_t> pids;
474  {
475  std::lock_guard<std::mutex> lock(dispatcher_transfers_mutex_);
476  pids.insert(registered_monitor_pids_[label]);
477  registered_monitor_pids_.erase(label);
478  }
479  TLOG(TLVL_DEBUG + 32) << "Calling ShutdownArtProcesses";
480  event_store_ptr_->ShutdownArtProcesses(pids);
481  TLOG(TLVL_DEBUG + 32) << "Done with ShutdownArtProcesses";
482  }
483  }
484  else
485  {
486  TLOG(TLVL_DEBUG + 32) << "Calling ReconfigureArt";
487  fhicl::ParameterSet generated_fhicl;
488  {
489  std::lock_guard<std::mutex> lock(dispatcher_transfers_mutex_);
490  generated_fhicl = generate_filter_fhicl_();
491  }
492  event_store_ptr_->ReconfigureArt(generated_fhicl);
493  TLOG(TLVL_DEBUG + 32) << "Done with ReconfigureArt";
494  }
495  }
496 }
std::string unregister_monitor(std::string const &label)
Delete the TransferInterface having the given unique label.
std::string register_monitor(fhicl::ParameterSet const &pset)
Create a new TransferInterface instance using the given configuration.
bool initializeDataReceiver(fhicl::ParameterSet const &pset, fhicl::ParameterSet const &data_pset, fhicl::ParameterSet const &metric_pset)
Initialize the DataReceiverCore (should be called from initialize() overrides.
bool initialize(fhicl::ParameterSet const &pset) override
Processes the initialize request.