artdaq  v2_03_02
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Pages
RootMPIOutput_module.cc
1 #include "art/Framework/Core/ModuleMacros.h"
2 #include "art/Framework/Core/OutputModule.h"
3 #include "art/Framework/Principal/EventPrincipal.h"
4 #include "art/Framework/Principal/OutputHandle.h"
5 #include "art/Framework/Principal/RunPrincipal.h"
6 #include "art/Framework/Principal/SubRunPrincipal.h"
7 #include "art/Framework/Services/Registry/ServiceHandle.h"
8 #if ART_HEX_VERSION >= 0x20703
9 # include <iterator>
10 #else
11 # include "art/Persistency/Provenance/BranchIDListHelper.h"
12 #endif
13 #include "art/Persistency/Provenance/BranchIDListRegistry.h"
14 #include "art/Persistency/Provenance/ProcessHistoryRegistry.h"
15 #include "art/Persistency/Provenance/ProductMetaData.h"
16 
17 #include "canvas/Persistency/Provenance/BranchDescription.h"
18 #include "canvas/Persistency/Provenance/BranchIDList.h"
19 #include "canvas/Persistency/Provenance/BranchKey.h"
20 #include "canvas/Persistency/Provenance/History.h"
21 #include "canvas/Persistency/Provenance/ParentageRegistry.h"
22 #include "canvas/Persistency/Provenance/ProcessConfiguration.h"
23 #include "canvas/Persistency/Provenance/ProcessConfigurationID.h"
24 #include "canvas/Persistency/Provenance/ProcessHistoryID.h"
25 #include "canvas/Persistency/Provenance/ProductList.h"
26 #include "canvas/Persistency/Provenance/ProductProvenance.h"
27 #include "canvas/Persistency/Provenance/RunAuxiliary.h"
28 #include "canvas/Persistency/Provenance/SubRunAuxiliary.h"
29 #include "canvas/Utilities/DebugMacros.h"
30 #include "canvas/Utilities/Exception.h"
31 #include "cetlib/column_width.h"
32 #include "cetlib/lpad.h"
33 #include "cetlib/rpad.h"
34 #include <algorithm>
35 #include "fhiclcpp/ParameterSet.h"
36 #include "fhiclcpp/ParameterSetID.h"
37 #include "fhiclcpp/ParameterSetRegistry.h"
38 
39 #include "artdaq/ArtModules/NetMonTransportService.h"
40 #include "artdaq/ArtModules/detail/ParentageMap.hh"
41 #include "artdaq/DAQdata/Globals.hh"
42 #include "artdaq/DAQdata/NetMonHeader.hh"
43 
44 #include <iomanip>
45 #include <iostream>
46 #include <sstream>
47 #include <string>
48 #include <vector>
49 
50 #include <unistd.h>
51 
52 #include <TClass.h>
53 #include <TMessage.h>
54 
55 #define TRACE_NAME "RootMPIOutput"
56 
57 # define CONST_WRITE
58 
59 namespace art
60 {
61  class RootMPIOutput;
62 }
63 
64 
70 class art::RootMPIOutput : public OutputModule
71 {
72 public:
80  explicit RootMPIOutput(fhicl::ParameterSet const& ps);
81 
86 
87 private:
88  virtual void openFile(FileBlock const&);
89 
90  virtual void closeFile();
91 
92  virtual void respondToCloseInputFile(FileBlock const&);
93 
94  virtual void respondToCloseOutputFiles(FileBlock const&);
95 
96  virtual void endJob();
97 
98  virtual void write(EventPrincipal CONST_WRITE&);
99 
100  virtual void writeRun(RunPrincipal CONST_WRITE&);
101 
102  virtual void writeSubRun(SubRunPrincipal CONST_WRITE&);
103 
104  void writeDataProducts(TBufferFile&, const Principal&,
105  std::vector<BranchKey*>&);
106 
107 private:
108  bool initMsgSent_;
109 };
110 
112 RootMPIOutput(fhicl::ParameterSet const& ps)
113  : OutputModule(ps)
114  , initMsgSent_(false)
115 {
116  TLOG_DEBUG("RootMPIOutput") << "Begin: RootMPIOutput::RootMPIOutput(ParameterSet const& ps)" << TLOG_ENDL;
117  ServiceHandle<NetMonTransportService> transport;
118  transport->connect();
119  TLOG_DEBUG("RootMPIOutput") << "End: RootMPIOutput::RootMPIOutput(ParameterSet const& ps)" << TLOG_ENDL;
120 }
121 
124 {
125  TLOG_DEBUG("RootMPIOutput") << "Begin: RootMPIOutput::~RootMPIOutput()" << TLOG_ENDL;
126  ServiceHandle<NetMonTransportService> transport;
127  transport->disconnect();
128  TLOG_DEBUG("RootMPIOutput") << "End: RootMPIOutput::~RootMPIOutput()" << TLOG_ENDL;
129 }
130 
131 void
132 art::RootMPIOutput::
133 openFile(FileBlock const&)
134 {
135  TRACE(5, "RootMPIOutput: Begin/End: RootMPIOutput::openFile(const FileBlock&)");
136 }
137 
138 void
139 art::RootMPIOutput::
140 closeFile()
141 {
142  TRACE(5, "RootMPIOutput: Begin/End: RootMPIOutput::closeFile()");
143 }
144 
145 void
146 art::RootMPIOutput::
147 respondToCloseInputFile(FileBlock const&)
148 {
149  TRACE(5, "RootMPIOutput: Begin/End: RootMPIOutput::"
150  "respondToCloseOutputFiles(FileBlock const&)");
151 }
152 
153 void
154 art::RootMPIOutput::
155 respondToCloseOutputFiles(FileBlock const&)
156 {
157  TRACE(5, "RootMPIOutput: Begin/End: RootMPIOutput::"
158  "respondToCloseOutputFiles(FileBlock const&)");
159 }
160 
161 static
162 void
163 send_shutdown_message()
164 {
165  // 4/1/2016, ELF: Apparently, all this function does is make things not work.
166  // At this point in the state machine, RHandles and SHandles have already been
167  // destructed. Calling sendMessage will cause SHandles to reconnect itself,
168  // but the other end will never recieve the message.
169 }
170 
171 void
172 art::RootMPIOutput::
173 endJob()
174 {
175  TRACE(5, "RootMPIOutput: Begin: RootMPIOutput::endJob()");
176  send_shutdown_message();
177  TRACE(5, "RootMPIOutput: End: RootMPIOutput::endJob()");
178 }
179 
180 //#pragma GCC push_options
181 //#pragma GCC optimize ("O0")
182 static
183 void
184 send_init_message()
185 {
186  TRACE(5, "RootMPIOutput: Begin: RootMPIOutput static send_init_message()");
187  //
188  // Get the classes we will need.
189  //
190  //static TClass* string_class = TClass::GetClass("std::string");
191  //if (string_class == nullptr) {
192  // throw art::Exception(art::errors::DictionaryNotFound) <<
193  // "RootMPIOutput static send_init_message(): "
194  // "Could not get TClass for std::string!";
195  //}
196  static TClass* product_list_class = TClass::GetClass(
197  "std::map<art::BranchKey,art::BranchDescription>");
198  if (product_list_class == nullptr)
199  {
200  throw art::Exception(art::errors::DictionaryNotFound) <<
201  "RootMPIOutput static send_init_message(): "
202  "Could not get TClass for "
203  "map<art::BranchKey,art::BranchDescription>!";
204  }
205  //typedef std::map<const ProcessHistoryID,ProcessHistory> ProcessHistoryMap;
206  //TClass* process_history_map_class = TClass::GetClass(
207  // "std::map<const art::ProcessHistoryID,art::ProcessHistory>");
208  //FIXME: Replace the "2" here with a use of the proper enum value!
209  static TClass* process_history_map_class = TClass::GetClass(
210  "std::map<const art::Hash<2>,art::ProcessHistory>");
211  if (process_history_map_class == nullptr)
212  {
213  throw art::Exception(art::errors::DictionaryNotFound) <<
214  "RootMPIOutput static send_init_message(): "
215  "Could not get class for "
216  "std::map<const art::Hash<2>,art::ProcessHistory>!";
217  }
218  //static TClass* parentage_map_class = TClass::GetClass(
219  // "std::map<const art::ParentageID,art::Parentage>");
220  //FIXME: Replace the "5" here with a use of the proper enum value!
221  static TClass* parentage_map_class = TClass::GetClass("art::ParentageMap");
222  if (parentage_map_class == nullptr)
223  {
224  throw art::Exception(art::errors::DictionaryNotFound) <<
225  "RootMPIOutput static send_init_message(): "
226  "Could not get class for ParentageMap.";
227  }
228  //
229  // Construct and send the init message.
230  //
231  TBufferFile msg(TBuffer::kWrite);
232  msg.SetWriteMode();
233  //
234  // Stream the message type code.
235  //
236  TRACE(5, "RootMPIOutput: RootMPIOutput static send_init_message(): "
237  "Streaming message type code ...");
238  msg.WriteULong(1);
239  TRACE(5, "RootMPIOutput: RootMPIOutput static send_init_message(): "
240  "Finished streaming message type code.");
241 
242  //
243  // Stream the ParameterSetRegistry.
244  //
245  unsigned long ps_cnt = fhicl::ParameterSetRegistry::size();
246  TRACE(5, "RootMPIOutput: RootMPIOutput static send_init_message(): parameter set count: " + std::to_string(ps_cnt));
247  msg.WriteULong(ps_cnt);
248  TRACE(5, "RootMPIOutput: RootMPIOutput static send_init_message(): Streaming parameter sets ...");
249  for (
250 # if ART_HEX_VERSION >= 0x20703
251  auto I = std::begin(fhicl::ParameterSetRegistry::get()),
252  E = std::end(fhicl::ParameterSetRegistry::get());
253 # else
254  auto I = fhicl::ParameterSetRegistry::begin(),
255  E = fhicl::ParameterSetRegistry::end();
256 # endif
257  I != E; ++I)
258  {
259  std::string pset_str = I->second.to_string();
260  //msg.WriteObjectAny(&pset_str, string_class);
261  msg.WriteStdString(pset_str);
262  }
263  TRACE(5, "RootMPIOutput: RootMPIOutput static send_init_message(): Finished streaming parameter sets.");
264 
265  //
266  // Stream the MasterProductRegistry.
267  //
268  TRACE(5, "RootMPIOutput: RootMPIOutput static send_init_message(): Streaming MasterProductRegistry ...");
269  art::ProductList productList(
270  art::ProductMetaData::instance().productList());
271  msg.WriteObjectAny(&productList, product_list_class);
272  TRACE(5, "RootMPIOutput: RootMPIOutput static send_init_message(): Finished streaming MasterProductRegistry.");
273 
274  //
275  // Dump The BranchIDListRegistry
276  //
277  if (art::debugit() >= 2)
278  {
279  //typedef vector<BranchID::value_type> BranchIDList
280  //typedef vector<BranchIDList> BranchIDLists
281  //std::vector<std::vector<art::BranchID::value_type>>
282 # if ART_HEX_VERSION >= 0x20703
283  art::BranchIDLists const * bilr =
284  &art::BranchIDListRegistry::instance().data();
285 # else
286  art::BranchIDLists* bilr =
287  &art::BranchIDListRegistry::instance()->data();
288 # endif
289  FDEBUG(2) << "RootMPIOutput static send_init_message(): "
290  "Content of BranchIDLists\n";
291  int max_bli = bilr->size();
292  FDEBUG(2) << "RootMPIOutput static send_init_message(): "
293  "max_bli: " << max_bli << '\n';
294  for (int i = 0; i < max_bli; ++i)
295  {
296  int max_prdidx = (*bilr)[i].size();
297  FDEBUG(2) << "RootMPIOutput static send_init_message(): "
298  "max_prdidx: " << max_prdidx << '\n';
299  for (int j = 0; j < max_prdidx; ++j)
300  {
301  FDEBUG(2) << "RootMPIOutput static send_init_message(): "
302  "bli: " << i
303  << " prdidx: " << j
304  << " bid: 0x" << std::hex
305  << static_cast<unsigned long>((*bilr)[i][j])
306  << std::dec << '\n';
307  }
308  }
309  }
310 
311 # if ART_HEX_VERSION >= 0x20703
312  art::ProcessHistoryMap phr;
313  for (auto const& pr : art::ProcessHistoryRegistry::get()) {
314  phr.emplace(pr);
315  }
316 # endif
317  //
318  // Dump the ProcessHistoryRegistry.
319  //
320  if (art::debugit() >= 1)
321  {
322  TRACE(5, "RootMPIOutput: RootMPIOutput static send_init_message(): "
323  "Dumping ProcessHistoryRegistry ...");
324  //typedef std::map<const ProcessHistoryID,ProcessHistory>
325  // ProcessHistoryMap;
326 # if ART_HEX_VERSION < 0x20703
327  art::ProcessHistoryMap const& phr = art::ProcessHistoryRegistry::get();
328 # endif
329  TRACE(5, "RootMPIOutput: RootMPIOutput static send_init_message(): "
330  "phr: size: " + std::to_string(phr.size()));
331  for (auto I = phr.begin(), E = phr.end(); I != E; ++I)
332  {
333  std::ostringstream OS;
334  I->first.print(OS);
335  TRACE(5, "RootMPIOutput: RootMPIOutput static send_init_message(): "
336  "phr: id: '" + OS.str() + "'");
337  }
338  }
339  //
340  // Stream the ProcessHistoryRegistry.
341  //
342  TRACE(5, "RootMPIOutput: RootMPIOutput static send_init_message(): "
343  "Streaming ProcessHistoryRegistry ...");
344  //typedef std::map<const ProcessHistoryID,ProcessHistory>
345  // ProcessHistoryMap;
346 # if ART_HEX_VERSION >= 0x20703
347  const art::ProcessHistoryMap& phm = phr;
348 # else
349  const art::ProcessHistoryMap& phm = art::ProcessHistoryRegistry::get();
350 # endif
351  TRACE(5, "RootMPIOutput: RootMPIOutput static send_init_message(): "
352  "phm: size: " + std::to_string(phm.size()));
353  msg.WriteObjectAny(&phm, process_history_map_class);
354  TRACE(5, "RootMPIOutput: RootMPIOutput static send_init_message(): "
355  "Finished streaming ProcessHistoryRegistry.");
356 
357  //
358  // Stream the ParentageRegistry.
359  //
360  TRACE(5, "RootMPIOutput: static send_init_message(): "
361  "Streaming ParentageRegistry ...");
362 # if ART_HEX_VERSION >= 0x20703
363  art::ParentageMap parentageMap{};
364  for (auto const& pr : art::ParentageRegistry::get()) {
365  parentageMap.emplace(pr.first, pr.second);
366  }
367 # else
368  const art::ParentageMap& parentageMap = art::ParentageRegistry::get();
369 # endif
370 
371  msg.WriteObjectAny(&parentageMap, parentage_map_class);
372 
373  TRACE(5, "RootMPIOutput: static send_init_message(): Finished streaming ParentageRegistry.");
374 
375  //
376  //
377  // Send init message.
378  //
379  art::ServiceHandle<NetMonTransportService> transport;
380  TRACE(5, "RootMPIOutput: RootMPIOutput static send_init_message(): "
381  "Sending the init message to "
382  + std::to_string(transport->dataReceiverCount()) +
383  " data receivers ...");
384  for (size_t idx = 0; idx < transport->dataReceiverCount(); ++idx)
385  {
386  transport->sendMessage(idx, artdaq::Fragment::InitFragmentType, msg);
387  }
388  TRACE(5, "RootMPIOutput: RootMPIOutput static send_init_message(): "
389  "Init message(s) sent.");
390 
391  TRACE(5, "RootMPIOutput: End: RootMPIOutput static send_init_message()");
392 }
393 
394 //#pragma GCC pop_options
395 
396 void
397 art::RootMPIOutput::
398 writeDataProducts(TBufferFile& msg, const Principal& principal,
399  std::vector<BranchKey*>& bkv)
400 {
401  TRACE(5, "RootMPIOutput: Begin: RootMPIOutput::writeDataProducts(...)");
402  //
403  // Fetch the class dictionaries we need for
404  // writing out the data products.
405  //
406  static TClass* branch_key_class = TClass::GetClass("art::BranchKey");
407  if (branch_key_class == nullptr)
408  {
409  throw art::Exception(art::errors::DictionaryNotFound) <<
410  "RootMPIOutput::writeDataProducts(...): "
411  "Could not get TClass for art::BranchKey!";
412  }
413  static TClass* prdprov_class = TClass::GetClass("art::ProductProvenance");
414  if (prdprov_class == nullptr)
415  {
416  throw art::Exception(art::errors::DictionaryNotFound) <<
417  "RootMPIOutput::writeDataProducts(...): "
418  "Could not get TClass for art::ProductProvenance!";
419  }
420  //
421  // Calculate the data product count.
422  //
423  unsigned long prd_cnt = 0;
424  //std::map<art::BranchID, std::shared_ptr<art::Group>>::const_iterator
425  for (auto I = principal.begin(), E = principal.end(); I != E; ++I)
426  {
427  if (I->second->productUnavailable() || !selected(I->second->productDescription()))
428  {
429  continue;
430  }
431  ++prd_cnt;
432  }
433  //
434  // Write the data product count.
435  //
436  TRACE(5, "RootMPIOutput: RootMPIOutput::writeDataProducts(...): "
437  "Streaming product count: " + std::to_string(prd_cnt));
438  msg.WriteULong(prd_cnt);
439  TRACE(5, "RootMPIOutput: RootMPIOutput::writeDataProducts(...): "
440  "Finished streaming product count.");
441 
442  //
443  // Loop over the groups in the RunPrincipal and
444  // write out the data products.
445  //
446  // Note: We need this vector of keys because the ROOT I/O mechanism
447  // requires that each object inserted in the message has a
448  // unique address, so we force that by holding on to each
449  // branch key manufactured in the loop until after we are
450  // done constructing the message.
451  //
452  bkv.reserve(prd_cnt);
453  //std::map<art::BranchID, std::shared_ptr<art::Group>>::const_iterator
454  for (auto I = principal.begin(), E = principal.end(); I != E; ++I)
455  {
456  if (I->second->productUnavailable() || !selected(I->second->productDescription()))
457  {
458  continue;
459  }
460  const BranchDescription& bd(I->second->productDescription());
461  bkv.push_back(new BranchKey(bd));
462  if (art::debugit() >= 2)
463  {
464  FDEBUG(2) << "RootMPIOutput::writeDataProducts(...): "
465  "Dumping branch key of class: '"
466  << bkv.back()->friendlyClassName_
467  << "' modlbl: '"
468  << bkv.back()->moduleLabel_
469  << "' instnm: '"
470  << bkv.back()->productInstanceName_
471  << "' procnm: '"
472  << bkv.back()->processName_
473  << "'";
474  }
475  TRACE(5, "RootMPIOutput: RootMPIOutput::writeDataProducts(...): "
476  "Streaming branch key of class: '"
477  + bd.producedClassName()
478  + "' modlbl: '"
479  + bd.moduleLabel()
480  + "' instnm: '"
481  + bd.productInstanceName()
482  + "' procnm: '"
483  + bd.processName()
484  + "'");
485  msg.WriteObjectAny(bkv.back(), branch_key_class);
486  TRACE(5, "RootMPIOutput: RootMPIOutput::writeDataProducts(...): "
487  "Streaming product of class: '"
488  + bd.producedClassName()
489  + "' modlbl: '"
490  + bd.moduleLabel()
491  + "' instnm: '"
492  + bd.productInstanceName()
493  + "' procnm: '"
494  + bd.processName()
495  + "'");
496  OutputHandle oh = principal.getForOutput(bd.branchID(), true);
497  const EDProduct* prd = oh.wrapper();
498  msg.WriteObjectAny(prd, TClass::GetClass(bd.wrappedName().c_str()));
499  TRACE(5, "RootMPIOutput: RootMPIOutput::writeDataProducts(...): "
500  "Streaming product provenance of class: '"
501  + bd.producedClassName()
502  + "' modlbl: '"
503  + bd.moduleLabel()
504  + "' instnm: '"
505  + bd.productInstanceName()
506  + "' procnm: '"
507  + bd.processName()
508  + "'");
509  const ProductProvenance* prdprov =
510  I->second->productProvenancePtr().get();
511  msg.WriteObjectAny(prdprov, prdprov_class);
512  }
513  TRACE(5, "RootMPIOutput: End: RootMPIOutput::writeDataProducts(...)");
514 }
515 
516 void
517 art::RootMPIOutput::
518 write(CONST_WRITE EventPrincipal& ep)
519 {
520  //
521  // Write an Event message.
522  //
523  TRACE(5, "RootMPIOutput: Begin: RootMPIOutput::"
524  "write(const EventPrincipal& ep)");
525  if (!initMsgSent_)
526  {
527  send_init_message();
528  initMsgSent_ = true;
529  }
530  //
531  // Get root classes needed for I/O.
532  //
533  static TClass* run_aux_class = TClass::GetClass("art::RunAuxiliary");
534  if (run_aux_class == nullptr)
535  {
536  throw art::Exception(art::errors::DictionaryNotFound) <<
537  "RootMPIOutput::write(const EventPrincipal& ep): "
538  "Could not get TClass for art::RunAuxiliary!";
539  }
540  static TClass* subrun_aux_class = TClass::GetClass("art::SubRunAuxiliary");
541  if (subrun_aux_class == nullptr)
542  {
543  throw art::Exception(art::errors::DictionaryNotFound) <<
544  "RootMPIOutput::write(const EventPrincipal& ep): "
545  "Could not get TClass for art::SubRunAuxiliary!";
546  }
547  static TClass* event_aux_class = TClass::GetClass("art::EventAuxiliary");
548  if (event_aux_class == nullptr)
549  {
550  throw art::Exception(art::errors::DictionaryNotFound) <<
551  "RootMPIOutput::write(const EventPrincipal& ep): "
552  "Could not get TClass for art::EventAuxiliary!";
553  }
554  static TClass* history_class = TClass::GetClass("art::History");
555  if (history_class == nullptr)
556  {
557  throw art::Exception(art::errors::DictionaryNotFound) <<
558  "RootMPIOutput::write(const EventPrincipal& ep): "
559  "Could not get TClass for art::History!";
560  }
561  //
562  // Setup message buffer.
563  //
564  TBufferFile msg(TBuffer::kWrite);
565  msg.SetWriteMode();
566  //
567  // Write message type code.
568  //
569  TRACE(5, "RootMPIOutput: RootMPIOutput::write(const EventPrincipal& ep): "
570  "Streaming message type code ...");
571  msg.WriteULong(4);
572  TRACE(5, "RootMPIOutput: RootMPIOutput::write(const EventPrincipal& ep): "
573  "Finished streaming message type code.");
574 
575  //
576  // Write RunAuxiliary.
577  //
578  TRACE(5, "RootMPIOutput: RootMPIOutput::write(const EventPrincipal& ep): "
579  "Streaming RunAuxiliary ...");
580  msg.WriteObjectAny(&ep.subRunPrincipal().runPrincipal().aux(),
581  run_aux_class);
582  TRACE(5, "RootMPIOutput: RootMPIOutput::write(const EventPrincipal& ep): "
583  "Finished streaming RunAuxiliary.");
584 
585  //
586  // Write SubRunAuxiliary.
587  //
588  TRACE(5, "RootMPIOutput: RootMPIOutput::write(const EventPrincipal& ep): "
589  "Streaming SubRunAuxiliary ...");
590  msg.WriteObjectAny(&ep.subRunPrincipal().aux(),
591  subrun_aux_class);
592  TRACE(5, "RootMPIOutput: RootMPIOutput::write(const EventPrincipal& ep): "
593  "Finished streaming SubRunAuxiliary.");
594 
595  //
596  // Write EventAuxiliary.
597  //
598  {
599  TRACE(5, "RootMPIOutput: RootMPIOutput::write(const EventPrincipal& ep): "
600  "Streaming EventAuxiliary ...");
601  msg.WriteObjectAny(&ep.aux(), event_aux_class);
602  TRACE(5, "RootMPIOutput: RootMPIOutput::write(const EventPrincipal& ep): "
603  "Finished streaming EventAuxiliary.");
604  }
605  //
606  // Write History.
607  //
608  {
609  TRACE(5, "RootMPIOutput: RootMPIOutput::write(const EventPrincipal& ep): "
610  "Streaming History ...");
611  msg.WriteObjectAny(&ep.history(), history_class);
612  TRACE(5, "RootMPIOutput: RootMPIOutput::write(const EventPrincipal& ep): "
613  "Finished streaming History.");
614  }
615  //
616  // Write data products.
617  //
618  std::vector<BranchKey*> bkv;
619  writeDataProducts(msg, ep, bkv);
620  //
621  // Send message.
622  //
623  {
624  ServiceHandle<NetMonTransportService> transport;
625  TRACE(5, "RootMPIOutput: RootMPIOutput::write(const EventPrincipal& ep): "
626  "Sending a message ...");
627  transport->sendMessage(ep.id().event(), artdaq::Fragment::DataFragmentType, msg);
628  TRACE(5, "RootMPIOutput: RootMPIOutput::write(const EventPrincipal& ep): "
629  "Message sent.");
630  }
631  //
632  // Delete the branch keys we created for the message.
633  //
634  for (auto I = bkv.begin(), E = bkv.end(); I != E; ++I)
635  {
636  delete *I;
637  *I = 0;
638  }
639  TRACE(5, "RootMPIOutput: End: RootMPIOutput::write(const EventPrincipal& ep)");
640 }
641 
642 void
643 art::RootMPIOutput::
644 writeRun(CONST_WRITE RunPrincipal& rp)
645 {
646  //
647  // Write an EndRun message.
648  //
649  TRACE(5, "RootMPIOutput: Begin: RootMPIOutput::writeRun(const RunPrincipal& rp)");
650  (void)rp;
651  if (!initMsgSent_)
652  {
653  send_init_message();
654  initMsgSent_ = true;
655  }
656 #if 0
657  //
658  // Fetch the class dictionaries we need for
659  // writing out the auxiliary information.
660  //
661  static TClass* run_aux_class = TClass::GetClass("art::RunAuxiliary");
662  assert(run_aux_class != nullptr && "writeRun: Could not get TClass for "
663  "art::RunAuxiliary!");
664  //
665  // Begin preparing message.
666  //
667  TBufferFile msg(TBuffer::kWrite);
668  msg.SetWriteMode();
669  //
670  // Write message type code.
671  //
672  {
673  TRACE(5, "RootMPIOutput: writeRun: streaming message type code ...");
674  msg.WriteULong(2);
675  TRACE(5, "RootMPIOutput: writeRun: finished streaming message type code.");
676  }
677  //
678  // Write RunAuxiliary.
679  //
680  {
681  TRACE(5, "RootMPIOutput: writeRun: streaming RunAuxiliary ...");
682  if (art::debugit() >= 1) {
683  TRACE(5, "RootMPIOutput: writeRun: dumping ProcessHistoryRegistry ...");
684  //typedef std::map<const ProcessHistoryID,ProcessHistory>
685  // ProcessHistoryMap;
686  art::ProcessHistoryMap const& phr =
687  art::ProcessHistoryRegistry::get();
688  TRACE(5, "RootMPIOutput: writeRun: phr: size: " << phr.size() << '\n';
689  for (auto I = phr.begin(), E = phr.end(); I != E; ++I) {
690  std::ostringstream OS;
691  I->first.print(OS);
692  TRACE(5, "RootMPIOutput: writeRun: phr: id: '" << OS.str() << "'");
693  OS.str("");
694  TRACE(5, "RootMPIOutput: writeRun: phr: data.size(): "
695  << I->second.data().size() << '\n';
696  if (I->second.data().size()) {
697  I->second.data().back().id().print(OS);
698  TRACE(5, "RootMPIOutput: writeRun: phr: data.back().id(): '"
699  << OS.str() << "'");
700  }
701  }
702  if (!rp.aux().processHistoryID().isValid()) {
703  TRACE(5, "RootMPIOutput: writeRun: ProcessHistoryID: 'INVALID'");
704  }
705  else {
706  std::ostringstream OS;
707  rp.aux().processHistoryID().print(OS);
708  TRACE(5, "RootMPIOutput: writeRun: ProcessHistoryID: '"
709  << OS.str() << "'");
710  OS.str("");
711  const ProcessHistory& processHistory =
712  ProcessHistoryRegistry::get(rp.aux().processHistoryID());
713  if (processHistory.data().size()) {
714  // FIXME: Print something special on invalid id() here!
715  processHistory.data().back().id().print(OS);
716  TRACE(5, "RootMPIOutput: writeRun: ProcessConfigurationID: '"
717  << OS.str() << "'");
718  OS.str("");
719  TRACE(5, "RootMPIOutput: writeRun: ProcessConfiguration: '"
720  << processHistory.data().back() << '\n';
721  }
722  }
723  }
724  msg.WriteObjectAny(&rp.aux(), run_aux_class);
725  TRACE(5, "RootMPIOutput: writeRun: streamed RunAuxiliary.");
726  }
727  //
728  // Write data products.
729  //
730  std::vector<BranchKey*> bkv;
731  writeDataProducts(msg, rp, bkv);
732  //
733  // Send message.
734  //
735  {
736  ServiceHandle<NetMonTransportService> transport;
737  TRACE(5, "RootMPIOutput: writeRun: sending a message ...");
738  transport->sendMessage(0, artdaq::Fragment::EndOfRunFragmentType, msg);
739  TRACE(5, "RootMPIOutput: writeRun: message sent.");
740  }
741  //
742  // Delete the branch keys we created for the message.
743  //
744  for (auto I = bkv.begin(), E = bkv.end(); I != E; ++I) {
745  delete *I;
746  *I = 0;
747  }
748 #endif // 0
749  TRACE(5, "RootMPIOutput: End: RootMPIOutput::writeRun(const RunPrincipal& rp)");
750 }
751 
752 void
753 art::RootMPIOutput::writeSubRun(CONST_WRITE SubRunPrincipal& srp)
754 {
755  //
756  // Write an EndSubRun message.
757  //
758  TRACE(5, "RootMPIOutput: Begin: RootMPIOutput::"
759  "writeSubRun(const SubRunPrincipal& srp)");
760  if (!initMsgSent_)
761  {
762  send_init_message();
763  initMsgSent_ = true;
764  }
765  //
766  // Fetch the class dictionaries we need for
767  // writing out the auxiliary information.
768  //
769  static TClass* subrun_aux_class = TClass::GetClass("art::SubRunAuxiliary");
770  if (subrun_aux_class == nullptr)
771  {
772  throw art::Exception(art::errors::DictionaryNotFound) <<
773  "RootMPIOutput::writeSubRun: "
774  "Could not get TClass for art::SubRunAuxiliary!";
775  }
776  //
777  // Begin preparing message.
778  //
779  TBufferFile msg(TBuffer::kWrite);
780  msg.SetWriteMode();
781  //
782  // Write message type code.
783  //
784  {
785  TRACE(5, "RootMPIOutput: RootMPIOutput::writeSubRun: "
786  "streaming message type code ...");
787  msg.WriteULong(3);
788  TRACE(5, "RootMPIOutput: RootMPIOutput::writeSubRun: "
789  "finished streaming message type code.");
790  }
791  //
792  // Write SubRunAuxiliary.
793  //
794  {
795  TRACE(5, "RootMPIOutput: RootMPIOutput::writeSubRun: "
796  "streaming SubRunAuxiliary ...");
797  if (art::debugit() >= 1)
798  {
799  TRACE(5, "RootMPIOutput: RootMPIOutput::writeSubRun: "
800  "dumping ProcessHistoryRegistry ...");
801  //typedef std::map<const ProcessHistoryID,ProcessHistory>
802  // ProcessHistoryMap;
803 # if ART_HEX_VERSION >= 0x20703
804  for (auto I = std::begin(art::ProcessHistoryRegistry::get())
805  , E = std::end(art::ProcessHistoryRegistry::get()); I != E; ++I)
806 # else
807  art::ProcessHistoryMap const& phr =
808  art::ProcessHistoryRegistry::get();
809  TRACE(5, "RootMPIOutput: RootMPIOutput::writeSubRun: "
810  "phr: size: " + std::to_string(phr.size()));
811  for (auto I = phr.begin(), E = phr.end(); I != E; ++I)
812 # endif
813  {
814  std::ostringstream OS;
815  I->first.print(OS);
816  TRACE(5, "RootMPIOutput: RootMPIOutput::writeSubRun: "
817  "phr: id: '" + OS.str() + "'");
818  OS.str("");
819  TRACE(5, "RootMPIOutput: RootMPIOutput::writeSubRun: "
820  "phr: data.size(): %zu",I->second.data().size() );
821  if (I->second.data().size())
822  {
823  I->second.data().back().id().print(OS);
824  TRACE(5, "RootMPIOutput: RootMPIOutput::writeSubRun: "
825  "phr: data.back().id(): '"
826  + OS.str() + "'");
827  }
828  }
829  if (!srp.aux().processHistoryID().isValid())
830  {
831  TRACE(5, "RootMPIOutput: RootMPIOutput::writeSubRun: "
832  "ProcessHistoryID: 'INVALID'");
833  }
834  else
835  {
836  std::ostringstream OS;
837  srp.aux().processHistoryID().print(OS);
838  TRACE(5, "RootMPIOutput: RootMPIOutput::writeSubRun: ProcessHistoryID: '"
839  + OS.str() + "'");
840  OS.str("");
841 # if ART_HEX_VERSION >= 0x20703
842  ProcessHistory processHistory;
843  ProcessHistoryRegistry::get(srp.aux().processHistoryID(),processHistory);
844 # else
845  const ProcessHistory& processHistory =
846  ProcessHistoryRegistry::get(srp.aux().processHistoryID());
847 # endif
848  if (processHistory.data().size())
849  {
850  // FIXME: Print something special on invalid id() here!
851  processHistory.data().back().id().print(OS);
852  TRACE(5, "RootMPIOutput: RootMPIOutput::writeSubRun: "
853  "ProcessConfigurationID: '"
854  + OS.str() + "'");
855  OS.str("");
856  OS << processHistory.data().back();
857  TRACE(5, "RootMPIOutput: RootMPIOutput::writeSubRun: "
858  "ProcessConfiguration: '"
859  + OS.str());
860  }
861  }
862  }
863  msg.WriteObjectAny(&srp.aux(), subrun_aux_class);
864  TRACE(5, "RootMPIOutput: RootMPIOutput::writeSubRun: streamed SubRunAuxiliary.");
865  }
866  //
867  // Write data products.
868  //
869  std::vector<BranchKey*> bkv;
870  writeDataProducts(msg, srp, bkv);
871  //
872  // Send message.
873  //
874  ServiceHandle<NetMonTransportService> transport;
875  TRACE(5, "RootMPIOutput: RootMPIOutput::writeSubRun: "
876  "Sending the EndOfSubrun message to "
877  + std::to_string(transport->dataReceiverCount())
878  + " data receivers ...");
879  for (size_t idx = 0; idx < transport->dataReceiverCount(); ++idx)
880  {
881  transport->sendMessage(idx, artdaq::Fragment::EndOfSubrunFragmentType, msg);
882  }
883  TRACE(5, "RootMPIOutput: RootMPIOutput::writeSubRun: "
884  "EndOfSubrun message(s) sent.");
885 
886  // Disconnecting will cause EOD fragments to be generated which will
887  // allow components downstream to flush data and clean up.
888  transport->disconnect();
889 
890  //
891  // Delete the branch keys we created for the message.
892  //
893  for (auto I = bkv.begin(), E = bkv.end(); I != E; ++I)
894  {
895  delete *I;
896  *I = 0;
897  }
898  TRACE(5, "RootMPIOutput: End: RootMPIOutput::writeSubRun(const SubRunPrincipal& srp)");
899 }
900 
901 DEFINE_ART_MODULE(art::RootMPIOutput)
RootMPIOutput(fhicl::ParameterSet const &ps)
RootMPIOutput Constructor.
~RootMPIOutput()
RootMPIOutput Destructor.
An art::OutputModule which sends events using DataSenderManager. This module is designed for transpor...