artdaq  v2_03_00
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Pages
RootMPIOutput_module.cc
1 #include "art/Framework/Core/ModuleMacros.h"
2 #include "art/Framework/Core/OutputModule.h"
3 #include "art/Framework/Principal/EventPrincipal.h"
4 #include "art/Framework/Principal/OutputHandle.h"
5 #include "art/Framework/Principal/RunPrincipal.h"
6 #include "art/Framework/Principal/SubRunPrincipal.h"
7 #include "art/Framework/Services/Registry/ServiceHandle.h"
8 #include "art/Persistency/Provenance/BranchIDListHelper.h"
9 #include "art/Persistency/Provenance/BranchIDListRegistry.h"
10 #include "art/Persistency/Provenance/ProcessHistoryRegistry.h"
11 #include "art/Persistency/Provenance/ProductMetaData.h"
12 
13 #include "canvas/Persistency/Provenance/BranchDescription.h"
14 #include "canvas/Persistency/Provenance/BranchIDList.h"
15 #include "canvas/Persistency/Provenance/BranchKey.h"
16 #include "canvas/Persistency/Provenance/History.h"
17 #include "canvas/Persistency/Provenance/ParentageRegistry.h"
18 #include "canvas/Persistency/Provenance/ProcessConfiguration.h"
19 #include "canvas/Persistency/Provenance/ProcessConfigurationID.h"
20 #include "canvas/Persistency/Provenance/ProcessHistoryID.h"
21 #include "canvas/Persistency/Provenance/ProductList.h"
22 #include "canvas/Persistency/Provenance/ProductProvenance.h"
23 #include "canvas/Persistency/Provenance/RunAuxiliary.h"
24 #include "canvas/Persistency/Provenance/SubRunAuxiliary.h"
25 #include "canvas/Utilities/DebugMacros.h"
26 #include "canvas/Utilities/Exception.h"
27 #include "cetlib/column_width.h"
28 #include "cetlib/lpad.h"
29 #include "cetlib/rpad.h"
30 #include <algorithm>
31 #include "fhiclcpp/ParameterSet.h"
32 #include "fhiclcpp/ParameterSetID.h"
33 #include "fhiclcpp/ParameterSetRegistry.h"
34 
35 #include "artdaq/DAQdata/Globals.hh"
36 #include "artdaq/ArtModules/NetMonTransportService.h"
37 #include "artdaq/DAQdata/NetMonHeader.hh"
38 
39 #include <iomanip>
40 #include <iostream>
41 #include <sstream>
42 #include <string>
43 #include <vector>
44 
45 #include "unistd.h"
46 
47 #include "TClass.h"
48 #include "TMessage.h"
49 
50 #define TRACE_NAME "RootMPIOutput"
51 
52 # define CONST_WRITE
53 
54 namespace art
55 {
56  class RootMPIOutput;
57 }
58 
64 class art::RootMPIOutput : public OutputModule
65 {
66 public:
74  explicit RootMPIOutput(fhicl::ParameterSet const& ps);
75 
80 
81 private:
82  virtual void openFile(FileBlock const&);
83 
84  virtual void closeFile();
85 
86  virtual void respondToCloseInputFile(FileBlock const&);
87 
88  virtual void respondToCloseOutputFiles(FileBlock const&);
89 
90  virtual void endJob();
91 
92  virtual void write(EventPrincipal CONST_WRITE&);
93 
94  virtual void writeRun(RunPrincipal CONST_WRITE&);
95 
96  virtual void writeSubRun(SubRunPrincipal CONST_WRITE&);
97 
98  void writeDataProducts(TBufferFile&, const Principal&,
99  std::vector<BranchKey*>&);
100 
101 private:
102  bool initMsgSent_;
103 };
104 
106 RootMPIOutput(fhicl::ParameterSet const& ps)
107  : OutputModule(ps)
108  , initMsgSent_(false)
109 {
110  TLOG_DEBUG("RootMPIOutput") << "Begin: RootMPIOutput::RootMPIOutput(ParameterSet const& ps)" << TLOG_ENDL;
111  ServiceHandle<NetMonTransportService> transport;
112  transport->connect();
113  TLOG_DEBUG("RootMPIOutput") << "End: RootMPIOutput::RootMPIOutput(ParameterSet const& ps)" << TLOG_ENDL;
114 }
115 
118 {
119  TLOG_DEBUG("RootMPIOutput") << "Begin: RootMPIOutput::~RootMPIOutput()" << TLOG_ENDL;
120  ServiceHandle<NetMonTransportService> transport;
121  transport->disconnect();
122  TLOG_DEBUG("RootMPIOutput") << "End: RootMPIOutput::~RootMPIOutput()" << TLOG_ENDL;
123 }
124 
125 void
126 art::RootMPIOutput::
127 openFile(FileBlock const&)
128 {
129  TRACE(5, "RootMPIOutput: Begin/End: RootMPIOutput::openFile(const FileBlock&)");
130 }
131 
132 void
133 art::RootMPIOutput::
134 closeFile()
135 {
136  TRACE(5, "RootMPIOutput: Begin/End: RootMPIOutput::closeFile()");
137 }
138 
139 void
140 art::RootMPIOutput::
141 respondToCloseInputFile(FileBlock const&)
142 {
143  TRACE(5, "RootMPIOutput: Begin/End: RootMPIOutput::"
144  "respondToCloseOutputFiles(FileBlock const&)");
145 }
146 
147 void
148 art::RootMPIOutput::
149 respondToCloseOutputFiles(FileBlock const&)
150 {
151  TRACE(5, "RootMPIOutput: Begin/End: RootMPIOutput::"
152  "respondToCloseOutputFiles(FileBlock const&)");
153 }
154 
155 static
156 void
157 send_shutdown_message()
158 {
159  // 4/1/2016, ELF: Apparently, all this function does is make things not work.
160  // At this point in the state machine, RHandles and SHandles have already been
161  // destructed. Calling sendMessage will cause SHandles to reconnect itself,
162  // but the other end will never recieve the message.
163 #if 0
164  TRACE(5, "RootMPIOutput: Begin: RootMPIOutput static send_shutdown_message()");
165  //
166  // Construct and send the shutdown message.
167  //
168  TBufferFile msg(TBuffer::kWrite);
169  msg.SetWriteMode();
170  //
171  // Stream the message type code.
172  //
173  {
174  TRACE(5, "RootMPIOutput: RootMPIOutput static send_shutdown_message: "
175  "streaming shutdown message type code ...");
176  msg.WriteULong(5);
177  TRACE(5, "RootMPIOutput: RootMPIOutput static send_shutdown_message: "
178  "finished streaming shutdown message type code.");
179  }
180  //
181  //
182  // Send the shutdown message.
183  //
184  {
185  art::ServiceHandle<NetMonTransportService> transport;
186  TRACE(5, "RootMPIOutput: RootMPIOutput static send_shutdown_message: "
187  "sending the shutdown message ...");
188  transport->sendMessage(0, artdaq::Fragment::ShutdownFragmentType, msg);
189  TRACE(5, "RootMPIOutput: RootMPIOutput static send_shutdown_message: sent the shutdown message.");
190  }
191  TRACE(5, "RootMPIOutput: End: RootMPIOutput static send_shutdown_message()");
192 #endif
193 }
194 
195 void
196 art::RootMPIOutput::
197 endJob()
198 {
199  TRACE(5, "RootMPIOutput: Begin: RootMPIOutput::endJob()");
200  send_shutdown_message();
201  TRACE(5, "RootMPIOutput: End: RootMPIOutput::endJob()");
202 }
203 
204 //#pragma GCC push_options
205 //#pragma GCC optimize ("O0")
206 static
207 void
208 send_init_message()
209 {
210  TRACE(5, "RootMPIOutput: Begin: RootMPIOutput static send_init_message()");
211  //
212  // Get the classes we will need.
213  //
214  //static TClass* string_class = TClass::GetClass("std::string");
215  //if (string_class == nullptr) {
216  // throw art::Exception(art::errors::DictionaryNotFound) <<
217  // "RootMPIOutput static send_init_message(): "
218  // "Could not get TClass for std::string!";
219  //}
220  static TClass* product_list_class = TClass::GetClass(
221  "std::map<art::BranchKey,art::BranchDescription>");
222  if (product_list_class == nullptr)
223  {
224  throw art::Exception(art::errors::DictionaryNotFound) <<
225  "RootMPIOutput static send_init_message(): "
226  "Could not get TClass for "
227  "map<art::BranchKey,art::BranchDescription>!";
228  }
229  //typedef std::map<const ProcessHistoryID,ProcessHistory> ProcessHistoryMap;
230  //TClass* process_history_map_class = TClass::GetClass(
231  // "std::map<const art::ProcessHistoryID,art::ProcessHistory>");
232  //FIXME: Replace the "2" here with a use of the proper enum value!
233  static TClass* process_history_map_class = TClass::GetClass(
234  "std::map<const art::Hash<2>,art::ProcessHistory>");
235  if (process_history_map_class == nullptr)
236  {
237  throw art::Exception(art::errors::DictionaryNotFound) <<
238  "RootMPIOutput static send_init_message(): "
239  "Could not get class for "
240  "std::map<const art::Hash<2>,art::ProcessHistory>!";
241  }
242  //static TClass* parentage_map_class = TClass::GetClass(
243  // "std::map<const art::ParentageID,art::Parentage>");
244  //FIXME: Replace the "5" here with a use of the proper enum value!
245  static TClass* parentage_map_class = TClass::GetClass(
246  "std::map<const art::Hash<5>,art::Parentage>");
247  if (parentage_map_class == nullptr)
248  {
249  throw art::Exception(art::errors::DictionaryNotFound) <<
250  "RootMPIOutput static send_init_message(): "
251  "Could not get class for "
252  "std::map<const art::Hash<5>,art::Parentage>!";
253  }
254  //
255  // Construct and send the init message.
256  //
257  TBufferFile msg(TBuffer::kWrite);
258  msg.SetWriteMode();
259  //
260  // Stream the message type code.
261  //
262  {
263  TRACE(5, "RootMPIOutput: RootMPIOutput static send_init_message(): "
264  "Streaming message type code ...");
265  msg.WriteULong(1);
266  TRACE(5, "RootMPIOutput: RootMPIOutput static send_init_message(): "
267  "Finished streaming message type code.");
268  }
269  //
270  // Stream the ParameterSetRegistry.
271  //
272  {
273  unsigned long ps_cnt = fhicl::ParameterSetRegistry::size();
274  TRACE(5, "RootMPIOutput: RootMPIOutput static send_init_message(): parameter set count: " + std::to_string(ps_cnt));
275  msg.WriteULong(ps_cnt);
276  TRACE(5, "RootMPIOutput: RootMPIOutput static send_init_message(): Streaming parameter sets ...");
277  for (auto I = fhicl::ParameterSetRegistry::begin(),
278  E = fhicl::ParameterSetRegistry::end(); I != E; ++I)
279  {
280  std::string pset_str = I->second.to_string();
281  //msg.WriteObjectAny(&pset_str, string_class);
282  msg.WriteStdString(pset_str);
283  }
284  TRACE(5, "RootMPIOutput: RootMPIOutput static send_init_message(): Finished streaming parameter sets.");
285  }
286  //
287  // Stream the MasterProductRegistry.
288  //
289  {
290  TRACE(5, "RootMPIOutput: RootMPIOutput static send_init_message(): Streaming MasterProductRegistry ...");
291  art::ProductList productList(
292  art::ProductMetaData::instance().productList());
293  msg.WriteObjectAny(&productList, product_list_class);
294  TRACE(5, "RootMPIOutput: RootMPIOutput static send_init_message(): Finished streaming MasterProductRegistry.");
295  }
296  //
297  // Dump The BranchIDListRegistry
298  //
299  if (art::debugit() >= 2)
300  {
301  //typedef vector<BranchID::value_type> BranchIDList
302  //typedef vector<BranchIDList> BranchIDLists
303  //std::vector<std::vector<art::BranchID::value_type>>
304  art::BranchIDLists* bilr =
305  &art::BranchIDListRegistry::instance()->data();
306  FDEBUG(2) << "RootMPIOutput static send_init_message(): "
307  "Content of BranchIDLists\n";
308  int max_bli = bilr->size();
309  FDEBUG(2) << "RootMPIOutput static send_init_message(): "
310  "max_bli: " << max_bli << '\n';
311  for (int i = 0; i < max_bli; ++i)
312  {
313  int max_prdidx = (*bilr)[i].size();
314  FDEBUG(2) << "RootMPIOutput static send_init_message(): "
315  "max_prdidx: " << max_prdidx << '\n';
316  for (int j = 0; j < max_prdidx; ++j)
317  {
318  FDEBUG(2) << "RootMPIOutput static send_init_message(): "
319  "bli: " << i
320  << " prdidx: " << j
321  << " bid: 0x" << std::hex
322  << static_cast<unsigned long>((*bilr)[i][j])
323  << std::dec << '\n';
324  }
325  }
326  }
327  //
328  // Dump the ProcessHistoryRegistry.
329  //
330  if (art::debugit() >= 1)
331  {
332  TRACE(5, "RootMPIOutput: RootMPIOutput static send_init_message(): "
333  "Dumping ProcessHistoryRegistry ...");
334  //typedef std::map<const ProcessHistoryID,ProcessHistory>
335  // ProcessHistoryMap;
336  art::ProcessHistoryMap const& phr = art::ProcessHistoryRegistry::get();
337  TRACE(5, "RootMPIOutput: RootMPIOutput static send_init_message(): "
338  "phr: size: " + std::to_string(phr.size()));
339  for (auto I = phr.begin(), E = phr.end(); I != E; ++I)
340  {
341  std::ostringstream OS;
342  I->first.print(OS);
343  TRACE(5, "RootMPIOutput: RootMPIOutput static send_init_message(): "
344  "phr: id: '" + OS.str() + "'");
345  }
346  }
347  //
348  // Stream the ProcessHistoryRegistry.
349  //
350  {
351  TRACE(5, "RootMPIOutput: RootMPIOutput static send_init_message(): "
352  "Streaming ProcessHistoryRegistry ...");
353  //typedef std::map<const ProcessHistoryID,ProcessHistory>
354  // ProcessHistoryMap;
355  const art::ProcessHistoryMap& phm = art::ProcessHistoryRegistry::get();
356  TRACE(5, "RootMPIOutput: RootMPIOutput static send_init_message(): "
357  "phm: size: " + std::to_string(phm.size()));
358  msg.WriteObjectAny(&phm, process_history_map_class);
359  TRACE(5, "RootMPIOutput: RootMPIOutput static send_init_message(): "
360  "Finished streaming ProcessHistoryRegistry.");
361  }
362  //
363  // Stream the ParentageRegistry.
364  //
365  {
366  TRACE(5, "RootMPIOutput: RootMPIOutput static send_init_message(): "
367  "Streaming ParentageRegistry ...");
368  //typedef std::map<const ParentageID,Parentage> ParentageMap
369  const art::ParentageMap& parentageMap = art::ParentageRegistry::get();
370  msg.WriteObjectAny(&parentageMap, parentage_map_class);
371  TRACE(5, "RootMPIOutput: RootMPIOutput static send_init_message(): "
372  "Finished streaming ParentageRegistry.");
373  }
374  //
375  //
376  // Send init message.
377  //
378  {
379  art::ServiceHandle<NetMonTransportService> transport;
380  TRACE(5, "RootMPIOutput: RootMPIOutput static send_init_message(): "
381  "Sending the init message to "
382  + std::to_string(transport->dataReceiverCount()) +
383  " data receivers ...");
384  for (size_t idx = 0; idx < transport->dataReceiverCount(); ++idx)
385  {
386  transport->sendMessage(idx, artdaq::Fragment::InitFragmentType, msg);
387  }
388  TRACE(5, "RootMPIOutput: RootMPIOutput static send_init_message(): "
389  "Init message(s) sent.");
390  }
391  TRACE(5, "RootMPIOutput: End: RootMPIOutput static send_init_message()");
392 }
393 
394 //#pragma GCC pop_options
395 
396 void
397 art::RootMPIOutput::
398 writeDataProducts(TBufferFile& msg, const Principal& principal,
399  std::vector<BranchKey*>& bkv)
400 {
401  TRACE(5, "RootMPIOutput: Begin: RootMPIOutput::writeDataProducts(...)");
402  //
403  // Fetch the class dictionaries we need for
404  // writing out the data products.
405  //
406  static TClass* branch_key_class = TClass::GetClass("art::BranchKey");
407  if (branch_key_class == nullptr)
408  {
409  throw art::Exception(art::errors::DictionaryNotFound) <<
410  "RootMPIOutput::writeDataProducts(...): "
411  "Could not get TClass for art::BranchKey!";
412  }
413  static TClass* prdprov_class = TClass::GetClass("art::ProductProvenance");
414  if (prdprov_class == nullptr)
415  {
416  throw art::Exception(art::errors::DictionaryNotFound) <<
417  "RootMPIOutput::writeDataProducts(...): "
418  "Could not get TClass for art::ProductProvenance!";
419  }
420  //
421  // Calculate the data product count.
422  //
423  unsigned long prd_cnt = 0;
424  //std::map<art::BranchID, std::shared_ptr<art::Group>>::const_iterator
425  for (auto I = principal.begin(), E = principal.end(); I != E; ++I)
426  {
427  if (I->second->productUnavailable() || !selected(I->second->productDescription()))
428  {
429  continue;
430  }
431  ++prd_cnt;
432  }
433  //
434  // Write the data product count.
435  //
436  {
437  TRACE(5, "RootMPIOutput: RootMPIOutput::writeDataProducts(...): "
438  "Streaming product count: " + std::to_string(prd_cnt));
439  msg.WriteULong(prd_cnt);
440  TRACE(5, "RootMPIOutput: RootMPIOutput::writeDataProducts(...): "
441  "Finished streaming product count.");
442  }
443  //
444  // Loop over the groups in the RunPrincipal and
445  // write out the data products.
446  //
447  // Note: We need this vector of keys because the ROOT I/O mechanism
448  // requires that each object inserted in the message has a
449  // unique address, so we force that by holding on to each
450  // branch key manufactured in the loop until after we are
451  // done constructing the message.
452  //
453  bkv.reserve(prd_cnt);
454  //std::map<art::BranchID, std::shared_ptr<art::Group>>::const_iterator
455  for (auto I = principal.begin(), E = principal.end(); I != E; ++I)
456  {
457  if (I->second->productUnavailable() || !selected(I->second->productDescription()))
458  {
459  continue;
460  }
461  const BranchDescription& bd(I->second->productDescription());
462  bkv.push_back(new BranchKey(bd));
463  if (art::debugit() >= 2)
464  {
465  FDEBUG(2) << "RootMPIOutput::writeDataProducts(...): "
466  "Dumping branch key of class: '"
467  << bkv.back()->friendlyClassName_
468  << "' modlbl: '"
469  << bkv.back()->moduleLabel_
470  << "' instnm: '"
471  << bkv.back()->productInstanceName_
472  << "' procnm: '"
473  << bkv.back()->processName_
474  << "'";
475  }
476  {
477  TRACE(5, "RootMPIOutput: RootMPIOutput::writeDataProducts(...): "
478  "Streaming branch key of class: '"
479  + bd.producedClassName()
480  + "' modlbl: '"
481  + bd.moduleLabel()
482  + "' instnm: '"
483  + bd.productInstanceName()
484  + "' procnm: '"
485  + bd.processName()
486  + "'");
487  msg.WriteObjectAny(bkv.back(), branch_key_class);
488  }
489  {
490  TRACE(5, "RootMPIOutput: RootMPIOutput::writeDataProducts(...): "
491  "Streaming product of class: '"
492  + bd.producedClassName()
493  + "' modlbl: '"
494  + bd.moduleLabel()
495  + "' instnm: '"
496  + bd.productInstanceName()
497  + "' procnm: '"
498  + bd.processName()
499  + "'");
500  OutputHandle oh = principal.getForOutput(bd.branchID(), true);
501  const EDProduct* prd = oh.wrapper();
502  msg.WriteObjectAny(prd, TClass::GetClass(bd.wrappedName().c_str()));
503  }
504  {
505  TRACE(5, "RootMPIOutput: RootMPIOutput::writeDataProducts(...): "
506  "Streaming product provenance of class: '"
507  + bd.producedClassName()
508  + "' modlbl: '"
509  + bd.moduleLabel()
510  + "' instnm: '"
511  + bd.productInstanceName()
512  + "' procnm: '"
513  + bd.processName()
514  + "'");
515  const ProductProvenance* prdprov =
516  I->second->productProvenancePtr().get();
517  msg.WriteObjectAny(prdprov, prdprov_class);
518  }
519  }
520  TRACE(5, "RootMPIOutput: End: RootMPIOutput::writeDataProducts(...)");
521 }
522 
523 void
524 art::RootMPIOutput::
525 write(CONST_WRITE EventPrincipal& ep)
526 {
527  //
528  // Write an Event message.
529  //
530  TRACE(5, "RootMPIOutput: Begin: RootMPIOutput::"
531  "write(const EventPrincipal& ep)");
532  if (!initMsgSent_)
533  {
534  send_init_message();
535  initMsgSent_ = true;
536  }
537  //
538  // Get root classes needed for I/O.
539  //
540  static TClass* run_aux_class = TClass::GetClass("art::RunAuxiliary");
541  if (run_aux_class == nullptr)
542  {
543  throw art::Exception(art::errors::DictionaryNotFound) <<
544  "RootMPIOutput::write(const EventPrincipal& ep): "
545  "Could not get TClass for art::RunAuxiliary!";
546  }
547  static TClass* subrun_aux_class = TClass::GetClass("art::SubRunAuxiliary");
548  if (subrun_aux_class == nullptr)
549  {
550  throw art::Exception(art::errors::DictionaryNotFound) <<
551  "RootMPIOutput::write(const EventPrincipal& ep): "
552  "Could not get TClass for art::SubRunAuxiliary!";
553  }
554  static TClass* event_aux_class = TClass::GetClass("art::EventAuxiliary");
555  if (event_aux_class == nullptr)
556  {
557  throw art::Exception(art::errors::DictionaryNotFound) <<
558  "RootMPIOutput::write(const EventPrincipal& ep): "
559  "Could not get TClass for art::EventAuxiliary!";
560  }
561  static TClass* history_class = TClass::GetClass("art::History");
562  if (history_class == nullptr)
563  {
564  throw art::Exception(art::errors::DictionaryNotFound) <<
565  "RootMPIOutput::write(const EventPrincipal& ep): "
566  "Could not get TClass for art::History!";
567  }
568  //
569  // Setup message buffer.
570  //
571  TBufferFile msg(TBuffer::kWrite);
572  msg.SetWriteMode();
573  //
574  // Write message type code.
575  //
576  {
577  TRACE(5, "RootMPIOutput: RootMPIOutput::write(const EventPrincipal& ep): "
578  "Streaming message type code ...");
579  msg.WriteULong(4);
580  TRACE(5, "RootMPIOutput: RootMPIOutput::write(const EventPrincipal& ep): "
581  "Finished streaming message type code.");
582  }
583  //
584  // Write RunAuxiliary.
585  //
586  {
587  TRACE(5, "RootMPIOutput: RootMPIOutput::write(const EventPrincipal& ep): "
588  "Streaming RunAuxiliary ...");
589  msg.WriteObjectAny(&ep.subRunPrincipal().runPrincipal().aux(),
590  run_aux_class);
591  TRACE(5, "RootMPIOutput: RootMPIOutput::write(const EventPrincipal& ep): "
592  "Finished streaming RunAuxiliary.");
593  }
594  //
595  // Write SubRunAuxiliary.
596  //
597  {
598  TRACE(5, "RootMPIOutput: RootMPIOutput::write(const EventPrincipal& ep): "
599  "Streaming SubRunAuxiliary ...");
600  msg.WriteObjectAny(&ep.subRunPrincipal().aux(),
601  subrun_aux_class);
602  TRACE(5, "RootMPIOutput: RootMPIOutput::write(const EventPrincipal& ep): "
603  "Finished streaming SubRunAuxiliary.");
604  }
605  //
606  // Write EventAuxiliary.
607  //
608  {
609  TRACE(5, "RootMPIOutput: RootMPIOutput::write(const EventPrincipal& ep): "
610  "Streaming EventAuxiliary ...");
611  msg.WriteObjectAny(&ep.aux(), event_aux_class);
612  TRACE(5, "RootMPIOutput: RootMPIOutput::write(const EventPrincipal& ep): "
613  "Finished streaming EventAuxiliary.");
614  }
615  //
616  // Write History.
617  //
618  {
619  TRACE(5, "RootMPIOutput: RootMPIOutput::write(const EventPrincipal& ep): "
620  "Streaming History ...");
621  msg.WriteObjectAny(&ep.history(), history_class);
622  TRACE(5, "RootMPIOutput: RootMPIOutput::write(const EventPrincipal& ep): "
623  "Finished streaming History.");
624  }
625  //
626  // Write data products.
627  //
628  std::vector<BranchKey*> bkv;
629  writeDataProducts(msg, ep, bkv);
630  //
631  // Send message.
632  //
633  {
634  ServiceHandle<NetMonTransportService> transport;
635  TRACE(5, "RootMPIOutput: RootMPIOutput::write(const EventPrincipal& ep): "
636  "Sending a message ...");
637  transport->sendMessage(ep.id().event(), artdaq::Fragment::DataFragmentType, msg);
638  TRACE(5, "RootMPIOutput: RootMPIOutput::write(const EventPrincipal& ep): "
639  "Message sent.");
640  }
641  //
642  // Delete the branch keys we created for the message.
643  //
644  for (auto I = bkv.begin(), E = bkv.end(); I != E; ++I)
645  {
646  delete *I;
647  *I = 0;
648  }
649  TRACE(5, "RootMPIOutput: End: RootMPIOutput::write(const EventPrincipal& ep)");
650 }
651 
652 void
653 art::RootMPIOutput::
654 writeRun(CONST_WRITE RunPrincipal& rp)
655 {
656  //
657  // Write an EndRun message.
658  //
659  TRACE(5, "RootMPIOutput: Begin: RootMPIOutput::writeRun(const RunPrincipal& rp)");
660  (void)rp;
661  if (!initMsgSent_)
662  {
663  send_init_message();
664  initMsgSent_ = true;
665  }
666 #if 0
667  //
668  // Fetch the class dictionaries we need for
669  // writing out the auxiliary information.
670  //
671  static TClass* run_aux_class = TClass::GetClass("art::RunAuxiliary");
672  assert(run_aux_class != nullptr && "writeRun: Could not get TClass for "
673  "art::RunAuxiliary!");
674  //
675  // Begin preparing message.
676  //
677  TBufferFile msg(TBuffer::kWrite);
678  msg.SetWriteMode();
679  //
680  // Write message type code.
681  //
682  {
683  TRACE(5, "RootMPIOutput: writeRun: streaming message type code ...");
684  msg.WriteULong(2);
685  TRACE(5, "RootMPIOutput: writeRun: finished streaming message type code.");
686  }
687  //
688  // Write RunAuxiliary.
689  //
690  {
691  TRACE(5, "RootMPIOutput: writeRun: streaming RunAuxiliary ...");
692  if (art::debugit() >= 1) {
693  TRACE(5, "RootMPIOutput: writeRun: dumping ProcessHistoryRegistry ...");
694  //typedef std::map<const ProcessHistoryID,ProcessHistory>
695  // ProcessHistoryMap;
696  art::ProcessHistoryMap const& phr =
697  art::ProcessHistoryRegistry::get();
698  TRACE(5, "RootMPIOutput: writeRun: phr: size: " << phr.size() << '\n';
699  for (auto I = phr.begin(), E = phr.end(); I != E; ++I) {
700  std::ostringstream OS;
701  I->first.print(OS);
702  TRACE(5, "RootMPIOutput: writeRun: phr: id: '" << OS.str() << "'");
703  OS.str("");
704  TRACE(5, "RootMPIOutput: writeRun: phr: data.size(): "
705  << I->second.data().size() << '\n';
706  if (I->second.data().size()) {
707  I->second.data().back().id().print(OS);
708  TRACE(5, "RootMPIOutput: writeRun: phr: data.back().id(): '"
709  << OS.str() << "'");
710  }
711  }
712  if (!rp.aux().processHistoryID().isValid()) {
713  TRACE(5, "RootMPIOutput: writeRun: ProcessHistoryID: 'INVALID'");
714  }
715  else {
716  std::ostringstream OS;
717  rp.aux().processHistoryID().print(OS);
718  TRACE(5, "RootMPIOutput: writeRun: ProcessHistoryID: '"
719  << OS.str() << "'");
720  OS.str("");
721  const ProcessHistory& processHistory =
722  ProcessHistoryRegistry::get(rp.aux().processHistoryID());
723  if (processHistory.data().size()) {
724  // FIXME: Print something special on invalid id() here!
725  processHistory.data().back().id().print(OS);
726  TRACE(5, "RootMPIOutput: writeRun: ProcessConfigurationID: '"
727  << OS.str() << "'");
728  OS.str("");
729  TRACE(5, "RootMPIOutput: writeRun: ProcessConfiguration: '"
730  << processHistory.data().back() << '\n';
731  }
732  }
733  }
734  msg.WriteObjectAny(&rp.aux(), run_aux_class);
735  TRACE(5, "RootMPIOutput: writeRun: streamed RunAuxiliary.");
736  }
737  //
738  // Write data products.
739  //
740  std::vector<BranchKey*> bkv;
741  writeDataProducts(msg, rp, bkv);
742  //
743  // Send message.
744  //
745  {
746  ServiceHandle<NetMonTransportService> transport;
747  TRACE(5, "RootMPIOutput: writeRun: sending a message ...");
748  transport->sendMessage(0, artdaq::Fragment::EndOfRunFragmentType, msg);
749  TRACE(5, "RootMPIOutput: writeRun: message sent.");
750  }
751  //
752  // Delete the branch keys we created for the message.
753  //
754  for (auto I = bkv.begin(), E = bkv.end(); I != E; ++I) {
755  delete *I;
756  *I = 0;
757  }
758 #endif // 0
759  TRACE(5, "RootMPIOutput: End: RootMPIOutput::writeRun(const RunPrincipal& rp)");
760 }
761 
762 void
763 art::RootMPIOutput::writeSubRun(CONST_WRITE SubRunPrincipal& srp)
764 {
765  //
766  // Write an EndSubRun message.
767  //
768  TRACE(5, "RootMPIOutput: Begin: RootMPIOutput::"
769  "writeSubRun(const SubRunPrincipal& srp)");
770  if (!initMsgSent_)
771  {
772  send_init_message();
773  initMsgSent_ = true;
774  }
775  //
776  // Fetch the class dictionaries we need for
777  // writing out the auxiliary information.
778  //
779  static TClass* subrun_aux_class = TClass::GetClass("art::SubRunAuxiliary");
780  if (subrun_aux_class == nullptr)
781  {
782  throw art::Exception(art::errors::DictionaryNotFound) <<
783  "RootMPIOutput::writeSubRun: "
784  "Could not get TClass for art::SubRunAuxiliary!";
785  }
786  //
787  // Begin preparing message.
788  //
789  TBufferFile msg(TBuffer::kWrite);
790  msg.SetWriteMode();
791  //
792  // Write message type code.
793  //
794  {
795  TRACE(5, "RootMPIOutput: RootMPIOutput::writeSubRun: "
796  "streaming message type code ...");
797  msg.WriteULong(3);
798  TRACE(5, "RootMPIOutput: RootMPIOutput::writeSubRun: "
799  "finished streaming message type code.");
800  }
801  //
802  // Write SubRunAuxiliary.
803  //
804  {
805  TRACE(5, "RootMPIOutput: RootMPIOutput::writeSubRun: "
806  "streaming SubRunAuxiliary ...");
807  if (art::debugit() >= 1)
808  {
809  TRACE(5, "RootMPIOutput: RootMPIOutput::writeSubRun: "
810  "dumping ProcessHistoryRegistry ...");
811  //typedef std::map<const ProcessHistoryID,ProcessHistory>
812  // ProcessHistoryMap;
813  art::ProcessHistoryMap const& phr =
814  art::ProcessHistoryRegistry::get();
815  TRACE(5, "RootMPIOutput: RootMPIOutput::writeSubRun: "
816  "phr: size: " + std::to_string(phr.size()));
817  for (auto I = phr.begin(), E = phr.end(); I != E; ++I)
818  {
819  std::ostringstream OS;
820  I->first.print(OS);
821  TRACE(5, "RootMPIOutput: RootMPIOutput::writeSubRun: "
822  "phr: id: '" + OS.str() + "'");
823  OS.str("");
824  TRACE(5, "RootMPIOutput: RootMPIOutput::writeSubRun: "
825  "phr: data.size(): "
826  + std::to_string(I->second.data().size()));
827  if (I->second.data().size())
828  {
829  I->second.data().back().id().print(OS);
830  TRACE(5, "RootMPIOutput: RootMPIOutput::writeSubRun: "
831  "phr: data.back().id(): '"
832  + OS.str() + "'");
833  }
834  }
835  if (!srp.aux().processHistoryID().isValid())
836  {
837  TRACE(5, "RootMPIOutput: RootMPIOutput::writeSubRun: "
838  "ProcessHistoryID: 'INVALID'");
839  }
840  else
841  {
842  std::ostringstream OS;
843  srp.aux().processHistoryID().print(OS);
844  TRACE(5, "RootMPIOutput: RootMPIOutput::writeSubRun: ProcessHistoryID: '"
845  + OS.str() + "'");
846  OS.str("");
847  const ProcessHistory& processHistory =
848  ProcessHistoryRegistry::get(srp.aux().processHistoryID());
849  if (processHistory.data().size())
850  {
851  // FIXME: Print something special on invalid id() here!
852  processHistory.data().back().id().print(OS);
853  TRACE(5, "RootMPIOutput: RootMPIOutput::writeSubRun: "
854  "ProcessConfigurationID: '"
855  + OS.str() + "'");
856  OS.str("");
857  OS << processHistory.data().back();
858  TRACE(5, "RootMPIOutput: RootMPIOutput::writeSubRun: "
859  "ProcessConfiguration: '"
860  + OS.str());
861  }
862  }
863  }
864  msg.WriteObjectAny(&srp.aux(), subrun_aux_class);
865  TRACE(5, "RootMPIOutput: RootMPIOutput::writeSubRun: streamed SubRunAuxiliary.");
866  }
867  //
868  // Write data products.
869  //
870  std::vector<BranchKey*> bkv;
871  writeDataProducts(msg, srp, bkv);
872  //
873  // Send message.
874  //
875  {
876  ServiceHandle<NetMonTransportService> transport;
877  TRACE(5, "RootMPIOutput: RootMPIOutput::writeSubRun: "
878  "Sending the EndOfSubrun message to "
879  + std::to_string(transport->dataReceiverCount())
880  + " data receivers ...");
881  for (size_t idx = 0; idx < transport->dataReceiverCount(); ++idx)
882  {
883  transport->sendMessage(idx, artdaq::Fragment::EndOfSubrunFragmentType, msg);
884  }
885  TRACE(5, "RootMPIOutput: RootMPIOutput::writeSubRun: "
886  "EndOfSubrun message(s) sent.");
887 
888  // Disconnecting will cause EOD fragments to be generated which will
889  // allow components downstream to flush data and clean up.
890  transport->disconnect();
891  }
892  //
893  // Delete the branch keys we created for the message.
894  //
895  for (auto I = bkv.begin(), E = bkv.end(); I != E; ++I)
896  {
897  delete *I;
898  *I = 0;
899  }
900  TRACE(5, "RootMPIOutput: End: RootMPIOutput::writeSubRun(const SubRunPrincipal& srp)");
901 }
902 
903 DEFINE_ART_MODULE(art::RootMPIOutput)
RootMPIOutput(fhicl::ParameterSet const &ps)
RootMPIOutput Constructor.
~RootMPIOutput()
RootMPIOutput Destructor.
An art::OutputModule which sends events using DataSenderManager. This module is designed for transpor...