2 #include "otsdaq/ARTDAQSupervisor/ARTDAQSupervisor.hh"
4 #include "artdaq-core/Utilities/configureMessageFacility.hh"
5 #include "artdaq/BuildInfo/GetPackageBuildInfo.hh"
6 #include "artdaq/DAQdata/Globals.hh"
7 #include "cetlib_except/exception.h"
8 #include "fhiclcpp/make_ParameterSet.h"
15 #include <boost/exception/all.hpp>
16 #include <boost/filesystem.hpp>
24 #define ARTDAQ_FCL_PATH std::string(__ENV__("USER_DATA")) + "/" + "ARTDAQConfigurations/"
25 #define FAKE_CONFIG_NAME "ots_config"
26 #define DAQINTERFACE_PORT \
27 std::atoi(__ENV__("ARTDAQ_BASE_PORT")) + \
28 (partition_ * std::atoi(__ENV__("ARTDAQ_PORTS_PER_PARTITION")))
31 static std::unordered_map<int, struct sigaction> old_actions =
32 std::unordered_map<int, struct sigaction>();
33 static bool sighandler_init =
false;
34 static void signal_handler(
int signum)
37 TRACE_STREAMER(TLVL_ERROR, &(
"ARTDAQsupervisor")[0], 0, 0, 0)
38 <<
"A signal of type " << signum
39 <<
" was caught by ARTDAQSupervisor. Shutting down DAQInterface, "
40 "then proceeding with default handlers!";
46 pthread_sigmask(SIG_UNBLOCK, NULL, &set);
47 pthread_sigmask(SIG_UNBLOCK, &set, NULL);
49 TRACE_STREAMER(TLVL_ERROR, &(
"SharedMemoryManager")[0], 0, 0, 0)
50 <<
"Calling default signal handler";
53 sigaction(signum, &old_actions[signum], NULL);
54 kill(getpid(), signum);
60 sigaction(SIGINT, &old_actions[SIGINT], NULL);
61 kill(getpid(), SIGINT);
67 static std::mutex sighandler_mutex;
68 std::unique_lock<std::mutex> lk(sighandler_mutex);
72 sighandler_init =
true;
74 std::vector<int> signals = {
85 for(
auto signal : signals)
87 struct sigaction old_action;
88 sigaction(signal, NULL, &old_action);
92 if(old_action.sa_handler != SIG_IGN)
94 struct sigaction action;
95 action.sa_handler = signal_handler;
96 sigemptyset(&action.sa_mask);
97 for(
auto sigblk : signals)
99 sigaddset(&action.sa_mask, sigblk);
105 sigaction(signal, &action, NULL);
106 old_actions[signal] = old_action;
113 ARTDAQSupervisor::ARTDAQSupervisor(xdaq::ApplicationStub* stub)
115 , daqinterface_ptr_(NULL)
116 , partition_(getSupervisorProperty(
"partition", 0))
117 , daqinterface_state_(
"notrunning")
118 , runner_thread_(nullptr)
120 __SUP_COUT__ <<
"Constructor." << __E__;
122 INIT_MF(
"ARTDAQSupervisor");
123 init_sighandler(
this);
126 auto settings_file = __ENV__(
"DAQINTERFACE_SETTINGS");
127 std::ofstream o(settings_file, std::ios::trunc);
129 o <<
"log_directory: "
130 << getSupervisorProperty(
"log_directory", std::string(__ENV__(
"OTSDAQ_LOG_DIR")))
132 o <<
"record_directory: "
133 << getSupervisorProperty(
"record_directory", ARTDAQ_FCL_PATH) << std::endl;
134 o <<
"package_hashes_to_save: "
135 << getSupervisorProperty(
"package_hashes_to_save",
"[artdaq]") << std::endl;
137 o <<
"productsdir_for_bash_scripts: "
138 << getSupervisorProperty(
"productsdir_for_bash_scripts",
139 std::string(__ENV__(
"OTS_PRODUCTS")))
141 o <<
"boardreader timeout: " << getSupervisorProperty(
"boardreader_timeout", 30)
143 o <<
"eventbuilder timeout: " << getSupervisorProperty(
"eventbuilder_timeout", 30)
145 o <<
"datalogger timeout: " << getSupervisorProperty(
"datalogger_timeout", 30)
147 o <<
"dispatcher timeout: " << getSupervisorProperty(
"dispatcher_timeout", 30)
149 o <<
"max_fragment_size_bytes: "
150 << getSupervisorProperty(
"max_fragment_size_bytes", 1048576) << std::endl;
151 o <<
"transfer_plugin_to_use: "
152 << getSupervisorProperty(
"transfer_plugin_to_use",
"Autodetect") << std::endl;
153 o <<
"all_events_to_all_dispatchers: " << std::boolalpha
154 << getSupervisorProperty(
"all_events_to_all_dispatchers",
true) << std::endl;
155 o <<
"data_directory_override: "
156 << getSupervisorProperty(
"data_directory_override",
157 std::string(__ENV__(
"ARTDAQ_OUTPUT_DIR")))
159 o <<
"max_configurations_to_list: "
160 << getSupervisorProperty(
"max_configurations_to_list", 10) << std::endl;
161 o <<
"disable_unique_rootfile_labels: "
162 << getSupervisorProperty(
"disable_unique_rootfile_labels",
false) << std::endl;
163 o <<
"use_messageviewer: " << std::boolalpha
164 << getSupervisorProperty(
"use_messageviewer",
false) << std::endl;
165 o <<
"fake_messagefacility: " << std::boolalpha
166 << getSupervisorProperty(
"fake_messagefacility",
false) << std::endl;
167 o <<
"advanced_memory_usage: " << std::boolalpha
168 << getSupervisorProperty(
"advanced_memory_usage",
false) << std::endl;
171 __SUP_COUT__ <<
"Constructed." << __E__;
175 ARTDAQSupervisor::~ARTDAQSupervisor(
void)
177 __SUP_COUT__ <<
"Destructor." << __E__;
179 __SUP_COUT__ <<
"Destructed." << __E__;
183 void ARTDAQSupervisor::init(
void)
187 __SUP_COUT__ <<
"Initializing..." << __E__;
189 std::lock_guard<std::mutex> lk(daqinterface_mutex_);
192 artdaq::configureMessageFacility(
"ARTDAQSupervisor");
193 __SUP_COUT__ <<
"artdaq MF configured." << __E__;
196 char* daqinterface_dir = getenv(
"ARTDAQ_DAQINTERFACE_DIR");
197 if(daqinterface_dir == NULL)
199 __SS__ <<
"ARTDAQ_DAQINTERFACE_DIR environment variable not set! This "
200 "means that DAQInterface has not been setup!"
206 __SUP_COUT__ <<
"Initializing Python" << __E__;
209 __SUP_COUT__ <<
"Adding DAQInterface directory to PYTHON_PATH" << __E__;
210 PyObject* sysPath = PySys_GetObject((
char*)
"path");
211 PyObject* programName = PyString_FromString(daqinterface_dir);
212 PyList_Append(sysPath, programName);
213 Py_DECREF(programName);
215 __SUP_COUT__ <<
"Creating Module name" << __E__;
216 PyObject* pName = PyString_FromString(
"rc.control.daqinterface");
219 __SUP_COUT__ <<
"Importing module" << __E__;
220 PyObject* pModule = PyImport_Import(pName);
226 __SS__ <<
"Failed to load rc.control.daqinterface" << __E__;
231 __SUP_COUT__ <<
"Loading python module dictionary" << __E__;
232 PyObject* pDict = PyModule_GetDict(pModule);
236 __SS__ <<
"Unable to load module dictionary" << __E__;
243 __SUP_COUT__ <<
"Getting DAQInterface object pointer" << __E__;
244 PyObject* di_obj_ptr = PyDict_GetItemString(pDict,
"DAQInterface");
246 __SUP_COUT__ <<
"Filling out DAQInterface args struct" << __E__;
247 PyObject* pArgs = PyTuple_New(0);
249 PyObject* kwargs = Py_BuildValue(
"{s:s, s:s, s:i, s:i, s:s, s:s}",
263 __SUP_COUT__ <<
"Calling DAQInterface Object Constructor" << __E__;
264 daqinterface_ptr_ = PyObject_Call(di_obj_ptr, pArgs, kwargs);
266 Py_DECREF(di_obj_ptr);
274 __SUP_COUT__ <<
"Initialized." << __E__;
278 void ARTDAQSupervisor::destroy(
void)
280 __SUP_COUT__ <<
"Destroying..." << __E__;
282 if(daqinterface_ptr_ != NULL)
284 __SUP_COUT__ <<
"Calling recover transition" << __E__;
285 std::lock_guard<std::mutex> lk(daqinterface_mutex_);
286 PyObject* pName = PyString_FromString(
"do_recover");
287 PyObject* res = PyObject_CallMethodObjArgs(daqinterface_ptr_, pName, NULL);
289 __SUP_COUT__ <<
"Making sure that correct state has been reached" << __E__;
291 while(daqinterface_state_ !=
"stopped")
294 __SUP_COUT__ <<
"State is " << daqinterface_state_
295 <<
", waiting 1s and retrying..." << __E__;
299 Py_XDECREF(daqinterface_ptr_);
300 daqinterface_ptr_ = NULL;
304 __SUP_COUT__ <<
"Destroyed." << __E__;
308 void ARTDAQSupervisor::transitionConfiguring(toolbox::Event::Reference event)
310 __SUP_COUT__ <<
"transitionConfiguring" << __E__;
313 if(RunControlStateMachine::getIterationIndex() == 0 &&
314 RunControlStateMachine::getSubIterationIndex() == 0)
317 SOAPUtilities::translate(theStateMachine_.getCurrentMessage())
319 .getValue(
"ConfigurationTableGroupName"),
320 TableGroupKey(SOAPUtilities::translate(theStateMachine_.getCurrentMessage())
322 .getValue(
"ConfigurationTableGroupKey")));
324 __SUP_COUT__ <<
"Configuration table group name: " << theGroup.first
325 <<
" key: " << theGroup.second << __E__;
327 theConfigurationManager_->loadTableGroup(
328 theGroup.first, theGroup.second,
true );
331 std::thread([](
ARTDAQSupervisor* as) { ARTDAQSupervisor::configuringThread(as); },
335 __SUP_COUT__ <<
"Configuring thread started." << __E__;
337 RunControlStateMachine::indicateSubIterationWork();
341 std::string errorMessage = theStateMachine_.getErrorMessage();
342 __SUP_COUTV__(errorMessage);
343 __SUP_COUTV__(theProgressBar_.read());
344 __SUP_COUTV__(theProgressBar_.isComplete());
348 if(errorMessage !=
"")
350 __SUP_SS__ <<
"Error was caught in configuring thread: " << errorMessage
352 __SUP_COUT_ERR__ <<
"\n" << ss.str();
354 theStateMachine_.setErrorMessage(ss.str());
355 throw toolbox::fsm::exception::Exception(
358 "CoreSupervisorBase::transitionConfiguring" ,
364 if(!theProgressBar_.isComplete())
366 RunControlStateMachine::indicateSubIterationWork();
370 __SUP_COUT_INFO__ <<
"Complete configuring transition!" << __E__;
377 void ARTDAQSupervisor::configuringThread(
ARTDAQSupervisor* theArtdaqSupervisor)
try
379 ProgressBar& progressBar = theArtdaqSupervisor->theProgressBar_;
381 const std::string& uid =
382 theArtdaqSupervisor->theConfigurationManager_
384 ConfigurationManager::XDAQ_APPLICATION_TABLE_NAME +
"/" +
385 theArtdaqSupervisor->CorePropertySupervisorBase::getSupervisorUID() +
386 "/" +
"LinkToSupervisorTable")
389 __COUT__ <<
"Supervisor uid is " << uid <<
", getting supervisor table node" << __E__;
391 const std::string mfSubject_ =
392 theArtdaqSupervisor->supervisorClassNoNamespace_ +
"-" + uid;
394 ConfigurationTree theSupervisorNode = theArtdaqSupervisor->getSupervisorTableNode();
397 std::map<ARTDAQTableBase::ARTDAQAppType, std::list<ARTDAQTableBase::ProcessInfo>> processes;
401 ARTDAQTableBase::extractArtdaqInfo(
406 theArtdaqSupervisor->CorePropertySupervisorBase::getSupervisorProperty<
size_t>(
407 "max_fragment_size_bytes", ARTDAQTableBase::DEFAULT_MAX_FRAGMENT_SIZE),
410 std::list<ARTDAQTableBase::ProcessInfo>& readerInfo =
411 processes[ARTDAQTableBase::ARTDAQAppType::BoardReader];
412 std::list<ARTDAQTableBase::ProcessInfo>& builderInfo =
413 processes[ARTDAQTableBase::ARTDAQAppType::EventBuilder];
414 std::list<ARTDAQTableBase::ProcessInfo>& loggerInfo =
415 processes[ARTDAQTableBase::ARTDAQAppType::DataLogger];
416 std::list<ARTDAQTableBase::ProcessInfo>& dispatcherInfo =
417 processes[ARTDAQTableBase::ARTDAQAppType::Dispatcher];
420 if(readerInfo.size() == 0)
422 __GEN_SS__ <<
"There must be at least one enabled BoardReader!" << __E__;
426 if(builderInfo.size() == 0)
428 __GEN_SS__ <<
"There must be at least one enabled EventBuilder!" << __E__;
435 __GEN_COUT__ <<
"Writing boot.txt" << __E__;
437 int debugLevel = theSupervisorNode.getNode(
"DAQInterfaceDebugLevel").getValue<
int>();
438 std::string setupScript = theSupervisorNode.getNode(
"DAQSetupScript").getValue();
440 std::ofstream o(ARTDAQ_FCL_PATH +
"/boot.txt", std::ios::trunc);
441 o <<
"DAQ setup script: " << setupScript << std::endl;
442 o <<
"debug level: " << debugLevel << std::endl;
445 if(subsystems.size() > 1)
447 for(
auto& ss : subsystems)
451 o <<
"Subsystem id: " << ss.first << std::endl;
452 if(ss.second.destination != 0)
454 o <<
"Subsystem destination: " << ss.second.destination << std::endl;
456 for(
auto& sss : ss.second.sources)
458 o <<
"Subsystem source: " << sss << std::endl;
464 for(
auto& builder : builderInfo)
466 o <<
"EventBuilder host: " << builder.hostname << std::endl;
467 o <<
"EventBuilder label: " << builder.label << std::endl;
468 if(builder.subsystem != 1)
470 o <<
"EventBuilder subsystem: " << builder.subsystem << std::endl;
474 for(
auto& logger : loggerInfo)
476 o <<
"DataLogger host: " << logger.hostname << std::endl;
477 o <<
"DataLogger label: " << logger.label << std::endl;
478 if(logger.subsystem != 1)
480 o <<
"DataLogger subsystem: " << logger.subsystem << std::endl;
484 for(
auto& dispatcher : dispatcherInfo)
486 o <<
"Dispatcher host: " << dispatcher.hostname << std::endl;
487 o <<
"Dispatcher label: " << dispatcher.label << std::endl;
488 if(dispatcher.subsystem != 1)
490 o <<
"Dispatcher subsystem: " << dispatcher.subsystem << std::endl;
498 __GEN_COUT__ <<
"Building configuration directory" << __E__;
500 boost::system::error_code ignored;
501 boost::filesystem::remove_all(ARTDAQ_FCL_PATH + FAKE_CONFIG_NAME, ignored);
502 mkdir((ARTDAQ_FCL_PATH + FAKE_CONFIG_NAME).c_str(), 0755);
504 for(
auto& reader : readerInfo)
507 (ARTDAQ_FCL_PATH +
"boardReader-" + reader.label +
".fcl").c_str(),
508 (ARTDAQ_FCL_PATH + FAKE_CONFIG_NAME +
"/" + reader.label +
".fcl").c_str());
510 for(
auto& builder : builderInfo)
513 (ARTDAQ_FCL_PATH +
"builder-" + builder.label +
".fcl").c_str(),
514 (ARTDAQ_FCL_PATH + FAKE_CONFIG_NAME +
"/" + builder.label +
".fcl").c_str());
516 for(
auto& logger : loggerInfo)
519 (ARTDAQ_FCL_PATH +
"datalogger-" + logger.label +
".fcl").c_str(),
520 (ARTDAQ_FCL_PATH + FAKE_CONFIG_NAME +
"/" + logger.label +
".fcl").c_str());
522 for(
auto& dispatcher : dispatcherInfo)
524 symlink((ARTDAQ_FCL_PATH +
"dispatcher-" + dispatcher.label +
".fcl").c_str(),
525 (ARTDAQ_FCL_PATH + FAKE_CONFIG_NAME +
"/" + dispatcher.label +
".fcl")
531 std::lock_guard<std::mutex> lk(theArtdaqSupervisor->daqinterface_mutex_);
532 theArtdaqSupervisor->getDAQState_();
533 if(theArtdaqSupervisor->daqinterface_state_ !=
"stopped" &&
534 theArtdaqSupervisor->daqinterface_state_ !=
"")
536 __GEN_SS__ <<
"Cannot configure DAQInterface because it is in the wrong state"
537 <<
" (" << theArtdaqSupervisor->daqinterface_state_ <<
" != stopped)!"
542 __GEN_COUT__ <<
"Calling setdaqcomps" << __E__;
543 __GEN_COUT__ <<
"Status before setdaqcomps: "
544 << theArtdaqSupervisor->daqinterface_state_ << __E__;
545 PyObject* pName1 = PyString_FromString(
"setdaqcomps");
547 PyObject* readerDict = PyDict_New();
548 for(
auto& reader : readerInfo)
550 PyObject* readerName = PyString_FromString(reader.label.c_str());
552 PyObject* readerData = PyList_New(3);
553 PyObject* readerHost = PyString_FromString(reader.hostname.c_str());
554 PyObject* readerPort = PyString_FromString(
"-1");
555 PyObject* readerSubsystem =
556 PyString_FromString(std::to_string(reader.subsystem).c_str());
557 PyList_SetItem(readerData, 0, readerHost);
558 PyList_SetItem(readerData, 1, readerPort);
559 PyList_SetItem(readerData, 2, readerSubsystem);
560 PyDict_SetItem(readerDict, readerName, readerData);
562 PyObject* res1 = PyObject_CallMethodObjArgs(
563 theArtdaqSupervisor->daqinterface_ptr_, pName1, readerDict, NULL);
564 Py_DECREF(readerDict);
569 __GEN_SS__ <<
"Error calling setdaqcomps transition" << __E__;
572 theArtdaqSupervisor->getDAQState_();
573 __GEN_COUT__ <<
"Status after setdaqcomps: "
574 << theArtdaqSupervisor->daqinterface_state_ << __E__;
577 __GEN_COUT__ <<
"Calling do_boot" << __E__;
578 __GEN_COUT__ <<
"Status before boot: " << theArtdaqSupervisor->daqinterface_state_
580 PyObject* pName2 = PyString_FromString(
"do_boot");
581 PyObject* pStateArgs1 = PyString_FromString((ARTDAQ_FCL_PATH +
"/boot.txt").c_str());
582 PyObject* res2 = PyObject_CallMethodObjArgs(
583 theArtdaqSupervisor->daqinterface_ptr_, pName2, pStateArgs1, NULL);
588 __GEN_SS__ <<
"Error calling boot transition" << __E__;
592 theArtdaqSupervisor->getDAQState_();
593 if(theArtdaqSupervisor->daqinterface_state_ !=
"booted")
595 __GEN_SS__ <<
"DAQInterface boot transition failed!" << __E__;
598 __GEN_COUT__ <<
"Status after boot: " << theArtdaqSupervisor->daqinterface_state_
602 __GEN_COUT__ <<
"Calling do_config" << __E__;
603 __GEN_COUT__ <<
"Status before config: " << theArtdaqSupervisor->daqinterface_state_
605 PyObject* pName3 = PyString_FromString(
"do_config");
606 PyObject* pStateArgs2 = Py_BuildValue(
"[s]", FAKE_CONFIG_NAME);
607 PyObject* res3 = PyObject_CallMethodObjArgs(
608 theArtdaqSupervisor->daqinterface_ptr_, pName3, pStateArgs2, NULL);
613 __GEN_SS__ <<
"Error calling config transition" << __E__;
616 theArtdaqSupervisor->getDAQState_();
617 if(theArtdaqSupervisor->daqinterface_state_ !=
"ready")
619 __GEN_SS__ <<
"DAQInterface config transition failed!" << __E__;
622 __GEN_COUT__ <<
"Status after config: " << theArtdaqSupervisor->daqinterface_state_
624 progressBar.complete();
625 __GEN_COUT__ <<
"Configured." << __E__;
628 catch(
const std::runtime_error& e)
630 __SS__ <<
"Error was caught while configuring: " << e.what() << __E__;
631 __COUT_ERR__ <<
"\n" << ss.str();
632 theArtdaqSupervisor->theStateMachine_.setErrorMessage(ss.str());
636 __SS__ <<
"Unknown error was caught while configuring. Please checked the logs."
638 __COUT_ERR__ <<
"\n" << ss.str();
639 theArtdaqSupervisor->theStateMachine_.setErrorMessage(ss.str());
643 void ARTDAQSupervisor::transitionHalting(toolbox::Event::Reference event)
646 __SUP_COUT__ <<
"Halting..." << __E__;
647 std::lock_guard<std::mutex> lk(daqinterface_mutex_);
649 __SUP_COUT__ <<
"Status before halt: " << daqinterface_state_ << __E__;
651 PyObject* pName = PyString_FromString(
"do_command");
652 PyObject* pArg = PyString_FromString(
"Shutdown");
653 PyObject* res = PyObject_CallMethodObjArgs(daqinterface_ptr_, pName, pArg, NULL);
658 __SS__ <<
"Error calling Shutdown transition" << __E__;
663 __SUP_COUT__ <<
"Status after halt: " << daqinterface_state_ << __E__;
664 __SUP_COUT__ <<
"Halted." << __E__;
666 catch(
const std::runtime_error& e)
668 const std::string transitionName =
"Halting";
670 if(theStateMachine_.getProvenanceStateName() ==
671 RunControlStateMachine::FAILED_STATE_NAME ||
672 theStateMachine_.getProvenanceStateName() ==
673 RunControlStateMachine::HALTED_STATE_NAME)
675 __SUP_COUT_INFO__ <<
"Error was caught while halting (but ignoring because "
676 "previous state was '"
677 << RunControlStateMachine::FAILED_STATE_NAME
678 <<
"'): " << e.what() << __E__;
682 __SUP_SS__ <<
"Error was caught while " << transitionName <<
": " << e.what()
684 __SUP_COUT_ERR__ <<
"\n" << ss.str();
685 theStateMachine_.setErrorMessage(ss.str());
686 throw toolbox::fsm::exception::Exception(
689 "ARTDAQSupervisorBase::transition" + transitionName ,
697 const std::string transitionName =
"Halting";
699 if(theStateMachine_.getProvenanceStateName() ==
700 RunControlStateMachine::FAILED_STATE_NAME ||
701 theStateMachine_.getProvenanceStateName() ==
702 RunControlStateMachine::HALTED_STATE_NAME)
704 __SUP_COUT_INFO__ <<
"Unknown error was caught while halting (but ignoring "
705 "because previous state was '"
706 << RunControlStateMachine::FAILED_STATE_NAME <<
"')."
711 __SUP_SS__ <<
"Unknown error was caught while " << transitionName
712 <<
". Please checked the logs." << __E__;
713 __SUP_COUT_ERR__ <<
"\n" << ss.str();
714 theStateMachine_.setErrorMessage(ss.str());
715 throw toolbox::fsm::exception::Exception(
718 "ARTDAQSupervisorBase::transition" + transitionName ,
726 void ARTDAQSupervisor::transitionInitializing(toolbox::Event::Reference event)
728 __SUP_COUT__ <<
"Initializing..." << __E__;
730 __SUP_COUT__ <<
"Initialized." << __E__;
734 void ARTDAQSupervisor::transitionPausing(toolbox::Event::Reference event)
736 __SUP_COUT__ <<
"Pausing..." << __E__;
737 std::lock_guard<std::mutex> lk(daqinterface_mutex_);
740 __SUP_COUT__ <<
"Status before pause: " << daqinterface_state_ << __E__;
742 PyObject* pName = PyString_FromString(
"do_command");
743 PyObject* pArg = PyString_FromString(
"Pause");
744 PyObject* res = PyObject_CallMethodObjArgs(daqinterface_ptr_, pName, pArg, NULL);
749 __SS__ <<
"Error calling Pause transition" << __E__;
754 __SUP_COUT__ <<
"Status after pause: " << daqinterface_state_ << __E__;
756 __SUP_COUT__ <<
"Paused." << __E__;
760 void ARTDAQSupervisor::transitionResuming(toolbox::Event::Reference event)
762 __SUP_COUT__ <<
"Resuming..." << __E__;
763 std::lock_guard<std::mutex> lk(daqinterface_mutex_);
766 __SUP_COUT__ <<
"Status before resume: " << daqinterface_state_ << __E__;
767 PyObject* pName = PyString_FromString(
"do_command");
768 PyObject* pArg = PyString_FromString(
"Resume");
769 PyObject* res = PyObject_CallMethodObjArgs(daqinterface_ptr_, pName, pArg, NULL);
774 __SS__ <<
"Error calling Resume transition" << __E__;
778 __SUP_COUT__ <<
"Status after resume: " << daqinterface_state_ << __E__;
779 __SUP_COUT__ <<
"Resumed." << __E__;
783 void ARTDAQSupervisor::transitionStarting(toolbox::Event::Reference event)
785 __SUP_COUT__ <<
"Starting..." << __E__;
787 std::lock_guard<std::mutex> lk(daqinterface_mutex_);
789 __SUP_COUT__ <<
"Status before start: " << daqinterface_state_ << __E__;
790 auto runNumber = SOAPUtilities::translate(theStateMachine_.getCurrentMessage())
792 .getValue(
"RunNumber");
794 PyObject* pName = PyString_FromString(
"do_start_running");
795 int run_number = std::stoi(runNumber);
796 PyObject* pStateArgs = PyInt_FromLong(run_number);
798 PyObject_CallMethodObjArgs(daqinterface_ptr_, pName, pStateArgs, NULL);
803 __SS__ <<
"Error calling start transition" << __E__;
807 __SUP_COUT__ <<
"Status after start: " << daqinterface_state_ << __E__;
808 if(daqinterface_state_ !=
"running")
810 __SS__ <<
"DAQInterface start transition failed!" << __E__;
815 __SUP_COUT__ <<
"Started." << __E__;
819 void ARTDAQSupervisor::transitionStopping(toolbox::Event::Reference event)
821 __SUP_COUT__ <<
"Stopping..." << __E__;
822 std::lock_guard<std::mutex> lk(daqinterface_mutex_);
824 __SUP_COUT__ <<
"Status before stop: " << daqinterface_state_ << __E__;
825 PyObject* pName = PyString_FromString(
"do_stop_running");
826 PyObject* res = PyObject_CallMethodObjArgs(daqinterface_ptr_, pName, NULL);
831 __SS__ <<
"Error calling stop transition" << __E__;
835 __SUP_COUT__ <<
"Status after stop: " << daqinterface_state_ << __E__;
836 __SUP_COUT__ <<
"Stopped." << __E__;
840 void ots::ARTDAQSupervisor::enteringError(toolbox::Event::Reference event)
842 __SUP_COUT__ <<
"Entering error recovery state" << __E__;
843 std::lock_guard<std::mutex> lk(daqinterface_mutex_);
845 __SUP_COUT__ <<
"Status before error: " << daqinterface_state_ << __E__;
847 PyObject* pName = PyString_FromString(
"do_recover");
848 PyObject* res = PyObject_CallMethodObjArgs(daqinterface_ptr_, pName, NULL);
853 __SS__ <<
"Error calling recover transition" << __E__;
857 __SUP_COUT__ <<
"Status after error: " << daqinterface_state_ << __E__;
858 __SUP_COUT__ <<
"EnteringError DONE." << __E__;
863 void ots::ARTDAQSupervisor::getDAQState_()
867 PyObject* pName = PyString_FromString(
"state");
868 PyObject* pArg = PyString_FromString(
"DAQInterface");
869 PyObject* res = PyObject_CallMethodObjArgs(daqinterface_ptr_, pName, pArg, NULL);
874 __SS__ <<
"Error calling state function" << __E__;
878 daqinterface_state_ = std::string(PyString_AsString(res));
883 void ots::ARTDAQSupervisor::daqinterfaceRunner_()
885 TLOG(TLVL_TRACE) <<
"Runner thread starting";
886 runner_running_ =
true;
887 while(runner_running_)
889 if(daqinterface_ptr_ != NULL)
891 std::unique_lock<std::mutex> lk(daqinterface_mutex_);
893 std::string state_before = daqinterface_state_;
895 if(daqinterface_state_ ==
"running" || daqinterface_state_ ==
"ready" ||
896 daqinterface_state_ ==
"booted")
900 TLOG(TLVL_TRACE) <<
"Calling DAQInterface::check_proc_heartbeats";
901 PyObject* pName = PyString_FromString(
"check_proc_heartbeats");
903 PyObject_CallMethodObjArgs(daqinterface_ptr_, pName, NULL);
905 <<
"Done with DAQInterface::check_proc_heartbeats call";
909 runner_running_ =
false;
912 __SS__ <<
"Error calling check_proc_heartbeats function" << __E__;
917 catch(cet::exception& ex)
919 runner_running_ =
false;
922 __SS__ <<
"An cet::exception occurred while calling "
923 "check_proc_heartbeats function: "
924 << ex.explain_self() << __E__;
928 catch(std::exception& ex)
930 runner_running_ =
false;
933 __SS__ <<
"An std::exception occurred while calling "
934 "check_proc_heartbeats function: "
935 << ex.what() << __E__;
941 runner_running_ =
false;
944 __SS__ <<
"An unknown Error occurred while calling runner function"
951 if(daqinterface_state_ != state_before)
953 runner_running_ =
false;
955 __SS__ <<
"DAQInterface state unexpectedly changed from "
956 << state_before <<
" to " << daqinterface_state_
957 <<
". Check supervisor log file for more info!" << __E__;
969 runner_running_ =
false;
970 TLOG(TLVL_TRACE) <<
"Runner thread complete";
974 void ots::ARTDAQSupervisor::stop_runner_()
976 runner_running_ =
false;
977 if(runner_thread_ && runner_thread_->joinable())
979 runner_thread_->join();
980 runner_thread_.reset(
nullptr);
985 void ots::ARTDAQSupervisor::start_runner_()
989 std::make_unique<std::thread>(&ots::ARTDAQSupervisor::daqinterfaceRunner_,
this);