1 #include "otsdaq-core/Supervisor/Iterator.h"
2 #include "otsdaq-core/MessageFacility/MessageFacility.h"
3 #include "otsdaq-core/Macros/CoutHeaderMacros.h"
4 #include "otsdaq-core/Supervisor/Supervisor.h"
5 #include "otsdaq-core/WebUsersUtilities/WebUsers.h"
7 #include "otsdaq-core/ConfigurationInterface/ConfigurationManagerRW.h"
14 #define __MF_SUBJECT__ "SupervisorIterator"
20 : workloopRunning_ (false)
21 , activePlanIsRunning_ (false)
22 , iteratorBusy_ (false)
23 , commandPlay_(false), commandPause_(false), commandHalt_(false)
24 , activePlanName_ (
"")
25 , activeCommandIndex_ (-1)
26 , activeCommandStartTime_ (0)
27 , theSupervisor_ (supervisor)
29 __MOUT__ <<
"Iterator constructed." << __E__;
30 __COUT__ <<
"Iterator constructed." << __E__;
35 Iterator::~Iterator(
void)
40 void Iterator::IteratorWorkLoop(
Iterator *iterator)
43 __MOUT__ <<
"Iterator work loop starting..." << __E__;
44 __COUT__ <<
"Iterator work loop starting..." << __E__;
50 if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ <<
"Waiting for iterator access" << __E__;
51 std::lock_guard<std::mutex> lock(iterator->accessMutex_);
52 if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ <<
"Have iterator access" << __E__;
54 iterator->errorMessage_ =
"";
59 IteratorWorkLoopStruct theIteratorStruct(iterator,
60 &theConfigurationManager);
65 std::vector<IterateConfiguration::Command> commands;
87 if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ <<
"Waiting for iterator access" << __E__;
88 std::lock_guard<std::mutex> lock(iterator->accessMutex_);
89 if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ <<
"Have iterator access" << __E__;
91 if(iterator->commandPlay_)
93 iterator->commandPlay_ =
false;
95 if(!iterator->activePlanIsRunning_)
99 iterator->activePlanIsRunning_ =
true;
100 iterator->iteratorBusy_ =
true;
102 if(theIteratorStruct.activePlan_ != iterator->activePlanName_)
104 __COUT__ <<
"New plan name encountered old=" << theIteratorStruct.activePlan_ <<
105 " vs new=" << iterator->activePlanName_ << __E__;
106 theIteratorStruct.commandIndex_ = -1;
109 theIteratorStruct.activePlan_ = iterator->activePlanName_;
110 iterator->lastStartedPlanName_ = iterator->activePlanName_;
112 if(theIteratorStruct.commandIndex_ == (
unsigned int)-1)
114 __COUT__ <<
"Starting plan '" << theIteratorStruct.activePlan_ <<
".'" << __E__;
115 __MOUT__ <<
"Starting plan '" << theIteratorStruct.activePlan_ <<
".'" << __E__;
119 theIteratorStruct.doResumeAction_ =
true;
120 __COUT__ <<
"Continuing plan '" << theIteratorStruct.activePlan_ <<
"' at command index " <<
121 theIteratorStruct.commandIndex_ <<
". " << __E__;
122 __MOUT__ <<
"Continuing plan '" << theIteratorStruct.activePlan_ <<
"' at command index " <<
123 theIteratorStruct.commandIndex_ <<
". " << __E__;
127 else if(iterator->commandPause_ && !theIteratorStruct.doPauseAction_)
129 theIteratorStruct.doPauseAction_ =
true;
130 iterator->commandPause_ =
false;
132 else if(iterator->commandHalt_ && !theIteratorStruct.doHaltAction_)
134 theIteratorStruct.doHaltAction_ =
true;
135 iterator->commandHalt_ =
false;
138 theIteratorStruct.running_ = iterator->activePlanIsRunning_;
140 if(iterator->activeCommandIndex_ !=
141 theIteratorStruct.commandIndex_)
143 iterator->activeCommandIndex_ = theIteratorStruct.commandIndex_;
144 iterator->activeCommandStartTime_ = time(0);
154 if(theIteratorStruct.doPauseAction_)
161 __COUT__ <<
"Waiting to pause..." << __E__;
162 while(!iterator->checkCommand(&theIteratorStruct))
163 __COUT__ <<
"Waiting to pause..." << __E__;
165 __COUT__ <<
"Completeing pause..." << __E__;
167 theIteratorStruct.doPauseAction_ =
false;
171 if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ <<
"Waiting for iterator access" << __E__;
172 std::lock_guard<std::mutex> lock(iterator->accessMutex_);
173 if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ <<
"Have iterator access" << __E__;
175 iterator->activePlanIsRunning_ =
false;
177 __COUT__ <<
"Paused plan '" << theIteratorStruct.activePlan_ <<
"' at command index " <<
178 theIteratorStruct.commandIndex_ <<
". " << __E__;
179 __MOUT__ <<
"Paused plan '" << theIteratorStruct.activePlan_ <<
"' at command index " <<
180 theIteratorStruct.commandIndex_ <<
". " << __E__;
184 else if(theIteratorStruct.doHaltAction_)
191 __COUT__ <<
"Waiting to halt..." << __E__;
192 while(!iterator->checkCommand(&theIteratorStruct))
193 __COUT__ <<
"Waiting to halt..." << __E__;
195 __COUT__ <<
"Completeing halt..." << __E__;
197 theIteratorStruct.doHaltAction_ =
false;
200 iterator->haltStateMachine(
201 iterator->theSupervisor_,
202 theIteratorStruct.fsmName_);
206 if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ <<
"Waiting for iterator access" << __E__;
207 std::lock_guard<std::mutex> lock(iterator->accessMutex_);
208 if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ <<
"Have iterator access" << __E__;
210 iterator->activePlanIsRunning_ =
false;
211 iterator->iteratorBusy_ =
false;
213 __COUT__ <<
"Halted plan '" << theIteratorStruct.activePlan_ <<
"' at command index " <<
214 theIteratorStruct.commandIndex_ <<
". " << __E__;
215 __MOUT__ <<
"Halted plan '" << theIteratorStruct.activePlan_ <<
"' at command index " <<
216 theIteratorStruct.commandIndex_ <<
". " << __E__;
218 theIteratorStruct.activePlan_ =
"";
219 theIteratorStruct.commandIndex_ = -1;
231 if(theIteratorStruct.running_ &&
232 theIteratorStruct.activePlan_ !=
"")
234 if(theIteratorStruct.commandIndex_ == (
unsigned int)-1)
236 __COUT__ <<
"Get commands" << __E__;
238 theIteratorStruct.commandIndex_ = 0;
240 theIteratorStruct.cfgMgr_->init();
243 theIteratorStruct.commands_ = itConfig->getPlanCommands(theIteratorStruct.cfgMgr_,
244 theIteratorStruct.activePlan_);
246 for(
auto& command:theIteratorStruct.commands_)
248 __COUT__ <<
"command " <<
249 command.type_ << __E__;
250 __COUT__ <<
"table " <<
251 IterateConfiguration::commandToTableMap_.at(command.type_) << __E__;
252 __COUT__ <<
"param count = " << command.params_.size() << __E__;
254 for(
auto& param:command.params_)
256 __COUT__ <<
"\t param " <<
257 param.first <<
" : " <<
258 param.second << __E__;
263 if(!theIteratorStruct.commandBusy_)
265 if(theIteratorStruct.commandIndex_ < theIteratorStruct.commands_.size())
268 theIteratorStruct.commandBusy_ =
true;
270 __COUT__ <<
"Iterator starting command " << theIteratorStruct.commandIndex_+1 <<
": " <<
271 theIteratorStruct.commands_[theIteratorStruct.commandIndex_].type_ << __E__;
272 __MOUT__ <<
"Iterator starting command " << theIteratorStruct.commandIndex_+1 <<
": " <<
273 theIteratorStruct.commands_[theIteratorStruct.commandIndex_].type_ << __E__;
290 iterator->startCommand(&theIteratorStruct);
292 else if(theIteratorStruct.commandIndex_ ==
293 theIteratorStruct.commands_.size())
295 __COUT__ <<
"Finished Iteration Plan '" << theIteratorStruct.activePlan_ << __E__;
296 __MOUT__ <<
"Finished Iteration Plan '" << theIteratorStruct.activePlan_ << __E__;
300 if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ <<
"Waiting for iterator access" << __E__;
301 std::lock_guard<std::mutex> lock(iterator->accessMutex_);
302 if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ <<
"Have iterator access" << __E__;
305 iterator->activePlanIsRunning_ =
false;
306 iterator->iteratorBusy_ =
false;
308 iterator->lastStartedPlanName_ = theIteratorStruct.activePlan_;
309 theIteratorStruct.activePlan_ =
"";
310 theIteratorStruct.commandIndex_ = -1;
313 else if(theIteratorStruct.commandBusy_)
316 if(iterator->checkCommand(&theIteratorStruct))
318 theIteratorStruct.commandBusy_ =
false;
320 ++theIteratorStruct.commandIndex_;
322 __COUT__ <<
"Ready for next command. Done with " << theIteratorStruct.commandIndex_ <<
" of " <<
323 theIteratorStruct.commands_.size() << __E__;
324 __MOUT__ <<
"Iterator ready for next command. Done with " << theIteratorStruct.commandIndex_ <<
" of " <<
325 theIteratorStruct.commands_.size() << __E__;
329 if(theIteratorStruct.doResumeAction_)
330 theIteratorStruct.doResumeAction_ =
false;
348 iterator->workloopRunning_ =
false;
350 catch(
const std::runtime_error &e)
352 __SS__ <<
"Encountered error in Iterator thread:\n" << e.what() << __E__;
353 __COUT_ERR__ << ss.str();
357 if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ <<
"Waiting for iterator access" << __E__;
358 std::lock_guard<std::mutex> lock(iterator->accessMutex_);
359 if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ <<
"Have iterator access" << __E__;
361 iterator->workloopRunning_ =
false;
362 iterator->errorMessage_ = ss.str();
367 __SS__ <<
"Encountered unknown error in Iterator thread." << __E__;
368 __COUT_ERR__ << ss.str();
372 if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ <<
"Waiting for iterator access" << __E__;
373 std::lock_guard<std::mutex> lock(iterator->accessMutex_);
374 if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ <<
"Have iterator access" << __E__;
376 iterator->workloopRunning_ =
false;
377 iterator->errorMessage_ = ss.str();
381 void Iterator::startCommand(IteratorWorkLoopStruct *iteratorStruct)
386 if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX) __COUT__ <<
"Waiting for FSM access" << __E__;
387 std::lock_guard<std::mutex> lock(iteratorStruct->theIterator_->theSupervisor_->stateMachineAccessMutex_);
388 if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX) __COUT__ <<
"Have FSM access" << __E__;
392 if(iteratorStruct->commandIndex_ >= iteratorStruct->commands_.size())
394 __SS__ <<
"Out of range commandIndex = " << iteratorStruct->commandIndex_ <<
395 " in size = " << iteratorStruct->commands_.size() << __E__;
396 throw std::runtime_error(ss.str());
400 std::string type = iteratorStruct->commands_[iteratorStruct->commandIndex_].type_;
401 if(type == IterateConfiguration::COMMAND_BEGIN_LABEL)
403 return startCommandBeginLabel(iteratorStruct);
405 else if(type == IterateConfiguration::COMMAND_CHOOSE_FSM)
407 return startCommandChooseFSM(
409 iteratorStruct->commands_
410 [iteratorStruct->commandIndex_].params_
411 [IterateConfiguration::commandChooseFSMParams_.NameOfFSM_]);
413 else if(type == IterateConfiguration::COMMAND_CONFIGURE_ACTIVE_GROUP)
415 return startCommandConfigureActive(iteratorStruct);
417 else if(type == IterateConfiguration::COMMAND_CONFIGURE_ALIAS)
419 return startCommandConfigureAlias(
421 iteratorStruct->commands_
422 [iteratorStruct->commandIndex_].params_
423 [IterateConfiguration::commandConfigureAliasParams_.SystemAlias_]);
425 else if(type == IterateConfiguration::COMMAND_CONFIGURE_GROUP)
427 return startCommandConfigureGroup(iteratorStruct);
429 else if(type == IterateConfiguration::COMMAND_EXECUTE_FE_MACRO)
434 else if(type == IterateConfiguration::COMMAND_EXECUTE_MACRO)
439 else if(type == IterateConfiguration::COMMAND_MODIFY_ACTIVE_GROUP)
441 return startCommandModifyActive(iteratorStruct);
443 else if(type == IterateConfiguration::COMMAND_REPEAT_LABEL)
445 return startCommandRepeatLabel(iteratorStruct);
447 else if(type == IterateConfiguration::COMMAND_RUN)
449 return startCommandRun(iteratorStruct);
453 __SS__ <<
"Attempt to start unrecognized command type = " << type << __E__;
454 __COUT_ERR__ << ss.str();
455 throw std::runtime_error(ss.str());
463 bool Iterator::checkCommand(IteratorWorkLoopStruct *iteratorStruct)
466 if(iteratorStruct->commandIndex_ >= iteratorStruct->commands_.size())
468 __COUT__ <<
"Out of range commandIndex = " << iteratorStruct->commandIndex_ <<
469 " in size = " << iteratorStruct->commands_.size() << __E__;
473 std::string type = iteratorStruct->commands_[iteratorStruct->commandIndex_].type_;
474 if(type == IterateConfiguration::COMMAND_BEGIN_LABEL)
479 else if(type == IterateConfiguration::COMMAND_CHOOSE_FSM)
484 else if(type == IterateConfiguration::COMMAND_CONFIGURE_ALIAS ||
485 type == IterateConfiguration::COMMAND_CONFIGURE_ACTIVE_GROUP ||
486 type == IterateConfiguration::COMMAND_CONFIGURE_GROUP)
488 return checkCommandConfigure(iteratorStruct);
490 else if(type == IterateConfiguration::COMMAND_EXECUTE_FE_MACRO)
495 else if(type == IterateConfiguration::COMMAND_EXECUTE_MACRO)
500 else if(type == IterateConfiguration::COMMAND_MODIFY_ACTIVE_GROUP)
505 else if(type == IterateConfiguration::COMMAND_REPEAT_LABEL)
510 else if(type == IterateConfiguration::COMMAND_RUN)
512 return checkCommandRun(iteratorStruct);
516 __SS__ <<
"Attempt to check unrecognized command type = " << type << __E__;
517 __COUT_ERR__ << ss.str();
518 throw std::runtime_error(ss.str());
523 void Iterator::startCommandChooseFSM(IteratorWorkLoopStruct *iteratorStruct,
524 const std::string& fsmName)
526 __COUT__ <<
"fsmName " << fsmName << __E__;
529 iteratorStruct->fsmName_ = fsmName;
530 iteratorStruct->theIterator_->lastFsmName_ = fsmName;
541 iteratorStruct->fsmRunAlias_ =
"Run";
544 ConfigurationTree configLinkNode = iteratorStruct->cfgMgr_->getSupervisorConfigurationNode(
545 iteratorStruct->theIterator_->theSupervisor_->supervisorContextUID_,
546 iteratorStruct->theIterator_->theSupervisor_->supervisorApplicationUID_);
548 if(!configLinkNode.isDisconnected())
552 ConfigurationTree fsmLinkNode = configLinkNode.getNode(
"LinkToStateMachineConfiguration");
553 if(!fsmLinkNode.isDisconnected())
554 iteratorStruct->fsmRunAlias_ =
555 fsmLinkNode.getNode(fsmName +
"/RunDisplayAlias").getValue<std::string>();
557 __COUT_INFO__ <<
"FSM Link disconnected." << __E__;
559 catch(std::runtime_error &e)
562 __COUT_INFO__ <<
"No state machine Run alias. Ignoring and assuming alias of '" <<
563 iteratorStruct->fsmRunAlias_ <<
".'" << __E__;
567 __COUT_ERR__ <<
"Unknown error. Should never happen." << __E__;
569 __COUT_INFO__ <<
"No state machine Run alias. Ignoring and assuming alias of '" <<
570 iteratorStruct->fsmRunAlias_ <<
".'" << __E__;
574 __COUT_INFO__ <<
"FSM Link disconnected." << __E__;
576 __COUT__ <<
"fsmRunAlias_ = " << iteratorStruct->fsmRunAlias_ << __E__;
583 iteratorStruct->fsmNextRunNumber_ = iteratorStruct->theIterator_->theSupervisor_->getNextRunNumber(
584 iteratorStruct->fsmName_);
586 if(iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getCurrentStateName() ==
"Running" ||
587 iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getCurrentStateName() ==
"Paused")
588 --iteratorStruct->fsmNextRunNumber_;
590 __COUT__ <<
"fsmNextRunNumber_ = " << iteratorStruct->fsmNextRunNumber_ << __E__;
595 bool Iterator::haltStateMachine(
Supervisor* theSupervisor,
const std::string& fsmName)
597 std::vector<std::string> fsmCommandParameters;
598 std::string errorStr =
"";
599 std::string currentState = theSupervisor->theStateMachine_.getCurrentStateName();
601 if(currentState ==
"Initialized" ||
602 currentState ==
"Halted")
604 __COUT__ <<
"Do nothing. Already halted." << __E__;
607 else if(currentState ==
"Running")
608 errorStr = theSupervisor->attemptStateMachineTransition(
611 WebUsers::DEFAULT_ITERATOR_USERNAME ,
612 WebUsers::DEFAULT_ITERATOR_USERNAME,
613 fsmCommandParameters);
615 errorStr = theSupervisor->attemptStateMachineTransition(
618 WebUsers::DEFAULT_ITERATOR_USERNAME ,
619 WebUsers::DEFAULT_ITERATOR_USERNAME,
620 fsmCommandParameters);
624 __SS__ <<
"Iterator failed to halt because of the following error: " << errorStr;
625 throw std::runtime_error(ss.str());
629 __COUT__ <<
"FSM in transition = " << theSupervisor->theStateMachine_.isInTransition() << __E__;
630 __COUT__ <<
"haltStateMachine launched." << __E__;
635 void Iterator::startCommandBeginLabel(IteratorWorkLoopStruct *iteratorStruct)
637 __COUT__ <<
"Entering label '" <<
638 iteratorStruct->commands_
639 [iteratorStruct->commandIndex_].params_
640 [IterateConfiguration::commandBeginLabelParams_.Label_]
641 <<
"'..." << std::endl;
644 iteratorStruct->stepIndexStack_.push_back(0);
649 void Iterator::startCommandRepeatLabel(IteratorWorkLoopStruct *iteratorStruct)
653 int numOfRepetitions;
654 sscanf(iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
655 [IterateConfiguration::commandRepeatLabelParams_.NumberOfRepetitions_].c_str(),
656 "%d",&numOfRepetitions);
657 __COUT__ <<
"numOfRepetitions remaining = " << numOfRepetitions << __E__;
659 if(numOfRepetitions <= 0)
662 iteratorStruct->stepIndexStack_.pop_back();
670 ++(iteratorStruct->stepIndexStack_.back());
673 for(i=iteratorStruct->commandIndex_;i>0;--i)
674 if(iteratorStruct->commands_[i].type_ == IterateConfiguration::COMMAND_BEGIN_LABEL &&
675 iteratorStruct->commands_[iteratorStruct->commandIndex_].params_[IterateConfiguration::commandRepeatLabelParams_.Label_] ==
676 iteratorStruct->commands_[i].params_[IterateConfiguration::commandBeginLabelParams_.Label_])
break;
679 sprintf(repStr,
"%d",numOfRepetitions);
680 iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
681 [IterateConfiguration::commandRepeatLabelParams_.NumberOfRepetitions_] =
684 iteratorStruct->commandIndex_ = i;
685 __COUT__ <<
"Jumping back to commandIndex " << iteratorStruct->commandIndex_ << __E__;
689 void Iterator::startCommandRun(IteratorWorkLoopStruct *iteratorStruct)
692 iteratorStruct->fsmCommandParameters_.clear();
694 std::string errorStr =
"";
695 std::string currentState = iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getCurrentStateName();
699 if(currentState ==
"Configured")
700 errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
702 "Start",iteratorStruct->fsmName_,
703 WebUsers::DEFAULT_ITERATOR_USERNAME ,
704 WebUsers::DEFAULT_ITERATOR_USERNAME,
705 iteratorStruct->fsmCommandParameters_);
707 errorStr =
"Can only Run from the Configured state. The current state is " +
713 __SS__ <<
"Iterator failed to run because of the following error: " << errorStr;
714 throw std::runtime_error(ss.str());
718 __COUT__ <<
"FSM in transition = " << iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.isInTransition() << __E__;
719 __COUT__ <<
"startCommandRun success." << __E__;
723 void Iterator::startCommandConfigureActive(IteratorWorkLoopStruct *iteratorStruct)
725 __COUT__ <<
"startCommandConfigureActive " << __E__;
731 std::string group = iteratorStruct->cfgMgr_->getActiveGroupName();
734 __COUT__ <<
"group " << group << __E__;
735 __COUT__ <<
"key " << key << __E__;
739 std::stringstream systemAlias;
740 systemAlias <<
"GROUP:" << group <<
":" << key;
741 startCommandConfigureAlias(iteratorStruct,systemAlias.str());
745 void Iterator::startCommandConfigureGroup(IteratorWorkLoopStruct *iteratorStruct)
747 __COUT__ <<
"startCommandConfigureGroup " << __E__;
753 iteratorStruct->commands_
754 [iteratorStruct->commandIndex_].params_
755 [IterateConfiguration::commandConfigureGroupParams_.GroupName_];
757 iteratorStruct->commands_
758 [iteratorStruct->commandIndex_].params_
759 [IterateConfiguration::commandConfigureGroupParams_.GroupKey_])
762 __COUT__ <<
"group " << group << __E__;
763 __COUT__ <<
"key " << key << __E__;
767 std::stringstream systemAlias;
768 systemAlias <<
"GROUP:" << group <<
":" << key;
769 startCommandConfigureAlias(iteratorStruct,systemAlias.str());
773 void Iterator::startCommandConfigureAlias(IteratorWorkLoopStruct *iteratorStruct,
774 const std::string& systemAlias)
776 __COUT__ <<
"systemAlias " << systemAlias << __E__;
778 iteratorStruct->fsmCommandParameters_.clear();
779 iteratorStruct->fsmCommandParameters_.push_back(systemAlias);
781 std::string errorStr =
"";
782 std::string currentState = iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getCurrentStateName();
786 if(currentState ==
"Initial")
787 errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
789 "Initialize",iteratorStruct->fsmName_,
790 WebUsers::DEFAULT_ITERATOR_USERNAME ,
791 WebUsers::DEFAULT_ITERATOR_USERNAME,
792 iteratorStruct->fsmCommandParameters_);
793 else if(currentState ==
"Halted")
794 errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
796 "Configure",iteratorStruct->fsmName_,
797 WebUsers::DEFAULT_ITERATOR_USERNAME ,
798 WebUsers::DEFAULT_ITERATOR_USERNAME,
799 iteratorStruct->fsmCommandParameters_);
801 errorStr =
"Can only Configure from the Initial or Halted state. The current state is " +
808 __SS__ <<
"Iterator failed to configure with system alias '" <<
809 (iteratorStruct->fsmCommandParameters_.size()?
810 iteratorStruct->fsmCommandParameters_[0]:
"UNKNOWN") <<
811 "' because of the following error: " << errorStr;
812 throw std::runtime_error(ss.str());
816 __COUT__ <<
"FSM in transition = " << iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.isInTransition() << __E__;
817 __COUT__ <<
"startCommandConfigureAlias success." << __E__;
821 void Iterator::startCommandModifyActive(IteratorWorkLoopStruct *iteratorStruct)
842 bool doTrackGroupChanges =
false;
843 if(
"True" == iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
844 [IterateConfiguration::commandModifyActiveParams_.DoTrackGroupChanges_])
845 doTrackGroupChanges =
true;
847 const std::string& pathToField =
848 iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
849 [IterateConfiguration::commandModifyActiveParams_.RelativePathToField_];
850 const std::string& startValue =
851 iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
852 [IterateConfiguration::commandModifyActiveParams_.FieldStartValue_];
853 const std::string& stepSize =
854 iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
855 [IterateConfiguration::commandModifyActiveParams_.FieldIterationStepSize_];
857 const unsigned int stepIndex = iteratorStruct->stepIndexStack_.back();
861 __COUT__ <<
"doTrackGroupChanges " << (doTrackGroupChanges?
"yes":
"no") << std::endl;
862 __COUT__ <<
"stepIndex " << stepIndex << std::endl;
863 __COUT__ <<
"pathToField " << pathToField << std::endl;
864 __COUT__ <<
"startValue " << startValue << std::endl;
865 __COUT__ <<
"stepSize " << stepSize << std::endl;
877 bool Iterator::checkCommandRun(IteratorWorkLoopStruct *iteratorStruct)
885 if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX) __COUT__ <<
"Waiting for FSM access" << __E__;
886 std::lock_guard<std::mutex> lock(iteratorStruct->theIterator_->theSupervisor_->stateMachineAccessMutex_);
887 if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX) __COUT__ <<
"Have FSM access" << __E__;
889 if(iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.isInTransition())
892 iteratorStruct->fsmCommandParameters_.clear();
894 std::string errorStr =
"";
895 std::string currentState = iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getCurrentStateName();
900 if(iteratorStruct->doPauseAction_)
903 __COUT__ <<
"Transitioning FSM to Paused..." << __E__;
905 if(currentState ==
"Paused")
908 __COUT__ <<
"Transition to Paused complete." << __E__;
911 else if(currentState ==
"Running")
912 errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
914 "Pause",iteratorStruct->fsmName_,
915 WebUsers::DEFAULT_ITERATOR_USERNAME ,
916 WebUsers::DEFAULT_ITERATOR_USERNAME,
917 iteratorStruct->fsmCommandParameters_);
919 errorStr =
"Expected to be in Paused. Unexpectedly, the current state is " +
920 currentState +
". Last State Machine error message was as follows: " +
921 iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getErrorMessage();
925 __SS__ <<
"Iterator failed to pause because of the following error: " << errorStr;
926 throw std::runtime_error(ss.str());
930 else if(iteratorStruct->doHaltAction_)
933 __COUT__ <<
"Transitioning FSM to Halted..." << __E__;
935 if(currentState ==
"Halted")
938 __COUT__ <<
"Transition to Halted complete." << __E__;
941 else if(currentState ==
"Running" ||
942 currentState ==
"Paused")
943 errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
945 "Abort",iteratorStruct->fsmName_,
946 WebUsers::DEFAULT_ITERATOR_USERNAME ,
947 WebUsers::DEFAULT_ITERATOR_USERNAME,
948 iteratorStruct->fsmCommandParameters_);
950 errorStr =
"Expected to be in Halted. Unexpectedly, the current state is " +
951 currentState +
". Last State Machine error message was as follows: " +
952 iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getErrorMessage();
956 __SS__ <<
"Iterator failed to halt because of the following error: " << errorStr;
957 throw std::runtime_error(ss.str());
961 else if(iteratorStruct->doResumeAction_)
966 __COUT__ <<
"Transitioning FSM to Running..." << __E__;
968 if(currentState ==
"Paused")
969 errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
971 "Resume",iteratorStruct->fsmName_,
972 WebUsers::DEFAULT_ITERATOR_USERNAME ,
973 WebUsers::DEFAULT_ITERATOR_USERNAME,
974 iteratorStruct->fsmCommandParameters_);
978 __SS__ <<
"Iterator failed to run because of the following error: " << errorStr;
979 throw std::runtime_error(ss.str());
989 if(currentState !=
"Running")
991 if(iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
992 [IterateConfiguration::commandRunParams_.DurationInSeconds_] ==
994 currentState ==
"Configured")
997 __COUT__ <<
"Reached end of run " <<
998 iteratorStruct->fsmNextRunNumber_ << __E__;
1002 errorStr =
"Expected to be in Running. Unexpectedly, the current state is " +
1003 currentState +
". Last State Machine error message was as follows: " +
1004 iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getErrorMessage();
1008 bool waitOnRunningThreads =
false;
1009 if(
"True" == iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
1010 [IterateConfiguration::commandRunParams_.WaitOnRunningThreads_])
1011 waitOnRunningThreads =
true;
1013 time_t remainingDurationInSeconds;
1014 sscanf(iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
1015 [IterateConfiguration::commandRunParams_.DurationInSeconds_].c_str(),
1016 "%ld",&remainingDurationInSeconds);
1019 __COUT__ <<
"waitOnRunningThreads " << waitOnRunningThreads << __E__;
1020 __COUT__ <<
"remainingDurationInSeconds " << remainingDurationInSeconds << __E__;
1024 if(waitOnRunningThreads)
1028 iteratorStruct->theIterator_->theSupervisor_;
1030 bool allFrontEndsAreDone =
true;
1032 theSupervisor->theSupervisorDescriptorInfo_.getFEDescriptors())
1036 std::string status = theSupervisor->send(it.second,
1037 "WorkLoopStatusRequest");
1039 __COUT__ <<
"FESupervisor instance " << it.first <<
1040 " has status = " << status << std::endl;
1042 if(status !=
"Done")
1044 allFrontEndsAreDone =
false;
1048 catch (xdaq::exception::Exception& e)
1050 __SS__ <<
"Could not retrieve status from FESupervisor instance " <<
1051 it.first <<
"." << std::endl;
1052 __COUT_WARN__ << ss.str();
1053 errorStr = ss.str();
1057 __SS__ <<
"Iterator failed to run because of the following error: " << errorStr;
1058 throw std::runtime_error(ss.str());
1063 if(allFrontEndsAreDone)
1066 __COUT__ <<
"FE workloops all complete! Stopping run..." << __E__;
1068 errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
1070 "Stop",iteratorStruct->fsmName_,
1071 WebUsers::DEFAULT_ITERATOR_USERNAME ,
1072 WebUsers::DEFAULT_ITERATOR_USERNAME,
1073 iteratorStruct->fsmCommandParameters_);
1077 __SS__ <<
"Iterator failed to stop run because of the following error: " << errorStr;
1078 throw std::runtime_error(ss.str());
1082 iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
1083 [IterateConfiguration::commandRunParams_.DurationInSeconds_] =
1093 if(remainingDurationInSeconds > 1)
1095 --remainingDurationInSeconds;
1099 sprintf(str,
"%ld",remainingDurationInSeconds);
1100 iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
1101 [IterateConfiguration::commandRunParams_.DurationInSeconds_] =
1104 else if(remainingDurationInSeconds == 1)
1107 __COUT__ <<
"Time duration reached! Stopping run..." << __E__;
1109 errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
1111 "Stop",iteratorStruct->fsmName_,
1112 WebUsers::DEFAULT_ITERATOR_USERNAME ,
1113 WebUsers::DEFAULT_ITERATOR_USERNAME,
1114 iteratorStruct->fsmCommandParameters_);
1118 __SS__ <<
"Iterator failed to stop run because of the following error: " << errorStr;
1119 throw std::runtime_error(ss.str());
1123 iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
1124 [IterateConfiguration::commandRunParams_.DurationInSeconds_] =
1133 __SS__ <<
"Iterator failed to run because of the following error: " << errorStr;
1134 throw std::runtime_error(ss.str());
1141 bool Iterator::checkCommandConfigure(IteratorWorkLoopStruct *iteratorStruct)
1149 if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX) __COUT__ <<
"Waiting for FSM access" << __E__;
1150 std::lock_guard<std::mutex> lock(iteratorStruct->theIterator_->theSupervisor_->stateMachineAccessMutex_);
1151 if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX) __COUT__ <<
"Have FSM access" << __E__;
1153 if(iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.isInTransition())
1156 std::string errorStr =
"";
1157 std::string currentState = iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getCurrentStateName();
1159 if(currentState ==
"Halted")
1160 errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
1162 "Configure",iteratorStruct->fsmName_,
1163 WebUsers::DEFAULT_ITERATOR_USERNAME ,
1164 WebUsers::DEFAULT_ITERATOR_USERNAME,
1165 iteratorStruct->fsmCommandParameters_);
1166 else if(currentState !=
"Configured")
1167 errorStr =
"Expected to be in Configure. Unexpectedly, the current state is " +
1168 currentState +
"." +
". Last State Machine error message was as follows: " +
1169 iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getErrorMessage();
1172 __COUT__ <<
"checkCommandConfigureAlias complete." << __E__;
1178 __SS__ <<
"Iterator failed to configure with system alias '" <<
1179 (iteratorStruct->fsmCommandParameters_.size()?
1180 iteratorStruct->fsmCommandParameters_[0]:
"UNKNOWN") <<
1181 "' because of the following error: " << errorStr;
1182 throw std::runtime_error(ss.str());
1190 const std::string& command,
const std::string& parameter)
1192 if(command ==
"iteratePlay")
1194 playIterationPlan(xmldoc,parameter);
1197 else if(command ==
"iteratePause")
1199 pauseIterationPlan(xmldoc);
1202 else if(command ==
"iterateHalt")
1204 haltIterationPlan(xmldoc);
1207 else if(command ==
"getIterationPlanStatus")
1209 getIterationPlanStatus(xmldoc);
1216 if(theSupervisor_->VERBOSE_MUTEX) __COUT__ <<
"Waiting for iterator access" << __E__;
1217 std::lock_guard<std::mutex> lock(accessMutex_);
1218 if(theSupervisor_->VERBOSE_MUTEX) __COUT__ <<
"Have iterator access" << __E__;
1222 __SS__ <<
"Error - Can not accept request because the Iterator " <<
1224 "in control of State Machine progress. ";
1225 __COUT_ERR__ <<
"\n" << ss.str();
1226 __MOUT_ERR__ <<
"\n" << ss.str();
1228 xmldoc.addTextElementToData(
"state_tranisition_attempted",
"0");
1229 xmldoc.addTextElementToData(
"state_tranisition_attempted_err",
1239 void Iterator::playIterationPlan(
HttpXmlDocument& xmldoc,
const std::string& planName)
1241 __MOUT__ <<
"Attempting to play iteration plan '" << planName <<
".'" << __E__;
1242 __COUT__ <<
"Attempting to play iteration plan '" << planName <<
".'" << __E__;
1250 if(theSupervisor_->VERBOSE_MUTEX) __COUT__ <<
"Waiting for iterator access" << __E__;
1251 std::lock_guard<std::mutex> lock(accessMutex_);
1252 if(theSupervisor_->VERBOSE_MUTEX) __COUT__ <<
"Have iterator access" << __E__;
1254 if(!activePlanIsRunning_ && !commandPlay_)
1256 if(!workloopRunning_)
1260 workloopRunning_ =
true;
1263 std::thread([](
Iterator *iterator){ Iterator::IteratorWorkLoop(iterator); },
this).detach();
1266 activePlanName_ = planName;
1267 commandPlay_ =
true;
1271 __SS__ <<
"Invalid play command attempted. Can only play when the Iterator is inactive or paused." <<
1272 " If you would like to restart an iteration plan, first try halting." << __E__;
1273 __MOUT__ << ss.str();
1275 xmldoc.addTextElementToData(
"error_message", ss.str());
1277 __COUT__ <<
"Invalid play command attempted. " <<
1278 commandPlay_ <<
" " <<
1279 activePlanName_ << __E__;
1286 __MOUT__ <<
"Attempting to pause iteration plan '" << activePlanName_ <<
".'" << __E__;
1287 __COUT__ <<
"Attempting to pause iteration plan '" << activePlanName_ <<
".'" << __E__;
1293 if(theSupervisor_->VERBOSE_MUTEX) __COUT__ <<
"Waiting for iterator access" << __E__;
1294 std::lock_guard<std::mutex> lock(accessMutex_);
1295 if(theSupervisor_->VERBOSE_MUTEX) __COUT__ <<
"Have iterator access" << __E__;
1297 if(workloopRunning_ && activePlanIsRunning_ && !commandPause_)
1299 commandPause_ =
true;
1303 __SS__ <<
"Invalid pause command attempted. Can only pause when running." << __E__;
1304 __MOUT__ << ss.str();
1306 xmldoc.addTextElementToData(
"error_message", ss.str());
1308 __COUT__ <<
"Invalid pause command attempted. " <<
1309 workloopRunning_ <<
" " <<
1310 activePlanIsRunning_ <<
" " <<
1311 commandPause_ <<
" " <<
1312 activePlanName_ << __E__;
1319 __MOUT__ <<
"Attempting to halt iteration plan '" << activePlanName_ <<
".'" << __E__;
1320 __COUT__ <<
"Attempting to halt iteration plan '" << activePlanName_ <<
".'" << __E__;
1326 if(theSupervisor_->VERBOSE_MUTEX) __COUT__ <<
"Waiting for iterator access" << __E__;
1327 std::lock_guard<std::mutex> lock(accessMutex_);
1328 if(theSupervisor_->VERBOSE_MUTEX) __COUT__ <<
"Have iterator access" << __E__;
1330 if(activePlanIsRunning_ && !commandHalt_)
1332 if(workloopRunning_)
1334 __COUT__ <<
"Passing halt command to iterator thread." << __E__;
1335 commandHalt_ =
true;
1339 __COUT__ <<
"No thread, so conducting halt." << __E__;
1340 activePlanIsRunning_ =
false;
1341 iteratorBusy_ =
false;
1345 Iterator::haltStateMachine(theSupervisor_, lastFsmName_);
1347 catch(
const std::runtime_error& e)
1349 xmldoc.addTextElementToData(
"error_message", e.what());
1355 __COUT__ <<
"No thread, so conducting halt." << __E__;
1357 bool haltAttempted =
false;
1360 haltAttempted = Iterator::haltStateMachine(theSupervisor_, lastFsmName_);
1362 catch(
const std::runtime_error& e)
1364 haltAttempted =
false;
1369 __SS__ <<
"Invalid halt command attempted. Can only halt when there is an active iteration plan." << __E__;
1370 __MOUT__ << ss.str();
1372 xmldoc.addTextElementToData(
"error_message", ss.str());
1374 __COUT__ <<
"Invalid halt command attempted. " <<
1375 workloopRunning_ <<
" " <<
1376 activePlanIsRunning_ <<
" " <<
1377 commandHalt_ <<
" " <<
1378 activePlanName_ << __E__;
1381 __COUT__ <<
"Halt was attempted." << __E__;
1389 xmldoc.addTextElementToData(
"current_state", theSupervisor_->theStateMachine_.getCurrentStateName());
1390 xmldoc.addTextElementToData(
"in_transition", theSupervisor_->theStateMachine_.isInTransition() ?
"1" :
"0");
1391 if(theSupervisor_->theStateMachine_.isInTransition())
1392 xmldoc.addTextElementToData(
"transition_progress",
1393 theSupervisor_->theProgressBar_.readPercentageString());
1395 xmldoc.addTextElementToData(
"transition_progress",
"100");
1398 sprintf(tmp,
"%lu",theSupervisor_->theStateMachine_.getTimeInState());
1399 xmldoc.addTextElementToData(
"time_in_state", tmp);
1405 if(theSupervisor_->VERBOSE_MUTEX) __COUT__ <<
"Waiting for iterator access" << __E__;
1406 std::lock_guard<std::mutex> lock(accessMutex_);
1407 if(theSupervisor_->VERBOSE_MUTEX) __COUT__ <<
"Have iterator access" << __E__;
1409 xmldoc.addTextElementToData(
"active_plan", activePlanName_);
1410 xmldoc.addTextElementToData(
"last_started_plan", lastStartedPlanName_);
1411 xmldoc.addTextElementToData(
"last_finished_plan", lastFinishedPlanName_);
1413 sprintf(tmp,
"%u",activeCommandIndex_);
1414 xmldoc.addTextElementToData(
"current_command_index", tmp);
1415 sprintf(tmp,
"%ld",time(0) - activeCommandStartTime_);
1416 xmldoc.addTextElementToData(
"current_command_duration", tmp);
1418 if(activePlanIsRunning_ && iteratorBusy_)
1420 if(workloopRunning_)
1421 xmldoc.addTextElementToData(
"active_plan_status",
"Running");
1423 xmldoc.addTextElementToData(
"active_plan_status",
"Error");
1425 else if(!activePlanIsRunning_ && iteratorBusy_)
1426 xmldoc.addTextElementToData(
"active_plan_status",
"Paused");
1428 xmldoc.addTextElementToData(
"active_plan_status",
"Inactive");
1430 xmldoc.addTextElementToData(
"error_message", errorMessage_);