$treeview $search $mathjax $extrastylesheet
otsdaq
v2_03_00
$projectbrief
|
$projectbrief
|
$searchbox |
00001 #include "otsdaq-core/FiniteStateMachine/RunControlStateMachine.h" 00002 #include "otsdaq-core/MessageFacility/MessageFacility.h" 00003 00004 #include "otsdaq-core/Macros/CoutMacros.h" 00005 #include "otsdaq-core/Macros/StringMacros.h" 00006 00007 #include "otsdaq-core/SOAPUtilities/SOAPCommand.h" 00008 #include "otsdaq-core/SOAPUtilities/SOAPUtilities.h" 00009 00010 #include <toolbox/fsm/FailedEvent.h> 00011 #include <xdaq/NamespaceURI.h> 00012 #include <xoap/Method.h> 00013 00014 #include <iostream> 00015 00016 #undef __MF_SUBJECT__ 00017 #define __MF_SUBJECT__ std::string("FSM-") + theStateMachine_.getStateMachineName() 00018 00019 using namespace ots; 00020 00021 const std::string RunControlStateMachine::FAILED_STATE_NAME = "Failed"; 00022 00023 //======================================================================================================================== 00024 RunControlStateMachine::RunControlStateMachine(const std::string& name) 00025 : theStateMachine_(name) 00026 , asyncFailureReceived_(false) 00027 , asyncSoftFailureReceived_(false) 00028 { 00029 INIT_MF("RunControlStateMachine"); 00030 00031 theStateMachine_.addState( 00032 'I', "Initial", this, &RunControlStateMachine::stateInitial); 00033 theStateMachine_.addState('H', "Halted", this, &RunControlStateMachine::stateHalted); 00034 theStateMachine_.addState( 00035 'C', "Configured", this, &RunControlStateMachine::stateConfigured); 00036 theStateMachine_.addState( 00037 'R', "Running", this, &RunControlStateMachine::stateRunning); 00038 theStateMachine_.addState('P', "Paused", this, &RunControlStateMachine::statePaused); 00039 theStateMachine_.addState( 00040 'X', "Shutdown", this, &RunControlStateMachine::stateShutdown); 00041 // theStateMachine_.addState('v', "Recovering", this, 00042 // &RunControlStateMachine::stateRecovering); theStateMachine_.addState('T', 00043 // "TTSTestMode", this, &RunControlStateMachine::stateTTSTestMode); 00044 00045 // RAR added back in on 11/20/2016.. why was it removed.. 00046 // exceptions like.. 00047 // XCEPT_RAISE (toolbox::fsm::exception::Exception, ss.str());) 00048 // take state machine to "failed" otherwise 00049 theStateMachine_.setStateName('F', RunControlStateMachine::FAILED_STATE_NAME); // x 00050 theStateMachine_.setFailedStateTransitionAction( 00051 this, &RunControlStateMachine::enteringError); 00052 theStateMachine_.setFailedStateTransitionChanged(this, 00053 &RunControlStateMachine::inError); 00054 00055 // this line was added to get out of Failed state 00056 RunControlStateMachine::addStateTransition( 00057 'F', 'H', "Halt", "Halting", this, &RunControlStateMachine::transitionHalting); 00058 RunControlStateMachine::addStateTransition( 00059 'F', 00060 'X', 00061 "Shutdown", 00062 "Shutting Down", 00063 this, 00064 &RunControlStateMachine::transitionShuttingDown); 00065 00066 RunControlStateMachine::addStateTransition( 00067 'H', 00068 'C', 00069 "Configure", 00070 "Configuring", 00071 "ConfigurationAlias", 00072 this, 00073 &RunControlStateMachine::transitionConfiguring); 00074 RunControlStateMachine::addStateTransition( 00075 'H', 00076 'X', 00077 "Shutdown", 00078 "Shutting Down", 00079 this, 00080 &RunControlStateMachine::transitionShuttingDown); 00081 RunControlStateMachine::addStateTransition( 00082 'X', 00083 'I', 00084 "Startup", 00085 "Starting Up", 00086 this, 00087 &RunControlStateMachine::transitionStartingUp); 00088 00089 // Every state can transition to halted 00090 RunControlStateMachine::addStateTransition( 00091 'I', 00092 'H', 00093 "Initialize", 00094 "Initializing", 00095 this, 00096 &RunControlStateMachine::transitionInitializing); 00097 RunControlStateMachine::addStateTransition( 00098 'H', 'H', "Halt", "Halting", this, &RunControlStateMachine::transitionHalting); 00099 RunControlStateMachine::addStateTransition( 00100 'C', 'H', "Halt", "Halting", this, &RunControlStateMachine::transitionHalting); 00101 RunControlStateMachine::addStateTransition( 00102 'R', 'H', "Abort", "Aborting", this, &RunControlStateMachine::transitionHalting); 00103 RunControlStateMachine::addStateTransition( 00104 'P', 'H', "Abort", "Aborting", this, &RunControlStateMachine::transitionHalting); 00105 00106 RunControlStateMachine::addStateTransition( 00107 'R', 'P', "Pause", "Pausing", this, &RunControlStateMachine::transitionPausing); 00108 RunControlStateMachine::addStateTransition( 00109 'P', 00110 'R', 00111 "Resume", 00112 "Resuming", 00113 this, 00114 &RunControlStateMachine::transitionResuming); 00115 RunControlStateMachine::addStateTransition( 00116 'C', 'R', "Start", "Starting", this, &RunControlStateMachine::transitionStarting); 00117 RunControlStateMachine::addStateTransition( 00118 'R', 'C', "Stop", "Stopping", this, &RunControlStateMachine::transitionStopping); 00119 RunControlStateMachine::addStateTransition( 00120 'P', 'C', "Stop", "Stopping", this, &RunControlStateMachine::transitionStopping); 00121 00122 // NOTE!! There must be a defined message handler for each transition name created 00123 // above 00124 xoap::bind(this, 00125 &RunControlStateMachine::runControlMessageHandler, 00126 "Initialize", 00127 XDAQ_NS_URI); 00128 xoap::bind(this, 00129 &RunControlStateMachine::runControlMessageHandler, 00130 "Configure", 00131 XDAQ_NS_URI); 00132 xoap::bind( 00133 this, &RunControlStateMachine::runControlMessageHandler, "Start", XDAQ_NS_URI); 00134 xoap::bind( 00135 this, &RunControlStateMachine::runControlMessageHandler, "Stop", XDAQ_NS_URI); 00136 xoap::bind( 00137 this, &RunControlStateMachine::runControlMessageHandler, "Pause", XDAQ_NS_URI); 00138 xoap::bind( 00139 this, &RunControlStateMachine::runControlMessageHandler, "Resume", XDAQ_NS_URI); 00140 xoap::bind( 00141 this, &RunControlStateMachine::runControlMessageHandler, "Halt", XDAQ_NS_URI); 00142 xoap::bind( 00143 this, &RunControlStateMachine::runControlMessageHandler, "Abort", XDAQ_NS_URI); 00144 xoap::bind( 00145 this, &RunControlStateMachine::runControlMessageHandler, "Shutdown", XDAQ_NS_URI); 00146 xoap::bind( 00147 this, &RunControlStateMachine::runControlMessageHandler, "Startup", XDAQ_NS_URI); 00148 xoap::bind( 00149 this, &RunControlStateMachine::runControlMessageHandler, "Fail", XDAQ_NS_URI); 00150 xoap::bind( 00151 this, &RunControlStateMachine::runControlMessageHandler, "Error", XDAQ_NS_URI); 00152 00153 xoap::bind(this, 00154 &RunControlStateMachine::runControlMessageHandler, 00155 "AsyncError", 00156 XDAQ_NS_URI); 00157 xoap::bind(this, 00158 &RunControlStateMachine::runControlMessageHandler, 00159 "AsyncSoftError", 00160 XDAQ_NS_URI); 00161 00162 reset(); 00163 } 00164 00165 //======================================================================================================================== 00166 RunControlStateMachine::~RunControlStateMachine(void) {} 00167 00168 //======================================================================================================================== 00169 void RunControlStateMachine::reset(void) 00170 { 00171 __COUT__ << "Resetting RunControlStateMachine with name '" 00172 << theStateMachine_.getStateMachineName() << "'..." << __E__; 00173 theStateMachine_.setInitialState('I'); 00174 theStateMachine_.reset(); 00175 00176 theStateMachine_.setErrorMessage("", false /*append*/); // clear error message 00177 00178 asyncFailureReceived_ = false; 00179 asyncSoftFailureReceived_ = false; 00180 } 00181 00183 //(RunControlStateMachine::stateMachineFunction_t) 00184 // RunControlStateMachine::getTransitionName( const toolbox::fsm::State from, 00185 // const std::string& transition) 00186 //{ 00187 // auto itFrom = stateTransitionFunctionTable_.find(from); 00188 // if(itFrom == stateTransitionFunctionTable_.end()) 00189 // { 00190 // __SS__ << "Cannot find transition function from '" << from << 00191 // "' with transition '" << transition << "!'" << __E__; 00192 // XCEPT_RAISE (toolbox::fsm::exception::Exception, ss.str()); 00193 // } 00194 // 00195 // auto itTrans = itFrom->second.find(transition); 00196 // if(itTrans == itFrom->second.end()) 00197 // { 00198 // __SS__ << "Cannot find transition function from '" << from << 00199 // "' with transition '" << transition << "!'" << __E__; 00200 // XCEPT_RAISE (toolbox::fsm::exception::Exception, ss.str()); 00201 // } 00202 // 00203 // return itTrans->second; 00204 //} 00205 00206 //======================================================================================================================== 00207 // runControlMessageHandler 00208 // Handles the command broadcast message from the Gateway Supervisor 00209 // and maps the command to a transition function, allowing for multiple iteration 00210 // passes through the transition function. 00211 xoap::MessageReference RunControlStateMachine::runControlMessageHandler( 00212 xoap::MessageReference message) 00213 00214 { 00215 __COUT__ << "Received... \t" << SOAPUtilities::translate(message) << std::endl; 00216 00217 std::string command = SOAPUtilities::translate(message).getCommand(); 00218 00219 // get iteration index 00220 try 00221 { 00222 StringMacros::getNumber( 00223 SOAPUtilities::translate(message).getParameters().getValue("iterationIndex"), 00224 iterationIndex_); 00225 } 00226 catch(...) // ignore errors and set iteration index to 0 00227 { 00228 __COUT__ << "Defaulting iteration index to 0." << __E__; 00229 iterationIndex_ = 0; 00230 } 00231 // get subIteration index 00232 try 00233 { 00234 StringMacros::getNumber( 00235 SOAPUtilities::translate(message).getParameters().getValue( 00236 "subIterationIndex"), 00237 subIterationIndex_); 00238 } 00239 catch(...) // ignore errors and set subIteration index to 0 00240 { 00241 __COUT__ << "Defaulting subIterationIndex_ index to 0." << __E__; 00242 subIterationIndex_ = 0; 00243 } 00244 00245 // get retransmission indicator 00246 try 00247 { 00248 if(SOAPUtilities::translate(message).getParameters().getValue("retransmission") == 00249 "1") 00250 { 00251 // handle retransmission 00252 00253 // attempt to stop an error if last command was same 00254 if(lastIterationCommand_ == command && 00255 lastIterationIndex_ == iterationIndex_ && 00256 lastSubIterationIndex_ == subIterationIndex_) 00257 { 00258 __COUT__ 00259 << "Assuming a timeout occurred at Gateway waiting for a response. " 00260 << "Attempting to avoid error, by giving last result for command '" 00261 << command << "': " << lastIterationResult_ << __E__; 00262 return SOAPUtilities::makeSOAPMessageReference(lastIterationResult_); 00263 } 00264 else 00265 __COUT__ << "Looks like Gateway command '" << command 00266 << "' was lost - attempting to handle retransmission." << __E__; 00267 } 00268 } 00269 catch(...) // ignore errors for retransmission indicator (assume it is not a 00270 // retransmission) 00271 { 00272 ; 00273 } 00274 lastIterationIndex_ = iterationIndex_; 00275 lastSubIterationIndex_ = subIterationIndex_; 00276 00277 std::string currentState; 00278 if(iterationIndex_ == 0 && subIterationIndex_ == 0) 00279 { 00280 // this is the first iteration attempt for this transition 00281 theProgressBar_.reset(command, theStateMachine_.getStateMachineName()); 00282 currentState = theStateMachine_.getCurrentStateName(); 00283 __COUT__ << "Starting state for " << theStateMachine_.getStateMachineName() 00284 << " is " << currentState << " and attempting to " << command 00285 << std::endl; 00286 } 00287 else 00288 { 00289 currentState = theStateMachine_.getStateName(lastIterationState_); 00290 00291 __COUT__ << "Iteration index " << iterationIndex_ << "." << subIterationIndex_ 00292 << " for " << theStateMachine_.getStateMachineName() << " from " 00293 << currentState << " attempting to " << command << std::endl; 00294 } 00295 00296 RunControlStateMachine::theProgressBar_.step(); 00297 00298 std::string result = command + "Done"; 00299 lastIterationResult_ = result; 00300 00301 // if error is received, immediately go to fail state 00302 // likely error was sent by central FSM or external xoap 00303 if(command == "Error" || command == "Fail") 00304 { 00305 __SS__ << command << " was received! Halting immediately." << std::endl; 00306 __COUT_ERR__ << "\n" << ss.str(); 00307 00308 try 00309 { 00310 if(currentState == "Configured") 00311 theStateMachine_.execTransition("Halt", message); 00312 else if(currentState == "Running" || currentState == "Paused") 00313 theStateMachine_.execTransition("Abort", message); 00314 } 00315 catch(...) 00316 { 00317 __COUT_ERR__ << "Halting failed in reaction to " << command << "... ignoring." 00318 << __E__; 00319 } 00320 return SOAPUtilities::makeSOAPMessageReference(result); 00321 } 00322 else if(command == "AsyncError") 00323 { 00324 std::string errorMessage = 00325 SOAPUtilities::translate(message).getParameters().getValue("ErrorMessage"); 00326 00327 __SS__ << command << " was received! Error'ing immediately: " << errorMessage 00328 << std::endl; 00329 __COUT_ERR__ << "\n" << ss.str(); 00330 theStateMachine_.setErrorMessage(ss.str()); 00331 00332 asyncFailureReceived_ = true; // mark flag, to be used to abort next transition 00333 // determine any valid transition from where we are 00334 theStateMachine_.execTransition("fail"); 00335 // XCEPT_RAISE (toolbox::fsm::exception::Exception, ss.str()); 00336 00337 return SOAPUtilities::makeSOAPMessageReference(result); 00338 } 00339 else if(command == "AsyncSoftError") 00340 { 00341 std::string errorMessage = 00342 SOAPUtilities::translate(message).getParameters().getValue("ErrorMessage"); 00343 00344 __SS__ << command << " was received! Pause'ing immediately: " << errorMessage 00345 << std::endl; 00346 __COUT_ERR__ << "\n" << ss.str(); 00347 theStateMachine_.setErrorMessage(ss.str()); 00348 00349 if(!asyncSoftFailureReceived_) // launch pause only first time 00350 { 00351 asyncSoftFailureReceived_ = true; // mark flag, to be used to avoid double 00352 // pausing and identify pause was due to 00353 // soft error 00354 theStateMachine_.execTransition("Pause"); 00355 } 00356 00357 return SOAPUtilities::makeSOAPMessageReference(result); 00358 } 00359 00360 // if already Halted, respond to Initialize with "done" 00361 // (this avoids race conditions involved with artdaq mpi reset) 00362 if(command == "Initialize" && currentState == "Halted") 00363 { 00364 __COUT__ << "Already Initialized.. ignoring Initialize command." << std::endl; 00365 00366 theStateMachine_.setErrorMessage("", false /*append*/); // clear error message 00367 return SOAPUtilities::makeSOAPMessageReference(result); 00368 } 00369 00370 // handle normal transitions here 00371 try 00372 { 00373 theStateMachine_.setErrorMessage("", false /*append*/); // clear error message 00374 00375 iterationWorkFlag_ = false; 00376 subIterationWorkFlag_ = false; 00377 if(iterationIndex_ || subIterationIndex_) 00378 { 00379 __COUT__ << command << " iteration " << iterationIndex_ << "." 00380 << subIterationIndex_ << __E__; 00381 toolbox::Event::Reference event(new toolbox::Event(command, this)); 00382 00383 // call inheriting transition function based on last state and command 00384 { 00385 // e.g. transitionConfiguring(event); 00386 __COUT__ << "Iterating on the transition function from " << currentState 00387 << " through " << lastIterationCommand_ << __E__; 00388 00389 auto itFrom = stateTransitionFunctionTable_.find(lastIterationState_); 00390 if(itFrom == stateTransitionFunctionTable_.end()) 00391 { 00392 __SS__ << "Cannot find transition function from '" << currentState 00393 << "' with transition '" << lastIterationCommand_ << "!'" 00394 << __E__; 00395 __COUT_ERR__ << ss.str(); 00396 XCEPT_RAISE(toolbox::fsm::exception::Exception, ss.str()); 00397 } 00398 00399 auto itTransition = itFrom->second.find(lastIterationCommand_); 00400 if(itTransition == itFrom->second.end()) 00401 { 00402 __SS__ << "Cannot find transition function from '" << currentState 00403 << "' with transition '" << lastIterationCommand_ << "!'" 00404 << __E__; 00405 __COUT_ERR__ << ss.str(); 00406 XCEPT_RAISE(toolbox::fsm::exception::Exception, ss.str()); 00407 } 00408 00409 (this->*(itTransition->second))(event); // call the transition function 00410 } 00411 } 00412 else 00413 { 00414 // save the lookup parameters for the last function to be called for the case 00415 // of additional iterations 00416 lastIterationState_ = theStateMachine_.getCurrentState(); 00417 lastIterationCommand_ = command; 00418 00419 theStateMachine_.execTransition(command, message); 00420 } 00421 00422 if(subIterationWorkFlag_) // sub-iteration has priority over 'Working' 00423 { 00424 __COUTV__(subIterationWorkFlag_); 00425 result = 00426 command + "SubIterate"; // indicate another sub-iteration back to Gateway 00427 } 00428 else if(iterationWorkFlag_) 00429 { 00430 __COUTV__(iterationWorkFlag_); 00431 result = command + "Iterate"; // indicate another iteration back to Gateway 00432 } 00433 } 00434 catch(toolbox::fsm::exception::Exception& e) 00435 { 00436 __SS__ << "Run Control Message Handling Failed: " << e.what() << " " 00437 << theStateMachine_.getErrorMessage() << __E__; 00438 __COUT_ERR__ << ss.str(); 00439 theStateMachine_.setErrorMessage(ss.str()); 00440 00441 result = command + " " + RunControlStateMachine::FAILED_STATE_NAME + ": " + 00442 theStateMachine_.getErrorMessage(); 00443 } 00444 catch(...) 00445 { 00446 __SS__ << "Run Control Message Handling encountered an unknown error." 00447 << theStateMachine_.getErrorMessage() << __E__; 00448 __COUT_ERR__ << ss.str(); 00449 theStateMachine_.setErrorMessage(ss.str()); 00450 00451 result = command + " " + RunControlStateMachine::FAILED_STATE_NAME + ": " + 00452 theStateMachine_.getErrorMessage(); 00453 } 00454 00455 RunControlStateMachine::theProgressBar_.step(); 00456 00457 currentState = theStateMachine_.getCurrentStateName(); 00458 00459 if(currentState == RunControlStateMachine::FAILED_STATE_NAME) 00460 { 00461 result = command + " " + RunControlStateMachine::FAILED_STATE_NAME + ": " + 00462 theStateMachine_.getErrorMessage(); 00463 __COUT_ERR__ << "Unexpected Failure state for " 00464 << theStateMachine_.getStateMachineName() << " is " << currentState 00465 << std::endl; 00466 __COUT_ERR__ << "Error message was as follows: " 00467 << theStateMachine_.getErrorMessage() << std::endl; 00468 } 00469 00470 RunControlStateMachine::theProgressBar_.step(); 00471 00472 if(!iterationWorkFlag_) 00473 theProgressBar_.complete(); 00474 00475 __COUT__ << "Ending state for " << theStateMachine_.getStateMachineName() << " is " 00476 << currentState << std::endl; 00477 __COUT__ << "result = " << result << std::endl; 00478 lastIterationResult_ = result; 00479 return SOAPUtilities::makeSOAPMessageReference(result); 00480 }