00001 #include "otsdaq-core/GatewaySupervisor/Iterator.h"
00002 #include "otsdaq-core/MessageFacility/MessageFacility.h"
00003 #include "otsdaq-core/Macros/CoutHeaderMacros.h"
00004 #include "otsdaq-core/GatewaySupervisor/GatewaySupervisor.h"
00005 #include "otsdaq-core/CoreSupervisors/CoreSupervisorBase.h"
00006 #include "otsdaq-core/WebUsersUtilities/WebUsers.h"
00007
00008
00009 #include <iostream>
00010 #include <thread>
00011
00012 #undef __MF_SUBJECT__
00013 #define __MF_SUBJECT__ "Iterator"
00014
00015 using namespace ots;
00016
00017
00018 Iterator::Iterator(GatewaySupervisor* supervisor)
00019 : workloopRunning_ (false)
00020 , activePlanIsRunning_ (false)
00021 , iteratorBusy_ (false)
00022 , commandPlay_(false), commandPause_(false), commandHalt_(false)
00023 , activePlanName_ ("")
00024 , activeCommandIndex_ (-1)
00025 , activeCommandStartTime_ (0)
00026 , theSupervisor_ (supervisor)
00027 {
00028 __MOUT__ << "Iterator constructed." << __E__;
00029 __COUT__ << "Iterator constructed." << __E__;
00030
00031 }
00032
00033
00034 Iterator::~Iterator(void)
00035 {
00036 }
00037
00038
00039 void Iterator::IteratorWorkLoop(Iterator *iterator)
00040 try
00041 {
00042 __MOUT__ << "Iterator work loop starting..." << __E__;
00043 __COUT__ << "Iterator work loop starting..." << __E__;
00044
00045
00046 {
00047
00048
00049 if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for iterator access" << __E__;
00050 std::lock_guard<std::mutex> lock(iterator->accessMutex_);
00051 if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have iterator access" << __E__;
00052
00053 iterator->errorMessage_ = "";
00054 }
00055
00056
00057 ConfigurationManagerRW theConfigurationManager(WebUsers::DEFAULT_ITERATOR_USERNAME);
00058
00059 theConfigurationManager.getAllConfigurationInfo(true);
00060
00061 IteratorWorkLoopStruct theIteratorStruct(iterator,
00062 &theConfigurationManager);
00063
00064
00065 const IterateConfiguration* itConfig;
00066
00067 std::vector<IterateConfiguration::Command> commands;
00068
00069 while(1)
00070 {
00071
00072
00073
00074
00075
00076
00077
00078
00079
00080
00081
00082
00083
00084
00085
00086 {
00087
00088
00089 if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for iterator access" << __E__;
00090 std::lock_guard<std::mutex> lock(iterator->accessMutex_);
00091 if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have iterator access" << __E__;
00092
00093 if(iterator->commandPlay_)
00094 {
00095 iterator->commandPlay_ = false;
00096
00097 if(!iterator->activePlanIsRunning_)
00098 {
00099
00100
00101 iterator->activePlanIsRunning_ = true;
00102 iterator->iteratorBusy_ = true;
00103
00104 if(theIteratorStruct.activePlan_ != iterator->activePlanName_)
00105 {
00106 __COUT__ << "New plan name encountered old=" << theIteratorStruct.activePlan_ <<
00107 " vs new=" << iterator->activePlanName_ << __E__;
00108 theIteratorStruct.commandIndex_ = -1;
00109 }
00110
00111 theIteratorStruct.activePlan_ = iterator->activePlanName_;
00112 iterator->lastStartedPlanName_ = iterator->activePlanName_;
00113
00114 if(theIteratorStruct.commandIndex_ == (unsigned int)-1)
00115 {
00116 __COUT__ << "Starting plan '" << theIteratorStruct.activePlan_ << ".'" << __E__;
00117 __MOUT__ << "Starting plan '" << theIteratorStruct.activePlan_ << ".'" << __E__;
00118 }
00119 else
00120 {
00121 theIteratorStruct.doResumeAction_ = true;
00122 __COUT__ << "Continuing plan '" << theIteratorStruct.activePlan_ << "' at command index " <<
00123 theIteratorStruct.commandIndex_ << ". " << __E__;
00124 __MOUT__ << "Continuing plan '" << theIteratorStruct.activePlan_ << "' at command index " <<
00125 theIteratorStruct.commandIndex_ << ". " << __E__;
00126 }
00127 }
00128 }
00129 else if(iterator->commandPause_ && !theIteratorStruct.doPauseAction_)
00130 {
00131 theIteratorStruct.doPauseAction_ = true;
00132 iterator->commandPause_ = false;
00133 }
00134 else if(iterator->commandHalt_ && !theIteratorStruct.doHaltAction_)
00135 {
00136 theIteratorStruct.doHaltAction_ = true;
00137 iterator->commandHalt_ = false;
00138 }
00139
00140 theIteratorStruct.running_ = iterator->activePlanIsRunning_;
00141
00142 if(iterator->activeCommandIndex_ !=
00143 theIteratorStruct.commandIndex_)
00144 {
00145 iterator->activeCommandIndex_ = theIteratorStruct.commandIndex_;
00146 iterator->activeCommandStartTime_ = time(0);
00147
00148 if(theIteratorStruct.commandIndex_ < theIteratorStruct.commandIterations_.size())
00149 iterator->activeCommandIteration_ = theIteratorStruct.commandIterations_
00150 [theIteratorStruct.commandIndex_];
00151 else
00152 iterator->activeCommandIteration_ = -1;
00153
00154 iterator->depthIterationStack_.clear();
00155 for(const auto& depthIteration:theIteratorStruct.stepIndexStack_)
00156 iterator->depthIterationStack_.push_back(depthIteration);
00157
00158
00159
00160
00161 }
00162
00163 }
00164
00165
00168
00169
00170 if(theIteratorStruct.doPauseAction_)
00171 {
00172
00173
00174
00175
00176
00177 __COUT__ << "Waiting to pause..." << __E__;
00178 while(!iterator->checkCommand(&theIteratorStruct))
00179 __COUT__ << "Waiting to pause..." << __E__;
00180
00181 __COUT__ << "Completing pause..." << __E__;
00182
00183 theIteratorStruct.doPauseAction_ = false;
00184
00185
00186
00187 if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for iterator access" << __E__;
00188 std::lock_guard<std::mutex> lock(iterator->accessMutex_);
00189 if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have iterator access" << __E__;
00190
00191 iterator->activePlanIsRunning_ = false;
00192
00193 __COUT__ << "Paused plan '" << theIteratorStruct.activePlan_ << "' at command index " <<
00194 theIteratorStruct.commandIndex_ << ". " << __E__;
00195 __MOUT__ << "Paused plan '" << theIteratorStruct.activePlan_ << "' at command index " <<
00196 theIteratorStruct.commandIndex_ << ". " << __E__;
00197
00198 continue;
00199 }
00200 else if(theIteratorStruct.doHaltAction_)
00201 {
00202
00203
00204
00205
00206
00207 __COUT__ << "Waiting to halt..." << __E__;
00208 while(!iterator->checkCommand(&theIteratorStruct))
00209 __COUT__ << "Waiting to halt..." << __E__;
00210
00211 __COUT__ << "Completing halt..." << __E__;
00212
00213 theIteratorStruct.doHaltAction_ = false;
00214
00215 iterator->haltIterator(iterator,&theIteratorStruct);
00216
00217
00218
00219
00220
00221
00222
00223
00224
00225
00226
00227
00228
00229
00230
00231
00232
00233
00234
00235
00236
00237
00238
00239 continue;
00240 }
00241
00242
00245
00246
00247
00248
00249 if(theIteratorStruct.running_ &&
00250 theIteratorStruct.activePlan_ != "")
00251 {
00252 if(theIteratorStruct.commandIndex_ == (unsigned int)-1)
00253 {
00254
00255
00256 __COUT__ << "Get commands" << __E__;
00257
00258 theIteratorStruct.commandIndex_ = 0;
00259
00260 theIteratorStruct.cfgMgr_->init();
00261 itConfig = theIteratorStruct.cfgMgr_->__GET_CONFIG__(IterateConfiguration);
00262
00263 theIteratorStruct.commands_ = itConfig->getPlanCommands(
00264 theIteratorStruct.cfgMgr_,
00265 theIteratorStruct.activePlan_);
00266
00267
00268 theIteratorStruct.commandIterations_.clear();
00269 for(auto& command:theIteratorStruct.commands_)
00270 {
00271 theIteratorStruct.commandIterations_.push_back(0);
00272 __COUT__ << "command " <<
00273 command.type_ << __E__;
00274 __COUT__ << "table " <<
00275 IterateConfiguration::commandToTableMap_.at(command.type_) << __E__;
00276 __COUT__ << "param count = " << command.params_.size() << __E__;
00277
00278 for(auto& param:command.params_)
00279 {
00280 __COUT__ << "\t param " <<
00281 param.first << " : " <<
00282 param.second << __E__;
00283 }
00284 }
00285
00286
00287 theIteratorStruct.originalTrackChanges_ =
00288 ConfigurationInterface::isVersionTrackingEnabled();
00289 theIteratorStruct.originalConfigGroup_ =
00290 theIteratorStruct.cfgMgr_->getActiveGroupName();
00291 theIteratorStruct.originalConfigKey_ =
00292 theIteratorStruct.cfgMgr_->getActiveGroupKey();
00293
00294 __COUT__ << "originalTrackChanges " <<
00295 theIteratorStruct.originalTrackChanges_ << __E__;
00296 __COUT__ << "originalConfigGroup " <<
00297 theIteratorStruct.originalConfigGroup_ << __E__;
00298 __COUT__ << "originalConfigKey " <<
00299 theIteratorStruct.originalConfigKey_ << __E__;
00300
00301 }
00302
00303
00304 if(!theIteratorStruct.commandBusy_)
00305 {
00306 if(theIteratorStruct.commandIndex_ < theIteratorStruct.commands_.size())
00307 {
00308
00309 theIteratorStruct.commandBusy_ = true;
00310
00311 __COUT__ << "Iterator starting command " << theIteratorStruct.commandIndex_+1 << ": " <<
00312 theIteratorStruct.commands_[theIteratorStruct.commandIndex_].type_ << __E__;
00313 __MOUT__ << "Iterator starting command " << theIteratorStruct.commandIndex_+1 << ": " <<
00314 theIteratorStruct.commands_[theIteratorStruct.commandIndex_].type_ << __E__;
00315
00316 iterator->startCommand(&theIteratorStruct);
00317 }
00318 else if(theIteratorStruct.commandIndex_ ==
00319 theIteratorStruct.commands_.size())
00320 {
00321 __COUT__ << "Finished Iteration Plan '" << theIteratorStruct.activePlan_ << __E__;
00322 __MOUT__ << "Finished Iteration Plan '" << theIteratorStruct.activePlan_ << __E__;
00323
00324 __COUT__ << "Reverting track changes." << __E__;
00325 ConfigurationInterface::setVersionTrackingEnabled(theIteratorStruct.originalTrackChanges_);
00326
00327 __COUT__ << "Activating original group..." << __E__;
00328 try
00329 {
00330 theIteratorStruct.cfgMgr_->activateConfigurationGroup(
00331 theIteratorStruct.originalConfigGroup_,theIteratorStruct.originalConfigKey_);
00332 }
00333 catch(...)
00334 {
00335 __COUT_WARN__ << "Original group could not be activated." << __E__;
00336 }
00337
00338
00339 __COUT__ << "Completing halt..." << __E__;
00340
00341 iterator->haltIterator(iterator,&theIteratorStruct);
00342
00343
00344
00345
00346
00347
00348
00349
00350
00351
00352
00353
00354
00355
00356
00357
00358
00359
00360
00361
00362 }
00363 }
00364 else if(theIteratorStruct.commandBusy_)
00365 {
00366
00367 if(iterator->checkCommand(&theIteratorStruct))
00368 {
00369 theIteratorStruct.commandBusy_ = false;
00370
00371 ++theIteratorStruct.commandIndex_;
00372
00373 __COUT__ << "Ready for next command. Done with " << theIteratorStruct.commandIndex_ << " of " <<
00374 theIteratorStruct.commands_.size() << __E__;
00375 __MOUT__ << "Iterator ready for next command. Done with " << theIteratorStruct.commandIndex_ << " of " <<
00376 theIteratorStruct.commands_.size() << __E__;
00377 }
00378
00379
00380 if(theIteratorStruct.doResumeAction_)
00381 theIteratorStruct.doResumeAction_ = false;
00382 }
00383
00384
00385
00386 }
00387 else
00388 sleep(1);
00389
00392
00393
00394
00395
00396
00397 }
00398
00399 iterator->workloopRunning_ = false;
00400 }
00401 catch(const std::runtime_error &e)
00402 {
00403 __SS__ << "Encountered error in Iterator thread:\n" << e.what() << __E__;
00404 __COUT_ERR__ << ss.str();
00405
00406
00407
00408 if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for iterator access" << __E__;
00409 std::lock_guard<std::mutex> lock(iterator->accessMutex_);
00410 if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have iterator access" << __E__;
00411
00412 iterator->workloopRunning_ = false;
00413 iterator->errorMessage_ = ss.str();
00414
00415 }
00416 catch(...)
00417 {
00418 __SS__ << "Encountered unknown error in Iterator thread." << __E__;
00419 __COUT_ERR__ << ss.str();
00420
00421
00422
00423 if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for iterator access" << __E__;
00424 std::lock_guard<std::mutex> lock(iterator->accessMutex_);
00425 if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have iterator access" << __E__;
00426
00427 iterator->workloopRunning_ = false;
00428 iterator->errorMessage_ = ss.str();
00429 }
00430
00431
00432 void Iterator::startCommand(IteratorWorkLoopStruct *iteratorStruct)
00433 try
00434 {
00435
00436 {
00437 int i =0;
00438 for(const auto& depthIteration:iteratorStruct->stepIndexStack_)
00439 {
00440 __COUT__ << i++ << ":" << depthIteration << __E__;
00441
00442 }
00443 }
00444
00445
00446
00447
00448
00449 if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for FSM access" << __E__;
00450 std::lock_guard<std::mutex> lock(iteratorStruct->theIterator_->theSupervisor_->stateMachineAccessMutex_);
00451 if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have FSM access" << __E__;
00452
00453
00454
00455 if(iteratorStruct->commandIndex_ >= iteratorStruct->commands_.size())
00456 {
00457 __SS__ << "Out of range commandIndex = " << iteratorStruct->commandIndex_ <<
00458 " in size = " << iteratorStruct->commands_.size() << __E__;
00459 throw std::runtime_error(ss.str());
00460 }
00461
00462
00463 ++iteratorStruct->commandIterations_[iteratorStruct->commandIndex_];
00464
00465
00466 std::string type = iteratorStruct->commands_[iteratorStruct->commandIndex_].type_;
00467 if(type == IterateConfiguration::COMMAND_BEGIN_LABEL)
00468 {
00469 return startCommandBeginLabel(iteratorStruct);
00470 }
00471 else if(type == IterateConfiguration::COMMAND_CHOOSE_FSM)
00472 {
00473 return startCommandChooseFSM(
00474 iteratorStruct,
00475 iteratorStruct->commands_
00476 [iteratorStruct->commandIndex_].params_
00477 [IterateConfiguration::commandChooseFSMParams_.NameOfFSM_]);
00478 }
00479 else if(type == IterateConfiguration::COMMAND_CONFIGURE_ACTIVE_GROUP)
00480 {
00481 return startCommandConfigureActive(iteratorStruct);
00482 }
00483 else if(type == IterateConfiguration::COMMAND_CONFIGURE_ALIAS)
00484 {
00485 return startCommandConfigureAlias(
00486 iteratorStruct,
00487 iteratorStruct->commands_
00488 [iteratorStruct->commandIndex_].params_
00489 [IterateConfiguration::commandConfigureAliasParams_.SystemAlias_]);
00490 }
00491 else if(type == IterateConfiguration::COMMAND_CONFIGURE_GROUP)
00492 {
00493 return startCommandConfigureGroup(iteratorStruct);
00494 }
00495 else if(type == IterateConfiguration::COMMAND_EXECUTE_FE_MACRO)
00496 {
00497
00498 return;
00499 }
00500 else if(type == IterateConfiguration::COMMAND_EXECUTE_MACRO)
00501 {
00502
00503 return;
00504 }
00505 else if(type == IterateConfiguration::COMMAND_MODIFY_ACTIVE_GROUP)
00506 {
00507 return startCommandModifyActive(iteratorStruct);
00508 }
00509 else if(type == IterateConfiguration::COMMAND_REPEAT_LABEL)
00510 {
00511 return startCommandRepeatLabel(iteratorStruct);
00512 }
00513 else if(type == IterateConfiguration::COMMAND_RUN)
00514 {
00515 return startCommandRun(iteratorStruct);
00516 }
00517 else
00518 {
00519 __SS__ << "Attempt to start unrecognized command type = " << type << __E__;
00520 __COUT_ERR__ << ss.str();
00521 throw std::runtime_error(ss.str());
00522 }
00523 }
00524 catch(...)
00525 {
00526 __COUT__ << "Error caught. Reverting track changes." << __E__;
00527 ConfigurationInterface::setVersionTrackingEnabled(iteratorStruct->originalTrackChanges_);
00528
00529 __COUT__ << "Activating original group..." << __E__;
00530 try
00531 {
00532 iteratorStruct->cfgMgr_->activateConfigurationGroup(
00533 iteratorStruct->originalConfigGroup_,iteratorStruct->originalConfigKey_);
00534 }
00535 catch(...)
00536 {
00537 __COUT_WARN__ << "Original group could not be activated." << __E__;
00538 }
00539 throw;
00540 }
00541
00542
00543
00544
00545
00546
00547 bool Iterator::checkCommand(IteratorWorkLoopStruct *iteratorStruct)
00548 try
00549 {
00550
00551 if(iteratorStruct->commandIndex_ >= iteratorStruct->commands_.size())
00552 {
00553 __COUT__ << "Out of range commandIndex = " << iteratorStruct->commandIndex_ <<
00554 " in size = " << iteratorStruct->commands_.size() << __E__;
00555 return true;
00556 }
00557
00558 std::string type = iteratorStruct->commands_[iteratorStruct->commandIndex_].type_;
00559 if(type == IterateConfiguration::COMMAND_BEGIN_LABEL)
00560 {
00561
00562 return true;
00563 }
00564 else if(type == IterateConfiguration::COMMAND_CHOOSE_FSM)
00565 {
00566
00567 return true;
00568 }
00569 else if(type == IterateConfiguration::COMMAND_CONFIGURE_ALIAS ||
00570 type == IterateConfiguration::COMMAND_CONFIGURE_ACTIVE_GROUP ||
00571 type == IterateConfiguration::COMMAND_CONFIGURE_GROUP)
00572 {
00573 return checkCommandConfigure(iteratorStruct);
00574 }
00575 else if(type == IterateConfiguration::COMMAND_EXECUTE_FE_MACRO)
00576 {
00577
00578 return true;
00579 }
00580 else if(type == IterateConfiguration::COMMAND_EXECUTE_MACRO)
00581 {
00582
00583 return true;
00584 }
00585 else if(type == IterateConfiguration::COMMAND_MODIFY_ACTIVE_GROUP)
00586 {
00587
00588 return true;
00589 }
00590 else if(type == IterateConfiguration::COMMAND_REPEAT_LABEL)
00591 {
00592
00593 return true;
00594 }
00595 else if(type == IterateConfiguration::COMMAND_RUN)
00596 {
00597 return checkCommandRun(iteratorStruct);
00598 }
00599 else
00600 {
00601 __SS__ << "Attempt to check unrecognized command type = " << type << __E__;
00602 __COUT_ERR__ << ss.str();
00603 throw std::runtime_error(ss.str());
00604 }
00605 }
00606 catch(...)
00607 {
00608 __COUT__ << "Error caught. Reverting track changes." << __E__;
00609 ConfigurationInterface::setVersionTrackingEnabled(iteratorStruct->originalTrackChanges_);
00610
00611 __COUT__ << "Activating original group..." << __E__;
00612 try
00613 {
00614 iteratorStruct->cfgMgr_->activateConfigurationGroup(
00615 iteratorStruct->originalConfigGroup_,iteratorStruct->originalConfigKey_);
00616 }
00617 catch(...)
00618 {
00619 __COUT_WARN__ << "Original group could not be activated." << __E__;
00620 }
00621
00622 throw;
00623 }
00624
00625
00626 void Iterator::startCommandChooseFSM(IteratorWorkLoopStruct *iteratorStruct,
00627 const std::string& fsmName)
00628 {
00629 __COUT__ << "fsmName " << fsmName << __E__;
00630
00631
00632 iteratorStruct->fsmName_ = fsmName;
00633 iteratorStruct->theIterator_->lastFsmName_ = fsmName;
00634
00635
00636
00637
00638
00639
00640
00641
00643
00644 iteratorStruct->fsmRunAlias_ = "Run";
00645
00646
00647 ConfigurationTree configLinkNode = iteratorStruct->cfgMgr_->getSupervisorConfigurationNode(
00648 iteratorStruct->theIterator_->theSupervisor_->supervisorContextUID_,
00649 iteratorStruct->theIterator_->theSupervisor_->supervisorApplicationUID_);
00650
00651 if(!configLinkNode.isDisconnected())
00652 {
00653 try
00654 {
00655 ConfigurationTree fsmLinkNode = configLinkNode.getNode("LinkToStateMachineConfiguration");
00656 if(!fsmLinkNode.isDisconnected())
00657 iteratorStruct->fsmRunAlias_ =
00658 fsmLinkNode.getNode(fsmName + "/RunDisplayAlias").getValue<std::string>();
00659 else
00660 __COUT_INFO__ << "FSM Link disconnected." << __E__;
00661 }
00662 catch(std::runtime_error &e)
00663 {
00664
00665 __COUT_INFO__ << "No state machine Run alias. Ignoring and assuming alias of '" <<
00666 iteratorStruct->fsmRunAlias_ << ".'" << __E__;
00667
00668 }
00669 catch(...) {
00670 __COUT_ERR__ << "Unknown error. Should never happen." << __E__;
00671
00672 __COUT_INFO__ << "No state machine Run alias. Ignoring and assuming alias of '" <<
00673 iteratorStruct->fsmRunAlias_ << ".'" << __E__;
00674 }
00675 }
00676 else
00677 __COUT_INFO__ << "FSM Link disconnected." << __E__;
00678
00679 __COUT__ << "fsmRunAlias_ = " << iteratorStruct->fsmRunAlias_ << __E__;
00680
00681
00682
00683
00685
00686 iteratorStruct->fsmNextRunNumber_ = iteratorStruct->theIterator_->theSupervisor_->getNextRunNumber(
00687 iteratorStruct->fsmName_);
00688
00689 if(iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getCurrentStateName() == "Running" ||
00690 iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getCurrentStateName() == "Paused")
00691 --iteratorStruct->fsmNextRunNumber_;
00692
00693 __COUT__ << "fsmNextRunNumber_ = " << iteratorStruct->fsmNextRunNumber_ << __E__;
00694 }
00695
00696
00697
00698 bool Iterator::haltIterator(Iterator *iterator,
00699 IteratorWorkLoopStruct *iteratorStruct)
00700
00701 {
00702 GatewaySupervisor* theSupervisor = iterator->theSupervisor_;
00703 const std::string& fsmName = iterator->lastFsmName_;
00704
00705 std::vector<std::string> fsmCommandParameters;
00706 std::string errorStr = "";
00707 std::string currentState = theSupervisor->theStateMachine_.getCurrentStateName();
00708
00709 bool haltAttempted = true;
00710 if(currentState == "Initialized" ||
00711 currentState == "Halted")
00712 {
00713 __COUT__ << "Do nothing. Already halted." << __E__;
00714 haltAttempted = false;
00715 }
00716 else if(currentState == "Running")
00717 errorStr = theSupervisor->attemptStateMachineTransition(
00718 0,0,
00719 "Abort",fsmName,
00720 WebUsers::DEFAULT_ITERATOR_USERNAME ,
00721 WebUsers::DEFAULT_ITERATOR_USERNAME,
00722 fsmCommandParameters);
00723 else
00724 errorStr = theSupervisor->attemptStateMachineTransition(
00725 0,0,
00726 "Halt",fsmName,
00727 WebUsers::DEFAULT_ITERATOR_USERNAME ,
00728 WebUsers::DEFAULT_ITERATOR_USERNAME,
00729 fsmCommandParameters);
00730
00731 if(haltAttempted)
00732 {
00733 if(errorStr != "")
00734
00735 {
00736 __SS__ << "Iterator failed to halt because of the following error: " << errorStr;
00737 throw std::runtime_error(ss.str());
00738 }
00739
00740
00741 __COUT__ << "FSM in transition = " << theSupervisor->theStateMachine_.isInTransition() << __E__;
00742 __COUT__ << "halting state machine launched." << __E__;
00743 }
00744
00745
00746
00747
00748 __COUT__ << "Conducting Iterator halt." << __E__;
00749
00750
00751 if(iteratorStruct)
00752 {
00753 __COUT__ << "Reverting track changes." << __E__;
00754 ConfigurationInterface::setVersionTrackingEnabled(iteratorStruct->originalTrackChanges_);
00755
00756 __COUT__ << "Activating original group..." << __E__;
00757 try
00758 {
00759 iteratorStruct->cfgMgr_->activateConfigurationGroup(
00760 iteratorStruct->originalConfigGroup_,iteratorStruct->originalConfigKey_);
00761 }
00762 catch(...)
00763 {
00764 __COUT_WARN__ << "Original group could not be activated." << __E__;
00765 }
00766
00767 }
00768
00769
00770
00771 if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for iterator access" << __E__;
00772 std::lock_guard<std::mutex> lock(iterator->accessMutex_);
00773 if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have iterator access" << __E__;
00774
00775
00776
00777 iterator->activePlanIsRunning_ = false;
00778 iterator->iteratorBusy_ = false;
00779
00780
00781 iterator->activePlanName_ = "";
00782 iterator->activeCommandIndex_ = -1;
00783
00784 if(iteratorStruct)
00785 {
00786
00787 __COUT__ << "Halted plan '" << iteratorStruct->activePlan_ << "' at command index " <<
00788 iteratorStruct->commandIndex_ << ". " << __E__;
00789 __MOUT__ << "Halted plan '" << iteratorStruct->activePlan_ << "' at command index " <<
00790 iteratorStruct->commandIndex_ << ". " << __E__;
00791
00792 iterator->lastStartedPlanName_ = iteratorStruct->activePlan_;
00793 iteratorStruct->activePlan_ = "";
00794 iteratorStruct->commandIndex_ = -1;
00795 }
00796
00797 return haltAttempted;
00798 }
00799
00800
00801 void Iterator::startCommandBeginLabel(IteratorWorkLoopStruct *iteratorStruct)
00802 {
00803 __COUT__ << "Entering label '" <<
00804 iteratorStruct->commands_
00805 [iteratorStruct->commandIndex_].params_
00806 [IterateConfiguration::commandBeginLabelParams_.Label_]
00807 << "'..." << std::endl;
00808
00809
00810 iteratorStruct->stepIndexStack_.push_back(0);
00811 }
00812
00813
00814
00815 void Iterator::startCommandRepeatLabel(IteratorWorkLoopStruct *iteratorStruct)
00816 {
00817
00818
00819 int numOfRepetitions;
00820 sscanf(iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
00821 [IterateConfiguration::commandRepeatLabelParams_.NumberOfRepetitions_].c_str(),
00822 "%d",&numOfRepetitions);
00823 __COUT__ << "numOfRepetitions remaining = " << numOfRepetitions << __E__;
00824
00825 char repStr[200];
00826
00827 if(numOfRepetitions <= 0)
00828 {
00829
00830 sprintf(repStr,"%d",iteratorStruct->stepIndexStack_.back());
00831 iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
00832 [IterateConfiguration::commandRepeatLabelParams_.NumberOfRepetitions_] =
00833 repStr;
00834
00835
00836 iteratorStruct->stepIndexStack_.pop_back();
00837
00838
00839 return;
00840 }
00841
00842 --numOfRepetitions;
00843
00844
00845 ++(iteratorStruct->stepIndexStack_.back());
00846
00847 unsigned int i;
00848 for(i=iteratorStruct->commandIndex_;i>0;--i)
00849 if(iteratorStruct->commands_[i].type_ == IterateConfiguration::COMMAND_BEGIN_LABEL &&
00850 iteratorStruct->commands_[iteratorStruct->commandIndex_].params_[IterateConfiguration::commandRepeatLabelParams_.Label_] ==
00851 iteratorStruct->commands_[i].params_[IterateConfiguration::commandBeginLabelParams_.Label_]) break;
00852
00853 sprintf(repStr,"%d",numOfRepetitions);
00854 iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
00855 [IterateConfiguration::commandRepeatLabelParams_.NumberOfRepetitions_] =
00856 repStr;
00857
00858 iteratorStruct->commandIndex_ = i;
00859 __COUT__ << "Jumping back to commandIndex " << iteratorStruct->commandIndex_ << __E__;
00860 }
00861
00862
00863 void Iterator::startCommandRun(IteratorWorkLoopStruct *iteratorStruct)
00864 {
00865 iteratorStruct->runIsDone_ = false;
00866 iteratorStruct->fsmCommandParameters_.clear();
00867
00868 std::string errorStr = "";
00869 std::string currentState = iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getCurrentStateName();
00870
00871
00872
00873 if(currentState == "Configured")
00874 errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
00875 0,0,
00876 "Start",iteratorStruct->fsmName_,
00877 WebUsers::DEFAULT_ITERATOR_USERNAME ,
00878 WebUsers::DEFAULT_ITERATOR_USERNAME,
00879 iteratorStruct->fsmCommandParameters_);
00880 else
00881 errorStr = "Can only Run from the Configured state. The current state is " +
00882 currentState;
00883
00884
00885 if(errorStr != "")
00886 {
00887 __SS__ << "Iterator failed to run because of the following error: " << errorStr;
00888 throw std::runtime_error(ss.str());
00889 }
00890
00891
00892 __COUT__ << "FSM in transition = " << iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.isInTransition() << __E__;
00893 __COUT__ << "startCommandRun success." << __E__;
00894 }
00895
00896
00897 void Iterator::startCommandConfigureActive(IteratorWorkLoopStruct *iteratorStruct)
00898 {
00899 __COUT__ << "startCommandConfigureActive " << __E__;
00900
00901
00902
00903
00904
00905 std::string group = iteratorStruct->cfgMgr_->getActiveGroupName();
00906 ConfigurationGroupKey key = iteratorStruct->cfgMgr_->getActiveGroupKey();
00907
00908 __COUT__ << "group " << group << __E__;
00909 __COUT__ << "key " << key << __E__;
00910
00911
00912
00913 std::stringstream systemAlias;
00914 systemAlias << "GROUP:" << group << ":" << key;
00915 startCommandConfigureAlias(iteratorStruct,systemAlias.str());
00916 }
00917
00918
00919 void Iterator::startCommandConfigureGroup(IteratorWorkLoopStruct *iteratorStruct)
00920 {
00921 __COUT__ << "startCommandConfigureGroup " << __E__;
00922
00923
00924
00925
00926 std::string group =
00927 iteratorStruct->commands_
00928 [iteratorStruct->commandIndex_].params_
00929 [IterateConfiguration::commandConfigureGroupParams_.GroupName_];
00930 ConfigurationGroupKey key = ConfigurationGroupKey(
00931 iteratorStruct->commands_
00932 [iteratorStruct->commandIndex_].params_
00933 [IterateConfiguration::commandConfigureGroupParams_.GroupKey_])
00934 ;
00935
00936 __COUT__ << "group " << group << __E__;
00937 __COUT__ << "key " << key << __E__;
00938
00939
00940
00941 std::stringstream systemAlias;
00942 systemAlias << "GROUP:" << group << ":" << key;
00943 startCommandConfigureAlias(iteratorStruct,systemAlias.str());
00944 }
00945
00946
00947 void Iterator::startCommandConfigureAlias(IteratorWorkLoopStruct *iteratorStruct,
00948 const std::string& systemAlias)
00949 {
00950 __COUT__ << "systemAlias " << systemAlias << __E__;
00951
00952 iteratorStruct->fsmCommandParameters_.clear();
00953 iteratorStruct->fsmCommandParameters_.push_back(systemAlias);
00954
00955 std::string errorStr = "";
00956 std::string currentState = iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getCurrentStateName();
00957
00958
00959
00960 if(currentState == "Initial")
00961 errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
00962 0,0,
00963 "Initialize",iteratorStruct->fsmName_,
00964 WebUsers::DEFAULT_ITERATOR_USERNAME ,
00965 WebUsers::DEFAULT_ITERATOR_USERNAME,
00966 iteratorStruct->fsmCommandParameters_);
00967 else if(currentState == "Halted")
00968 errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
00969 0,0,
00970 "Configure",iteratorStruct->fsmName_,
00971 WebUsers::DEFAULT_ITERATOR_USERNAME ,
00972 WebUsers::DEFAULT_ITERATOR_USERNAME,
00973 iteratorStruct->fsmCommandParameters_);
00974 else if(currentState == "Configured")
00975 errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
00976 0,0,
00977 "Halt",iteratorStruct->fsmName_,
00978 WebUsers::DEFAULT_ITERATOR_USERNAME ,
00979 WebUsers::DEFAULT_ITERATOR_USERNAME,
00980 iteratorStruct->fsmCommandParameters_);
00981 else
00982 errorStr = "Can only Configure from the Initial or Halted state. The current state is " +
00983 currentState;
00984
00985
00986
00987 if(errorStr != "")
00988 {
00989 __SS__ << "Iterator failed to configure with system alias '" <<
00990 (iteratorStruct->fsmCommandParameters_.size()?
00991 iteratorStruct->fsmCommandParameters_[0]:"UNKNOWN") <<
00992 "' because of the following error: " << errorStr;
00993 throw std::runtime_error(ss.str());
00994 }
00995
00996
00997 __COUT__ << "FSM in transition = " << iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.isInTransition() << __E__;
00998 __COUT__ << "startCommandConfigureAlias success." << __E__;
00999 }
01000
01001
01002 void Iterator::startCommandModifyActive(IteratorWorkLoopStruct *iteratorStruct)
01003 {
01004
01005
01006
01007
01008
01009
01010
01011
01012
01013
01014
01015
01016
01017
01018
01019
01020
01021
01022
01023 bool doTrackGroupChanges = false;
01024 if("True" == iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
01025 [IterateConfiguration::commandModifyActiveParams_.DoTrackGroupChanges_])
01026 doTrackGroupChanges = true;
01027
01028 const std::string& startValueStr =
01029 iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
01030 [IterateConfiguration::commandModifyActiveParams_.FieldStartValue_];
01031 const std::string& stepSizeStr =
01032 iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
01033 [IterateConfiguration::commandModifyActiveParams_.FieldIterationStepSize_];
01034
01035 const unsigned int stepIndex = iteratorStruct->stepIndexStack_.back();
01036
01037
01038 __COUT__ << "doTrackGroupChanges " << (doTrackGroupChanges?"yes":"no") << std::endl;
01039 __COUT__ << "stepIndex " << stepIndex << std::endl;
01040
01041 ConfigurationInterface::setVersionTrackingEnabled(doTrackGroupChanges);
01042
01043
01044
01045 if(startValueStr.size() &&
01046 (startValueStr[startValueStr.size()-1] == 'f' ||
01047 startValueStr.find('.') != std::string::npos))
01048 {
01049
01050 double startValue = strtod(startValueStr.c_str(),0);
01051 double stepSize = strtod(stepSizeStr.c_str(),0);
01052
01053 __COUT__ << "startValue " << startValue << std::endl;
01054 __COUT__ << "stepSize " << stepSize << std::endl;
01055 __COUT__ << "currentValue " << startValue + stepSize*stepIndex << std::endl;
01056
01057 helpCommandModifyActive(
01058 iteratorStruct,
01059 startValue + stepSize*stepIndex,
01060 doTrackGroupChanges);
01061 }
01062 else
01063 {
01064 long int startValue;
01065
01066 if(startValueStr.size() > 2 && startValueStr[1] == 'x')
01067 startValue = strtol(startValueStr.c_str(),0,16);
01068 else if(startValueStr.size() > 1 && startValueStr[0] == 'b')
01069 startValue = strtol(startValueStr.substr(1).c_str(),0,2);
01070 else
01071 startValue = strtol(startValueStr.c_str(),0,10);
01072
01073 long int stepSize;
01074
01075 if(stepSizeStr.size() > 2 && stepSizeStr[1] == 'x')
01076 stepSize = strtol(stepSizeStr.c_str(),0,16);
01077 else if(stepSizeStr.size() > 1 && stepSizeStr[0] == 'b')
01078 stepSize = strtol(stepSizeStr.substr(1).c_str(),0,2);
01079 else
01080 stepSize = strtol(stepSizeStr.c_str(),0,10);
01081
01082 __COUT__ << "startValue " << startValue << std::endl;
01083 __COUT__ << "stepSize " << stepSize << std::endl;
01084 __COUT__ << "currentValue " << startValue + stepSize*stepIndex << std::endl;
01085
01086 helpCommandModifyActive(
01087 iteratorStruct,
01088 startValue + stepSize*stepIndex,
01089 doTrackGroupChanges);
01090 }
01091
01092 }
01093
01094
01095
01096
01097
01098
01099
01100
01101
01102
01103
01104 bool Iterator::checkCommandRun(IteratorWorkLoopStruct *iteratorStruct)
01105 {
01106 sleep(1);
01107
01108
01109
01110
01111
01112 if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for FSM access" << __E__;
01113 std::lock_guard<std::mutex> lock(iteratorStruct->theIterator_->theSupervisor_->stateMachineAccessMutex_);
01114 if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have FSM access" << __E__;
01115
01116 if(iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.isInTransition())
01117 return false;
01118
01119 iteratorStruct->fsmCommandParameters_.clear();
01120
01121 std::string errorStr = "";
01122 std::string currentState = iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getCurrentStateName();
01123
01124
01126
01127 if(iteratorStruct->doPauseAction_)
01128 {
01129
01130 __COUT__ << "Transitioning FSM to Paused..." << __E__;
01131
01132 if(currentState == "Paused")
01133 {
01134
01135 __COUT__ << "Transition to Paused complete." << __E__;
01136 return true;
01137 }
01138 else if(currentState == "Running")
01139 errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
01140 0,0,
01141 "Pause",iteratorStruct->fsmName_,
01142 WebUsers::DEFAULT_ITERATOR_USERNAME ,
01143 WebUsers::DEFAULT_ITERATOR_USERNAME,
01144 iteratorStruct->fsmCommandParameters_);
01145 else if(currentState == "Configured")
01146 {
01147
01148 __COUT__ << "In Configured state. No need to transition to Paused." << __E__;
01149 return true;
01150 }
01151 else
01152 errorStr = "Expected to be in Paused. Unexpectedly, the current state is " +
01153 currentState + ". Last State Machine error message was as follows: " +
01154 iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getErrorMessage();
01155
01156 if(errorStr != "")
01157 {
01158 __SS__ << "Iterator failed to pause because of the following error: " << errorStr;
01159 throw std::runtime_error(ss.str());
01160 }
01161 return false;
01162 }
01163 else if(iteratorStruct->doHaltAction_)
01164 {
01165
01166 __COUT__ << "Transitioning FSM to Halted..." << __E__;
01167
01168 if(currentState == "Halted")
01169 {
01170
01171 __COUT__ << "Transition to Halted complete." << __E__;
01172 return true;
01173 }
01174 else if(currentState == "Running" ||
01175 currentState == "Paused")
01176 errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
01177 0,0,
01178 "Abort",iteratorStruct->fsmName_,
01179 WebUsers::DEFAULT_ITERATOR_USERNAME ,
01180 WebUsers::DEFAULT_ITERATOR_USERNAME,
01181 iteratorStruct->fsmCommandParameters_);
01182 else if(currentState == "Configured")
01183 errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
01184 0,0,
01185 "Halt",iteratorStruct->fsmName_,
01186 WebUsers::DEFAULT_ITERATOR_USERNAME ,
01187 WebUsers::DEFAULT_ITERATOR_USERNAME,
01188 iteratorStruct->fsmCommandParameters_);
01189 else
01190 errorStr = "Expected to be in Halted. Unexpectedly, the current state is " +
01191 currentState + ". Last State Machine error message was as follows: " +
01192 iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getErrorMessage();
01193
01194 if(errorStr != "")
01195 {
01196 __SS__ << "Iterator failed to halt because of the following error: " << errorStr;
01197 throw std::runtime_error(ss.str());
01198 }
01199 return false;
01200 }
01201 else if(iteratorStruct->doResumeAction_)
01202 {
01203
01204
01205
01206 __COUT__ << "Transitioning FSM to Running..." << __E__;
01207
01208 if(currentState == "Paused")
01209 errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
01210 0,0,
01211 "Resume",iteratorStruct->fsmName_,
01212 WebUsers::DEFAULT_ITERATOR_USERNAME ,
01213 WebUsers::DEFAULT_ITERATOR_USERNAME,
01214 iteratorStruct->fsmCommandParameters_);
01215
01216 if(errorStr != "")
01217 {
01218 __SS__ << "Iterator failed to run because of the following error: " << errorStr;
01219 throw std::runtime_error(ss.str());
01220 }
01221 return false;
01222 }
01223
01224
01225
01227
01228
01229 if(currentState != "Running")
01230 {
01231 if(iteratorStruct->runIsDone_ &&
01232 currentState == "Configured")
01233 {
01234
01235 __COUT__ << "Reached end of run " <<
01236 iteratorStruct->fsmNextRunNumber_ << __E__;
01237 return true;
01238 }
01239
01240 errorStr = "Expected to be in Running. Unexpectedly, the current state is " +
01241 currentState + ". Last State Machine error message was as follows: " +
01242 iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getErrorMessage();
01243 }
01244 else
01245 {
01246 bool waitOnRunningThreads = false;
01247 if("True" == iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
01248 [IterateConfiguration::commandRunParams_.WaitOnRunningThreads_])
01249 waitOnRunningThreads = true;
01250
01251 time_t remainingDurationInSeconds;
01252 sscanf(iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
01253 [IterateConfiguration::commandRunParams_.DurationInSeconds_].c_str(),
01254 "%ld",&remainingDurationInSeconds);
01255
01256
01257 __COUT__ << "waitOnRunningThreads " << waitOnRunningThreads << __E__;
01258 __COUT__ << "remainingDurationInSeconds " << remainingDurationInSeconds << __E__;
01259
01261
01262 if(waitOnRunningThreads)
01263 {
01264
01265 GatewaySupervisor* theSupervisor =
01266 iteratorStruct->theIterator_->theSupervisor_;
01267
01268 bool allFrontEndsAreDone = true;
01269 for (auto& it :
01270 theSupervisor->allSupervisorInfo_.getAllFETypeSupervisorInfo())
01271 {
01272 try
01273 {
01274 std::string status = theSupervisor->send(it.second.getDescriptor(),
01275 "WorkLoopStatusRequest");
01276
01277 __COUT__ << "FESupervisor instance " << it.first <<
01278 " has status = " << status << std::endl;
01279
01280 if(status != CoreSupervisorBase::WORK_LOOP_DONE)
01281 {
01282 allFrontEndsAreDone = false;
01283 break;
01284 }
01285 }
01286 catch (xdaq::exception::Exception& e)
01287 {
01288 __SS__ << "Could not retrieve status from FESupervisor instance " <<
01289 it.first << "." << std::endl;
01290 __COUT_WARN__ << ss.str();
01291 errorStr = ss.str();
01292
01293 if(errorStr != "")
01294 {
01295 __SS__ << "Iterator failed to run because of the following error: " << errorStr;
01296 throw std::runtime_error(ss.str());
01297 }
01298 }
01299 }
01300
01301 if(allFrontEndsAreDone)
01302 {
01303
01304 __COUT__ << "FE workloops all complete! Stopping run..." << __E__;
01305
01306 errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
01307 0,0,
01308 "Stop",iteratorStruct->fsmName_,
01309 WebUsers::DEFAULT_ITERATOR_USERNAME ,
01310 WebUsers::DEFAULT_ITERATOR_USERNAME,
01311 iteratorStruct->fsmCommandParameters_);
01312
01313 if(errorStr != "")
01314 {
01315 __SS__ << "Iterator failed to stop run because of the following error: " << errorStr;
01316 throw std::runtime_error(ss.str());
01317 }
01318
01319
01320 iteratorStruct->runIsDone_ = true;
01321
01322 return false;
01323 }
01324
01325 }
01326
01328
01329 if(remainingDurationInSeconds > 1)
01330 {
01331 --remainingDurationInSeconds;
01332
01333
01334 char str[200];
01335 sprintf(str,"%ld",remainingDurationInSeconds);
01336 iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
01337 [IterateConfiguration::commandRunParams_.DurationInSeconds_] =
01338 str;
01339 }
01340 else if(remainingDurationInSeconds == 1)
01341 {
01342
01343 __COUT__ << "Time duration reached! Stopping run..." << __E__;
01344
01345 errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
01346 0,0,
01347 "Stop",iteratorStruct->fsmName_,
01348 WebUsers::DEFAULT_ITERATOR_USERNAME ,
01349 WebUsers::DEFAULT_ITERATOR_USERNAME,
01350 iteratorStruct->fsmCommandParameters_);
01351
01352 if(errorStr != "")
01353 {
01354 __SS__ << "Iterator failed to stop run because of the following error: " << errorStr;
01355 throw std::runtime_error(ss.str());
01356 }
01357
01358
01359 iteratorStruct->runIsDone_ = true;
01360
01361 return false;
01362 }
01363 }
01364
01365 if(errorStr != "")
01366 {
01367 __SS__ << "Iterator failed to run because of the following error: " << errorStr;
01368 throw std::runtime_error(ss.str());
01369 }
01370 return false;
01371 }
01372
01373
01374
01375 bool Iterator::checkCommandConfigure(IteratorWorkLoopStruct *iteratorStruct)
01376 {
01377 sleep(1);
01378
01379
01380
01381
01382
01383 if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for FSM access" << __E__;
01384 std::lock_guard<std::mutex> lock(iteratorStruct->theIterator_->theSupervisor_->stateMachineAccessMutex_);
01385 if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have FSM access" << __E__;
01386
01387 if(iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.isInTransition())
01388 return false;
01389
01390 std::string errorStr = "";
01391 std::string currentState = iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getCurrentStateName();
01392
01393 if(currentState == "Halted")
01394 errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
01395 0,0,
01396 "Configure",iteratorStruct->fsmName_,
01397 WebUsers::DEFAULT_ITERATOR_USERNAME ,
01398 WebUsers::DEFAULT_ITERATOR_USERNAME,
01399 iteratorStruct->fsmCommandParameters_);
01400 else if(currentState != "Configured")
01401 errorStr = "Expected to be in Configure. Unexpectedly, the current state is " +
01402 currentState + "." + ". Last State Machine error message was as follows: " +
01403 iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getErrorMessage();
01404 else
01405 {
01406 __COUT__ << "checkCommandConfigureAlias complete." << __E__;
01407
01408
01409 std::pair<std::string, ConfigurationGroupKey> newActiveGroup =
01410 iteratorStruct->cfgMgr_->getConfigurationGroupFromAlias(iteratorStruct->fsmCommandParameters_[0]);
01411 iteratorStruct->cfgMgr_->loadConfigurationGroup(newActiveGroup.first,newActiveGroup.second,true );
01412
01413
01414 __COUT__ << "originalTrackChanges " <<
01415 iteratorStruct->originalTrackChanges_ << __E__;
01416 __COUT__ << "originalConfigGroup " <<
01417 iteratorStruct->originalConfigGroup_ << __E__;
01418 __COUT__ << "originalConfigKey " <<
01419 iteratorStruct->originalConfigKey_ << __E__;
01420
01421 __COUT__ << "currentTrackChanges " <<
01422 ConfigurationInterface::isVersionTrackingEnabled() << __E__;
01423 __COUT__ << "originalConfigGroup " <<
01424 iteratorStruct->cfgMgr_->getActiveGroupName() << __E__;
01425 __COUT__ << "originalConfigKey " <<
01426 iteratorStruct->cfgMgr_->getActiveGroupKey() << __E__;
01427
01428 return true;
01429 }
01430
01431 if(errorStr != "")
01432 {
01433 __SS__ << "Iterator failed to configure with system alias '" <<
01434 (iteratorStruct->fsmCommandParameters_.size()?
01435 iteratorStruct->fsmCommandParameters_[0]:"UNKNOWN") <<
01436 "' because of the following error: " << errorStr;
01437 throw std::runtime_error(ss.str());
01438 }
01439 return false;
01440 }
01441
01442
01443
01444 bool Iterator::handleCommandRequest(HttpXmlDocument& xmldoc,
01445 const std::string& command, const std::string& parameter)
01446 {
01447 if(command == "iteratePlay")
01448 {
01449 playIterationPlan(xmldoc,parameter);
01450 return true;
01451 }
01452 else if(command == "iteratePause")
01453 {
01454 pauseIterationPlan(xmldoc);
01455 return true;
01456 }
01457 else if(command == "iterateHalt")
01458 {
01459 haltIterationPlan(xmldoc);
01460 return true;
01461 }
01462 else if(command == "getIterationPlanStatus")
01463 {
01464 getIterationPlanStatus(xmldoc);
01465 return true;
01466 }
01467 else
01468 {
01469
01470
01471 if(theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for iterator access" << __E__;
01472 std::lock_guard<std::mutex> lock(accessMutex_);
01473 if(theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have iterator access" << __E__;
01474
01475 if(iteratorBusy_)
01476 {
01477 __SS__ << "Error - Can not accept request because the Iterator " <<
01478 "is currently " <<
01479 "in control of State Machine progress. ";
01480 __COUT_ERR__ << "\n" << ss.str();
01481 __MOUT_ERR__ << "\n" << ss.str();
01482
01483 xmldoc.addTextElementToData("state_tranisition_attempted", "0");
01484 xmldoc.addTextElementToData("state_tranisition_attempted_err",
01485 ss.str());
01486
01487 return true;
01488 }
01489 }
01490 return false;
01491 }
01492
01493
01494 void Iterator::playIterationPlan(HttpXmlDocument& xmldoc, const std::string& planName)
01495 {
01496 __MOUT__ << "Attempting to play iteration plan '" << planName << ".'" << __E__;
01497 __COUT__ << "Attempting to play iteration plan '" << planName << ".'" << __E__;
01498
01499
01500
01501
01502
01503
01504
01505 if(theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for iterator access" << __E__;
01506 std::lock_guard<std::mutex> lock(accessMutex_);
01507 if(theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have iterator access" << __E__;
01508
01509 if(!activePlanIsRunning_ && !commandPlay_)
01510 {
01511 if(!workloopRunning_)
01512 {
01513
01514
01515 workloopRunning_ = true;
01516
01517
01518 std::thread([](Iterator *iterator){ Iterator::IteratorWorkLoop(iterator); },this).detach();
01519 }
01520
01521 activePlanName_ = planName;
01522 commandPlay_ = true;
01523 }
01524 else
01525 {
01526 __SS__ << "Invalid play command attempted. Can only play when the Iterator is inactive or paused." <<
01527 " If you would like to restart an iteration plan, first try halting." << __E__;
01528 __MOUT__ << ss.str();
01529
01530 xmldoc.addTextElementToData("error_message", ss.str());
01531
01532 __COUT__ << "Invalid play command attempted. " <<
01533 commandPlay_ << " " <<
01534 activePlanName_ << __E__;
01535 }
01536 }
01537
01538
01539 void Iterator::pauseIterationPlan(HttpXmlDocument& xmldoc)
01540 {
01541 __MOUT__ << "Attempting to pause iteration plan '" << activePlanName_ << ".'" << __E__;
01542 __COUT__ << "Attempting to pause iteration plan '" << activePlanName_ << ".'" << __E__;
01543
01544
01545
01546
01547
01548 if(theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for iterator access" << __E__;
01549 std::lock_guard<std::mutex> lock(accessMutex_);
01550 if(theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have iterator access" << __E__;
01551
01552 if(workloopRunning_ && activePlanIsRunning_ && !commandPause_)
01553 {
01554 commandPause_ = true;
01555 }
01556 else
01557 {
01558 __SS__ << "Invalid pause command attempted. Can only pause when running." << __E__;
01559 __MOUT__ << ss.str();
01560
01561 xmldoc.addTextElementToData("error_message", ss.str());
01562
01563 __COUT__ << "Invalid pause command attempted. " <<
01564 workloopRunning_ << " " <<
01565 activePlanIsRunning_ << " " <<
01566 commandPause_ << " " <<
01567 activePlanName_ << __E__;
01568 }
01569 }
01570
01571
01572 void Iterator::haltIterationPlan(HttpXmlDocument& xmldoc)
01573 {
01574 __MOUT__ << "Attempting to halt iteration plan '" << activePlanName_ << ".'" << __E__;
01575 __COUT__ << "Attempting to halt iteration plan '" << activePlanName_ << ".'" << __E__;
01576
01577
01578
01579
01580 if(workloopRunning_)
01581 {
01582
01583
01584 if(theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for iterator access" << __E__;
01585 std::lock_guard<std::mutex> lock(accessMutex_);
01586 if(theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have iterator access" << __E__;
01587
01588 __COUT__ << "activePlanIsRunning_: " << activePlanIsRunning_ << __E__;
01589 __COUT__ << "Passing halt command to iterator thread." << __E__;
01590 commandHalt_ = true;
01591
01592
01593 activePlanName_ = "";
01594 activeCommandIndex_ = -1;
01595 }
01596 else
01597 {
01598 __COUT__ << "No thread, so conducting halt." << __E__;
01599 Iterator::haltIterator(this);
01600
01601
01602
01603
01604
01605
01606
01607
01608
01609
01610
01611
01612
01613
01614
01615
01616
01617
01618
01619
01620
01621
01622
01623
01624
01625
01626
01627
01628
01629
01630
01631 }
01632
01633 }
01634
01635
01636
01637 void Iterator::getIterationPlanStatus(HttpXmlDocument& xmldoc)
01638 {
01639 xmldoc.addTextElementToData("current_state",
01640 theSupervisor_->theStateMachine_.isInTransition()?
01641 theSupervisor_->theStateMachine_.getCurrentTransitionName(
01642 theSupervisor_->stateMachineLastCommandInput_):
01643 theSupervisor_->theStateMachine_.getCurrentStateName());
01644
01645
01646 if(theSupervisor_->theStateMachine_.isInTransition())
01647 xmldoc.addTextElementToData("transition_progress",
01648 theSupervisor_->theProgressBar_.readPercentageString());
01649 else
01650 xmldoc.addTextElementToData("transition_progress", "100");
01651
01652 char tmp[20];
01653 sprintf(tmp,"%lu",theSupervisor_->theStateMachine_.getTimeInState());
01654 xmldoc.addTextElementToData("time_in_state", tmp);
01655
01656
01657
01658
01659
01660 if(theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for iterator access" << __E__;
01661 std::lock_guard<std::mutex> lock(accessMutex_);
01662 if(theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have iterator access" << __E__;
01663
01664 xmldoc.addTextElementToData("active_plan", activePlanName_);
01665 xmldoc.addTextElementToData("last_started_plan", lastStartedPlanName_);
01666 xmldoc.addTextElementToData("last_finished_plan", lastFinishedPlanName_);
01667
01668 sprintf(tmp,"%u",activeCommandIndex_);
01669 xmldoc.addTextElementToData("current_command_index", tmp);
01670 sprintf(tmp,"%ld",time(0) - activeCommandStartTime_);
01671 xmldoc.addTextElementToData("current_command_duration", tmp);
01672 sprintf(tmp,"%u",activeCommandIteration_);
01673 xmldoc.addTextElementToData("current_command_iteration", tmp);
01674
01675 for(const auto& depthIteration:depthIterationStack_)
01676 {
01677 sprintf(tmp,"%u",depthIteration);
01678 xmldoc.addTextElementToData("depth_iteration", tmp);
01679 }
01680
01681 if(activePlanIsRunning_ && iteratorBusy_)
01682 {
01683 if(workloopRunning_)
01684 xmldoc.addTextElementToData("active_plan_status", "Running");
01685 else
01686 xmldoc.addTextElementToData("active_plan_status", "Error");
01687 }
01688 else if(!activePlanIsRunning_ && iteratorBusy_)
01689 xmldoc.addTextElementToData("active_plan_status", "Paused");
01690 else
01691 xmldoc.addTextElementToData("active_plan_status", "Inactive");
01692
01693 xmldoc.addTextElementToData("error_message", errorMessage_);
01694 }
01695
01696