otsdaq  v1_01_04
 All Classes Namespaces Functions
Iterator.cc
1 #include "otsdaq-core/Supervisor/Iterator.h"
2 #include "otsdaq-core/MessageFacility/MessageFacility.h"
3 #include "otsdaq-core/Macros/CoutHeaderMacros.h"
4 #include "otsdaq-core/Supervisor/Supervisor.h"
5 #include "otsdaq-core/WebUsersUtilities/WebUsers.h"
6 
7 #include "otsdaq-core/ConfigurationInterface/ConfigurationManagerRW.h"
8 
9 
10 #include <iostream>
11 #include <thread> //for std::thread
12 
13 #undef __MF_SUBJECT__
14 #define __MF_SUBJECT__ "SupervisorIterator"
15 
16 using namespace ots;
17 
18 //========================================================================================================================
19 Iterator::Iterator(Supervisor* supervisor)
20 : workloopRunning_ (false)
21 , activePlanIsRunning_ (false)
22 , iteratorBusy_ (false)
23 , commandPlay_(false), commandPause_(false), commandHalt_(false)
24 , activePlanName_ ("")
25 , activeCommandIndex_ (-1)
26 , activeCommandStartTime_ (0)
27 , theSupervisor_ (supervisor)
28 {
29  __MOUT__ << "Iterator constructed." << __E__;
30  __COUT__ << "Iterator constructed." << __E__;
31 
32 }
33 
34 //========================================================================================================================
35 Iterator::~Iterator(void)
36 {
37 }
38 
39 //========================================================================================================================
40 void Iterator::IteratorWorkLoop(Iterator *iterator)
41 try
42 {
43  __MOUT__ << "Iterator work loop starting..." << __E__;
44  __COUT__ << "Iterator work loop starting..." << __E__;
45 
46  //mutex init scope
47  {
48  //lockout the messages array for the remainder of the scope
49  //this guarantees the reading thread can safely access the messages
50  if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for iterator access" << __E__;
51  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
52  if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have iterator access" << __E__;
53 
54  iterator->errorMessage_ = ""; //clear error message
55  }
56 
57 
58  ConfigurationManagerRW theConfigurationManager(WebUsers::DEFAULT_ITERATOR_USERNAME); //this is a restricted username
59  IteratorWorkLoopStruct theIteratorStruct(iterator,
60  &theConfigurationManager);
61 
62 
63  const IterateConfiguration* itConfig;
64 
65  std::vector<IterateConfiguration::Command> commands;
66 
67  while(1)
68  {
69  //Process:
70  // - always "listen" for commands
71  // - play: if no plan running, activePlanIsRunning_ = true,
72  // and start or continue plan based on name/commandIndex
73  // - pause: if plan playing, pause it, activePlanIsRunning_ = false
74  // and do not clear commandIndex or name, iteratorBusy_ = true
75  // - halt: if plan playing or not, activePlanIsRunning_ = false
76  // and clear commandIndex or name, iteratorBusy_ = false
77  // - when running
78  // - go through each command
79  // - start the command, commandBusy = true
80  // - check for complete, then commandBusy = false
81 
82  //start command handling
83  //define mutex scope
84  {
85  //lockout the messages array for the remainder of the scope
86  //this guarantees the reading thread can safely access the messages
87  if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for iterator access" << __E__;
88  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
89  if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have iterator access" << __E__;
90 
91  if(iterator->commandPlay_)
92  {
93  iterator->commandPlay_ = false; //clear
94 
95  if(!iterator->activePlanIsRunning_)
96  {
97  //valid PLAY command!
98 
99  iterator->activePlanIsRunning_ = true;
100  iterator->iteratorBusy_ = true;
101 
102  if(theIteratorStruct.activePlan_ != iterator->activePlanName_)
103  {
104  __COUT__ << "New plan name encountered old=" << theIteratorStruct.activePlan_ <<
105  " vs new=" << iterator->activePlanName_ << __E__;
106  theIteratorStruct.commandIndex_ = -1; //reset for new plan
107  }
108 
109  theIteratorStruct.activePlan_ = iterator->activePlanName_;
110  iterator->lastStartedPlanName_ = iterator->activePlanName_;
111 
112  if(theIteratorStruct.commandIndex_ == (unsigned int)-1)
113  {
114  __COUT__ << "Starting plan '" << theIteratorStruct.activePlan_ << ".'" << __E__;
115  __MOUT__ << "Starting plan '" << theIteratorStruct.activePlan_ << ".'" << __E__;
116  }
117  else
118  {
119  theIteratorStruct.doResumeAction_ = true;
120  __COUT__ << "Continuing plan '" << theIteratorStruct.activePlan_ << "' at command index " <<
121  theIteratorStruct.commandIndex_ << ". " << __E__;
122  __MOUT__ << "Continuing plan '" << theIteratorStruct.activePlan_ << "' at command index " <<
123  theIteratorStruct.commandIndex_ << ". " << __E__;
124  }
125  }
126  }
127  else if(iterator->commandPause_ && !theIteratorStruct.doPauseAction_)
128  {
129  theIteratorStruct.doPauseAction_ = true;
130  iterator->commandPause_ = false; //clear
131  }
132  else if(iterator->commandHalt_ && !theIteratorStruct.doHaltAction_)
133  {
134  theIteratorStruct.doHaltAction_ = true;
135  iterator->commandHalt_ = false; //clear
136  }
137 
138  theIteratorStruct.running_ = iterator->activePlanIsRunning_;
139 
140  if(iterator->activeCommandIndex_ != //update active command status if changed
141  theIteratorStruct.commandIndex_)
142  {
143  iterator->activeCommandIndex_ = theIteratorStruct.commandIndex_;
144  iterator->activeCommandStartTime_ = time(0); //reset on any change
145  }
146 
147  } //end command handling and iterator mutex
148 
149 
152  //do halt or pause action outside of iterator mutex
153 
154  if(theIteratorStruct.doPauseAction_)
155  {
156  //valid PAUSE command!
157 
158  //safely pause plan!
159  // i.e. check that command is complete
160 
161  __COUT__ << "Waiting to pause..." << __E__;
162  while(!iterator->checkCommand(&theIteratorStruct))
163  __COUT__ << "Waiting to pause..." << __E__;
164 
165  __COUT__ << "Completeing pause..." << __E__;
166 
167  theIteratorStruct.doPauseAction_ = false; //clear
168 
169  //lockout the messages array for the remainder of the scope
170  //this guarantees the reading thread can safely access the messages
171  if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for iterator access" << __E__;
172  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
173  if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have iterator access" << __E__;
174 
175  iterator->activePlanIsRunning_ = false;
176 
177  __COUT__ << "Paused plan '" << theIteratorStruct.activePlan_ << "' at command index " <<
178  theIteratorStruct.commandIndex_ << ". " << __E__;
179  __MOUT__ << "Paused plan '" << theIteratorStruct.activePlan_ << "' at command index " <<
180  theIteratorStruct.commandIndex_ << ". " << __E__;
181 
182  continue; //resume workloop
183  }
184  else if(theIteratorStruct.doHaltAction_)
185  {
186  //valid HALT command!
187 
188  //safely end plan!
189  // i.e. check that command is complete
190 
191  __COUT__ << "Waiting to halt..." << __E__;
192  while(!iterator->checkCommand(&theIteratorStruct))
193  __COUT__ << "Waiting to halt..." << __E__;
194 
195  __COUT__ << "Completeing halt..." << __E__;
196 
197  theIteratorStruct.doHaltAction_ = false; //clear
198 
199  //last ditch effort to make sure FSM is halted
200  iterator->haltStateMachine(
201  iterator->theSupervisor_,
202  theIteratorStruct.fsmName_);
203 
204  //lockout the messages array for the remainder of the scope
205  //this guarantees the reading thread can safely access the messages
206  if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for iterator access" << __E__;
207  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
208  if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have iterator access" << __E__;
209 
210  iterator->activePlanIsRunning_ = false;
211  iterator->iteratorBusy_ = false;
212 
213  __COUT__ << "Halted plan '" << theIteratorStruct.activePlan_ << "' at command index " <<
214  theIteratorStruct.commandIndex_ << ". " << __E__;
215  __MOUT__ << "Halted plan '" << theIteratorStruct.activePlan_ << "' at command index " <<
216  theIteratorStruct.commandIndex_ << ". " << __E__;
217 
218  theIteratorStruct.activePlan_ = ""; //clear
219  theIteratorStruct.commandIndex_ = -1; //clear
220 
221  continue; //resume workloop
222  }
223 
224 
227  // handle running
228  // __COUT__ << "thinking.." << theIteratorStruct.running_ << " " <<
229  // theIteratorStruct.activePlan_ << " cmd=" <<
230  // theIteratorStruct.commandIndex_ << __E__;
231  if(theIteratorStruct.running_ &&
232  theIteratorStruct.activePlan_ != "") //important, because after errors, still "running" until halt
233  {
234  if(theIteratorStruct.commandIndex_ == (unsigned int)-1)
235  {
236  __COUT__ << "Get commands" << __E__;
237 
238  theIteratorStruct.commandIndex_ = 0;
239 
240  theIteratorStruct.cfgMgr_->init(); //completely reset to re-align with any changes
241  itConfig = theIteratorStruct.cfgMgr_->__GET_CONFIG__(IterateConfiguration);
242 
243  theIteratorStruct.commands_ = itConfig->getPlanCommands(theIteratorStruct.cfgMgr_,
244  theIteratorStruct.activePlan_);
245 
246  for(auto& command:theIteratorStruct.commands_)
247  {
248  __COUT__ << "command " <<
249  command.type_ << __E__;
250  __COUT__ << "table " <<
251  IterateConfiguration::commandToTableMap_.at(command.type_) << __E__;
252  __COUT__ << "param count = " << command.params_.size() << __E__;
253 
254  for(auto& param:command.params_)
255  {
256  __COUT__ << "\t param " <<
257  param.first << " : " <<
258  param.second << __E__;
259  }
260  }
261  }
262 
263  if(!theIteratorStruct.commandBusy_)
264  {
265  if(theIteratorStruct.commandIndex_ < theIteratorStruct.commands_.size())
266  {
267  //execute command
268  theIteratorStruct.commandBusy_ = true;
269 
270  __COUT__ << "Iterator starting command " << theIteratorStruct.commandIndex_+1 << ": " <<
271  theIteratorStruct.commands_[theIteratorStruct.commandIndex_].type_ << __E__;
272  __MOUT__ << "Iterator starting command " << theIteratorStruct.commandIndex_+1 << ": " <<
273  theIteratorStruct.commands_[theIteratorStruct.commandIndex_].type_ << __E__;
274 
275  //FIXME
276  //FIXME
277  //FIXME
278  //for debugging modify commands: FIXME
279 // if(theIteratorStruct.commands_[theIteratorStruct.commandIndex_].type_ ==
280 // IterateConfiguration::COMMAND_CONFIGURE_ALIAS)
281 // theIteratorStruct.commands_[theIteratorStruct.commandIndex_].type_ =
282 // IterateConfiguration::COMMAND_MODIFY_ACTIVE_GROUP;
283 //
284 // if(theIteratorStruct.commands_[theIteratorStruct.commandIndex_].type_ ==
285 // IterateConfiguration::COMMAND_RUN)
286 // theIteratorStruct.commands_[theIteratorStruct.commandIndex_].type_ =
287 // IterateConfiguration::COMMAND_CONFIGURE_ACTIVE_GROUP;
288 
289 
290  iterator->startCommand(&theIteratorStruct);
291  }
292  else if(theIteratorStruct.commandIndex_ ==
293  theIteratorStruct.commands_.size()) //Done!
294  {
295  __COUT__ << "Finished Iteration Plan '" << theIteratorStruct.activePlan_ << __E__;
296  __MOUT__ << "Finished Iteration Plan '" << theIteratorStruct.activePlan_ << __E__;
297 
298  //lockout the messages array for the remainder of the scope
299  //this guarantees the reading thread can safely access the messages
300  if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for iterator access" << __E__;
301  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
302  if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have iterator access" << __E__;
303 
304  //similar to halt
305  iterator->activePlanIsRunning_ = false;
306  iterator->iteratorBusy_ = false;
307 
308  iterator->lastStartedPlanName_ = theIteratorStruct.activePlan_;
309  theIteratorStruct.activePlan_ = ""; //clear
310  theIteratorStruct.commandIndex_ = -1; //clear
311  }
312  }
313  else if(theIteratorStruct.commandBusy_)
314  {
315  //check for command completion
316  if(iterator->checkCommand(&theIteratorStruct))
317  {
318  theIteratorStruct.commandBusy_ = false; //command complete
319 
320  ++theIteratorStruct.commandIndex_;
321 
322  __COUT__ << "Ready for next command. Done with " << theIteratorStruct.commandIndex_ << " of " <<
323  theIteratorStruct.commands_.size() << __E__;
324  __MOUT__ << "Iterator ready for next command. Done with " << theIteratorStruct.commandIndex_ << " of " <<
325  theIteratorStruct.commands_.size() << __E__;
326  }
327 
328  //Note: check command gets one shot to resume
329  if(theIteratorStruct.doResumeAction_) //end resume action
330  theIteratorStruct.doResumeAction_ = false;
331  }
332 
333 
334 
335  } //end running
336  else
337  sleep(1); //when inactive sleep a lot
338 
341 
342  // __COUT__ << "end loop.." << theIteratorStruct.running_ << " " <<
343  // theIteratorStruct.activePlan_ << " cmd=" <<
344  // theIteratorStruct.commandIndex_ << __E__;
345 
346  }
347 
348  iterator->workloopRunning_ = false; //if we ever exit
349 }
350 catch(const std::runtime_error &e)
351 {
352  __SS__ << "Encountered error in Iterator thread:\n" << e.what() << __E__;
353  __COUT_ERR__ << ss.str();
354 
355  //lockout the messages array for the remainder of the scope
356  //this guarantees the reading thread can safely access the messages
357  if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for iterator access" << __E__;
358  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
359  if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have iterator access" << __E__;
360 
361  iterator->workloopRunning_ = false; //if we ever exit
362  iterator->errorMessage_ = ss.str();
363 
364 }
365 catch(...)
366 {
367  __SS__ << "Encountered unknown error in Iterator thread." << __E__;
368  __COUT_ERR__ << ss.str();
369 
370  //lockout the messages array for the remainder of the scope
371  //this guarantees the reading thread can safely access the messages
372  if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for iterator access" << __E__;
373  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
374  if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have iterator access" << __E__;
375 
376  iterator->workloopRunning_ = false; //if we ever exit
377  iterator->errorMessage_ = ss.str();
378 }
379 
380 //========================================================================================================================
381 void Iterator::startCommand(IteratorWorkLoopStruct *iteratorStruct)
382 {
383  //should be mutually exclusive with Supervisor main thread state machine accesses
384  //lockout the messages array for the remainder of the scope
385  //this guarantees the reading thread can safely access the messages
386  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for FSM access" << __E__;
387  std::lock_guard<std::mutex> lock(iteratorStruct->theIterator_->theSupervisor_->stateMachineAccessMutex_);
388  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have FSM access" << __E__;
389 
390 
391  //for out of range, throw exception - should never happen
392  if(iteratorStruct->commandIndex_ >= iteratorStruct->commands_.size())
393  {
394  __SS__ << "Out of range commandIndex = " << iteratorStruct->commandIndex_ <<
395  " in size = " << iteratorStruct->commands_.size() << __E__;
396  throw std::runtime_error(ss.str());
397  }
398 
399 
400  std::string type = iteratorStruct->commands_[iteratorStruct->commandIndex_].type_;
401  if(type == IterateConfiguration::COMMAND_BEGIN_LABEL)
402  {
403  return startCommandBeginLabel(iteratorStruct);
404  }
405  else if(type == IterateConfiguration::COMMAND_CHOOSE_FSM)
406  {
407  return startCommandChooseFSM(
408  iteratorStruct,
409  iteratorStruct->commands_
410  [iteratorStruct->commandIndex_].params_
411  [IterateConfiguration::commandChooseFSMParams_.NameOfFSM_]);
412  }
413  else if(type == IterateConfiguration::COMMAND_CONFIGURE_ACTIVE_GROUP)
414  {
415  return startCommandConfigureActive(iteratorStruct);
416  }
417  else if(type == IterateConfiguration::COMMAND_CONFIGURE_ALIAS)
418  {
419  return startCommandConfigureAlias(
420  iteratorStruct,
421  iteratorStruct->commands_
422  [iteratorStruct->commandIndex_].params_
423  [IterateConfiguration::commandConfigureAliasParams_.SystemAlias_]);
424  }
425  else if(type == IterateConfiguration::COMMAND_CONFIGURE_GROUP)
426  {
427  return startCommandConfigureGroup(iteratorStruct);
428  }
429  else if(type == IterateConfiguration::COMMAND_EXECUTE_FE_MACRO)
430  {
431  //TODO
432  return;
433  }
434  else if(type == IterateConfiguration::COMMAND_EXECUTE_MACRO)
435  {
436  //TODO
437  return;
438  }
439  else if(type == IterateConfiguration::COMMAND_MODIFY_ACTIVE_GROUP)
440  {
441  return startCommandModifyActive(iteratorStruct);
442  }
443  else if(type == IterateConfiguration::COMMAND_REPEAT_LABEL)
444  {
445  return startCommandRepeatLabel(iteratorStruct);
446  }
447  else if(type == IterateConfiguration::COMMAND_RUN)
448  {
449  return startCommandRun(iteratorStruct);
450  }
451  else
452  {
453  __SS__ << "Attempt to start unrecognized command type = " << type << __E__;
454  __COUT_ERR__ << ss.str();
455  throw std::runtime_error(ss.str());
456  }
457 }
458 
459 //========================================================================================================================
460 //checkCommand
461 // when busy for a while, start to sleep
462 // use sleep() or nanosleep()
463 bool Iterator::checkCommand(IteratorWorkLoopStruct *iteratorStruct)
464 {
465  //for out of range, return done
466  if(iteratorStruct->commandIndex_ >= iteratorStruct->commands_.size())
467  {
468  __COUT__ << "Out of range commandIndex = " << iteratorStruct->commandIndex_ <<
469  " in size = " << iteratorStruct->commands_.size() << __E__;
470  return true;
471  }
472 
473  std::string type = iteratorStruct->commands_[iteratorStruct->commandIndex_].type_;
474  if(type == IterateConfiguration::COMMAND_BEGIN_LABEL)
475  {
476  //do nothing
477  return true;
478  }
479  else if(type == IterateConfiguration::COMMAND_CHOOSE_FSM)
480  {
481  //do nothing
482  return true;
483  }
484  else if(type == IterateConfiguration::COMMAND_CONFIGURE_ALIAS ||
485  type == IterateConfiguration::COMMAND_CONFIGURE_ACTIVE_GROUP ||
486  type == IterateConfiguration::COMMAND_CONFIGURE_GROUP)
487  {
488  return checkCommandConfigure(iteratorStruct);
489  }
490  else if(type == IterateConfiguration::COMMAND_EXECUTE_FE_MACRO)
491  {
492  //do nothing
493  return true;
494  }
495  else if(type == IterateConfiguration::COMMAND_EXECUTE_MACRO)
496  {
497  //do nothing
498  return true;
499  }
500  else if(type == IterateConfiguration::COMMAND_MODIFY_ACTIVE_GROUP)
501  {
502  //do nothing
503  return true;
504  }
505  else if(type == IterateConfiguration::COMMAND_REPEAT_LABEL)
506  {
507  //do nothing
508  return true;
509  }
510  else if(type == IterateConfiguration::COMMAND_RUN)
511  {
512  return checkCommandRun(iteratorStruct);
513  }
514  else
515  {
516  __SS__ << "Attempt to check unrecognized command type = " << type << __E__;
517  __COUT_ERR__ << ss.str();
518  throw std::runtime_error(ss.str());
519  }
520 }
521 
522 //========================================================================================================================
523 void Iterator::startCommandChooseFSM(IteratorWorkLoopStruct *iteratorStruct,
524  const std::string& fsmName)
525 {
526  __COUT__ << "fsmName " << fsmName << __E__;
527 
528 
529  iteratorStruct->fsmName_ = fsmName;
530  iteratorStruct->theIterator_->lastFsmName_ = fsmName;
531 
532  //Translate fsmName
533  // to gives us run alias (fsmRunAlias_) and next run number (fsmNextRunNumber_)
534 
535 
536  //CAREFUL?? Threads
537 
538 
540 
541  iteratorStruct->fsmRunAlias_ = "Run"; //default to "Run"
542 
543  // get stateMachineAliasFilter if possible
544  ConfigurationTree configLinkNode = iteratorStruct->cfgMgr_->getSupervisorConfigurationNode(
545  iteratorStruct->theIterator_->theSupervisor_->supervisorContextUID_,
546  iteratorStruct->theIterator_->theSupervisor_->supervisorApplicationUID_);
547 
548  if(!configLinkNode.isDisconnected())
549  {
550  try //for backwards compatibility
551  {
552  ConfigurationTree fsmLinkNode = configLinkNode.getNode("LinkToStateMachineConfiguration");
553  if(!fsmLinkNode.isDisconnected())
554  iteratorStruct->fsmRunAlias_ =
555  fsmLinkNode.getNode(fsmName + "/RunDisplayAlias").getValue<std::string>();
556  else
557  __COUT_INFO__ << "FSM Link disconnected." << __E__;
558  }
559  catch(std::runtime_error &e)
560  {
561  //__COUT_INFO__ << e.what() << __E__;
562  __COUT_INFO__ << "No state machine Run alias. Ignoring and assuming alias of '" <<
563  iteratorStruct->fsmRunAlias_ << ".'" << __E__;
564 
565  }
566  catch(...) {
567  __COUT_ERR__ << "Unknown error. Should never happen." << __E__;
568 
569  __COUT_INFO__ << "No state machine Run alias. Ignoring and assuming alias of '" <<
570  iteratorStruct->fsmRunAlias_ << ".'" << __E__;
571  }
572  }
573  else
574  __COUT_INFO__ << "FSM Link disconnected." << __E__;
575 
576  __COUT__ << "fsmRunAlias_ = " << iteratorStruct->fsmRunAlias_ << __E__;
577 
578 
579 
580 
582 
583  iteratorStruct->fsmNextRunNumber_ = iteratorStruct->theIterator_->theSupervisor_->getNextRunNumber(
584  iteratorStruct->fsmName_);
585 
586  if(iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getCurrentStateName() == "Running" ||
587  iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getCurrentStateName() == "Paused")
588  --iteratorStruct->fsmNextRunNumber_; //current run number is one back
589 
590  __COUT__ << "fsmNextRunNumber_ = " << iteratorStruct->fsmNextRunNumber_ << __E__;
591 }
592 
593 //========================================================================================================================
594 // return true if an action was attempted
595 bool Iterator::haltStateMachine(Supervisor* theSupervisor, const std::string& fsmName)
596 {
597  std::vector<std::string> fsmCommandParameters;
598  std::string errorStr = "";
599  std::string currentState = theSupervisor->theStateMachine_.getCurrentStateName();
600 
601  if(currentState == "Initialized" ||
602  currentState == "Halted")
603  {
604  __COUT__ << "Do nothing. Already halted." << __E__;
605  return false;
606  }
607  else if(currentState == "Running")
608  errorStr = theSupervisor->attemptStateMachineTransition(
609  0,0,
610  "Abort",fsmName,
611  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
612  WebUsers::DEFAULT_ITERATOR_USERNAME,
613  fsmCommandParameters);
614  else
615  errorStr = theSupervisor->attemptStateMachineTransition(
616  0,0,
617  "Halt",fsmName,
618  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
619  WebUsers::DEFAULT_ITERATOR_USERNAME,
620  fsmCommandParameters);
621 
622  if(errorStr != "")
623  {
624  __SS__ << "Iterator failed to halt because of the following error: " << errorStr;
625  throw std::runtime_error(ss.str());
626  }
627 
628  //else successfully launched
629  __COUT__ << "FSM in transition = " << theSupervisor->theStateMachine_.isInTransition() << __E__;
630  __COUT__ << "haltStateMachine launched." << __E__;
631  return true;
632 }
633 
634 //========================================================================================================================
635 void Iterator::startCommandBeginLabel(IteratorWorkLoopStruct *iteratorStruct)
636 {
637  __COUT__ << "Entering label '" <<
638  iteratorStruct->commands_
639  [iteratorStruct->commandIndex_].params_
640  [IterateConfiguration::commandBeginLabelParams_.Label_]
641  << "'..." << std::endl;
642 
643  //add new step index to stack
644  iteratorStruct->stepIndexStack_.push_back(0);
645 }
646 
647 
648 //========================================================================================================================
649 void Iterator::startCommandRepeatLabel(IteratorWorkLoopStruct *iteratorStruct)
650 {
651  //search for first matching label backward and set command to there
652 
653  int numOfRepetitions;
654  sscanf(iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
655  [IterateConfiguration::commandRepeatLabelParams_.NumberOfRepetitions_].c_str(),
656  "%d",&numOfRepetitions);
657  __COUT__ << "numOfRepetitions remaining = " << numOfRepetitions << __E__;
658 
659  if(numOfRepetitions <= 0)
660  {
661  //remove step index from stack
662  iteratorStruct->stepIndexStack_.pop_back();
663 
664  return; //no more repetitions
665  }
666 
667  --numOfRepetitions;
668 
669  //increment step index in stack
670  ++(iteratorStruct->stepIndexStack_.back());
671 
672  unsigned int i;
673  for(i=iteratorStruct->commandIndex_;i>0;--i) //assume 0 is always the fallback option
674  if(iteratorStruct->commands_[i].type_ == IterateConfiguration::COMMAND_BEGIN_LABEL &&
675  iteratorStruct->commands_[iteratorStruct->commandIndex_].params_[IterateConfiguration::commandRepeatLabelParams_.Label_] ==
676  iteratorStruct->commands_[i].params_[IterateConfiguration::commandBeginLabelParams_.Label_]) break;
677 
678  char repStr[200];
679  sprintf(repStr,"%d",numOfRepetitions);
680  iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
681  [IterateConfiguration::commandRepeatLabelParams_.NumberOfRepetitions_] =
682  repStr; //re-store as string
683 
684  iteratorStruct->commandIndex_ = i;
685  __COUT__ << "Jumping back to commandIndex " << iteratorStruct->commandIndex_ << __E__;
686 }
687 
688 //========================================================================================================================
689 void Iterator::startCommandRun(IteratorWorkLoopStruct *iteratorStruct)
690 {
691 
692  iteratorStruct->fsmCommandParameters_.clear();
693 
694  std::string errorStr = "";
695  std::string currentState = iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getCurrentStateName();
696 
697  //execute first transition (may need two)
698 
699  if(currentState == "Configured")
700  errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
701  0,0,
702  "Start",iteratorStruct->fsmName_,
703  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
704  WebUsers::DEFAULT_ITERATOR_USERNAME,
705  iteratorStruct->fsmCommandParameters_);
706  else
707  errorStr = "Can only Run from the Configured state. The current state is " +
708  currentState;
709 
710 
711  if(errorStr != "")
712  {
713  __SS__ << "Iterator failed to run because of the following error: " << errorStr;
714  throw std::runtime_error(ss.str());
715  }
716 
717  //else successfully launched
718  __COUT__ << "FSM in transition = " << iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.isInTransition() << __E__;
719  __COUT__ << "startCommandRun success." << __E__;
720 }
721 
722 //========================================================================================================================
723 void Iterator::startCommandConfigureActive(IteratorWorkLoopStruct *iteratorStruct)
724 {
725  __COUT__ << "startCommandConfigureActive " << __E__;
726 
727  //steps:
728  // get active config group
729  // transition to configure with parameters describing group
730 
731  std::string group = iteratorStruct->cfgMgr_->getActiveGroupName();
732  ConfigurationGroupKey key = iteratorStruct->cfgMgr_->getActiveGroupKey();
733 
734  __COUT__ << "group " << group << __E__;
735  __COUT__ << "key " << key << __E__;
736 
737  //create special alias for this group using : separators
738 
739  std::stringstream systemAlias;
740  systemAlias << "GROUP:" << group << ":" << key;
741  startCommandConfigureAlias(iteratorStruct,systemAlias.str());
742 }
743 
744 //========================================================================================================================
745 void Iterator::startCommandConfigureGroup(IteratorWorkLoopStruct *iteratorStruct)
746 {
747  __COUT__ << "startCommandConfigureGroup " << __E__;
748 
749  //steps:
750  // transition to configure with parameters describing group
751 
752  std::string group =
753  iteratorStruct->commands_
754  [iteratorStruct->commandIndex_].params_
755  [IterateConfiguration::commandConfigureGroupParams_.GroupName_];
757  iteratorStruct->commands_
758  [iteratorStruct->commandIndex_].params_
759  [IterateConfiguration::commandConfigureGroupParams_.GroupKey_])
760  ;
761 
762  __COUT__ << "group " << group << __E__;
763  __COUT__ << "key " << key << __E__;
764 
765  //create special alias for this group using : separators
766 
767  std::stringstream systemAlias;
768  systemAlias << "GROUP:" << group << ":" << key;
769  startCommandConfigureAlias(iteratorStruct,systemAlias.str());
770 }
771 
772 //========================================================================================================================
773 void Iterator::startCommandConfigureAlias(IteratorWorkLoopStruct *iteratorStruct,
774  const std::string& systemAlias)
775 {
776  __COUT__ << "systemAlias " << systemAlias << __E__;
777 
778  iteratorStruct->fsmCommandParameters_.clear();
779  iteratorStruct->fsmCommandParameters_.push_back(systemAlias);
780 
781  std::string errorStr = "";
782  std::string currentState = iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getCurrentStateName();
783 
784  //execute first transition (may need two)
785 
786  if(currentState == "Initial")
787  errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
788  0,0,
789  "Initialize",iteratorStruct->fsmName_,
790  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
791  WebUsers::DEFAULT_ITERATOR_USERNAME,
792  iteratorStruct->fsmCommandParameters_);
793  else if(currentState == "Halted")
794  errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
795  0,0,
796  "Configure",iteratorStruct->fsmName_,
797  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
798  WebUsers::DEFAULT_ITERATOR_USERNAME,
799  iteratorStruct->fsmCommandParameters_);
800  else
801  errorStr = "Can only Configure from the Initial or Halted state. The current state is " +
802  currentState;
803 
804 
805 
806  if(errorStr != "")
807  {
808  __SS__ << "Iterator failed to configure with system alias '" <<
809  (iteratorStruct->fsmCommandParameters_.size()?
810  iteratorStruct->fsmCommandParameters_[0]:"UNKNOWN") <<
811  "' because of the following error: " << errorStr;
812  throw std::runtime_error(ss.str());
813  }
814 
815  //else successfully launched
816  __COUT__ << "FSM in transition = " << iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.isInTransition() << __E__;
817  __COUT__ << "startCommandConfigureAlias success." << __E__;
818 }
819 
820 //========================================================================================================================
821 void Iterator::startCommandModifyActive(IteratorWorkLoopStruct *iteratorStruct)
822 {
823  //Steps:
824  // 4 parameters commandModifyActiveParams_:
825  // const std::string DoTrackGroupChanges_ TrueFalse
826  // //targets
827  // const std::string RelativePathToField_ = "RelativePathToField";
828  // const std::string FieldStartValue_ = "FieldStartValue";
829  // const std::string FieldIterationStepSize_ = "FieldIterationStepSize";
830  //
831  // if tracking changes,
832  // create a new group
833  // for every enabled FE
834  // set field = start value + stepSize * currentStepIndex_
835  // activate group
836  // else
837  // load scratch group
838  // for every enabled FE
839  // set field = start value + stepSize * stepIndex
840  // activate group
841 
842  bool doTrackGroupChanges = false;
843  if("True" == iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
844  [IterateConfiguration::commandModifyActiveParams_.DoTrackGroupChanges_])
845  doTrackGroupChanges = true;
846 
847  const std::string& pathToField =
848  iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
849  [IterateConfiguration::commandModifyActiveParams_.RelativePathToField_];
850  const std::string& startValue =
851  iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
852  [IterateConfiguration::commandModifyActiveParams_.FieldStartValue_];
853  const std::string& stepSize =
854  iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
855  [IterateConfiguration::commandModifyActiveParams_.FieldIterationStepSize_];
856 
857  const unsigned int stepIndex = iteratorStruct->stepIndexStack_.back();
858 
859  ConfigurationManagerRW* cfgMgr = iteratorStruct->cfgMgr_;
860 
861  __COUT__ << "doTrackGroupChanges " << (doTrackGroupChanges?"yes":"no") << std::endl;
862  __COUT__ << "stepIndex " << stepIndex << std::endl;
863  __COUT__ << "pathToField " << pathToField << std::endl;
864  __COUT__ << "startValue " << startValue << std::endl;
865  __COUT__ << "stepSize " << stepSize << std::endl;
866 }
867 
868 //========================================================================================================================
869 //checkCommandRun
870 // return true if done
871 //
872 // Either will be done on (priority 1) running threads (for Frontends) ending
873 // or (priority 2 and ignored if <= 0) duration timeout
874 //
875 // Note: use command structure strings to maintain duration left
876 // Note: watch iterator->doPauseAction and iterator->doHaltAction and respond
877 bool Iterator::checkCommandRun(IteratorWorkLoopStruct *iteratorStruct)
878 {
879  sleep(1); //sleep to give FSM time to transition
880 
881  //all RunControlStateMachine access commands should be mutually exclusive with Supervisor main thread state machine accesses
882  //should be mutually exclusive with Supervisor main thread state machine accesses
883  //lockout the messages array for the remainder of the scope
884  //this guarantees the reading thread can safely access the messages
885  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for FSM access" << __E__;
886  std::lock_guard<std::mutex> lock(iteratorStruct->theIterator_->theSupervisor_->stateMachineAccessMutex_);
887  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have FSM access" << __E__;
888 
889  if(iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.isInTransition())
890  return false;
891 
892  iteratorStruct->fsmCommandParameters_.clear();
893 
894  std::string errorStr = "";
895  std::string currentState = iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getCurrentStateName();
896 
897 
899  //check for imposed actions and forced exits
900  if(iteratorStruct->doPauseAction_)
901  {
902  //transition to pause state
903  __COUT__ << "Transitioning FSM to Paused..." << __E__;
904 
905  if(currentState == "Paused")
906  {
907  //done with early pause exit!
908  __COUT__ << "Transition to Paused complete." << __E__;
909  return true;
910  }
911  else if(currentState == "Running") //launch transition to pause
912  errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
913  0,0,
914  "Pause",iteratorStruct->fsmName_,
915  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
916  WebUsers::DEFAULT_ITERATOR_USERNAME,
917  iteratorStruct->fsmCommandParameters_);
918  else
919  errorStr = "Expected to be in Paused. Unexpectedly, the current state is " +
920  currentState + ". Last State Machine error message was as follows: " +
921  iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getErrorMessage();
922 
923  if(errorStr != "")
924  {
925  __SS__ << "Iterator failed to pause because of the following error: " << errorStr;
926  throw std::runtime_error(ss.str());
927  }
928  return false;
929  }
930  else if(iteratorStruct->doHaltAction_)
931  {
932  //transition to halted state
933  __COUT__ << "Transitioning FSM to Halted..." << __E__;
934 
935  if(currentState == "Halted")
936  {
937  //done with early halt exit!
938  __COUT__ << "Transition to Halted complete." << __E__;
939  return true;
940  }
941  else if(currentState == "Running" || //launch transition to halt
942  currentState == "Paused")
943  errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
944  0,0,
945  "Abort",iteratorStruct->fsmName_,
946  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
947  WebUsers::DEFAULT_ITERATOR_USERNAME,
948  iteratorStruct->fsmCommandParameters_);
949  else
950  errorStr = "Expected to be in Halted. Unexpectedly, the current state is " +
951  currentState + ". Last State Machine error message was as follows: " +
952  iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getErrorMessage();
953 
954  if(errorStr != "")
955  {
956  __SS__ << "Iterator failed to halt because of the following error: " << errorStr;
957  throw std::runtime_error(ss.str());
958  }
959  return false;
960  }
961  else if(iteratorStruct->doResumeAction_)
962  {
963  //Note: check command gets one shot to resume
964 
965  //transition to running state
966  __COUT__ << "Transitioning FSM to Running..." << __E__;
967 
968  if(currentState == "Paused") //launch transition to running
969  errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
970  0,0,
971  "Resume",iteratorStruct->fsmName_,
972  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
973  WebUsers::DEFAULT_ITERATOR_USERNAME,
974  iteratorStruct->fsmCommandParameters_);
975 
976  if(errorStr != "")
977  {
978  __SS__ << "Iterator failed to run because of the following error: " << errorStr;
979  throw std::runtime_error(ss.str());
980  }
981  return false;
982  }
983 
984 
985 
987  //normal running
988 
989  if(currentState != "Running")
990  {
991  if(iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
992  [IterateConfiguration::commandRunParams_.DurationInSeconds_] ==
993  "DONE" &&
994  currentState == "Configured")
995  {
996  //indication of done
997  __COUT__ << "Reached end of run " <<
998  iteratorStruct->fsmNextRunNumber_ << __E__;
999  return true;
1000  }
1001 
1002  errorStr = "Expected to be in Running. Unexpectedly, the current state is " +
1003  currentState + ". Last State Machine error message was as follows: " +
1004  iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getErrorMessage();
1005  }
1006  else //else in Running state! Check for end of run
1007  {
1008  bool waitOnRunningThreads = false;
1009  if("True" == iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
1010  [IterateConfiguration::commandRunParams_.WaitOnRunningThreads_])
1011  waitOnRunningThreads = true;
1012 
1013  time_t remainingDurationInSeconds; //parameter converted during start to the stop linux timestamp
1014  sscanf(iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
1015  [IterateConfiguration::commandRunParams_.DurationInSeconds_].c_str(),
1016  "%ld",&remainingDurationInSeconds);
1017 
1018 
1019  __COUT__ << "waitOnRunningThreads " << waitOnRunningThreads << __E__;
1020  __COUT__ << "remainingDurationInSeconds " << remainingDurationInSeconds << __E__;
1021 
1023  //priority 1 is waiting on running threads
1024  if(waitOnRunningThreads)
1025  {
1026  // get status of all running FE workloops
1027  Supervisor* theSupervisor =
1028  iteratorStruct->theIterator_->theSupervisor_;
1029 
1030  bool allFrontEndsAreDone = true;
1031  for (auto& it :
1032  theSupervisor->theSupervisorDescriptorInfo_.getFEDescriptors())
1033  {
1034  try
1035  {
1036  std::string status = theSupervisor->send(it.second,
1037  "WorkLoopStatusRequest");
1038 
1039  __COUT__ << "FESupervisor instance " << it.first <<
1040  " has status = " << status << std::endl;
1041 
1042  if(status != "Done")
1043  {
1044  allFrontEndsAreDone = false;
1045  break;
1046  }
1047  }
1048  catch (xdaq::exception::Exception& e)
1049  {
1050  __SS__ << "Could not retrieve status from FESupervisor instance " <<
1051  it.first << "." << std::endl;
1052  __COUT_WARN__ << ss.str();
1053  errorStr = ss.str();
1054 
1055  if(errorStr != "")
1056  {
1057  __SS__ << "Iterator failed to run because of the following error: " << errorStr;
1058  throw std::runtime_error(ss.str());
1059  }
1060  }
1061  }
1062 
1063  if(allFrontEndsAreDone)
1064  {
1065  //need to end run!
1066  __COUT__ << "FE workloops all complete! Stopping run..." << __E__;
1067 
1068  errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
1069  0,0,
1070  "Stop",iteratorStruct->fsmName_,
1071  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1072  WebUsers::DEFAULT_ITERATOR_USERNAME,
1073  iteratorStruct->fsmCommandParameters_);
1074 
1075  if(errorStr != "")
1076  {
1077  __SS__ << "Iterator failed to stop run because of the following error: " << errorStr;
1078  throw std::runtime_error(ss.str());
1079  }
1080 
1081  //write indication of run done into duration
1082  iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
1083  [IterateConfiguration::commandRunParams_.DurationInSeconds_] =
1084  "DONE";
1085 
1086  return false;
1087  }
1088 
1089  }
1090 
1092  //priority 2 is duration, if <= 0 it is ignored
1093  if(remainingDurationInSeconds > 1)
1094  {
1095  --remainingDurationInSeconds;
1096 
1097  //write back to string
1098  char str[200];
1099  sprintf(str,"%ld",remainingDurationInSeconds);
1100  iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
1101  [IterateConfiguration::commandRunParams_.DurationInSeconds_] =
1102  str; //re-store as string
1103  }
1104  else if(remainingDurationInSeconds == 1)
1105  {
1106  //need to end run!
1107  __COUT__ << "Time duration reached! Stopping run..." << __E__;
1108 
1109  errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
1110  0,0,
1111  "Stop",iteratorStruct->fsmName_,
1112  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1113  WebUsers::DEFAULT_ITERATOR_USERNAME,
1114  iteratorStruct->fsmCommandParameters_);
1115 
1116  if(errorStr != "")
1117  {
1118  __SS__ << "Iterator failed to stop run because of the following error: " << errorStr;
1119  throw std::runtime_error(ss.str());
1120  }
1121 
1122  //write indication of run done into duration
1123  iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
1124  [IterateConfiguration::commandRunParams_.DurationInSeconds_] =
1125  "DONE";
1126 
1127  return false;
1128  }
1129  }
1130 
1131  if(errorStr != "")
1132  {
1133  __SS__ << "Iterator failed to run because of the following error: " << errorStr;
1134  throw std::runtime_error(ss.str());
1135  }
1136  return false;
1137 }
1138 
1139 //========================================================================================================================
1140 //return true if done
1141 bool Iterator::checkCommandConfigure(IteratorWorkLoopStruct *iteratorStruct)
1142 {
1143  sleep(1); //sleep to give FSM time to transition
1144 
1145  //all RunControlStateMachine access commands should be mutually exclusive with Supervisor main thread state machine accesses
1146  //should be mutually exclusive with Supervisor main thread state machine accesses
1147  //lockout the messages array for the remainder of the scope
1148  //this guarantees the reading thread can safely access the messages
1149  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for FSM access" << __E__;
1150  std::lock_guard<std::mutex> lock(iteratorStruct->theIterator_->theSupervisor_->stateMachineAccessMutex_);
1151  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have FSM access" << __E__;
1152 
1153  if(iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.isInTransition())
1154  return false;
1155 
1156  std::string errorStr = "";
1157  std::string currentState = iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getCurrentStateName();
1158 
1159  if(currentState == "Halted")
1160  errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
1161  0,0,
1162  "Configure",iteratorStruct->fsmName_,
1163  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1164  WebUsers::DEFAULT_ITERATOR_USERNAME,
1165  iteratorStruct->fsmCommandParameters_);
1166  else if(currentState != "Configured")
1167  errorStr = "Expected to be in Configure. Unexpectedly, the current state is " +
1168  currentState + "." + ". Last State Machine error message was as follows: " +
1169  iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getErrorMessage();
1170  else //else successfully done (in Configured state!)
1171  {
1172  __COUT__ << "checkCommandConfigureAlias complete." << __E__;
1173  return true;
1174  }
1175 
1176  if(errorStr != "")
1177  {
1178  __SS__ << "Iterator failed to configure with system alias '" <<
1179  (iteratorStruct->fsmCommandParameters_.size()?
1180  iteratorStruct->fsmCommandParameters_[0]:"UNKNOWN") <<
1181  "' because of the following error: " << errorStr;
1182  throw std::runtime_error(ss.str());
1183  }
1184  return false;
1185 }
1186 
1187 
1188 //========================================================================================================================
1189 bool Iterator::handleCommandRequest(HttpXmlDocument& xmldoc,
1190  const std::string& command, const std::string& parameter)
1191 {
1192  if(command == "iteratePlay")
1193  {
1194  playIterationPlan(xmldoc,parameter);
1195  return true;
1196  }
1197  else if(command == "iteratePause")
1198  {
1199  pauseIterationPlan(xmldoc);
1200  return true;
1201  }
1202  else if(command == "iterateHalt")
1203  {
1204  haltIterationPlan(xmldoc);
1205  return true;
1206  }
1207  else if(command == "getIterationPlanStatus")
1208  {
1209  getIterationPlanStatus(xmldoc);
1210  return true;
1211  }
1212  else //return true if iterator has control of state machine
1213  {
1214  //lockout the messages array for the remainder of the scope
1215  //this guarantees the reading thread can safely access the messages
1216  if(theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for iterator access" << __E__;
1217  std::lock_guard<std::mutex> lock(accessMutex_);
1218  if(theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have iterator access" << __E__;
1219 
1220  if(iteratorBusy_)
1221  {
1222  __SS__ << "Error - Can not accept request because the Iterator " <<
1223  "is currently " <<
1224  "in control of State Machine progress. ";
1225  __COUT_ERR__ << "\n" << ss.str();
1226  __MOUT_ERR__ << "\n" << ss.str();
1227 
1228  xmldoc.addTextElementToData("state_tranisition_attempted", "0"); //indicate to GUI transition NOT attempted
1229  xmldoc.addTextElementToData("state_tranisition_attempted_err",
1230  ss.str()); //indicate to GUI transition NOT attempted
1231 
1232  return true; //to block other commands
1233  }
1234  }
1235  return false;
1236 }
1237 
1238 //========================================================================================================================
1239 void Iterator::playIterationPlan(HttpXmlDocument& xmldoc, const std::string& planName)
1240 {
1241  __MOUT__ << "Attempting to play iteration plan '" << planName << ".'" << __E__;
1242  __COUT__ << "Attempting to play iteration plan '" << planName << ".'" << __E__;
1243 
1244 
1245 
1246  //setup "play" command
1247 
1248  //lockout the messages array for the remainder of the scope
1249  //this guarantees the reading thread can safely access the messages
1250  if(theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for iterator access" << __E__;
1251  std::lock_guard<std::mutex> lock(accessMutex_);
1252  if(theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have iterator access" << __E__;
1253 
1254  if(!activePlanIsRunning_ && !commandPlay_)
1255  {
1256  if(!workloopRunning_)
1257  {
1258  //start thread with member variables initialized
1259 
1260  workloopRunning_ = true;
1261 
1262  //must start thread first
1263  std::thread([](Iterator *iterator){ Iterator::IteratorWorkLoop(iterator); },this).detach();
1264  }
1265 
1266  activePlanName_ = planName;
1267  commandPlay_ = true;
1268  }
1269  else
1270  {
1271  __SS__ << "Invalid play command attempted. Can only play when the Iterator is inactive or paused." <<
1272  " If you would like to restart an iteration plan, first try halting." << __E__;
1273  __MOUT__ << ss.str();
1274 
1275  xmldoc.addTextElementToData("error_message", ss.str());
1276 
1277  __COUT__ << "Invalid play command attempted. " <<
1278  commandPlay_ << " " <<
1279  activePlanName_ << __E__;
1280  }
1281 }
1282 
1283 //========================================================================================================================
1284 void Iterator::pauseIterationPlan(HttpXmlDocument& xmldoc)
1285 {
1286  __MOUT__ << "Attempting to pause iteration plan '" << activePlanName_ << ".'" << __E__;
1287  __COUT__ << "Attempting to pause iteration plan '" << activePlanName_ << ".'" << __E__;
1288 
1289  //setup "pause" command
1290 
1291  //lockout the messages array for the remainder of the scope
1292  //this guarantees the reading thread can safely access the messages
1293  if(theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for iterator access" << __E__;
1294  std::lock_guard<std::mutex> lock(accessMutex_);
1295  if(theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have iterator access" << __E__;
1296 
1297  if(workloopRunning_ && activePlanIsRunning_ && !commandPause_)
1298  {
1299  commandPause_ = true;
1300  }
1301  else
1302  {
1303  __SS__ << "Invalid pause command attempted. Can only pause when running." << __E__;
1304  __MOUT__ << ss.str();
1305 
1306  xmldoc.addTextElementToData("error_message", ss.str());
1307 
1308  __COUT__ << "Invalid pause command attempted. " <<
1309  workloopRunning_ << " " <<
1310  activePlanIsRunning_ << " " <<
1311  commandPause_ << " " <<
1312  activePlanName_ << __E__;
1313  }
1314 }
1315 
1316 //========================================================================================================================
1317 void Iterator::haltIterationPlan(HttpXmlDocument& xmldoc)
1318 {
1319  __MOUT__ << "Attempting to halt iteration plan '" << activePlanName_ << ".'" << __E__;
1320  __COUT__ << "Attempting to halt iteration plan '" << activePlanName_ << ".'" << __E__;
1321 
1322  //setup "halt" command
1323 
1324  //lockout the messages array for the remainder of the scope
1325  //this guarantees the reading thread can safely access the messages
1326  if(theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for iterator access" << __E__;
1327  std::lock_guard<std::mutex> lock(accessMutex_);
1328  if(theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have iterator access" << __E__;
1329 
1330  if(activePlanIsRunning_ && !commandHalt_)
1331  {
1332  if(workloopRunning_)
1333  {
1334  __COUT__ << "Passing halt command to iterator thread." << __E__;
1335  commandHalt_ = true;
1336  }
1337  else //no thread, so reset 'Error' without command to thread
1338  {
1339  __COUT__ << "No thread, so conducting halt." << __E__;
1340  activePlanIsRunning_ = false;
1341  iteratorBusy_ = false;
1342 
1343  try
1344  {
1345  Iterator::haltStateMachine(theSupervisor_, lastFsmName_);
1346  }
1347  catch(const std::runtime_error& e)
1348  {
1349  xmldoc.addTextElementToData("error_message", e.what());
1350  }
1351  }
1352  }
1353  else
1354  {
1355  __COUT__ << "No thread, so conducting halt." << __E__;
1356 
1357  bool haltAttempted = false;
1358  try
1359  {
1360  haltAttempted = Iterator::haltStateMachine(theSupervisor_, lastFsmName_);
1361  }
1362  catch(const std::runtime_error& e)
1363  {
1364  haltAttempted = false;
1365  }
1366 
1367  if(!haltAttempted) //then show error
1368  {
1369  __SS__ << "Invalid halt command attempted. Can only halt when there is an active iteration plan." << __E__;
1370  __MOUT__ << ss.str();
1371 
1372  xmldoc.addTextElementToData("error_message", ss.str());
1373 
1374  __COUT__ << "Invalid halt command attempted. " <<
1375  workloopRunning_ << " " <<
1376  activePlanIsRunning_ << " " <<
1377  commandHalt_ << " " <<
1378  activePlanName_ << __E__;
1379  }
1380  else
1381  __COUT__ << "Halt was attempted." << __E__;
1382  }
1383 }
1384 
1385 //========================================================================================================================
1386 // return state machine and iterator status
1387 void Iterator::getIterationPlanStatus(HttpXmlDocument& xmldoc)
1388 {
1389  xmldoc.addTextElementToData("current_state", theSupervisor_->theStateMachine_.getCurrentStateName());
1390  xmldoc.addTextElementToData("in_transition", theSupervisor_->theStateMachine_.isInTransition() ? "1" : "0");
1391  if(theSupervisor_->theStateMachine_.isInTransition())
1392  xmldoc.addTextElementToData("transition_progress",
1393  theSupervisor_->theProgressBar_.readPercentageString());
1394  else
1395  xmldoc.addTextElementToData("transition_progress", "100");
1396 
1397  char tmp[20];
1398  sprintf(tmp,"%lu",theSupervisor_->theStateMachine_.getTimeInState());
1399  xmldoc.addTextElementToData("time_in_state", tmp);
1400 
1401 
1402 
1403  //lockout the messages array for the remainder of the scope
1404  //this guarantees the reading thread can safely access the messages
1405  if(theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for iterator access" << __E__;
1406  std::lock_guard<std::mutex> lock(accessMutex_);
1407  if(theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have iterator access" << __E__;
1408 
1409  xmldoc.addTextElementToData("active_plan", activePlanName_);
1410  xmldoc.addTextElementToData("last_started_plan", lastStartedPlanName_);
1411  xmldoc.addTextElementToData("last_finished_plan", lastFinishedPlanName_);
1412 
1413  sprintf(tmp,"%u",activeCommandIndex_);
1414  xmldoc.addTextElementToData("current_command_index", tmp);
1415  sprintf(tmp,"%ld",time(0) - activeCommandStartTime_);
1416  xmldoc.addTextElementToData("current_command_duration", tmp);
1417 
1418  if(activePlanIsRunning_ && iteratorBusy_)
1419  {
1420  if(workloopRunning_)
1421  xmldoc.addTextElementToData("active_plan_status", "Running");
1422  else
1423  xmldoc.addTextElementToData("active_plan_status", "Error");
1424  }
1425  else if(!activePlanIsRunning_ && iteratorBusy_)
1426  xmldoc.addTextElementToData("active_plan_status", "Paused");
1427  else
1428  xmldoc.addTextElementToData("active_plan_status", "Inactive");
1429 
1430  xmldoc.addTextElementToData("error_message", errorMessage_);
1431 }
1432 
1433