otsdaq  v2_04_02
Iterator.cc
1 #include "otsdaq/GatewaySupervisor/Iterator.h"
2 #include "otsdaq/CoreSupervisors/CoreSupervisorBase.h"
3 #include "otsdaq/GatewaySupervisor/GatewaySupervisor.h"
4 #include "otsdaq/Macros/CoutMacros.h"
5 #include "otsdaq/MessageFacility/MessageFacility.h"
6 #include "otsdaq/WebUsersUtilities/WebUsers.h"
7 
8 #include <iostream>
9 #include <thread> //for std::thread
10 
11 #undef __MF_SUBJECT__
12 #define __MF_SUBJECT__ "Iterator"
13 
14 using namespace ots;
15 
16 //========================================================================================================================
17 Iterator::Iterator(GatewaySupervisor* supervisor)
18  : workloopRunning_(false)
19  , activePlanIsRunning_(false)
20  , iteratorBusy_(false)
21  , commandPlay_(false)
22  , commandPause_(false)
23  , commandHalt_(false)
24  , activePlanName_("")
25  , activeCommandIndex_(-1)
26  , activeCommandStartTime_(0)
27  , theSupervisor_(supervisor)
28 {
29  __MOUT__ << "Iterator constructed." << __E__;
30  __COUT__ << "Iterator constructed." << __E__;
31 }
32 
33 //========================================================================================================================
34 Iterator::~Iterator(void) {}
35 
36 //========================================================================================================================
37 void Iterator::IteratorWorkLoop(Iterator* iterator) try
38 {
39  __MOUT__ << "Iterator work loop starting..." << __E__;
40  __COUT__ << "Iterator work loop starting..." << __E__;
41 
42  // mutex init scope
43  {
44  // lockout the messages array for the remainder of the scope
45  // this guarantees the reading thread can safely access the messages
46  if(iterator->theSupervisor_->VERBOSE_MUTEX)
47  __COUT__ << "Waiting for iterator access" << __E__;
48  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
49  if(iterator->theSupervisor_->VERBOSE_MUTEX)
50  __COUT__ << "Have iterator access" << __E__;
51 
52  iterator->errorMessage_ = ""; // clear error message
53  }
54 
55  ConfigurationManagerRW theConfigurationManager(
56  WebUsers::DEFAULT_ITERATOR_USERNAME); // this is a restricted username
57  // theConfigurationManager.init();
58  theConfigurationManager.getAllTableInfo(true); // to prep all info
59 
60  IteratorWorkLoopStruct theIteratorStruct(iterator, &theConfigurationManager);
61 
62  const IterateTable* itConfig;
63 
64  std::vector<IterateTable::Command> commands;
65 
66  while(1)
67  {
68  // Process:
69  // - always "listen" for commands
70  // - play: if no plan running, activePlanIsRunning_ = true,
71  // and start or continue plan based on name/commandIndex
72  // - pause: if plan playing, pause it, activePlanIsRunning_ = false
73  // and do not clear commandIndex or name, iteratorBusy_ = true
74  // - halt: if plan playing or not, activePlanIsRunning_ = false
75  // and clear commandIndex or name, iteratorBusy_ = false
76  // - when running
77  // - go through each command
78  // - start the command, commandBusy = true
79  // - check for complete, then commandBusy = false
80 
81  // start command handling
82  // define mutex scope
83  {
84  // lockout the messages array for the remainder of the scope
85  // this guarantees the reading thread can safely access the messages
86  if(iterator->theSupervisor_->VERBOSE_MUTEX)
87  __COUT__ << "Waiting for iterator access" << __E__;
88  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
89  if(iterator->theSupervisor_->VERBOSE_MUTEX)
90  __COUT__ << "Have iterator access" << __E__;
91 
92  if(iterator->commandPlay_)
93  {
94  iterator->commandPlay_ = false; // clear
95 
96  if(!iterator->activePlanIsRunning_)
97  {
98  // valid PLAY command!
99 
100  iterator->activePlanIsRunning_ = true;
101  iterator->iteratorBusy_ = true;
102 
103  if(theIteratorStruct.activePlan_ != iterator->activePlanName_)
104  {
105  __COUT__ << "New plan name encountered old="
106  << theIteratorStruct.activePlan_
107  << " vs new=" << iterator->activePlanName_ << __E__;
108  theIteratorStruct.commandIndex_ = -1; // reset for new plan
109  }
110 
111  theIteratorStruct.activePlan_ = iterator->activePlanName_;
112  iterator->lastStartedPlanName_ = iterator->activePlanName_;
113 
114  if(theIteratorStruct.commandIndex_ == (unsigned int)-1)
115  {
116  __COUT__ << "Starting plan '" << theIteratorStruct.activePlan_
117  << ".'" << __E__;
118  __MOUT__ << "Starting plan '" << theIteratorStruct.activePlan_
119  << ".'" << __E__;
120  }
121  else
122  {
123  theIteratorStruct.doResumeAction_ = true;
124  __COUT__ << "Continuing plan '" << theIteratorStruct.activePlan_
125  << "' at command index "
126  << theIteratorStruct.commandIndex_ << ". " << __E__;
127  __MOUT__ << "Continuing plan '" << theIteratorStruct.activePlan_
128  << "' at command index "
129  << theIteratorStruct.commandIndex_ << ". " << __E__;
130  }
131  }
132  }
133  else if(iterator->commandPause_ && !theIteratorStruct.doPauseAction_)
134  {
135  theIteratorStruct.doPauseAction_ = true;
136  iterator->commandPause_ = false; // clear
137  }
138  else if(iterator->commandHalt_ && !theIteratorStruct.doHaltAction_)
139  {
140  theIteratorStruct.doHaltAction_ = true;
141  iterator->commandHalt_ = false; // clear
142  }
143 
144  theIteratorStruct.running_ = iterator->activePlanIsRunning_;
145 
146  if(iterator
147  ->activeCommandIndex_ != // update active command status if changed
148  theIteratorStruct.commandIndex_)
149  {
150  iterator->activeCommandIndex_ = theIteratorStruct.commandIndex_;
151  iterator->activeCommandStartTime_ = time(0); // reset on any change
152 
153  if(theIteratorStruct.commandIndex_ <
154  theIteratorStruct.commandIterations_.size())
155  iterator->activeCommandIteration_ =
156  theIteratorStruct
157  .commandIterations_[theIteratorStruct.commandIndex_];
158  else
159  iterator->activeCommandIteration_ = -1;
160 
161  iterator->depthIterationStack_.clear();
162  for(const auto& depthIteration : theIteratorStruct.stepIndexStack_)
163  iterator->depthIterationStack_.push_back(depthIteration);
164  // if(theIteratorStruct.stepIndexStack_.size())
165  // iterator->activeLoopIteration_ =
166  // theIteratorStruct.stepIndexStack_.back(); else
167  // iterator->activeLoopIteration_ = -1;
168  }
169 
170  } // end command handling and iterator mutex
171 
174  // do halt or pause action outside of iterator mutex
175 
176  if(theIteratorStruct.doPauseAction_)
177  {
178  // valid PAUSE command!
179 
180  // safely pause plan!
181  // i.e. check that command is complete
182 
183  __COUT__ << "Waiting to pause..." << __E__;
184  while(!iterator->checkCommand(&theIteratorStruct))
185  __COUT__ << "Waiting to pause..." << __E__;
186 
187  __COUT__ << "Completing pause..." << __E__;
188 
189  theIteratorStruct.doPauseAction_ = false; // clear
190 
191  // lockout the messages array for the remainder of the scope
192  // this guarantees the reading thread can safely access the messages
193  if(iterator->theSupervisor_->VERBOSE_MUTEX)
194  __COUT__ << "Waiting for iterator access" << __E__;
195  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
196  if(iterator->theSupervisor_->VERBOSE_MUTEX)
197  __COUT__ << "Have iterator access" << __E__;
198 
199  iterator->activePlanIsRunning_ = false;
200 
201  __COUT__ << "Paused plan '" << theIteratorStruct.activePlan_
202  << "' at command index " << theIteratorStruct.commandIndex_ << ". "
203  << __E__;
204  __MOUT__ << "Paused plan '" << theIteratorStruct.activePlan_
205  << "' at command index " << theIteratorStruct.commandIndex_ << ". "
206  << __E__;
207 
208  continue; // resume workloop
209  }
210  else if(theIteratorStruct.doHaltAction_)
211  {
212  // valid HALT command!
213 
214  // safely end plan!
215  // i.e. check that command is complete
216 
217  __COUT__ << "Waiting to halt..." << __E__;
218  while(!iterator->checkCommand(&theIteratorStruct))
219  __COUT__ << "Waiting to halt..." << __E__;
220 
221  __COUT__ << "Completing halt..." << __E__;
222 
223  theIteratorStruct.doHaltAction_ = false; // clear
224 
225  iterator->haltIterator(iterator, &theIteratorStruct);
226 
227  // //last ditch effort to make sure FSM is halted
228  // iterator->haltIterator(
229  // iterator->theSupervisor_,
230  // theIteratorStruct.fsmName_);
231  //
232  // //lockout the messages array for the remainder of the scope
233  // //this guarantees the reading thread can safely access the
234  // messages
235  // if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting
236  // for iterator access" << __E__; std::lock_guard<std::mutex>
237  // lock(iterator->accessMutex_);
238  // if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have
239  // iterator access" << __E__;
240  //
241  // iterator->activePlanIsRunning_ = false;
242  // iterator->iteratorBusy_ = false;
243  //
244  // __COUT__ << "Halted plan '" << theIteratorStruct.activePlan_ << "'
245  // at command index " << theIteratorStruct.commandIndex_ <<
246  //". " << __E__;
247  // __MOUT__ << "Halted plan '" << theIteratorStruct.activePlan_ << "'
248  // at command index " << theIteratorStruct.commandIndex_ <<
249  //". " << __E__;
250  //
251  // theIteratorStruct.activePlan_ = ""; //clear
252  // theIteratorStruct.commandIndex_ = -1; //clear
253 
254  continue; // resume workloop
255  }
256 
259  // handle running
260  // __COUT__ << "thinking.." << theIteratorStruct.running_ << " " <<
261  // theIteratorStruct.activePlan_ << " cmd=" <<
262  // theIteratorStruct.commandIndex_ << __E__;
263  if(theIteratorStruct.running_ &&
264  theIteratorStruct.activePlan_ !=
265  "") // important, because after errors, still "running" until halt
266  {
267  if(theIteratorStruct.commandIndex_ == (unsigned int)-1)
268  {
269  // initialize the running plan
270 
271  __COUT__ << "Get commands" << __E__;
272 
273  theIteratorStruct.commandIndex_ = 0;
274 
275  theIteratorStruct.cfgMgr_
276  ->init(); // completely reset to re-align with any changes
277  itConfig = theIteratorStruct.cfgMgr_->__GET_CONFIG__(IterateTable);
278 
279  theIteratorStruct.commands_ = itConfig->getPlanCommands(
280  theIteratorStruct.cfgMgr_, theIteratorStruct.activePlan_);
281 
282  // reset commandIteration counts
283  theIteratorStruct.commandIterations_.clear();
284  for(auto& command : theIteratorStruct.commands_)
285  {
286  theIteratorStruct.commandIterations_.push_back(0);
287  __COUT__ << "command " << command.type_ << __E__;
288  __COUT__ << "table "
289  << IterateTable::commandToTableMap_.at(command.type_)
290  << __E__;
291  __COUT__ << "param count = " << command.params_.size() << __E__;
292 
293  for(auto& param : command.params_)
294  {
295  __COUT__ << "\t param " << param.first << " : " << param.second
296  << __E__;
297  }
298  }
299 
300  theIteratorStruct.originalTrackChanges_ =
301  ConfigurationInterface::isVersionTrackingEnabled();
302  theIteratorStruct.originalConfigGroup_ =
303  theIteratorStruct.cfgMgr_->getActiveGroupName();
304  theIteratorStruct.originalConfigKey_ =
305  theIteratorStruct.cfgMgr_->getActiveGroupKey();
306 
307  __COUT__ << "originalTrackChanges "
308  << theIteratorStruct.originalTrackChanges_ << __E__;
309  __COUT__ << "originalConfigGroup "
310  << theIteratorStruct.originalConfigGroup_ << __E__;
311  __COUT__ << "originalConfigKey " << theIteratorStruct.originalConfigKey_
312  << __E__;
313 
314  } // end initial section
315 
316  if(!theIteratorStruct.commandBusy_)
317  {
318  if(theIteratorStruct.commandIndex_ < theIteratorStruct.commands_.size())
319  {
320  // execute command
321  theIteratorStruct.commandBusy_ = true;
322 
323  __COUT__ << "Iterator starting command "
324  << theIteratorStruct.commandIndex_ + 1 << ": "
325  << theIteratorStruct
326  .commands_[theIteratorStruct.commandIndex_]
327  .type_
328  << __E__;
329  __MOUT__ << "Iterator starting command "
330  << theIteratorStruct.commandIndex_ + 1 << ": "
331  << theIteratorStruct
332  .commands_[theIteratorStruct.commandIndex_]
333  .type_
334  << __E__;
335 
336  iterator->startCommand(&theIteratorStruct);
337  }
338  else if(theIteratorStruct.commandIndex_ ==
339  theIteratorStruct.commands_.size()) // Done!
340  {
341  __COUT__ << "Finished Iteration Plan '"
342  << theIteratorStruct.activePlan_ << __E__;
343  __MOUT__ << "Finished Iteration Plan '"
344  << theIteratorStruct.activePlan_ << __E__;
345 
346  __COUT__ << "Reverting track changes." << __E__;
347  ConfigurationInterface::setVersionTrackingEnabled(
348  theIteratorStruct.originalTrackChanges_);
349 
350  __COUT__ << "Activating original group..." << __E__;
351  try
352  {
353  theIteratorStruct.cfgMgr_->activateTableGroup(
354  theIteratorStruct.originalConfigGroup_,
355  theIteratorStruct.originalConfigKey_);
356  }
357  catch(...)
358  {
359  __COUT_WARN__ << "Original group could not be activated."
360  << __E__;
361  }
362 
363  // leave FSM halted
364  __COUT__ << "Completing halt..." << __E__;
365 
366  iterator->haltIterator(iterator, &theIteratorStruct);
367 
368  //
369  // //leave FSM halted
370  // iterator->haltIterator(
371  // iterator->theSupervisor_,
372  // theIteratorStruct.fsmName_);
373  //
374  // //lockout the messages array for the remainder of
375  // the scope
376  // //this guarantees the reading thread can safely
377  // access the messages
378  // if(iterator->theSupervisor_->VERBOSE_MUTEX)
379  //__COUT__
380  //<< "Waiting for iterator access" << __E__;
381  // std::lock_guard<std::mutex>
382  // lock(iterator->accessMutex_);
383  // if(iterator->theSupervisor_->VERBOSE_MUTEX)
384  //__COUT__
385  //<< "Have iterator access" << __E__;
386  //
387  // //similar to halt
388  // iterator->activePlanIsRunning_ = false;
389  // iterator->iteratorBusy_ = false;
390  //
391  // iterator->lastStartedPlanName_ =
392  // theIteratorStruct.activePlan_;
393  // theIteratorStruct.activePlan_ = ""; //clear
394  // theIteratorStruct.commandIndex_ = -1; //clear
395  }
396  }
397  else if(theIteratorStruct.commandBusy_)
398  {
399  // check for command completion
400  if(iterator->checkCommand(&theIteratorStruct))
401  {
402  theIteratorStruct.commandBusy_ = false; // command complete
403 
404  ++theIteratorStruct.commandIndex_;
405 
406  __COUT__ << "Ready for next command. Done with "
407  << theIteratorStruct.commandIndex_ << " of "
408  << theIteratorStruct.commands_.size() << __E__;
409  __MOUT__ << "Iterator ready for next command. Done with "
410  << theIteratorStruct.commandIndex_ << " of "
411  << theIteratorStruct.commands_.size() << __E__;
412  }
413 
414  // Note: check command gets one shot to resume
415  if(theIteratorStruct.doResumeAction_) // end resume action
416  theIteratorStruct.doResumeAction_ = false;
417  }
418 
419  } // end running
420  else
421  sleep(1); // when inactive sleep a lot
422 
425 
426  // __COUT__ << "end loop.." << theIteratorStruct.running_ << " " <<
427  // theIteratorStruct.activePlan_ << " cmd=" <<
428  // theIteratorStruct.commandIndex_ << __E__;
429  }
430 
431  iterator->workloopRunning_ = false; // if we ever exit
432 }
433 catch(const std::runtime_error& e)
434 {
435  __SS__ << "Encountered error in Iterator thread:\n" << e.what() << __E__;
436  __COUT_ERR__ << ss.str();
437 
438  // lockout the messages array for the remainder of the scope
439  // this guarantees the reading thread can safely access the messages
440  if(iterator->theSupervisor_->VERBOSE_MUTEX)
441  __COUT__ << "Waiting for iterator access" << __E__;
442  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
443  if(iterator->theSupervisor_->VERBOSE_MUTEX)
444  __COUT__ << "Have iterator access" << __E__;
445 
446  iterator->workloopRunning_ = false; // if we ever exit
447  iterator->errorMessage_ = ss.str();
448 }
449 catch(...)
450 {
451  __SS__ << "Encountered unknown error in Iterator thread." << __E__;
452  __COUT_ERR__ << ss.str();
453 
454  // lockout the messages array for the remainder of the scope
455  // this guarantees the reading thread can safely access the messages
456  if(iterator->theSupervisor_->VERBOSE_MUTEX)
457  __COUT__ << "Waiting for iterator access" << __E__;
458  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
459  if(iterator->theSupervisor_->VERBOSE_MUTEX)
460  __COUT__ << "Have iterator access" << __E__;
461 
462  iterator->workloopRunning_ = false; // if we ever exit
463  iterator->errorMessage_ = ss.str();
464 } // end IteratorWorkLoop()
465 
466 //========================================================================================================================
467 void Iterator::startCommand(IteratorWorkLoopStruct* iteratorStruct) try
468 {
469  {
470  int i = 0;
471  for(const auto& depthIteration : iteratorStruct->stepIndexStack_)
472  {
473  __COUT__ << i++ << ":" << depthIteration << __E__;
474  }
475  }
476 
477  // should be mutually exclusive with GatewaySupervisor main thread state machine
478  // accesses lockout the messages array for the remainder of the scope this
479  // guarantees the reading thread can safely access the messages
480  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX)
481  __COUT__ << "Waiting for FSM access" << __E__;
482  std::lock_guard<std::mutex> lock(
483  iteratorStruct->theIterator_->theSupervisor_->stateMachineAccessMutex_);
484  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX)
485  __COUT__ << "Have FSM access" << __E__;
486 
487  // for out of range, throw exception - should never happen
488  if(iteratorStruct->commandIndex_ >= iteratorStruct->commands_.size())
489  {
490  __SS__ << "Out of range commandIndex = " << iteratorStruct->commandIndex_
491  << " in size = " << iteratorStruct->commands_.size() << __E__;
492  __SS_THROW__;
493  }
494 
495  // increment iteration count for command
496  ++iteratorStruct->commandIterations_[iteratorStruct->commandIndex_];
497 
498  std::string type = iteratorStruct->commands_[iteratorStruct->commandIndex_].type_;
499  if(type == IterateTable::COMMAND_BEGIN_LABEL)
500  {
501  return startCommandBeginLabel(iteratorStruct);
502  }
503  else if(type == IterateTable::COMMAND_CHOOSE_FSM)
504  {
505  return startCommandChooseFSM(
506  iteratorStruct,
507  iteratorStruct->commands_[iteratorStruct->commandIndex_]
508  .params_[IterateTable::commandChooseFSMParams_.NameOfFSM_]);
509  }
510  else if(type == IterateTable::COMMAND_CONFIGURE_ACTIVE_GROUP)
511  {
512  return startCommandConfigureActive(iteratorStruct);
513  }
514  else if(type == IterateTable::COMMAND_CONFIGURE_ALIAS)
515  {
516  return startCommandConfigureAlias(
517  iteratorStruct,
518  iteratorStruct->commands_[iteratorStruct->commandIndex_]
519  .params_[IterateTable::commandConfigureAliasParams_.SystemAlias_]);
520  }
521  else if(type == IterateTable::COMMAND_CONFIGURE_GROUP)
522  {
523  return startCommandConfigureGroup(iteratorStruct);
524  }
525  else if(type == IterateTable::COMMAND_EXECUTE_FE_MACRO)
526  {
527  return startCommandMacro(iteratorStruct, true /*isFEMacro*/);
528  }
529  else if(type == IterateTable::COMMAND_EXECUTE_MACRO)
530  {
531  return startCommandMacro(iteratorStruct, false /*isFEMacro*/);
532  }
533  else if(type == IterateTable::COMMAND_MODIFY_ACTIVE_GROUP)
534  {
535  return startCommandModifyActive(iteratorStruct);
536  }
537  else if(type == IterateTable::COMMAND_REPEAT_LABEL)
538  {
539  return startCommandRepeatLabel(iteratorStruct);
540  }
541  else if(type == IterateTable::COMMAND_RUN)
542  {
543  return startCommandRun(iteratorStruct);
544  }
545  else
546  {
547  __SS__ << "Failed attempt to start unrecognized command type = " << type << __E__;
548  __COUT_ERR__ << ss.str();
549  __SS_THROW__;
550  }
551 }
552 catch(...)
553 {
554  __COUT__ << "Error caught. Reverting track changes." << __E__;
555  ConfigurationInterface::setVersionTrackingEnabled(
556  iteratorStruct->originalTrackChanges_);
557 
558  __COUT__ << "Activating original group..." << __E__;
559  try
560  {
561  iteratorStruct->cfgMgr_->activateTableGroup(iteratorStruct->originalConfigGroup_,
562  iteratorStruct->originalConfigKey_);
563  }
564  catch(...)
565  {
566  __COUT_WARN__ << "Original group could not be activated." << __E__;
567  }
568  throw;
569 } // end startCommand()
570 
571 //========================================================================================================================
572 // checkCommand
573 // when busy for a while, start to sleep
574 // use sleep() or nanosleep()
575 bool Iterator::checkCommand(IteratorWorkLoopStruct* iteratorStruct) try
576 {
577  // for out of range, return done
578  if(iteratorStruct->commandIndex_ >= iteratorStruct->commands_.size())
579  {
580  __COUT__ << "Out of range commandIndex = " << iteratorStruct->commandIndex_
581  << " in size = " << iteratorStruct->commands_.size() << __E__;
582  return true;
583  }
584 
585  std::string type = iteratorStruct->commands_[iteratorStruct->commandIndex_].type_;
586  if(type == IterateTable::COMMAND_BEGIN_LABEL)
587  {
588  // do nothing
589  return true;
590  }
591  else if(type == IterateTable::COMMAND_CHOOSE_FSM)
592  {
593  // do nothing
594  return true;
595  }
596  else if(type == IterateTable::COMMAND_CONFIGURE_ALIAS ||
597  type == IterateTable::COMMAND_CONFIGURE_ACTIVE_GROUP ||
598  type == IterateTable::COMMAND_CONFIGURE_GROUP)
599  {
600  return checkCommandConfigure(iteratorStruct);
601  }
602  else if(type == IterateTable::COMMAND_EXECUTE_FE_MACRO)
603  {
604  return checkCommandMacro(iteratorStruct, true /*isFEMacro*/);
605  }
606  else if(type == IterateTable::COMMAND_EXECUTE_MACRO)
607  {
608  return checkCommandMacro(iteratorStruct, false /*isFEMacro*/);
609  }
610  else if(type == IterateTable::COMMAND_MODIFY_ACTIVE_GROUP)
611  {
612  // do nothing
613  return true;
614  }
615  else if(type == IterateTable::COMMAND_REPEAT_LABEL)
616  {
617  // do nothing
618  return true;
619  }
620  else if(type == IterateTable::COMMAND_RUN)
621  {
622  return checkCommandRun(iteratorStruct);
623  }
624  else
625  {
626  __SS__ << "Attempt to check unrecognized command type = " << type << __E__;
627  __COUT_ERR__ << ss.str();
628  __SS_THROW__;
629  }
630 }
631 catch(...)
632 {
633  __COUT__ << "Error caught. Reverting track changes." << __E__;
634  ConfigurationInterface::setVersionTrackingEnabled(
635  iteratorStruct->originalTrackChanges_);
636 
637  __COUT__ << "Activating original group..." << __E__;
638  try
639  {
640  iteratorStruct->cfgMgr_->activateTableGroup(iteratorStruct->originalConfigGroup_,
641  iteratorStruct->originalConfigKey_);
642  }
643  catch(...)
644  {
645  __COUT_WARN__ << "Original group could not be activated." << __E__;
646  }
647 
648  throw;
649 } // end checkCommand()
650 
651 //========================================================================================================================
652 void Iterator::startCommandChooseFSM(IteratorWorkLoopStruct* iteratorStruct,
653  const std::string& fsmName)
654 {
655  __COUT__ << "fsmName " << fsmName << __E__;
656 
657  iteratorStruct->fsmName_ = fsmName;
658  iteratorStruct->theIterator_->lastFsmName_ = fsmName;
659 
660  // Translate fsmName
661  // to gives us run alias (fsmRunAlias_) and next run number (fsmNextRunNumber_)
662 
663  // CAREFUL?? Threads
664 
666 
667  iteratorStruct->fsmRunAlias_ = "Run"; // default to "Run"
668 
669  // get stateMachineAliasFilter if possible
670  ConfigurationTree configLinkNode = iteratorStruct->cfgMgr_->getSupervisorTableNode(
671  iteratorStruct->theIterator_->theSupervisor_->getContextUID(),
672  iteratorStruct->theIterator_->theSupervisor_->getSupervisorUID());
673 
674  if(!configLinkNode.isDisconnected())
675  {
676  try // for backwards compatibility
677  {
678  ConfigurationTree fsmLinkNode =
679  configLinkNode.getNode("LinkToStateMachineTable");
680  if(!fsmLinkNode.isDisconnected())
681  iteratorStruct->fsmRunAlias_ =
682  fsmLinkNode.getNode(fsmName + "/RunDisplayAlias")
683  .getValue<std::string>();
684  else
685  __COUT_INFO__ << "FSM Link disconnected." << __E__;
686  }
687  catch(std::runtime_error& e)
688  {
689  //__COUT_INFO__ << e.what() << __E__;
690  __COUT_INFO__
691  << "No state machine Run alias. Ignoring and assuming alias of '"
692  << iteratorStruct->fsmRunAlias_ << ".'" << __E__;
693  }
694  catch(...)
695  {
696  __COUT_ERR__ << "Unknown error. Should never happen." << __E__;
697 
698  __COUT_INFO__
699  << "No state machine Run alias. Ignoring and assuming alias of '"
700  << iteratorStruct->fsmRunAlias_ << ".'" << __E__;
701  }
702  }
703  else
704  __COUT_INFO__ << "FSM Link disconnected." << __E__;
705 
706  __COUT__ << "fsmRunAlias_ = " << iteratorStruct->fsmRunAlias_ << __E__;
707 
709 
710  iteratorStruct->fsmNextRunNumber_ =
711  iteratorStruct->theIterator_->theSupervisor_->getNextRunNumber(
712  iteratorStruct->fsmName_);
713 
714  if(iteratorStruct->theIterator_->theSupervisor_->theStateMachine_
715  .getCurrentStateName() == "Running" ||
716  iteratorStruct->theIterator_->theSupervisor_->theStateMachine_
717  .getCurrentStateName() == "Paused")
718  --iteratorStruct->fsmNextRunNumber_; // current run number is one back
719 
720  __COUT__ << "fsmNextRunNumber_ = " << iteratorStruct->fsmNextRunNumber_ << __E__;
721 } // end startCommandChooseFSM()
722 
723 //========================================================================================================================
724 // return true if an action was attempted
725 bool Iterator::haltIterator(Iterator* iterator, IteratorWorkLoopStruct* iteratorStruct)
726 //(GatewaySupervisor* theSupervisor, const std::string& fsmName)
727 {
728  GatewaySupervisor* theSupervisor = iterator->theSupervisor_;
729  const std::string& fsmName = iterator->lastFsmName_;
730 
731  std::vector<std::string> fsmCommandParameters;
732  std::string errorStr = "";
733  std::string currentState = theSupervisor->theStateMachine_.getCurrentStateName();
734 
735  bool haltAttempted = true;
736  if(currentState == "Initialized" || currentState == "Halted")
737  {
738  __COUT__ << "Do nothing. Already halted." << __E__;
739  haltAttempted = false;
740  }
741  else if(currentState == "Running")
742  errorStr = theSupervisor->attemptStateMachineTransition(
743  0,
744  0,
745  "Abort",
746  fsmName,
747  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
748  WebUsers::DEFAULT_ITERATOR_USERNAME,
749  fsmCommandParameters);
750  else
751  errorStr = theSupervisor->attemptStateMachineTransition(
752  0,
753  0,
754  "Halt",
755  fsmName,
756  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
757  WebUsers::DEFAULT_ITERATOR_USERNAME,
758  fsmCommandParameters);
759 
760  if(haltAttempted)
761  {
762  if(errorStr != "")
763  {
764  __SS__ << "Iterator failed to halt because of the following error: "
765  << errorStr;
766  __SS_THROW__;
767  }
768 
769  // else successfully launched
770  __COUT__ << "FSM in transition = "
771  << theSupervisor->theStateMachine_.isInTransition() << __E__;
772  __COUT__ << "halting state machine launched." << __E__;
773  }
774 
775  // finish up halting the iterator
776  __COUT__ << "Conducting Iterator halt." << __E__;
777 
778  if(iteratorStruct)
779  {
780  __COUT__ << "Reverting track changes." << __E__;
781  ConfigurationInterface::setVersionTrackingEnabled(
782  iteratorStruct->originalTrackChanges_);
783 
784  __COUT__ << "Activating original group..." << __E__;
785  try
786  {
787  iteratorStruct->cfgMgr_->activateTableGroup(
788  iteratorStruct->originalConfigGroup_, iteratorStruct->originalConfigKey_);
789  }
790  catch(...)
791  {
792  __COUT_WARN__ << "Original group could not be activated." << __E__;
793  }
794  }
795 
796  // lockout the messages array for the remainder of the scope
797  // this guarantees the reading thread can safely access the messages
798  if(iterator->theSupervisor_->VERBOSE_MUTEX)
799  __COUT__ << "Waiting for iterator access" << __E__;
800  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
801  if(iterator->theSupervisor_->VERBOSE_MUTEX)
802  __COUT__ << "Have iterator access" << __E__;
803 
804  iterator->activePlanIsRunning_ = false;
805  iterator->iteratorBusy_ = false;
806 
807  // clear
808  iterator->activePlanName_ = "";
809  iterator->activeCommandIndex_ = -1;
810 
811  if(iteratorStruct)
812  {
813  __COUT__ << "Halted plan '" << iteratorStruct->activePlan_
814  << "' at command index " << iteratorStruct->commandIndex_ << ". "
815  << __E__;
816  __MOUT__ << "Halted plan '" << iteratorStruct->activePlan_
817  << "' at command index " << iteratorStruct->commandIndex_ << ". "
818  << __E__;
819 
820  iterator->lastStartedPlanName_ = iteratorStruct->activePlan_;
821  iteratorStruct->activePlan_ = ""; // clear
822  iteratorStruct->commandIndex_ = -1; // clear
823  }
824 
825  return haltAttempted;
826 } // end haltIterator()
827 
828 //========================================================================================================================
829 void Iterator::startCommandBeginLabel(IteratorWorkLoopStruct* iteratorStruct)
830 {
831  __COUT__ << "Entering label '"
832  << iteratorStruct->commands_[iteratorStruct->commandIndex_]
833  .params_[IterateTable::commandBeginLabelParams_.Label_]
834  << "'..." << std::endl;
835 
836  // add new step index to stack
837  iteratorStruct->stepIndexStack_.push_back(0);
838 } // end startCommandBeginLabel()
839 
840 //========================================================================================================================
841 void Iterator::startCommandRepeatLabel(IteratorWorkLoopStruct* iteratorStruct)
842 {
843  // search for first matching label backward and set command to there
844 
845  int numOfRepetitions;
846  sscanf(iteratorStruct->commands_[iteratorStruct->commandIndex_]
847  .params_[IterateTable::commandRepeatLabelParams_.NumberOfRepetitions_]
848  .c_str(),
849  "%d",
850  &numOfRepetitions);
851  __COUT__ << "numOfRepetitions remaining = " << numOfRepetitions << __E__;
852 
853  char repStr[200];
854 
855  if(numOfRepetitions <= 0)
856  {
857  // write original number of repetitions value back
858  sprintf(repStr, "%d", iteratorStruct->stepIndexStack_.back());
859  iteratorStruct->commands_[iteratorStruct->commandIndex_]
860  .params_[IterateTable::commandRepeatLabelParams_.NumberOfRepetitions_] =
861  repStr; // re-store as string
862 
863  // remove step index from stack
864  iteratorStruct->stepIndexStack_.pop_back();
865 
866  return; // no more repetitions
867  }
868 
869  --numOfRepetitions;
870 
871  // increment step index in stack
872  ++(iteratorStruct->stepIndexStack_.back());
873 
874  unsigned int i;
875  for(i = iteratorStruct->commandIndex_; i > 0;
876  --i) // assume 0 is always the fallback option
877  if(iteratorStruct->commands_[i].type_ == IterateTable::COMMAND_BEGIN_LABEL &&
878  iteratorStruct->commands_[iteratorStruct->commandIndex_]
879  .params_[IterateTable::commandRepeatLabelParams_.Label_] ==
880  iteratorStruct->commands_[i]
881  .params_[IterateTable::commandBeginLabelParams_.Label_])
882  break;
883 
884  sprintf(repStr, "%d", numOfRepetitions);
885  iteratorStruct->commands_[iteratorStruct->commandIndex_]
886  .params_[IterateTable::commandRepeatLabelParams_.NumberOfRepetitions_] =
887  repStr; // re-store as string
888 
889  iteratorStruct->commandIndex_ = i;
890  __COUT__ << "Jumping back to commandIndex " << iteratorStruct->commandIndex_ << __E__;
891 } // end startCommandRepeatLabel()
892 
893 //========================================================================================================================
894 void Iterator::startCommandRun(IteratorWorkLoopStruct* iteratorStruct)
895 {
896  iteratorStruct->runIsDone_ = false;
897  iteratorStruct->fsmCommandParameters_.clear();
898 
899  std::string errorStr = "";
900  std::string currentState = iteratorStruct->theIterator_->theSupervisor_
901  ->theStateMachine_.getCurrentStateName();
902 
903  // execute first transition (may need two)
904 
905  if(currentState == "Configured")
906  errorStr =
907  iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
908  0,
909  0,
910  "Start",
911  iteratorStruct->fsmName_,
912  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
913  WebUsers::DEFAULT_ITERATOR_USERNAME,
914  iteratorStruct->fsmCommandParameters_);
915  else
916  errorStr = "Can only Run from the Configured state. The current state is " +
917  currentState;
918 
919  if(errorStr != "")
920  {
921  __SS__ << "Iterator failed to run because of the following error: " << errorStr;
922  __SS_THROW__;
923  }
924 
925  // save original duration
926  sscanf(iteratorStruct->commands_[iteratorStruct->commandIndex_]
927  .params_[IterateTable::commandRunParams_.DurationInSeconds_]
928  .c_str(),
929  "%ld",
930  &iteratorStruct->originalDurationInSeconds_);
931  __COUTV__(iteratorStruct->originalDurationInSeconds_);
932 
933  // else successfully launched
934  __COUT__
935  << "FSM in transition = "
936  << iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.isInTransition()
937  << __E__;
938  __COUT__ << "startCommandRun success." << __E__;
939 } // end startCommandRun()
940 
941 //========================================================================================================================
942 void Iterator::startCommandConfigureActive(IteratorWorkLoopStruct* iteratorStruct)
943 {
944  __COUT__ << "startCommandConfigureActive " << __E__;
945 
946  // steps:
947  // get active config group
948  // transition to configure with parameters describing group
949 
950  std::string group = iteratorStruct->cfgMgr_->getActiveGroupName();
951  TableGroupKey key = iteratorStruct->cfgMgr_->getActiveGroupKey();
952 
953  __COUT__ << "group " << group << __E__;
954  __COUT__ << "key " << key << __E__;
955 
956  // create special alias for this group using : separators
957 
958  std::stringstream systemAlias;
959  systemAlias << "GROUP:" << group << ":" << key;
960  startCommandConfigureAlias(iteratorStruct, systemAlias.str());
961 } // end startCommandConfigureActive()
962 
963 //========================================================================================================================
964 void Iterator::startCommandConfigureGroup(IteratorWorkLoopStruct* iteratorStruct)
965 {
966  __COUT__ << "startCommandConfigureGroup " << __E__;
967 
968  // steps:
969  // transition to configure with parameters describing group
970 
971  std::string group =
972  iteratorStruct->commands_[iteratorStruct->commandIndex_]
973  .params_[IterateTable::commandConfigureGroupParams_.GroupName_];
974  TableGroupKey key =
975  TableGroupKey(iteratorStruct->commands_[iteratorStruct->commandIndex_]
976  .params_[IterateTable::commandConfigureGroupParams_.GroupKey_]);
977 
978  __COUT__ << "group " << group << __E__;
979  __COUT__ << "key " << key << __E__;
980 
981  // create special alias for this group using : separators
982 
983  std::stringstream systemAlias;
984  systemAlias << "GROUP:" << group << ":" << key;
985  startCommandConfigureAlias(iteratorStruct, systemAlias.str());
986 } // end startCommandConfigureGroup()
987 
988 //========================================================================================================================
989 void Iterator::startCommandConfigureAlias(IteratorWorkLoopStruct* iteratorStruct,
990  const std::string& systemAlias)
991 {
992  __COUT__ << "systemAlias " << systemAlias << __E__;
993 
994  iteratorStruct->fsmCommandParameters_.clear();
995  iteratorStruct->fsmCommandParameters_.push_back(systemAlias);
996 
997  std::string errorStr = "";
998  std::string currentState = iteratorStruct->theIterator_->theSupervisor_
999  ->theStateMachine_.getCurrentStateName();
1000 
1001  // execute first transition (may need two in conjunction with checkCommandConfigure())
1002 
1003  if(currentState == "Initial")
1004  errorStr =
1005  iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
1006  0,
1007  0,
1008  "Initialize",
1009  iteratorStruct->fsmName_,
1010  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1011  WebUsers::DEFAULT_ITERATOR_USERNAME,
1012  iteratorStruct->fsmCommandParameters_);
1013  else if(currentState == "Halted")
1014  errorStr =
1015  iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
1016  0,
1017  0,
1018  "Configure",
1019  iteratorStruct->fsmName_,
1020  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1021  WebUsers::DEFAULT_ITERATOR_USERNAME,
1022  iteratorStruct->fsmCommandParameters_);
1023  else if(currentState == "Configured")
1024  errorStr =
1025  iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
1026  0,
1027  0,
1028  "Halt",
1029  iteratorStruct->fsmName_,
1030  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1031  WebUsers::DEFAULT_ITERATOR_USERNAME,
1032  iteratorStruct->fsmCommandParameters_);
1033  else
1034  errorStr =
1035  "Can only Configure from the Initial or Halted state. The current state is " +
1036  currentState;
1037 
1038  if(errorStr != "")
1039  {
1040  __SS__ << "Iterator failed to configure with system alias '"
1041  << (iteratorStruct->fsmCommandParameters_.size()
1042  ? iteratorStruct->fsmCommandParameters_[0]
1043  : "UNKNOWN")
1044  << "' because of the following error: " << errorStr;
1045  __SS_THROW__;
1046  }
1047 
1048  // else successfully launched
1049  __COUT__
1050  << "FSM in transition = "
1051  << iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.isInTransition()
1052  << __E__;
1053  __COUT__ << "startCommandConfigureAlias success." << __E__;
1054 } // end startCommandConfigureAlias()
1055 
1056 //========================================================================================================================
1057 void Iterator::startCommandMacro(IteratorWorkLoopStruct* iteratorStruct,
1058  bool isFrontEndMacro)
1059 {
1060  // Steps:
1061  // 4 parameters CommandExecuteFEMacroParams:
1062  // //targets
1063  // const std::string FEMacroName_ = "FEMacroName";
1064  // //macro parameters (table/groupID)
1065 
1066  const std::string& macroName =
1067  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1068  .params_[IterateTable::commandExecuteMacroParams_.MacroName_];
1069  const std::string& enableSavingOutput =
1070  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1071  .params_[IterateTable::commandExecuteMacroParams_.EnableSavingOutput_];
1072  const std::string& outputFilePath =
1073  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1074  .params_[IterateTable::commandExecuteMacroParams_.OutputFilePath_];
1075  const std::string& outputFileRadix =
1076  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1077  .params_[IterateTable::commandExecuteMacroParams_.OutputFileRadix_];
1078  const std::string& inputArgs =
1079  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1080  .params_[IterateTable::commandExecuteMacroParams_.MacroArgumentString_];
1081 
1082  __COUTV__(macroName);
1083  __COUTV__(enableSavingOutput);
1084  __COUTV__(outputFilePath);
1085  __COUTV__(outputFileRadix);
1086  __COUTV__(inputArgs);
1087 
1088  // send request to MacroMaker a la FEVInterface::runFrontEndMacro
1089  // but need to pass iteration information, so that the call is launched by just one
1090  // send to each front end. Front-ends must immediately respond that is started
1091  // FEVInterfacesManager.. must start a thread for running the macro iterations
1092  // Then check for complete.
1093 
1094  iteratorStruct->targetsDone_.clear(); // reset
1095 
1096  __COUTV__(iteratorStruct->commands_[iteratorStruct->commandIndex_].targets_.size());
1097  for(const auto& target :
1098  iteratorStruct->commands_[iteratorStruct->commandIndex_].targets_)
1099  {
1100  __COUT__ << "target " << target.table_ << ":" << target.UID_ << __E__;
1101 
1102  // for each target, init to not done
1103  iteratorStruct->targetsDone_.push_back(false);
1104 
1105  xoap::MessageReference message =
1106  SOAPUtilities::makeSOAPMessageReference("FECommunication");
1107 
1108  SOAPParameters parameters;
1109  std::string type = isFrontEndMacro ? "feMacroMultiDimensionalStart"
1110  : "macroMultiDimensionalStart";
1111  parameters.addParameter("type", type);
1112  parameters.addParameter("requester", WebUsers::DEFAULT_ITERATOR_USERNAME);
1113  parameters.addParameter("targetInterfaceID", target.UID_);
1114  parameters.addParameter(isFrontEndMacro ? "feMacroName" : "macroName", macroName);
1115  parameters.addParameter("enableSavingOutput", enableSavingOutput);
1116  parameters.addParameter("outputFilePath", outputFilePath);
1117  parameters.addParameter("outputFileRadix", outputFileRadix);
1118  parameters.addParameter("inputArgs", inputArgs);
1119  SOAPUtilities::addParameters(message, parameters);
1120 
1121  __COUT__ << "Sending FE communication: " << SOAPUtilities::translate(message)
1122  << __E__;
1123 
1124  xoap::MessageReference replyMessage =
1125  iteratorStruct->theIterator_->theSupervisor_
1126  ->SOAPMessenger::sendWithSOAPReply(
1127  iteratorStruct->theIterator_->theSupervisor_->allSupervisorInfo_
1128  .getAllMacroMakerTypeSupervisorInfo()
1129  .begin()
1130  ->second.getDescriptor(),
1131  message);
1132 
1133  __COUT__ << "Response received: " << SOAPUtilities::translate(replyMessage)
1134  << __E__;
1135 
1136  SOAPParameters rxParameters;
1137  rxParameters.addParameter("Error");
1138  std::string response = SOAPUtilities::receive(replyMessage, rxParameters);
1139 
1140  std::string error = rxParameters.getValue("Error");
1141 
1142  if(response != type + "Done" || error != "")
1143  {
1144  // error occurred!
1145  __SS__ << "Error transmitting request to target interface '" << target.UID_
1146  << "' from '" << WebUsers::DEFAULT_ITERATOR_USERNAME << ".' Response '"
1147  << response << "' with error: " << error << __E__;
1148  __SS_THROW__;
1149  }
1150  } // end target loop
1151 
1152 } // end startCommandMacro()
1153 
1154 //========================================================================================================================
1155 bool Iterator::checkCommandMacro(IteratorWorkLoopStruct* iteratorStruct,
1156  bool isFrontEndMacro)
1157 {
1158  sleep(1);
1159 
1160  // Steps:
1161  // 4 parameters CommandExecuteFEMacroParams:
1162  // //targets
1163  // const std::string FEMacroName_ = "FEMacroName";
1164  // //macro parameters (table/groupID)
1165 
1166  const std::string& macroName =
1167  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1168  .params_[IterateTable::commandExecuteMacroParams_.MacroName_];
1169 
1170  __COUTV__(macroName);
1171 
1172  // send request to MacroMaker to check completion of macro
1173  // as targets are identified complete, remove targets_ from vector
1174 
1175  for(unsigned int i = 0;
1176  i < iteratorStruct->commands_[iteratorStruct->commandIndex_].targets_.size();
1177  ++i)
1178  {
1180  iteratorStruct->commands_[iteratorStruct->commandIndex_].targets_[i];
1181 
1182  __COUT__ << "target " << target.table_ << ":" << target.UID_ << __E__;
1183 
1184  xoap::MessageReference message =
1185  SOAPUtilities::makeSOAPMessageReference("FECommunication");
1186 
1187  SOAPParameters parameters;
1188  std::string type = isFrontEndMacro ? "feMacroMultiDimensionalCheck"
1189  : "macroMultiDimensionalCheck";
1190  parameters.addParameter("type", type);
1191  parameters.addParameter("requester", WebUsers::DEFAULT_ITERATOR_USERNAME);
1192  parameters.addParameter("targetInterfaceID", target.UID_);
1193  parameters.addParameter(isFrontEndMacro ? "feMacroName" : "macroName", macroName);
1194  SOAPUtilities::addParameters(message, parameters);
1195 
1196  __COUT__ << "Sending FE communication: " << SOAPUtilities::translate(message)
1197  << __E__;
1198 
1199  xoap::MessageReference replyMessage =
1200  iteratorStruct->theIterator_->theSupervisor_
1201  ->SOAPMessenger::sendWithSOAPReply(
1202  iteratorStruct->theIterator_->theSupervisor_->allSupervisorInfo_
1203  .getAllMacroMakerTypeSupervisorInfo()
1204  .begin()
1205  ->second.getDescriptor(),
1206  message);
1207 
1208  __COUT__ << "Response received: " << SOAPUtilities::translate(replyMessage)
1209  << __E__;
1210 
1211  SOAPParameters rxParameters;
1212  rxParameters.addParameter("Error");
1213  rxParameters.addParameter("Done");
1214  std::string response = SOAPUtilities::receive(replyMessage, rxParameters);
1215 
1216  std::string error = rxParameters.getValue("Error");
1217  bool done = rxParameters.getValue("Done") == "1";
1218 
1219  if(response != type + "Done" || error != "")
1220  {
1221  // error occurred!
1222  __SS__ << "Error transmitting request to target interface '" << target.UID_
1223  << "' from '" << WebUsers::DEFAULT_ITERATOR_USERNAME << ".' Response '"
1224  << response << "' with error: " << error << __E__;
1225  __SS_THROW__;
1226  }
1227 
1228  if(!done) // still more to do so give up checking
1229  return false;
1230 
1231  // mark target done
1232  iteratorStruct->targetsDone_[i] = true;
1233 
1234  // iteratorStruct->commands_[iteratorStruct->commandIndex_].targets_.erase(
1235  // targetIt--); //go back after delete
1236 
1237  } // end target loop
1238 
1239  // if here all targets are done
1240  return true;
1241 } // end checkCommandMacro()
1242 
1243 //========================================================================================================================
1244 void Iterator::startCommandModifyActive(IteratorWorkLoopStruct* iteratorStruct)
1245 {
1246  // Steps:
1247  // 4 parameters commandModifyActiveParams_:
1248  // const std::string DoTrackGroupChanges_ TrueFalse
1249  // //targets
1250  // const std::string RelativePathToField_ = "RelativePathToField";
1251  // const std::string FieldStartValue_ = "FieldStartValue";
1252  // const std::string FieldIterationStepSize_ = "FieldIterationStepSize";
1253  //
1254  // if tracking changes,
1255  // create a new group
1256  // for every enabled FE
1257  // set field = start value + stepSize * currentStepIndex_
1258  // activate group
1259  // else
1260  // load scratch group
1261  // for every enabled FE
1262  // set field = start value + stepSize * stepIndex
1263  // activate group
1264 
1265  bool doTrackGroupChanges = false;
1266  if("True" ==
1267  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1268  .params_[IterateTable::commandModifyActiveParams_.DoTrackGroupChanges_])
1269  doTrackGroupChanges = true;
1270 
1271  const std::string& startValueStr =
1272  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1273  .params_[IterateTable::commandModifyActiveParams_.FieldStartValue_];
1274  const std::string& stepSizeStr =
1275  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1276  .params_[IterateTable::commandModifyActiveParams_.FieldIterationStepSize_];
1277 
1278  const unsigned int stepIndex = iteratorStruct->stepIndexStack_.back();
1279 
1280  __COUT__ << "doTrackGroupChanges " << (doTrackGroupChanges ? "yes" : "no")
1281  << std::endl;
1282  __COUT__ << "stepIndex " << stepIndex << std::endl;
1283 
1284  ConfigurationInterface::setVersionTrackingEnabled(doTrackGroupChanges);
1285 
1286  // two approaches: double or long handling
1287 
1288  if(startValueStr.size() && (startValueStr[startValueStr.size() - 1] == 'f' ||
1289  startValueStr.find('.') != std::string::npos))
1290  {
1291  // handle as double
1292  double startValue = strtod(startValueStr.c_str(), 0);
1293  double stepSize = strtod(stepSizeStr.c_str(), 0);
1294 
1295  __COUT__ << "startValue " << startValue << std::endl;
1296  __COUT__ << "stepSize " << stepSize << std::endl;
1297  __COUT__ << "currentValue " << startValue + stepSize * stepIndex << std::endl;
1298 
1299  helpCommandModifyActive(
1300  iteratorStruct, startValue + stepSize * stepIndex, doTrackGroupChanges);
1301  }
1302  else // handle as long
1303  {
1304  long int startValue;
1305  long int stepSize;
1306 
1307  StringMacros::getNumber(startValueStr, startValue);
1308  StringMacros::getNumber(startValueStr, stepSize);
1309 
1310  // long int startValue;
1311  //
1312  // if(startValueStr.size() > 2 && startValueStr[1] == 'x') //assume hex value
1313  // startValue = strtol(startValueStr.c_str(),0,16);
1314  // else if(startValueStr.size() > 1 && startValueStr[0] == 'b') //assume
1315  // binary value startValue =
1316  // strtol(startValueStr.substr(1).c_str(),0,2);
1318  // strtol(startValueStr.c_str(),0,10);
1319  //
1320  // long int stepSize;
1321  //
1322  // if(stepSizeStr.size() > 2 && stepSizeStr[1] == 'x') //assume hex value
1323  // stepSize = strtol(stepSizeStr.c_str(),0,16);
1324  // else if(stepSizeStr.size() > 1 && stepSizeStr[0] == 'b') //assume binary
1325  // value stepSize = strtol(stepSizeStr.substr(1).c_str(),0,2); //skip
1326  // first 'b' character else stepSize =
1327  // strtol(stepSizeStr.c_str(),0,10);
1328 
1329  __COUT__ << "startValue " << startValue << std::endl;
1330  __COUT__ << "stepSize " << stepSize << std::endl;
1331  __COUT__ << "currentValue " << startValue + stepSize * stepIndex << std::endl;
1332 
1333  helpCommandModifyActive(
1334  iteratorStruct, startValue + stepSize * stepIndex, doTrackGroupChanges);
1335  }
1336 
1337 } // end startCommandModifyActive()
1338 
1339 //========================================================================================================================
1340 // checkCommandRun
1341 // return true if done
1342 //
1343 // Either will be done on (priority 1) running threads (for Frontends) ending
1344 // or (priority 2 and ignored if <= 0) duration timeout
1345 //
1346 // Note: use command structure strings to maintain duration left
1347 // Note: watch iterator->doPauseAction and iterator->doHaltAction and respond
1348 bool Iterator::checkCommandRun(IteratorWorkLoopStruct* iteratorStruct)
1349 {
1350  sleep(1); // sleep to give FSM time to transition
1351 
1352  // all RunControlStateMachine access commands should be mutually exclusive with
1353  // GatewaySupervisor main thread state machine accesses should be mutually exclusive
1354  // with GatewaySupervisor main thread state machine accesses lockout the messages
1355  // array for the remainder of the scope this guarantees the reading thread can safely
1356  // access the messages
1357  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX)
1358  __COUT__ << "Waiting for FSM access" << __E__;
1359  std::lock_guard<std::mutex> lock(
1360  iteratorStruct->theIterator_->theSupervisor_->stateMachineAccessMutex_);
1361  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX)
1362  __COUT__ << "Have FSM access" << __E__;
1363 
1364  if(iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.isInTransition())
1365  return false;
1366 
1367  iteratorStruct->fsmCommandParameters_.clear();
1368 
1369  std::string errorStr = "";
1370  std::string currentState = iteratorStruct->theIterator_->theSupervisor_
1371  ->theStateMachine_.getCurrentStateName();
1372 
1374  // check for imposed actions and forced exits
1375  if(iteratorStruct->doPauseAction_)
1376  {
1377  // transition to pause state
1378  __COUT__ << "Transitioning FSM to Paused..." << __E__;
1379 
1380  if(currentState == "Paused")
1381  {
1382  // done with early pause exit!
1383  __COUT__ << "Transition to Paused complete." << __E__;
1384  return true;
1385  }
1386  else if(currentState == "Running") // launch transition to pause
1387  errorStr = iteratorStruct->theIterator_->theSupervisor_
1388  ->attemptStateMachineTransition(
1389  0,
1390  0,
1391  "Pause",
1392  iteratorStruct->fsmName_,
1393  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1394  WebUsers::DEFAULT_ITERATOR_USERNAME,
1395  iteratorStruct->fsmCommandParameters_);
1396  else if(currentState == "Configured")
1397  {
1398  // no need to pause state machine, no run going on
1399  __COUT__ << "In Configured state. No need to transition to Paused." << __E__;
1400  return true;
1401  }
1402  else
1403  errorStr = "Expected to be in Paused. Unexpectedly, the current state is " +
1404  currentState +
1405  ". Last State Machine error message was as follows: " +
1406  iteratorStruct->theIterator_->theSupervisor_->theStateMachine_
1407  .getErrorMessage();
1408 
1409  if(errorStr != "")
1410  {
1411  __SS__ << "Iterator failed to pause because of the following error: "
1412  << errorStr;
1413  __SS_THROW__;
1414  }
1415  return false;
1416  }
1417  else if(iteratorStruct->doHaltAction_)
1418  {
1419  // transition to halted state
1420  __COUT__ << "Transitioning FSM to Halted..." << __E__;
1421 
1422  if(currentState == "Halted")
1423  {
1424  // done with early halt exit!
1425  __COUT__ << "Transition to Halted complete." << __E__;
1426  return true;
1427  }
1428  else if(currentState == "Running" || // launch transition to halt
1429  currentState == "Paused")
1430  errorStr = iteratorStruct->theIterator_->theSupervisor_
1431  ->attemptStateMachineTransition(
1432  0,
1433  0,
1434  "Abort",
1435  iteratorStruct->fsmName_,
1436  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1437  WebUsers::DEFAULT_ITERATOR_USERNAME,
1438  iteratorStruct->fsmCommandParameters_);
1439  else if(currentState == "Configured") // launch transition to halt
1440  errorStr = iteratorStruct->theIterator_->theSupervisor_
1441  ->attemptStateMachineTransition(
1442  0,
1443  0,
1444  "Halt",
1445  iteratorStruct->fsmName_,
1446  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1447  WebUsers::DEFAULT_ITERATOR_USERNAME,
1448  iteratorStruct->fsmCommandParameters_);
1449  else
1450  errorStr = "Expected to be in Halted. Unexpectedly, the current state is " +
1451  currentState +
1452  ". Last State Machine error message was as follows: " +
1453  iteratorStruct->theIterator_->theSupervisor_->theStateMachine_
1454  .getErrorMessage();
1455 
1456  if(errorStr != "")
1457  {
1458  __SS__ << "Iterator failed to halt because of the following error: "
1459  << errorStr;
1460  __SS_THROW__;
1461  }
1462  return false;
1463  }
1464  else if(iteratorStruct->doResumeAction_)
1465  {
1466  // Note: check command gets one shot to resume
1467 
1468  // transition to running state
1469  __COUT__ << "Transitioning FSM to Running..." << __E__;
1470 
1471  if(currentState == "Paused") // launch transition to running
1472  errorStr = iteratorStruct->theIterator_->theSupervisor_
1473  ->attemptStateMachineTransition(
1474  0,
1475  0,
1476  "Resume",
1477  iteratorStruct->fsmName_,
1478  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1479  WebUsers::DEFAULT_ITERATOR_USERNAME,
1480  iteratorStruct->fsmCommandParameters_);
1481 
1482  if(errorStr != "")
1483  {
1484  __SS__ << "Iterator failed to run because of the following error: "
1485  << errorStr;
1486  __SS_THROW__;
1487  }
1488  return false;
1489  }
1490 
1492  // normal running
1493 
1494  if(currentState != "Running")
1495  {
1496  if(iteratorStruct->runIsDone_ && currentState == "Configured")
1497  {
1498  // indication of done
1499  __COUT__ << "Reached end of run " << iteratorStruct->fsmNextRunNumber_
1500  << __E__;
1501  return true;
1502  }
1503 
1504  errorStr = "Expected to be in Running. Unexpectedly, the current state is " +
1505  currentState + ". Last State Machine error message was as follows: " +
1506  iteratorStruct->theIterator_->theSupervisor_->theStateMachine_
1507  .getErrorMessage();
1508  }
1509  else // else in Running state! Check for end of run
1510  {
1511  bool waitOnRunningThreads = false;
1512  if("True" == iteratorStruct->commands_[iteratorStruct->commandIndex_]
1513  .params_[IterateTable::commandRunParams_.WaitOnRunningThreads_])
1514  waitOnRunningThreads = true;
1515 
1516  time_t remainingDurationInSeconds;
1517  sscanf(iteratorStruct->commands_[iteratorStruct->commandIndex_]
1518  .params_[IterateTable::commandRunParams_.DurationInSeconds_]
1519  .c_str(),
1520  "%ld",
1521  &remainingDurationInSeconds);
1522 
1523  __COUT__ << "waitOnRunningThreads " << waitOnRunningThreads << __E__;
1524  __COUT__ << "remainingDurationInSeconds " << remainingDurationInSeconds << __E__;
1525 
1527  // priority 1 is waiting on running threads
1528  if(waitOnRunningThreads)
1529  {
1530  // get status of all running FE workloops
1531  GatewaySupervisor* theSupervisor =
1532  iteratorStruct->theIterator_->theSupervisor_;
1533 
1534  bool allFrontEndsAreDone = true;
1535  for(auto& it : theSupervisor->allSupervisorInfo_.getAllFETypeSupervisorInfo())
1536  {
1537  try
1538  {
1539  std::string status = theSupervisor->send(it.second.getDescriptor(),
1540  "WorkLoopStatusRequest");
1541 
1542  __COUT__ << "FESupervisor instance " << it.first
1543  << " has status = " << status << std::endl;
1544 
1545  if(status != CoreSupervisorBase::WORK_LOOP_DONE)
1546  {
1547  allFrontEndsAreDone = false;
1548  break;
1549  }
1550  }
1551  catch(xdaq::exception::Exception& e)
1552  {
1553  __SS__ << "Could not retrieve status from FESupervisor instance "
1554  << it.first << "." << std::endl;
1555  __COUT_WARN__ << ss.str();
1556  errorStr = ss.str();
1557 
1558  if(errorStr != "")
1559  {
1560  __SS__
1561  << "Iterator failed to run because of the following error: "
1562  << errorStr;
1563  __SS_THROW__;
1564  }
1565  }
1566  }
1567 
1568  if(allFrontEndsAreDone)
1569  {
1570  // need to end run!
1571  __COUT__ << "FE workloops all complete! Stopping run..." << __E__;
1572 
1573  errorStr = iteratorStruct->theIterator_->theSupervisor_
1574  ->attemptStateMachineTransition(
1575  0,
1576  0,
1577  "Stop",
1578  iteratorStruct->fsmName_,
1579  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1580  WebUsers::DEFAULT_ITERATOR_USERNAME,
1581  iteratorStruct->fsmCommandParameters_);
1582 
1583  if(errorStr != "")
1584  {
1585  __SS__
1586  << "Iterator failed to stop run because of the following error: "
1587  << errorStr;
1588  __SS_THROW__;
1589  }
1590 
1591  // write indication of run done into duration
1592  iteratorStruct->runIsDone_ = true;
1593 
1594  return false;
1595  }
1596  }
1597 
1599  // priority 2 is duration, if <= 0 it is ignored
1600  if(remainingDurationInSeconds > 1)
1601  {
1602  --remainingDurationInSeconds;
1603 
1604  // write back to string
1605  char str[200];
1606  sprintf(str, "%ld", remainingDurationInSeconds);
1607  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1608  .params_[IterateTable::commandRunParams_.DurationInSeconds_] =
1609  str; // re-store as string
1610  }
1611  else if(remainingDurationInSeconds == 1)
1612  {
1613  // need to end run!
1614  __COUT__ << "Time duration reached! Stopping run..." << __E__;
1615 
1616  errorStr = iteratorStruct->theIterator_->theSupervisor_
1617  ->attemptStateMachineTransition(
1618  0,
1619  0,
1620  "Stop",
1621  iteratorStruct->fsmName_,
1622  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1623  WebUsers::DEFAULT_ITERATOR_USERNAME,
1624  iteratorStruct->fsmCommandParameters_);
1625 
1626  if(errorStr != "")
1627  {
1628  __SS__ << "Iterator failed to stop run because of the following error: "
1629  << errorStr;
1630  __SS_THROW__;
1631  }
1632 
1633  // write indication of run done
1634  iteratorStruct->runIsDone_ = true;
1635 
1636  // write original duration back to string
1637  char str[200];
1638  sprintf(str, "%ld", iteratorStruct->originalDurationInSeconds_);
1639  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1640  .params_[IterateTable::commandRunParams_.DurationInSeconds_] =
1641  str; // re-store as string
1642  return false;
1643  }
1644  }
1645 
1646  if(errorStr != "")
1647  {
1648  __SS__ << "Iterator failed to run because of the following error: " << errorStr;
1649  __SS_THROW__;
1650  }
1651  return false;
1652 } // end checkCommandRun()
1653 
1654 //========================================================================================================================
1655 // return true if done
1656 bool Iterator::checkCommandConfigure(IteratorWorkLoopStruct* iteratorStruct)
1657 {
1658  sleep(1); // sleep to give FSM time to transition
1659 
1660  // all RunControlStateMachine access commands should be mutually exclusive with
1661  // GatewaySupervisor main thread state machine accesses should be mutually exclusive
1662  // with GatewaySupervisor main thread state machine accesses lockout the messages
1663  // array for the remainder of the scope this guarantees the reading thread can safely
1664  // access the messages
1665  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX)
1666  __COUT__ << "Waiting for FSM access" << __E__;
1667  std::lock_guard<std::mutex> lock(
1668  iteratorStruct->theIterator_->theSupervisor_->stateMachineAccessMutex_);
1669  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX)
1670  __COUT__ << "Have FSM access" << __E__;
1671 
1672  if(iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.isInTransition())
1673  return false;
1674 
1675  std::string errorStr = "";
1676  std::string currentState = iteratorStruct->theIterator_->theSupervisor_
1677  ->theStateMachine_.getCurrentStateName();
1678 
1679  if(currentState == "Halted")
1680  errorStr =
1681  iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
1682  0,
1683  0,
1684  "Configure",
1685  iteratorStruct->fsmName_,
1686  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1687  WebUsers::DEFAULT_ITERATOR_USERNAME,
1688  iteratorStruct->fsmCommandParameters_);
1689  else if(currentState != "Configured")
1690  errorStr = "Expected to be in Configure. Unexpectedly, the current state is " +
1691  currentState + "." +
1692  ". Last State Machine error message was as follows: " +
1693  iteratorStruct->theIterator_->theSupervisor_->theStateMachine_
1694  .getErrorMessage();
1695  else // else successfully done (in Configured state!)
1696  {
1697  __COUT__ << "checkCommandConfigureAlias complete." << __E__;
1698 
1699  // also activate the Iterator config manager to mimic active config
1700  std::pair<std::string, TableGroupKey> newActiveGroup =
1701  iteratorStruct->cfgMgr_->getTableGroupFromAlias(
1702  iteratorStruct->fsmCommandParameters_[0]);
1703  iteratorStruct->cfgMgr_->loadTableGroup(
1704  newActiveGroup.first, newActiveGroup.second, true /*activate*/);
1705 
1706  __COUT__ << "originalTrackChanges " << iteratorStruct->originalTrackChanges_
1707  << __E__;
1708  __COUT__ << "originalConfigGroup " << iteratorStruct->originalConfigGroup_
1709  << __E__;
1710  __COUT__ << "originalConfigKey " << iteratorStruct->originalConfigKey_ << __E__;
1711 
1712  __COUT__ << "currentTrackChanges "
1713  << ConfigurationInterface::isVersionTrackingEnabled() << __E__;
1714  __COUT__ << "originalConfigGroup "
1715  << iteratorStruct->cfgMgr_->getActiveGroupName() << __E__;
1716  __COUT__ << "originalConfigKey " << iteratorStruct->cfgMgr_->getActiveGroupKey()
1717  << __E__;
1718 
1719  return true;
1720  }
1721 
1722  if(errorStr != "")
1723  {
1724  __SS__ << "Iterator failed to configure with system alias '"
1725  << (iteratorStruct->fsmCommandParameters_.size()
1726  ? iteratorStruct->fsmCommandParameters_[0]
1727  : "UNKNOWN")
1728  << "' because of the following error: " << errorStr;
1729  __SS_THROW__;
1730  }
1731  return false;
1732 }
1733 
1734 //========================================================================================================================
1735 bool Iterator::handleCommandRequest(HttpXmlDocument& xmldoc,
1736  const std::string& command,
1737  const std::string& parameter)
1738 {
1739  //__COUTV__(command);
1740  if(command == "iteratePlay")
1741  {
1742  playIterationPlan(xmldoc, parameter);
1743  return true;
1744  }
1745  else if(command == "iteratePause")
1746  {
1747  pauseIterationPlan(xmldoc);
1748  return true;
1749  }
1750  else if(command == "iterateHalt")
1751  {
1752  haltIterationPlan(xmldoc);
1753  return true;
1754  }
1755  else if(command == "getIterationPlanStatus")
1756  {
1757  getIterationPlanStatus(xmldoc);
1758  return true;
1759  }
1760  else // return true if iterator has control of state machine
1761  {
1762  // lockout the messages array for the remainder of the scope
1763  // this guarantees the reading thread can safely access the messages
1764  if(theSupervisor_->VERBOSE_MUTEX)
1765  __COUT__ << "Waiting for iterator access" << __E__;
1766  std::lock_guard<std::mutex> lock(accessMutex_);
1767  if(theSupervisor_->VERBOSE_MUTEX)
1768  __COUT__ << "Have iterator access" << __E__;
1769 
1770  if(iteratorBusy_)
1771  {
1772  __SS__ << "Error - Can not accept request because the Iterator "
1773  << "is currently "
1774  << "in control of State Machine progress. ";
1775  __COUT_ERR__ << "\n" << ss.str();
1776  __MOUT_ERR__ << "\n" << ss.str();
1777 
1778  xmldoc.addTextElementToData("state_tranisition_attempted",
1779  "0"); // indicate to GUI transition NOT attempted
1780  xmldoc.addTextElementToData(
1781  "state_tranisition_attempted_err",
1782  ss.str()); // indicate to GUI transition NOT attempted
1783 
1784  return true; // to block other commands
1785  }
1786  }
1787  return false;
1788 }
1789 
1790 //========================================================================================================================
1791 void Iterator::playIterationPlan(HttpXmlDocument& xmldoc, const std::string& planName)
1792 {
1793  __MOUT__ << "Attempting to play iteration plan '" << planName << ".'" << __E__;
1794  __COUT__ << "Attempting to play iteration plan '" << planName << ".'" << __E__;
1795 
1796  // setup "play" command
1797 
1798  // lockout the messages array for the remainder of the scope
1799  // this guarantees the reading thread can safely access the messages
1800  if(theSupervisor_->VERBOSE_MUTEX)
1801  __COUT__ << "Waiting for iterator access" << __E__;
1802  std::lock_guard<std::mutex> lock(accessMutex_);
1803  if(theSupervisor_->VERBOSE_MUTEX)
1804  __COUT__ << "Have iterator access" << __E__;
1805 
1806  if(!activePlanIsRunning_ && !commandPlay_)
1807  {
1808  if(!workloopRunning_)
1809  {
1810  // start thread with member variables initialized
1811 
1812  workloopRunning_ = true;
1813 
1814  // must start thread first
1815  std::thread([](Iterator* iterator) { Iterator::IteratorWorkLoop(iterator); },
1816  this)
1817  .detach();
1818  }
1819 
1820  activePlanName_ = planName;
1821  commandPlay_ = true;
1822  }
1823  else
1824  {
1825  __SS__ << "Invalid play command attempted. Can only play when the Iterator is "
1826  "inactive or paused."
1827  << " If you would like to restart an iteration plan, first try halting."
1828  << __E__;
1829  __MOUT__ << ss.str();
1830 
1831  xmldoc.addTextElementToData("error_message", ss.str());
1832 
1833  __COUT__ << "Invalid play command attempted. " << commandPlay_ << " "
1834  << activePlanName_ << __E__;
1835  }
1836 }
1837 
1838 //========================================================================================================================
1839 void Iterator::pauseIterationPlan(HttpXmlDocument& xmldoc)
1840 {
1841  __MOUT__ << "Attempting to pause iteration plan '" << activePlanName_ << ".'"
1842  << __E__;
1843  __COUT__ << "Attempting to pause iteration plan '" << activePlanName_ << ".'"
1844  << __E__;
1845 
1846  // setup "pause" command
1847 
1848  // lockout the messages array for the remainder of the scope
1849  // this guarantees the reading thread can safely access the messages
1850  if(theSupervisor_->VERBOSE_MUTEX)
1851  __COUT__ << "Waiting for iterator access" << __E__;
1852  std::lock_guard<std::mutex> lock(accessMutex_);
1853  if(theSupervisor_->VERBOSE_MUTEX)
1854  __COUT__ << "Have iterator access" << __E__;
1855 
1856  if(workloopRunning_ && activePlanIsRunning_ && !commandPause_)
1857  {
1858  commandPause_ = true;
1859  }
1860  else
1861  {
1862  __SS__ << "Invalid pause command attempted. Can only pause when running."
1863  << __E__;
1864  __MOUT__ << ss.str();
1865 
1866  xmldoc.addTextElementToData("error_message", ss.str());
1867 
1868  __COUT__ << "Invalid pause command attempted. " << workloopRunning_ << " "
1869  << activePlanIsRunning_ << " " << commandPause_ << " " << activePlanName_
1870  << __E__;
1871  }
1872 }
1873 
1874 //========================================================================================================================
1875 void Iterator::haltIterationPlan(HttpXmlDocument& xmldoc)
1876 {
1877  __MOUT__ << "Attempting to halt iteration plan '" << activePlanName_ << ".'" << __E__;
1878  __COUT__ << "Attempting to halt iteration plan '" << activePlanName_ << ".'" << __E__;
1879 
1880  // setup "halt" command
1881 
1882  if(workloopRunning_)
1883  {
1884  // lockout the messages array for the remainder of the scope
1885  // this guarantees the reading thread can safely access the messages
1886  if(theSupervisor_->VERBOSE_MUTEX)
1887  __COUT__ << "Waiting for iterator access" << __E__;
1888  std::lock_guard<std::mutex> lock(accessMutex_);
1889  if(theSupervisor_->VERBOSE_MUTEX)
1890  __COUT__ << "Have iterator access" << __E__;
1891 
1892  __COUT__ << "activePlanIsRunning_: " << activePlanIsRunning_ << __E__;
1893  __COUT__ << "Passing halt command to iterator thread." << __E__;
1894  commandHalt_ = true;
1895 
1896  // clear
1897  activePlanName_ = "";
1898  activeCommandIndex_ = -1;
1899  }
1900  else // no thread, so halt (and reset Error') without command to thread
1901  {
1902  __COUT__ << "No thread, so conducting halt." << __E__;
1903  Iterator::haltIterator(this);
1904  //
1905  // activePlanIsRunning_ = false;
1906  // iteratorBusy_ = false;
1907  //
1908  // bool haltAttempted = false;
1909  // try
1910  // {
1911  // haltAttempted = Iterator::haltIterator(theSupervisor_, lastFsmName_);
1912  // }
1913  // catch(const std::runtime_error& e)
1914  // {
1915  // haltAttempted = false;
1916  // __COUT__ << "Halt error: " << e.what() << __E__;
1917  // }
1918  //
1919  // if(!haltAttempted) //then show error
1920  // {
1921  // __SS__ << "Invalid halt command attempted. Can only halt when there is
1922  // an active iteration plan." << __E__;
1923  // __MOUT_ERR__ << ss.str();
1924  //
1925  // xmldoc.addTextElementToData("error_message", ss.str());
1926  //
1927  // __COUT__ << "Invalid halt command attempted. " <<
1928  // workloopRunning_ << " " <<
1929  // activePlanIsRunning_ << " " <<
1930  // commandHalt_ << " " <<
1931  // activePlanName_ << __E__;
1932  // }
1933  // else
1934  // __COUT__ << "Halt was attempted." << __E__;
1935  }
1936 }
1937 
1938 //========================================================================================================================
1939 // return state machine and iterator status
1940 void Iterator::getIterationPlanStatus(HttpXmlDocument& xmldoc)
1941 {
1942  xmldoc.addTextElementToData(
1943  "current_state",
1944  theSupervisor_->theStateMachine_.isInTransition()
1945  ? theSupervisor_->theStateMachine_.getCurrentTransitionName(
1946  theSupervisor_->stateMachineLastCommandInput_)
1947  : theSupervisor_->theStateMachine_.getCurrentStateName());
1948 
1949  // xmldoc.addTextElementToData("in_transition",
1950  // theSupervisor_->theStateMachine_.isInTransition() ? "1" : "0");
1951  if(theSupervisor_->theStateMachine_.isInTransition())
1952  xmldoc.addTextElementToData(
1953  "transition_progress",
1954  theSupervisor_->theProgressBar_.readPercentageString());
1955  else
1956  xmldoc.addTextElementToData("transition_progress", "100");
1957 
1958  char tmp[20];
1959  sprintf(tmp, "%lu", theSupervisor_->theStateMachine_.getTimeInState());
1960  xmldoc.addTextElementToData("time_in_state", tmp);
1961 
1962  // lockout the messages array for the remainder of the scope
1963  // this guarantees the reading thread can safely access the messages
1964  if(theSupervisor_->VERBOSE_MUTEX)
1965  __COUT__ << "Waiting for iterator access" << __E__;
1966  std::lock_guard<std::mutex> lock(accessMutex_);
1967  if(theSupervisor_->VERBOSE_MUTEX)
1968  __COUT__ << "Have iterator access" << __E__;
1969 
1970  xmldoc.addTextElementToData("active_plan", activePlanName_);
1971  xmldoc.addTextElementToData("last_started_plan", lastStartedPlanName_);
1972  xmldoc.addTextElementToData("last_finished_plan", lastFinishedPlanName_);
1973 
1974  sprintf(tmp, "%u", activeCommandIndex_);
1975  xmldoc.addTextElementToData("current_command_index", tmp);
1976  sprintf(tmp, "%ld", time(0) - activeCommandStartTime_);
1977  xmldoc.addTextElementToData("current_command_duration", tmp);
1978  sprintf(tmp, "%u", activeCommandIteration_);
1979  xmldoc.addTextElementToData("current_command_iteration", tmp);
1980 
1981  for(const auto& depthIteration : depthIterationStack_)
1982  {
1983  sprintf(tmp, "%u", depthIteration);
1984  xmldoc.addTextElementToData("depth_iteration", tmp);
1985  }
1986 
1987  if(activePlanIsRunning_ && iteratorBusy_)
1988  {
1989  if(workloopRunning_)
1990  xmldoc.addTextElementToData("active_plan_status", "Running");
1991  else
1992  xmldoc.addTextElementToData("active_plan_status", "Error");
1993  }
1994  else if(!activePlanIsRunning_ && iteratorBusy_)
1995  xmldoc.addTextElementToData("active_plan_status", "Paused");
1996  else
1997  xmldoc.addTextElementToData("active_plan_status", "Inactive");
1998 
1999  xmldoc.addTextElementToData("error_message", errorMessage_);
2000 }