otsdaq  v2_04_01
Iterator.cc
1 #include "otsdaq-core/GatewaySupervisor/Iterator.h"
2 #include "otsdaq-core/CoreSupervisors/CoreSupervisorBase.h"
3 #include "otsdaq-core/GatewaySupervisor/GatewaySupervisor.h"
4 #include "otsdaq-core/Macros/CoutMacros.h"
5 #include "otsdaq-core/MessageFacility/MessageFacility.h"
6 #include "otsdaq-core/WebUsersUtilities/WebUsers.h"
7 
8 #include <iostream>
9 #include <thread> //for std::thread
10 
11 #undef __MF_SUBJECT__
12 #define __MF_SUBJECT__ "Iterator"
13 
14 using namespace ots;
15 
16 //========================================================================================================================
17 Iterator::Iterator(GatewaySupervisor* supervisor)
18  : workloopRunning_(false)
19  , activePlanIsRunning_(false)
20  , iteratorBusy_(false)
21  , commandPlay_(false)
22  , commandPause_(false)
23  , commandHalt_(false)
24  , activePlanName_("")
25  , activeCommandIndex_(-1)
26  , activeCommandStartTime_(0)
27  , theSupervisor_(supervisor)
28 {
29  __MOUT__ << "Iterator constructed." << __E__;
30  __COUT__ << "Iterator constructed." << __E__;
31 }
32 
33 //========================================================================================================================
34 Iterator::~Iterator(void) {}
35 
36 //========================================================================================================================
37 void Iterator::IteratorWorkLoop(Iterator* iterator) try
38 {
39  __MOUT__ << "Iterator work loop starting..." << __E__;
40  __COUT__ << "Iterator work loop starting..." << __E__;
41 
42  // mutex init scope
43  {
44  // lockout the messages array for the remainder of the scope
45  // this guarantees the reading thread can safely access the messages
46  if(iterator->theSupervisor_->VERBOSE_MUTEX)
47  __COUT__ << "Waiting for iterator access" << __E__;
48  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
49  if(iterator->theSupervisor_->VERBOSE_MUTEX)
50  __COUT__ << "Have iterator access" << __E__;
51 
52  iterator->errorMessage_ = ""; // clear error message
53  }
54 
55  ConfigurationManagerRW theConfigurationManager(
56  WebUsers::DEFAULT_ITERATOR_USERNAME); // this is a restricted username
57  // theConfigurationManager.init();
58  theConfigurationManager.getAllTableInfo(true); // to prep all info
59 
60  IteratorWorkLoopStruct theIteratorStruct(iterator, &theConfigurationManager);
61 
62  const IterateTable* itConfig;
63 
64  std::vector<IterateTable::Command> commands;
65 
66  while(1)
67  {
68  // Process:
69  // - always "listen" for commands
70  // - play: if no plan running, activePlanIsRunning_ = true,
71  // and start or continue plan based on name/commandIndex
72  // - pause: if plan playing, pause it, activePlanIsRunning_ = false
73  // and do not clear commandIndex or name, iteratorBusy_ = true
74  // - halt: if plan playing or not, activePlanIsRunning_ = false
75  // and clear commandIndex or name, iteratorBusy_ = false
76  // - when running
77  // - go through each command
78  // - start the command, commandBusy = true
79  // - check for complete, then commandBusy = false
80 
81  // start command handling
82  // define mutex scope
83  {
84  // lockout the messages array for the remainder of the scope
85  // this guarantees the reading thread can safely access the messages
86  if(iterator->theSupervisor_->VERBOSE_MUTEX)
87  __COUT__ << "Waiting for iterator access" << __E__;
88  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
89  if(iterator->theSupervisor_->VERBOSE_MUTEX)
90  __COUT__ << "Have iterator access" << __E__;
91 
92  if(iterator->commandPlay_)
93  {
94  iterator->commandPlay_ = false; // clear
95 
96  if(!iterator->activePlanIsRunning_)
97  {
98  // valid PLAY command!
99 
100  iterator->activePlanIsRunning_ = true;
101  iterator->iteratorBusy_ = true;
102 
103  if(theIteratorStruct.activePlan_ != iterator->activePlanName_)
104  {
105  __COUT__ << "New plan name encountered old="
106  << theIteratorStruct.activePlan_
107  << " vs new=" << iterator->activePlanName_ << __E__;
108  theIteratorStruct.commandIndex_ = -1; // reset for new plan
109  }
110 
111  theIteratorStruct.activePlan_ = iterator->activePlanName_;
112  iterator->lastStartedPlanName_ = iterator->activePlanName_;
113 
114  if(theIteratorStruct.commandIndex_ == (unsigned int)-1)
115  {
116  __COUT__ << "Starting plan '" << theIteratorStruct.activePlan_
117  << ".'" << __E__;
118  __MOUT__ << "Starting plan '" << theIteratorStruct.activePlan_
119  << ".'" << __E__;
120  }
121  else
122  {
123  theIteratorStruct.doResumeAction_ = true;
124  __COUT__ << "Continuing plan '" << theIteratorStruct.activePlan_
125  << "' at command index "
126  << theIteratorStruct.commandIndex_ << ". " << __E__;
127  __MOUT__ << "Continuing plan '" << theIteratorStruct.activePlan_
128  << "' at command index "
129  << theIteratorStruct.commandIndex_ << ". " << __E__;
130  }
131  }
132  }
133  else if(iterator->commandPause_ && !theIteratorStruct.doPauseAction_)
134  {
135  theIteratorStruct.doPauseAction_ = true;
136  iterator->commandPause_ = false; // clear
137  }
138  else if(iterator->commandHalt_ && !theIteratorStruct.doHaltAction_)
139  {
140  theIteratorStruct.doHaltAction_ = true;
141  iterator->commandHalt_ = false; // clear
142  }
143 
144  theIteratorStruct.running_ = iterator->activePlanIsRunning_;
145 
146  if(iterator
147  ->activeCommandIndex_ != // update active command status if changed
148  theIteratorStruct.commandIndex_)
149  {
150  iterator->activeCommandIndex_ = theIteratorStruct.commandIndex_;
151  iterator->activeCommandStartTime_ = time(0); // reset on any change
152 
153  if(theIteratorStruct.commandIndex_ <
154  theIteratorStruct.commandIterations_.size())
155  iterator->activeCommandIteration_ =
156  theIteratorStruct
157  .commandIterations_[theIteratorStruct.commandIndex_];
158  else
159  iterator->activeCommandIteration_ = -1;
160 
161  iterator->depthIterationStack_.clear();
162  for(const auto& depthIteration : theIteratorStruct.stepIndexStack_)
163  iterator->depthIterationStack_.push_back(depthIteration);
164  // if(theIteratorStruct.stepIndexStack_.size())
165  // iterator->activeLoopIteration_ =
166  // theIteratorStruct.stepIndexStack_.back(); else
167  // iterator->activeLoopIteration_ = -1;
168  }
169 
170  } // end command handling and iterator mutex
171 
174  // do halt or pause action outside of iterator mutex
175 
176  if(theIteratorStruct.doPauseAction_)
177  {
178  // valid PAUSE command!
179 
180  // safely pause plan!
181  // i.e. check that command is complete
182 
183  __COUT__ << "Waiting to pause..." << __E__;
184  while(!iterator->checkCommand(&theIteratorStruct))
185  __COUT__ << "Waiting to pause..." << __E__;
186 
187  __COUT__ << "Completing pause..." << __E__;
188 
189  theIteratorStruct.doPauseAction_ = false; // clear
190 
191  // lockout the messages array for the remainder of the scope
192  // this guarantees the reading thread can safely access the messages
193  if(iterator->theSupervisor_->VERBOSE_MUTEX)
194  __COUT__ << "Waiting for iterator access" << __E__;
195  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
196  if(iterator->theSupervisor_->VERBOSE_MUTEX)
197  __COUT__ << "Have iterator access" << __E__;
198 
199  iterator->activePlanIsRunning_ = false;
200 
201  __COUT__ << "Paused plan '" << theIteratorStruct.activePlan_
202  << "' at command index " << theIteratorStruct.commandIndex_ << ". "
203  << __E__;
204  __MOUT__ << "Paused plan '" << theIteratorStruct.activePlan_
205  << "' at command index " << theIteratorStruct.commandIndex_ << ". "
206  << __E__;
207 
208  continue; // resume workloop
209  }
210  else if(theIteratorStruct.doHaltAction_)
211  {
212  // valid HALT command!
213 
214  // safely end plan!
215  // i.e. check that command is complete
216 
217  __COUT__ << "Waiting to halt..." << __E__;
218  while(!iterator->checkCommand(&theIteratorStruct))
219  __COUT__ << "Waiting to halt..." << __E__;
220 
221  __COUT__ << "Completing halt..." << __E__;
222 
223  theIteratorStruct.doHaltAction_ = false; // clear
224 
225  iterator->haltIterator(iterator, &theIteratorStruct);
226 
227  // //last ditch effort to make sure FSM is halted
228  // iterator->haltIterator(
229  // iterator->theSupervisor_,
230  // theIteratorStruct.fsmName_);
231  //
232  // //lockout the messages array for the remainder of the scope
233  // //this guarantees the reading thread can safely access the
234  // messages
235  // if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting
236  // for iterator access" << __E__; std::lock_guard<std::mutex>
237  // lock(iterator->accessMutex_);
238  // if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have
239  // iterator access" << __E__;
240  //
241  // iterator->activePlanIsRunning_ = false;
242  // iterator->iteratorBusy_ = false;
243  //
244  // __COUT__ << "Halted plan '" << theIteratorStruct.activePlan_ << "'
245  // at command index " << theIteratorStruct.commandIndex_ <<
246  //". " << __E__;
247  // __MOUT__ << "Halted plan '" << theIteratorStruct.activePlan_ << "'
248  // at command index " << theIteratorStruct.commandIndex_ <<
249  //". " << __E__;
250  //
251  // theIteratorStruct.activePlan_ = ""; //clear
252  // theIteratorStruct.commandIndex_ = -1; //clear
253 
254  continue; // resume workloop
255  }
256 
259  // handle running
260  // __COUT__ << "thinking.." << theIteratorStruct.running_ << " " <<
261  // theIteratorStruct.activePlan_ << " cmd=" <<
262  // theIteratorStruct.commandIndex_ << __E__;
263  if(theIteratorStruct.running_ &&
264  theIteratorStruct.activePlan_ !=
265  "") // important, because after errors, still "running" until halt
266  {
267  if(theIteratorStruct.commandIndex_ == (unsigned int)-1)
268  {
269  // initialize the running plan
270 
271  __COUT__ << "Get commands" << __E__;
272 
273  theIteratorStruct.commandIndex_ = 0;
274 
275  theIteratorStruct.cfgMgr_
276  ->init(); // completely reset to re-align with any changes
277  itConfig = theIteratorStruct.cfgMgr_->__GET_CONFIG__(IterateTable);
278 
279  theIteratorStruct.commands_ = itConfig->getPlanCommands(
280  theIteratorStruct.cfgMgr_, theIteratorStruct.activePlan_);
281 
282  // reset commandIteration counts
283  theIteratorStruct.commandIterations_.clear();
284  for(auto& command : theIteratorStruct.commands_)
285  {
286  theIteratorStruct.commandIterations_.push_back(0);
287  __COUT__ << "command " << command.type_ << __E__;
288  __COUT__ << "table "
289  << IterateTable::commandToTableMap_.at(command.type_)
290  << __E__;
291  __COUT__ << "param count = " << command.params_.size() << __E__;
292 
293  for(auto& param : command.params_)
294  {
295  __COUT__ << "\t param " << param.first << " : " << param.second
296  << __E__;
297  }
298  }
299 
300  theIteratorStruct.originalTrackChanges_ =
301  ConfigurationInterface::isVersionTrackingEnabled();
302  theIteratorStruct.originalConfigGroup_ =
303  theIteratorStruct.cfgMgr_->getActiveGroupName();
304  theIteratorStruct.originalConfigKey_ =
305  theIteratorStruct.cfgMgr_->getActiveGroupKey();
306 
307  __COUT__ << "originalTrackChanges "
308  << theIteratorStruct.originalTrackChanges_ << __E__;
309  __COUT__ << "originalConfigGroup "
310  << theIteratorStruct.originalConfigGroup_ << __E__;
311  __COUT__ << "originalConfigKey " << theIteratorStruct.originalConfigKey_
312  << __E__;
313 
314  } // end initial section
315 
316  if(!theIteratorStruct.commandBusy_)
317  {
318  if(theIteratorStruct.commandIndex_ < theIteratorStruct.commands_.size())
319  {
320  // execute command
321  theIteratorStruct.commandBusy_ = true;
322 
323  __COUT__ << "Iterator starting command "
324  << theIteratorStruct.commandIndex_ + 1 << ": "
325  << theIteratorStruct
326  .commands_[theIteratorStruct.commandIndex_]
327  .type_
328  << __E__;
329  __MOUT__ << "Iterator starting command "
330  << theIteratorStruct.commandIndex_ + 1 << ": "
331  << theIteratorStruct
332  .commands_[theIteratorStruct.commandIndex_]
333  .type_
334  << __E__;
335 
336  iterator->startCommand(&theIteratorStruct);
337  }
338  else if(theIteratorStruct.commandIndex_ ==
339  theIteratorStruct.commands_.size()) // Done!
340  {
341  __COUT__ << "Finished Iteration Plan '"
342  << theIteratorStruct.activePlan_ << __E__;
343  __MOUT__ << "Finished Iteration Plan '"
344  << theIteratorStruct.activePlan_ << __E__;
345 
346  __COUT__ << "Reverting track changes." << __E__;
347  ConfigurationInterface::setVersionTrackingEnabled(
348  theIteratorStruct.originalTrackChanges_);
349 
350  __COUT__ << "Activating original group..." << __E__;
351  try
352  {
353  theIteratorStruct.cfgMgr_->activateTableGroup(
354  theIteratorStruct.originalConfigGroup_,
355  theIteratorStruct.originalConfigKey_);
356  }
357  catch(...)
358  {
359  __COUT_WARN__ << "Original group could not be activated."
360  << __E__;
361  }
362 
363  // leave FSM halted
364  __COUT__ << "Completing halt..." << __E__;
365 
366  iterator->haltIterator(iterator, &theIteratorStruct);
367 
368  //
369  // //leave FSM halted
370  // iterator->haltIterator(
371  // iterator->theSupervisor_,
372  // theIteratorStruct.fsmName_);
373  //
374  // //lockout the messages array for the remainder of
375  // the scope
376  // //this guarantees the reading thread can safely
377  // access the messages
378  // if(iterator->theSupervisor_->VERBOSE_MUTEX)
379  //__COUT__
380  //<< "Waiting for iterator access" << __E__;
381  // std::lock_guard<std::mutex>
382  // lock(iterator->accessMutex_);
383  // if(iterator->theSupervisor_->VERBOSE_MUTEX)
384  //__COUT__
385  //<< "Have iterator access" << __E__;
386  //
387  // //similar to halt
388  // iterator->activePlanIsRunning_ = false;
389  // iterator->iteratorBusy_ = false;
390  //
391  // iterator->lastStartedPlanName_ =
392  // theIteratorStruct.activePlan_;
393  // theIteratorStruct.activePlan_ = ""; //clear
394  // theIteratorStruct.commandIndex_ = -1; //clear
395  }
396  }
397  else if(theIteratorStruct.commandBusy_)
398  {
399  // check for command completion
400  if(iterator->checkCommand(&theIteratorStruct))
401  {
402  theIteratorStruct.commandBusy_ = false; // command complete
403 
404  ++theIteratorStruct.commandIndex_;
405 
406  __COUT__ << "Ready for next command. Done with "
407  << theIteratorStruct.commandIndex_ << " of "
408  << theIteratorStruct.commands_.size() << __E__;
409  __MOUT__ << "Iterator ready for next command. Done with "
410  << theIteratorStruct.commandIndex_ << " of "
411  << theIteratorStruct.commands_.size() << __E__;
412  }
413 
414  // Note: check command gets one shot to resume
415  if(theIteratorStruct.doResumeAction_) // end resume action
416  theIteratorStruct.doResumeAction_ = false;
417  }
418 
419  } // end running
420  else
421  sleep(1); // when inactive sleep a lot
422 
425 
426  // __COUT__ << "end loop.." << theIteratorStruct.running_ << " " <<
427  // theIteratorStruct.activePlan_ << " cmd=" <<
428  // theIteratorStruct.commandIndex_ << __E__;
429  }
430 
431  iterator->workloopRunning_ = false; // if we ever exit
432 }
433 catch(const std::runtime_error& e)
434 {
435  __SS__ << "Encountered error in Iterator thread:\n" << e.what() << __E__;
436  __COUT_ERR__ << ss.str();
437 
438  // lockout the messages array for the remainder of the scope
439  // this guarantees the reading thread can safely access the messages
440  if(iterator->theSupervisor_->VERBOSE_MUTEX)
441  __COUT__ << "Waiting for iterator access" << __E__;
442  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
443  if(iterator->theSupervisor_->VERBOSE_MUTEX)
444  __COUT__ << "Have iterator access" << __E__;
445 
446  iterator->workloopRunning_ = false; // if we ever exit
447  iterator->errorMessage_ = ss.str();
448 }
449 catch(...)
450 {
451  __SS__ << "Encountered unknown error in Iterator thread." << __E__;
452  __COUT_ERR__ << ss.str();
453 
454  // lockout the messages array for the remainder of the scope
455  // this guarantees the reading thread can safely access the messages
456  if(iterator->theSupervisor_->VERBOSE_MUTEX)
457  __COUT__ << "Waiting for iterator access" << __E__;
458  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
459  if(iterator->theSupervisor_->VERBOSE_MUTEX)
460  __COUT__ << "Have iterator access" << __E__;
461 
462  iterator->workloopRunning_ = false; // if we ever exit
463  iterator->errorMessage_ = ss.str();
464 } // end IteratorWorkLoop()
465 
466 //========================================================================================================================
467 void Iterator::startCommand(IteratorWorkLoopStruct* iteratorStruct) try
468 {
469  {
470  int i = 0;
471  for(const auto& depthIteration : iteratorStruct->stepIndexStack_)
472  {
473  __COUT__ << i++ << ":" << depthIteration << __E__;
474  }
475  }
476 
477  // should be mutually exclusive with GatewaySupervisor main thread state machine
478  // accesses lockout the messages array for the remainder of the scope this
479  // guarantees the reading thread can safely access the messages
480  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX)
481  __COUT__ << "Waiting for FSM access" << __E__;
482  std::lock_guard<std::mutex> lock(
483  iteratorStruct->theIterator_->theSupervisor_->stateMachineAccessMutex_);
484  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX)
485  __COUT__ << "Have FSM access" << __E__;
486 
487  // for out of range, throw exception - should never happen
488  if(iteratorStruct->commandIndex_ >= iteratorStruct->commands_.size())
489  {
490  __SS__ << "Out of range commandIndex = " << iteratorStruct->commandIndex_
491  << " in size = " << iteratorStruct->commands_.size() << __E__;
492  __SS_THROW__;
493  }
494 
495  // increment iteration count for command
496  ++iteratorStruct->commandIterations_[iteratorStruct->commandIndex_];
497 
498  std::string type = iteratorStruct->commands_[iteratorStruct->commandIndex_].type_;
499  if(type == IterateTable::COMMAND_BEGIN_LABEL)
500  {
501  return startCommandBeginLabel(iteratorStruct);
502  }
503  else if(type == IterateTable::COMMAND_CHOOSE_FSM)
504  {
505  return startCommandChooseFSM(
506  iteratorStruct,
507  iteratorStruct->commands_[iteratorStruct->commandIndex_]
508  .params_[IterateTable::commandChooseFSMParams_.NameOfFSM_]);
509  }
510  else if(type == IterateTable::COMMAND_CONFIGURE_ACTIVE_GROUP)
511  {
512  return startCommandConfigureActive(iteratorStruct);
513  }
514  else if(type == IterateTable::COMMAND_CONFIGURE_ALIAS)
515  {
516  return startCommandConfigureAlias(
517  iteratorStruct,
518  iteratorStruct->commands_[iteratorStruct->commandIndex_]
519  .params_[IterateTable::commandConfigureAliasParams_.SystemAlias_]);
520  }
521  else if(type == IterateTable::COMMAND_CONFIGURE_GROUP)
522  {
523  return startCommandConfigureGroup(iteratorStruct);
524  }
525  else if(type == IterateTable::COMMAND_EXECUTE_FE_MACRO)
526  {
527  return startCommandMacro(iteratorStruct, true /*isFEMacro*/);
528  }
529  else if(type == IterateTable::COMMAND_EXECUTE_MACRO)
530  {
531  return startCommandMacro(iteratorStruct, false /*isFEMacro*/);
532  }
533  else if(type == IterateTable::COMMAND_MODIFY_ACTIVE_GROUP)
534  {
535  return startCommandModifyActive(iteratorStruct);
536  }
537  else if(type == IterateTable::COMMAND_REPEAT_LABEL)
538  {
539  return startCommandRepeatLabel(iteratorStruct);
540  }
541  else if(type == IterateTable::COMMAND_RUN)
542  {
543  return startCommandRun(iteratorStruct);
544  }
545  else
546  {
547  __SS__ << "Failed attempt to start unrecognized command type = " << type << __E__;
548  __COUT_ERR__ << ss.str();
549  __SS_THROW__;
550  }
551 }
552 catch(...)
553 {
554  __COUT__ << "Error caught. Reverting track changes." << __E__;
555  ConfigurationInterface::setVersionTrackingEnabled(
556  iteratorStruct->originalTrackChanges_);
557 
558  __COUT__ << "Activating original group..." << __E__;
559  try
560  {
561  iteratorStruct->cfgMgr_->activateTableGroup(iteratorStruct->originalConfigGroup_,
562  iteratorStruct->originalConfigKey_);
563  }
564  catch(...)
565  {
566  __COUT_WARN__ << "Original group could not be activated." << __E__;
567  }
568  throw;
569 } // end startCommand()
570 
571 //========================================================================================================================
572 // checkCommand
573 // when busy for a while, start to sleep
574 // use sleep() or nanosleep()
575 bool Iterator::checkCommand(IteratorWorkLoopStruct* iteratorStruct) try
576 {
577  // for out of range, return done
578  if(iteratorStruct->commandIndex_ >= iteratorStruct->commands_.size())
579  {
580  __COUT__ << "Out of range commandIndex = " << iteratorStruct->commandIndex_
581  << " in size = " << iteratorStruct->commands_.size() << __E__;
582  return true;
583  }
584 
585  std::string type = iteratorStruct->commands_[iteratorStruct->commandIndex_].type_;
586  if(type == IterateTable::COMMAND_BEGIN_LABEL)
587  {
588  // do nothing
589  return true;
590  }
591  else if(type == IterateTable::COMMAND_CHOOSE_FSM)
592  {
593  // do nothing
594  return true;
595  }
596  else if(type == IterateTable::COMMAND_CONFIGURE_ALIAS ||
597  type == IterateTable::COMMAND_CONFIGURE_ACTIVE_GROUP ||
598  type == IterateTable::COMMAND_CONFIGURE_GROUP)
599  {
600  return checkCommandConfigure(iteratorStruct);
601  }
602  else if(type == IterateTable::COMMAND_EXECUTE_FE_MACRO)
603  {
604  return checkCommandMacro(iteratorStruct, true /*isFEMacro*/);
605  }
606  else if(type == IterateTable::COMMAND_EXECUTE_MACRO)
607  {
608  return checkCommandMacro(iteratorStruct, false /*isFEMacro*/);
609  }
610  else if(type == IterateTable::COMMAND_MODIFY_ACTIVE_GROUP)
611  {
612  // do nothing
613  return true;
614  }
615  else if(type == IterateTable::COMMAND_REPEAT_LABEL)
616  {
617  // do nothing
618  return true;
619  }
620  else if(type == IterateTable::COMMAND_RUN)
621  {
622  return checkCommandRun(iteratorStruct);
623  }
624  else
625  {
626  __SS__ << "Attempt to check unrecognized command type = " << type << __E__;
627  __COUT_ERR__ << ss.str();
628  __SS_THROW__;
629  }
630 }
631 catch(...)
632 {
633  __COUT__ << "Error caught. Reverting track changes." << __E__;
634  ConfigurationInterface::setVersionTrackingEnabled(
635  iteratorStruct->originalTrackChanges_);
636 
637  __COUT__ << "Activating original group..." << __E__;
638  try
639  {
640  iteratorStruct->cfgMgr_->activateTableGroup(iteratorStruct->originalConfigGroup_,
641  iteratorStruct->originalConfigKey_);
642  }
643  catch(...)
644  {
645  __COUT_WARN__ << "Original group could not be activated." << __E__;
646  }
647 
648  throw;
649 } // end checkCommand()
650 
651 //========================================================================================================================
652 void Iterator::startCommandChooseFSM(IteratorWorkLoopStruct* iteratorStruct,
653  const std::string& fsmName)
654 {
655  __COUT__ << "fsmName " << fsmName << __E__;
656 
657  iteratorStruct->fsmName_ = fsmName;
658  iteratorStruct->theIterator_->lastFsmName_ = fsmName;
659 
660  // Translate fsmName
661  // to gives us run alias (fsmRunAlias_) and next run number (fsmNextRunNumber_)
662 
663  // CAREFUL?? Threads
664 
666 
667  iteratorStruct->fsmRunAlias_ = "Run"; // default to "Run"
668 
669  // get stateMachineAliasFilter if possible
670  ConfigurationTree configLinkNode = iteratorStruct->cfgMgr_->getSupervisorTableNode(
671  iteratorStruct->theIterator_->theSupervisor_->getContextUID(),
672  iteratorStruct->theIterator_->theSupervisor_->getSupervisorUID());
673 
674  if(!configLinkNode.isDisconnected())
675  {
676  try // for backwards compatibility
677  {
678  ConfigurationTree fsmLinkNode =
679  configLinkNode.getNode("LinkToStateMachineTable");
680  if(!fsmLinkNode.isDisconnected())
681  iteratorStruct->fsmRunAlias_ =
682  fsmLinkNode.getNode(fsmName + "/RunDisplayAlias")
683  .getValue<std::string>();
684  else
685  __COUT_INFO__ << "FSM Link disconnected." << __E__;
686  }
687  catch(std::runtime_error& e)
688  {
689  //__COUT_INFO__ << e.what() << __E__;
690  __COUT_INFO__
691  << "No state machine Run alias. Ignoring and assuming alias of '"
692  << iteratorStruct->fsmRunAlias_ << ".'" << __E__;
693  }
694  catch(...)
695  {
696  __COUT_ERR__ << "Unknown error. Should never happen." << __E__;
697 
698  __COUT_INFO__
699  << "No state machine Run alias. Ignoring and assuming alias of '"
700  << iteratorStruct->fsmRunAlias_ << ".'" << __E__;
701  }
702  }
703  else
704  __COUT_INFO__ << "FSM Link disconnected." << __E__;
705 
706  __COUT__ << "fsmRunAlias_ = " << iteratorStruct->fsmRunAlias_ << __E__;
707 
709 
710  iteratorStruct->fsmNextRunNumber_ =
711  iteratorStruct->theIterator_->theSupervisor_->getNextRunNumber(
712  iteratorStruct->fsmName_);
713 
714  if(iteratorStruct->theIterator_->theSupervisor_->theStateMachine_
715  .getCurrentStateName() == "Running" ||
716  iteratorStruct->theIterator_->theSupervisor_->theStateMachine_
717  .getCurrentStateName() == "Paused")
718  --iteratorStruct->fsmNextRunNumber_; // current run number is one back
719 
720  __COUT__ << "fsmNextRunNumber_ = " << iteratorStruct->fsmNextRunNumber_ << __E__;
721 } // end startCommandChooseFSM()
722 
723 //========================================================================================================================
724 // return true if an action was attempted
725 bool Iterator::haltIterator(Iterator* iterator, IteratorWorkLoopStruct* iteratorStruct)
726 //(GatewaySupervisor* theSupervisor, const std::string& fsmName)
727 {
728  GatewaySupervisor* theSupervisor = iterator->theSupervisor_;
729  const std::string& fsmName = iterator->lastFsmName_;
730 
731  std::vector<std::string> fsmCommandParameters;
732  std::string errorStr = "";
733  std::string currentState = theSupervisor->theStateMachine_.getCurrentStateName();
734 
735  bool haltAttempted = true;
736  if(currentState == "Initialized" || currentState == "Halted")
737  {
738  __COUT__ << "Do nothing. Already halted." << __E__;
739  haltAttempted = false;
740  }
741  else if(currentState == "Running")
742  errorStr = theSupervisor->attemptStateMachineTransition(
743  0,
744  0,
745  "Abort",
746  fsmName,
747  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
748  WebUsers::DEFAULT_ITERATOR_USERNAME,
749  fsmCommandParameters);
750  else
751  errorStr = theSupervisor->attemptStateMachineTransition(
752  0,
753  0,
754  "Halt",
755  fsmName,
756  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
757  WebUsers::DEFAULT_ITERATOR_USERNAME,
758  fsmCommandParameters);
759 
760  if(haltAttempted)
761  {
762  if(errorStr != "")
763  {
764  __SS__ << "Iterator failed to halt because of the following error: "
765  << errorStr;
766  __SS_THROW__;
767  }
768 
769  // else successfully launched
770  __COUT__ << "FSM in transition = "
771  << theSupervisor->theStateMachine_.isInTransition() << __E__;
772  __COUT__ << "halting state machine launched." << __E__;
773  }
774 
775  // finish up halting the iterator
776  __COUT__ << "Conducting Iterator halt." << __E__;
777 
778  if(iteratorStruct)
779  {
780  __COUT__ << "Reverting track changes." << __E__;
781  ConfigurationInterface::setVersionTrackingEnabled(
782  iteratorStruct->originalTrackChanges_);
783 
784  __COUT__ << "Activating original group..." << __E__;
785  try
786  {
787  iteratorStruct->cfgMgr_->activateTableGroup(
788  iteratorStruct->originalConfigGroup_, iteratorStruct->originalConfigKey_);
789  }
790  catch(...)
791  {
792  __COUT_WARN__ << "Original group could not be activated." << __E__;
793  }
794  }
795 
796  // lockout the messages array for the remainder of the scope
797  // this guarantees the reading thread can safely access the messages
798  if(iterator->theSupervisor_->VERBOSE_MUTEX)
799  __COUT__ << "Waiting for iterator access" << __E__;
800  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
801  if(iterator->theSupervisor_->VERBOSE_MUTEX)
802  __COUT__ << "Have iterator access" << __E__;
803 
804  iterator->activePlanIsRunning_ = false;
805  iterator->iteratorBusy_ = false;
806 
807  // clear
808  iterator->activePlanName_ = "";
809  iterator->activeCommandIndex_ = -1;
810 
811  if(iteratorStruct)
812  {
813  __COUT__ << "Halted plan '" << iteratorStruct->activePlan_
814  << "' at command index " << iteratorStruct->commandIndex_ << ". "
815  << __E__;
816  __MOUT__ << "Halted plan '" << iteratorStruct->activePlan_
817  << "' at command index " << iteratorStruct->commandIndex_ << ". "
818  << __E__;
819 
820  iterator->lastStartedPlanName_ = iteratorStruct->activePlan_;
821  iteratorStruct->activePlan_ = ""; // clear
822  iteratorStruct->commandIndex_ = -1; // clear
823  }
824 
825  return haltAttempted;
826 } // end haltIterator()
827 
828 //========================================================================================================================
829 void Iterator::startCommandBeginLabel(IteratorWorkLoopStruct* iteratorStruct)
830 {
831  __COUT__ << "Entering label '"
832  << iteratorStruct->commands_[iteratorStruct->commandIndex_]
833  .params_[IterateTable::commandBeginLabelParams_.Label_]
834  << "'..." << std::endl;
835 
836  // add new step index to stack
837  iteratorStruct->stepIndexStack_.push_back(0);
838 } // end startCommandBeginLabel()
839 
840 //========================================================================================================================
841 void Iterator::startCommandRepeatLabel(IteratorWorkLoopStruct* iteratorStruct)
842 {
843  // search for first matching label backward and set command to there
844 
845  int numOfRepetitions;
846  sscanf(iteratorStruct->commands_[iteratorStruct->commandIndex_]
847  .params_[IterateTable::commandRepeatLabelParams_.NumberOfRepetitions_]
848  .c_str(),
849  "%d",
850  &numOfRepetitions);
851  __COUT__ << "numOfRepetitions remaining = " << numOfRepetitions << __E__;
852 
853  char repStr[200];
854 
855  if(numOfRepetitions <= 0)
856  {
857  // write original number of repetitions value back
858  sprintf(repStr, "%d", iteratorStruct->stepIndexStack_.back());
859  iteratorStruct->commands_[iteratorStruct->commandIndex_]
860  .params_[IterateTable::commandRepeatLabelParams_.NumberOfRepetitions_] =
861  repStr; // re-store as string
862 
863  // remove step index from stack
864  iteratorStruct->stepIndexStack_.pop_back();
865 
866  return; // no more repetitions
867  }
868 
869  --numOfRepetitions;
870 
871  // increment step index in stack
872  ++(iteratorStruct->stepIndexStack_.back());
873 
874  unsigned int i;
875  for(i = iteratorStruct->commandIndex_; i > 0;
876  --i) // assume 0 is always the fallback option
877  if(iteratorStruct->commands_[i].type_ == IterateTable::COMMAND_BEGIN_LABEL &&
878  iteratorStruct->commands_[iteratorStruct->commandIndex_]
879  .params_[IterateTable::commandRepeatLabelParams_.Label_] ==
880  iteratorStruct->commands_[i]
881  .params_[IterateTable::commandBeginLabelParams_.Label_])
882  break;
883 
884  sprintf(repStr, "%d", numOfRepetitions);
885  iteratorStruct->commands_[iteratorStruct->commandIndex_]
886  .params_[IterateTable::commandRepeatLabelParams_.NumberOfRepetitions_] =
887  repStr; // re-store as string
888 
889  iteratorStruct->commandIndex_ = i;
890  __COUT__ << "Jumping back to commandIndex " << iteratorStruct->commandIndex_ << __E__;
891 } // end startCommandRepeatLabel()
892 
893 //========================================================================================================================
894 void Iterator::startCommandRun(IteratorWorkLoopStruct* iteratorStruct)
895 {
896  iteratorStruct->runIsDone_ = false;
897  iteratorStruct->fsmCommandParameters_.clear();
898 
899  std::string errorStr = "";
900  std::string currentState = iteratorStruct->theIterator_->theSupervisor_
901  ->theStateMachine_.getCurrentStateName();
902 
903  // execute first transition (may need two)
904 
905  if(currentState == "Configured")
906  errorStr =
907  iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
908  0,
909  0,
910  "Start",
911  iteratorStruct->fsmName_,
912  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
913  WebUsers::DEFAULT_ITERATOR_USERNAME,
914  iteratorStruct->fsmCommandParameters_);
915  else
916  errorStr = "Can only Run from the Configured state. The current state is " +
917  currentState;
918 
919  if(errorStr != "")
920  {
921  __SS__ << "Iterator failed to run because of the following error: " << errorStr;
922  __SS_THROW__;
923  }
924 
925  // else successfully launched
926  __COUT__
927  << "FSM in transition = "
928  << iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.isInTransition()
929  << __E__;
930  __COUT__ << "startCommandRun success." << __E__;
931 } // end startCommandRun()
932 
933 //========================================================================================================================
934 void Iterator::startCommandConfigureActive(IteratorWorkLoopStruct* iteratorStruct)
935 {
936  __COUT__ << "startCommandConfigureActive " << __E__;
937 
938  // steps:
939  // get active config group
940  // transition to configure with parameters describing group
941 
942  std::string group = iteratorStruct->cfgMgr_->getActiveGroupName();
943  TableGroupKey key = iteratorStruct->cfgMgr_->getActiveGroupKey();
944 
945  __COUT__ << "group " << group << __E__;
946  __COUT__ << "key " << key << __E__;
947 
948  // create special alias for this group using : separators
949 
950  std::stringstream systemAlias;
951  systemAlias << "GROUP:" << group << ":" << key;
952  startCommandConfigureAlias(iteratorStruct, systemAlias.str());
953 } // end startCommandConfigureActive()
954 
955 //========================================================================================================================
956 void Iterator::startCommandConfigureGroup(IteratorWorkLoopStruct* iteratorStruct)
957 {
958  __COUT__ << "startCommandConfigureGroup " << __E__;
959 
960  // steps:
961  // transition to configure with parameters describing group
962 
963  std::string group =
964  iteratorStruct->commands_[iteratorStruct->commandIndex_]
965  .params_[IterateTable::commandConfigureGroupParams_.GroupName_];
966  TableGroupKey key =
967  TableGroupKey(iteratorStruct->commands_[iteratorStruct->commandIndex_]
968  .params_[IterateTable::commandConfigureGroupParams_.GroupKey_]);
969 
970  __COUT__ << "group " << group << __E__;
971  __COUT__ << "key " << key << __E__;
972 
973  // create special alias for this group using : separators
974 
975  std::stringstream systemAlias;
976  systemAlias << "GROUP:" << group << ":" << key;
977  startCommandConfigureAlias(iteratorStruct, systemAlias.str());
978 } // end startCommandConfigureGroup()
979 
980 //========================================================================================================================
981 void Iterator::startCommandConfigureAlias(IteratorWorkLoopStruct* iteratorStruct,
982  const std::string& systemAlias)
983 {
984  __COUT__ << "systemAlias " << systemAlias << __E__;
985 
986  iteratorStruct->fsmCommandParameters_.clear();
987  iteratorStruct->fsmCommandParameters_.push_back(systemAlias);
988 
989  std::string errorStr = "";
990  std::string currentState = iteratorStruct->theIterator_->theSupervisor_
991  ->theStateMachine_.getCurrentStateName();
992 
993  // execute first transition (may need two in conjunction with checkCommandConfigure())
994 
995  if(currentState == "Initial")
996  errorStr =
997  iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
998  0,
999  0,
1000  "Initialize",
1001  iteratorStruct->fsmName_,
1002  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1003  WebUsers::DEFAULT_ITERATOR_USERNAME,
1004  iteratorStruct->fsmCommandParameters_);
1005  else if(currentState == "Halted")
1006  errorStr =
1007  iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
1008  0,
1009  0,
1010  "Configure",
1011  iteratorStruct->fsmName_,
1012  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1013  WebUsers::DEFAULT_ITERATOR_USERNAME,
1014  iteratorStruct->fsmCommandParameters_);
1015  else if(currentState == "Configured")
1016  errorStr =
1017  iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
1018  0,
1019  0,
1020  "Halt",
1021  iteratorStruct->fsmName_,
1022  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1023  WebUsers::DEFAULT_ITERATOR_USERNAME,
1024  iteratorStruct->fsmCommandParameters_);
1025  else
1026  errorStr =
1027  "Can only Configure from the Initial or Halted state. The current state is " +
1028  currentState;
1029 
1030  if(errorStr != "")
1031  {
1032  __SS__ << "Iterator failed to configure with system alias '"
1033  << (iteratorStruct->fsmCommandParameters_.size()
1034  ? iteratorStruct->fsmCommandParameters_[0]
1035  : "UNKNOWN")
1036  << "' because of the following error: " << errorStr;
1037  __SS_THROW__;
1038  }
1039 
1040  // else successfully launched
1041  __COUT__
1042  << "FSM in transition = "
1043  << iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.isInTransition()
1044  << __E__;
1045  __COUT__ << "startCommandConfigureAlias success." << __E__;
1046 } // end startCommandConfigureAlias()
1047 
1048 //========================================================================================================================
1049 void Iterator::startCommandMacro(IteratorWorkLoopStruct* iteratorStruct,
1050  bool isFrontEndMacro)
1051 {
1052  // Steps:
1053  // 4 parameters CommandExecuteFEMacroParams:
1054  // //targets
1055  // const std::string FEMacroName_ = "FEMacroName";
1056  // //macro parameters (table/groupID)
1057 
1058  const std::string& macroName =
1059  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1060  .params_[IterateTable::commandExecuteMacroParams_.MacroName_];
1061  const std::string& enableSavingOutput =
1062  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1063  .params_[IterateTable::commandExecuteMacroParams_.EnableSavingOutput_];
1064  const std::string& outputFilePath =
1065  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1066  .params_[IterateTable::commandExecuteMacroParams_.OutputFilePath_];
1067  const std::string& outputFileRadix =
1068  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1069  .params_[IterateTable::commandExecuteMacroParams_.OutputFileRadix_];
1070  const std::string& inputArgs =
1071  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1072  .params_[IterateTable::commandExecuteMacroParams_.MacroArgumentString_];
1073 
1074  __COUTV__(macroName);
1075  __COUTV__(enableSavingOutput);
1076  __COUTV__(outputFilePath);
1077  __COUTV__(outputFileRadix);
1078  __COUTV__(inputArgs);
1079 
1080  // send request to MacroMaker a la FEVInterface::runFrontEndMacro
1081  // but need to pass iteration information, so that the call is launched by just one
1082  // send to each front end. Front-ends must immediately respond that is started
1083  // FEVInterfacesManager.. must start a thread for running the macro iterations
1084  // Then check for complete.
1085 
1086  iteratorStruct->targetsDone_.clear(); // reset
1087 
1088  __COUTV__(iteratorStruct->commands_[iteratorStruct->commandIndex_].targets_.size());
1089  for(const auto& target :
1090  iteratorStruct->commands_[iteratorStruct->commandIndex_].targets_)
1091  {
1092  __COUT__ << "target " << target.table_ << ":" << target.UID_ << __E__;
1093 
1094  // for each target, init to not done
1095  iteratorStruct->targetsDone_.push_back(false);
1096 
1097  xoap::MessageReference message =
1098  SOAPUtilities::makeSOAPMessageReference("FECommunication");
1099 
1100  SOAPParameters parameters;
1101  std::string type = isFrontEndMacro ? "feMacroMultiDimensionalStart"
1102  : "macroMultiDimensionalStart";
1103  parameters.addParameter("type", type);
1104  parameters.addParameter("requester", WebUsers::DEFAULT_ITERATOR_USERNAME);
1105  parameters.addParameter("targetInterfaceID", target.UID_);
1106  parameters.addParameter(isFrontEndMacro ? "feMacroName" : "macroName", macroName);
1107  parameters.addParameter("enableSavingOutput", enableSavingOutput);
1108  parameters.addParameter("outputFilePath", outputFilePath);
1109  parameters.addParameter("outputFileRadix", outputFileRadix);
1110  parameters.addParameter("inputArgs", inputArgs);
1111  SOAPUtilities::addParameters(message, parameters);
1112 
1113  __COUT__ << "Sending FE communication: " << SOAPUtilities::translate(message)
1114  << __E__;
1115 
1116  xoap::MessageReference replyMessage =
1117  iteratorStruct->theIterator_->theSupervisor_
1118  ->SOAPMessenger::sendWithSOAPReply(
1119  iteratorStruct->theIterator_->theSupervisor_->allSupervisorInfo_
1120  .getAllMacroMakerTypeSupervisorInfo()
1121  .begin()
1122  ->second.getDescriptor(),
1123  message);
1124 
1125  __COUT__ << "Response received: " << SOAPUtilities::translate(replyMessage)
1126  << __E__;
1127 
1128  SOAPParameters rxParameters;
1129  rxParameters.addParameter("Error");
1130  std::string response = SOAPUtilities::receive(replyMessage, rxParameters);
1131 
1132  std::string error = rxParameters.getValue("Error");
1133 
1134  if(response != type + "Done" || error != "")
1135  {
1136  // error occurred!
1137  __SS__ << "Error transmitting request to target interface '" << target.UID_
1138  << "' from '" << WebUsers::DEFAULT_ITERATOR_USERNAME << ".' Response '"
1139  << response << "' with error: " << error << __E__;
1140  __SS_THROW__;
1141  }
1142  } // end target loop
1143 
1144 } // end startCommandMacro()
1145 
1146 //========================================================================================================================
1147 bool Iterator::checkCommandMacro(IteratorWorkLoopStruct* iteratorStruct,
1148  bool isFrontEndMacro)
1149 {
1150  sleep(1);
1151 
1152  // Steps:
1153  // 4 parameters CommandExecuteFEMacroParams:
1154  // //targets
1155  // const std::string FEMacroName_ = "FEMacroName";
1156  // //macro parameters (table/groupID)
1157 
1158  const std::string& macroName =
1159  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1160  .params_[IterateTable::commandExecuteMacroParams_.MacroName_];
1161 
1162  __COUTV__(macroName);
1163 
1164  // send request to MacroMaker to check completion of macro
1165  // as targets are identified complete, remove targets_ from vector
1166 
1167  for(unsigned int i = 0;
1168  i < iteratorStruct->commands_[iteratorStruct->commandIndex_].targets_.size();
1169  ++i)
1170  {
1172  iteratorStruct->commands_[iteratorStruct->commandIndex_].targets_[i];
1173 
1174  __COUT__ << "target " << target.table_ << ":" << target.UID_ << __E__;
1175 
1176  xoap::MessageReference message =
1177  SOAPUtilities::makeSOAPMessageReference("FECommunication");
1178 
1179  SOAPParameters parameters;
1180  std::string type = isFrontEndMacro ? "feMacroMultiDimensionalCheck"
1181  : "macroMultiDimensionalCheck";
1182  parameters.addParameter("type", type);
1183  parameters.addParameter("requester", WebUsers::DEFAULT_ITERATOR_USERNAME);
1184  parameters.addParameter("targetInterfaceID", target.UID_);
1185  parameters.addParameter(isFrontEndMacro ? "feMacroName" : "macroName", macroName);
1186  SOAPUtilities::addParameters(message, parameters);
1187 
1188  __COUT__ << "Sending FE communication: " << SOAPUtilities::translate(message)
1189  << __E__;
1190 
1191  xoap::MessageReference replyMessage =
1192  iteratorStruct->theIterator_->theSupervisor_
1193  ->SOAPMessenger::sendWithSOAPReply(
1194  iteratorStruct->theIterator_->theSupervisor_->allSupervisorInfo_
1195  .getAllMacroMakerTypeSupervisorInfo()
1196  .begin()
1197  ->second.getDescriptor(),
1198  message);
1199 
1200  __COUT__ << "Response received: " << SOAPUtilities::translate(replyMessage)
1201  << __E__;
1202 
1203  SOAPParameters rxParameters;
1204  rxParameters.addParameter("Error");
1205  rxParameters.addParameter("Done");
1206  std::string response = SOAPUtilities::receive(replyMessage, rxParameters);
1207 
1208  std::string error = rxParameters.getValue("Error");
1209  bool done = rxParameters.getValue("Done") == "1";
1210 
1211  if(response != type + "Done" || error != "")
1212  {
1213  // error occurred!
1214  __SS__ << "Error transmitting request to target interface '" << target.UID_
1215  << "' from '" << WebUsers::DEFAULT_ITERATOR_USERNAME << ".' Response '"
1216  << response << "' with error: " << error << __E__;
1217  __SS_THROW__;
1218  }
1219 
1220  if(!done) // still more to do so give up checking
1221  return false;
1222 
1223  // mark target done
1224  iteratorStruct->targetsDone_[i] = true;
1225 
1226  // iteratorStruct->commands_[iteratorStruct->commandIndex_].targets_.erase(
1227  // targetIt--); //go back after delete
1228 
1229  } // end target loop
1230 
1231  // if here all targets are done
1232  return true;
1233 } // end checkCommandMacro()
1234 
1235 //========================================================================================================================
1236 void Iterator::startCommandModifyActive(IteratorWorkLoopStruct* iteratorStruct)
1237 {
1238  // Steps:
1239  // 4 parameters commandModifyActiveParams_:
1240  // const std::string DoTrackGroupChanges_ TrueFalse
1241  // //targets
1242  // const std::string RelativePathToField_ = "RelativePathToField";
1243  // const std::string FieldStartValue_ = "FieldStartValue";
1244  // const std::string FieldIterationStepSize_ = "FieldIterationStepSize";
1245  //
1246  // if tracking changes,
1247  // create a new group
1248  // for every enabled FE
1249  // set field = start value + stepSize * currentStepIndex_
1250  // activate group
1251  // else
1252  // load scratch group
1253  // for every enabled FE
1254  // set field = start value + stepSize * stepIndex
1255  // activate group
1256 
1257  bool doTrackGroupChanges = false;
1258  if("True" ==
1259  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1260  .params_[IterateTable::commandModifyActiveParams_.DoTrackGroupChanges_])
1261  doTrackGroupChanges = true;
1262 
1263  const std::string& startValueStr =
1264  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1265  .params_[IterateTable::commandModifyActiveParams_.FieldStartValue_];
1266  const std::string& stepSizeStr =
1267  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1268  .params_[IterateTable::commandModifyActiveParams_.FieldIterationStepSize_];
1269 
1270  const unsigned int stepIndex = iteratorStruct->stepIndexStack_.back();
1271 
1272  __COUT__ << "doTrackGroupChanges " << (doTrackGroupChanges ? "yes" : "no")
1273  << std::endl;
1274  __COUT__ << "stepIndex " << stepIndex << std::endl;
1275 
1276  ConfigurationInterface::setVersionTrackingEnabled(doTrackGroupChanges);
1277 
1278  // two approaches: double or long handling
1279 
1280  if(startValueStr.size() && (startValueStr[startValueStr.size() - 1] == 'f' ||
1281  startValueStr.find('.') != std::string::npos))
1282  {
1283  // handle as double
1284  double startValue = strtod(startValueStr.c_str(), 0);
1285  double stepSize = strtod(stepSizeStr.c_str(), 0);
1286 
1287  __COUT__ << "startValue " << startValue << std::endl;
1288  __COUT__ << "stepSize " << stepSize << std::endl;
1289  __COUT__ << "currentValue " << startValue + stepSize * stepIndex << std::endl;
1290 
1291  helpCommandModifyActive(
1292  iteratorStruct, startValue + stepSize * stepIndex, doTrackGroupChanges);
1293  }
1294  else // handle as long
1295  {
1296  long int startValue;
1297  long int stepSize;
1298 
1299  StringMacros::getNumber(startValueStr, startValue);
1300  StringMacros::getNumber(startValueStr, stepSize);
1301 
1302  // long int startValue;
1303  //
1304  // if(startValueStr.size() > 2 && startValueStr[1] == 'x') //assume hex value
1305  // startValue = strtol(startValueStr.c_str(),0,16);
1306  // else if(startValueStr.size() > 1 && startValueStr[0] == 'b') //assume
1307  // binary value startValue =
1308  // strtol(startValueStr.substr(1).c_str(),0,2);
1310  // strtol(startValueStr.c_str(),0,10);
1311  //
1312  // long int stepSize;
1313  //
1314  // if(stepSizeStr.size() > 2 && stepSizeStr[1] == 'x') //assume hex value
1315  // stepSize = strtol(stepSizeStr.c_str(),0,16);
1316  // else if(stepSizeStr.size() > 1 && stepSizeStr[0] == 'b') //assume binary
1317  // value stepSize = strtol(stepSizeStr.substr(1).c_str(),0,2); //skip
1318  // first 'b' character else stepSize =
1319  // strtol(stepSizeStr.c_str(),0,10);
1320 
1321  __COUT__ << "startValue " << startValue << std::endl;
1322  __COUT__ << "stepSize " << stepSize << std::endl;
1323  __COUT__ << "currentValue " << startValue + stepSize * stepIndex << std::endl;
1324 
1325  helpCommandModifyActive(
1326  iteratorStruct, startValue + stepSize * stepIndex, doTrackGroupChanges);
1327  }
1328 
1329 } // end startCommandModifyActive()
1330 
1331 //========================================================================================================================
1332 // checkCommandRun
1333 // return true if done
1334 //
1335 // Either will be done on (priority 1) running threads (for Frontends) ending
1336 // or (priority 2 and ignored if <= 0) duration timeout
1337 //
1338 // Note: use command structure strings to maintain duration left
1339 // Note: watch iterator->doPauseAction and iterator->doHaltAction and respond
1340 bool Iterator::checkCommandRun(IteratorWorkLoopStruct* iteratorStruct)
1341 {
1342  sleep(1); // sleep to give FSM time to transition
1343 
1344  // all RunControlStateMachine access commands should be mutually exclusive with
1345  // GatewaySupervisor main thread state machine accesses should be mutually exclusive
1346  // with GatewaySupervisor main thread state machine accesses lockout the messages
1347  // array for the remainder of the scope this guarantees the reading thread can safely
1348  // access the messages
1349  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX)
1350  __COUT__ << "Waiting for FSM access" << __E__;
1351  std::lock_guard<std::mutex> lock(
1352  iteratorStruct->theIterator_->theSupervisor_->stateMachineAccessMutex_);
1353  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX)
1354  __COUT__ << "Have FSM access" << __E__;
1355 
1356  if(iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.isInTransition())
1357  return false;
1358 
1359  iteratorStruct->fsmCommandParameters_.clear();
1360 
1361  std::string errorStr = "";
1362  std::string currentState = iteratorStruct->theIterator_->theSupervisor_
1363  ->theStateMachine_.getCurrentStateName();
1364 
1366  // check for imposed actions and forced exits
1367  if(iteratorStruct->doPauseAction_)
1368  {
1369  // transition to pause state
1370  __COUT__ << "Transitioning FSM to Paused..." << __E__;
1371 
1372  if(currentState == "Paused")
1373  {
1374  // done with early pause exit!
1375  __COUT__ << "Transition to Paused complete." << __E__;
1376  return true;
1377  }
1378  else if(currentState == "Running") // launch transition to pause
1379  errorStr = iteratorStruct->theIterator_->theSupervisor_
1380  ->attemptStateMachineTransition(
1381  0,
1382  0,
1383  "Pause",
1384  iteratorStruct->fsmName_,
1385  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1386  WebUsers::DEFAULT_ITERATOR_USERNAME,
1387  iteratorStruct->fsmCommandParameters_);
1388  else if(currentState == "Configured")
1389  {
1390  // no need to pause state machine, no run going on
1391  __COUT__ << "In Configured state. No need to transition to Paused." << __E__;
1392  return true;
1393  }
1394  else
1395  errorStr = "Expected to be in Paused. Unexpectedly, the current state is " +
1396  currentState +
1397  ". Last State Machine error message was as follows: " +
1398  iteratorStruct->theIterator_->theSupervisor_->theStateMachine_
1399  .getErrorMessage();
1400 
1401  if(errorStr != "")
1402  {
1403  __SS__ << "Iterator failed to pause because of the following error: "
1404  << errorStr;
1405  __SS_THROW__;
1406  }
1407  return false;
1408  }
1409  else if(iteratorStruct->doHaltAction_)
1410  {
1411  // transition to halted state
1412  __COUT__ << "Transitioning FSM to Halted..." << __E__;
1413 
1414  if(currentState == "Halted")
1415  {
1416  // done with early halt exit!
1417  __COUT__ << "Transition to Halted complete." << __E__;
1418  return true;
1419  }
1420  else if(currentState == "Running" || // launch transition to halt
1421  currentState == "Paused")
1422  errorStr = iteratorStruct->theIterator_->theSupervisor_
1423  ->attemptStateMachineTransition(
1424  0,
1425  0,
1426  "Abort",
1427  iteratorStruct->fsmName_,
1428  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1429  WebUsers::DEFAULT_ITERATOR_USERNAME,
1430  iteratorStruct->fsmCommandParameters_);
1431  else if(currentState == "Configured") // launch transition to halt
1432  errorStr = iteratorStruct->theIterator_->theSupervisor_
1433  ->attemptStateMachineTransition(
1434  0,
1435  0,
1436  "Halt",
1437  iteratorStruct->fsmName_,
1438  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1439  WebUsers::DEFAULT_ITERATOR_USERNAME,
1440  iteratorStruct->fsmCommandParameters_);
1441  else
1442  errorStr = "Expected to be in Halted. Unexpectedly, the current state is " +
1443  currentState +
1444  ". Last State Machine error message was as follows: " +
1445  iteratorStruct->theIterator_->theSupervisor_->theStateMachine_
1446  .getErrorMessage();
1447 
1448  if(errorStr != "")
1449  {
1450  __SS__ << "Iterator failed to halt because of the following error: "
1451  << errorStr;
1452  __SS_THROW__;
1453  }
1454  return false;
1455  }
1456  else if(iteratorStruct->doResumeAction_)
1457  {
1458  // Note: check command gets one shot to resume
1459 
1460  // transition to running state
1461  __COUT__ << "Transitioning FSM to Running..." << __E__;
1462 
1463  if(currentState == "Paused") // launch transition to running
1464  errorStr = iteratorStruct->theIterator_->theSupervisor_
1465  ->attemptStateMachineTransition(
1466  0,
1467  0,
1468  "Resume",
1469  iteratorStruct->fsmName_,
1470  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1471  WebUsers::DEFAULT_ITERATOR_USERNAME,
1472  iteratorStruct->fsmCommandParameters_);
1473 
1474  if(errorStr != "")
1475  {
1476  __SS__ << "Iterator failed to run because of the following error: "
1477  << errorStr;
1478  __SS_THROW__;
1479  }
1480  return false;
1481  }
1482 
1484  // normal running
1485 
1486  if(currentState != "Running")
1487  {
1488  if(iteratorStruct->runIsDone_ && currentState == "Configured")
1489  {
1490  // indication of done
1491  __COUT__ << "Reached end of run " << iteratorStruct->fsmNextRunNumber_
1492  << __E__;
1493  return true;
1494  }
1495 
1496  errorStr = "Expected to be in Running. Unexpectedly, the current state is " +
1497  currentState + ". Last State Machine error message was as follows: " +
1498  iteratorStruct->theIterator_->theSupervisor_->theStateMachine_
1499  .getErrorMessage();
1500  }
1501  else // else in Running state! Check for end of run
1502  {
1503  bool waitOnRunningThreads = false;
1504  if("True" == iteratorStruct->commands_[iteratorStruct->commandIndex_]
1505  .params_[IterateTable::commandRunParams_.WaitOnRunningThreads_])
1506  waitOnRunningThreads = true;
1507 
1508  time_t remainingDurationInSeconds; // parameter converted during start to the
1509  // stop linux timestamp
1510  sscanf(iteratorStruct->commands_[iteratorStruct->commandIndex_]
1511  .params_[IterateTable::commandRunParams_.DurationInSeconds_]
1512  .c_str(),
1513  "%ld",
1514  &remainingDurationInSeconds);
1515 
1516  __COUT__ << "waitOnRunningThreads " << waitOnRunningThreads << __E__;
1517  __COUT__ << "remainingDurationInSeconds " << remainingDurationInSeconds << __E__;
1518 
1520  // priority 1 is waiting on running threads
1521  if(waitOnRunningThreads)
1522  {
1523  // get status of all running FE workloops
1524  GatewaySupervisor* theSupervisor =
1525  iteratorStruct->theIterator_->theSupervisor_;
1526 
1527  bool allFrontEndsAreDone = true;
1528  for(auto& it : theSupervisor->allSupervisorInfo_.getAllFETypeSupervisorInfo())
1529  {
1530  try
1531  {
1532  std::string status = theSupervisor->send(it.second.getDescriptor(),
1533  "WorkLoopStatusRequest");
1534 
1535  __COUT__ << "FESupervisor instance " << it.first
1536  << " has status = " << status << std::endl;
1537 
1538  if(status != CoreSupervisorBase::WORK_LOOP_DONE)
1539  {
1540  allFrontEndsAreDone = false;
1541  break;
1542  }
1543  }
1544  catch(xdaq::exception::Exception& e)
1545  {
1546  __SS__ << "Could not retrieve status from FESupervisor instance "
1547  << it.first << "." << std::endl;
1548  __COUT_WARN__ << ss.str();
1549  errorStr = ss.str();
1550 
1551  if(errorStr != "")
1552  {
1553  __SS__
1554  << "Iterator failed to run because of the following error: "
1555  << errorStr;
1556  __SS_THROW__;
1557  }
1558  }
1559  }
1560 
1561  if(allFrontEndsAreDone)
1562  {
1563  // need to end run!
1564  __COUT__ << "FE workloops all complete! Stopping run..." << __E__;
1565 
1566  errorStr = iteratorStruct->theIterator_->theSupervisor_
1567  ->attemptStateMachineTransition(
1568  0,
1569  0,
1570  "Stop",
1571  iteratorStruct->fsmName_,
1572  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1573  WebUsers::DEFAULT_ITERATOR_USERNAME,
1574  iteratorStruct->fsmCommandParameters_);
1575 
1576  if(errorStr != "")
1577  {
1578  __SS__
1579  << "Iterator failed to stop run because of the following error: "
1580  << errorStr;
1581  __SS_THROW__;
1582  }
1583 
1584  // write indication of run done into duration
1585  iteratorStruct->runIsDone_ = true;
1586 
1587  return false;
1588  }
1589  }
1590 
1592  // priority 2 is duration, if <= 0 it is ignored
1593  if(remainingDurationInSeconds > 1)
1594  {
1595  --remainingDurationInSeconds;
1596 
1597  // write back to string
1598  char str[200];
1599  sprintf(str, "%ld", remainingDurationInSeconds);
1600  iteratorStruct->commands_[iteratorStruct->commandIndex_]
1601  .params_[IterateTable::commandRunParams_.DurationInSeconds_] =
1602  str; // re-store as string
1603  }
1604  else if(remainingDurationInSeconds == 1)
1605  {
1606  // need to end run!
1607  __COUT__ << "Time duration reached! Stopping run..." << __E__;
1608 
1609  errorStr = iteratorStruct->theIterator_->theSupervisor_
1610  ->attemptStateMachineTransition(
1611  0,
1612  0,
1613  "Stop",
1614  iteratorStruct->fsmName_,
1615  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1616  WebUsers::DEFAULT_ITERATOR_USERNAME,
1617  iteratorStruct->fsmCommandParameters_);
1618 
1619  if(errorStr != "")
1620  {
1621  __SS__ << "Iterator failed to stop run because of the following error: "
1622  << errorStr;
1623  __SS_THROW__;
1624  }
1625 
1626  // write indication of run done into duration
1627  iteratorStruct->runIsDone_ = true;
1628 
1629  return false;
1630  }
1631  }
1632 
1633  if(errorStr != "")
1634  {
1635  __SS__ << "Iterator failed to run because of the following error: " << errorStr;
1636  __SS_THROW__;
1637  }
1638  return false;
1639 }
1640 
1641 //========================================================================================================================
1642 // return true if done
1643 bool Iterator::checkCommandConfigure(IteratorWorkLoopStruct* iteratorStruct)
1644 {
1645  sleep(1); // sleep to give FSM time to transition
1646 
1647  // all RunControlStateMachine access commands should be mutually exclusive with
1648  // GatewaySupervisor main thread state machine accesses should be mutually exclusive
1649  // with GatewaySupervisor main thread state machine accesses lockout the messages
1650  // array for the remainder of the scope this guarantees the reading thread can safely
1651  // access the messages
1652  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX)
1653  __COUT__ << "Waiting for FSM access" << __E__;
1654  std::lock_guard<std::mutex> lock(
1655  iteratorStruct->theIterator_->theSupervisor_->stateMachineAccessMutex_);
1656  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX)
1657  __COUT__ << "Have FSM access" << __E__;
1658 
1659  if(iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.isInTransition())
1660  return false;
1661 
1662  std::string errorStr = "";
1663  std::string currentState = iteratorStruct->theIterator_->theSupervisor_
1664  ->theStateMachine_.getCurrentStateName();
1665 
1666  if(currentState == "Halted")
1667  errorStr =
1668  iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
1669  0,
1670  0,
1671  "Configure",
1672  iteratorStruct->fsmName_,
1673  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1674  WebUsers::DEFAULT_ITERATOR_USERNAME,
1675  iteratorStruct->fsmCommandParameters_);
1676  else if(currentState != "Configured")
1677  errorStr = "Expected to be in Configure. Unexpectedly, the current state is " +
1678  currentState + "." +
1679  ". Last State Machine error message was as follows: " +
1680  iteratorStruct->theIterator_->theSupervisor_->theStateMachine_
1681  .getErrorMessage();
1682  else // else successfully done (in Configured state!)
1683  {
1684  __COUT__ << "checkCommandConfigureAlias complete." << __E__;
1685 
1686  // also activate the Iterator config manager to mimic active config
1687  std::pair<std::string, TableGroupKey> newActiveGroup =
1688  iteratorStruct->cfgMgr_->getTableGroupFromAlias(
1689  iteratorStruct->fsmCommandParameters_[0]);
1690  iteratorStruct->cfgMgr_->loadTableGroup(
1691  newActiveGroup.first, newActiveGroup.second, true /*activate*/);
1692 
1693  __COUT__ << "originalTrackChanges " << iteratorStruct->originalTrackChanges_
1694  << __E__;
1695  __COUT__ << "originalConfigGroup " << iteratorStruct->originalConfigGroup_
1696  << __E__;
1697  __COUT__ << "originalConfigKey " << iteratorStruct->originalConfigKey_ << __E__;
1698 
1699  __COUT__ << "currentTrackChanges "
1700  << ConfigurationInterface::isVersionTrackingEnabled() << __E__;
1701  __COUT__ << "originalConfigGroup "
1702  << iteratorStruct->cfgMgr_->getActiveGroupName() << __E__;
1703  __COUT__ << "originalConfigKey " << iteratorStruct->cfgMgr_->getActiveGroupKey()
1704  << __E__;
1705 
1706  return true;
1707  }
1708 
1709  if(errorStr != "")
1710  {
1711  __SS__ << "Iterator failed to configure with system alias '"
1712  << (iteratorStruct->fsmCommandParameters_.size()
1713  ? iteratorStruct->fsmCommandParameters_[0]
1714  : "UNKNOWN")
1715  << "' because of the following error: " << errorStr;
1716  __SS_THROW__;
1717  }
1718  return false;
1719 }
1720 
1721 //========================================================================================================================
1722 bool Iterator::handleCommandRequest(HttpXmlDocument& xmldoc,
1723  const std::string& command,
1724  const std::string& parameter)
1725 {
1726  __COUTV__(command);
1727  if(command == "iteratePlay")
1728  {
1729  playIterationPlan(xmldoc, parameter);
1730  return true;
1731  }
1732  else if(command == "iteratePause")
1733  {
1734  pauseIterationPlan(xmldoc);
1735  return true;
1736  }
1737  else if(command == "iterateHalt")
1738  {
1739  haltIterationPlan(xmldoc);
1740  return true;
1741  }
1742  else if(command == "getIterationPlanStatus")
1743  {
1744  getIterationPlanStatus(xmldoc);
1745  return true;
1746  }
1747  else // return true if iterator has control of state machine
1748  {
1749  // lockout the messages array for the remainder of the scope
1750  // this guarantees the reading thread can safely access the messages
1751  if(theSupervisor_->VERBOSE_MUTEX)
1752  __COUT__ << "Waiting for iterator access" << __E__;
1753  std::lock_guard<std::mutex> lock(accessMutex_);
1754  if(theSupervisor_->VERBOSE_MUTEX)
1755  __COUT__ << "Have iterator access" << __E__;
1756 
1757  if(iteratorBusy_)
1758  {
1759  __SS__ << "Error - Can not accept request because the Iterator "
1760  << "is currently "
1761  << "in control of State Machine progress. ";
1762  __COUT_ERR__ << "\n" << ss.str();
1763  __MOUT_ERR__ << "\n" << ss.str();
1764 
1765  xmldoc.addTextElementToData("state_tranisition_attempted",
1766  "0"); // indicate to GUI transition NOT attempted
1767  xmldoc.addTextElementToData(
1768  "state_tranisition_attempted_err",
1769  ss.str()); // indicate to GUI transition NOT attempted
1770 
1771  return true; // to block other commands
1772  }
1773  }
1774  return false;
1775 }
1776 
1777 //========================================================================================================================
1778 void Iterator::playIterationPlan(HttpXmlDocument& xmldoc, const std::string& planName)
1779 {
1780  __MOUT__ << "Attempting to play iteration plan '" << planName << ".'" << __E__;
1781  __COUT__ << "Attempting to play iteration plan '" << planName << ".'" << __E__;
1782 
1783  // setup "play" command
1784 
1785  // lockout the messages array for the remainder of the scope
1786  // this guarantees the reading thread can safely access the messages
1787  if(theSupervisor_->VERBOSE_MUTEX)
1788  __COUT__ << "Waiting for iterator access" << __E__;
1789  std::lock_guard<std::mutex> lock(accessMutex_);
1790  if(theSupervisor_->VERBOSE_MUTEX)
1791  __COUT__ << "Have iterator access" << __E__;
1792 
1793  if(!activePlanIsRunning_ && !commandPlay_)
1794  {
1795  if(!workloopRunning_)
1796  {
1797  // start thread with member variables initialized
1798 
1799  workloopRunning_ = true;
1800 
1801  // must start thread first
1802  std::thread([](Iterator* iterator) { Iterator::IteratorWorkLoop(iterator); },
1803  this)
1804  .detach();
1805  }
1806 
1807  activePlanName_ = planName;
1808  commandPlay_ = true;
1809  }
1810  else
1811  {
1812  __SS__ << "Invalid play command attempted. Can only play when the Iterator is "
1813  "inactive or paused."
1814  << " If you would like to restart an iteration plan, first try halting."
1815  << __E__;
1816  __MOUT__ << ss.str();
1817 
1818  xmldoc.addTextElementToData("error_message", ss.str());
1819 
1820  __COUT__ << "Invalid play command attempted. " << commandPlay_ << " "
1821  << activePlanName_ << __E__;
1822  }
1823 }
1824 
1825 //========================================================================================================================
1826 void Iterator::pauseIterationPlan(HttpXmlDocument& xmldoc)
1827 {
1828  __MOUT__ << "Attempting to pause iteration plan '" << activePlanName_ << ".'"
1829  << __E__;
1830  __COUT__ << "Attempting to pause iteration plan '" << activePlanName_ << ".'"
1831  << __E__;
1832 
1833  // setup "pause" command
1834 
1835  // lockout the messages array for the remainder of the scope
1836  // this guarantees the reading thread can safely access the messages
1837  if(theSupervisor_->VERBOSE_MUTEX)
1838  __COUT__ << "Waiting for iterator access" << __E__;
1839  std::lock_guard<std::mutex> lock(accessMutex_);
1840  if(theSupervisor_->VERBOSE_MUTEX)
1841  __COUT__ << "Have iterator access" << __E__;
1842 
1843  if(workloopRunning_ && activePlanIsRunning_ && !commandPause_)
1844  {
1845  commandPause_ = true;
1846  }
1847  else
1848  {
1849  __SS__ << "Invalid pause command attempted. Can only pause when running."
1850  << __E__;
1851  __MOUT__ << ss.str();
1852 
1853  xmldoc.addTextElementToData("error_message", ss.str());
1854 
1855  __COUT__ << "Invalid pause command attempted. " << workloopRunning_ << " "
1856  << activePlanIsRunning_ << " " << commandPause_ << " " << activePlanName_
1857  << __E__;
1858  }
1859 }
1860 
1861 //========================================================================================================================
1862 void Iterator::haltIterationPlan(HttpXmlDocument& xmldoc)
1863 {
1864  __MOUT__ << "Attempting to halt iteration plan '" << activePlanName_ << ".'" << __E__;
1865  __COUT__ << "Attempting to halt iteration plan '" << activePlanName_ << ".'" << __E__;
1866 
1867  // setup "halt" command
1868 
1869  if(workloopRunning_)
1870  {
1871  // lockout the messages array for the remainder of the scope
1872  // this guarantees the reading thread can safely access the messages
1873  if(theSupervisor_->VERBOSE_MUTEX)
1874  __COUT__ << "Waiting for iterator access" << __E__;
1875  std::lock_guard<std::mutex> lock(accessMutex_);
1876  if(theSupervisor_->VERBOSE_MUTEX)
1877  __COUT__ << "Have iterator access" << __E__;
1878 
1879  __COUT__ << "activePlanIsRunning_: " << activePlanIsRunning_ << __E__;
1880  __COUT__ << "Passing halt command to iterator thread." << __E__;
1881  commandHalt_ = true;
1882 
1883  // clear
1884  activePlanName_ = "";
1885  activeCommandIndex_ = -1;
1886  }
1887  else // no thread, so halt (and reset Error') without command to thread
1888  {
1889  __COUT__ << "No thread, so conducting halt." << __E__;
1890  Iterator::haltIterator(this);
1891  //
1892  // activePlanIsRunning_ = false;
1893  // iteratorBusy_ = false;
1894  //
1895  // bool haltAttempted = false;
1896  // try
1897  // {
1898  // haltAttempted = Iterator::haltIterator(theSupervisor_, lastFsmName_);
1899  // }
1900  // catch(const std::runtime_error& e)
1901  // {
1902  // haltAttempted = false;
1903  // __COUT__ << "Halt error: " << e.what() << __E__;
1904  // }
1905  //
1906  // if(!haltAttempted) //then show error
1907  // {
1908  // __SS__ << "Invalid halt command attempted. Can only halt when there is
1909  // an active iteration plan." << __E__;
1910  // __MOUT_ERR__ << ss.str();
1911  //
1912  // xmldoc.addTextElementToData("error_message", ss.str());
1913  //
1914  // __COUT__ << "Invalid halt command attempted. " <<
1915  // workloopRunning_ << " " <<
1916  // activePlanIsRunning_ << " " <<
1917  // commandHalt_ << " " <<
1918  // activePlanName_ << __E__;
1919  // }
1920  // else
1921  // __COUT__ << "Halt was attempted." << __E__;
1922  }
1923 }
1924 
1925 //========================================================================================================================
1926 // return state machine and iterator status
1927 void Iterator::getIterationPlanStatus(HttpXmlDocument& xmldoc)
1928 {
1929  xmldoc.addTextElementToData(
1930  "current_state",
1931  theSupervisor_->theStateMachine_.isInTransition()
1932  ? theSupervisor_->theStateMachine_.getCurrentTransitionName(
1933  theSupervisor_->stateMachineLastCommandInput_)
1934  : theSupervisor_->theStateMachine_.getCurrentStateName());
1935 
1936  // xmldoc.addTextElementToData("in_transition",
1937  // theSupervisor_->theStateMachine_.isInTransition() ? "1" : "0");
1938  if(theSupervisor_->theStateMachine_.isInTransition())
1939  xmldoc.addTextElementToData(
1940  "transition_progress",
1941  theSupervisor_->theProgressBar_.readPercentageString());
1942  else
1943  xmldoc.addTextElementToData("transition_progress", "100");
1944 
1945  char tmp[20];
1946  sprintf(tmp, "%lu", theSupervisor_->theStateMachine_.getTimeInState());
1947  xmldoc.addTextElementToData("time_in_state", tmp);
1948 
1949  // lockout the messages array for the remainder of the scope
1950  // this guarantees the reading thread can safely access the messages
1951  if(theSupervisor_->VERBOSE_MUTEX)
1952  __COUT__ << "Waiting for iterator access" << __E__;
1953  std::lock_guard<std::mutex> lock(accessMutex_);
1954  if(theSupervisor_->VERBOSE_MUTEX)
1955  __COUT__ << "Have iterator access" << __E__;
1956 
1957  xmldoc.addTextElementToData("active_plan", activePlanName_);
1958  xmldoc.addTextElementToData("last_started_plan", lastStartedPlanName_);
1959  xmldoc.addTextElementToData("last_finished_plan", lastFinishedPlanName_);
1960 
1961  sprintf(tmp, "%u", activeCommandIndex_);
1962  xmldoc.addTextElementToData("current_command_index", tmp);
1963  sprintf(tmp, "%ld", time(0) - activeCommandStartTime_);
1964  xmldoc.addTextElementToData("current_command_duration", tmp);
1965  sprintf(tmp, "%u", activeCommandIteration_);
1966  xmldoc.addTextElementToData("current_command_iteration", tmp);
1967 
1968  for(const auto& depthIteration : depthIterationStack_)
1969  {
1970  sprintf(tmp, "%u", depthIteration);
1971  xmldoc.addTextElementToData("depth_iteration", tmp);
1972  }
1973 
1974  if(activePlanIsRunning_ && iteratorBusy_)
1975  {
1976  if(workloopRunning_)
1977  xmldoc.addTextElementToData("active_plan_status", "Running");
1978  else
1979  xmldoc.addTextElementToData("active_plan_status", "Error");
1980  }
1981  else if(!activePlanIsRunning_ && iteratorBusy_)
1982  xmldoc.addTextElementToData("active_plan_status", "Paused");
1983  else
1984  xmldoc.addTextElementToData("active_plan_status", "Inactive");
1985 
1986  xmldoc.addTextElementToData("error_message", errorMessage_);
1987 }