otsdaq  v2_01_00
Iterator.cc
1 #include "otsdaq-core/GatewaySupervisor/Iterator.h"
2 #include "otsdaq-core/MessageFacility/MessageFacility.h"
3 #include "otsdaq-core/Macros/CoutMacros.h"
4 #include "otsdaq-core/GatewaySupervisor/GatewaySupervisor.h"
5 #include "otsdaq-core/CoreSupervisors/CoreSupervisorBase.h"
6 #include "otsdaq-core/WebUsersUtilities/WebUsers.h"
7 
8 
9 #include <iostream>
10 #include <thread> //for std::thread
11 
12 #undef __MF_SUBJECT__
13 #define __MF_SUBJECT__ "Iterator"
14 
15 using namespace ots;
16 
17 //========================================================================================================================
18 Iterator::Iterator(GatewaySupervisor* supervisor)
19 : workloopRunning_ (false)
20 , activePlanIsRunning_ (false)
21 , iteratorBusy_ (false)
22 , commandPlay_(false), commandPause_(false), commandHalt_(false)
23 , activePlanName_ ("")
24 , activeCommandIndex_ (-1)
25 , activeCommandStartTime_ (0)
26 , theSupervisor_ (supervisor)
27 {
28  __MOUT__ << "Iterator constructed." << __E__;
29  __COUT__ << "Iterator constructed." << __E__;
30 
31 }
32 
33 //========================================================================================================================
34 Iterator::~Iterator(void)
35 {
36 }
37 
38 //========================================================================================================================
39 void Iterator::IteratorWorkLoop(Iterator *iterator)
40 try
41 {
42  __MOUT__ << "Iterator work loop starting..." << __E__;
43  __COUT__ << "Iterator work loop starting..." << __E__;
44 
45  //mutex init scope
46  {
47  //lockout the messages array for the remainder of the scope
48  //this guarantees the reading thread can safely access the messages
49  if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for iterator access" << __E__;
50  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
51  if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have iterator access" << __E__;
52 
53  iterator->errorMessage_ = ""; //clear error message
54  }
55 
56 
57  ConfigurationManagerRW theConfigurationManager(WebUsers::DEFAULT_ITERATOR_USERNAME); //this is a restricted username
58  //theConfigurationManager.init();
59  theConfigurationManager.getAllConfigurationInfo(true); // to prep all info
60 
61  IteratorWorkLoopStruct theIteratorStruct(iterator,
62  &theConfigurationManager);
63 
64 
65  const IterateConfiguration* itConfig;
66 
67  std::vector<IterateConfiguration::Command> commands;
68 
69  while(1)
70  {
71  //Process:
72  // - always "listen" for commands
73  // - play: if no plan running, activePlanIsRunning_ = true,
74  // and start or continue plan based on name/commandIndex
75  // - pause: if plan playing, pause it, activePlanIsRunning_ = false
76  // and do not clear commandIndex or name, iteratorBusy_ = true
77  // - halt: if plan playing or not, activePlanIsRunning_ = false
78  // and clear commandIndex or name, iteratorBusy_ = false
79  // - when running
80  // - go through each command
81  // - start the command, commandBusy = true
82  // - check for complete, then commandBusy = false
83 
84  //start command handling
85  //define mutex scope
86  {
87  //lockout the messages array for the remainder of the scope
88  //this guarantees the reading thread can safely access the messages
89  if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for iterator access" << __E__;
90  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
91  if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have iterator access" << __E__;
92 
93  if(iterator->commandPlay_)
94  {
95  iterator->commandPlay_ = false; //clear
96 
97  if(!iterator->activePlanIsRunning_)
98  {
99  //valid PLAY command!
100 
101  iterator->activePlanIsRunning_ = true;
102  iterator->iteratorBusy_ = true;
103 
104  if(theIteratorStruct.activePlan_ != iterator->activePlanName_)
105  {
106  __COUT__ << "New plan name encountered old=" << theIteratorStruct.activePlan_ <<
107  " vs new=" << iterator->activePlanName_ << __E__;
108  theIteratorStruct.commandIndex_ = -1; //reset for new plan
109  }
110 
111  theIteratorStruct.activePlan_ = iterator->activePlanName_;
112  iterator->lastStartedPlanName_ = iterator->activePlanName_;
113 
114  if(theIteratorStruct.commandIndex_ == (unsigned int)-1)
115  {
116  __COUT__ << "Starting plan '" << theIteratorStruct.activePlan_ << ".'" << __E__;
117  __MOUT__ << "Starting plan '" << theIteratorStruct.activePlan_ << ".'" << __E__;
118  }
119  else
120  {
121  theIteratorStruct.doResumeAction_ = true;
122  __COUT__ << "Continuing plan '" << theIteratorStruct.activePlan_ << "' at command index " <<
123  theIteratorStruct.commandIndex_ << ". " << __E__;
124  __MOUT__ << "Continuing plan '" << theIteratorStruct.activePlan_ << "' at command index " <<
125  theIteratorStruct.commandIndex_ << ". " << __E__;
126  }
127  }
128  }
129  else if(iterator->commandPause_ && !theIteratorStruct.doPauseAction_)
130  {
131  theIteratorStruct.doPauseAction_ = true;
132  iterator->commandPause_ = false; //clear
133  }
134  else if(iterator->commandHalt_ && !theIteratorStruct.doHaltAction_)
135  {
136  theIteratorStruct.doHaltAction_ = true;
137  iterator->commandHalt_ = false; //clear
138  }
139 
140  theIteratorStruct.running_ = iterator->activePlanIsRunning_;
141 
142  if(iterator->activeCommandIndex_ != //update active command status if changed
143  theIteratorStruct.commandIndex_)
144  {
145  iterator->activeCommandIndex_ = theIteratorStruct.commandIndex_;
146  iterator->activeCommandStartTime_ = time(0); //reset on any change
147 
148  if(theIteratorStruct.commandIndex_ < theIteratorStruct.commandIterations_.size())
149  iterator->activeCommandIteration_ = theIteratorStruct.commandIterations_
150  [theIteratorStruct.commandIndex_];
151  else
152  iterator->activeCommandIteration_ = -1;
153 
154  iterator->depthIterationStack_.clear();
155  for(const auto& depthIteration:theIteratorStruct.stepIndexStack_)
156  iterator->depthIterationStack_.push_back(depthIteration);
157  //if(theIteratorStruct.stepIndexStack_.size())
158  // iterator->activeLoopIteration_ = theIteratorStruct.stepIndexStack_.back();
159  //else
160  // iterator->activeLoopIteration_ = -1;
161  }
162 
163  } //end command handling and iterator mutex
164 
165 
168  //do halt or pause action outside of iterator mutex
169 
170  if(theIteratorStruct.doPauseAction_)
171  {
172  //valid PAUSE command!
173 
174  //safely pause plan!
175  // i.e. check that command is complete
176 
177  __COUT__ << "Waiting to pause..." << __E__;
178  while(!iterator->checkCommand(&theIteratorStruct))
179  __COUT__ << "Waiting to pause..." << __E__;
180 
181  __COUT__ << "Completing pause..." << __E__;
182 
183  theIteratorStruct.doPauseAction_ = false; //clear
184 
185  //lockout the messages array for the remainder of the scope
186  //this guarantees the reading thread can safely access the messages
187  if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for iterator access" << __E__;
188  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
189  if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have iterator access" << __E__;
190 
191  iterator->activePlanIsRunning_ = false;
192 
193  __COUT__ << "Paused plan '" << theIteratorStruct.activePlan_ << "' at command index " <<
194  theIteratorStruct.commandIndex_ << ". " << __E__;
195  __MOUT__ << "Paused plan '" << theIteratorStruct.activePlan_ << "' at command index " <<
196  theIteratorStruct.commandIndex_ << ". " << __E__;
197 
198  continue; //resume workloop
199  }
200  else if(theIteratorStruct.doHaltAction_)
201  {
202  //valid HALT command!
203 
204  //safely end plan!
205  // i.e. check that command is complete
206 
207  __COUT__ << "Waiting to halt..." << __E__;
208  while(!iterator->checkCommand(&theIteratorStruct))
209  __COUT__ << "Waiting to halt..." << __E__;
210 
211  __COUT__ << "Completing halt..." << __E__;
212 
213  theIteratorStruct.doHaltAction_ = false; //clear
214 
215  iterator->haltIterator(iterator,&theIteratorStruct);
216 
217 // //last ditch effort to make sure FSM is halted
218 // iterator->haltIterator(
219 // iterator->theSupervisor_,
220 // theIteratorStruct.fsmName_);
221 //
222 // //lockout the messages array for the remainder of the scope
223 // //this guarantees the reading thread can safely access the messages
224 // if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for iterator access" << __E__;
225 // std::lock_guard<std::mutex> lock(iterator->accessMutex_);
226 // if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have iterator access" << __E__;
227 //
228 // iterator->activePlanIsRunning_ = false;
229 // iterator->iteratorBusy_ = false;
230 //
231 // __COUT__ << "Halted plan '" << theIteratorStruct.activePlan_ << "' at command index " <<
232 // theIteratorStruct.commandIndex_ << ". " << __E__;
233 // __MOUT__ << "Halted plan '" << theIteratorStruct.activePlan_ << "' at command index " <<
234 // theIteratorStruct.commandIndex_ << ". " << __E__;
235 //
236 // theIteratorStruct.activePlan_ = ""; //clear
237 // theIteratorStruct.commandIndex_ = -1; //clear
238 
239  continue; //resume workloop
240  }
241 
242 
245  // handle running
246  // __COUT__ << "thinking.." << theIteratorStruct.running_ << " " <<
247  // theIteratorStruct.activePlan_ << " cmd=" <<
248  // theIteratorStruct.commandIndex_ << __E__;
249  if(theIteratorStruct.running_ &&
250  theIteratorStruct.activePlan_ != "") //important, because after errors, still "running" until halt
251  {
252  if(theIteratorStruct.commandIndex_ == (unsigned int)-1)
253  {
254  //initialize the running plan
255 
256  __COUT__ << "Get commands" << __E__;
257 
258  theIteratorStruct.commandIndex_ = 0;
259 
260  theIteratorStruct.cfgMgr_->init(); //completely reset to re-align with any changes
261  itConfig = theIteratorStruct.cfgMgr_->__GET_CONFIG__(IterateConfiguration);
262 
263  theIteratorStruct.commands_ = itConfig->getPlanCommands(
264  theIteratorStruct.cfgMgr_,
265  theIteratorStruct.activePlan_);
266 
267  //reset commandIteration counts
268  theIteratorStruct.commandIterations_.clear();
269  for(auto& command:theIteratorStruct.commands_)
270  {
271  theIteratorStruct.commandIterations_.push_back(0);
272  __COUT__ << "command " <<
273  command.type_ << __E__;
274  __COUT__ << "table " <<
275  IterateConfiguration::commandToTableMap_.at(command.type_) << __E__;
276  __COUT__ << "param count = " << command.params_.size() << __E__;
277 
278  for(auto& param:command.params_)
279  {
280  __COUT__ << "\t param " <<
281  param.first << " : " <<
282  param.second << __E__;
283  }
284  }
285 
286 
287  theIteratorStruct.originalTrackChanges_ =
288  ConfigurationInterface::isVersionTrackingEnabled();
289  theIteratorStruct.originalConfigGroup_ =
290  theIteratorStruct.cfgMgr_->getActiveGroupName();
291  theIteratorStruct.originalConfigKey_ =
292  theIteratorStruct.cfgMgr_->getActiveGroupKey();
293 
294  __COUT__ << "originalTrackChanges " <<
295  theIteratorStruct.originalTrackChanges_ << __E__;
296  __COUT__ << "originalConfigGroup " <<
297  theIteratorStruct.originalConfigGroup_ << __E__;
298  __COUT__ << "originalConfigKey " <<
299  theIteratorStruct.originalConfigKey_ << __E__;
300 
301  } //end initial section
302 
303 
304  if(!theIteratorStruct.commandBusy_)
305  {
306  if(theIteratorStruct.commandIndex_ < theIteratorStruct.commands_.size())
307  {
308  //execute command
309  theIteratorStruct.commandBusy_ = true;
310 
311  __COUT__ << "Iterator starting command " << theIteratorStruct.commandIndex_+1 << ": " <<
312  theIteratorStruct.commands_[theIteratorStruct.commandIndex_].type_ << __E__;
313  __MOUT__ << "Iterator starting command " << theIteratorStruct.commandIndex_+1 << ": " <<
314  theIteratorStruct.commands_[theIteratorStruct.commandIndex_].type_ << __E__;
315 
316  iterator->startCommand(&theIteratorStruct);
317  }
318  else if(theIteratorStruct.commandIndex_ ==
319  theIteratorStruct.commands_.size()) //Done!
320  {
321  __COUT__ << "Finished Iteration Plan '" << theIteratorStruct.activePlan_ << __E__;
322  __MOUT__ << "Finished Iteration Plan '" << theIteratorStruct.activePlan_ << __E__;
323 
324  __COUT__ << "Reverting track changes." << __E__;
325  ConfigurationInterface::setVersionTrackingEnabled(theIteratorStruct.originalTrackChanges_);
326 
327  __COUT__ << "Activating original group..." << __E__;
328  try
329  {
330  theIteratorStruct.cfgMgr_->activateConfigurationGroup(
331  theIteratorStruct.originalConfigGroup_,theIteratorStruct.originalConfigKey_);
332  }
333  catch(...)
334  {
335  __COUT_WARN__ << "Original group could not be activated." << __E__;
336  }
337 
338  //leave FSM halted
339  __COUT__ << "Completing halt..." << __E__;
340 
341  iterator->haltIterator(iterator,&theIteratorStruct);
342 
343 //
344 // //leave FSM halted
345 // iterator->haltIterator(
346 // iterator->theSupervisor_,
347 // theIteratorStruct.fsmName_);
348 //
349 // //lockout the messages array for the remainder of the scope
350 // //this guarantees the reading thread can safely access the messages
351 // if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for iterator access" << __E__;
352 // std::lock_guard<std::mutex> lock(iterator->accessMutex_);
353 // if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have iterator access" << __E__;
354 //
355 // //similar to halt
356 // iterator->activePlanIsRunning_ = false;
357 // iterator->iteratorBusy_ = false;
358 //
359 // iterator->lastStartedPlanName_ = theIteratorStruct.activePlan_;
360 // theIteratorStruct.activePlan_ = ""; //clear
361 // theIteratorStruct.commandIndex_ = -1; //clear
362  }
363  }
364  else if(theIteratorStruct.commandBusy_)
365  {
366  //check for command completion
367  if(iterator->checkCommand(&theIteratorStruct))
368  {
369  theIteratorStruct.commandBusy_ = false; //command complete
370 
371  ++theIteratorStruct.commandIndex_;
372 
373  __COUT__ << "Ready for next command. Done with " << theIteratorStruct.commandIndex_ << " of " <<
374  theIteratorStruct.commands_.size() << __E__;
375  __MOUT__ << "Iterator ready for next command. Done with " << theIteratorStruct.commandIndex_ << " of " <<
376  theIteratorStruct.commands_.size() << __E__;
377  }
378 
379  //Note: check command gets one shot to resume
380  if(theIteratorStruct.doResumeAction_) //end resume action
381  theIteratorStruct.doResumeAction_ = false;
382  }
383 
384 
385 
386  } //end running
387  else
388  sleep(1); //when inactive sleep a lot
389 
392 
393  // __COUT__ << "end loop.." << theIteratorStruct.running_ << " " <<
394  // theIteratorStruct.activePlan_ << " cmd=" <<
395  // theIteratorStruct.commandIndex_ << __E__;
396 
397  }
398 
399  iterator->workloopRunning_ = false; //if we ever exit
400 }
401 catch(const std::runtime_error &e)
402 {
403  __SS__ << "Encountered error in Iterator thread:\n" << e.what() << __E__;
404  __COUT_ERR__ << ss.str();
405 
406  //lockout the messages array for the remainder of the scope
407  //this guarantees the reading thread can safely access the messages
408  if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for iterator access" << __E__;
409  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
410  if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have iterator access" << __E__;
411 
412  iterator->workloopRunning_ = false; //if we ever exit
413  iterator->errorMessage_ = ss.str();
414 
415 }
416 catch(...)
417 {
418  __SS__ << "Encountered unknown error in Iterator thread." << __E__;
419  __COUT_ERR__ << ss.str();
420 
421  //lockout the messages array for the remainder of the scope
422  //this guarantees the reading thread can safely access the messages
423  if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for iterator access" << __E__;
424  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
425  if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have iterator access" << __E__;
426 
427  iterator->workloopRunning_ = false; //if we ever exit
428  iterator->errorMessage_ = ss.str();
429 }
430 
431 //========================================================================================================================
432 void Iterator::startCommand(IteratorWorkLoopStruct *iteratorStruct)
433 try
434 {
435 
436  {
437  int i =0;
438  for(const auto& depthIteration:iteratorStruct->stepIndexStack_)
439  {
440  __COUT__ << i++ << ":" << depthIteration << __E__;
441 
442  }
443  }
444 
445 
446  //should be mutually exclusive with GatewaySupervisor main thread state machine accesses
447  //lockout the messages array for the remainder of the scope
448  //this guarantees the reading thread can safely access the messages
449  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for FSM access" << __E__;
450  std::lock_guard<std::mutex> lock(iteratorStruct->theIterator_->theSupervisor_->stateMachineAccessMutex_);
451  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have FSM access" << __E__;
452 
453 
454  //for out of range, throw exception - should never happen
455  if(iteratorStruct->commandIndex_ >= iteratorStruct->commands_.size())
456  {
457  __SS__ << "Out of range commandIndex = " << iteratorStruct->commandIndex_ <<
458  " in size = " << iteratorStruct->commands_.size() << __E__;
459  throw std::runtime_error(ss.str());
460  }
461 
462  //increment iteration count for command
463  ++iteratorStruct->commandIterations_[iteratorStruct->commandIndex_];
464 
465 
466  std::string type = iteratorStruct->commands_[iteratorStruct->commandIndex_].type_;
467  if(type == IterateConfiguration::COMMAND_BEGIN_LABEL)
468  {
469  return startCommandBeginLabel(iteratorStruct);
470  }
471  else if(type == IterateConfiguration::COMMAND_CHOOSE_FSM)
472  {
473  return startCommandChooseFSM(
474  iteratorStruct,
475  iteratorStruct->commands_
476  [iteratorStruct->commandIndex_].params_
477  [IterateConfiguration::commandChooseFSMParams_.NameOfFSM_]);
478  }
479  else if(type == IterateConfiguration::COMMAND_CONFIGURE_ACTIVE_GROUP)
480  {
481  return startCommandConfigureActive(iteratorStruct);
482  }
483  else if(type == IterateConfiguration::COMMAND_CONFIGURE_ALIAS)
484  {
485  return startCommandConfigureAlias(
486  iteratorStruct,
487  iteratorStruct->commands_
488  [iteratorStruct->commandIndex_].params_
489  [IterateConfiguration::commandConfigureAliasParams_.SystemAlias_]);
490  }
491  else if(type == IterateConfiguration::COMMAND_CONFIGURE_GROUP)
492  {
493  return startCommandConfigureGroup(iteratorStruct);
494  }
495  else if(type == IterateConfiguration::COMMAND_EXECUTE_FE_MACRO)
496  {
497  //TODO
498  return;
499  }
500  else if(type == IterateConfiguration::COMMAND_EXECUTE_MACRO)
501  {
502  //TODO
503  return;
504  }
505  else if(type == IterateConfiguration::COMMAND_MODIFY_ACTIVE_GROUP)
506  {
507  return startCommandModifyActive(iteratorStruct);
508  }
509  else if(type == IterateConfiguration::COMMAND_REPEAT_LABEL)
510  {
511  return startCommandRepeatLabel(iteratorStruct);
512  }
513  else if(type == IterateConfiguration::COMMAND_RUN)
514  {
515  return startCommandRun(iteratorStruct);
516  }
517  else
518  {
519  __SS__ << "Attempt to start unrecognized command type = " << type << __E__;
520  __COUT_ERR__ << ss.str();
521  throw std::runtime_error(ss.str());
522  }
523 }
524 catch(...)
525 {
526  __COUT__ << "Error caught. Reverting track changes." << __E__;
527  ConfigurationInterface::setVersionTrackingEnabled(iteratorStruct->originalTrackChanges_);
528 
529  __COUT__ << "Activating original group..." << __E__;
530  try
531  {
532  iteratorStruct->cfgMgr_->activateConfigurationGroup(
533  iteratorStruct->originalConfigGroup_,iteratorStruct->originalConfigKey_);
534  }
535  catch(...)
536  {
537  __COUT_WARN__ << "Original group could not be activated." << __E__;
538  }
539  throw;
540 }
541 
542 
543 //========================================================================================================================
544 //checkCommand
545 // when busy for a while, start to sleep
546 // use sleep() or nanosleep()
547 bool Iterator::checkCommand(IteratorWorkLoopStruct *iteratorStruct)
548 try
549 {
550  //for out of range, return done
551  if(iteratorStruct->commandIndex_ >= iteratorStruct->commands_.size())
552  {
553  __COUT__ << "Out of range commandIndex = " << iteratorStruct->commandIndex_ <<
554  " in size = " << iteratorStruct->commands_.size() << __E__;
555  return true;
556  }
557 
558  std::string type = iteratorStruct->commands_[iteratorStruct->commandIndex_].type_;
559  if(type == IterateConfiguration::COMMAND_BEGIN_LABEL)
560  {
561  //do nothing
562  return true;
563  }
564  else if(type == IterateConfiguration::COMMAND_CHOOSE_FSM)
565  {
566  //do nothing
567  return true;
568  }
569  else if(type == IterateConfiguration::COMMAND_CONFIGURE_ALIAS ||
570  type == IterateConfiguration::COMMAND_CONFIGURE_ACTIVE_GROUP ||
571  type == IterateConfiguration::COMMAND_CONFIGURE_GROUP)
572  {
573  return checkCommandConfigure(iteratorStruct);
574  }
575  else if(type == IterateConfiguration::COMMAND_EXECUTE_FE_MACRO)
576  {
577  //do nothing
578  return true;
579  }
580  else if(type == IterateConfiguration::COMMAND_EXECUTE_MACRO)
581  {
582  //do nothing
583  return true;
584  }
585  else if(type == IterateConfiguration::COMMAND_MODIFY_ACTIVE_GROUP)
586  {
587  //do nothing
588  return true;
589  }
590  else if(type == IterateConfiguration::COMMAND_REPEAT_LABEL)
591  {
592  //do nothing
593  return true;
594  }
595  else if(type == IterateConfiguration::COMMAND_RUN)
596  {
597  return checkCommandRun(iteratorStruct);
598  }
599  else
600  {
601  __SS__ << "Attempt to check unrecognized command type = " << type << __E__;
602  __COUT_ERR__ << ss.str();
603  throw std::runtime_error(ss.str());
604  }
605 }
606 catch(...)
607 {
608  __COUT__ << "Error caught. Reverting track changes." << __E__;
609  ConfigurationInterface::setVersionTrackingEnabled(iteratorStruct->originalTrackChanges_);
610 
611  __COUT__ << "Activating original group..." << __E__;
612  try
613  {
614  iteratorStruct->cfgMgr_->activateConfigurationGroup(
615  iteratorStruct->originalConfigGroup_,iteratorStruct->originalConfigKey_);
616  }
617  catch(...)
618  {
619  __COUT_WARN__ << "Original group could not be activated." << __E__;
620  }
621 
622  throw;
623 }
624 
625 //========================================================================================================================
626 void Iterator::startCommandChooseFSM(IteratorWorkLoopStruct *iteratorStruct,
627  const std::string& fsmName)
628 {
629  __COUT__ << "fsmName " << fsmName << __E__;
630 
631 
632  iteratorStruct->fsmName_ = fsmName;
633  iteratorStruct->theIterator_->lastFsmName_ = fsmName;
634 
635  //Translate fsmName
636  // to gives us run alias (fsmRunAlias_) and next run number (fsmNextRunNumber_)
637 
638 
639  //CAREFUL?? Threads
640 
641 
643 
644  iteratorStruct->fsmRunAlias_ = "Run"; //default to "Run"
645 
646  // get stateMachineAliasFilter if possible
647  ConfigurationTree configLinkNode = iteratorStruct->cfgMgr_->getSupervisorConfigurationNode(
648  iteratorStruct->theIterator_->theSupervisor_->supervisorContextUID_,
649  iteratorStruct->theIterator_->theSupervisor_->supervisorApplicationUID_);
650 
651  if(!configLinkNode.isDisconnected())
652  {
653  try //for backwards compatibility
654  {
655  ConfigurationTree fsmLinkNode = configLinkNode.getNode("LinkToStateMachineConfiguration");
656  if(!fsmLinkNode.isDisconnected())
657  iteratorStruct->fsmRunAlias_ =
658  fsmLinkNode.getNode(fsmName + "/RunDisplayAlias").getValue<std::string>();
659  else
660  __COUT_INFO__ << "FSM Link disconnected." << __E__;
661  }
662  catch(std::runtime_error &e)
663  {
664  //__COUT_INFO__ << e.what() << __E__;
665  __COUT_INFO__ << "No state machine Run alias. Ignoring and assuming alias of '" <<
666  iteratorStruct->fsmRunAlias_ << ".'" << __E__;
667 
668  }
669  catch(...) {
670  __COUT_ERR__ << "Unknown error. Should never happen." << __E__;
671 
672  __COUT_INFO__ << "No state machine Run alias. Ignoring and assuming alias of '" <<
673  iteratorStruct->fsmRunAlias_ << ".'" << __E__;
674  }
675  }
676  else
677  __COUT_INFO__ << "FSM Link disconnected." << __E__;
678 
679  __COUT__ << "fsmRunAlias_ = " << iteratorStruct->fsmRunAlias_ << __E__;
680 
681 
682 
683 
685 
686  iteratorStruct->fsmNextRunNumber_ = iteratorStruct->theIterator_->theSupervisor_->getNextRunNumber(
687  iteratorStruct->fsmName_);
688 
689  if(iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getCurrentStateName() == "Running" ||
690  iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getCurrentStateName() == "Paused")
691  --iteratorStruct->fsmNextRunNumber_; //current run number is one back
692 
693  __COUT__ << "fsmNextRunNumber_ = " << iteratorStruct->fsmNextRunNumber_ << __E__;
694 }
695 
696 //========================================================================================================================
697 // return true if an action was attempted
698 bool Iterator::haltIterator(Iterator *iterator,
699  IteratorWorkLoopStruct *iteratorStruct)
700 //(GatewaySupervisor* theSupervisor, const std::string& fsmName)
701 {
702  GatewaySupervisor* theSupervisor = iterator->theSupervisor_;
703  const std::string& fsmName = iterator->lastFsmName_;
704 
705  std::vector<std::string> fsmCommandParameters;
706  std::string errorStr = "";
707  std::string currentState = theSupervisor->theStateMachine_.getCurrentStateName();
708 
709  bool haltAttempted = true;
710  if(currentState == "Initialized" ||
711  currentState == "Halted")
712  {
713  __COUT__ << "Do nothing. Already halted." << __E__;
714  haltAttempted = false;
715  }
716  else if(currentState == "Running")
717  errorStr = theSupervisor->attemptStateMachineTransition(
718  0,0,
719  "Abort",fsmName,
720  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
721  WebUsers::DEFAULT_ITERATOR_USERNAME,
722  fsmCommandParameters);
723  else
724  errorStr = theSupervisor->attemptStateMachineTransition(
725  0,0,
726  "Halt",fsmName,
727  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
728  WebUsers::DEFAULT_ITERATOR_USERNAME,
729  fsmCommandParameters);
730 
731  if(haltAttempted)
732  {
733  if(errorStr != "")
734 
735  {
736  __SS__ << "Iterator failed to halt because of the following error: " << errorStr;
737  throw std::runtime_error(ss.str());
738  }
739 
740  //else successfully launched
741  __COUT__ << "FSM in transition = " << theSupervisor->theStateMachine_.isInTransition() << __E__;
742  __COUT__ << "halting state machine launched." << __E__;
743  }
744 
745 
746 
747  //finish up halting the iterator
748  __COUT__ << "Conducting Iterator halt." << __E__;
749 
750 
751  if(iteratorStruct)
752  {
753  __COUT__ << "Reverting track changes." << __E__;
754  ConfigurationInterface::setVersionTrackingEnabled(iteratorStruct->originalTrackChanges_);
755 
756  __COUT__ << "Activating original group..." << __E__;
757  try
758  {
759  iteratorStruct->cfgMgr_->activateConfigurationGroup(
760  iteratorStruct->originalConfigGroup_,iteratorStruct->originalConfigKey_);
761  }
762  catch(...)
763  {
764  __COUT_WARN__ << "Original group could not be activated." << __E__;
765  }
766 
767  }
768 
769  //lockout the messages array for the remainder of the scope
770  //this guarantees the reading thread can safely access the messages
771  if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for iterator access" << __E__;
772  std::lock_guard<std::mutex> lock(iterator->accessMutex_);
773  if(iterator->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have iterator access" << __E__;
774 
775 
776 
777  iterator->activePlanIsRunning_ = false;
778  iterator->iteratorBusy_ = false;
779 
780  //clear
781  iterator->activePlanName_ = "";
782  iterator->activeCommandIndex_ = -1;
783 
784  if(iteratorStruct)
785  {
786 
787  __COUT__ << "Halted plan '" << iteratorStruct->activePlan_ << "' at command index " <<
788  iteratorStruct->commandIndex_ << ". " << __E__;
789  __MOUT__ << "Halted plan '" << iteratorStruct->activePlan_ << "' at command index " <<
790  iteratorStruct->commandIndex_ << ". " << __E__;
791 
792  iterator->lastStartedPlanName_ = iteratorStruct->activePlan_;
793  iteratorStruct->activePlan_ = ""; //clear
794  iteratorStruct->commandIndex_ = -1; //clear
795  }
796 
797  return haltAttempted;
798 }
799 
800 //========================================================================================================================
801 void Iterator::startCommandBeginLabel(IteratorWorkLoopStruct *iteratorStruct)
802 {
803  __COUT__ << "Entering label '" <<
804  iteratorStruct->commands_
805  [iteratorStruct->commandIndex_].params_
806  [IterateConfiguration::commandBeginLabelParams_.Label_]
807  << "'..." << std::endl;
808 
809  //add new step index to stack
810  iteratorStruct->stepIndexStack_.push_back(0);
811 }
812 
813 
814 //========================================================================================================================
815 void Iterator::startCommandRepeatLabel(IteratorWorkLoopStruct *iteratorStruct)
816 {
817  //search for first matching label backward and set command to there
818 
819  int numOfRepetitions;
820  sscanf(iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
821  [IterateConfiguration::commandRepeatLabelParams_.NumberOfRepetitions_].c_str(),
822  "%d",&numOfRepetitions);
823  __COUT__ << "numOfRepetitions remaining = " << numOfRepetitions << __E__;
824 
825  char repStr[200];
826 
827  if(numOfRepetitions <= 0)
828  {
829  //write original number of repetitions value back
830  sprintf(repStr,"%d",iteratorStruct->stepIndexStack_.back());
831  iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
832  [IterateConfiguration::commandRepeatLabelParams_.NumberOfRepetitions_] =
833  repStr; //re-store as string
834 
835  //remove step index from stack
836  iteratorStruct->stepIndexStack_.pop_back();
837 
838 
839  return; //no more repetitions
840  }
841 
842  --numOfRepetitions;
843 
844  //increment step index in stack
845  ++(iteratorStruct->stepIndexStack_.back());
846 
847  unsigned int i;
848  for(i=iteratorStruct->commandIndex_;i>0;--i) //assume 0 is always the fallback option
849  if(iteratorStruct->commands_[i].type_ == IterateConfiguration::COMMAND_BEGIN_LABEL &&
850  iteratorStruct->commands_[iteratorStruct->commandIndex_].params_[IterateConfiguration::commandRepeatLabelParams_.Label_] ==
851  iteratorStruct->commands_[i].params_[IterateConfiguration::commandBeginLabelParams_.Label_]) break;
852 
853  sprintf(repStr,"%d",numOfRepetitions);
854  iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
855  [IterateConfiguration::commandRepeatLabelParams_.NumberOfRepetitions_] =
856  repStr; //re-store as string
857 
858  iteratorStruct->commandIndex_ = i;
859  __COUT__ << "Jumping back to commandIndex " << iteratorStruct->commandIndex_ << __E__;
860 }
861 
862 //========================================================================================================================
863 void Iterator::startCommandRun(IteratorWorkLoopStruct *iteratorStruct)
864 {
865  iteratorStruct->runIsDone_ = false;
866  iteratorStruct->fsmCommandParameters_.clear();
867 
868  std::string errorStr = "";
869  std::string currentState = iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getCurrentStateName();
870 
871  //execute first transition (may need two)
872 
873  if(currentState == "Configured")
874  errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
875  0,0,
876  "Start",iteratorStruct->fsmName_,
877  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
878  WebUsers::DEFAULT_ITERATOR_USERNAME,
879  iteratorStruct->fsmCommandParameters_);
880  else
881  errorStr = "Can only Run from the Configured state. The current state is " +
882  currentState;
883 
884 
885  if(errorStr != "")
886  {
887  __SS__ << "Iterator failed to run because of the following error: " << errorStr;
888  throw std::runtime_error(ss.str());
889  }
890 
891  //else successfully launched
892  __COUT__ << "FSM in transition = " << iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.isInTransition() << __E__;
893  __COUT__ << "startCommandRun success." << __E__;
894 }
895 
896 //========================================================================================================================
897 void Iterator::startCommandConfigureActive(IteratorWorkLoopStruct *iteratorStruct)
898 {
899  __COUT__ << "startCommandConfigureActive " << __E__;
900 
901  //steps:
902  // get active config group
903  // transition to configure with parameters describing group
904 
905  std::string group = iteratorStruct->cfgMgr_->getActiveGroupName();
906  ConfigurationGroupKey key = iteratorStruct->cfgMgr_->getActiveGroupKey();
907 
908  __COUT__ << "group " << group << __E__;
909  __COUT__ << "key " << key << __E__;
910 
911  //create special alias for this group using : separators
912 
913  std::stringstream systemAlias;
914  systemAlias << "GROUP:" << group << ":" << key;
915  startCommandConfigureAlias(iteratorStruct,systemAlias.str());
916 }
917 
918 //========================================================================================================================
919 void Iterator::startCommandConfigureGroup(IteratorWorkLoopStruct *iteratorStruct)
920 {
921  __COUT__ << "startCommandConfigureGroup " << __E__;
922 
923  //steps:
924  // transition to configure with parameters describing group
925 
926  std::string group =
927  iteratorStruct->commands_
928  [iteratorStruct->commandIndex_].params_
929  [IterateConfiguration::commandConfigureGroupParams_.GroupName_];
931  iteratorStruct->commands_
932  [iteratorStruct->commandIndex_].params_
933  [IterateConfiguration::commandConfigureGroupParams_.GroupKey_])
934  ;
935 
936  __COUT__ << "group " << group << __E__;
937  __COUT__ << "key " << key << __E__;
938 
939  //create special alias for this group using : separators
940 
941  std::stringstream systemAlias;
942  systemAlias << "GROUP:" << group << ":" << key;
943  startCommandConfigureAlias(iteratorStruct,systemAlias.str());
944 }
945 
946 //========================================================================================================================
947 void Iterator::startCommandConfigureAlias(IteratorWorkLoopStruct *iteratorStruct,
948  const std::string& systemAlias)
949 {
950  __COUT__ << "systemAlias " << systemAlias << __E__;
951 
952  iteratorStruct->fsmCommandParameters_.clear();
953  iteratorStruct->fsmCommandParameters_.push_back(systemAlias);
954 
955  std::string errorStr = "";
956  std::string currentState = iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getCurrentStateName();
957 
958  //execute first transition (may need two in conjunction with checkCommandConfigure())
959 
960  if(currentState == "Initial")
961  errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
962  0,0,
963  "Initialize",iteratorStruct->fsmName_,
964  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
965  WebUsers::DEFAULT_ITERATOR_USERNAME,
966  iteratorStruct->fsmCommandParameters_);
967  else if(currentState == "Halted")
968  errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
969  0,0,
970  "Configure",iteratorStruct->fsmName_,
971  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
972  WebUsers::DEFAULT_ITERATOR_USERNAME,
973  iteratorStruct->fsmCommandParameters_);
974  else if(currentState == "Configured")
975  errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
976  0,0,
977  "Halt",iteratorStruct->fsmName_,
978  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
979  WebUsers::DEFAULT_ITERATOR_USERNAME,
980  iteratorStruct->fsmCommandParameters_);
981  else
982  errorStr = "Can only Configure from the Initial or Halted state. The current state is " +
983  currentState;
984 
985 
986 
987  if(errorStr != "")
988  {
989  __SS__ << "Iterator failed to configure with system alias '" <<
990  (iteratorStruct->fsmCommandParameters_.size()?
991  iteratorStruct->fsmCommandParameters_[0]:"UNKNOWN") <<
992  "' because of the following error: " << errorStr;
993  throw std::runtime_error(ss.str());
994  }
995 
996  //else successfully launched
997  __COUT__ << "FSM in transition = " << iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.isInTransition() << __E__;
998  __COUT__ << "startCommandConfigureAlias success." << __E__;
999 }
1000 
1001 //========================================================================================================================
1002 void Iterator::startCommandModifyActive(IteratorWorkLoopStruct *iteratorStruct)
1003 {
1004  //Steps:
1005  // 4 parameters commandModifyActiveParams_:
1006  // const std::string DoTrackGroupChanges_ TrueFalse
1007  // //targets
1008  // const std::string RelativePathToField_ = "RelativePathToField";
1009  // const std::string FieldStartValue_ = "FieldStartValue";
1010  // const std::string FieldIterationStepSize_ = "FieldIterationStepSize";
1011  //
1012  // if tracking changes,
1013  // create a new group
1014  // for every enabled FE
1015  // set field = start value + stepSize * currentStepIndex_
1016  // activate group
1017  // else
1018  // load scratch group
1019  // for every enabled FE
1020  // set field = start value + stepSize * stepIndex
1021  // activate group
1022 
1023  bool doTrackGroupChanges = false;
1024  if("True" == iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
1025  [IterateConfiguration::commandModifyActiveParams_.DoTrackGroupChanges_])
1026  doTrackGroupChanges = true;
1027 
1028  const std::string& startValueStr =
1029  iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
1030  [IterateConfiguration::commandModifyActiveParams_.FieldStartValue_];
1031  const std::string& stepSizeStr =
1032  iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
1033  [IterateConfiguration::commandModifyActiveParams_.FieldIterationStepSize_];
1034 
1035  const unsigned int stepIndex = iteratorStruct->stepIndexStack_.back();
1036 
1037 
1038  __COUT__ << "doTrackGroupChanges " << (doTrackGroupChanges?"yes":"no") << std::endl;
1039  __COUT__ << "stepIndex " << stepIndex << std::endl;
1040 
1041  ConfigurationInterface::setVersionTrackingEnabled(doTrackGroupChanges);
1042 
1043  //two approaches: double or long handling
1044 
1045  if(startValueStr.size() &&
1046  (startValueStr[startValueStr.size()-1] == 'f' ||
1047  startValueStr.find('.') != std::string::npos))
1048  {
1049  //handle as double
1050  double startValue = strtod(startValueStr.c_str(),0);
1051  double stepSize = strtod(stepSizeStr.c_str(),0);
1052 
1053  __COUT__ << "startValue " << startValue << std::endl;
1054  __COUT__ << "stepSize " << stepSize << std::endl;
1055  __COUT__ << "currentValue " << startValue + stepSize*stepIndex << std::endl;
1056 
1057  helpCommandModifyActive(
1058  iteratorStruct,
1059  startValue + stepSize*stepIndex,
1060  doTrackGroupChanges);
1061  }
1062  else //handle as long
1063  {
1064  long int startValue;
1065 
1066  if(startValueStr.size() > 2 && startValueStr[1] == 'x') //assume hex value
1067  startValue = strtol(startValueStr.c_str(),0,16);
1068  else if(startValueStr.size() > 1 && startValueStr[0] == 'b') //assume binary value
1069  startValue = strtol(startValueStr.substr(1).c_str(),0,2); //skip first 'b' character
1070  else
1071  startValue = strtol(startValueStr.c_str(),0,10);
1072 
1073  long int stepSize;
1074 
1075  if(stepSizeStr.size() > 2 && stepSizeStr[1] == 'x') //assume hex value
1076  stepSize = strtol(stepSizeStr.c_str(),0,16);
1077  else if(stepSizeStr.size() > 1 && stepSizeStr[0] == 'b') //assume binary value
1078  stepSize = strtol(stepSizeStr.substr(1).c_str(),0,2); //skip first 'b' character
1079  else
1080  stepSize = strtol(stepSizeStr.c_str(),0,10);
1081 
1082  __COUT__ << "startValue " << startValue << std::endl;
1083  __COUT__ << "stepSize " << stepSize << std::endl;
1084  __COUT__ << "currentValue " << startValue + stepSize*stepIndex << std::endl;
1085 
1086  helpCommandModifyActive(
1087  iteratorStruct,
1088  startValue + stepSize*stepIndex,
1089  doTrackGroupChanges);
1090  }
1091 
1092 }
1093 
1094 
1095 //========================================================================================================================
1096 //checkCommandRun
1097 // return true if done
1098 //
1099 // Either will be done on (priority 1) running threads (for Frontends) ending
1100 // or (priority 2 and ignored if <= 0) duration timeout
1101 //
1102 // Note: use command structure strings to maintain duration left
1103 // Note: watch iterator->doPauseAction and iterator->doHaltAction and respond
1104 bool Iterator::checkCommandRun(IteratorWorkLoopStruct *iteratorStruct)
1105 {
1106  sleep(1); //sleep to give FSM time to transition
1107 
1108  //all RunControlStateMachine access commands should be mutually exclusive with GatewaySupervisor main thread state machine accesses
1109  //should be mutually exclusive with GatewaySupervisor main thread state machine accesses
1110  //lockout the messages array for the remainder of the scope
1111  //this guarantees the reading thread can safely access the messages
1112  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for FSM access" << __E__;
1113  std::lock_guard<std::mutex> lock(iteratorStruct->theIterator_->theSupervisor_->stateMachineAccessMutex_);
1114  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have FSM access" << __E__;
1115 
1116  if(iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.isInTransition())
1117  return false;
1118 
1119  iteratorStruct->fsmCommandParameters_.clear();
1120 
1121  std::string errorStr = "";
1122  std::string currentState = iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getCurrentStateName();
1123 
1124 
1126  //check for imposed actions and forced exits
1127  if(iteratorStruct->doPauseAction_)
1128  {
1129  //transition to pause state
1130  __COUT__ << "Transitioning FSM to Paused..." << __E__;
1131 
1132  if(currentState == "Paused")
1133  {
1134  //done with early pause exit!
1135  __COUT__ << "Transition to Paused complete." << __E__;
1136  return true;
1137  }
1138  else if(currentState == "Running") //launch transition to pause
1139  errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
1140  0,0,
1141  "Pause",iteratorStruct->fsmName_,
1142  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1143  WebUsers::DEFAULT_ITERATOR_USERNAME,
1144  iteratorStruct->fsmCommandParameters_);
1145  else if(currentState == "Configured")
1146  {
1147  //no need to pause state machine, no run going on
1148  __COUT__ << "In Configured state. No need to transition to Paused." << __E__;
1149  return true;
1150  }
1151  else
1152  errorStr = "Expected to be in Paused. Unexpectedly, the current state is " +
1153  currentState + ". Last State Machine error message was as follows: " +
1154  iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getErrorMessage();
1155 
1156  if(errorStr != "")
1157  {
1158  __SS__ << "Iterator failed to pause because of the following error: " << errorStr;
1159  throw std::runtime_error(ss.str());
1160  }
1161  return false;
1162  }
1163  else if(iteratorStruct->doHaltAction_)
1164  {
1165  //transition to halted state
1166  __COUT__ << "Transitioning FSM to Halted..." << __E__;
1167 
1168  if(currentState == "Halted")
1169  {
1170  //done with early halt exit!
1171  __COUT__ << "Transition to Halted complete." << __E__;
1172  return true;
1173  }
1174  else if(currentState == "Running" || //launch transition to halt
1175  currentState == "Paused")
1176  errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
1177  0,0,
1178  "Abort",iteratorStruct->fsmName_,
1179  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1180  WebUsers::DEFAULT_ITERATOR_USERNAME,
1181  iteratorStruct->fsmCommandParameters_);
1182  else if(currentState == "Configured") //launch transition to halt
1183  errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
1184  0,0,
1185  "Halt",iteratorStruct->fsmName_,
1186  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1187  WebUsers::DEFAULT_ITERATOR_USERNAME,
1188  iteratorStruct->fsmCommandParameters_);
1189  else
1190  errorStr = "Expected to be in Halted. Unexpectedly, the current state is " +
1191  currentState + ". Last State Machine error message was as follows: " +
1192  iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getErrorMessage();
1193 
1194  if(errorStr != "")
1195  {
1196  __SS__ << "Iterator failed to halt because of the following error: " << errorStr;
1197  throw std::runtime_error(ss.str());
1198  }
1199  return false;
1200  }
1201  else if(iteratorStruct->doResumeAction_)
1202  {
1203  //Note: check command gets one shot to resume
1204 
1205  //transition to running state
1206  __COUT__ << "Transitioning FSM to Running..." << __E__;
1207 
1208  if(currentState == "Paused") //launch transition to running
1209  errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
1210  0,0,
1211  "Resume",iteratorStruct->fsmName_,
1212  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1213  WebUsers::DEFAULT_ITERATOR_USERNAME,
1214  iteratorStruct->fsmCommandParameters_);
1215 
1216  if(errorStr != "")
1217  {
1218  __SS__ << "Iterator failed to run because of the following error: " << errorStr;
1219  throw std::runtime_error(ss.str());
1220  }
1221  return false;
1222  }
1223 
1224 
1225 
1227  //normal running
1228 
1229  if(currentState != "Running")
1230  {
1231  if(iteratorStruct->runIsDone_ &&
1232  currentState == "Configured")
1233  {
1234  //indication of done
1235  __COUT__ << "Reached end of run " <<
1236  iteratorStruct->fsmNextRunNumber_ << __E__;
1237  return true;
1238  }
1239 
1240  errorStr = "Expected to be in Running. Unexpectedly, the current state is " +
1241  currentState + ". Last State Machine error message was as follows: " +
1242  iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getErrorMessage();
1243  }
1244  else //else in Running state! Check for end of run
1245  {
1246  bool waitOnRunningThreads = false;
1247  if("True" == iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
1248  [IterateConfiguration::commandRunParams_.WaitOnRunningThreads_])
1249  waitOnRunningThreads = true;
1250 
1251  time_t remainingDurationInSeconds; //parameter converted during start to the stop linux timestamp
1252  sscanf(iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
1253  [IterateConfiguration::commandRunParams_.DurationInSeconds_].c_str(),
1254  "%ld",&remainingDurationInSeconds);
1255 
1256 
1257  __COUT__ << "waitOnRunningThreads " << waitOnRunningThreads << __E__;
1258  __COUT__ << "remainingDurationInSeconds " << remainingDurationInSeconds << __E__;
1259 
1261  //priority 1 is waiting on running threads
1262  if(waitOnRunningThreads)
1263  {
1264  // get status of all running FE workloops
1265  GatewaySupervisor* theSupervisor =
1266  iteratorStruct->theIterator_->theSupervisor_;
1267 
1268  bool allFrontEndsAreDone = true;
1269  for (auto& it :
1270  theSupervisor->allSupervisorInfo_.getAllFETypeSupervisorInfo())
1271  {
1272  try
1273  {
1274  std::string status = theSupervisor->send(it.second.getDescriptor(),
1275  "WorkLoopStatusRequest");
1276 
1277  __COUT__ << "FESupervisor instance " << it.first <<
1278  " has status = " << status << std::endl;
1279 
1280  if(status != CoreSupervisorBase::WORK_LOOP_DONE)
1281  {
1282  allFrontEndsAreDone = false;
1283  break;
1284  }
1285  }
1286  catch (xdaq::exception::Exception& e)
1287  {
1288  __SS__ << "Could not retrieve status from FESupervisor instance " <<
1289  it.first << "." << std::endl;
1290  __COUT_WARN__ << ss.str();
1291  errorStr = ss.str();
1292 
1293  if(errorStr != "")
1294  {
1295  __SS__ << "Iterator failed to run because of the following error: " << errorStr;
1296  throw std::runtime_error(ss.str());
1297  }
1298  }
1299  }
1300 
1301  if(allFrontEndsAreDone)
1302  {
1303  //need to end run!
1304  __COUT__ << "FE workloops all complete! Stopping run..." << __E__;
1305 
1306  errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
1307  0,0,
1308  "Stop",iteratorStruct->fsmName_,
1309  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1310  WebUsers::DEFAULT_ITERATOR_USERNAME,
1311  iteratorStruct->fsmCommandParameters_);
1312 
1313  if(errorStr != "")
1314  {
1315  __SS__ << "Iterator failed to stop run because of the following error: " << errorStr;
1316  throw std::runtime_error(ss.str());
1317  }
1318 
1319  //write indication of run done into duration
1320  iteratorStruct->runIsDone_ = true;
1321 
1322  return false;
1323  }
1324 
1325  }
1326 
1328  //priority 2 is duration, if <= 0 it is ignored
1329  if(remainingDurationInSeconds > 1)
1330  {
1331  --remainingDurationInSeconds;
1332 
1333  //write back to string
1334  char str[200];
1335  sprintf(str,"%ld",remainingDurationInSeconds);
1336  iteratorStruct->commands_[iteratorStruct->commandIndex_].params_
1337  [IterateConfiguration::commandRunParams_.DurationInSeconds_] =
1338  str; //re-store as string
1339  }
1340  else if(remainingDurationInSeconds == 1)
1341  {
1342  //need to end run!
1343  __COUT__ << "Time duration reached! Stopping run..." << __E__;
1344 
1345  errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
1346  0,0,
1347  "Stop",iteratorStruct->fsmName_,
1348  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1349  WebUsers::DEFAULT_ITERATOR_USERNAME,
1350  iteratorStruct->fsmCommandParameters_);
1351 
1352  if(errorStr != "")
1353  {
1354  __SS__ << "Iterator failed to stop run because of the following error: " << errorStr;
1355  throw std::runtime_error(ss.str());
1356  }
1357 
1358  //write indication of run done into duration
1359  iteratorStruct->runIsDone_ = true;
1360 
1361  return false;
1362  }
1363  }
1364 
1365  if(errorStr != "")
1366  {
1367  __SS__ << "Iterator failed to run because of the following error: " << errorStr;
1368  throw std::runtime_error(ss.str());
1369  }
1370  return false;
1371 }
1372 
1373 //========================================================================================================================
1374 //return true if done
1375 bool Iterator::checkCommandConfigure(IteratorWorkLoopStruct *iteratorStruct)
1376 {
1377  sleep(1); //sleep to give FSM time to transition
1378 
1379  //all RunControlStateMachine access commands should be mutually exclusive with GatewaySupervisor main thread state machine accesses
1380  //should be mutually exclusive with GatewaySupervisor main thread state machine accesses
1381  //lockout the messages array for the remainder of the scope
1382  //this guarantees the reading thread can safely access the messages
1383  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for FSM access" << __E__;
1384  std::lock_guard<std::mutex> lock(iteratorStruct->theIterator_->theSupervisor_->stateMachineAccessMutex_);
1385  if(iteratorStruct->theIterator_->theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have FSM access" << __E__;
1386 
1387  if(iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.isInTransition())
1388  return false;
1389 
1390  std::string errorStr = "";
1391  std::string currentState = iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getCurrentStateName();
1392 
1393  if(currentState == "Halted")
1394  errorStr = iteratorStruct->theIterator_->theSupervisor_->attemptStateMachineTransition(
1395  0,0,
1396  "Configure",iteratorStruct->fsmName_,
1397  WebUsers::DEFAULT_ITERATOR_USERNAME /*fsmWindowName*/,
1398  WebUsers::DEFAULT_ITERATOR_USERNAME,
1399  iteratorStruct->fsmCommandParameters_);
1400  else if(currentState != "Configured")
1401  errorStr = "Expected to be in Configure. Unexpectedly, the current state is " +
1402  currentState + "." + ". Last State Machine error message was as follows: " +
1403  iteratorStruct->theIterator_->theSupervisor_->theStateMachine_.getErrorMessage();
1404  else //else successfully done (in Configured state!)
1405  {
1406  __COUT__ << "checkCommandConfigureAlias complete." << __E__;
1407 
1408  //also activate the Iterator config manager to mimic active config
1409  std::pair<std::string, ConfigurationGroupKey> newActiveGroup =
1410  iteratorStruct->cfgMgr_->getConfigurationGroupFromAlias(iteratorStruct->fsmCommandParameters_[0]);
1411  iteratorStruct->cfgMgr_->loadConfigurationGroup(newActiveGroup.first,newActiveGroup.second,true /*activate*/);
1412 
1413 
1414  __COUT__ << "originalTrackChanges " <<
1415  iteratorStruct->originalTrackChanges_ << __E__;
1416  __COUT__ << "originalConfigGroup " <<
1417  iteratorStruct->originalConfigGroup_ << __E__;
1418  __COUT__ << "originalConfigKey " <<
1419  iteratorStruct->originalConfigKey_ << __E__;
1420 
1421  __COUT__ << "currentTrackChanges " <<
1422  ConfigurationInterface::isVersionTrackingEnabled() << __E__;
1423  __COUT__ << "originalConfigGroup " <<
1424  iteratorStruct->cfgMgr_->getActiveGroupName() << __E__;
1425  __COUT__ << "originalConfigKey " <<
1426  iteratorStruct->cfgMgr_->getActiveGroupKey() << __E__;
1427 
1428  return true;
1429  }
1430 
1431  if(errorStr != "")
1432  {
1433  __SS__ << "Iterator failed to configure with system alias '" <<
1434  (iteratorStruct->fsmCommandParameters_.size()?
1435  iteratorStruct->fsmCommandParameters_[0]:"UNKNOWN") <<
1436  "' because of the following error: " << errorStr;
1437  throw std::runtime_error(ss.str());
1438  }
1439  return false;
1440 }
1441 
1442 
1443 //========================================================================================================================
1444 bool Iterator::handleCommandRequest(HttpXmlDocument& xmldoc,
1445  const std::string& command, const std::string& parameter)
1446 {
1447  if(command == "iteratePlay")
1448  {
1449  playIterationPlan(xmldoc,parameter);
1450  return true;
1451  }
1452  else if(command == "iteratePause")
1453  {
1454  pauseIterationPlan(xmldoc);
1455  return true;
1456  }
1457  else if(command == "iterateHalt")
1458  {
1459  haltIterationPlan(xmldoc);
1460  return true;
1461  }
1462  else if(command == "getIterationPlanStatus")
1463  {
1464  getIterationPlanStatus(xmldoc);
1465  return true;
1466  }
1467  else //return true if iterator has control of state machine
1468  {
1469  //lockout the messages array for the remainder of the scope
1470  //this guarantees the reading thread can safely access the messages
1471  if(theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for iterator access" << __E__;
1472  std::lock_guard<std::mutex> lock(accessMutex_);
1473  if(theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have iterator access" << __E__;
1474 
1475  if(iteratorBusy_)
1476  {
1477  __SS__ << "Error - Can not accept request because the Iterator " <<
1478  "is currently " <<
1479  "in control of State Machine progress. ";
1480  __COUT_ERR__ << "\n" << ss.str();
1481  __MOUT_ERR__ << "\n" << ss.str();
1482 
1483  xmldoc.addTextElementToData("state_tranisition_attempted", "0"); //indicate to GUI transition NOT attempted
1484  xmldoc.addTextElementToData("state_tranisition_attempted_err",
1485  ss.str()); //indicate to GUI transition NOT attempted
1486 
1487  return true; //to block other commands
1488  }
1489  }
1490  return false;
1491 }
1492 
1493 //========================================================================================================================
1494 void Iterator::playIterationPlan(HttpXmlDocument& xmldoc, const std::string& planName)
1495 {
1496  __MOUT__ << "Attempting to play iteration plan '" << planName << ".'" << __E__;
1497  __COUT__ << "Attempting to play iteration plan '" << planName << ".'" << __E__;
1498 
1499 
1500 
1501  //setup "play" command
1502 
1503  //lockout the messages array for the remainder of the scope
1504  //this guarantees the reading thread can safely access the messages
1505  if(theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for iterator access" << __E__;
1506  std::lock_guard<std::mutex> lock(accessMutex_);
1507  if(theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have iterator access" << __E__;
1508 
1509  if(!activePlanIsRunning_ && !commandPlay_)
1510  {
1511  if(!workloopRunning_)
1512  {
1513  //start thread with member variables initialized
1514 
1515  workloopRunning_ = true;
1516 
1517  //must start thread first
1518  std::thread([](Iterator *iterator){ Iterator::IteratorWorkLoop(iterator); },this).detach();
1519  }
1520 
1521  activePlanName_ = planName;
1522  commandPlay_ = true;
1523  }
1524  else
1525  {
1526  __SS__ << "Invalid play command attempted. Can only play when the Iterator is inactive or paused." <<
1527  " If you would like to restart an iteration plan, first try halting." << __E__;
1528  __MOUT__ << ss.str();
1529 
1530  xmldoc.addTextElementToData("error_message", ss.str());
1531 
1532  __COUT__ << "Invalid play command attempted. " <<
1533  commandPlay_ << " " <<
1534  activePlanName_ << __E__;
1535  }
1536 }
1537 
1538 //========================================================================================================================
1539 void Iterator::pauseIterationPlan(HttpXmlDocument& xmldoc)
1540 {
1541  __MOUT__ << "Attempting to pause iteration plan '" << activePlanName_ << ".'" << __E__;
1542  __COUT__ << "Attempting to pause iteration plan '" << activePlanName_ << ".'" << __E__;
1543 
1544  //setup "pause" command
1545 
1546  //lockout the messages array for the remainder of the scope
1547  //this guarantees the reading thread can safely access the messages
1548  if(theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for iterator access" << __E__;
1549  std::lock_guard<std::mutex> lock(accessMutex_);
1550  if(theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have iterator access" << __E__;
1551 
1552  if(workloopRunning_ && activePlanIsRunning_ && !commandPause_)
1553  {
1554  commandPause_ = true;
1555  }
1556  else
1557  {
1558  __SS__ << "Invalid pause command attempted. Can only pause when running." << __E__;
1559  __MOUT__ << ss.str();
1560 
1561  xmldoc.addTextElementToData("error_message", ss.str());
1562 
1563  __COUT__ << "Invalid pause command attempted. " <<
1564  workloopRunning_ << " " <<
1565  activePlanIsRunning_ << " " <<
1566  commandPause_ << " " <<
1567  activePlanName_ << __E__;
1568  }
1569 }
1570 
1571 //========================================================================================================================
1572 void Iterator::haltIterationPlan(HttpXmlDocument& xmldoc)
1573 {
1574  __MOUT__ << "Attempting to halt iteration plan '" << activePlanName_ << ".'" << __E__;
1575  __COUT__ << "Attempting to halt iteration plan '" << activePlanName_ << ".'" << __E__;
1576 
1577  //setup "halt" command
1578 
1579 
1580  if(workloopRunning_)
1581  {
1582  //lockout the messages array for the remainder of the scope
1583  //this guarantees the reading thread can safely access the messages
1584  if(theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for iterator access" << __E__;
1585  std::lock_guard<std::mutex> lock(accessMutex_);
1586  if(theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have iterator access" << __E__;
1587 
1588  __COUT__ << "activePlanIsRunning_: " << activePlanIsRunning_ << __E__;
1589  __COUT__ << "Passing halt command to iterator thread." << __E__;
1590  commandHalt_ = true;
1591 
1592  //clear
1593  activePlanName_ = "";
1594  activeCommandIndex_ = -1;
1595  }
1596  else //no thread, so halt (and reset Error') without command to thread
1597  {
1598  __COUT__ << "No thread, so conducting halt." << __E__;
1599  Iterator::haltIterator(this);
1600 //
1601 // activePlanIsRunning_ = false;
1602 // iteratorBusy_ = false;
1603 //
1604 // bool haltAttempted = false;
1605 // try
1606 // {
1607 // haltAttempted = Iterator::haltIterator(theSupervisor_, lastFsmName_);
1608 // }
1609 // catch(const std::runtime_error& e)
1610 // {
1611 // haltAttempted = false;
1612 // __COUT__ << "Halt error: " << e.what() << __E__;
1613 // }
1614 //
1615 // if(!haltAttempted) //then show error
1616 // {
1617 // __SS__ << "Invalid halt command attempted. Can only halt when there is an active iteration plan." << __E__;
1618 // __MOUT_ERR__ << ss.str();
1619 //
1620 // xmldoc.addTextElementToData("error_message", ss.str());
1621 //
1622 // __COUT__ << "Invalid halt command attempted. " <<
1623 // workloopRunning_ << " " <<
1624 // activePlanIsRunning_ << " " <<
1625 // commandHalt_ << " " <<
1626 // activePlanName_ << __E__;
1627 // }
1628 // else
1629 // __COUT__ << "Halt was attempted." << __E__;
1630 
1631  }
1632 
1633 }
1634 
1635 //========================================================================================================================
1636 // return state machine and iterator status
1637 void Iterator::getIterationPlanStatus(HttpXmlDocument& xmldoc)
1638 {
1639  xmldoc.addTextElementToData("current_state",
1640  theSupervisor_->theStateMachine_.isInTransition()?
1641  theSupervisor_->theStateMachine_.getCurrentTransitionName(
1642  theSupervisor_->stateMachineLastCommandInput_):
1643  theSupervisor_->theStateMachine_.getCurrentStateName());
1644 
1645  //xmldoc.addTextElementToData("in_transition", theSupervisor_->theStateMachine_.isInTransition() ? "1" : "0");
1646  if(theSupervisor_->theStateMachine_.isInTransition())
1647  xmldoc.addTextElementToData("transition_progress",
1648  theSupervisor_->theProgressBar_.readPercentageString());
1649  else
1650  xmldoc.addTextElementToData("transition_progress", "100");
1651 
1652  char tmp[20];
1653  sprintf(tmp,"%lu",theSupervisor_->theStateMachine_.getTimeInState());
1654  xmldoc.addTextElementToData("time_in_state", tmp);
1655 
1656 
1657 
1658  //lockout the messages array for the remainder of the scope
1659  //this guarantees the reading thread can safely access the messages
1660  if(theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Waiting for iterator access" << __E__;
1661  std::lock_guard<std::mutex> lock(accessMutex_);
1662  if(theSupervisor_->VERBOSE_MUTEX) __COUT__ << "Have iterator access" << __E__;
1663 
1664  xmldoc.addTextElementToData("active_plan", activePlanName_);
1665  xmldoc.addTextElementToData("last_started_plan", lastStartedPlanName_);
1666  xmldoc.addTextElementToData("last_finished_plan", lastFinishedPlanName_);
1667 
1668  sprintf(tmp,"%u",activeCommandIndex_);
1669  xmldoc.addTextElementToData("current_command_index", tmp);
1670  sprintf(tmp,"%ld",time(0) - activeCommandStartTime_);
1671  xmldoc.addTextElementToData("current_command_duration", tmp);
1672  sprintf(tmp,"%u",activeCommandIteration_);
1673  xmldoc.addTextElementToData("current_command_iteration", tmp);
1674 
1675  for(const auto& depthIteration:depthIterationStack_)
1676  {
1677  sprintf(tmp,"%u",depthIteration);
1678  xmldoc.addTextElementToData("depth_iteration", tmp);
1679  }
1680 
1681  if(activePlanIsRunning_ && iteratorBusy_)
1682  {
1683  if(workloopRunning_)
1684  xmldoc.addTextElementToData("active_plan_status", "Running");
1685  else
1686  xmldoc.addTextElementToData("active_plan_status", "Error");
1687  }
1688  else if(!activePlanIsRunning_ && iteratorBusy_)
1689  xmldoc.addTextElementToData("active_plan_status", "Paused");
1690  else
1691  xmldoc.addTextElementToData("active_plan_status", "Inactive");
1692 
1693  xmldoc.addTextElementToData("error_message", errorMessage_);
1694 }
1695 
1696