otsdaq_utilities  v2_02_00
ConsoleSupervisor.cc
1 #include "otsdaq-utilities/Console/ConsoleSupervisor.h"
2 #include "otsdaq-core/MessageFacility/MessageFacility.h"
3 #include "otsdaq-core/Macros/CoutMacros.h"
4 #include "otsdaq-core/CgiDataUtilities/CgiDataUtilities.h"
5 #include "otsdaq-core/XmlUtilities/HttpXmlDocument.h"
6 #include "otsdaq-core/NetworkUtilities/ReceiverSocket.h"
7 #include <xdaq/NamespaceURI.h>
8 
9 #include <iostream>
10 #include <fstream>
11 #include <string>
12 #include <dirent.h> //for DIR
13 #include <sys/stat.h> //for mkdir
14 #include <thread> //for std::thread
15 
16 using namespace ots;
17 
18 // UDP Message Format:
19 // UDPMESSAGE|TIMESTAMP|SEQNUM|HOSTNAME|HOSTADDR|SEVERITY|CATEGORY|APPLICATION|PID|ITERATION|MODULE|(FILE|LINE)|MESSAGE
20 // FILE and LINE are only printed for s67+
21 
22 XDAQ_INSTANTIATOR_IMPL(ConsoleSupervisor)
23 
24 
25 #define USER_CONSOLE_PREF_PATH std::string(getenv("SERVICE_DATA_PATH")) + "/ConsolePreferences/"
26 #define USERS_PREFERENCES_FILETYPE "pref"
27 
28 #define QUIET_CFG_FILE std::string(getenv("USER_DATA")) + "/MessageFacilityConfigurations/QuietForwarder.cfg"
29 
30 #define CONSOLE_SPECIAL_ERROR std::string("||0|||Error|Console|-1||ConsoleSupervisor|")
31 #define CONSOLE_SPECIAL_WARNING std::string("||0|||Warning|Console|-1||ConsoleSupervisor|")
32 
33 #undef __MF_SUBJECT__
34 #define __MF_SUBJECT__ "Console"
35 
36 
37 //========================================================================================================================
38 ConsoleSupervisor::ConsoleSupervisor(xdaq::ApplicationStub* stub)
39 : CoreSupervisorBase (stub)
40 , writePointer_ (0)
41 , messageCount_ (0)
42 {
43  __SUP_COUT__ << "Constructor started." << __E__;
44 
45  INIT_MF("ConsoleSupervisor");
46 
47  //attempt to make directory structure (just in case)
48  mkdir(((std::string)USER_CONSOLE_PREF_PATH).c_str(), 0755);
49 
50  init();
51 
52  __SUP_COUT__ << "Constructor complete." << __E__;
53 }
54 
55 //========================================================================================================================
56 ConsoleSupervisor::~ConsoleSupervisor(void)
57 {
58  destroy();
59 }
60 //========================================================================================================================
61 void ConsoleSupervisor::init(void)
62 {
63  //start mf msg listener
64  std::thread([](ConsoleSupervisor *cs){ ConsoleSupervisor::messageFacilityReceiverWorkLoop(cs); },this).detach();
65 }
66 
67 //========================================================================================================================
68 void ConsoleSupervisor::destroy(void)
69 {
70  //called by destructor
71 }
72 
73 //========================================================================================================================
74 //messageFacilityReceiverWorkLoop ~~
75 // Thread for printing Message Facility messages without decorations
76 // Note: Uses std::mutex to avoid conflict with reading thread.
77 void ConsoleSupervisor::messageFacilityReceiverWorkLoop(ConsoleSupervisor *cs)
78 {
79  __COUT__ << std::endl;
80 
81  std::string configFile = QUIET_CFG_FILE;
82  FILE *fp = fopen(configFile.c_str(),"r");
83  if(!fp)
84  {
85  __SS__ << "File with port info could not be loaded: " <<
86  QUIET_CFG_FILE << std::endl;
87  __COUT__ << "\n" << ss.str();
88  __SS_THROW__;
89  }
90  char tmp[100];
91  fgets(tmp,100,fp); //receive port (ignore)
92  fgets(tmp,100,fp); //destination port *** used here ***
93  int myport;
94  sscanf(tmp,"%*s %d",&myport);
95 
96  fgets(tmp,100,fp); //destination ip *** used here ***
97  char myip[100];
98  sscanf(tmp,"%*s %s",myip);
99  fclose(fp);
100 
101  ReceiverSocket rsock(myip,myport); //Take Port from Configuration
102  try
103  {
104  rsock.initialize();
105  }
106  catch(...)
107  {
108  //lockout the messages array for the remainder of the scope
109  //this guarantees the reading thread can safely access the messages
110  std::lock_guard<std::mutex> lock(cs->messageMutex_);
111 
112  //NOTE: if we do not want this to be fatal, do not throw here, just print out
113 
114  if(1) //generate special message and throw for failed socket
115  {
116  __SS__ << "FATAL Console error. Could not initialize socket on port " <<
117  myport << ". Perhaps the port is already in use? Check for multiple stale instances of otsdaq processes, or notify admins." <<
118  " Multiple instances of otsdaq on the same node should be possible, but port numbers must be unique." << std::endl;
119  __SS_THROW__;
120  }
121 
122  //generate special message to indicate failed socket
123  __SS__ << "FATAL Console error. Could not initialize socket on port " <<
124  myport << ". Perhaps it is already in use? Exiting Console receive loop." << std::endl;
125  __COUT__ << ss.str();
126 
127 
128  cs->messages_[cs->writePointer_].set(CONSOLE_SPECIAL_ERROR +
129  ss.str(),
130  cs->messageCount_++);
131 
132 
133  if(++cs->writePointer_ == cs->messages_.size()) //handle wrap-around
134  cs->writePointer_ = 0;
135 
136  return;
137  }
138 
139  std::string buffer;
140  int i = 0;
141  int heartbeatCount = 0;
142  int selfGeneratedMessageCount = 0;
143 
144  std::map<unsigned int, unsigned int> sourceLastSequenceID; //map from sourceID to lastSequenceID to identify missed messages
145  long long newSourceId;
146  unsigned int newSequenceId;
147 
148  while(1)
149  {
150  //if receive succeeds display message
151 
152  if(rsock.receive(buffer,1 /*timeoutSeconds*/,0/*timeoutUSeconds*/,
153  false /*verbose*/) != -1)
154  {
155  if(i != 200)
156  {
157  __COUT__ << "Console has first message." << std::endl;
158  i = 200; //mark so things are good for all time. (this indicates things are configured to be sent here)
159 
160  __MOUT__ << "DEBUG messages look like this." << std::endl;
161  __MOUT_INFO__ << "INFO messages look like this." << std::endl;
162  __MOUT_WARN__ << "WARNING messages look like this." << std::endl;
163  __MOUT_ERR__ << "ERROR messages look like this." << std::endl;
164 
165 
166 
167  // //to debug special packets
168  // __SS__ << "???";
169  // cs->messages_[cs->writePointer_].set(CONSOLE_SPECIAL_ERROR +
170  // ss.str(),
171  // cs->messageCount_++);
172  //
173  // if(++cs->writePointer_ == cs->messages_.size()) //handle wrap-around
174  // cs->writePointer_ = 0;
175  }
176 
177  if(selfGeneratedMessageCount)
178  --selfGeneratedMessageCount; //decrement internal message count
179  else
180  heartbeatCount = 0; //reset heartbeat if external messages are coming through
181 
182  //__COUT__ << buffer << std::endl;
183 
184  //lockout the messages array for the remainder of the scope
185  //this guarantees the reading thread can safely access the messages
186  std::lock_guard<std::mutex> lock(cs->messageMutex_);
187 
188  cs->messages_[cs->writePointer_].set(buffer,cs->messageCount_++);
189 
190 
191  //check if sequence ID is out of order
192  newSourceId = cs->messages_[cs->writePointer_].getSourceIDAsNumber();
193  newSequenceId = cs->messages_[cs->writePointer_].getSequenceIDAsNumber();
194 
195  //__COUT__ << "newSourceId: " << newSourceId << std::endl;
196  //__COUT__ << "newSequenceId: " << newSequenceId << std::endl;
197 
198  if(newSourceId != -1 && sourceLastSequenceID.find(newSourceId) !=
199  sourceLastSequenceID.end() && //ensure not first packet received
200  ((newSequenceId == 0 &&
201  sourceLastSequenceID[newSourceId] != (unsigned int)-1) || //wrap around case
202  newSequenceId != sourceLastSequenceID[newSourceId] + 1)) //normal sequence case
203  {
204  //missed some messages!
205  __SS__ << "Missed packets from " <<
206  cs->messages_[cs->writePointer_].getSource() << "! Sequence IDs " <<
207  sourceLastSequenceID[newSourceId] <<
208  " to " << newSequenceId << "." << std::endl;
209  std::cout << ss.str();
210 
211  if(++cs->writePointer_ == cs->messages_.size()) //handle wrap-around
212  cs->writePointer_ = 0;
213 
214  //generate special message to indicate missed packets
215  cs->messages_[cs->writePointer_].set(CONSOLE_SPECIAL_WARNING +
216  ss.str(),
217  cs->messageCount_++);
218  }
219 
220  //save the new last sequence ID
221  sourceLastSequenceID[newSourceId] = newSequenceId;
222 
223  if(++cs->writePointer_ == cs->messages_.size()) //handle wrap-around
224  cs->writePointer_ = 0;
225 
226  }
227  else
228  {
229  if(i < 120) //if nothing received for 120 seconds, then something is wrong with Console configuration
230  ++i;
231 
232  sleep(1); //sleep one second, if timeout
233 
234  //every 60 heartbeatCount (2 seconds each = 1 sleep and 1 timeout) print a heartbeat message
235  if(i != 200 || //show first message, if not already a message
236  (heartbeatCount < 60*5 && heartbeatCount%60 == 59)) //every ~2 min for first 5 messages
237  {
238  ++selfGeneratedMessageCount; //increment internal message count
239  __MOUT__ << "Console is alive and waiting... (if no messages, next heartbeat is in approximately two minutes)" << std::endl;
240  }
241  else if(heartbeatCount%(60*30) == 59) //approx every hour
242  {
243  ++selfGeneratedMessageCount; //increment internal message count
244  __MOUT__ << "Console is alive and waiting a long time... (if no messages, next heartbeat is in approximately one hour)" << std::endl;
245  }
246 
247  ++heartbeatCount;
248  }
249 
250  //if nothing received for 2 minutes seconds, then something is wrong with Console configuration
251  // after 5 seconds there is a self-send. Which will at least confirm configuration.
252  // OR if 5 generated messages and never cleared.. then the forwarding is not working.
253  if(i==120 || selfGeneratedMessageCount == 5)
254  {
255  __COUT__ << "No messages received at Console Supervisor. Exiting Console messageFacilityReceiverWorkLoop" << std::endl;
256  break; //assume something wrong, and break loop
257  }
258  }
259 
260 } //end messageFacilityReceiverWorkLoop
261 
262 //========================================================================================================================
263 void ConsoleSupervisor::defaultPage(xgi::Input * in, xgi::Output * out )
264 {
265  __SUP_COUT__ << "ApplicationDescriptor LID=" << getApplicationDescriptor()->getLocalId() << std::endl;
266  *out << "<!DOCTYPE HTML><html lang='en'><frameset col='100%' row='100%'><frame src='/WebPath/html/Console.html?urn=" <<
267  getApplicationDescriptor()->getLocalId() << "'></frameset></html>";
268 }
269 
270 //========================================================================================================================
271 //forceSupervisorPropertyValues
272 // override to force supervisor property values (and ignore user settings)
273 void ConsoleSupervisor::forceSupervisorPropertyValues()
274 {
275  CorePropertySupervisorBase::setSupervisorProperty(CorePropertySupervisorBase::SUPERVISOR_PROPERTIES.AutomatedRequestTypes,
276  "GetConsoleMsgs");
277 // CorePropertySupervisorBase::setSupervisorProperty(CorePropertySupervisorBase::SUPERVISOR_PROPERTIES.NeedUsernameRequestTypes,
278 // "SaveUserPreferences | LoadUserPreferences");
279 }
280 
281 //========================================================================================================================
282 // Request
283 // Handles Web Interface requests to Console supervisor.
284 // Does not refresh cookie for automatic update checks.
285 void ConsoleSupervisor::request(const std::string& requestType, cgicc::Cgicc& cgiIn,
286  HttpXmlDocument& xmlOut, const WebUsers::RequestUserInfo& userInfo)
287 {
288  //__SUP_COUT__ << "requestType " << requestType << std::endl;
289 
290  //Commands:
291  //GetConsoleMsgs
292  //SaveUserPreferences
293  //LoadUserPreferences
294 
295 
296  //Note: to report to logbook admin status use xmlOut.addTextElementToData(XML_ADMIN_STATUS,refreshTempStr_);
297 
298  if(requestType == "GetConsoleMsgs")
299  {
300  //lindex of -1 means first time and user just gets update lcount and lindex
301  std::string lastUpdateCountStr = CgiDataUtilities::postData(cgiIn,"lcount");
302  std::string lastUpdateIndexStr = CgiDataUtilities::postData(cgiIn,"lindex");
303 
304  if(lastUpdateCountStr == "" || lastUpdateIndexStr == "")
305  {
306  __SUP_COUT_ERR__ << "Invalid Parameters! lastUpdateCount=" << lastUpdateCountStr <<
307  ", lastUpdateIndex=" << lastUpdateIndexStr << std::endl;
308  xmlOut.addTextElementToData("Error","Error - Invalid parameters for GetConsoleMsgs.");
309  return;
310  }
311 
312  clock_t lastUpdateCount;
313  sscanf(lastUpdateCountStr.c_str(),"%ld",&lastUpdateCount);
314 
315  unsigned int lastUpdateIndex;
316  sscanf(lastUpdateIndexStr.c_str(),"%u",&lastUpdateIndex);
317 // __SUP_COUT__ << "lastUpdateCount=" << lastUpdateCount <<
318 // ", lastUpdateIndex=" << lastUpdateIndex << std::endl;
319 
320  insertMessageRefresh(&xmlOut,lastUpdateCount,lastUpdateIndex);
321  }
322  else if(requestType == "SaveUserPreferences")
323  {
324  int colorIndex = CgiDataUtilities::postDataAsInt(cgiIn,"colorIndex");
325  int showSideBar = CgiDataUtilities::postDataAsInt(cgiIn,"showSideBar");
326  int noWrap = CgiDataUtilities::postDataAsInt(cgiIn,"noWrap");
327  int messageOnly = CgiDataUtilities::postDataAsInt(cgiIn,"messageOnly");
328  int hideLineNumers = CgiDataUtilities::postDataAsInt(cgiIn,"hideLineNumers");
329 
330  __SUP_COUT__ << "requestType " << requestType << std::endl;
331  __SUP_COUT__ << "colorIndex: " << colorIndex << std::endl;
332  __SUP_COUT__ << "showSideBar: " << showSideBar << std::endl;
333  __SUP_COUT__ << "noWrap: " << noWrap << std::endl;
334  __SUP_COUT__ << "messageOnly: " << messageOnly << std::endl;
335  __SUP_COUT__ << "hideLineNumers: " << hideLineNumers << std::endl;
336 
337  if(userInfo.username_ == "") //should never happen?
338  {
339  __SUP_COUT_ERR__ << "Invalid user found! user=" << userInfo.username_ << std::endl;
340  xmlOut.addTextElementToData("Error","Error - InvauserInfo.username_user found.");
341  return;
342  }
343 
344  std::string fn = (std::string)USER_CONSOLE_PREF_PATH + userInfo.username_ + "." + (std::string)USERS_PREFERENCES_FILETYPE;
345 
346  __SUP_COUT__ << "Save preferences: " << fn << std::endl;
347  FILE *fp = fopen(fn.c_str(),"w");
348  if(!fp)
349  {__SS__;__THROW__(ss.str()+"Could not open file: " + fn);}
350  fprintf(fp,"colorIndex %d\n",colorIndex);
351  fprintf(fp,"showSideBar %d\n",showSideBar);
352  fprintf(fp,"noWrap %d\n",noWrap);
353  fprintf(fp,"messageOnly %d\n",messageOnly);
354  fprintf(fp,"hideLineNumers %d\n",hideLineNumers);
355  fclose(fp);
356  }
357  else if(requestType == "LoadUserPreferences")
358  {
359  __SUP_COUT__ << "requestType " << requestType << std::endl;
360 
361  unsigned int colorIndex,showSideBar,noWrap,messageOnly,hideLineNumers;
362 
363  if(userInfo.username_ == "") //should never happen?
364  {
365  __SUP_COUT_ERR__ << "Invalid user found! user=" << userInfo.username_ << std::endl;
366  xmlOut.addTextElementToData("Error","Error - Invalid user found.");
367  return;
368  }
369 
370  std::string fn = (std::string)USER_CONSOLE_PREF_PATH + userInfo.username_ + "." + (std::string)USERS_PREFERENCES_FILETYPE;
371 
372  __SUP_COUT__ << "Load preferences: " << fn << std::endl;
373 
374  FILE *fp = fopen(fn.c_str(),"r");
375  if(!fp)
376  {
377  //return defaults
378  __SUP_COUT__ << "Returning defaults." << std::endl;
379  xmlOut.addTextElementToData("colorIndex","0");
380  xmlOut.addTextElementToData("showSideBar","0");
381  xmlOut.addTextElementToData("noWrap","1");
382  xmlOut.addTextElementToData("messageOnly","0");
383  xmlOut.addTextElementToData("hideLineNumers","1");
384  return;
385  }
386  fscanf(fp,"%*s %u",&colorIndex);
387  fscanf(fp,"%*s %u",&showSideBar);
388  fscanf(fp,"%*s %u",&noWrap);
389  fscanf(fp,"%*s %u",&messageOnly);
390  fscanf(fp,"%*s %u",&hideLineNumers);
391  fclose(fp);
392  __SUP_COUT__ << "colorIndex: " << colorIndex << std::endl;
393  __SUP_COUT__ << "showSideBar: " << showSideBar << std::endl;
394  __SUP_COUT__ << "noWrap: " << noWrap << std::endl;
395  __SUP_COUT__ << "messageOnly: " << messageOnly << std::endl;
396  __SUP_COUT__ << "hideLineNumers: " << hideLineNumers << std::endl;
397 
398  char tmpStr[20];
399  sprintf(tmpStr,"%u",colorIndex);
400  xmlOut.addTextElementToData("colorIndex",tmpStr);
401  sprintf(tmpStr,"%u",showSideBar);
402  xmlOut.addTextElementToData("showSideBar",tmpStr);
403  sprintf(tmpStr,"%u",noWrap);
404  xmlOut.addTextElementToData("noWrap",tmpStr);
405  sprintf(tmpStr,"%u",messageOnly);
406  xmlOut.addTextElementToData("messageOnly",tmpStr);
407  sprintf(tmpStr,"%u",hideLineNumers);
408  xmlOut.addTextElementToData("hideLineNumers",tmpStr);
409  }
410 
411 }
412 
413 
414 //========================================================================================================================
415 //ConsoleSupervisor::insertMessageRefresh()
416 // if lastUpdateClock is current, return nothing
417 // else return new messages
418 // (note: lastUpdateIndex==(unsigned int)-1 first time and returns as much as possible// nothing but lastUpdateClock)
419 //
420 // format of xml:
421 //
422 // <last_update_count/>
423 // <last_update_index/>
424 // <messages>
425 // <message_FIELDNAME*/>
426  //"Level"
427  //"Label"
428  //"Source"
429  //"Msg"
430  //"Time"
431  //"Count"
432 // </messages>
433 //
434 // NOTE: Uses std::mutex to avoid conflict with writing thread. (this is the reading thread)
435 void ConsoleSupervisor::insertMessageRefresh(HttpXmlDocument *xmlOut,
436  const time_t lastUpdateCount, const unsigned int lastUpdateIndex)
437 {
438  //__SUP_COUT__ << std::endl;
439 
440  //validate lastUpdateIndex
441  if(lastUpdateIndex > messages_.size() &&
442  lastUpdateIndex != (unsigned int)-1)
443  {
444  __SS__ << "Invalid lastUpdateIndex: " << lastUpdateIndex <<
445  " messagesArray size = " << messages_.size() << std::endl;
446  __SS_THROW__;
447  }
448 
449  //lockout the messages array for the remainder of the scope
450  //this guarantees the reading thread can safely access the messages
451  std::lock_guard<std::mutex> lock(messageMutex_);
452 
453  //newest available read pointer is defined as always one behind writePointer_
454  refreshReadPointer_ = (writePointer_ + messages_.size() - 1) % messages_.size();
455 
456  sprintf(refreshTempStr_,"%lu",messages_[refreshReadPointer_].getCount());
457  xmlOut->addTextElementToData("last_update_count",refreshTempStr_);
458  sprintf(refreshTempStr_,"%u",refreshReadPointer_);
459  xmlOut->addTextElementToData("last_update_index",refreshTempStr_);
460 
461  if(!messages_[refreshReadPointer_].getTime()) //if no data, then no data
462  return;
463 
464  //else, send all messages since last_update_count, from last_update_index(refreshReadPointer_) on
465  if(lastUpdateIndex != (unsigned int)-1 && //if not first time, and index-count are valid then start at next message
466  messages_[lastUpdateIndex].getCount() == lastUpdateCount)
467  refreshReadPointer_ = (lastUpdateIndex+1) % messages_.size();
468  else if(messages_[writePointer_].getTime()) //check that writePointer_ message has been initialized, therefore has wrapped around at least once already
469  {
470  //This means we have had many messages and that some were missed since last update
471  // (give as many messages as we can!)
472  xmlOut->addTextElementToData("message_overflow","1");
473  __SUP_COUT__ << "Overflow was detected!" << std::endl;
474  refreshReadPointer_ = (writePointer_+1) % messages_.size();
475  }
476  else //user does not have valid index, and writePointer_ has not wrapped around, so give all new messages
477  refreshReadPointer_ = 0;
478 
479 // __SUP_COUT__ << "refreshReadPointer_: " << refreshReadPointer_ << std::endl;
480 // __SUP_COUT__ << "lastUpdateCount: " << lastUpdateCount << std::endl;
481 // __SUP_COUT__ << "writePointer_: " << writePointer_ << std::endl;
482 
483  //return anything from refreshReadPointer_ to writePointer_
484  //all should have a clock greater than lastUpdateClock
485 
486  refreshParent_ = xmlOut->addTextElementToData("messages","");
487 
488  bool requestOutOfSync = false;
489  std::string requestOutOfSyncMsg;
490  //output oldest to new (from refreshReadPointer_ to writePointer_-1, inclusive)
491  for(/*refreshReadPointer_=<first index to read>*/;
492  refreshReadPointer_ != writePointer_;
493  refreshReadPointer_ = (refreshReadPointer_+1) % messages_.size())
494  {
495  if(messages_[refreshReadPointer_].getCount() < lastUpdateCount)
496  {
497  if(!requestOutOfSync) //record out of sync message once only
498  {
499  requestOutOfSync = true;
500  __SS__ << "Request is out of sync! Message count should be more recent than update clock! " <<
501  messages_[refreshReadPointer_].getCount() << " < " <<
502  lastUpdateCount << std::endl;
503  requestOutOfSyncMsg = ss.str();
504  }
505  //assume these messages are new (due to a system restart)
506  //continue;
507  }
508 
509  //for all fields, give value
510  for(refreshIndex_=0; refreshIndex_ < messages_[refreshReadPointer_].fields.size();++refreshIndex_)
511  xmlOut->addTextElementToParent("message_" +
512  messages_[refreshReadPointer_].fields[refreshIndex_].fieldName,
513  messages_[refreshReadPointer_].getField(refreshIndex_), refreshParent_);
514 
515  //give timestamp also
516  sprintf(refreshTempStr_,"%lu",messages_[refreshReadPointer_].getTime());
517  xmlOut->addTextElementToParent("message_Time",
518  refreshTempStr_, refreshParent_);
519  //give clock also
520  sprintf(refreshTempStr_,"%lu",messages_[refreshReadPointer_].getCount());
521  xmlOut->addTextElementToParent("message_Count",
522  refreshTempStr_, refreshParent_);
523  }
524 
525  if(requestOutOfSync) //if request was out of sync, show message
526  __SUP_COUT__ << requestOutOfSyncMsg;
527 }
528 
529 
530 
531 
532 
533 
534 
535 
536