otsdaq_utilities  v2_04_02
ConsoleSupervisor.cc
1 #include "otsdaq-utilities/Console/ConsoleSupervisor.h"
2 #include <xdaq/NamespaceURI.h>
3 #include "otsdaq/CgiDataUtilities/CgiDataUtilities.h"
4 #include "otsdaq/Macros/CoutMacros.h"
5 #include "otsdaq/MessageFacility/MessageFacility.h"
6 #include "otsdaq/NetworkUtilities/ReceiverSocket.h"
7 #include "otsdaq/XmlUtilities/HttpXmlDocument.h"
8 
9 #include <dirent.h> //for DIR
10 #include <sys/stat.h> //for mkdir
11 #include <fstream>
12 #include <iostream>
13 #include <string>
14 #include <thread> //for std::thread
15 
16 using namespace ots;
17 
18 // UDP Message Format:
19 // UDPMESSAGE|TIMESTAMP|SEQNUM|HOSTNAME|HOSTADDR|SEVERITY|CATEGORY|APPLICATION|PID|ITERATION|MODULE|(FILE|LINE)|MESSAGE
20 // FILE and LINE are only printed for s67+
21 
22 XDAQ_INSTANTIATOR_IMPL(ConsoleSupervisor)
23 
24 #define USER_CONSOLE_PREF_PATH \
25  std::string(__ENV__("SERVICE_DATA_PATH")) + "/ConsolePreferences/"
26 #define USERS_PREFERENCES_FILETYPE "pref"
27 
28 #define QUIET_CFG_FILE \
29  std::string(__ENV__("USER_DATA")) + \
30  "/MessageFacilityConfigurations/" \
31  "QuietForwarder.cfg"
32 
33 #define CONSOLE_SPECIAL_ERROR \
34  std::string("|30-Aug-2019 15:30:17 CDT|0|||Error|Console||-1||ConsoleSupervisor|") + \
35  std::string(__FILE__) + std::string("|") + std::to_string(__LINE__) + \
36  std::string("|")
37 #define CONSOLE_SPECIAL_WARNING \
38  std::string( \
39  "|30-Aug-2019 15:30:17 CDT|0|||Warning|Console||-1||ConsoleSupervisor|") + \
40  std::string(__FILE__) + std::string("|") + std::to_string(__LINE__) + \
41  std::string("|")
42 
43 #undef __MF_SUBJECT__
44 #define __MF_SUBJECT__ "Console"
45 
46 //========================================================================================================================
47 ConsoleSupervisor::ConsoleSupervisor(xdaq::ApplicationStub* stub)
48  : CoreSupervisorBase(stub), messageCount_(0), maxMessageCount_(100000)
49 {
50  __SUP_COUT__ << "Constructor started." << __E__;
51 
52  INIT_MF("ConsoleSupervisor");
53 
54  // attempt to make directory structure (just in case)
55  mkdir(((std::string)USER_CONSOLE_PREF_PATH).c_str(), 0755);
56 
57  init();
58 
59  __SUP_COUT__ << "Constructor complete." << __E__;
60 }
61 
62 //========================================================================================================================
63 ConsoleSupervisor::~ConsoleSupervisor(void) { destroy(); }
64 //========================================================================================================================
65 void ConsoleSupervisor::init(void)
66 {
67  // start mf msg listener
68  std::thread(
69  [](ConsoleSupervisor* cs) {
70  ConsoleSupervisor::messageFacilityReceiverWorkLoop(cs);
71  },
72  this)
73  .detach();
74 } // end init()
75 
76 //========================================================================================================================
77 void ConsoleSupervisor::destroy(void)
78 {
79  // called by destructor
80 } // end destroy()
81 
82 //========================================================================================================================
83 // messageFacilityReceiverWorkLoop ~~
84 // Thread for printing Message Facility messages without decorations
85 // Note: Uses std::mutex to avoid conflict with reading thread.
86 void ConsoleSupervisor::messageFacilityReceiverWorkLoop(ConsoleSupervisor* cs) try
87 {
88  __COUT__ << std::endl;
89 
90  std::string configFile = QUIET_CFG_FILE;
91  FILE* fp = fopen(configFile.c_str(), "r");
92  if(!fp)
93  {
94  __SS__ << "File with port info could not be loaded: " << QUIET_CFG_FILE
95  << std::endl;
96  __COUT__ << "\n" << ss.str();
97  __SS_THROW__;
98  }
99  char tmp[100];
100  fgets(tmp, 100, fp); // receive port (ignore)
101  fgets(tmp, 100, fp); // destination port *** used here ***
102  int myport;
103  sscanf(tmp, "%*s %d", &myport);
104 
105  fgets(tmp, 100, fp); // destination ip *** used here ***
106  char myip[100];
107  sscanf(tmp, "%*s %s", myip);
108  fclose(fp);
109 
110  ReceiverSocket rsock(myip, myport); // Take Port from Configuration
111  try
112  {
113  rsock.initialize();
114  }
115  catch(...)
116  {
117  // lockout the messages array for the remainder of the scope
118  // this guarantees the reading thread can safely access the messages
119  std::lock_guard<std::mutex> lock(cs->messageMutex_);
120 
121  // NOTE: if we do not want this to be fatal, do not throw here, just print out
122 
123  if(1) // generate special message and throw for failed socket
124  {
125  __SS__ << "FATAL Console error. Could not initialize socket on port "
126  << myport
127  << ". Perhaps the port is already in use? Check for multiple stale "
128  "instances of otsdaq processes, or notify admins."
129  << " Multiple instances of otsdaq on the same node should be "
130  "possible, but port numbers must be unique."
131  << std::endl;
132  __SS_THROW__;
133  }
134 
135  // generate special message to indicate failed socket
136  __SS__ << "FATAL Console error. Could not initialize socket on port " << myport
137  << ". Perhaps it is already in use? Exiting Console receive loop."
138  << std::endl;
139  __COUT__ << ss.str();
140 
141  cs->messages_.emplace_back(CONSOLE_SPECIAL_ERROR + ss.str(), cs->messageCount_++);
142 
143  if(cs->messages_.size() > cs->maxMessageCount_)
144  {
145  cs->messages_.erase(cs->messages_.begin());
146  }
147 
148  return;
149  }
150 
151  std::string buffer;
152  int i = 0;
153  int heartbeatCount = 0;
154  int selfGeneratedMessageCount = 0;
155 
156  std::map<unsigned int, unsigned int>
157  sourceLastSequenceID; // map from sourceID to
158  // lastSequenceID to
159  // identify missed messages
160  long long newSourceId;
161  unsigned int newSequenceId;
162 
163  // force a starting message
164  __MCOUT__("DEBUG messages look like this." << __E__);
165 
166  while(1)
167  {
168  // if receive succeeds display message
169 
170  //__COUTV__(i);
171 
172  if(rsock.receive(
173  buffer, 1 /*timeoutSeconds*/, 0 /*timeoutUSeconds*/, false /*verbose*/) !=
174  -1)
175  {
176  if(i != 200)
177  {
178  __COUT__ << "Console has first message." << __E__;
179  i = 200; // mark so things are good for all time. (this indicates things
180  // are configured to be sent here)
181 
182  __MOUT_INFO__ << "INFO messages look like this." << __E__;
183  __MOUT_WARN__ << "WARNING messages look like this." << __E__;
184  __MOUT_ERR__ << "ERROR messages look like this." << __E__;
185 
186  // //to debug special packets
187  // __SS__ << "???";
188  // cs->messages_[cs->writePointer_].set(CONSOLE_SPECIAL_ERROR
189  //+ ss.str(),
190  // cs->messageCount_++);
191  //
192  // if(++cs->writePointer_ == cs->messages_.size()) //handle
193  // wrap-around cs->writePointer_ = 0;
194  }
195 
196  if(selfGeneratedMessageCount)
197  --selfGeneratedMessageCount; // decrement internal message count
198  else // reset heartbeat if external messages are coming through
199  heartbeatCount = 0;
200 
201  //__COUT__ << buffer << std::endl;
202 
203  // lockout the messages array for the remainder of the scope
204  // this guarantees the reading thread can safely access the messages
205  std::lock_guard<std::mutex> lock(cs->messageMutex_);
206 
207  cs->messages_.emplace_back(buffer, cs->messageCount_++);
208 
209  // check if sequence ID is out of order
210  newSourceId = cs->messages_.back().getSourceIDAsNumber();
211  newSequenceId = cs->messages_.back().getSequenceIDAsNumber();
212 
213  //__COUT__ << "newSourceId: " << newSourceId << std::endl;
214  //__COUT__ << "newSequenceId: " << newSequenceId << std::endl;
215 
216  if(newSourceId != -1 &&
217  sourceLastSequenceID.find(newSourceId) !=
218  sourceLastSequenceID.end() && // ensure not first packet received
219  ((newSequenceId == 0 && sourceLastSequenceID[newSourceId] !=
220  (unsigned int)-1) || // wrap around case
221  newSequenceId !=
222  sourceLastSequenceID[newSourceId] + 1)) // normal sequence case
223  {
224  // missed some messages!
225  __SS__ << "Missed packets from " << cs->messages_.back().getSource()
226  << "! Sequence IDs " << sourceLastSequenceID[newSourceId] << " to "
227  << newSequenceId << "." << std::endl;
228  std::cout << ss.str();
229 
230  // generate special message to indicate missed packets
231  cs->messages_.emplace_back(CONSOLE_SPECIAL_WARNING + ss.str(),
232  cs->messageCount_++);
233  }
234 
235  // save the new last sequence ID
236  sourceLastSequenceID[newSourceId] = newSequenceId;
237 
238  while(cs->messages_.size() > 0 && cs->messages_.size() > cs->maxMessageCount_)
239  {
240  cs->messages_.erase(cs->messages_.begin());
241  }
242  }
243  else
244  {
245  if(i < 120) // if nothing received for 120 seconds, then something is wrong
246  // with Console configuration
247  ++i;
248 
249  sleep(1); // sleep one second, if timeout
250 
251  // every 60 heartbeatCount (2 seconds each = 1 sleep and 1 timeout) print a
252  // heartbeat message
253  if(i != 200 || // show first message, if not already a message
254  //(heartbeatCount < 60 * 5 &&
255  heartbeatCount % 60 == 59 //)
256  ) // every ~2 min for first 5 messages
257  {
258  ++selfGeneratedMessageCount; // increment internal message count
259  __MOUT__ << "Console is alive and waiting... (if no messages, next "
260  "heartbeat is in approximately two minutes)"
261  << std::endl;
262  }
263  else if(heartbeatCount % (60 * 30) == 59) // approx every hour
264  {
265  ++selfGeneratedMessageCount; // increment internal message count
266  __MOUT__ << "Console is alive and waiting a long time... (if no "
267  "messages, next heartbeat is in approximately one hour)"
268  << std::endl;
269  }
270 
271  ++heartbeatCount;
272  }
273 
274  // if nothing received for 2 minutes seconds, then something is wrong with Console
275  // configuration after 5 seconds there is a self-send. Which will at least
276  // confirm configuration. OR if 5 generated messages and never cleared.. then
277  // the forwarding is not working.
278  if(i == 120 || selfGeneratedMessageCount == 5)
279  {
280  __COUTV__(i);
281  __COUTV__(selfGeneratedMessageCount);
282  __COUT__ << "No messages received at Console Supervisor. Exiting Console "
283  "messageFacilityReceiverWorkLoop"
284  << std::endl;
285  break; // assume something wrong, and break loop
286  }
287  }
288 
289 } // end messageFacilityReceiverWorkLoop()
290 catch(const std::runtime_error& e)
291 {
292  __COUT_ERR__ << "Error caught at Console Supervisor thread: " << e.what() << __E__;
293 }
294 catch(...)
295 {
296  __COUT_ERR__ << "Unknown error caught at Console Supervisor thread." << __E__;
297 }
298 
299 //========================================================================================================================
300 void ConsoleSupervisor::defaultPage(xgi::Input* in, xgi::Output* out)
301 {
302  __SUP_COUT__ << "ApplicationDescriptor LID="
303  << getApplicationDescriptor()->getLocalId() << std::endl;
304  *out << "<!DOCTYPE HTML><html lang='en'><frameset col='100%' row='100%'><frame "
305  "src='/WebPath/html/Console.html?urn="
306  << getApplicationDescriptor()->getLocalId() << "'></frameset></html>";
307 } // end defaultPage()
308 
309 //========================================================================================================================
310 // forceSupervisorPropertyValues
311 // override to force supervisor property values (and ignore user settings)
312 void ConsoleSupervisor::forceSupervisorPropertyValues()
313 {
314  CorePropertySupervisorBase::setSupervisorProperty(
315  CorePropertySupervisorBase::SUPERVISOR_PROPERTIES.AutomatedRequestTypes,
316  "GetConsoleMsgs");
317  // CorePropertySupervisorBase::setSupervisorProperty(CorePropertySupervisorBase::SUPERVISOR_PROPERTIES.NeedUsernameRequestTypes,
318  // "SaveUserPreferences | LoadUserPreferences");
319 } // end forceSupervisorPropertyValues()
320 
321 //========================================================================================================================
322 // Request
323 // Handles Web Interface requests to Console supervisor.
324 // Does not refresh cookie for automatic update checks.
325 void ConsoleSupervisor::request(const std::string& requestType,
326  cgicc::Cgicc& cgiIn,
327  HttpXmlDocument& xmlOut,
328  const WebUsers::RequestUserInfo& userInfo)
329 {
330  //__SUP_COUT__ << "requestType " << requestType << std::endl;
331 
332  // Commands:
333  // GetConsoleMsgs
334  // SaveUserPreferences
335  // LoadUserPreferences
336 
337  // Note: to report to logbook admin status use
338  // xmlOut.addTextElementToData(XML_ADMIN_STATUS,refreshTempStr_);
339 
340  if(requestType == "GetConsoleMsgs")
341  {
342  // lindex of -1 means first time and user just gets update lcount and lindex
343  std::string lastUpdateCountStr = CgiDataUtilities::postData(cgiIn, "lcount");
344 
345  if(lastUpdateCountStr == "")
346  {
347  __SUP_COUT_ERR__ << "Invalid Parameters! lastUpdateCount="
348  << lastUpdateCountStr << std::endl;
349  xmlOut.addTextElementToData("Error",
350  "Error - Invalid parameters for GetConsoleMsgs.");
351  return;
352  }
353 
354  size_t lastUpdateCount = std::stoull(lastUpdateCountStr);
355 
356  // __SUP_COUT__ << "lastUpdateCount=" << lastUpdateCount << std::endl;
357 
358  insertMessageRefresh(&xmlOut, lastUpdateCount);
359  }
360  else if(requestType == "SaveUserPreferences")
361  {
362  int colorIndex = CgiDataUtilities::postDataAsInt(cgiIn, "colorIndex");
363  int showSideBar = CgiDataUtilities::postDataAsInt(cgiIn, "showSideBar");
364  int noWrap = CgiDataUtilities::postDataAsInt(cgiIn, "noWrap");
365  int messageOnly = CgiDataUtilities::postDataAsInt(cgiIn, "messageOnly");
366  int hideLineNumers = CgiDataUtilities::postDataAsInt(cgiIn, "hideLineNumers");
367 
368  __SUP_COUT__ << "requestType " << requestType << std::endl;
369  __SUP_COUT__ << "colorIndex: " << colorIndex << std::endl;
370  __SUP_COUT__ << "showSideBar: " << showSideBar << std::endl;
371  __SUP_COUT__ << "noWrap: " << noWrap << std::endl;
372  __SUP_COUT__ << "messageOnly: " << messageOnly << std::endl;
373  __SUP_COUT__ << "hideLineNumers: " << hideLineNumers << std::endl;
374 
375  if(userInfo.username_ == "") // should never happen?
376  {
377  __SUP_COUT_ERR__ << "Invalid user found! user=" << userInfo.username_
378  << std::endl;
379  xmlOut.addTextElementToData("Error",
380  "Error - InvauserInfo.username_user found.");
381  return;
382  }
383 
384  std::string fn = (std::string)USER_CONSOLE_PREF_PATH + userInfo.username_ + "." +
385  (std::string)USERS_PREFERENCES_FILETYPE;
386 
387  __SUP_COUT__ << "Save preferences: " << fn << std::endl;
388  FILE* fp = fopen(fn.c_str(), "w");
389  if(!fp)
390  {
391  __SS__;
392  __THROW__(ss.str() + "Could not open file: " + fn);
393  }
394  fprintf(fp, "colorIndex %d\n", colorIndex);
395  fprintf(fp, "showSideBar %d\n", showSideBar);
396  fprintf(fp, "noWrap %d\n", noWrap);
397  fprintf(fp, "messageOnly %d\n", messageOnly);
398  fprintf(fp, "hideLineNumers %d\n", hideLineNumers);
399  fclose(fp);
400  }
401  else if(requestType == "LoadUserPreferences")
402  {
403  __SUP_COUT__ << "requestType " << requestType << std::endl;
404 
405  unsigned int colorIndex, showSideBar, noWrap, messageOnly, hideLineNumers;
406 
407  if(userInfo.username_ == "") // should never happen?
408  {
409  __SUP_COUT_ERR__ << "Invalid user found! user=" << userInfo.username_
410  << std::endl;
411  xmlOut.addTextElementToData("Error", "Error - Invalid user found.");
412  return;
413  }
414 
415  std::string fn = (std::string)USER_CONSOLE_PREF_PATH + userInfo.username_ + "." +
416  (std::string)USERS_PREFERENCES_FILETYPE;
417 
418  __SUP_COUT__ << "Load preferences: " << fn << std::endl;
419 
420  FILE* fp = fopen(fn.c_str(), "r");
421  if(!fp)
422  {
423  // return defaults
424  __SUP_COUT__ << "Returning defaults." << std::endl;
425  xmlOut.addTextElementToData("colorIndex", "0");
426  xmlOut.addTextElementToData("showSideBar", "0");
427  xmlOut.addTextElementToData("noWrap", "1");
428  xmlOut.addTextElementToData("messageOnly", "0");
429  xmlOut.addTextElementToData("hideLineNumers", "1");
430  return;
431  }
432  fscanf(fp, "%*s %u", &colorIndex);
433  fscanf(fp, "%*s %u", &showSideBar);
434  fscanf(fp, "%*s %u", &noWrap);
435  fscanf(fp, "%*s %u", &messageOnly);
436  fscanf(fp, "%*s %u", &hideLineNumers);
437  fclose(fp);
438  __SUP_COUT__ << "colorIndex: " << colorIndex << std::endl;
439  __SUP_COUT__ << "showSideBar: " << showSideBar << std::endl;
440  __SUP_COUT__ << "noWrap: " << noWrap << std::endl;
441  __SUP_COUT__ << "messageOnly: " << messageOnly << std::endl;
442  __SUP_COUT__ << "hideLineNumers: " << hideLineNumers << std::endl;
443 
444  char tmpStr[20];
445  sprintf(tmpStr, "%u", colorIndex);
446  xmlOut.addTextElementToData("colorIndex", tmpStr);
447  sprintf(tmpStr, "%u", showSideBar);
448  xmlOut.addTextElementToData("showSideBar", tmpStr);
449  sprintf(tmpStr, "%u", noWrap);
450  xmlOut.addTextElementToData("noWrap", tmpStr);
451  sprintf(tmpStr, "%u", messageOnly);
452  xmlOut.addTextElementToData("messageOnly", tmpStr);
453  sprintf(tmpStr, "%u", hideLineNumers);
454  xmlOut.addTextElementToData("hideLineNumers", tmpStr);
455  }
456  else
457  {
458  __SUP_SS__ << "requestType Request, " << requestType << ", not recognized."
459  << __E__;
460  __SUP_SS_THROW__;
461  }
462 } // end request()
463 
464 //========================================================================================================================
465 // ConsoleSupervisor::insertMessageRefresh()
466 // if lastUpdateClock is current, return nothing
467 // else return new messages
468 // (note: lastUpdateIndex==(unsigned int)-1 first time and returns as much as possible//
469 // nothing but lastUpdateClock)
470 //
471 // format of xml:
472 //
473 // <last_update_count/>
474 // <last_update_index/>
475 // <messages>
476 // <message_FIELDNAME*/>
477 //"Level"
478 //"Label"
479 //"Source"
480 //"Msg"
481 //"Time"
482 //"Count"
483 // </messages>
484 //
485 // NOTE: Uses std::mutex to avoid conflict with writing thread. (this is the reading
486 // thread)
487 void ConsoleSupervisor::insertMessageRefresh(HttpXmlDocument* xmlOut,
488  const size_t lastUpdateCount)
489 {
490  //__SUP_COUT__ << std::endl;
491 
492  if(messages_.size() == 0)
493  return;
494 
495  // validate lastUpdateCount
496  if(lastUpdateCount > messages_.back().getCount() && lastUpdateCount != (size_t)-1)
497  {
498  __SS__ << "Invalid lastUpdateCount: " << lastUpdateCount
499  << " messagesArray size = " << messages_.back().getCount() << std::endl;
500  __SS_THROW__;
501  }
502 
503  // lockout the messages array for the remainder of the scope
504  // this guarantees the reading thread can safely access the messages
505  std::lock_guard<std::mutex> lock(messageMutex_);
506 
507  xmlOut->addTextElementToData("last_update_count",
508  std::to_string(messages_.back().getCount()));
509 
510  refreshParent_ = xmlOut->addTextElementToData("messages", "");
511 
512  bool requestOutOfSync = false;
513  std::string requestOutOfSyncMsg;
514 
515  size_t refreshReadPointer = 0;
516  if(lastUpdateCount != (size_t)-1)
517  {
518  while(refreshReadPointer < messages_.size() &&
519  messages_[refreshReadPointer].getCount() <= lastUpdateCount)
520  {
521  ++refreshReadPointer;
522  }
523  }
524 
525  if(refreshReadPointer >= messages_.size())
526  return;
527 
528  if(messages_.size() - refreshReadPointer > 250)
529  {
530  __SUP_COUT__ << "Only sending latest 250 messages!";
531 
532  auto oldrrp = refreshReadPointer;
533  refreshReadPointer = messages_.size() - 250;
534 
535  // generate special message to indicate failed socket
536  __SS__ << "Skipping " << (refreshReadPointer - oldrrp)
537  << " messages because the web console has fallen behind!" << std::endl;
538  __COUT__ << ss.str();
539 
540  ConsoleMessageStruct msg(CONSOLE_SPECIAL_WARNING + ss.str(), lastUpdateCount);
541  auto it = messages_.begin();
542  std::advance(it, refreshReadPointer + 1);
543  messages_.insert(it, msg);
544  }
545 
546  // output oldest to new
547  for(; refreshReadPointer < messages_.size(); ++refreshReadPointer)
548  {
549  auto msg = messages_[refreshReadPointer];
550  if(msg.getCount() < lastUpdateCount)
551  {
552  if(!requestOutOfSync) // record out of sync message once only
553  {
554  requestOutOfSync = true;
555  __SS__ << "Request is out of sync! Message count should be more recent "
556  "than update clock! "
557  << msg.getCount() << " < " << lastUpdateCount << std::endl;
558  requestOutOfSyncMsg = ss.str();
559  }
560  // assume these messages are new (due to a system restart)
561  // continue;
562  }
563 
564  // for all fields, give value
565  for(auto& field : msg.fields)
566  {
567  if(field.second.fieldName == "Source")
568  continue; // skip, not userful
569  if(field.second.fieldName == "SourceID")
570  continue; // skip, not userful
571 
572  xmlOut->addTextElementToParent("message_" + field.second.fieldName,
573  field.second.fieldValue,
574  refreshParent_);
575  }
576 
577  // give timestamp also
578  xmlOut->addTextElementToParent("message_Time", msg.getTime(), refreshParent_);
579  // give clock also
580  xmlOut->addTextElementToParent(
581  "message_Count", std::to_string(msg.getCount()), refreshParent_);
582  }
583 
584  if(requestOutOfSync) // if request was out of sync, show message
585  __SUP_COUT__ << requestOutOfSyncMsg;
586 }