1 #include "otsdaq-utilities/Console/ConsoleSupervisor.h"
2 #include <xdaq/NamespaceURI.h>
3 #include "otsdaq/CgiDataUtilities/CgiDataUtilities.h"
4 #include "otsdaq/Macros/CoutMacros.h"
5 #include "otsdaq/MessageFacility/MessageFacility.h"
6 #include "otsdaq/NetworkUtilities/ReceiverSocket.h"
7 #include "otsdaq/XmlUtilities/HttpXmlDocument.h"
24 #define USER_CONSOLE_PREF_PATH \
25 std::string(__ENV__("SERVICE_DATA_PATH")) + "/ConsolePreferences/"
26 #define USERS_PREFERENCES_FILETYPE "pref"
28 #define QUIET_CFG_FILE \
29 std::string(__ENV__("USER_DATA")) + \
30 "/MessageFacilityConfigurations/" \
33 #define CONSOLE_SPECIAL_ERROR \
34 std::string("|30-Aug-2019 15:30:17 CDT|0|||Error|Console||-1||ConsoleSupervisor|") + \
35 std::string(__FILE__) + std::string("|") + std::to_string(__LINE__) + \
37 #define CONSOLE_SPECIAL_WARNING \
39 "|30-Aug-2019 15:30:17 CDT|0|||Warning|Console||-1||ConsoleSupervisor|") + \
40 std::string(__FILE__) + std::string("|") + std::to_string(__LINE__) + \
44 #define __MF_SUBJECT__ "Console"
47 ConsoleSupervisor::ConsoleSupervisor(xdaq::ApplicationStub* stub)
48 : CoreSupervisorBase(stub), messageCount_(0), maxMessageCount_(100000)
50 __SUP_COUT__ <<
"Constructor started." << __E__;
52 INIT_MF(
"ConsoleSupervisor");
55 mkdir(((std::string)USER_CONSOLE_PREF_PATH).c_str(), 0755);
59 __SUP_COUT__ <<
"Constructor complete." << __E__;
63 ConsoleSupervisor::~ConsoleSupervisor(
void) { destroy(); }
65 void ConsoleSupervisor::init(
void)
70 ConsoleSupervisor::messageFacilityReceiverWorkLoop(cs);
77 void ConsoleSupervisor::destroy(
void)
86 void ConsoleSupervisor::messageFacilityReceiverWorkLoop(
ConsoleSupervisor* cs)
try
88 __COUT__ << std::endl;
90 std::string configFile = QUIET_CFG_FILE;
91 FILE* fp = fopen(configFile.c_str(),
"r");
94 __SS__ <<
"File with port info could not be loaded: " << QUIET_CFG_FILE
96 __COUT__ <<
"\n" << ss.str();
103 sscanf(tmp,
"%*s %d", &myport);
107 sscanf(tmp,
"%*s %s", myip);
110 ReceiverSocket rsock(myip, myport);
119 std::lock_guard<std::mutex> lock(cs->messageMutex_);
125 __SS__ <<
"FATAL Console error. Could not initialize socket on port "
127 <<
". Perhaps the port is already in use? Check for multiple stale "
128 "instances of otsdaq processes, or notify admins."
129 <<
" Multiple instances of otsdaq on the same node should be "
130 "possible, but port numbers must be unique."
136 __SS__ <<
"FATAL Console error. Could not initialize socket on port " << myport
137 <<
". Perhaps it is already in use? Exiting Console receive loop."
139 __COUT__ << ss.str();
141 cs->messages_.emplace_back(CONSOLE_SPECIAL_ERROR + ss.str(), cs->messageCount_++);
143 if(cs->messages_.size() > cs->maxMessageCount_)
145 cs->messages_.erase(cs->messages_.begin());
153 int heartbeatCount = 0;
154 int selfGeneratedMessageCount = 0;
156 std::map<unsigned int, unsigned int>
157 sourceLastSequenceID;
160 long long newSourceId;
161 unsigned int newSequenceId;
164 __MCOUT__(
"DEBUG messages look like this." << __E__);
173 buffer, 1 , 0 ,
false ) !=
178 __COUT__ <<
"Console has first message." << __E__;
182 __MOUT_INFO__ <<
"INFO messages look like this." << __E__;
183 __MOUT_WARN__ <<
"WARNING messages look like this." << __E__;
184 __MOUT_ERR__ <<
"ERROR messages look like this." << __E__;
196 if(selfGeneratedMessageCount)
197 --selfGeneratedMessageCount;
205 std::lock_guard<std::mutex> lock(cs->messageMutex_);
207 cs->messages_.emplace_back(buffer, cs->messageCount_++);
210 newSourceId = cs->messages_.back().getSourceIDAsNumber();
211 newSequenceId = cs->messages_.back().getSequenceIDAsNumber();
216 if(newSourceId != -1 &&
217 sourceLastSequenceID.find(newSourceId) !=
218 sourceLastSequenceID.end() &&
219 ((newSequenceId == 0 && sourceLastSequenceID[newSourceId] !=
222 sourceLastSequenceID[newSourceId] + 1))
225 __SS__ <<
"Missed packets from " << cs->messages_.back().getSource()
226 <<
"! Sequence IDs " << sourceLastSequenceID[newSourceId] <<
" to "
227 << newSequenceId <<
"." << std::endl;
228 std::cout << ss.str();
231 cs->messages_.emplace_back(CONSOLE_SPECIAL_WARNING + ss.str(),
232 cs->messageCount_++);
236 sourceLastSequenceID[newSourceId] = newSequenceId;
238 while(cs->messages_.size() > 0 && cs->messages_.size() > cs->maxMessageCount_)
240 cs->messages_.erase(cs->messages_.begin());
255 heartbeatCount % 60 == 59
258 ++selfGeneratedMessageCount;
259 __MOUT__ <<
"Console is alive and waiting... (if no messages, next "
260 "heartbeat is in approximately two minutes)"
263 else if(heartbeatCount % (60 * 30) == 59)
265 ++selfGeneratedMessageCount;
266 __MOUT__ <<
"Console is alive and waiting a long time... (if no "
267 "messages, next heartbeat is in approximately one hour)"
278 if(i == 120 || selfGeneratedMessageCount == 5)
281 __COUTV__(selfGeneratedMessageCount);
282 __COUT__ <<
"No messages received at Console Supervisor. Exiting Console "
283 "messageFacilityReceiverWorkLoop"
290 catch(
const std::runtime_error& e)
292 __COUT_ERR__ <<
"Error caught at Console Supervisor thread: " << e.what() << __E__;
296 __COUT_ERR__ <<
"Unknown error caught at Console Supervisor thread." << __E__;
300 void ConsoleSupervisor::defaultPage(xgi::Input* in, xgi::Output* out)
302 __SUP_COUT__ <<
"ApplicationDescriptor LID="
303 << getApplicationDescriptor()->getLocalId() << std::endl;
304 *out <<
"<!DOCTYPE HTML><html lang='en'><frameset col='100%' row='100%'><frame "
305 "src='/WebPath/html/Console.html?urn="
306 << getApplicationDescriptor()->getLocalId() <<
"'></frameset></html>";
312 void ConsoleSupervisor::forceSupervisorPropertyValues()
314 CorePropertySupervisorBase::setSupervisorProperty(
315 CorePropertySupervisorBase::SUPERVISOR_PROPERTIES.AutomatedRequestTypes,
325 void ConsoleSupervisor::request(
const std::string& requestType,
327 HttpXmlDocument& xmlOut,
328 const WebUsers::RequestUserInfo& userInfo)
340 if(requestType ==
"GetConsoleMsgs")
343 std::string lastUpdateCountStr = CgiDataUtilities::postData(cgiIn,
"lcount");
345 if(lastUpdateCountStr ==
"")
347 __SUP_COUT_ERR__ <<
"Invalid Parameters! lastUpdateCount="
348 << lastUpdateCountStr << std::endl;
349 xmlOut.addTextElementToData(
"Error",
350 "Error - Invalid parameters for GetConsoleMsgs.");
354 size_t lastUpdateCount = std::stoull(lastUpdateCountStr);
358 insertMessageRefresh(&xmlOut, lastUpdateCount);
360 else if(requestType ==
"SaveUserPreferences")
362 int colorIndex = CgiDataUtilities::postDataAsInt(cgiIn,
"colorIndex");
363 int showSideBar = CgiDataUtilities::postDataAsInt(cgiIn,
"showSideBar");
364 int noWrap = CgiDataUtilities::postDataAsInt(cgiIn,
"noWrap");
365 int messageOnly = CgiDataUtilities::postDataAsInt(cgiIn,
"messageOnly");
366 int hideLineNumers = CgiDataUtilities::postDataAsInt(cgiIn,
"hideLineNumers");
368 __SUP_COUT__ <<
"requestType " << requestType << std::endl;
369 __SUP_COUT__ <<
"colorIndex: " << colorIndex << std::endl;
370 __SUP_COUT__ <<
"showSideBar: " << showSideBar << std::endl;
371 __SUP_COUT__ <<
"noWrap: " << noWrap << std::endl;
372 __SUP_COUT__ <<
"messageOnly: " << messageOnly << std::endl;
373 __SUP_COUT__ <<
"hideLineNumers: " << hideLineNumers << std::endl;
375 if(userInfo.username_ ==
"")
377 __SUP_COUT_ERR__ <<
"Invalid user found! user=" << userInfo.username_
379 xmlOut.addTextElementToData(
"Error",
380 "Error - InvauserInfo.username_user found.");
384 std::string fn = (std::string)USER_CONSOLE_PREF_PATH + userInfo.username_ +
"." +
385 (std::string)USERS_PREFERENCES_FILETYPE;
387 __SUP_COUT__ <<
"Save preferences: " << fn << std::endl;
388 FILE* fp = fopen(fn.c_str(),
"w");
392 __THROW__(ss.str() +
"Could not open file: " + fn);
394 fprintf(fp,
"colorIndex %d\n", colorIndex);
395 fprintf(fp,
"showSideBar %d\n", showSideBar);
396 fprintf(fp,
"noWrap %d\n", noWrap);
397 fprintf(fp,
"messageOnly %d\n", messageOnly);
398 fprintf(fp,
"hideLineNumers %d\n", hideLineNumers);
401 else if(requestType ==
"LoadUserPreferences")
403 __SUP_COUT__ <<
"requestType " << requestType << std::endl;
405 unsigned int colorIndex, showSideBar, noWrap, messageOnly, hideLineNumers;
407 if(userInfo.username_ ==
"")
409 __SUP_COUT_ERR__ <<
"Invalid user found! user=" << userInfo.username_
411 xmlOut.addTextElementToData(
"Error",
"Error - Invalid user found.");
415 std::string fn = (std::string)USER_CONSOLE_PREF_PATH + userInfo.username_ +
"." +
416 (std::string)USERS_PREFERENCES_FILETYPE;
418 __SUP_COUT__ <<
"Load preferences: " << fn << std::endl;
420 FILE* fp = fopen(fn.c_str(),
"r");
424 __SUP_COUT__ <<
"Returning defaults." << std::endl;
425 xmlOut.addTextElementToData(
"colorIndex",
"0");
426 xmlOut.addTextElementToData(
"showSideBar",
"0");
427 xmlOut.addTextElementToData(
"noWrap",
"1");
428 xmlOut.addTextElementToData(
"messageOnly",
"0");
429 xmlOut.addTextElementToData(
"hideLineNumers",
"1");
432 fscanf(fp,
"%*s %u", &colorIndex);
433 fscanf(fp,
"%*s %u", &showSideBar);
434 fscanf(fp,
"%*s %u", &noWrap);
435 fscanf(fp,
"%*s %u", &messageOnly);
436 fscanf(fp,
"%*s %u", &hideLineNumers);
438 __SUP_COUT__ <<
"colorIndex: " << colorIndex << std::endl;
439 __SUP_COUT__ <<
"showSideBar: " << showSideBar << std::endl;
440 __SUP_COUT__ <<
"noWrap: " << noWrap << std::endl;
441 __SUP_COUT__ <<
"messageOnly: " << messageOnly << std::endl;
442 __SUP_COUT__ <<
"hideLineNumers: " << hideLineNumers << std::endl;
445 sprintf(tmpStr,
"%u", colorIndex);
446 xmlOut.addTextElementToData(
"colorIndex", tmpStr);
447 sprintf(tmpStr,
"%u", showSideBar);
448 xmlOut.addTextElementToData(
"showSideBar", tmpStr);
449 sprintf(tmpStr,
"%u", noWrap);
450 xmlOut.addTextElementToData(
"noWrap", tmpStr);
451 sprintf(tmpStr,
"%u", messageOnly);
452 xmlOut.addTextElementToData(
"messageOnly", tmpStr);
453 sprintf(tmpStr,
"%u", hideLineNumers);
454 xmlOut.addTextElementToData(
"hideLineNumers", tmpStr);
458 __SUP_SS__ <<
"requestType Request, " << requestType <<
", not recognized."
487 void ConsoleSupervisor::insertMessageRefresh(HttpXmlDocument* xmlOut,
488 const size_t lastUpdateCount)
492 if(messages_.size() == 0)
496 if(lastUpdateCount > messages_.back().getCount() && lastUpdateCount != (size_t)-1)
498 __SS__ <<
"Invalid lastUpdateCount: " << lastUpdateCount
499 <<
" messagesArray size = " << messages_.back().getCount() << std::endl;
505 std::lock_guard<std::mutex> lock(messageMutex_);
507 xmlOut->addTextElementToData(
"last_update_count",
508 std::to_string(messages_.back().getCount()));
510 refreshParent_ = xmlOut->addTextElementToData(
"messages",
"");
512 bool requestOutOfSync =
false;
513 std::string requestOutOfSyncMsg;
515 size_t refreshReadPointer = 0;
516 if(lastUpdateCount != (
size_t)-1)
518 while(refreshReadPointer < messages_.size() &&
519 messages_[refreshReadPointer].getCount() <= lastUpdateCount)
521 ++refreshReadPointer;
525 if(refreshReadPointer >= messages_.size())
528 if(messages_.size() - refreshReadPointer > 250)
530 __SUP_COUT__ <<
"Only sending latest 250 messages!";
532 auto oldrrp = refreshReadPointer;
533 refreshReadPointer = messages_.size() - 250;
536 __SS__ <<
"Skipping " << (refreshReadPointer - oldrrp)
537 <<
" messages because the web console has fallen behind!" << std::endl;
538 __COUT__ << ss.str();
540 ConsoleMessageStruct msg(CONSOLE_SPECIAL_WARNING + ss.str(), lastUpdateCount);
541 auto it = messages_.begin();
542 std::advance(it, refreshReadPointer + 1);
543 messages_.insert(it, msg);
547 for(; refreshReadPointer < messages_.size(); ++refreshReadPointer)
549 auto msg = messages_[refreshReadPointer];
550 if(msg.getCount() < lastUpdateCount)
552 if(!requestOutOfSync)
554 requestOutOfSync =
true;
555 __SS__ <<
"Request is out of sync! Message count should be more recent "
556 "than update clock! "
557 << msg.getCount() <<
" < " << lastUpdateCount << std::endl;
558 requestOutOfSyncMsg = ss.str();
565 for(
auto& field : msg.fields)
567 if(field.second.fieldName ==
"Source")
569 if(field.second.fieldName ==
"SourceID")
572 xmlOut->addTextElementToParent(
"message_" + field.second.fieldName,
573 field.second.fieldValue,
578 xmlOut->addTextElementToParent(
"message_Time", msg.getTime(), refreshParent_);
580 xmlOut->addTextElementToParent(
581 "message_Count", std::to_string(msg.getCount()), refreshParent_);
585 __SUP_COUT__ << requestOutOfSyncMsg;