Merge pull request #100 from sy-c/master

v2.9.0

Author: sy-c

CMakeLists.txt

@@ -254,6 +254,7 @@ add_executable(
   src/infoLoggerServer.cxx
   src/InfoLoggerDispatch.cxx
   src/InfoLoggerDispatchBrowser.cxx
+  src/InfoLoggerDispatchStats.cxx
   src/ConfigInfoLoggerServer.cxx  
   src/infoLoggerMessageDecode.c
   src/InfoLoggerMessageHelper.cxx

doc/README.md

@@ -11,6 +11,7 @@ The following executables, presented with the _nicknames_ used below, are part o
   - o2-infologger-admindb or _infoLoggerAdminDB_: to maintain the logging database, i.e. create, archive, clean or destroy the database content.
   - o2-infologger-newdb : helper script for the initial set-up of the logging database, in particular for the definition of access credentials.
   - o2-infologger-tester : a tool to check the logging chain, from injection to DB storage and online subscription.
+  - o2-infologger-alert
 
 The following libraries are also provided, to inject logs into the system:
 
@@ -314,3 +315,13 @@ achieved on CentOS 7 with e.g. (as root):
   - in the infoLoggerD configuration section, set: `msgQueueReset=1` (this is permanent, done on each startup of infoLoggerD, which might not be what you want)
   - when starting infoLoggerD process from the command line (not with the systemctl service), add option: `-o msgQueueReset=1`
   - create a file named [msgQueuePath].reset (by default, msgQueuePath=/tmp/infoLoggerD/infoLoggerD.queue), e.g. `touch /tmp/infoLoggerD/infoLoggerD.queue.reset`. This will reset the queue on next startup (by hand or with e.g. service infoLoggerD restart), and the reset file will also be deleted (which ensures cleanup is done once only).
+
+
+* o2-infologger-alert
+
+  This is a daemon connecting to the stream of online messages and generating alerts (themselves log message + telegraf metric) based on the definition of some logic rules to detect known bad situations possibly requiring intervention. This is intended to raise the awareness of the shift crew to some online operational issues, and trigger a response.
+ 
+* o2-infologger-server statistics
+
+  Messages are indexed by the server, and published as a TCL list on a socket (eg port 6103), to allow categorizing messages and presenting a high-level view of current logging activity.
+  See the configuration parameters to define window size, publish interval, and amount of history kept.

doc/releaseNotes.md

@@ -174,3 +174,11 @@ This file describes the main feature changes for each InfoLogger released versio
 
 # v2.8.3 - 11/06/2025
 - Compilation / security fix. (string handling)
+
+# v2.9.0 - 11/11/2025
+- o2-infologger-server:
+  - improved handling of SQL insert errors, messages dropped after retry.
+  - added indexing of messages to publish stats
+- o2-infologger-alert service
+- o2-infologger-browser:
+  - added some extra startup option, to preconfigure filters

o2-infologger-alert.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=O2 infoLogger alert daemon, generating alerts from log messages.
+After=network.target 
+
+[Service]
+Type=simple
+TimeoutSec=5
+ExecStart=/opt/o2-InfoLogger/bin/o2-infologger-alert -z /etc/o2.d/infologger/infoLogger.cfg
+ExecStop=/usr/bin/kill -s SIGTERM $MAINPID
+GuessMainPID=1
+Restart=always
+
+[Install]
+WantedBy=multi-user.target

src/ConfigInfoLoggerServer.cxx

@@ -38,5 +38,13 @@ void ConfigInfoLoggerServer::readFromConfigFile(ConfigFile& config)
 
   config.getOptionalValue<int>(INFOLOGGER_CONFIG_SECTION_NAME_SERVER ".serverPortTx", serverPortTx);
   config.getOptionalValue<int>(INFOLOGGER_CONFIG_SECTION_NAME_SERVER ".maxClientsTx", maxClientsTx);
+  
+  config.getOptionalValue<int>(INFOLOGGER_CONFIG_SECTION_NAME_SERVER ".statsEnabled", statsEnabled);
+  config.getOptionalValue<int>(INFOLOGGER_CONFIG_SECTION_NAME_SERVER ".statsPort", statsPort);
+  config.getOptionalValue<int>(INFOLOGGER_CONFIG_SECTION_NAME_SERVER ".statsMaxClients", statsMaxClients);
+  config.getOptionalValue<int>(INFOLOGGER_CONFIG_SECTION_NAME_SERVER ".statsPublishInterval", statsPublishInterval);
+  config.getOptionalValue<int>(INFOLOGGER_CONFIG_SECTION_NAME_SERVER ".statsResetInterval", statsResetInterval);
+  config.getOptionalValue<int>(INFOLOGGER_CONFIG_SECTION_NAME_SERVER ".statsHistory", statsHistory);
+  
 }
 

src/ConfigInfoLoggerServer.h

@@ -50,6 +50,14 @@ class ConfigInfoLoggerServer
   // settings for infoBrowser clients
   int serverPortTx = INFOLOGGER_DEFAULT_SERVER_TX_PORT;
   int maxClientsTx = 100;
+
+  // settings for infoLoggerStats clients
+  int statsEnabled = 1; // flag to enable/disable feature
+  int statsPort = INFOLOGGER_DEFAULT_SERVER_STATS_PORT; // TCP/IP port number
+  int statsMaxClients = 5; // max number of clients connections allowed
+  int statsPublishInterval = 5 ; // publish interval time (seconds)
+  int statsResetInterval = 60; // size of the stats window (seconds)
+  int statsHistory = 600; // backlog of stats kept and published (seconds)
 };
 
 #endif // SRC_CONFIGINFOLOGGERSERVER_H_

src/InfoLoggerDispatch.h

@@ -96,5 +96,18 @@ class InfoLoggerDispatchSQL : public InfoLoggerDispatch
   std::unique_ptr<InfoLoggerDispatchSQLImpl> dPtr;
 };
 
+// a class to dispatch stats about online messages
+class InfoLoggerDispatchStatsImpl;
+class InfoLoggerDispatchStats : public InfoLoggerDispatch
+{
+ public:
+  InfoLoggerDispatchStats(ConfigInfoLoggerServer* theConfig, SimpleLog* theLog);
+  ~InfoLoggerDispatchStats();
+  int customMessageProcess(std::shared_ptr<InfoLoggerMessageList> msg);
+  int customLoop();
+
+ private:
+  std::unique_ptr<InfoLoggerDispatchStatsImpl> dPtr;
+};
 #endif
 

src/InfoLoggerDispatchBrowser.cxx

@@ -82,7 +82,8 @@ InfoLoggerDispatchOnlineBrowser::InfoLoggerDispatchOnlineBrowser(ConfigInfoLogge
     throw __LINE__;
   }
   //theLog.info("%s() success\n",__FUNCTION__);
-  
+  theLog->info("Publishing online messages on port %d", theConfig->serverPortTx);
+
   // enable customloop callback
   isReady = true;
 }

src/InfoLoggerDispatchSQL.cxx

@@ -12,6 +12,7 @@
 #include "InfoLoggerDispatch.h"
 #include <mysql.h>
 #include <mysqld_error.h>
+#include <errmsg.h>
 #include "utility.h"
 #include "infoLoggerMessage.h"
 #include <unistd.h>
@@ -61,6 +62,9 @@ class InfoLoggerDispatchSQLImpl
   int commitTimeout = 1000000; // time between commits
   Timer commitTimer;           // timer for transaction
   int commitNumberOfMsg;       // number of messages since last commit
+
+  int numberOfSuccessiveFailures = 0; // count consecutive insert failures
+  int maxNumberOfRetries = 1;         // number of retries allowed
 };
 
 void InfoLoggerDispatchSQLImpl::start()
@@ -384,13 +388,31 @@ int InfoLoggerDispatchSQLImpl::customMessageProcess(std::shared_ptr<InfoLoggerMe
       // Do the insertion
       if (mysql_stmt_execute(stmt)) {
         parent->logError("mysql_stmt_exec() failed: (%d) %s", mysql_errno(db), mysql_error(db));
+	unsigned int err = mysql_errno(db);
         // column too long
-        if (mysql_errno(db) == ER_DATA_TOO_LONG) {
+        if ( err == ER_DATA_TOO_LONG) {
+          return returnDroppedMessage(msg, m);
+        }
+        // column with wrong value
+        if ( err == ER_TRUNCATED_WRONG_VALUE_FOR_FIELD) {
           return returnDroppedMessage(msg, m);
         }
-        // retry with new connection - usually it means server was down
-        disconnectDB();
-        return returnDelayedMessage();
+	// server gone - retry with new connection
+	if (( err == CR_SERVER_LOST ) || ( err == CR_SERVER_GONE_ERROR )) {
+          disconnectDB();
+          return returnDelayedMessage();
+        }
+
+	numberOfSuccessiveFailures++;
+	if (numberOfSuccessiveFailures <= maxNumberOfRetries) {
+          disconnectDB();
+          return returnDelayedMessage();
+	}
+        numberOfSuccessiveFailures = 0;
+
+	// by default: drop message
+        parent->logError("Unhandled error code %d after %d attempts", mysql_errno(db), maxNumberOfRetries);
+	return returnDroppedMessage(msg, m);
       }
 
       insertCount++;