Merge pull request #6444 from DIRACGridBot/cherry-pick-2-b879c174d-integration

[sweep:integration] fix: remove warning messages when initializing ElasticDB parameters

Author: fstagni

integration_tests.py

@@ -621,6 +621,8 @@ def _make_config(modules, flags, release_var, editable):
         "DB_HOST": DB_HOST,
         "DB_PORT": DB_PORT,
         # ElasticSearch settings
+        "NoSQLDB_USER": "elastic",
+        "NoSQLDB_PASSWORD": "changeme",
         "NoSQLDB_HOST": "elasticsearch",
         "NoSQLDB_PORT": "9200",
         # Hostnames

src/DIRAC/ConfigurationSystem/Client/Utilities.py

@@ -369,15 +369,6 @@ def getDBParameters(fullname):
     """Retrieve Database parameters from CS
 
     :param str fullname: should be of the form <System>/<DBname>
-           defaultHost is the host to return if the option is not found in the CS.
-           Not used as the method will fail if it cannot be found
-           defaultPort is the port to return if the option is not found in the CS
-           defaultUser is the user to return if the option is not found in the CS.
-           Not usePassword is the password to return if the option is not found in the CS.
-           Not used as the method will fail if it cannot be found
-           defaultDB is the db to return if the option is not found in the CS.
-           Not used as the method will fail if it cannot be found
-           defaultQueueSize is the QueueSize to return if the option is not found in the CS
 
     :return: S_OK(dict)/S_ERROR() - dictionary with the keys: 'host', 'port', 'user', 'password',
                                     'db' and 'queueSize'
@@ -386,6 +377,25 @@ def getDBParameters(fullname):
     cs_path = getDatabaseSection(fullname)
     parameters = {}
 
+    # Check mandatory parameters first: Password, User, Host and DBName
+    result = gConfig.getOption(cs_path + "/Password")
+    if not result["OK"]:
+        # No individual password found, try at the common place
+        result = gConfig.getOption("/Systems/Databases/Password")
+        if not result["OK"]:
+            return S_ERROR("Failed to get the configuration parameter: Password")
+    dbPass = result["Value"]
+    parameters["Password"] = dbPass
+
+    result = gConfig.getOption(cs_path + "/User")
+    if not result["OK"]:
+        # No individual user name found, try at the common place
+        result = gConfig.getOption("/Systems/Databases/User")
+        if not result["OK"]:
+            return S_ERROR("Failed to get the configuration parameter: User")
+    dbUser = result["Value"]
+    parameters["User"] = dbUser
+
     result = gConfig.getOption(cs_path + "/Host")
     if not result["OK"]:
         # No host name found, try at the common place
@@ -401,6 +411,13 @@ def getDBParameters(fullname):
             dbHost = "localhost"
     parameters["Host"] = dbHost
 
+    result = gConfig.getOption(cs_path + "/DBName")
+    if not result["OK"]:
+        return S_ERROR("Failed to get the configuration parameter: DBName")
+    dbName = result["Value"]
+    parameters["DBName"] = dbName
+
+    # Check optional parameters: Port
     # Mysql standard
     dbPort = 3306
     result = gConfig.getOption(cs_path + "/Port")
@@ -413,30 +430,6 @@ def getDBParameters(fullname):
         dbPort = int(result["Value"])
     parameters["Port"] = dbPort
 
-    result = gConfig.getOption(cs_path + "/User")
-    if not result["OK"]:
-        # No individual user name found, try at the common place
-        result = gConfig.getOption("/Systems/Databases/User")
-        if not result["OK"]:
-            return S_ERROR("Failed to get the configuration parameter: User")
-    dbUser = result["Value"]
-    parameters["User"] = dbUser
-
-    result = gConfig.getOption(cs_path + "/Password")
-    if not result["OK"]:
-        # No individual password found, try at the common place
-        result = gConfig.getOption("/Systems/Databases/Password")
-        if not result["OK"]:
-            return S_ERROR("Failed to get the configuration parameter: Password")
-    dbPass = result["Value"]
-    parameters["Password"] = dbPass
-
-    result = gConfig.getOption(cs_path + "/DBName")
-    if not result["OK"]:
-        return S_ERROR("Failed to get the configuration parameter: DBName")
-    dbName = result["Value"]
-    parameters["DBName"] = dbName
-
     return S_OK(parameters)
 
 
@@ -451,6 +444,26 @@ def getElasticDBParameters(fullname):
     cs_path = getDatabaseSection(fullname)
     parameters = {}
 
+    # Check mandatory parameters first: Password, User
+    result = gConfig.getOption(cs_path + "/Password")
+    if not result["OK"]:
+        # No individual password found, try at the common place
+        result = gConfig.getOption("/Systems/NoSQLDatabases/Password")
+        if not result["OK"]:
+            return S_ERROR("Failed to get the configuration parameter: Password.")
+    dbPass = result["Value"]
+    parameters["Password"] = dbPass
+
+    result = gConfig.getOption(cs_path + "/User")
+    if not result["OK"]:
+        # No individual user name found, try at the common place
+        result = gConfig.getOption("/Systems/NoSQLDatabases/User")
+        if not result["OK"]:
+            return S_ERROR("Failed to get the configuration parameter: User.")
+    dbUser = result["Value"]
+    parameters["User"] = dbUser
+
+    # Check optional parameters: Host, Port, SSL, CRT, ca_certs, client_key, client_cert
     result = gConfig.getOption(cs_path + "/Host")
     if not result["OK"]:
         # No host name found, try at the common place
@@ -484,36 +497,6 @@ def getElasticDBParameters(fullname):
         dbPort = int(result["Value"])
     parameters["Port"] = dbPort
 
-    result = gConfig.getOption(cs_path + "/User")
-    if not result["OK"]:
-        # No individual user name found, try at the common place
-        result = gConfig.getOption("/Systems/NoSQLDatabases/User")
-        if not result["OK"]:
-            gLogger.warn(
-                "Failed to get the configuration parameter: User. Assuming no user/password is provided/needed"
-            )
-            dbUser = None
-        else:
-            dbUser = result["Value"]
-    else:
-        dbUser = result["Value"]
-    parameters["User"] = dbUser
-
-    result = gConfig.getOption(cs_path + "/Password")
-    if not result["OK"]:
-        # No individual password found, try at the common place
-        result = gConfig.getOption("/Systems/NoSQLDatabases/Password")
-        if not result["OK"]:
-            gLogger.warn(
-                "Failed to get the configuration parameter: Password. Assuming no user/password is provided/needed"
-            )
-            dbPass = None
-        else:
-            dbPass = result["Value"]
-    else:
-        dbPass = result["Value"]
-    parameters["Password"] = dbPass
-
     result = gConfig.getOption(cs_path + "/SSL")
     if not result["OK"]:
         # No SSL option found, try at the common place

src/DIRAC/Core/Base/ElasticDB.py

@@ -27,9 +27,8 @@ def __init__(self, dbname, fullName, indexPrefix="", parentLogger=None):
         dbParameters = result["Value"]
         self._dbHost = dbParameters["Host"]
         self._dbPort = dbParameters["Port"]
-        # we can have db which does not have any authentication...
-        self.__user = dbParameters.get("User", "")
-        self.__dbPassword = dbParameters.get("Password", "")
+        self.__user = dbParameters["User"]
+        self.__dbPassword = dbParameters["Password"]
         self.__useSSL = dbParameters.get("SSL", True)
         self.__useCRT = dbParameters.get("CRT", True)
         self.__ca_certs = dbParameters.get("ca_certs", None)

src/DIRAC/Core/Utilities/ElasticSearchDB.py

@@ -137,22 +137,23 @@ def __init__(
         if user and password:
             sLog.debug("Specified username and password")
             if port:
-                self.__url = "https://%s:%s@%s:%d" % (user, password, host, port)
+                self.__url = f"://{user}:{password}@{host}:{port}"
             else:
-                self.__url = f"https://{user}:{password}@{host}"
+                self.__url = f"://{user}:{password}@{host}"
         else:
             sLog.debug("Username and password not specified")
             if port:
-                self.__url = "http://%s:%d" % (host, port)
+                self.__url = f"://{host}:{port}"
             else:
-                self.__url = "http://%s" % host
+                self.__url = f"://{host}"
 
         if port:
             sLog.verbose(f"Connecting to {host}:{port}, useSSL = {useSSL}")
         else:
             sLog.verbose(f"Connecting to {host}, useSSL = {useSSL}")
 
         if useSSL:
+            self.__url = "https" + self.__url
             if ca_certs:
                 casFile = ca_certs
             else:
@@ -169,6 +170,7 @@ def __init__(
                 self.__url, timeout=self.__timeout, use_ssl=True, verify_certs=True, ca_certs=casFile
             )
         elif useCRT:
+            self.__url = "https" + self.__url
             self.client = Elasticsearch(
                 self.__url,
                 timeout=self.__timeout,
@@ -179,6 +181,7 @@ def __init__(
                 client_key=client_key,
             )
         else:
+            self.__url = "http" + self.__url
             self.client = Elasticsearch(self.__url, timeout=self.__timeout)
 
         # Before we use the database we try to connect

src/DIRAC/Core/Utilities/ServerUtils.py

@@ -0,0 +1,24 @@
+"""
+  Provide uniform interface to backend for local and remote clients.return
+
+  There's a pretty big assumption here: that DB and Handler expose the same calls, with identical signatures.
+  This is not always the case.
+"""
+
+
+def getDBOrClient(DB, serverName):
+    """Tries to instantiate the DB object and returns it if we manage to connect to the DB,
+    otherwise returns a Client of the server
+    """
+    from DIRAC import gLogger
+    from DIRAC.Core.Base.Client import Client
+
+    try:
+        database = DB()
+        if database._connected:
+            return database
+    except Exception:
+        pass
+
+    gLogger.info(f"Can not connect to DB will use {serverName}")
+    return Client(url=serverName)

src/DIRAC/MonitoringSystem/Client/ServerUtils.py

@@ -3,23 +3,7 @@
 It always try to insert the records directly. In case of failure the monitoring client is used...
 """
 
-
-def getDBOrClient(DB, serverName):
-    """Tries to instantiate the DB object and returns it if we manage to connect to the DB,
-    otherwise returns a Client of the server
-    """
-    from DIRAC import gLogger
-    from DIRAC.Core.Base.Client import Client
-
-    try:
-        database = DB()
-        if database._connected:
-            return database
-    except Exception:
-        pass
-
-    gLogger.info(f"Can not connect to DB will use {serverName}")
-    return Client(url=serverName)
+from DIRAC.Core.Utilities.ServerUtils import getDBOrClient
 
 
 def getMonitoringDB():

src/DIRAC/WorkloadManagementSystem/Client/ServerUtils.py

@@ -1,28 +1,9 @@
 """
-  Provide uniform interface to backend for local and remote clients.return
-
-  There's a pretty big assumption here: that DB and Handler expose the same calls, with identical signatures.
-  This is not exactly the case for WMS DBs and services.
+This module is used to create an appropriate object which can be used to insert records to the WMS.
+It always try to insert the records directly. In case of failure a WMS client is used...
 """
 
-
-def getDBOrClient(DB, serverName):
-    """Tries to instantiate the DB object
-    and returns it if we manage to connect to the DB,
-    otherwise returns a Client of the server
-    """
-    from DIRAC import gLogger
-    from DIRAC.Core.Base.Client import Client
-
-    try:
-        myDB = DB()
-        if myDB._connected:
-            return myDB
-    except Exception:
-        pass
-
-    gLogger.info("Can not connect to DB will use %s" % serverName)
-    return Client(url=serverName)
+from DIRAC.Core.Utilities.ServerUtils import getDBOrClient
 
 
 def getPilotAgentsDB():

tests/CI/docker-compose.yml

@@ -22,7 +22,7 @@ services:
       - 9200:9200
     env_file: "${ES_VER}.env"
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://localhost:9200"]
+      test: ["CMD", "curl", "-f", "-u", "elastic:changeme", "http://localhost:9200"]
       interval: 5s
       timeout: 2s
       retries: 15

tests/CI/envs/elasticsearch:7.9.1.env

@@ -3,6 +3,8 @@ cluster.routing.allocation.disk.threshold_enabled=true
 cluster.routing.allocation.disk.watermark.flood_stage=200mb
 cluster.routing.allocation.disk.watermark.low=500mb
 cluster.routing.allocation.disk.watermark.high=300mb
+xpack.security.enabled=true
+ELASTIC_PASSWORD="changeme"
 # Elasticsearch allocates 1GB of memory by default. As resources are limited
 # and elasticsearch performance isn't critical in CI, limit this to 256MB
 "ES_JAVA_OPTS=-Xms256m -Xmx256m"