Merge pull request #8005 from fstagni/90_use_containerized_certs_creation

[9.0] use containerized certs creation

Author: fstagni

.github/workflows/integration.yml

@@ -33,10 +33,8 @@ jobs:
         include:
           - TEST_NAME: "MariaDB 11.4"
             ARGS: MYSQL_VER=mariadb:11.4.3
-          - TEST_NAME: "HTTPS and MySQL8"
-            ARGS: TEST_HTTPS=Yes MYSQL_VER=mysql:8.0.40
-          - TEST_NAME: "Force DEncode"
-            ARGS: DIRAC_USE_JSON_ENCODE=NO
+          - TEST_NAME: "Force DEncode and MySQL8"
+            ARGS: DIRAC_USE_JSON_ENCODE=NO MYSQL_VER=mysql:8.0.40
           - TEST_NAME: "Backward Compatibility"
             ARGS: CLIENT_INSTALLATION_BRANCH=rel-v8r0
 

docs/source/DeveloperGuide/DevelopmentEnvironment/DeveloperInstallation/stuffThatRun.rst

@@ -93,13 +93,9 @@ the private key. You will need two different sets certificates and the CA certif
 The following commands should do the trick for you, by creating a fake CA, a fake user certificate, and a fake host certificate::
 
    cd $DEVROOT/DIRAC
-   git checkout release/integration
-   source tests/Jenkins/utilities.sh
-   generateCA
-   generateCertificates 365
-   generateUserCredentials 365
+   docker run ghcr.io/diracgrid/diracx/certificates-generation:latest
    mkdir -p ~/.globus/
-   cp $DEVROOT/user/*.{pem,key} ~/.globus/
+   docker cp certificates-generation:/ca/certs/client.{pem,key} ~/.globus/
    mv ~/.globus/client.key ~/.globus/userkey.pem
    mv ~/.globus/client.pem ~/.globus/usercert.pem
 

integration_tests.py

@@ -378,7 +378,7 @@ def install_server():
     # for the diracx container to use
     # It needs to be started and running before the DIRAC server installation
     # because after installing the databases, the install server script
-    # calls dirac-login.
+    # calls dirac-proxy-init.
     # At this point we need the new CS to have been updated
     # already else the token exchange fails.
 
@@ -396,91 +396,6 @@ def install_server():
         check=True,
     )
 
-    typer.secho("Copying credentials and certificates", fg=c.GREEN)
-    base_cmd = _build_docker_cmd("client", tty=False)
-    subprocess.run(
-        base_cmd
-        + [
-            "mkdir",
-            "-p",
-            "/home/dirac/ServerInstallDIR/user",
-            "/home/dirac/ClientInstallDIR/etc",
-            "/home/dirac/.globus",
-        ],
-        check=True,
-    )
-    for path in [
-        "etc/grid-security",
-        "user/client.pem",
-        "user/client.key",
-        f"/tmp/x509up_u{os.getuid()}",
-    ]:
-        source = os.path.join("/home/dirac/ServerInstallDIR", path)
-        ret = subprocess.run(
-            ["docker", "cp", f"server:{source}", "-"],
-            check=True,
-            text=False,
-            stdout=subprocess.PIPE,
-        )
-        if path.startswith("user/"):
-            dest = f"client:/home/dirac/ServerInstallDIR/{os.path.dirname(path)}"
-        elif path.startswith("/"):
-            dest = f"client:{os.path.dirname(path)}"
-        else:
-            dest = f"client:/home/dirac/ClientInstallDIR/{os.path.dirname(path)}"
-        subprocess.run(["docker", "cp", "-", dest], check=True, text=False, input=ret.stdout)
-    subprocess.run(
-        base_cmd
-        + [
-            "bash",
-            "-c",
-            "cp /home/dirac/ServerInstallDIR/user/client.* /home/dirac/.globus/",
-        ],
-        check=True,
-    )
-
-    base_cmd = _build_docker_cmd("pilot", tty=False)
-    subprocess.run(
-        base_cmd
-        + [
-            "mkdir",
-            "-p",
-            "/home/dirac/ServerInstallDIR/user",
-            "/home/dirac/PilotInstallDIR/etc",
-            "/home/dirac/.globus",
-        ],
-        check=True,
-    )
-    for path in [
-        "etc/grid-security",
-        "user/client.pem",
-        "user/client.key",
-        f"/tmp/x509up_u{os.getuid()}",
-    ]:
-        source = os.path.join("/home/dirac/ServerInstallDIR", path)
-        ret = subprocess.run(
-            ["docker", "cp", f"server:{source}", "-"],
-            check=True,
-            text=False,
-            stdout=subprocess.PIPE,
-        )
-        if path.startswith("user/"):
-            dest = f"pilot:/home/dirac/ServerInstallDIR/{os.path.dirname(path)}"
-        elif path.startswith("/"):
-            dest = f"pilot:{os.path.dirname(path)}"
-        else:
-            dest = f"pilot:/home/dirac/PilotInstallDIR/{os.path.dirname(path)}"
-        subprocess.run(["docker", "cp", "-", dest], check=True, text=False, input=ret.stdout)
-    subprocess.run(
-        base_cmd
-        + [
-            "bash",
-            "-c",
-            "cp /home/dirac/ServerInstallDIR/user/client.* /home/dirac/.globus/",
-        ],
-        check=True,
-    )
-
 
 @app.command()
 def install_client():

src/DIRAC/Core/scripts/dirac_install_db.py

@@ -2,6 +2,8 @@
 """
 Create a new DB in the MySQL server
 """
+from DIRAC import exit as DIRACExit
+from DIRAC import gConfig, gLogger
 from DIRAC.Core.Base.Script import Script
 
 
@@ -12,24 +14,40 @@ def main():
     _, args = Script.parseCommandLine()
 
     # Script imports
-    from DIRAC import gConfig
+    from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import useServerCertificate
+    from DIRAC.Core.Security.ProxyInfo import getProxyInfo
     from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller
     from DIRAC.FrameworkSystem.Utilities import MonitoringUtilities
 
+    user = "DIRAC"
+
     gComponentInstaller.exitOnError = True
     gComponentInstaller.getMySQLPasswords()
     for db in args:
         result = gComponentInstaller.installDatabase(db)
         if not result["OK"]:
-            print(f"ERROR: failed to correctly install {db}", result["Message"])
-            continue
+            gLogger.error(f"Failed to correctly install {db}:", result["Message"])
+            DIRACExit(1)
         extension, system = result["Value"]
-        gComponentInstaller.addDatabaseOptionsToCS(gConfig, system, db, overwrite=True)
+        result = gComponentInstaller.addDatabaseOptionsToCS(gConfig, system, db, overwrite=True)
+        if not result["OK"]:
+            gLogger.error("Failed to add database options to CS:", result["Message"])
+            DIRACExit(1)
 
         if db != "InstalledComponentsDB":
-            result = MonitoringUtilities.monitorInstallation("DB", system, db)
+            # get the user that installed the DB
+            if not useServerCertificate():
+                result = getProxyInfo()
+                if not result["OK"]:
+                    return result
+                proxyInfo = result["Value"]
+                if "username" in proxyInfo:
+                    user = proxyInfo["username"]
+
+            result = MonitoringUtilities.monitorInstallation("DB", system, db, user=user)
             if not result["OK"]:
-                print(f"ERROR: failed to register installation in database: {result['Message']}")
+                gLogger.error("Failed to register installation in database:", result["Message"])
+                DIRACExit(1)
 
 
 if __name__ == "__main__":

src/DIRAC/DataManagementSystem/scripts/dirac_admin_allow_se.py

@@ -17,6 +17,7 @@ def main():
     remove = False
     site = ""
     mute = False
+    userName = ""
 
     Script.registerSwitch("r", "AllowRead", "     Allow only reading from the storage element")
     Script.registerSwitch("w", "AllowWrite", "     Allow only writing to the storage element")
@@ -25,6 +26,7 @@ def main():
     Script.registerSwitch("a", "All", "    Allow all access to the storage element")
     Script.registerSwitch("m", "Mute", "     Do not send email")
     Script.registerSwitch("S:", "Site=", "     Allow all SEs associated to site")
+    Script.registerSwitch("t:", "tokenOwner=", "     Optional Name of the token owner")
     # Registering arguments will automatically add their description to the help menu
     Script.registerArgument(["seGroupList: list of SEs or comma-separated SEs"])
 
@@ -48,6 +50,8 @@ def main():
             mute = True
         if switch[0].lower() in ("s", "site"):
             site = switch[1]
+        if switch[0] in ("t", "tokenOwner"):
+            userName = switch[1]
 
     # imports
     from DIRAC import gLogger
@@ -69,15 +73,16 @@ def main():
     ses = resolveSEGroup(ses)
     diracAdmin = DiracAdmin()
 
-    res = getProxyInfo()
-    if not res["OK"]:
-        gLogger.error("Failed to get proxy information", res["Message"])
-        DIRAC.exit(2)
-
-    userName = res["Value"].get("username")
     if not userName:
-        gLogger.error("Failed to get username for proxy")
-        DIRAC.exit(2)
+        res = getProxyInfo()
+        if not res["OK"]:
+            gLogger.error("Failed to get proxy information", res["Message"])
+            DIRAC.exit(2)
+
+        userName = res["Value"].get("username")
+        if not userName:
+            gLogger.error("Failed to get username for proxy")
+            DIRAC.exit(2)
 
     if site:
         res = getSites()

src/DIRAC/DataManagementSystem/scripts/dirac_admin_ban_se.py

@@ -18,6 +18,7 @@ def main():
     remove = True
     sites = []
     mute = False
+    userName = ""
 
     Script.registerSwitch("r", "BanRead", "     Ban only reading from the storage element")
     Script.registerSwitch("w", "BanWrite", "     Ban writing to the storage element")
@@ -28,6 +29,7 @@ def main():
     Script.registerSwitch(
         "S:", "Site=", "     Ban all SEs associate to site (note that if writing is allowed, check is always allowed)"
     )
+    Script.registerSwitch("t:", "tokenOwner=", "     Optional Name of the token owner")
     # Registering arguments will automatically add their description to the help menu
     Script.registerArgument(["seGroupList: list of SEs or comma-separated SEs"])
 
@@ -56,6 +58,8 @@ def main():
             mute = True
         if switch[0].lower() in ("s", "site"):
             sites = switch[1].split(",")
+        if switch[0] in ("t", "tokenOwner"):
+            userName = switch[1]
 
     # from DIRAC.ConfigurationSystem.Client.CSAPI           import CSAPI
     from DIRAC import gLogger
@@ -68,15 +72,16 @@ def main():
     ses = resolveSEGroup(ses)
     diracAdmin = DiracAdmin()
 
-    res = getProxyInfo()
-    if not res["OK"]:
-        gLogger.error("Failed to get proxy information", res["Message"])
-        DIRAC.exit(2)
-
-    userName = res["Value"].get("username")
     if not userName:
-        gLogger.error("Failed to get username for proxy")
-        DIRAC.exit(2)
+        res = getProxyInfo()
+        if not res["OK"]:
+            gLogger.error("Failed to get proxy information", res["Message"])
+            DIRAC.exit(2)
+
+        userName = res["Value"].get("username")
+        if not userName:
+            gLogger.error("Failed to get username for proxy")
+            DIRAC.exit(2)
 
     for site in sites:
         res = DMSHelpers().getSEsForSite(site)

src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py

@@ -69,7 +69,6 @@
 from DIRAC.ConfigurationSystem.Client import PathFinder
 from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
 from DIRAC.ConfigurationSystem.Client.Helpers import (
-    CSGlobals,
     cfgInstallPath,
     cfgInstallSection,
     cfgPath,
@@ -85,6 +84,7 @@
     PRODUCTION_MANAGEMENT,
     PROXY_MANAGEMENT,
     SERVICE_ADMINISTRATOR,
+    SITE_MANAGER,
     TRUSTED_HOST,
 )
 from DIRAC.Core.Utilities.Extensions import (
@@ -432,6 +432,8 @@ def _getCentralCfg(self, installCfg):
         defaultHostProperties = [
             TRUSTED_HOST,
             CS_ADMINISTRATOR,
+            SERVICE_ADMINISTRATOR,
+            SITE_MANAGER,
             JOB_ADMINISTRATOR,
             FULL_DELEGATION,
             PROXY_MANAGEMENT,

src/DIRAC/FrameworkSystem/Client/SystemAdministratorClientCLI.py

@@ -11,7 +11,6 @@
 import time
 
 from DIRAC import gConfig, gLogger
-from DIRAC.ConfigurationSystem.Client.Helpers import CSGlobals
 from DIRAC.Core.Base.CLI import CLI, colorize
 from DIRAC.Core.Security.ProxyInfo import getProxyInfo
 from DIRAC.Core.Utilities import List
@@ -623,6 +622,11 @@ def do_install(self, args):
           install agent <system> <agent> [-m <ModuleName>] [-p <Option>=<Value>] [-p <Option>=<Value>] ...
           install executor <system> <executor> [-m <ModuleName>] [-p <Option>=<Value>] [-p <Option>=<Value>] ...
         """
+        result = getProxyInfo()
+        if not result["OK"]:
+            self._errMsg(result["Message"])
+        user = result["Value"]["username"]
+
         argss = args.split()
         hostSetup = extension = None
         if not argss:
@@ -673,7 +677,7 @@ def do_install(self, args):
 
                 if database != "InstalledComponentsDB":
                     result = MonitoringUtilities.monitorInstallation(
-                        "DB", system.replace("System", ""), database, cpu=cpu, hostname=hostname
+                        "DB", system.replace("System", ""), database, cpu=cpu, hostname=hostname, user=user
                     )
                     if not result["OK"]:
                         self._errMsg(result["Message"])
@@ -786,14 +790,14 @@ def do_install(self, args):
                     return
 
                 result = MonitoringUtilities.monitorInstallation(
-                    "DB", system, "InstalledComponentsDB", cpu=cpu, hostname=hostname
+                    "DB", system, "InstalledComponentsDB", cpu=cpu, hostname=hostname, user=user
                 )
                 if not result["OK"]:
                     self._errMsg(f"Error registering installation into database: {result['Message']}")
                     return
 
             result = MonitoringUtilities.monitorInstallation(
-                option, system, component, module, cpu=cpu, hostname=hostname
+                option, system, component, module, cpu=cpu, hostname=hostname, user=user
             )
             if not result["OK"]:
                 self._errMsg(f"Error registering installation into database: {result['Message']}")
@@ -820,6 +824,7 @@ def do_uninstall(self, args):
         result = getProxyInfo()
         if not result["OK"]:
             self._errMsg(result["Message"])
+        user = result["Value"]["username"]
 
         option = argss[0]
         if option == "db":
@@ -842,7 +847,7 @@ def do_uninstall(self, args):
                 self._errMsg(result["Message"])
                 return
             system = result["Value"][component]["System"]
-            result = MonitoringUtilities.monitorUninstallation(system, component, hostname=hostname, cpu=cpu)
+            result = MonitoringUtilities.monitorUninstallation(system, component, hostname=hostname, cpu=cpu, user=user)
             if not result["OK"]:
                 self._errMsg(result["Message"])
                 return
@@ -937,7 +942,7 @@ def do_uninstall(self, args):
             else:
                 cpu = result["Value"]["CPUModel"]
             hostname = self.host
-            result = MonitoringUtilities.monitorUninstallation(system, component, hostname=hostname, cpu=cpu)
+            result = MonitoringUtilities.monitorUninstallation(system, component, hostname=hostname, cpu=cpu, user=user)
             if not result["OK"]:
                 return result
 

src/DIRAC/FrameworkSystem/ConfigTemplate.cfg

@@ -167,6 +167,7 @@ Services
       componentExists = authenticated
       getComponents = authenticated
       hostExists = authenticated
+      installationExists = authenticated
       getHosts = authenticated
       installationExists = authenticated
       getInstallations = authenticated
@@ -184,6 +185,7 @@ Services
       componentExists = authenticated
       getComponents = authenticated
       hostExists = authenticated
+      installationExists = authenticated
       getHosts = authenticated
       installationExists = authenticated
       getInstallations = authenticated