feat: Add legacy adaptor in CI

Squashed commits:
[7ae3428] fix: Fixed to support better other branches
[2066814] feat: Add diracx repo choice in the integration test
[7859c6b] fix: Use JobStateUpdateClient instead of deactivating it
[e3b1d68] fix: Resolved dirac-proxy-init issue with multi-VO
[d2f99c0] fix: Resolved bug for setJobAttributes
[fe09e19] fix: Fixed ClientWMS test and reversed TEST_DIRACX flag
[1447a20] fix: Use opensearch db for job parametersDB
[f6116aa] fix: Made mariadb works better
[c4ab3f2] fix: Fixed CI to use sandoxes
[a5a02bc] fix: Activate job endpoints in CI
[a1bc1b1] fix: Applied changes to the ci workflows
[282efcb] refactor: Moved down diracx installation to fix auth
[ac22d9d] feat: Add DiracX tests into DIRAC CI, and use only latest tag.
[0ebf454] fix: Moved up futureServices argument to prevent "Missing mandatory /DiracX/URL configuration"
[d98b705] fix: Fixed small bug and refactored ClientSelector
[8ce234c] fix: Small fixes with docker compose cmd and removed grep
[08d9eb1] fix: Add filters to remove 'Client' from  services name
[bc44891] fix: Remove error when a future client is not available to warn instead
[a1f553d] fix: Changed to make the legacy adaptor automatic.
[3fb8c4d] fix: Fixed destroy command to support MacOSX
[42df58d] fix: Going back, removed CI, and going to add it to DiracX
[28b9789] fix: Small fix (forgot to change to docker**compose)
[61384af] feat: Add in CI new legacy adapted services.
[a3a7f5b] feat: Add legacy adaptor in CI

Author: Robin VAN DE MERGHEL

.github/workflows/integration.yml

@@ -37,6 +37,8 @@ jobs:
             ARGS: DIRAC_USE_JSON_ENCODE=NO MYSQL_VER=mysql:8.0.40
           - TEST_NAME: "Backward Compatibility"
             ARGS: CLIENT_INSTALLATION_BRANCH=rel-v8r0 PILOT_INSTALLATION_BRANCH=rel-v8r0
+          - TEST_NAME: "Test DiracX latest"
+            ARGS: TEST_DIRACX=Yes
 
     steps:
     - uses: actions/checkout@v4

integration_tests.py

@@ -25,6 +25,7 @@
 DEFAULT_HOST_OS = "el9"
 DEFAULT_MYSQL_VER = "mysql:8.4.4"
 DEFAULT_ES_VER = "opensearchproject/opensearch:2.18.0"
+DEFAULT_ES_PLATFORM = "linux/amd64"
 DEFAULT_IAM_VER = "indigoiam/iam-login-service:v1.10.2"
 FEATURE_VARIABLES = {
     "DIRACOSVER": "master",
@@ -71,6 +72,8 @@
 }
 LOG_PATTERN = re.compile(r"^[\d\-]{10} [\d:]{8} UTC [^\s]+ ([A-Z]+):")
 
+DOCKER_COMPOSE_CMD = shlex.split(os.environ.get("DOCKER_COMPOSE_CMD", "docker compose"))
+
 
 class NaturalOrderGroup(typer.core.TyperGroup):
     """Group for showing subcommands in the correct order"""
@@ -193,8 +196,8 @@ def destroy():
     typer.secho("Shutting down and removing containers", err=True, fg=c.GREEN)
     with _gen_docker_compose(DEFAULT_MODULES) as docker_compose_fn:
         os.execvpe(
-            "docker",
-            ["docker", "compose", "-f", docker_compose_fn, "down", "--remove-orphans", "-t", "0", "--volumes"],
+            DOCKER_COMPOSE_CMD[0],
+            [*DOCKER_COMPOSE_CMD, "-f", docker_compose_fn, "down", "--remove-orphans", "-t", "0", "--volumes"],
             _make_env({}),
         )
 
@@ -253,7 +256,7 @@ def prepare_environment(
     typer.secho("Running docker compose to create containers", fg=c.GREEN)
     with _gen_docker_compose(modules, diracx_dist_dir=diracx_dist_dir) as docker_compose_fn:
         subprocess.run(
-            ["docker", "compose", "-f", docker_compose_fn, "up", "-d", "dirac-server", "dirac-client", "dirac-pilot"]
+            [*DOCKER_COMPOSE_CMD, "-f", docker_compose_fn, "up", "-d", "dirac-server", "dirac-client", "dirac-pilot"]
             + extra_services,
             check=True,
             env=docker_compose_env,
@@ -360,7 +363,7 @@ def prepare_environment(
         subStderr = open(docker_compose_fn_final / "stderr", "w")
 
         subprocess.Popen(
-            ["docker", "compose", "-f", docker_compose_fn_final / "docker-compose.yml", "up", "-d", "diracx"],
+            [*DOCKER_COMPOSE_CMD, "-f", docker_compose_fn_final / "docker-compose.yml", "up", "-d", "diracx"],
             env=docker_compose_env,
             stdin=None,
             stdout=subStdout,
@@ -569,7 +572,7 @@ def _gen_docker_compose(modules, *, diracx_dist_dir=None):
     # Load the docker compose configuration and mount the necessary volumes
     input_fn = Path(__file__).parent / "tests/CI/docker-compose.yml"
     docker_compose = yaml.safe_load(input_fn.read_text())
-    # diracx-wait-for-db needs the volume to be able to run the witing script
+    # diracx-wait-for-db needs the volume to be able to run the waiting script
     for ctn in ("dirac-server", "dirac-client", "dirac-pilot", "diracx-wait-for-db"):
         if "volumes" not in docker_compose["services"][ctn]:
             docker_compose["services"][ctn]["volumes"] = []
@@ -619,7 +622,7 @@ def _gen_docker_compose(modules, *, diracx_dist_dir=None):
 def _check_containers_running(*, is_up=True):
     with _gen_docker_compose(DEFAULT_MODULES) as docker_compose_fn:
         running_containers = subprocess.run(
-            ["docker", "compose", "-f", docker_compose_fn, "ps", "-q", "-a"],
+            [*DOCKER_COMPOSE_CMD, "-f", docker_compose_fn, "ps", "-q", "-a"],
             stdout=subprocess.PIPE,
             env=_make_env({}),
             # docker compose ps has a non-zero exit code when no containers are running
@@ -701,6 +704,7 @@ def _make_env(flags):
     else:
         env["MYSQL_ADMIN_COMMAND"] = "mysqladmin"
     env["ES_VER"] = flags.pop("ES_VER", DEFAULT_ES_VER)
+    env["ES_PLATFORM"] = flags.pop("ES_PLATFORM", DEFAULT_ES_PLATFORM)
     env["IAM_VER"] = flags.pop("IAM_VER", DEFAULT_IAM_VER)
     if "CVMFS_DIR" not in env or not Path(env["CVMFS_DIR"]).is_dir():
         typer.secho(f"CVMFS_DIR environment value: {env.get('CVMFS_DIR', 'NOT SET')}", fg=c.YELLOW)

src/DIRAC/ConfigurationSystem/Client/Helpers/Registry.py

@@ -457,7 +457,7 @@ def getVOForGroup(group):
 
     :return: str
     """
-    return getVO() or gConfig.getValue(f"{gBaseRegistrySection}/Groups/{group}/VO", "")
+    return gConfig.getValue(f"{gBaseRegistrySection}/Groups/{group}/VO", "") or getVO()
 
 
 def getIdPForGroup(group):

src/DIRAC/Core/Security/DiracX.py

@@ -47,7 +47,7 @@
 def addTokenToPEM(pemPath, group):
     from DIRAC.Core.Base.Client import Client
 
-    vo = Registry.getVOMSVOForGroup(group)
+    vo = Registry.getVOForGroup(group)
     if not vo:
         gLogger.error(f"ERROR: Could not find VO for group {group}, DiracX will not work!")
     disabledVOs = gConfig.getValue("/DiracX/DisabledVOs", [])

src/DIRAC/Core/Tornado/Client/ClientSelector.py

@@ -17,7 +17,6 @@
 from DIRAC.Core.DISET.TransferClient import TransferClient
 from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient
 
-
 sLog = gLogger.getSubLogger(__name__)
 
 
@@ -66,28 +65,30 @@ def ClientSelector(disetClient, *args, **kwargs):  # We use same interface as RP
         # If we are not already given a URL, resolve it
         if serviceName.startswith(("http", "dip")):
             completeUrl = serviceName
-        elif useLegacyAdapter(serviceName):
-            sLog.debug(f"Using legacy adapter for service {serviceName}")
-            if diracxClient is None:
-                raise NotImplementedError(
-                    "DiracX is enabled but no diracxClient is provided, do you need to update your client?"
-                )
-            return diracxClient()
         else:
+            isServiceLegacyAdapted = useLegacyAdapter(serviceName)
+            if isServiceLegacyAdapted and diracxClient is not None:
+                sLog.debug(f"Using legacy adapter for service {serviceName}")
+                return diracxClient()
+            else:
+                if isServiceLegacyAdapted:
+                    # Only warn to deactivate it without making the code crash.
+                    sLog.warn(
+                        f"FutureClient for {serviceName} is provided but not activated, do you need to update your client?"
+                    )
+
             completeUrl = getServiceURL(serviceName)
             sLog.debug(f"URL resolved: {completeUrl}")
 
         if completeUrl.startswith("http"):
             sLog.debug(f"Using HTTPS for service {serviceName}")
-            rpc = tornadoClient(*args, **kwargs)
-        else:
-            rpc = disetClient(*args, **kwargs)
+            return tornadoClient(*args, **kwargs)
     except Exception as e:  # pylint: disable=broad-except
         # If anything went wrong in the resolution, we return default RPCClient
         # So the behaviour is exactly the same as before implementation of Tornado
         sLog.warn("Could not select DISET or Tornado client", f"{repr(e)}")
-        rpc = disetClient(*args, **kwargs)
-    return rpc
+        sLog.exception()
+    return disetClient(*args, **kwargs)
 
 
 # Client to use for RPC selection

src/DIRAC/Core/Utilities/Extensions.py

@@ -73,6 +73,15 @@ def findServices(modules):
     return findModules(modules, "Service", "*Handler")
 
 
+def findFutureServices(modules):
+    """Find the legacy adapted services for one or more DIRAC extension(s)
+
+    :param list/str/module module: One or more Python modules or Python module names
+    :returns: list of tuples of the form (SystemName, ServiceName)
+    """
+    return findModules(modules, "FutureClient")
+
+
 @iterateThenSort
 def findDatabases(module):
     """Find the DB SQL schema defintions for one or more DIRAC extension(s)
@@ -182,7 +191,7 @@ def parseArgs():
     parser = argparse.ArgumentParser()
     subparsers = parser.add_subparsers(required=True, dest="function")
     defaultExtensions = extensionsByPriority()
-    for func in [findSystems, findAgents, findExecutors, findServices, findDatabases]:
+    for func in [findSystems, findAgents, findExecutors, findServices, findDatabases, findFutureServices]:
         subparser = subparsers.add_parser(func.__name__)
         subparser.add_argument("--extensions", nargs="+", default=defaultExtensions)
         subparser.set_defaults(func=func)

src/DIRAC/WorkloadManagementSystem/Client/JobMonitoringClient.py

@@ -18,7 +18,8 @@ def __init__(self, **kwargs):
         super().__init__(**kwargs)
         self.setServer("WorkloadManagement/JobMonitoring")
 
-    diracxClient = futureJobMonitoringClient
+    # Set to NULL to avoid using it in ClientSelector
+    diracxClient = None
 
     @ignoreEncodeWarning
     def getJobsStatus(self, jobIDs):

src/DIRAC/WorkloadManagementSystem/FutureClient/JobStateUpdateClient.py

@@ -77,11 +77,11 @@ def setJobApplicationStatus(self, jobID: str | int, appStatus: str, source: str
     def setJobAttribute(self, jobID: str | int, attribute: str, value: str):
         with DiracXClient() as api:
             if attribute == "Status":
-                api.jobs.set_job_statuses(
+                return api.jobs.set_job_statuses(
                     {jobID: {datetime.now(tz=timezone.utc): {"Status": value}}},
                 )
             else:
-                api.jobs.patch_metadata({jobID: {attribute: value}})
+                return api.jobs.patch_metadata({jobID: {attribute: value}})
 
     @stripValueIfOK
     @convertToReturnValue

tests/CI/check_db_initialized.sh

@@ -1,22 +1,43 @@
 #!/bin/bash
-dbMissing=true;
-allDBs=(AccountingDB FTS3DB JobDB JobLoggingDB PilotAgentsDB ProductionDB ProxyDB ReqDB ResourceManagementDB ResourceStatusDB SandboxMetadataDB StorageManagementDB TaskQueueDB TransformationDB)
-while ${dbMissing};
-do
-    dbMissing=false;
-    allExistingDBs=$(mysql -uDirac -pDirac -h mysql -P 3306 -e "show databases;");
-    for db in "${allDBs[@]}";
-    do
-        if grep -q "${db}" <<< "${allExistingDBs}";
-        then
-            echo "${db} OK";
-        else
-            echo "${db} not created";
-            dbMissing=true;
-        fi;
-    done;
-    if ${dbMissing};
-    then
-        sleep 1;
+DB_USER="Dirac"
+DB_PASS="Dirac"
+DB_HOST="mysql"
+DB_PORT=3306
+DB_CMD=""
+
+# Detect available client: maria or mysql
+if command -v mariadb >/dev/null 2>&1; then
+    DB_CMD="mariadb -u${DB_USER} -p${DB_PASS} -h${DB_HOST} -P${DB_PORT}"
+elif command -v mysql >/dev/null 2>&1; then
+    DB_CMD="mysql -u${DB_USER} -p${DB_PASS} -h${DB_HOST} -P${DB_PORT}"
+else
+    echo "❌ Neither mysql nor mariadb client found in PATH."
+    exit 1
+fi
+
+echo "Using client: ${DB_CMD%% *}"
+
+dbMissing=true
+allDBs=(
+  AccountingDB FTS3DB JobDB JobLoggingDB PilotAgentsDB ProductionDB
+  ProxyDB ReqDB ResourceManagementDB ResourceStatusDB
+  SandboxMetadataDB StorageManagementDB TaskQueueDB TransformationDB
+)
+
+while $dbMissing; do
+  dbMissing=false
+  allExistingDBs=$($DB_CMD -e "SHOW DATABASES;" 2>/dev/null)
+
+  for db in "${allDBs[@]}"; do
+    if grep -q "^${db}$" <<<"$allExistingDBs"; then
+      echo "✅ ${db} exists"
+    else
+      echo "⚠️ ${db} not created yet"
+      dbMissing=true
     fi
+  done
+
+  $dbMissing && sleep 10
 done
+
+echo "🎉 All databases are present."