Merge pull request #7143 from chaen/getToken_integration

[v8.1] [diracx] Get token and run integration tests

Author: fstagni

docs/diracdoctools/__init__.py

@@ -9,6 +9,7 @@
     "_arc",
     "arc",
     "cmreslogging",
+    "diracx",
     "fts3",
     "gfal2",
     "git",

docs/source/AdministratorGuide/ServerInstallations/environment_variable_configuration.rst

@@ -18,6 +18,12 @@ DIRAC_DEPRECATED_FAIL
   If set, the use of functions or objects that are marked ``@deprecated`` will fail. Useful for example in continuous
   integration tests against future versions of DIRAC
 
+DIRAC_ENABLE_DIRACX_JOB_MONITORING
+  If set, calls the diracx job monitoring service. Off by default.
+
+DIRAC_ENABLE_DIRACX_LOGIN
+  If set, retrieve a DiracX token when calling dirac-proxy-init or dirac-login
+
 DIRAC_FEWER_CFG_LOCKS
   If ``true`` or ``yes`` or ``on`` or ``1`` or ``y`` or ``t``, DIRAC will reduce the number of locks used when accessing the CS for better performance (default, ``no``).
 

integration_tests.py

@@ -29,13 +29,16 @@
     "DIRACOSVER": "master",
     "DIRACOS_TARBALL_PATH": None,
     "TEST_HTTPS": "Yes",
+    "TEST_DIRACX": "No",
     "DIRAC_FEWER_CFG_LOCKS": None,
     "DIRAC_USE_JSON_ENCODE": None,
     "INSTALLATION_BRANCH": "",
 }
-DEFAULT_MODULES = {
-    "DIRAC": Path(__file__).parent.absolute(),
-}
+DIRACX_OPTIONS = (
+    "DIRAC_ENABLE_DIRACX_LOGIN",
+    "DIRAC_ENABLE_DIRACX_JOB_MONITORING",
+)
+DEFAULT_MODULES = {"DIRAC": Path(__file__).parent.absolute()}
 
 # Static configuration
 DB_USER = "Dirac"
@@ -180,7 +183,7 @@ def destroy():
     with _gen_docker_compose(DEFAULT_MODULES) as docker_compose_fn:
         os.execvpe(
             "docker-compose",
-            ["docker-compose", "-f", docker_compose_fn, "down", "--remove-orphans", "-t", "0"],
+            ["docker-compose", "-f", docker_compose_fn, "down", "--remove-orphans", "-t", "0", "--volumes"],
             _make_env({}),
         )
 
@@ -193,7 +196,6 @@ def prepare_environment(
     release_var: Optional[str] = None,
 ):
     """Prepare the local environment for installing DIRAC."""
-
     _check_containers_running(is_up=False)
     if editable is None:
         editable = sys.stdout.isatty()
@@ -224,7 +226,7 @@ def prepare_environment(
     typer.secho("Running docker-compose to create containers", fg=c.GREEN)
     with _gen_docker_compose(modules) as docker_compose_fn:
         subprocess.run(
-            ["docker-compose", "-f", docker_compose_fn, "up", "-d"],
+            ["docker-compose", "-f", docker_compose_fn, "up", "-d", "dirac-server", "dirac-client"],
             check=True,
             env=docker_compose_env,
         )
@@ -313,6 +315,27 @@ def prepare_environment(
             )
             subprocess.run(command, check=True, shell=True)
 
+    docker_compose_fn_final = Path(tempfile.mkdtemp()) / "ci"
+    typer.secho("Running docker-compose to create DiracX containers", fg=c.GREEN)
+    typer.secho(f"Will leave a folder behind: {docker_compose_fn_final}", fg=c.YELLOW)
+
+    with _gen_docker_compose(modules) as docker_compose_fn:
+        # We cannot use the temporary directory created in the context manager because
+        # we don't stay in the contect manager (Popen)
+        # So we need something that outlives it.
+        shutil.copytree(docker_compose_fn.parent, docker_compose_fn_final, dirs_exist_ok=True)
+        # We use Popen because we don't want to wait for this command to finish.
+        # It is going to start all the diracx containers, including one which waits
+        # for the DIRAC installation to be over.
+        subprocess.Popen(
+            ["docker-compose", "-f", docker_compose_fn_final / "docker-compose.yml", "up", "-d", "diracx"],
+            env=docker_compose_env,
+            stdin=None,
+            stdout=None,
+            stderr=None,
+            close_fds=True,
+        )
+
 
 @app.command()
 def install_server():
@@ -326,6 +349,15 @@ def install_server():
         check=True,
     )
 
+    # This runs a continuous loop that exports the config in yaml
+    # for the diracx container to use
+    typer.secho("Starting configuration export loop for diracx", fg=c.GREEN)
+    base_cmd = _build_docker_cmd("server", tty=False, daemon=True)
+    subprocess.run(
+        base_cmd + ["bash", "/home/dirac/LocalRepo/ALTERNATIVE_MODULES/DIRAC/tests/CI/exportCSLoop.sh"],
+        check=True,
+    )
+
     typer.secho("Copying credentials and certificates", fg=c.GREEN)
     base_cmd = _build_docker_cmd("client", tty=False)
     subprocess.run(
@@ -508,13 +540,24 @@ def _gen_docker_compose(modules):
     # Load the docker-compose configuration and mount the necessary volumes
     input_fn = Path(__file__).parent / "tests/CI/docker-compose.yml"
     docker_compose = yaml.safe_load(input_fn.read_text())
+    # diracx-wait-for-db needs the volume to be able to run the witing script
+    for ctn in ("dirac-server", "dirac-client", "diracx-wait-for-db"):
+        if "volumes" not in docker_compose["services"][ctn]:
+            docker_compose["services"][ctn]["volumes"] = []
     volumes = [f"{path}:/home/dirac/LocalRepo/ALTERNATIVE_MODULES/{name}" for name, path in modules.items()]
     volumes += [f"{path}:/home/dirac/LocalRepo/TestCode/{name}" for name, path in modules.items()]
-    docker_compose["services"]["dirac-server"]["volumes"] = volumes[:]
-    docker_compose["services"]["dirac-client"]["volumes"] = volumes[:]
+    docker_compose["services"]["dirac-server"]["volumes"].extend(volumes[:])
+    docker_compose["services"]["dirac-client"]["volumes"].extend(volumes[:])
+    docker_compose["services"]["diracx-wait-for-db"]["volumes"].extend(volumes[:])
+
+    module_configs = _load_module_configs(modules)
+    if "diracx" in module_configs:
+        docker_compose["services"]["diracx"]["volumes"].append(
+            f"{modules['diracx']}/src/diracx:{module_configs['diracx']['install-location']}"
+        )
 
     # Add any extension services
-    for module_name, module_configs in _load_module_configs(modules).items():
+    for module_name, module_configs in module_configs.items():
         for service_name, service_config in module_configs["extra-services"].items():
             typer.secho(f"Adding service {service_name} for {module_name}", err=True, fg=c.GREEN)
             docker_compose["services"][service_name] = service_config.copy()
@@ -981,6 +1024,8 @@ def _make_config(modules, flags, release_var, editable):
         "CLIENT_HOST": "client",
         # Test specific variables
         "WORKSPACE": "/home/dirac",
+        # DiracX variable
+        "DIRACX_URL": "http://diracx:8000",
     }
 
     if editable:
@@ -1006,6 +1051,12 @@ def _make_config(modules, flags, release_var, editable):
         except KeyError:
             typer.secho(f"Required feature variable {key!r} is missing", err=True, fg=c.RED)
             raise typer.Exit(code=1)
+
+    # If we test DiracX, enable all the options
+    if config["TEST_DIRACX"].lower() in ("yes", "true"):
+        for key in DIRACX_OPTIONS:
+            config[key] = "Yes"
+
     config["TESTREPO"] = [f"/home/dirac/LocalRepo/TestCode/{name}" for name in modules]
     config["ALTERNATIVE_MODULES"] = [f"/home/dirac/LocalRepo/ALTERNATIVE_MODULES/{name}" for name in modules]
 
@@ -1027,7 +1078,7 @@ def _load_module_configs(modules):
     return module_ci_configs
 
 
-def _build_docker_cmd(container_name, *, use_root=False, cwd="/home/dirac", tty=True):
+def _build_docker_cmd(container_name, *, use_root=False, cwd="/home/dirac", tty=True, daemon=False):
     if use_root or os.getuid() == 0:
         user = "root"
     else:
@@ -1042,6 +1093,8 @@ def _build_docker_cmd(container_name, *, use_root=False, cwd="/home/dirac", tty=
                 err=True,
                 fg=c.YELLOW,
             )
+    if daemon:
+        cmd += ["-d"]
     cmd += [
         "-e=TERM=xterm-color",
         "-e=INSTALLROOT=/home/dirac",

src/DIRAC/ConfigurationSystem/Client/CSAPI.py

@@ -612,25 +612,15 @@ def getOpsSection():
             Where is the shifters section?
             """
             vo = CSGlobals.getVO()
-            setup = CSGlobals.getSetup()
 
             if vo:
-                res = gConfig.getSections(f"/Operations/{vo}/{setup}/Shifter")
+                res = gConfig.getSections(f"/Operations/{vo}/Shifter")
                 if res["OK"]:
-                    return S_OK(f"/Operations/{vo}/{setup}/Shifter")
+                    return S_OK(f"/Operations/{vo}/Shifter")
 
-                res = gConfig.getSections(f"/Operations/{vo}/Defaults/Shifter")
-                if res["OK"]:
-                    return S_OK(f"/Operations/{vo}/Defaults/Shifter")
-
-            else:
-                res = gConfig.getSections(f"/Operations/{setup}/Shifter")
-                if res["OK"]:
-                    return S_OK(f"/Operations/{setup}/Shifter")
-
-                res = gConfig.getSections("/Operations/Defaults/Shifter")
-                if res["OK"]:
-                    return S_OK("/Operations/Defaults/Shifter")
+            res = gConfig.getSections("/Operations/Defaults/Shifter")
+            if res["OK"]:
+                return S_OK("/Operations/Defaults/Shifter")
 
             return S_ERROR("No shifter section")
 
@@ -671,7 +661,7 @@ def getOpsSection():
                 gLogger.info("Adding shifter section")
                 vo = CSGlobals.getVO()
                 if vo:
-                    section = f"/Operations/{vo}/Defaults/Shifter"
+                    section = f"/Operations/{vo}/Shifter"
                 else:
                     section = "/Operations/Defaults/Shifter"
                 res = self.__csMod.createSection(section)

src/DIRAC/ConfigurationSystem/Client/Helpers/Operations.py

@@ -6,56 +6,19 @@
 
       Operations/
           Defaults/
+                someOption = someValue
+                aSecondOption = aSecondValue
+          specificVo/
               someSection/
-                  someOption = someValue
-                  aSecondOption = aSecondValue
-          Production/
-              someSection/
-                  someOption = someValueInProduction
-                  aSecondOption = aSecondValueInProduction
-          Certification/
-              someSection/
-                  someOption = someValueInCertification
+                  someOption = someValueInVO
 
     The following calls would give different results based on the setup::
 
       Operations().getValue('someSection/someOption')
-        - someValueInProduction if we are in 'Production' setup
-        - someValueInCertification if we are in 'Certification' setup
-
-      Operations().getValue('someSection/aSecondOption')
-        - aSecondValueInProduction if we are in 'Production' setup
-        - aSecondValue if we are in 'Certification' setup    <- looking in Defaults
-                                                                since there's no Certification/someSection/aSecondOption
-
+        - someValueInVO if we are in 'specificVo' vo
+        - someValue if we are in any other VO
 
-    At the same time, for multi-VO installations, it is also possible to specify different options per-VO,
-    like the following::
-
-      Operations/
-          aVOName/
-              Defaults/
-                  someSection/
-                      someOption = someValue
-                      aSecondOption = aSecondValue
-              Production/
-                  someSection/
-                      someOption = someValueInProduction
-                      aSecondOption = aSecondValueInProduction
-              Certification/
-                  someSection/
-                      someOption = someValueInCertification
-          anotherVOName/
-              Defaults/
-                  someSectionName/
-                      someOptionX = someValueX
-                      aSecondOption = aSecondValue
-              setupName/
-                  someSection/
-                      someOption = someValueInProduction
-                      aSecondOption = aSecondValueInProduction
-
-    For this case it becomes then important for the Operations() objects to know the VO name
+    It becomes then important for the Operations() objects to know the VO name
     for which we want the information, and this can be done in the following ways.
 
     1. by specifying the VO name directly::
@@ -98,9 +61,7 @@ def __init__(self, vo=False, group=False, setup=False):
         """
         self.__uVO = vo
         self.__uGroup = group
-        self.__uSetup = setup
         self.__vo = False
-        self.__setup = False
         self.__discoverSettings()
 
     def __discoverSettings(self):
@@ -119,12 +80,6 @@ def __discoverSettings(self):
             result = getVOfromProxyGroup()
             if result["OK"]:
                 self.__vo = result["Value"]
-        # Set the setup
-        self.__setup = False
-        if self.__uSetup:
-            self.__setup = self.__uSetup
-        else:
-            self.__setup = CSGlobals.getSetup()
 
     def __getCache(self):
         Operations.__cacheLock.acquire()
@@ -134,7 +89,7 @@ def __getCache(self):
                 Operations.__cache = {}
                 Operations.__cacheVersion = currentVersion
 
-            cacheKey = (self.__vo, self.__setup)
+            cacheKey = (self.__vo,)
             if cacheKey in Operations.__cache:
                 return Operations.__cache[cacheKey]
 
@@ -155,14 +110,13 @@ def __getCache(self):
                 pass
 
     def __getSearchPaths(self):
-        paths = ["/Operations/Defaults", f"/Operations/{self.__setup}"]
+        paths = ["/Operations/Defaults"]
         if not self.__vo:
             globalVO = CSGlobals.getVO()
             if not globalVO:
                 return paths
             self.__vo = CSGlobals.getVO()
-        paths.append(f"/Operations/{self.__vo}/Defaults")
-        paths.append(f"/Operations/{self.__vo}/{self.__setup}")
+        paths.append(f"/Operations/{self.__vo}/")
         return paths
 
     def getValue(self, optionPath, defaultValue=None):
@@ -202,26 +156,6 @@ def getOptionsDict(self, sectionPath):
             data[opName] = sectionCFG[opName]
         return S_OK(data)
 
-    def getPath(self, option, vo=False, setup=False):
-        """
-        Generate the CS path for an option:
-
-          - if vo is not defined, the helper's vo will be used for multi VO installations
-          - if setup evaluates False (except None) -> The helpers setup will  be used
-          - if setup is defined -> whatever is defined will be used as setup
-          - if setup is None -> Defaults will be used
-
-        :param option: path with respect to the Operations standard path
-        :type option: string
-        """
-
-        for path in self.__getSearchPaths():
-            optionPath = os.path.join(path, option)
-            value = gConfig.getValue(optionPath, "NoValue")
-            if value != "NoValue":
-                return optionPath
-        return ""
-
     def getMonitoringBackends(self, monitoringType=None):
         """
         Chooses the type of backend to use (Monitoring and/or Accounting) depending on the MonitoringType.