feat (diracx): make diracx installation mandatory

chaen · chrisburr · commit cfe2fc4780a4 · 2023-11-27T20:42:46.000+01:00
diff --git a/integration_tests.py b/integration_tests.py
@@ -563,7 +563,7 @@ def _gen_docker_compose(modules, *, diracx_dist_dir=None):
 
     module_configs = _load_module_configs(modules)
     if diracx_dist_dir is not None:
-        for container_name in ["diracx-init-cs", "diracx-wait-for-db", "diracx"]:
+        for container_name in ["dirac-client", "dirac-server", "diracx-init-cs", "diracx-wait-for-db", "diracx"]:
             docker_compose["services"][container_name]["volumes"].append(f"{diracx_dist_dir}:/diracx_sources")
             docker_compose["services"][container_name].setdefault("environment", []).append(
                 "DIRACX_CUSTOM_SOURCE_PREFIXES=/diracx_sources"
diff --git a/setup.cfg b/setup.cfg
@@ -29,6 +29,8 @@ install_requires =
     cachetools
     certifi
     diraccfg
+    diracx-client
+    diracx-core
     db12
     fts3
     gfal2-python
diff --git a/src/DIRAC/Core/Security/DiracX.py b/src/DIRAC/Core/Security/DiracX.py
@@ -19,7 +19,7 @@
 from diracx.core.preferences import DiracxPreferences
 from diracx.core.utils import serialize_credentials
 
-from DIRAC import gConfig
+from DIRAC import gConfig, S_ERROR
 from DIRAC.ConfigurationSystem.Client.Helpers import Registry
 from DIRAC.Core.Security.Locations import getDefaultProxyLocation
 from DIRAC.Core.Utilities.ReturnValues import convertToReturnValue, returnValueOrRaise
@@ -31,7 +31,7 @@
 
 
 @convertToReturnValue
-def addProxyToPEM(pemPath, group):
+def addTokenToPEM(pemPath, group):
     from DIRAC.Core.Base.Client import Client
 
     vo = Registry.getVOMSVOForGroup(group)
@@ -41,10 +41,6 @@ def addProxyToPEM(pemPath, group):
             Client(url="Framework/ProxyManager", proxyLocation=pemPath).exchangeProxyForToken()
         )
 
-        diracxUrl = gConfig.getValue("/DiracX/URL")
-        if not diracxUrl:
-            return S_ERROR("Missing mandatory /DiracX/URL configuration")
-
         token = TokenResponse(
             access_token=token_content["access_token"],
             expires_in=token_content["expires_in"],
diff --git a/src/DIRAC/Core/Tornado/Client/private/TornadoBaseClient.py b/src/DIRAC/Core/Tornado/Client/private/TornadoBaseClient.py
@@ -511,7 +511,7 @@ def _request(self, retry=0, outputFile=None, **kwargs):
         # getting certificate
         # Do we use the server certificate ?
         if self.kwargs[self.KW_USE_CERTIFICATES]:
-            # TODO: Does this code path need to work with DiracX?
+            # TODO: make this code path work with DiracX for Agents and possibly webapp ?
             auth = {"cert": Locations.getHostCertificateAndKeyLocation()}
 
         # Use access token?
diff --git a/src/DIRAC/FrameworkSystem/Client/ProxyManagerClient.py b/src/DIRAC/FrameworkSystem/Client/ProxyManagerClient.py
@@ -10,7 +10,7 @@
 from DIRAC.ConfigurationSystem.Client.Helpers import Registry
 from DIRAC.Core.Utilities import ThreadSafe, DIRACSingleton
 from DIRAC.Core.Utilities.DictCache import DictCache
-from DIRAC.Core.Security.DiracX import addProxyToPEM
+from DIRAC.Core.Security.DiracX import addTokenToPEM
 from DIRAC.Core.Security.ProxyFile import multiProxyArgument, deleteMultiProxy
 from DIRAC.Core.Security.X509Chain import X509Chain  # pylint: disable=import-error
 from DIRAC.Core.Security.X509Request import X509Request  # pylint: disable=import-error
@@ -550,7 +550,7 @@ def dumpProxyToFile(self, chain, destinationFile=None, requiredTimeLeft=600):
         filename = retVal["Value"]
         if not (result := chain.getDIRACGroup())["OK"]:
             return result
-        if not (result := addProxyToPEM(filename, result["Value"]))["OK"]:
+        if not (result := addTokenToPEM(filename, result["Value"]))["OK"]:  # pylint: disable=unsubscriptable-object
             return result
         self.__filesCache.add(cHash, chain.getRemainingSecs()["Value"], filename)
         return S_OK(filename)
@@ -665,7 +665,7 @@ def renewProxy(self, proxyToBeRenewed=None, minLifeTime=3600, newProxyLifeTime=4
                 return result
             if not (result := chain.getDIRACGroup())["OK"]:
                 return result
-            if not (result := addProxyToPEM(filename, result["Value"]))["OK"]:
+            if not (result := addTokenToPEM(filename, result["Value"]))["OK"]:  # pylint: disable=unsubscriptable-object
                 return result
             return S_OK(filename)
 
diff --git a/src/DIRAC/FrameworkSystem/Service/ProxyManagerHandler.py b/src/DIRAC/FrameworkSystem/Service/ProxyManagerHandler.py
@@ -12,7 +12,7 @@
 from DIRAC.Core.Security import Properties
 from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader
 from DIRAC.ConfigurationSystem.Client.Helpers import Registry
-
+from DIRAC.FrameworkSystem.Utilities.diracx import get_token
 
 DEFAULT_MAIL_FROM = "proxymanager@diracgrid.org"
 
@@ -412,7 +412,6 @@ def export_getVOMSProxyWithToken(self, userDN, userGroup, requestPem, requiredLi
     @convertToReturnValue
     def export_exchangeProxyForToken(self):
         """Exchange a proxy for an equivalent token to be used with diracx"""
-        from DIRAC.FrameworkSystem.Utilities.diracx import get_token
 
         credDict = self.getRemoteCredentials()
         return get_token(
diff --git a/src/DIRAC/FrameworkSystem/Utilities/diracx.py b/src/DIRAC/FrameworkSystem/Utilities/diracx.py
@@ -1,4 +1,3 @@
-# pylint: disable=import-error
 import requests
 
 from cachetools import TTLCache, cached
diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_admin_get_proxy.py b/src/DIRAC/FrameworkSystem/scripts/dirac_admin_get_proxy.py
@@ -15,7 +15,7 @@
 import DIRAC
 from DIRAC import gLogger, S_OK, S_ERROR
 from DIRAC.Core.Base.Script import Script
-from DIRAC.Core.Security.DiracX import addProxyToPEM
+from DIRAC.Core.Security.DiracX import addTokenToPEM
 from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager
 from DIRAC.ConfigurationSystem.Client.Helpers import Registry
 
@@ -162,7 +162,7 @@ def main():
         DIRAC.exit(2)
     if not (result := chain.getDIRACGroup())["OK"]:
         return result
-    if not (result := addProxyToPEM(params.proxyPath, result["Value"]))["OK"]:
+    if not (result := addTokenToPEM(params.proxyPath, result["Value"]))["OK"]:  # pylint: disable=unsubscriptable-object
         return result
     gLogger.notice(f"Proxy downloaded to {params.proxyPath}")
     DIRAC.exit(0)
diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_login.py b/src/DIRAC/FrameworkSystem/scripts/dirac_login.py
@@ -25,7 +25,7 @@
 from DIRAC import gConfig, gLogger, S_OK, S_ERROR
 from DIRAC.Core.Security.Locations import getDefaultProxyLocation, getCertificateAndKeyLocation
 from DIRAC.Core.Security.VOMS import VOMS
-from DIRAC.Core.Security.DiracX import addProxyToPEM
+from DIRAC.Core.Security.DiracX import addTokenToPEM
 from DIRAC.Core.Security.ProxyFile import writeToProxyFile
 from DIRAC.Core.Security.ProxyInfo import getProxyInfo, formatProxyInfoAsString
 from DIRAC.Core.Security.X509Chain import X509Chain  # pylint: disable=import-error
@@ -315,7 +315,7 @@ def loginWithCertificate(self):
                     return res
 
             # Get a token for use with diracx
-            if not (result := addProxyToPEM(self.outputFile, self.group))["OK"]:
+            if not (result := addTokenToPEM(self.outputFile, self.group))["OK"]:
                 return result
 
         return S_OK()
diff --git a/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_init.py b/src/DIRAC/FrameworkSystem/scripts/dirac_proxy_init.py
@@ -18,7 +18,7 @@
 from DIRAC.Core.Base.Script import Script
 from DIRAC.FrameworkSystem.Client import ProxyGeneration, ProxyUpload
 from DIRAC.Core.Security import X509Chain, ProxyInfo, VOMS
-from DIRAC.Core.Security.DiracX import addProxyToPEM
+from DIRAC.Core.Security.DiracX import addTokenToPEM
 from DIRAC.Core.Security.Locations import getCAsLocation, getDefaultProxyLocation
 from DIRAC.ConfigurationSystem.Client.Helpers import Registry
 from DIRAC.FrameworkSystem.Client.BundleDeliveryClient import BundleDeliveryClient
@@ -223,7 +223,7 @@ def doTheMagic(self):
         resultProxyWithVOMS = pI.addVOMSExtIfNeeded()
 
         proxyLoc = self.__piParams.proxyLoc or getDefaultProxyLocation()
-        if not (result := addProxyToPEM(proxyLoc, self.__piParams.diracGroup))["OK"]:
+        if not (result := addTokenToPEM(proxyLoc, self.__piParams.diracGroup))["OK"]:
             return result
 
         if not resultProxyWithVOMS["OK"]:
diff --git a/src/DIRAC/Interfaces/Utilities/DCommands.py b/src/DIRAC/Interfaces/Utilities/DCommands.py
@@ -15,6 +15,7 @@
 from DIRAC import S_OK, S_ERROR, gConfig, gLogger
 from DIRAC.Core.Security.Locations import getCAsLocation
 from DIRAC.Core.Security import Locations, VOMS
+from DIRAC.Core.Security.DiracX import addTokenToPEM
 from DIRAC.Core.Utilities.PrettyPrint import printTable
 from DIRAC.FrameworkSystem.Client.BundleDeliveryClient import BundleDeliveryClient
 from DIRAC.ConfigurationSystem.Client.Helpers import Registry
@@ -507,8 +508,8 @@ def proxyInit(self):
             # silently skip VOMS errors
             pass
 
-        if not (result := addProxyToPEM(filename, params.diracGroup))["OK"]:
-            raise Exception(result["Message"])
+        if not (result := addTokenToPEM(filename, params.diracGroup))["OK"]:  # pylint: disable=unsubscriptable-object
+            raise Exception(result["Message"])  # pylint: disable=unsubscriptable-object
 
     def addVomsExt(self, proxy):
         retVal = self.getEnv("group_name")
diff --git a/src/DIRAC/WorkloadManagementSystem/FutureClient/JobMonitoringClient.py b/src/DIRAC/WorkloadManagementSystem/FutureClient/JobMonitoringClient.py
@@ -1,6 +1,5 @@
-# pylint: disable=import-error
 from diracx.client import DiracClient
-from diracx.client.models import JobSearchParams
+
 
 from DIRAC.Core.Utilities.ReturnValues import convertToReturnValue
 
diff --git a/src/DIRAC/WorkloadManagementSystem/Service/SandboxStoreHandler.py b/src/DIRAC/WorkloadManagementSystem/Service/SandboxStoreHandler.py
@@ -22,13 +22,16 @@
 from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader
 from DIRAC.DataManagementSystem.Client.DataManager import DataManager
 from DIRAC.DataManagementSystem.Service.StorageElementHandler import getDiskSpace
+from DIRAC.FrameworkSystem.Utilities.diracx import TheImpersonator
 from DIRAC.RequestManagementSystem.Client.File import File
 from DIRAC.RequestManagementSystem.Client.Operation import Operation
 from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient
 from DIRAC.RequestManagementSystem.Client.Request import Request
 from DIRAC.Resources.Storage.StorageElement import StorageElement
 from DIRAC.Core.Utilities.File import getGlobbedTotalSize
 
+from diracx.client.models import SandboxInfo
+
 
 class SandboxStoreHandlerMixin:
     __purgeCount = -1
@@ -113,9 +116,6 @@ def _getFromClient(self, fileId, token, fileSize, fileHelper=None, data=""):
 
         disabledVOs = gConfig.getValue("/DiracX/DisabledVOs", [])
         if self._useDiracXBackend and vo not in disabledVOs:
-            from DIRAC.FrameworkSystem.Utilities.diracx import TheImpersonator
-            from diracx.client.models import SandboxInfo  # pylint: disable=import-error
-
             gLogger.info("Forwarding to DiracX")
             with tempfile.TemporaryFile(mode="w+b") as tar_fh:
                 result = fileHelper.networkToDataSink(tar_fh, maxFileSize=self._maxUploadBytes)
@@ -490,8 +490,6 @@ def _sendToClient(self, fileID, token, fileHelper=None, raw=False):
         # If the PFN starts with S3, we know it has been uploaded to the
         # S3 sandbox store, so download it from there before sending it
         if filePath.startswith("/S3"):
-            from DIRAC.FrameworkSystem.Utilities.diracx import TheImpersonator
-
             with TheImpersonator(credDict) as client:
                 res = client.jobs.get_sandbox_file(pfn=filePath)
                 r = requests.get(res.url)
diff --git a/tests/Jenkins/dirac_ci.sh b/tests/Jenkins/dirac_ci.sh
@@ -133,8 +133,13 @@ installSite() {
   cd -
 
 
-  echo "==> Done installing, now configuring"
+  echo "==> Sourcing bashrc"
   source "${SERVERINSTALLDIR}/bashrc"
+
+  echo "==> Installing main branch of diracx"
+  installDIRACX core client cli
+
+  echo "==> Done installing, now configuring"
   configureArgs=()
   if [[ "${TEST_DIRACX:-}" = "Yes" ]]; then
     configureArgs+=("--LegacyExchangeApiKey=diracx:legacy:InsecureChangeMe")
diff --git a/tests/Jenkins/utilities.sh b/tests/Jenkins/utilities.sh
@@ -263,6 +263,11 @@ installDIRAC() {
     done
   fi
 
+
+
+  echo "==> Installing main branch of diracx"
+  installDIRACX core client
+
   echo "$DIRAC"
   echo "$PATH"
 
@@ -276,6 +281,25 @@ installDIRAC() {
   echo '==> Done installDIRAC'
 }
 
+##############################################################################
+# Install DiracX either from wheels or from github
+# Arguments: list of DiracX submodule module names to install (core, client, etc.)
+
+function installDIRACX() {
+  for wheel_name in "$@"; do
+    if [[ -n "${DIRACX_CUSTOM_SOURCE_PREFIXES:-}" ]]; then
+      wheels=( $(find "${DIRACX_CUSTOM_SOURCE_PREFIXES}" -name "diracx_${wheel_name}-*.whl") )
+      if [[ ! ${#wheels[@]} -eq 1 ]]; then
+          echo "ERROR: Multiple wheels found for ${package_name} in ${dir}"
+          exit 1
+      fi
+      pip install "${wheels[0]}"
+    else
+      pip install "git+https://github.com/DIRACGrid/diracx.git@main#egg=diracx-${wheel_name}&subdirectory=diracx-${wheel_name}"
+    fi
+  done
+}
+
 ##############################################################################
 # This function submits a job or more (it assumes a DIRAC client is installed)
 # it needs the following environment variables:

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,3 @@`
`1`		`-# pylint: disable=import-error`
`2`	`1`	`import requests`
`3`	`2`
`4`	`3`	`from cachetools import TTLCache, cached`