Merge pull request DIRACGrid#8050 from fstagni/cherry-pick-2-9af438937-integration

[sweep:integration] dirac-apptainer-exec: creating the correct environment inside the container

Author: fstagni

src/DIRAC/Core/LCG/GOCDBClient.py

@@ -210,11 +210,11 @@ def getHostnameDowntime(self, hostname, startDate=None, ongoing=False):
         if ongoing:
             params += "&ongoing_only=yes"
 
-        caPath = getCAsLocation()
-
         try:
             response = requests.get(
-                "https://goc.egi.eu/gocdbpi/public/?method=get_downtime&topentity=" + params, verify=caPath, timeout=20
+                "https://goc.egi.eu/gocdbpi/public/?method=get_downtime&topentity=" + params,
+                verify=getCAsLocation(),
+                timeout=20,
             )
             response.raise_for_status()
         except requests.exceptions.RequestException as e:
@@ -269,8 +269,7 @@ def _downTimeCurlDownload(self, entity=None, startDate=None):
                 gocdb_ep = gocdb_ep + "&topentity=" + entity
         gocdb_ep = gocdb_ep + when + gocdbpi_startDate + "&scope="
 
-        caPath = getCAsLocation()
-        dtPage = requests.get(gocdb_ep, verify=caPath, timeout=20)
+        dtPage = requests.get(gocdb_ep, verify=getCAsLocation(), timeout=20)
 
         dt = dtPage.text
 
@@ -294,8 +293,7 @@ def _getServiceEndpointCurlDownload(self, granularity, entity):
         # GOCDB-PI query
         gocdb_ep = "https://goc.egi.eu/gocdbpi/public/?method=get_service_endpoint&" + granularity + "=" + entity
 
-        caPath = getCAsLocation()
-        service_endpoint_page = requests.get(gocdb_ep, verify=caPath, timeout=20)
+        service_endpoint_page = requests.get(gocdb_ep, verify=getCAsLocation(), timeout=20)
 
         return service_endpoint_page.text
 

src/DIRAC/Core/Security/IAMService.py

@@ -3,11 +3,7 @@
 
 import requests
 
-from DIRAC import gConfig, gLogger, S_OK, S_ERROR
-from DIRAC.Core.Utilities import DErrno
-from DIRAC.Core.Security.Locations import getProxyLocation, getCAsLocation
-from DIRAC.Core.Utilities.Decorators import deprecated
-from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOOption
+from DIRAC import S_OK, gConfig, gLogger
 from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import getVO
 
 

src/DIRAC/Core/Security/Locations.py

@@ -1,6 +1,7 @@
 """ Collection of utilities for locating certs, proxy, CAs
 """
 import os
+
 import DIRAC
 from DIRAC import gConfig
 
@@ -24,12 +25,6 @@ def getProxyLocation():
     if os.path.isfile(f"/tmp/{proxyName}"):
         return f"/tmp/{proxyName}"
 
-    # No gridproxy found
-    return False
-
-
-# Retrieve CA's location
-
 
 def getCAsLocation():
     """Retrieve the CA's files location"""
@@ -63,22 +58,36 @@ def getCAsLocationNoConfig():
     casPath = "/etc/grid-security/certificates"
     if os.path.isdir(casPath):
         return casPath
-    # No CA's location found
-    return False
-
-
-# Retrieve CA's location
-
-
-def getCAsDefaultLocation():
-    """Retrievethe CAs Location inside DIRAC etc directory"""
     # rootPath./etc/grid-security/certificates
     casPath = f"{DIRAC.rootPath}/etc/grid-security/certificates"
-    return casPath
+    if os.path.isdir(casPath):
+        return casPath
 
 
-# TODO: Static depending on files specified on CS
-# Retrieve certificate
+def getVOMSLocation():
+    """Retrieve the CA's files location"""
+    # Grid-Security
+    retVal = gConfig.getOption(f"{g_SecurityConfPath}/Grid-Security")
+    if retVal["OK"]:
+        vomsPath = f"{retVal['Value']}/vomsdir"
+        if os.path.isdir(vomsPath):
+            return vomsPath
+    # Look up the X509_VOMS_DIR environment variable
+    if "X509_VOMS_DIR" in os.environ:
+        vomsPath = os.environ["X509_VOMS_DIR"]
+        return vomsPath
+    # rootPath./etc/grid-security/vomsdir
+    vomsPath = f"{DIRAC.rootPath}/etc/grid-security/vomsdir"
+    if os.path.isdir(vomsPath):
+        return vomsPath
+    # /etc/grid-security/vomsdir
+    vomsPath = "/etc/grid-security/vomsdir"
+    if os.path.isdir(vomsPath):
+        return vomsPath
+    # rootPath./etc/grid-security/vomsdir
+    vomsPath = f"{DIRAC.rootPath}/etc/grid-security/vomsdir"
+    if os.path.isdir(vomsPath):
+        return vomsPath
 
 
 def getHostCertificateAndKeyLocation(specificLocation=None):

src/DIRAC/Core/Security/VOMSService.py

@@ -64,8 +64,6 @@ def getUsers(self):
         if not self.urls:
             return S_ERROR(DErrno.ENOAUTH, "No VOMS server defined")
 
-        userProxy = getProxyLocation()
-        caPath = getCAsLocation()
         rawUserList = []
         result = None
         for url in self.urls:
@@ -79,8 +77,8 @@ def getUsers(self):
                     result = requests.get(
                         url,
                         headers={"X-VOMS-CSRF-GUARD": "y"},
-                        cert=userProxy,
-                        verify=caPath,
+                        cert=getProxyLocation(),
+                        verify=getCAsLocation(),
                         params={"startIndex": str(startIndex), "pageSize": "100"},
                     )
                 except requests.ConnectionError as exc:

src/DIRAC/Core/scripts/dirac_apptainer_exec.py

@@ -2,42 +2,29 @@
 """
 
 import os
-import shutil
 import sys
 
 import DIRAC
 from DIRAC import S_ERROR, gConfig, gLogger
 from DIRAC.Core.Base.Script import Script
+from DIRAC.Core.Security.Locations import getCAsLocation, getProxyLocation, getVOMSLocation
 from DIRAC.Core.Utilities.Subprocess import systemCall
 
 CONTAINER_WRAPPER = """#!/bin/bash
 
-echo "Starting inner container wrapper scripts (no install) at `date`."
 export DIRAC=%(dirac_env_var)s
 export DIRACOS=%(diracos_env_var)s
-# In any case we need to find a bashrc, and a cfg
+export X509_USER_PROXY=/etc/proxy
+export X509_CERT_DIR=/etc/grid-security/certificates
+export X509_VOMS_DIR=/etc/grid-security/vomsdir
+export DIRACSYSCONFIG=%(etc_dir)s/dirac.cfg
 source %(rc_script)s
 %(command)s
-echo "Finishing inner container wrapper scripts at `date`."
 """
 
 CONTAINER_DEFROOT = ""  # Should add something like "/cvmfs/dirac.egi.eu/container/apptainer/alma9/x86_64"
 
 
-def getEnv():
-    """Gets the environment for use within the container.
-    We blank almost everything to prevent contamination from the host system.
-    """
-
-    payloadEnv = {k: v for k, v in os.environ.items()}
-    payloadEnv["TMP"] = "/tmp"
-    payloadEnv["TMPDIR"] = "/tmp"
-    payloadEnv["X509_USER_PROXY"] = os.path.join("tmp", "proxy")
-    payloadEnv["DIRACSYSCONFIG"] = os.path.join("tmp", "dirac.cfg")
-
-    return payloadEnv
-
-
 @Script()
 def main():
     Script.registerArgument(" command: Command to execute inside the container")
@@ -50,41 +37,45 @@ def main():
         if switch[0].lower() == "i" or switch[0].lower() == "image":
             user_image = switch[1]
 
+    etc_dir = os.path.join(DIRAC.rootPath, "etc")
+
     wrapSubs = {
         "dirac_env_var": os.environ.get("DIRAC", os.getcwd()),
         "diracos_env_var": os.environ.get("DIRACOS", os.getcwd()),
+        "etc_dir": etc_dir,
     }
     wrapSubs["rc_script"] = os.path.join(os.path.realpath(sys.base_prefix), "diracosrc")
     wrapSubs["command"] = command
-    shutil.copyfile("dirac.cfg", os.path.join("tmp", "dirac.cfg"))
 
-    wrapLoc = os.path.join("tmp", "dirac_container.sh")
-    rawfd = os.open(wrapLoc, os.O_WRONLY | os.O_CREAT, 0o700)
+    rawfd = os.open("dirac_container.sh", os.O_WRONLY | os.O_CREAT, 0o700)
     fd = os.fdopen(rawfd, "w")
     fd.write(CONTAINER_WRAPPER % wrapSubs)
     fd.close()
 
-    innerCmd = os.path.join("tmp", "dirac_container.sh")
     cmd = ["apptainer", "exec"]
     cmd.extend(["--contain"])  # use minimal /dev and empty other directories (e.g. /tmp and $HOME)
     cmd.extend(["--ipc"])  # run container in a new IPC namespace
-    cmd.extend(["--workdir", "/tmp"])  # working directory to be used for /tmp, /var/tmp and $HOME
-    cmd.extend(["--home", "/tmp"])  # Avoid using small tmpfs for default $HOME and use scratch /tmp instead
-    cmd.extend(["--bind", "{0}:{0}:ro".format(os.path.join(os.path.realpath(sys.base_prefix)))])
+    cmd.extend(["--bind", f"{os.getcwd()}:/mnt"])  # bind current directory for dirac_container.sh
+    cmd.extend(["--bind", f"{getProxyLocation()}:/etc/proxy"])  # bind proxy file
+    cmd.extend(["--bind", f"{getCAsLocation()}:/etc/grid-security/certificates"])  # X509_CERT_DIR
+    cmd.extend(["--bind", f"{getVOMSLocation()}:/etc/grid-security/vomsdir"])  # X509_VOMS_DIR
+    cmd.extend(["--bind", "{0}:{0}:ro".format(etc_dir)])  # etc dir for dirac.cfg
+    cmd.extend(["--bind", "{0}:{0}:ro".format(os.path.join(os.path.realpath(sys.base_prefix)))])  # code dir
 
     rootImage = user_image or gConfig.getValue("/Resources/Computing/Singularity/ContainerRoot") or CONTAINER_DEFROOT
 
     if os.path.isdir(rootImage) or os.path.isfile(rootImage):
-        cmd.extend([rootImage, innerCmd])
+        cmd.extend([rootImage, "/mnt/dirac_container.sh"])
     else:
         # if we are here is because there's no image, or it is not accessible (e.g. not on CVMFS)
         gLogger.error("Apptainer image to exec not found: ", rootImage)
         return S_ERROR("Failed to find Apptainer image to exec")
 
-    gLogger.debug(f"Execute Apptainer command: {cmd}")
-    result = systemCall(0, cmd, env=getEnv())
+    gLogger.debug(f"Execute Apptainer command: {' '.join(cmd)}")
+    result = systemCall(0, cmd)
     if not result["OK"]:
         DIRAC.exit(1)
+    gLogger.notice(result["Value"][1])
 
 
 if __name__ == "__main__":

src/DIRAC/FrameworkSystem/Client/BundleDeliveryClient.py

@@ -143,13 +143,9 @@ def syncCAs(self):
         if "X509_CERT_DIR" in os.environ:
             X509_CERT_DIR = os.environ["X509_CERT_DIR"]
             del os.environ["X509_CERT_DIR"]
-        casLocation = Locations.getCAsLocation()
-        if not casLocation:
-            casLocation = Locations.getCAsDefaultLocation()
-        result = self.syncDir("CAs", casLocation)
         if X509_CERT_DIR:
             os.environ["X509_CERT_DIR"] = X509_CERT_DIR
-        return result
+        return self.syncDir("CAs", Locations.getCAsLocation())
 
     def syncCRLs(self):
         """Synchronize CRLs
@@ -160,10 +156,9 @@ def syncCRLs(self):
         if "X509_CERT_DIR" in os.environ:
             X509_CERT_DIR = os.environ["X509_CERT_DIR"]
             del os.environ["X509_CERT_DIR"]
-        result = self.syncDir("CRLs", Locations.getCAsLocation())
         if X509_CERT_DIR:
             os.environ["X509_CERT_DIR"] = X509_CERT_DIR
-        return result
+        return self.syncDir("CRLs", Locations.getCAsLocation())
 
     def getCAs(self):
         """This method can be used to create the CAs. If the file can not be created,

src/DIRAC/Resources/Storage/OccupancyPlugins/WLCGAccountingHTTPJson.py

@@ -41,9 +41,7 @@ def _downloadJsonFile(self, occupancyLFN, filePath):
         """
         try:
             with open(filePath, "w") as fd:
-                caPath = getCAsLocation()
-                userProxy = getProxyLocation()
-                res = requests.get(occupancyLFN, cert=userProxy, verify=caPath, timeout=30)
+                res = requests.get(occupancyLFN, cert=getProxyLocation(), verify=getCAsLocation(), timeout=30)
                 res.raise_for_status()
                 fd.write(res.text)
         except Exception as e:

src/DIRAC/Resources/Storage/StorageElement.py

@@ -65,14 +65,13 @@ def __call__(self, name, protocolSections=None, vo=None, hideExceptions=False):
         # Because the gfal2 context caches the proxy location,
         # we also use the proxy location as a key.
         # In practice, there should almost always be one, except for the REA
-        # If we see its memory consumtpion exploding, this might be a place to look
-        proxyLoc = getProxyLocation()
+        # If we see its memory consumption exploding, this might be a place to look
 
         # ensure protocolSections is hashable! (tuple)
         if isinstance(protocolSections, list):
             protocolSections = tuple(protocolSections)
 
-        argTuple = (tId, name, protocolSections, vo, proxyLoc)
+        argTuple = (tId, name, protocolSections, vo, getProxyLocation())
         seObj = self.seCache.get(argTuple)
 
         if not seObj:

tests/System/client_core.sh

@@ -81,3 +81,15 @@ if ! dirac-configuration-dump-local-cache; then
    exit 1
 fi
 echo " "
+
+echo " "
+echo "======  dirac-apptainer-exec dirac-platform"
+if ! dirac-apptainer-exec dirac-platform; then
+   exit 1
+fi
+
+echo " "
+echo "======  dirac-apptainer-exec dirac-proxy-info --help"
+if ! dirac-apptainer-exec dirac-proxy-info --help; then
+   exit 1
+fi