add support to eessi_container.sh and create_tarball.sh for using unionfs overlay tool

Author: boegel

create_tarball.sh

@@ -2,6 +2,10 @@
 
 set -e
 
+# with fuse-overlayfs as overlay tool, removed files are indicated with .wh.*
+# with unionfs as overlay tool, removed files are indicated with *_HIDDEN~
+REMOVED_FILES_PATTERN='/\.wh\.|/[^/]*_HIDDEN\~'
+
 base_dir=$(dirname $(realpath $0))
 
 if [ $# -ne 5 ]; then
@@ -32,6 +36,10 @@ fi
 cvmfs_repo_name=${cvmfs_repo%"/${EESSI_DEV_PROJECT}"}
 cvmfs_repo_name=${cvmfs_repo_name#/cvmfs/}
 overlay_upper_dir="${eessi_tmpdir}/${cvmfs_repo_name}/overlay-upper${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT}"
+# if unionfs overlay tool was used, we need to peek into .unionfs subdirectory
+if [ -d "${overlay_upper_dir}/.unionfs" ]; then
+    overlay_upper_dir="${overlay_upper_dir}/.unionfs"
+fi
 
 software_dir_overlay="${overlay_upper_dir}/versions/${eessi_version}"
 if [ ! -d ${software_dir_overlay} ]; then
@@ -49,30 +57,30 @@ module_files_list=${tmpdir}/module_files.list.txt
 if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then
     # include Lmod cache and configuration file (lmodrc.lua),
     # skip whiteout files and backup copies of Lmod cache (spiderT.old.*)
-    find ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod -type f | egrep -v '/\.wh\.|spiderT.old' >> ${files_list}
+    find ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod -type f | egrep -v "${REMOVED_FILES_PATTERN}|spiderT.old" >> ${files_list}
 fi
 
 # include scripts that were copied by install_scripts.sh, which we want to ship in EESSI repository
 if [ -d ${eessi_version}/scripts ]; then
-    find ${eessi_version}/scripts -type f | grep -v '/\.wh\.' >> ${files_list}
+    find ${eessi_version}/scripts -type f | grep -v "${REMOVED_FILES_PATTERN}" >> ${files_list}
 fi
 
 # also include init, which is also copied by install_scripts.sh
 if [ -d ${eessi_version}/init ]; then
-    find ${eessi_version}/init -type f | grep -v '/\.wh\.' >> ${files_list}
+    find ${eessi_version}/init -type f | grep -v "${REMOVED_FILES_PATTERN}" >> ${files_list}
 fi
 
 # consider both CPU-only and accelerator subdirectories
 for subdir in ${cpu_arch_subdir} ${cpu_arch_subdir}/accel/${accel_subdir}; do
 
     if [ -d ${eessi_version}/software/${os}/${subdir}/modules ]; then
         # module files
-        find ${eessi_version}/software/${os}/${subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list} || true  # Make sure we don't exit because of set -e if grep doesn't return a match
+        find ${eessi_version}/software/${os}/${subdir}/modules -type f | grep -v "${REMOVED_FILES_PATTERN}" >> ${files_list} || true  # Make sure we don't exit because of set -e if grep doesn't return a match
         # module symlinks
-        find ${eessi_version}/software/${os}/${subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list} || true  # Make sure we don't exit because of set -e if grep doesn't return a match
+        find ${eessi_version}/software/${os}/${subdir}/modules -type l | grep -v "${REMOVED_FILES_PATTERN}" >> ${files_list} || true  # Make sure we don't exit because of set -e if grep doesn't return a match
         # module files and symlinks
         find ${eessi_version}/software/${os}/${subdir}/modules/all -type f -o -type l \
-            | grep -v '/\.wh\.' | grep -v '/\.modulerc\.lua' | sed -e 's/.lua$//' | sed -e 's@.*/modules/all/@@g' | sort -u \
+            | grep -v "${REMOVED_FILES_PATTERN}" | grep -v '/\.modulerc\.lua' | sed -e 's/.lua$//' | sed -e 's@.*/modules/all/@@g' | sort -u \
             >> ${module_files_list}
     fi
 
@@ -86,7 +94,7 @@ for subdir in ${cpu_arch_subdir} ${cpu_arch_subdir}/accel/${accel_subdir}; do
         for package_version in $(cat ${module_files_list}); do
             echo "handling ${package_version}"
             ls -d ${eessi_version}/software/${os}/${subdir}/software/${package_version} \
-                | grep -v '/\.wh\.' >> ${files_list} || true  # Make sure we don't exit because of set -e if grep doesn't return a match
+                | grep -v "${REMOVED_FILES_PATTERN}" >> ${files_list} || true  # Make sure we don't exit because of set -e if grep doesn't return a match
         done
     fi
 done

eessi_container.sh

@@ -47,6 +47,7 @@ HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9))
 HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10))
 RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11))
 NVIDIA_MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 12))
+OVERLAY_TOOL_EXITCODE=$((${ANY_ERROR_EXITCODE} << 13))
 
 # CernVM-FS settings
 CVMFS_VAR_LIB="var-lib-cvmfs"
@@ -89,6 +90,9 @@ display_help() {
   echo "  -n | --nvidia MODE      - configure the container to work with NVIDIA GPUs,"
   echo "                            MODE==install for a CUDA installation, MODE==run to"
   echo "                            attach a GPU, MODE==all for both [default: false]"
+  echo "  -o | --overlay-tool ARG - tool to use to create (read-only or writable) overlay;"
+  echo "                            selected tool *must* be available in container image being used;"
+  echo "                            can be 'fuse-overlayfs' or 'unionfs' [default: fuse-overlayfs]"
   echo "  -p | --pass-through ARG - argument to pass through to the launch of the"
   echo "                            container; can be given multiple times [default: not set]"
   echo "  -r | --repository CFG   - configuration file or identifier defining the"
@@ -128,6 +132,7 @@ VERBOSE=0
 STORAGE=
 LIST_REPOS=0
 MODE="shell"
+OVERLAY_TOOL="fuse-overlayfs"
 PASS_THROUGH=()
 SETUP_NVIDIA=0
 REPOSITORIES=()
@@ -185,6 +190,10 @@ while [[ $# -gt 0 ]]; do
       NVIDIA_MODE="$2"
       shift 2
       ;;
+    -o|--overlay-tool)
+      OVERLAY_TOOL="$2"
+      shift 2
+      ;;
     -p|--pass-through)
       PASS_THROUGH+=("$2")
       shift 2
@@ -779,7 +788,7 @@ do
             # below); the overlay-upper directory can only exist because it is part of
             # the ${RESUME} directory or tarball
             # to be able to see the contents of the read-write session we have to mount
-            # the fuse-overlayfs (in read-only mode) on top of the CernVM-FS repository
+            # the overlay (in read-only mode) on top of the CernVM-FS repository
 
             echo "While processing '${cvmfs_repo_name}' to be mounted 'read-only' we detected an overlay-upper"
             echo "  directory (${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper) likely from a previous"
@@ -791,14 +800,25 @@ do
             EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}")
 
             # now, put the overlay-upper read-only on top of the repo and make it available under the usual prefix /cvmfs
-            EESSI_READONLY_OVERLAY="container:fuse-overlayfs"
-            # The contents of the previous session are available under
-            #   ${EESSI_TMPDIR} which is bind mounted to ${TMP_IN_CONTAINER}.
-            #   Hence, we have to use ${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper
-            # the left-most directory given for the lowerdir argument is put on top,
-            #   and with no upperdir=... the whole overlayfs is made available read-only
-            EESSI_READONLY_OVERLAY+=" -o lowerdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper:/cvmfs_ro/${cvmfs_repo_name}"
-            EESSI_READONLY_OVERLAY+=" /cvmfs/${cvmfs_repo_name}"
+            if [[ "${OVERLAY_TOOL}" == "fuse-overlayfs" ]]; then
+                EESSI_READONLY_OVERLAY="container:fuse-overlayfs"
+                # The contents of the previous session are available under
+                #   ${EESSI_TMPDIR} which is bind mounted to ${TMP_IN_CONTAINER}.
+                #   Hence, we have to use ${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper
+                # the left-most directory given for the lowerdir argument is put on top,
+                #   and with no upperdir=... the whole overlayfs is made available read-only
+                EESSI_READONLY_OVERLAY+=" -o lowerdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper:/cvmfs_ro/${cvmfs_repo_name}"
+                EESSI_READONLY_OVERLAY+=" /cvmfs/${cvmfs_repo_name}"
+            elif [[ "${OVERLAY_TOOL}" == "unionfs" ]]; then
+                EESSI_READONLY_OVERLAY="container:unionfs"
+                # cow stands for 'copy-on-write'
+                EESSI_READONLY_OVERLAY+=" -o cow"
+                EESSI_READONLY_OVERLAY+=" /cvmfs_ro/software.eessi.io=RO"
+                EESSI_READONLY_OVERLAY+=" /cvmfs/${cvmfs_repo_name}"
+            else
+                echo -e "ERROR: unknown overlay tool specified: ${OVERLAY_TOOL}"
+                exit ${OVERLAY_TOOL_EXITCODE}
+            fi
             export EESSI_READONLY_OVERLAY
 
             EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY_OVERLAY}")
@@ -824,11 +844,23 @@ do
 
         EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}")
 
-        EESSI_WRITABLE_OVERLAY="container:fuse-overlayfs"
-        EESSI_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${cvmfs_repo_name}"
-        EESSI_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper"
-        EESSI_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-work"
-        EESSI_WRITABLE_OVERLAY+=" /cvmfs/${cvmfs_repo_name}"
+        if [[ "${OVERLAY_TOOL}" == "fuse-overlayfs" ]]; then
+            EESSI_WRITABLE_OVERLAY="container:fuse-overlayfs"
+            EESSI_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${cvmfs_repo_name}"
+            EESSI_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper"
+            EESSI_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-work"
+            EESSI_WRITABLE_OVERLAY+=" /cvmfs/${cvmfs_repo_name}"
+        elif [[ "${OVERLAY_TOOL}" == "unionfs" ]]; then
+            # files touched are reflected under /cvmfs/<repo>/.unionfs/
+            EESSI_WRITABLE_OVERLAY="container:unionfs"
+            # cow stands for 'copy-on-write'
+            EESSI_WRITABLE_OVERLAY+=" -o cow"
+            EESSI_WRITABLE_OVERLAY+=" ${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper=RW:/cvmfs_ro/software.eessi.io=RO"
+            EESSI_WRITABLE_OVERLAY+=" /cvmfs/${cvmfs_repo_name}"
+        else
+            echo -e "ERROR: unknown overlay tool specified: ${OVERLAY_TOOL}"
+            exit ${OVERLAY_TOOL_EXITCODE}
+        fi
         export EESSI_WRITABLE_OVERLAY
 
         EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_WRITABLE_OVERLAY}")