supported distro detection for docker-in-docker script (#1371)

* work in progress distro detection

* regex check

* clearer condition

* additional logging

* Update script-library/docker-in-docker-debian.sh

Co-authored-by: Chuck Lantz <[email protected]>

* add impish to dind settings.env

* rename docker settings to work for both dind and dfd

* add check to dfd

* add impish

* add jammy

* better regression testing for moby (when distro supported)

* remove jammy temporarily

* echo whether testing moby

Co-authored-by: Chuck Lantz <[email protected]>

Author: joshspicer

.github/workflows/script-library-pr-debian.yml

@@ -13,7 +13,7 @@ jobs:
     if: "!contains(github.event.head_commit.message, 'Automated update') && !contains(github.event.head_commit.message, 'CI ignore')"
     strategy:
       matrix:
-        os: ["debian:bullseye", "debian:buster", "ubuntu:focal", "ubuntu:bionic"]
+        os: ["debian:bullseye", "debian:buster", "ubuntu:focal", "ubuntu:bionic"]  # TODO: Add  "ubuntu:jammy"
         defaults: ["true", "false"]
       fail-fast: true
     runs-on: ubuntu-latest

migrations/readme-metadata-to-json.py

@@ -106,17 +106,17 @@ def template(defId, displayName, description, categories, defType, platforms, op
         options = 'lc rc'
 
     subst = '''
-		lc
+		l-c
 			"id": "{}",
 			"displayName": "{}",
 			"description": "{}",
 			"categories": [ "{}" ],
 			"platforms": [ "{}" ],
 			"type": "{}",
 			"options": {}
-		rc,'''.format(defId, displayName, description, categories, platforms, defType, options)
+		r-c,'''.format(defId, displayName, description, categories, platforms, defType, options)
 
-    return subst.replace('lc', '{').replace('rc', '}')
+    return subst.replace('l-c', '{').replace('r-c', '}')
 
 
 exceptions = 0

script-library/docker-debian.sh

@@ -17,6 +17,9 @@ USE_MOBY=${5:-"true"}
 DOCKER_VERSION=${6:-"latest"}
 DOCKER_DASH_COMPOSE_VERSION=${7:-"v1"} # v1 or v2
 MICROSOFT_GPG_KEYS_URI="https://packages.microsoft.com/keys/microsoft.asc"
+DOCKER_DASH_COMPOSE_VERSION="1"
+DOCKER_MOBY_ARCHIVE_VERSION_CODENAMES="buster bullseye bionic focal jammy"
+DOCKER_LICENSED_ARCHIVE_VERSION_CODENAMES="buster bullseye bionic focal hirsute impish jammy"
 
 set -e
 
@@ -125,6 +128,26 @@ fi
 # Fetch host/container arch.
 architecture="$(dpkg --print-architecture)"
 
+# Check if distro is suppported
+if [ "${USE_MOBY}" = "true" ]; then
+    # 'get_common_setting' allows attribute to be updated remotely
+    get_common_setting DOCKER_MOBY_ARCHIVE_VERSION_CODENAMES
+    if [[ "${DOCKER_MOBY_ARCHIVE_VERSION_CODENAMES}" != *"${VERSION_CODENAME}"* ]]; then
+        err "Unsupported  distribution version '${VERSION_CODENAME}'. To resolve, either: (1) set feature option '\"moby\": false' , or (2) choose a compatible OS distribution"
+        err "Support distributions include:  ${DOCKER_MOBY_ARCHIVE_VERSION_CODENAMES}"
+        exit 1
+    fi
+    echo "Distro codename  '${VERSION_CODENAME}'  matched filter  '${DOCKER_MOBY_ARCHIVE_VERSION_CODENAMES}'"
+else
+    get_common_setting DOCKER_LICENSED_ARCHIVE_VERSION_CODENAMES
+    if [[ "${DOCKER_LICENSED_ARCHIVE_VERSION_CODENAMES}" != *"${VERSION_CODENAME}"* ]]; then
+        err "Unsupported distribution version '${VERSION_CODENAME}'. To resolve, please choose a compatible OS distribution"
+        err "Support distributions include:  ${DOCKER_LICENSED_ARCHIVE_VERSION_CODENAMES}"
+        exit 1
+    fi
+    echo "Distro codename  '${VERSION_CODENAME}'  matched filter  '${DOCKER_LICENSED_ARCHIVE_VERSION_CODENAMES}'"
+fi
+
 # Set up the necessary apt repos (either Microsoft's or Docker's)
 if [ "${USE_MOBY}" = "true" ]; then
 

script-library/docker-in-docker-debian.sh

@@ -15,14 +15,28 @@ USE_MOBY=${3:-"true"}
 DOCKER_VERSION=${4:-"latest"} # The Docker/Moby Engine + CLI should match in version
 DOCKER_DASH_COMPOSE_VERSION=${5:-"v1"} # v1 or v2
 MICROSOFT_GPG_KEYS_URI="https://packages.microsoft.com/keys/microsoft.asc"
+DOCKER_DASH_COMPOSE_VERSION="1"
+DOCKER_MOBY_ARCHIVE_VERSION_CODENAMES="buster bullseye bionic focal jammy"
+DOCKER_LICENSED_ARCHIVE_VERSION_CODENAMES="buster bullseye bionic focal hirsute impish jammy"
 
+# Default: Exit on any failure.
 set -e
 
+# Setup STDERR.
+err() {
+    echo "(!) $*" >&2
+}
+
 if [ "$(id -u)" -ne 0 ]; then
-    echo -e 'Script must be run as root. Use sudo, su, or add "USER root" to your Dockerfile before running this script.'
+    err 'Script must be run as root. Use sudo, su, or add "USER root" to your Dockerfile before running this script.'
     exit 1
 fi
 
+###################
+# Helper Functions
+# See: https://github.com/microsoft/vscode-dev-containers/blob/main/script-library/shared/utils.sh
+###################
+
 # Determine the appropriate non-root user
 if [ "${USERNAME}" = "auto" ] || [ "${USERNAME}" = "automatic" ]; then
     USERNAME=""
@@ -97,20 +111,50 @@ find_version_from_git_tags() {
             declare -g ${variable_name}="$(echo "${version_list}" | head -n 1)"
         else
             set +e
-            declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")"
+                declare -g ${variable_name}="$(echo "${version_list}" | grep -E -m 1 "^${requested_version//./\\.}([\\.\\s]|$)")"
             set -e
         fi
     fi
     if [ -z "${!variable_name}" ] || ! echo "${version_list}" | grep "^${!variable_name//./\\.}$" > /dev/null 2>&1; then
-        echo -e "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2
+        err "Invalid ${variable_name} value: ${requested_version}\nValid values:\n${version_list}" >&2
         exit 1
     fi
     echo "${variable_name}=${!variable_name}"
 }
 
+###########################################
+# Start docker-in-docker installation
+###########################################
+
 # Ensure apt is in non-interactive to avoid prompts
 export DEBIAN_FRONTEND=noninteractive
 
+
+# Source /etc/os-release to get OS info
+. /etc/os-release
+# Fetch host/container arch.
+architecture="$(dpkg --print-architecture)"
+
+# Check if distro is suppported
+if [ "${USE_MOBY}" = "true" ]; then
+    # 'get_common_setting' allows attribute to be updated remotely
+    get_common_setting DOCKER_MOBY_ARCHIVE_VERSION_CODENAMES
+    if [[ "${DOCKER_MOBY_ARCHIVE_VERSION_CODENAMES}" != *"${VERSION_CODENAME}"* ]]; then
+        err "Unsupported  distribution version '${VERSION_CODENAME}'. To resolve, either: (1) set feature option '\"moby\": false' , or (2) choose a compatible OS distribution"
+        err "Support distributions include:  ${DOCKER_MOBY_ARCHIVE_VERSION_CODENAMES}"
+        exit 1
+    fi
+    echo "Distro codename  '${VERSION_CODENAME}'  matched filter  '${DOCKER_MOBY_ARCHIVE_VERSION_CODENAMES}'"
+else
+    get_common_setting DOCKER_LICENSED_ARCHIVE_VERSION_CODENAMES
+    if [[ "${DOCKER_LICENSED_ARCHIVE_VERSION_CODENAMES}" != *"${VERSION_CODENAME}"* ]]; then
+        err "Unsupported distribution version '${VERSION_CODENAME}'. To resolve, please choose a compatible OS distribution"
+        err "Support distributions include:  ${DOCKER_LICENSED_ARCHIVE_VERSION_CODENAMES}"
+        exit 1
+    fi
+    echo "Distro codename  '${VERSION_CODENAME}'  matched filter  '${DOCKER_LICENSED_ARCHIVE_VERSION_CODENAMES}'"
+fi
+
 # Install dependencies
 check_packages apt-transport-https curl ca-certificates pigz iptables gnupg2 dirmngr
 if ! type git > /dev/null 2>&1; then
@@ -124,10 +168,7 @@ if type iptables-legacy > /dev/null 2>&1; then
     update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy
 fi
 
-# Source /etc/os-release to get OS info
-. /etc/os-release
-# Fetch host/container arch.
-architecture="$(dpkg --print-architecture)"
+
 
 # Set up the necessary apt repos (either Microsoft's or Docker's)
 if [ "${USE_MOBY}" = "true" ]; then
@@ -165,11 +206,11 @@ else
     # Regex needs to handle debian package version number format: https://www.systutorials.com/docs/linux/man/5-deb-version/
     docker_version_regex="^(.+:)?${docker_version_dot_plus_escaped}([\\.\\+ ~:-]|$)"
     set +e # Don't exit if finding version fails - will handle gracefully
-    cli_version_suffix="=$(apt-cache madison ${cli_package_name} | awk -F"|" '{print $2}' | sed -e 's/^[ \t]*//' | grep -E -m 1 "${docker_version_regex}")"
-    engine_version_suffix="=$(apt-cache madison ${engine_package_name} | awk -F"|" '{print $2}' | sed -e 's/^[ \t]*//' | grep -E -m 1 "${docker_version_regex}")"
+        cli_version_suffix="=$(apt-cache madison ${cli_package_name} | awk -F"|" '{print $2}' | sed -e 's/^[ \t]*//' | grep -E -m 1 "${docker_version_regex}")"
+        engine_version_suffix="=$(apt-cache madison ${engine_package_name} | awk -F"|" '{print $2}' | sed -e 's/^[ \t]*//' | grep -E -m 1 "${docker_version_regex}")"
     set -e
     if [ -z "${engine_version_suffix}" ] || [ "${engine_version_suffix}" = "=" ] || [ -z "${cli_version_suffix}" ] || [ "${cli_version_suffix}" = "=" ] ; then
-        echo "(!) No full or partial Docker / Moby version match found for \"${DOCKER_VERSION}\" on OS ${ID} ${VERSION_CODENAME} (${architecture}). Available versions:"
+        err "No full or partial Docker / Moby version match found for \"${DOCKER_VERSION}\" on OS ${ID} ${VERSION_CODENAME} (${architecture}). Available versions:"
         apt-cache madison ${cli_package_name} | awk -F"|" '{print $2}' | grep -oP '^(.+:)?\K.+'
         exit 1
     fi
@@ -182,8 +223,17 @@ if type docker > /dev/null 2>&1 && type dockerd > /dev/null 2>&1; then
     echo "Docker / Moby CLI and Engine already installed."
 else
     if [ "${USE_MOBY}" = "true" ]; then
-        apt-get -y install --no-install-recommends moby-cli${cli_version_suffix} moby-buildx moby-engine${engine_version_suffix}
-        apt-get -y install --no-install-recommends moby-compose || echo "(*) Package moby-compose (Docker Compose v2) not available for OS ${ID} ${VERSION_CODENAME} (${architecture}). Skipping."
+        # Install engine
+        set +e # Handle error gracefully
+            apt-get -y install --no-install-recommends moby-cli${cli_version_suffix} moby-buildx moby-engine${engine_version_suffix}
+            if [ $? -ne 0 ]; then
+                err "Packages for moby not available in OS ${ID} ${VERSION_CODENAME} (${architecture}). To resolve, either: (1) set feature option '\"moby\": false' , or (2) choose a compatible OS version (eg: 'ubuntu-20.04')."
+                exit 1
+            fi
+        set -e
+
+        # Install compose
+        apt-get -y install --no-install-recommends moby-compose || err "Package moby-compose (Docker Compose v2) not available for OS ${ID} ${VERSION_CODENAME} (${architecture}). Skipping."
     else
         apt-get -y install --no-install-recommends docker-ce-cli${cli_version_suffix} docker-ce${engine_version_suffix}
     fi
@@ -344,3 +394,5 @@ EOF
 
 chmod +x /usr/local/share/docker-init.sh
 chown ${USERNAME}:root /usr/local/share/docker-init.sh
+
+echo 'docker-in-docker-debian script has completed!'

script-library/shared/settings.env

@@ -52,3 +52,5 @@ H17gsBJr+opwJw/Zio2LMjQBOqlm3K1A4zFTh7wBC7He6KPQea1p2XAMgtvATtNe
 YLZATHZKTJyiqA==
 =vYOk
 -----END PGP PUBLIC KEY BLOCK-----"
+DOCKER_IN_DOCKER_MOBY_ARCHIVE_VERSION_CODENAMES="buster bullseye bionic focal jammy"
+DOCKER_IN_DOCKER_LICENSED_ARCHIVE_VERSION_CODENAMES="buster bullseye bionic focal hirsute impish jammy"

script-library/test/regression/run-scripts.sh

@@ -42,6 +42,22 @@ run_script()
     echo "**** Done! ****"
 }
 
+get_common_setting() {
+    # if [ "${common_settings_file_loaded}" != "true" ]; then
+    #     curl -sfL "https://aka.ms/vscode-dev-containers/script-library/settings.env" 2>/dev/null -o /tmp/vsdc-settings.env || echo "Could not download settings file. Skipping."
+    #     common_settings_file_loaded=true
+    # fi
+    cp  ${SCRIPT_DIR}/shared/settings.env  /tmp/vsdc-settings.env
+
+    if [ -f "/tmp/vsdc-settings.env" ]; then
+        local multi_line=""
+        if [ "$2" = "true" ]; then multi_line="-z"; fi
+        local result="$(grep ${multi_line} -oP "$1=\"?\K[^\"]+" /tmp/vsdc-settings.env | tr -d '\0')"
+        if [ ! -z "${result}" ]; then declare -g $1="${result}"; fi
+    fi
+    echo "$1=${!1}"
+}
+
 # Determine distro scripts to use
 . /etc/os-release
 architecture="$(uname -m)"
@@ -96,11 +112,19 @@ if [ "${DISTRO}" = "debian" ]; then
     run_script terraform "0.15.0 0.12.1"
     run_script sshd "2223 ${USERNAME} true random"
     run_script desktop-lite "${USERNAME} changeme false"
-    docker_version="20.10"
-    if [ "${VERSION_CODENAME}" = "stretch" ]; then
-        docker_version="19.03"
+
+    # docker-in-docker
+    get_common_setting DOCKER_MOBY_ARCHIVE_VERSION_CODENAMES
+    if [[ "${DOCKER_MOBY_ARCHIVE_VERSION_CODENAMES}" != *"${VERSION_CODENAME}"* ]]; then
+        # Do not use Moby
+        echo 'testing moby: false'
+        run_script docker-in-docker "false ${USERNAME} false latest v2"
+    else
+        # Use Moby
+         echo 'testing moby: true'
+        run_script docker-in-docker "false ${USERNAME} true latest v2"
     fi
-    run_script docker-in-docker "false ${USERNAME} false ${docker_version} v2"
+
     run_script powershell
     if [ "${architecture}" = "amd64" ] || [ "${architecture}" = "x86_64" ] || [ "${architecture}" = "arm64" ] || [ "${architecture}" = "aarch64" ]; then
         run_script java "13.0.2.j9-adpt /usr/local/sdkman2 ${USERNAME} false"
@@ -111,6 +135,7 @@ if [ "${DISTRO}" = "debian" ]; then
     run_script dotnet "3.1 true ${USERNAME} false /opt/dotnet dotnet"
 fi
 
+# TODO: Most of this script does not execute since 'docker-in-docker' is run above
 if [ "${DISTRO}" != "alpine" ]; then
     run_script docker "true /var/run/docker-host.sock /var/run/docker.sock ${USERNAME}"
 fi