diff --git a/.gitignore b/.gitignore index 87618221..69f77654 100644 --- a/.gitignore +++ b/.gitignore @@ -185,3 +185,6 @@ infra/tests/* # Unikraft .unikraft + +# mise-en-place +.mise.toml diff --git a/images/chromium-headful/Dockerfile b/images/chromium-headful/Dockerfile index 55d88330..654526c9 100644 --- a/images/chromium-headful/Dockerfile +++ b/images/chromium-headful/Dockerfile @@ -7,12 +7,16 @@ ENV CGO_ENABLED=0 COPY server/go.mod ./ COPY server/go.sum ./ -RUN go mod download +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + go mod download COPY server/ . # Build kernel-images API -RUN GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH:-amd64} \ +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH:-amd64} \ go build -ldflags="-s -w" -o /out/kernel-images-api ./cmd/api # Build chromium launcher @@ -23,19 +27,22 @@ RUN GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH:-amd64} \ FROM node:22-bullseye-slim AS client WORKDIR /src COPY images/chromium-headful/client/package*.json ./ -RUN npm install +RUN --mount=type=cache,target=/root/.npm npm install COPY images/chromium-headful/client/ . -RUN npm run build +RUN --mount=type=cache,target=/root/.npm npm run build # xorg dependencies FROM docker.io/ubuntu:22.04 AS xorg-deps WORKDIR /xorg ENV DEBIAN_FRONTEND=noninteractive -RUN set -eux; \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private,id=ubuntu2204-aptcache \ + --mount=type=cache,target=/var/lib/apt,sharing=private,id=ubuntu2204-aptlib \ + rm -f /etc/apt/apt.conf.d/docker-clean; \ + echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache; \ + set -eux; \ apt-get update; \ - apt-get install -y \ - git gcc pkgconf autoconf automake libtool make xorg-dev xutils-dev \ - && rm -rf /var/lib/apt/lists/*; + apt-get --no-install-recommends -y install \ + git gcc pkgconf autoconf automake libtool make xorg-dev xutils-dev; COPY images/chromium-headful/xorg-deps/ /xorg/ # build xf86-video-dummy v0.3.8 with RandR support RUN set -eux; \ @@ -61,9 +68,14 @@ FROM docker.io/ubuntu:22.04 ENV DEBIAN_FRONTEND=noninteractive ENV DEBIAN_PRIORITY=high -RUN apt-get update && \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private,id=ubuntu2204-aptcache \ + --mount=type=cache,target=/var/lib/apt,sharing=private,id=ubuntu2204-aptlib \ + rm -f /etc/apt/apt.conf.d/docker-clean; \ + echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache; \ + apt-get update && \ apt-get -y upgrade && \ - apt-get -y install \ + apt-get --no-install-recommends -y install \ + gpg-agent \ # UI Requirements xvfb \ xterm \ @@ -95,38 +107,55 @@ RUN apt-get update && \ software-properties-common && \ # Userland apps sudo add-apt-repository ppa:mozillateam/ppa && \ - sudo apt-get install -y --no-install-recommends \ - libreoffice \ + sudo apt-get --no-install-recommends -y install \ x11-apps \ - xpdf \ - gedit \ - xpaint \ tint2 \ - galculator \ - pcmanfm \ wget \ xdg-utils \ libvulkan1 \ fonts-liberation \ - unzip && \ - apt-get clean + unzip; # install ffmpeg manually since the version available in apt is from the 4.x branch due to #drama. # as of writing these static builds will be the latest 7.0.x release. -RUN set -eux; \ - URL="https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz"; \ - echo "Downloading FFmpeg static build from $URL"; \ - curl -fsSL "$URL" -o /tmp/ffmpeg.tar.xz; \ - tar -xJf /tmp/ffmpeg.tar.xz -C /tmp; \ - install -m755 /tmp/ffmpeg-*/ffmpeg /usr/local/bin/ffmpeg; \ - install -m755 /tmp/ffmpeg-*/ffprobe /usr/local/bin/ffprobe; \ +RUN --mount=type=cache,target=/tmp/cache/ffmpeg,id=ffmpeg \ + <<-'EOT' + set -eux + URL="https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz" + echo "Downloading FFmpeg MD5 checksum" + if ! curl --connect-timeout 10 -fsSL "${URL}.md5" -o /tmp/cache/ffmpeg/ffmpeg.tar.xz.md5; then + echo "Failed to connect to ffmpeg static build provider for MD5 checksum." + echo "Checking for cached version to use." + if [ ! -f /tmp/cache/ffmpeg/ffmpeg.tar.xz.md5 ]; then + echo "Unable to locate cached MD5 checksum. Exiting." + exit 1 + else + echo "Found cached MD5 checksum." + fi + fi + sed -i -e 's/ .*$/ \/tmp\/cache\/ffmpeg\/ffmpeg.tar.xz/' /tmp/cache/ffmpeg/ffmpeg.tar.xz.md5 + echo "Checking cache for FFmpeg archive and validating MD5 checksum" + if md5sum --check /tmp/cache/ffmpeg/ffmpeg.tar.xz.md5; then + echo "Checksum validated, using cached FFmpeg archive" + else + echo "Downloading FFmpeg static build from $URL" + curl -fsSL "$URL" -o /tmp/cache/ffmpeg/ffmpeg.tar.xz + echo "Validating MD5 checksum of FFmpeg static build download" + md5sum --check /tmp/cache/ffmpeg/ffmpeg.tar.xz.md5 + fi + tar -xJf /tmp/cache/ffmpeg/ffmpeg.tar.xz -C /tmp + install -m755 /tmp/ffmpeg-*/ffmpeg /usr/local/bin/ffmpeg + install -m755 /tmp/ffmpeg-*/ffprobe /usr/local/bin/ffprobe rm -rf /tmp/ffmpeg* +EOT # runtime ENV USERNAME=root -RUN set -eux; \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private,id=ubuntu2204-aptcache \ + --mount=type=cache,target=/var/lib/apt,sharing=private,id=ubuntu2204-aptlib \ + set -eux; \ apt-get update; \ - apt-get install -y --no-install-recommends \ + apt-get --no-install-recommends -y install \ wget ca-certificates python2 supervisor xclip xdotool \ pulseaudio dbus-x11 xserver-xorg-video-dummy \ libcairo2 libxcb1 libxrandr2 libxv1 libopus0 libvpx7 \ @@ -138,7 +167,7 @@ RUN set -eux; \ # install libxcvt0 (not available in debian:bullseye) ARCH=$(dpkg --print-architecture); \ wget http://ftp.de.debian.org/debian/pool/main/libx/libxcvt/libxcvt0_0.1.2-1_${ARCH}.deb; \ - apt-get install --no-install-recommends ./libxcvt0_0.1.2-1_${ARCH}.deb; \ + apt-get --no-install-recommends install ./libxcvt0_0.1.2-1_${ARCH}.deb; \ rm ./libxcvt0_0.1.2-1_${ARCH}.deb; \ # # workaround for an X11 problem: http://blog.tigerteufel.de/?p=476 @@ -153,14 +182,17 @@ RUN set -eux; \ /home/$USERNAME/.local/share/xorg; \ chmod 1777 /var/log/neko; \ chown $USERNAME /var/log/neko/ /tmp/runtime-$USERNAME; \ - chown -R $USERNAME:$USERNAME /home/$USERNAME; \ - # clean up - apt-get clean -y; \ - rm -rf /var/lib/apt/lists/* /var/cache/apt/ + chown -R $USERNAME:$USERNAME /home/$USERNAME; # install chromium and sqlite3 for debugging the cookies file -RUN add-apt-repository -y ppa:xtradeb/apps -RUN apt update -y && apt install -y chromium sqlite3 +RUN --mount=type=cache,target=/var/cache/apt,sharing=private,id=ubuntu2204-aptcache \ + --mount=type=cache,target=/var/lib/apt,sharing=private,id=ubuntu2204-aptlib \ + add-apt-repository -y ppa:xtradeb/apps; +RUN --mount=type=cache,target=/var/cache/apt,sharing=private,id=ubuntu2204-aptcache \ + --mount=type=cache,target=/var/lib/apt,sharing=private,id=ubuntu2204-aptlib \ + apt update -y && \ + apt -y install chromium && \ + apt --no-install-recommends -y install sqlite3; # install Node.js 22.x by copying from the node:22-bullseye-slim stage COPY --from=node-22 /usr/local/bin/node /usr/local/bin/node @@ -174,7 +206,7 @@ RUN set -eux; \ fi # Install TypeScript and Playwright globally -RUN npm install -g typescript playwright-core tsx +RUN --mount=type=cache,target=/root/.npm npm install -g typescript playwright-core tsx # setup desktop env & app ENV DISPLAY_NUM=1 diff --git a/images/chromium-headful/client/Dockerfile b/images/chromium-headful/client/Dockerfile index da1f452e..85e77c36 100644 --- a/images/chromium-headful/client/Dockerfile +++ b/images/chromium-headful/client/Dockerfile @@ -6,12 +6,12 @@ WORKDIR /src # # install dependencies COPY package*.json ./ -RUN npm install +RUN --mount=type=cache,target=/root/.npm npm install # # build client COPY . . -RUN npm run build +RUN --mount=type=cache,target=/root/.npm npm run build # # artifacts from this stage diff --git a/images/chromium-headful/xorg-deps/Dockerfile b/images/chromium-headful/xorg-deps/Dockerfile index 2e770363..5e38145c 100644 --- a/images/chromium-headful/xorg-deps/Dockerfile +++ b/images/chromium-headful/xorg-deps/Dockerfile @@ -4,11 +4,15 @@ FROM $BASE_IMAGE AS xorg-deps WORKDIR /xorg ENV DEBIAN_FRONTEND=noninteractive -RUN set -eux; \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private,id=ubuntu2204-aptcache \ + --mount=type=cache,target=/var/lib/apt,sharing=private,id=ubuntu2204-aptlib \ + rm -f /etc/apt/apt.conf.d/docker-clean; \ + echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache; \ + set -eux; \ apt-get update; \ - apt-get install -y \ - git gcc pkgconf autoconf automake libtool make xorg-dev xutils-dev \ - && rm -rf /var/lib/apt/lists/*; + apt-get --no-install-recommends -y install \ + git gcc pkgconf autoconf automake libtool make xorg-dev xutils-dev; + COPY . /xorg/ diff --git a/images/chromium-headful/xorg-deps/xf86-input-neko/Dockerfile b/images/chromium-headful/xorg-deps/xf86-input-neko/Dockerfile index 4eb666e0..864de108 100644 --- a/images/chromium-headful/xorg-deps/xf86-input-neko/Dockerfile +++ b/images/chromium-headful/xorg-deps/xf86-input-neko/Dockerfile @@ -2,11 +2,14 @@ FROM debian:bullseye-slim ENV DEBIAN_FRONTEND=noninteractive -RUN set -eux; \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private,id=debian-bullseye-aptcache \ + --mount=type=cache,target=/var/lib/apt,sharing=private,id=debian-bullseye-aptlib \ + rm -f /etc/apt/apt.conf.d/docker-clean; \ + echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache; \ + set -eux; \ apt-get update; \ - apt-get install -y \ - gcc pkgconf autoconf automake libtool make xorg-dev xutils-dev \ - && rm -rf /var/lib/apt/lists/*; + apt-get install --no-install-recommends -y \ + gcc pkgconf autoconf automake libtool make xorg-dev xutils-dev; WORKDIR /app diff --git a/images/chromium-headless/image/Dockerfile b/images/chromium-headless/image/Dockerfile index dbf63b9c..23fd86dc 100644 --- a/images/chromium-headless/image/Dockerfile +++ b/images/chromium-headless/image/Dockerfile @@ -9,12 +9,16 @@ ENV CGO_ENABLED=0 # Go module dependencies first for better layer caching COPY server/go.mod ./ COPY server/go.sum ./ -RUN go mod download +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + go mod download COPY server/ . # Build kernel-images API -RUN GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH:-amd64} \ +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH:-amd64} \ go build -ldflags="-s -w" -o /out/kernel-images-api ./cmd/api # Build chromium launcher @@ -23,10 +27,13 @@ RUN GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH:-amd64} \ FROM node:22-bullseye-slim AS node-22 FROM docker.io/ubuntu:22.04 - -RUN set -xe; \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=private,id=ubuntu2204-aptcache \ + --mount=type=cache,target=/var/lib/apt,sharing=private,id=ubuntu2204-aptlib \ + rm -f /etc/apt/apt.conf.d/docker-clean; \ + echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache; \ + set -xe; \ apt-get -yqq update; \ - apt-get -yqq install \ + apt-get -yqq --no-install-recommends install \ libcups2 \ libnss3 \ libatk1.0-0 \ @@ -43,12 +50,13 @@ RUN set -xe; \ libxrandr2 \ libgbm1 \ libnss3; \ - apt-get -yqq install \ + apt-get -yqq --no-install-recommends install \ ca-certificates \ curl \ build-essential \ libssl-dev \ git \ + gpg-agent \ dbus \ dbus-x11 \ xvfb \ @@ -58,21 +66,51 @@ RUN set -xe; \ supervisor; # install chromium and sqlite3 for debugging the cookies file -RUN add-apt-repository -y ppa:xtradeb/apps -RUN apt update -y && apt install -y chromium sqlite3 +RUN --mount=type=cache,target=/var/cache/apt,sharing=private,id=ubuntu2204-aptcache \ + --mount=type=cache,target=/var/lib/apt,sharing=private,id=ubuntu2204-aptlib \ + add-apt-repository -y ppa:xtradeb/apps +RUN --mount=type=cache,target=/var/cache/apt,sharing=private,id=ubuntu2204-aptcache \ + --mount=type=cache,target=/var/lib/apt,sharing=private,id=ubuntu2204-aptlib \ + apt-get update -y && \ + apt-get -y install chromium && \ + apt-get --no-install-recommends -y install sqlite3; # Install FFmpeg (latest static build) for the recording server -RUN set -eux; \ - URL="https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz"; \ - echo "Downloading FFmpeg static build from $URL"; \ - curl -fsSL "$URL" -o /tmp/ffmpeg.tar.xz; \ - tar -xJf /tmp/ffmpeg.tar.xz -C /tmp; \ - install -m755 /tmp/ffmpeg-*/ffmpeg /usr/local/bin/ffmpeg; \ - install -m755 /tmp/ffmpeg-*/ffprobe /usr/local/bin/ffprobe; \ +RUN --mount=type=cache,target=/tmp/cache/ffmpeg,id=ffmpeg \ + <<-'EOT' + set -eux + URL="https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz" + echo "Downloading FFmpeg MD5 checksum" + if ! curl --connect-timeout 10 -fsSL "${URL}.md5" -o /tmp/cache/ffmpeg/ffmpeg.tar.xz.md5; then + echo "Failed to connect to ffmpeg static build provider for MD5 checksum." + echo "Checking for cached version to use." + if [ ! -f /tmp/cache/ffmpeg/ffmpeg.tar.xz.md5 ]; then + echo "Unable to locate cached MD5 checksum. Exiting." + exit 1 + else + echo "Found cached MD5 checksum." + fi + fi + sed -i -e 's/ .*$/ \/tmp\/cache\/ffmpeg\/ffmpeg.tar.xz/' /tmp/cache/ffmpeg/ffmpeg.tar.xz.md5 + echo "Checking cache for FFmpeg archive and validating MD5 checksum" + if md5sum --check /tmp/cache/ffmpeg/ffmpeg.tar.xz.md5; then + echo "Checksum validated, using cached FFmpeg archive" + else + echo "Downloading FFmpeg static build from $URL" + curl -fsSL "$URL" -o /tmp/cache/ffmpeg/ffmpeg.tar.xz + echo "Validating MD5 checksum of FFmpeg static build download" + md5sum --check /tmp/cache/ffmpeg/ffmpeg.tar.xz.md5 + fi + tar -xJf /tmp/cache/ffmpeg/ffmpeg.tar.xz -C /tmp + install -m755 /tmp/ffmpeg-*/ffmpeg /usr/local/bin/ffmpeg + install -m755 /tmp/ffmpeg-*/ffprobe /usr/local/bin/ffprobe rm -rf /tmp/ffmpeg* +EOT # Remove upower to prevent spurious D-Bus activations and logs -RUN apt-get -yqq purge upower || true && rm -rf /var/lib/apt/lists/* +RUN --mount=type=cache,target=/var/cache/apt,sharing=private,id=ubuntu2204-aptcache \ + --mount=type=cache,target=/var/lib/apt,sharing=private,id=ubuntu2204-aptlib \ + apt-get -yqq purge upower || true # install Node.js 22.x by copying from the node:22-bullseye-slim stage COPY --from=node-22 /usr/local/bin/node /usr/local/bin/node