Merge pull request #2 from AnastaZIuk/buildkit

AnastaZIuk · web-flow · commit bd38b03a2507 · 2025-03-18T20:47:51.000+01:00
Docker -&gt; Buildkit + Containerd for Windows runners
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -14,61 +14,161 @@ jobs:
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4
-
-      - name: Restore Cached Image TAR
-        uses: actions/cache@v4
         with:
-          path: cache-image.tar
-          key: docker-image-${{ runner.os }}-${{ github.sha }}
-          restore-keys: |
-            docker-image-${{ runner.os }}-
-
-      - name: Load Cached Image
-        shell: pwsh
-        id: load_cache
-        run: |
-          if (Test-Path cache-image.tar) {
-            docker load -i cache-image.tar
-            echo "CACHE_HIT=true" >> $env:GITHUB_ENV
-          }
+          fetch-depth: 1
 
-      - name: Build Image Without Cache
-        if: env.CACHE_HIT != 'true'
+      - name: Setup environment
         shell: pwsh
         run: |
-          docker build -t app:latest .
-          
-      - name: Save Docker Image to TAR
-        if: env.CACHE_HIT != 'true'
-        shell: pwsh
-        run: |
-          docker save -o cache-image.tar app:latest
-      
-      - name: Cache Image TAR
-        if: env.CACHE_HIT != 'true'
-        uses: actions/cache@v4
+          Set-MpPreference -DisableRealtimeMonitoring $true   
+      - name: Restore BuildKit Cache
+        uses: actions/cache/restore@v4
+        id: restore-buildkit-cache
+        with:
+          path: |
+            buildkit
+          key: buildkit-${{ runner.os }}-
+          restore-keys: |
+            buildkit-${{ runner.os }}-
+      - name: Restore Daemon Cache
+        uses: actions/cache/restore@v4
+        id: restore-droot-cache
         with:
-          path: cache-image.tar
-          key: docker-image-${{ runner.os }}-${{ github.sha }}
+          path: |
+            droot
+          key: droot-${{ runner.os }}-
           restore-keys: |
-            docker-image-${{ runner.os }}-
+            droot-${{ runner.os }}-
+      - name: Log in to Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: dr-private.devsh.eu
+          username: ${{ secrets.DOCKER_USERNAME  }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
 
-      - name: Run Nano Container
+      - name: Setup buildkit
         shell: pwsh
         run: |
-          docker run -di --name orphan -v "${{ github.workspace }}:C:\app" app:latest
-
-      - name: Nano Container - Configure CMake Project
+            if (-Not (Test-Path "buildkit")) {
+                Write-Host "📂 'buildkit' directory not found. Downloading and extracting daemons..."
+                New-Item -ItemType Directory -Path "buildkit" -Force
+                Invoke-WebRequest -Uri "https://github.com/moby/buildkit/releases/download/v0.20.1/buildkit-v0.20.1.windows-amd64.tar.gz" -OutFile "buildkit.tar.gz"
+                tar -xf buildkit.tar.gz -C buildkit
+                
+                Invoke-WebRequest -Uri "https://github.com/containerd/containerd/releases/download/v2.0.4/containerd-2.0.4-windows-amd64.tar.gz" -OutFile "containerd.tar.gz"
+                tar -xf containerd.tar.gz -C buildkit
+            } else {
+                Write-Host "✅ 'buildkit' directory already exists. Skipping download."
+            }
+            .\cni\setup-nat.ps1
+            New-Item -ItemType Directory -Path "buildkit/logs" -Force
+            $Bin = "${{ github.workspace }}\buildkit\bin"
+            echo "PATH=$Bin;$env:PATH" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8
+      - name: Run daemons
         shell: pwsh
+        run: |    
+            function Wait-For {
+                param (
+                    [string]$Command,          
+                    [int]$TickWait = 1,    
+                    [int]$TotalTicks = 5 
+                )
+          
+                Write-Host "⏳ Waiting for command.. (Max retries: $TotalTicks, Interval: ${TickWait}s)"
+                
+                for ($i=1; $i -le $TotalTicks; $i++) {
+                    Write-Host "⏳ [$i/$TotalTicks]: $Command"
+                    if (Invoke-Expression $Command) {
+                        Write-Host "✅ Success!"
+                        return $true
+                    }
+                    Start-Sleep -Seconds $TickWait
+                }
+          
+                Write-Error "❌ Timeout: $Command did not succeed!"
+                exit 1
+            }
+            $logs = "buildkit/logs"
+            $cni = "${{ github.workspace }}\buildkit\cni"
+            $droot = "${{ github.workspace }}\droot"
+            Start-Job -ScriptBlock {
+                param($logs, $droot)
+                Start-Process containerd.exe `
+                -ArgumentList @("--root=$droot\containerd") `
+                  -RedirectStandardOutput "$logs/containerd.log" `
+                  -RedirectStandardError "$logs/containerd_error.log" `
+                  -NoNewWindow -PassThru
+            } -ArgumentList $logs, $droot
+            Wait-For "ctr --namespace buildkit image ls"
+            Start-Job -ScriptBlock {
+              param($logs, $cni, $droot)
+              Start-Process buildkitd.exe `
+              -ArgumentList @("--containerd-cni-config-path=$cni\0-containerd-nat.conf",
+                              "--containerd-cni-binary-dir=$cni", `
+                              "--root=$droot\buildkitd") `
+              -RedirectStandardOutput "$logs/buildkitd.log" `
+              -RedirectStandardError "$logs/buildkitd_error.log" `
+              -NoNewWindow -PassThru
+              } -ArgumentList $logs, $cni, $droot
+            Wait-For "buildctl du"
+      - name: Build base image
+        shell: cmd
         run: |
-          docker exec -w "C:\app\tests" -i orphan cmd /c cmake --preset configure-msvc-winsdk
-
-      - name: Nano Container - Build the Project
+            buildctl build --frontend=dockerfile.v0 --local context=. --local dockerfile=. ^
+              --output=type=image,name=dr-private.devsh.eu/actions/cache/nano/base,push=true ^
+              --export-cache=type=inline ^
+              --import-cache=type=registry,ref=dr-private.devsh.eu/actions/cache/nano/base ^
+              --opt target=buildtools
+      - name: Prune images
         shell: pwsh
         run: |
-          docker exec -w "C:\app\tests" -i orphan cmd /c cmake --build --preset build-msvc-winsdk --config Release
-
-      - name: Nano Container - Test the Project
+          buildctl prune
+      - name: Stop daemons
         shell: pwsh
         run: |
-          docker exec -w "C:\app\tests\build\src" -i orphan cmd /c ctest -C Release --stop-on-failure --verbose
+            Write-Host "🔴 Stopping background jobs..."
+            if (Get-Job) {
+                Get-Job | Stop-Job -Force
+                Get-Job | Remove-Job -Force
+                Write-Host "✅ Background jobs stopped!"
+            } else {
+                Write-Host "⚠️ No background jobs found."
+            }
+            Write-Host "🔴 Stopping containerd and buildkitd..."
+            if (Get-Process -Name "containerd" -ErrorAction SilentlyContinue) {
+                Stop-Process -Name "containerd" -Force
+                Write-Host "✅ containerd stopped!"
+            } else {
+                Write-Host "⚠️ containerd is not running."
+            }
+            
+            if (Get-Process -Name "buildkitd" -ErrorAction SilentlyContinue) {
+                Stop-Process -Name "buildkitd" -Force
+                Write-Host "✅ buildkitd stopped!"
+            } else {
+                Write-Host "⚠️ buildkitd is not running."
+            }
+      #- name: Build app image
+      #  shell: cmd
+      #  run: |
+      #      buildctl build --frontend=dockerfile.v0 --local context=. --local dockerfile=. ^            
+      #        --output type=image,name=nano/app,push=false ^
+      #        --export-cache type=local,dest=buildkit/.cache,mode=max ^
+      #        --import-cache type=local,src=buildkit/.cache ^
+      #        --opt target=nano
+            
+      - name: Save BuildKit Cache
+        id: save-buildkit-cache
+        uses: actions/cache/save@v4
+        with:
+          path: |
+            buildkit
+          key: buildkit-${{ runner.os }}-
+          
+      - name: Save Daemon Cache
+        id: save-droot-cache
+        uses: actions/cache/save@v4
+        with:
+          path: |
+            droot
+          key: droot-${{ runner.os }}-
diff --git a/Dockerfile b/Dockerfile
@@ -1,6 +1,6 @@
 # escape=`
 
-FROM --platform=windows/amd64 mcr.microsoft.com/windows/servercore:ltsc2022 AS BUILD_TOOLS
+FROM mcr.microsoft.com/windows/servercore:ltsc2022 AS buildtools
 
 ARG WINDOWS_11_SDK_VERSION="22621"
 ARG VC_VERSION="14.42.17.12"
@@ -25,7 +25,9 @@ echo "Error: Expected MSVC version directory %MSVC_VERSION% does not exist!" &&
 
 RUN (echo { "WINDOWS_SDK_VERSION": "10.0.%WINDOWS_11_SDK_VERSION%.0", "VC_VERSION": "%VC_VERSION%", "MSVC_VERSION": "%MSVC_VERSION%" }) > %BUILD_TOOLS_DIR%\env.json
 
-FROM mcr.microsoft.com/windows/nanoserver:ltsc2022
+FROM mcr.microsoft.com/windows/nanoserver:ltsc2022 AS nano
+
+SHELL ["cmd", "/S", "/C"]
 
 ARG BUILD_TOOLS_DIR="C:\BuildTools"
 ARG WINDOWS_KITS_10_DIR="C:\WindowsKits10SDK"
@@ -38,9 +40,6 @@ ARG NINJA_DIR="C:\Ninja"
 ARG NASM_VERSION=2.16.03
 ARG NASM_DIR="C:\Nasm"
 
-COPY --link --from=BUILD_TOOLS ["C:/Program Files (x86)/Windows Kits/10", "${WINDOWS_KITS_10_DIR}"]
-COPY --link --from=BUILD_TOOLS ["C:/artifacts", "${BUILD_TOOLS_DIR}"]
-
 ENV CMAKE_WINDOWS_KITS_10_DIR=${WINDOWS_KITS_10_DIR} BUILD_TOOLS_DIR=${BUILD_TOOLS_DIR} `
 CMAKE_VERSION=${CMAKE_VERSION} CMAKE_URL=https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-windows-x86_64.zip CMAKE_DIR=${CMAKE_DIR} `
 PYTHON_VERSION=${PYTHON_VERSION} PYTHON_URL=https://www.python.org/ftp/python/${PYTHON_VERSION}/python-${PYTHON_VERSION}-embed-amd64.zip PYTHON_DIR=${PYTHON_DIR} `
@@ -52,10 +51,11 @@ RUN cd C:\Temp && mkdir "%PYTHON_DIR%" && curl -SL --output python.zip %PYTHON_U
 RUN cd C:\Temp && mkdir "%NINJA_DIR%" && curl -SL --output ninja.zip %NINJA_URL% && tar -xf ninja.zip -C "%NINJA_DIR%" && del ninja.zip
 RUN cd C:\Temp && mkdir "%NASM_DIR%" && curl -SL --output nasm.zip %NASM_URL% && tar -xf nasm.zip -C "%NASM_DIR%" && del nasm.zip
 
+RUN echo "test"
+COPY --from=buildtools ["C:/artifacts", "${BUILD_TOOLS_DIR}"]
 RUN cd %BUILD_TOOLS_DIR% && "%PYTHON_DIR%\python.exe" -c "import json, os; env=json.load(open('./env.json')); [os.system(f'setx {k} \"{v}\"') for k,v in env.items()]" `
 && setx PATH "%CMAKE_DIR%\cmake-%CMAKE_VERSION%-windows-x86_64\bin;%PYTHON_DIR%;%NINJA_DIR%;%NASM_DIR%\nasm-%NASM_VERSION%;%PATH%" `
 && setx MSVC_TOOLSET_DIR "%BUILD_TOOLS_DIR%\VC\Tools\MSVC\%MSVC_VERSION%"
+COPY --from=buildtools ["C:/Program Files (x86)/Windows Kits/10", "${WINDOWS_KITS_10_DIR}"]
 
-SHELL ["cmd", "/c"]
 ENTRYPOINT ["cmd"]
-CMD ["/noexit"]
diff --git a/cni/setup-nat.ps1 b/cni/setup-nat.ps1
@@ -0,0 +1,47 @@
+# https://github.com/moby/buildkit/blob/248ff7c29ed979ef0c142b8166dd6d192cf0b215/docs/windows.md#cni--networking-setup
+$workspaceDir = Resolve-Path "$PSScriptRoot\.."
+$networkName = 'nat'
+$natInfo = Get-HnsNetwork -ErrorAction Ignore | Where-Object { $_.Name -eq $networkName }
+if ($null -eq $natInfo) {
+    throw "NAT network not found, check if you enabled containers, Hyper-V features and restarted the machine"
+}
+$gateway = $natInfo.Subnets[0].GatewayAddress
+$subnet = $natInfo.Subnets[0].AddressPrefix
+
+$cniVersion = "1.0.0"
+$cniPluginVersion = "0.3.1"
+$cniBinDir = "$workspaceDir\buildkit\cni"
+$cniConfPath = "$cniBinDir\0-containerd-nat.conf"
+
+if (-Not (Test-Path "$cniConfPath")) {
+    Write-Host "📂 'cni' directory not found. Downloading and extracting cni binaries..."
+
+    mkdir $cniBinDir -Force
+    curl.exe -LO https://github.com/microsoft/windows-container-networking/releases/download/v$cniPluginVersion/windows-container-networking-cni-amd64-v$cniPluginVersion.zip
+    Expand-Archive -Path windows-container-networking-cni-amd64-v$cniPluginVersion.zip -DestinationPath $cniBinDir -Force
+} else {
+    Write-Host "✅ 'cni' directory already exists. Skipping download."
+}
+
+$natConfig = @"
+{
+    "cniVersion": "$cniVersion",
+    "name": "$networkName",
+    "type": "nat",
+    "master": "Ethernet",
+    "ipam": {
+        "subnet": "$subnet",
+        "routes": [
+            {
+                "gateway": "$gateway"
+            }
+        ]
+    },
+    "capabilities": {
+        "portMappings": true,
+        "dns": true
+    }
+}
+"@
+Set-Content -Path $cniConfPath -Value $natConfig
+cat $cniConfPath
diff --git a/wsl/setup-wsl.ps1 b/wsl/setup-wsl.ps1
@@ -0,0 +1,72 @@
+param (
+    [string]$WslMsiUrl = "https://github.com/microsoft/WSL/releases/download/2.4.12/wsl.2.4.12.0.x64.msi",
+    [string]$AlpineZipUrl = "https://github.com/yuk7/AlpineWSL/releases/download/3.21.3-0/Alpine.zip",
+    [string]$LinuxKernelbzImageUrl = "https://github.com/Devsh-Graphics-Programming/WSL2-Linux-Kernel/releases/download/wsl2-kernel-13885376672/bzImage",
+    [string]$RegistryTag = "2.8.3"
+)
+
+$WorkspaceDir = Resolve-Path "$PSScriptRoot\.."
+$LinuxDir = "$WorkspaceDir\linux"
+if (!(Test-Path $LinuxDir)) { New-Item -ItemType Directory -Path $LinuxDir | Out-Null }
+
+$bzImagePath = "$LinuxDir\bzImage"
+$ext4Path = "$LinuxDir\ext4.vhdx"
+$wslInstaller = "$LinuxDir\wsl.msi"
+
+$bzImageCached = Test-Path $bzImagePath
+$ext4Cached = Test-Path $ext4Path
+$wslCached = Test-Path $wslInstaller
+
+if ($bzImageCached -and $ext4Cached -and $wslCached) {
+    echo "REGISTRY_CACHE_HIT=true" >> $env:GITHUB_ENV
+}
+
+if (-not $wslCached) {
+    Write-Host "$wslInstaller not found in cache, downloading..."
+    Invoke-WebRequest -Uri $WslMsiUrl -OutFile $wslInstaller
+} else {
+    Write-Host "$wslInstaller found in cache, skipping download."
+}
+
+Write-Host "Updating WSL..."
+Start-Process -Wait "msiexec.exe" -ArgumentList "/i `"$wslInstaller`" /qn /norestart" -NoNewWindow -PassThru
+
+wsl --set-default-version 2
+
+Write-Host "Updating .wslconfig..."
+$wslConfigPath = "$env:USERPROFILE\.wslconfig"
+$kernelPath = $bzImagePath.Replace("\", "\\")
+$wslConfigContent = @"
+[wsl2]
+kernel=$kernelPath
+"@
+$wslConfigContent | Set-Content -Path $wslConfigPath -Encoding UTF8 -Force
+
+Write-Host "==== .wslconfig CONTENT ====" -ForegroundColor Green
+Write-Host $wslConfigContent -ForegroundColor Green
+
+if (-not $bzImageCached) {
+    Write-Host "$bzImagePath not found in cache, downloading..."
+    Invoke-WebRequest -Uri $LinuxKernelbzImageUrl -OutFile $bzImagePath
+} else {
+    Write-Host "$bzImagePath found in cache, skipping download."
+}
+
+if (-not $ext4Cached) {
+    Write-Host "$ext4Path not found in cache, creating Alpine instance with Docker registry..."
+    Invoke-WebRequest -Uri $AlpineZipUrl -OutFile "$WorkspaceDir\Alpine.zip"
+    Expand-Archive -Path "$WorkspaceDir\Alpine.zip" -DestinationPath $LinuxDir -Force
+    Write-Output "`n" | & "$LinuxDir\Alpine.exe"
+    wsl -s Alpine
+    wsl apk add docker
+    wsl nohup dockerd > /dockerd-log 2>&1 &
+    & "$WorkspaceDir\wsl\wait-for-daemon.ps1"
+    wsl docker pull registry:$RegistryTag
+} else {
+    Write-Host "$ext4Path found in cache, skipping download & importing!"
+    wsl --import-in-place Alpine "$ext4Path"
+    wsl -s Alpine
+}
+
+Write-Host "==== LINUX WSL DIRECTORY STRUCTURE ====" -ForegroundColor Cyan
+Get-ChildItem -Path $LinuxDir -Recurse | ForEach-Object { $_.FullName.Replace($PWD.Path, "").Replace("\", "/") }
diff --git a/wsl/wait-for-daemon.ps1 b/wsl/wait-for-daemon.ps1
@@ -0,0 +1,24 @@
+param (
+    [int]$maxRetries = 15,
+    [int]$delaySeconds = 1
+)
+
+$linuxDockerReady = $false
+$i = 0
+
+while ($i -lt $maxRetries) {
+    Write-Host "[$i/$maxRetries]: Waiting for Linux Docker daemon..."
+    Start-Sleep -Seconds $delaySeconds
+    wsl docker info > $null 2>&1
+    if ($LASTEXITCODE -eq 0) {
+        Write-Host "Linux Docker daemon running!" -ForegroundColor Green
+        $linuxDockerReady = $true
+        break
+    }
+    $i++
+}
+
+if (-not $linuxDockerReady) {
+    Write-Host "Failed to connect to Linux Docker daemon!" -ForegroundColor Red
+    exit 1
+}
