From 390771bbb38c4fd5c9a062ad492820a94fc7e285 Mon Sep 17 00:00:00 2001 From: James Brink Date: Sun, 14 Dec 2025 21:12:52 -0700 Subject: [PATCH 1/2] feat: Add multi-arch Docker builds and auto-download template inputs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Multi-architecture Docker support: - Build CPU images for both amd64 and arm64 (Apple Silicon compatible) - CUDA images remain x86_64 only (NVIDIA requirement) - Use QEMU emulation in CI for cross-architecture builds - Create multi-arch manifests for seamless pulling on any platform - Update README with Apple Silicon Docker instructions Auto-download workflow template input files: - Add template_inputs.sh script to download example images on startup - Fetch manifest from GitHub workflow_templates repository - Non-blocking download (doesn't fail startup if network unavailable) - Cache manifest for 7 days to minimize network requests - Skip existing files for faster subsequent startups - Add curl and jq to runtime PATH via makeWrapper 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- .github/workflows/docker.yml | 208 +++++++++++++++++++++++++++-------- CLAUDE.md | 7 +- README.md | 38 +++++-- flake.nix | 28 ++++- scripts/install.sh | 12 +- scripts/template_inputs.sh | 133 ++++++++++++++++++++++ 6 files changed, 364 insertions(+), 62 deletions(-) create mode 100644 scripts/template_inputs.sh diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 3967725..ac7df57 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -28,7 +28,7 @@ env: IMAGE_NAME: ${{ github.repository }} jobs: - build-and-push: + build: runs-on: ubuntu-latest permissions: contents: read @@ -36,13 +36,25 @@ jobs: id-token: write strategy: + fail-fast: false matrix: - variant: - - name: cpu + include: + # CPU variants for both architectures + - variant: cpu + arch: x86_64-linux nix_target: dockerImage + platform: linux/amd64 tag_suffix: '' - - name: cuda + - variant: cpu + arch: aarch64-linux + nix_target: dockerImage + platform: linux/arm64 + tag_suffix: '' + # CUDA only for x86_64 (NVIDIA CUDA not available for ARM) + - variant: cuda + arch: x86_64-linux nix_target: dockerImageCuda + platform: linux/amd64 tag_suffix: '-cuda' steps: @@ -51,6 +63,14 @@ jobs: with: fetch-depth: 0 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: arm64 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Install Nix uses: cachix/install-nix-action@v31 with: @@ -58,6 +78,7 @@ jobs: extra_nix_config: | experimental-features = nix-command flakes accept-flake-config = true + extra-platforms = aarch64-linux - name: Setup Cachix uses: cachix/cachix-action@v15 @@ -81,28 +102,29 @@ jobs: VERSION=$(grep -m1 'comfyuiVersion = ' flake.nix | sed 's/.*"\(.*\)".*/\1/') echo "version=$VERSION" >> $GITHUB_OUTPUT - # Generate Docker tags - TAGS="" + # Determine architecture suffix for intermediate tags + if [[ "${{ matrix.arch }}" == "aarch64-linux" ]]; then + ARCH_SUFFIX="-arm64" + else + ARCH_SUFFIX="-amd64" + fi + echo "arch_suffix=$ARCH_SUFFIX" >> $GITHUB_OUTPUT + + # Generate Docker tags for this specific architecture if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then - TAGS="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest${{ matrix.variant.tag_suffix }}" - TAGS="$TAGS,${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${VERSION}${{ matrix.variant.tag_suffix }}" + TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest${{ matrix.tag_suffix }}${ARCH_SUFFIX}" elif [[ "${{ github.ref }}" == refs/tags/v* ]]; then TAG_VERSION=${GITHUB_REF#refs/tags/v} - TAGS="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${TAG_VERSION}${{ matrix.variant.tag_suffix }}" - TAGS="$TAGS,${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest${{ matrix.variant.tag_suffix }}" + TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${TAG_VERSION}${{ matrix.tag_suffix }}${ARCH_SUFFIX}" else - TAGS="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:pr-${{ github.event.pull_request.number }}${{ matrix.variant.tag_suffix }}" + TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:pr-${{ github.event.pull_request.number }}${{ matrix.tag_suffix }}${ARCH_SUFFIX}" fi - echo "tags=$TAGS" >> $GITHUB_OUTPUT - - # Generate labels - echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT - echo "revision=${{ github.sha }}" >> $GITHUB_OUTPUT + echo "tag=$TAG" >> $GITHUB_OUTPUT - name: Build Docker image with Nix run: | - echo "Building ${{ matrix.variant.name }} variant..." - nix build .#${{ matrix.variant.nix_target }} --print-build-logs + echo "Building ${{ matrix.variant }} variant for ${{ matrix.arch }}..." + nix build .#packages.${{ matrix.arch }}.${{ matrix.nix_target }} --print-build-logs # Load the image into Docker docker load < result @@ -110,51 +132,151 @@ jobs: # Show loaded image docker images | grep comfy-ui - - name: Tag and push Docker images + - name: Tag and push Docker image if: github.event_name != 'pull_request' run: | # Get the image ID from the loaded image - if [[ "${{ matrix.variant.name }}" == "cuda" ]]; then + if [[ "${{ matrix.variant }}" == "cuda" ]]; then IMAGE_ID=$(docker images comfy-ui:cuda -q | head -n1) else IMAGE_ID=$(docker images comfy-ui:latest -q | head -n1) fi - # Tag and push each tag - IFS=',' read -ra TAG_ARRAY <<< "${{ steps.meta.outputs.tags }}" - for tag in "${TAG_ARRAY[@]}"; do - echo "Tagging and pushing: $tag" - docker tag "$IMAGE_ID" "$tag" - docker push "$tag" - done + echo "Tagging and pushing: ${{ steps.meta.outputs.tag }}" + docker tag "$IMAGE_ID" "${{ steps.meta.outputs.tag }}" + docker push "${{ steps.meta.outputs.tag }}" - - name: Generate image summary - if: github.event_name != 'pull_request' + - name: Generate build summary run: | - echo "## Docker Image Published (${{ matrix.variant.name }})" >> $GITHUB_STEP_SUMMARY + echo "## Docker Image Built (${{ matrix.variant }} - ${{ matrix.arch }})" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "**Version:** ${{ steps.meta.outputs.version }}" >> $GITHUB_STEP_SUMMARY - echo "**Variant:** ${{ matrix.variant.name }}" >> $GITHUB_STEP_SUMMARY + echo "**Variant:** ${{ matrix.variant }}" >> $GITHUB_STEP_SUMMARY + echo "**Architecture:** ${{ matrix.arch }}" >> $GITHUB_STEP_SUMMARY + echo "**Platform:** ${{ matrix.platform }}" >> $GITHUB_STEP_SUMMARY + + # Create and push multi-arch manifests after all builds complete + create-manifests: + needs: build + runs-on: ubuntu-latest + if: github.event_name != 'pull_request' + permissions: + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Log in to Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + run: | + VERSION=$(grep -m1 'comfyuiVersion = ' flake.nix | sed 's/.*"\(.*\)".*/\1/') + echo "version=$VERSION" >> $GITHUB_OUTPUT + + - name: Create and push multi-arch manifest (CPU) + run: | + # Determine tags based on ref + if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then + MANIFEST_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest" + VERSION_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }}" + elif [[ "${{ github.ref }}" == refs/tags/v* ]]; then + TAG_VERSION=${GITHUB_REF#refs/tags/v} + MANIFEST_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${TAG_VERSION}" + VERSION_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest" + fi + + # Create multi-arch manifest for CPU variant + echo "Creating multi-arch manifest: $MANIFEST_TAG" + docker manifest create "$MANIFEST_TAG" \ + "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-amd64" \ + "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-arm64" + + docker manifest annotate "$MANIFEST_TAG" \ + "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-amd64" --arch amd64 + docker manifest annotate "$MANIFEST_TAG" \ + "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-arm64" --arch arm64 + + docker manifest push "$MANIFEST_TAG" + + # Also push version tag if on main + if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then + echo "Creating multi-arch manifest: $VERSION_TAG" + docker manifest create "$VERSION_TAG" \ + "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-amd64" \ + "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-arm64" + + docker manifest annotate "$VERSION_TAG" \ + "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-amd64" --arch amd64 + docker manifest annotate "$VERSION_TAG" \ + "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-arm64" --arch arm64 + + docker manifest push "$VERSION_TAG" + fi + + - name: Create and push CUDA manifest (x86_64 only) + run: | + # CUDA is only available for x86_64, so no multi-arch manifest needed + # Just create alias tags for consistency + if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then + CUDA_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-cuda" + VERSION_CUDA_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }}-cuda" + + # Pull the amd64 cuda image and retag + docker pull "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-cuda-amd64" + + docker tag "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-cuda-amd64" "$CUDA_TAG" + docker push "$CUDA_TAG" + + docker tag "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-cuda-amd64" "$VERSION_CUDA_TAG" + docker push "$VERSION_CUDA_TAG" + elif [[ "${{ github.ref }}" == refs/tags/v* ]]; then + TAG_VERSION=${GITHUB_REF#refs/tags/v} + CUDA_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${TAG_VERSION}-cuda" + LATEST_CUDA_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-cuda" + + docker pull "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${TAG_VERSION}-cuda-amd64" + + docker tag "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${TAG_VERSION}-cuda-amd64" "$CUDA_TAG" + docker push "$CUDA_TAG" + + docker tag "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${TAG_VERSION}-cuda-amd64" "$LATEST_CUDA_TAG" + docker push "$LATEST_CUDA_TAG" + fi + + - name: Generate manifest summary + run: | + echo "## Multi-Architecture Docker Images Published" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - echo "### Pull Commands:" >> $GITHUB_STEP_SUMMARY + echo "**Version:** ${{ steps.meta.outputs.version }}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### CPU (Multi-arch: amd64 + arm64)" >> $GITHUB_STEP_SUMMARY echo '```bash' >> $GITHUB_STEP_SUMMARY - IFS=',' read -ra TAG_ARRAY <<< "${{ steps.meta.outputs.tags }}" - for tag in "${TAG_ARRAY[@]}"; do - echo "docker pull $tag" >> $GITHUB_STEP_SUMMARY - done + echo "docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest" >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - echo "### Run Command:" >> $GITHUB_STEP_SUMMARY + echo "### CUDA (amd64 only)" >> $GITHUB_STEP_SUMMARY echo '```bash' >> $GITHUB_STEP_SUMMARY - if [[ "${{ matrix.variant.name }}" == "cuda" ]]; then - echo "docker run --gpus all -p 8188:8188 -v \$PWD/data:/data ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-cuda" >> $GITHUB_STEP_SUMMARY - else - echo "docker run -p 8188:8188 -v \$PWD/data:/data ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest" >> $GITHUB_STEP_SUMMARY - fi + echo "docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-cuda" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Run Commands" >> $GITHUB_STEP_SUMMARY + echo '```bash' >> $GITHUB_STEP_SUMMARY + echo "# CPU (works on both amd64 and arm64)" >> $GITHUB_STEP_SUMMARY + echo "docker run -p 8188:8188 -v \$PWD/data:/data ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "# CUDA (amd64 only)" >> $GITHUB_STEP_SUMMARY + echo "docker run --gpus all -p 8188:8188 -v \$PWD/data:/data ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-cuda" >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY update-description: - needs: build-and-push + needs: create-manifests runs-on: ubuntu-latest if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' permissions: diff --git a/CLAUDE.md b/CLAUDE.md index 51db5d3..d776fcd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -104,11 +104,13 @@ custom_nodes/ - Persistent custom node installations #### Docker Image Publishing (`.github/workflows/docker.yml`) - **Purpose**: Automatically build and publish Docker images to GitHub Container Registry - **Triggers**: Push to main, version tags (v*), pull requests -- **Build Matrix**: Both CPU and CUDA variants built in parallel +- **Multi-Architecture**: CPU images built for both amd64 and arm64 (via QEMU emulation) +- **Build Matrix**: CPU (multi-arch) and CUDA (x86_64 only) variants built in parallel - **Outputs**: Images published to `ghcr.io/utensils/comfyui-nix` - **Tags**: - Main branch: `latest`, `X.Y.Z` (from flake.nix) - Version tags: `vX.Y.Z`, `latest` + - Architecture-specific: `latest-amd64`, `latest-arm64` - Pull requests: `pr-N` (build only, no push) - **Caching**: Uses Cachix for Nix build caching (requires `CACHIX_AUTH_TOKEN` secret) @@ -126,7 +128,8 @@ custom_nodes/ - Persistent custom node installations - **Location**: GitHub Container Registry (ghcr.io) - **Public Access**: All images are publicly readable - **Namespace**: `ghcr.io/utensils/comfyui-nix` -- **Variants**: CPU (`:latest`) and CUDA (`:latest-cuda`) +- **Variants**: CPU (`:latest`, multi-arch) and CUDA (`:latest-cuda`, x86_64 only) +- **Architectures**: amd64 (x86_64) and arm64 (aarch64/Apple Silicon) for CPU images ## Code Style Guidelines diff --git a/README.md b/README.md index 80903ab..a257b9d 100644 --- a/README.md +++ b/README.md @@ -246,23 +246,23 @@ This structure ensures clear separation of concerns and makes the codebase easie ## Docker Support -This flake includes Docker support for running ComfyUI in a containerized environment while preserving all functionality. Both CPU and CUDA-enabled GPU images are available. +This flake includes Docker support for running ComfyUI in a containerized environment while preserving all functionality. Multi-architecture images are available for both x86_64 (amd64) and ARM64 (aarch64) platforms. ### Pre-built Images (GitHub Container Registry) Pre-built Docker images are automatically published to GitHub Container Registry on every release. This is the easiest way to get started: -#### Pull and Run CPU Version +#### Pull and Run CPU Version (Multi-arch: amd64 + arm64) ```bash -# Pull the latest CPU version +# Pull the latest CPU version (automatically selects correct architecture) docker pull ghcr.io/utensils/comfyui-nix:latest # Run the container docker run -p 8188:8188 -v "$PWD/data:/data" ghcr.io/utensils/comfyui-nix:latest ``` -#### Pull and Run CUDA (GPU) Version +#### Pull and Run CUDA (GPU) Version (x86_64 only) ```bash # Pull the latest CUDA version @@ -274,13 +274,31 @@ docker run --gpus all -p 8188:8188 -v "$PWD/data:/data" ghcr.io/utensils/comfyui #### Available Tags -- `latest` - Latest CPU version from main branch -- `latest-cuda` - Latest CUDA version from main branch -- `X.Y.Z` - Specific version (CPU) -- `X.Y.Z-cuda` - Specific version (CUDA) +- `latest` - Latest CPU version, multi-arch (amd64 + arm64) +- `latest-cuda` - Latest CUDA version (x86_64/amd64 only) +- `latest-amd64` - Latest CPU version for x86_64 +- `latest-arm64` - Latest CPU version for ARM64 +- `X.Y.Z` - Specific version (CPU, multi-arch) +- `X.Y.Z-cuda` - Specific version (CUDA, x86_64 only) Visit the [packages page](https://github.com/utensils/comfyui-nix/pkgs/container/comfyui-nix) to see all available versions. +### Apple Silicon (M1/M2/M3) Support + +The `latest` and `latest-arm64` tags work on Apple Silicon Macs via Docker Desktop: + +```bash +# Works on Apple Silicon Macs +docker run -p 8188:8188 -v "$PWD/data:/data" ghcr.io/utensils/comfyui-nix:latest +``` + +**Important**: Docker containers on macOS cannot access the Metal GPU (MPS). The Docker image runs **CPU-only** on Apple Silicon. For GPU acceleration on Apple Silicon, use `nix run` directly instead of Docker: + +```bash +# For GPU acceleration on Apple Silicon, use nix directly (not Docker) +nix run github:utensils/comfyui-nix +``` + ### Building the Docker Image Locally #### CPU Version @@ -374,11 +392,13 @@ The Docker image follows the same modular structure as the regular installation, Docker images are automatically built and published to GitHub Container Registry via GitHub Actions: - **Trigger events**: Push to main branch, version tags (v*), and pull requests -- **Build matrix**: Both CPU and CUDA variants are built in parallel +- **Multi-architecture**: CPU images built for both amd64 and arm64 (via QEMU emulation) +- **Build matrix**: CPU (multi-arch) and CUDA (x86_64 only) variants built in parallel - **Tagging strategy**: - Main branch pushes: `latest` and `X.Y.Z` (version from flake.nix) - Version tags: `vX.Y.Z` and `latest` - Pull requests: `pr-N` (for testing, not pushed to registry) + - Architecture-specific: `latest-amd64`, `latest-arm64` - **Registry**: All images are publicly accessible at `ghcr.io/utensils/comfyui-nix` - **Build cache**: Nix builds are cached using Cachix for faster CI runs diff --git a/flake.nix b/flake.nix index dfd3de7..9415eab 100644 --- a/flake.nix +++ b/flake.nix @@ -87,6 +87,11 @@ pythonEnv = pythonEnv; }; + templateInputsScript = pkgs.substituteAll { + src = ./scripts/template_inputs.sh; + pythonEnv = pythonEnv; + }; + # Main launcher script with substitutions launcherScript = pkgs.substituteAll { src = ./scripts/launcher.sh; @@ -106,6 +111,7 @@ cp ${installScript} $out/install.sh cp ${persistenceShScript} $out/persistence.sh cp ${runtimeScript} $out/runtime.sh + cp ${templateInputsScript} $out/template_inputs.sh cp ${launcherScript} $out/launcher.sh chmod +x $out/*.sh ''; @@ -152,12 +158,20 @@ # Copy all script files cp -r ${scriptDir}/* "$out/share/comfy-ui/scripts/" - # Install the launcher script - ln -s "$out/share/comfy-ui/scripts/launcher.sh" "$out/bin/comfy-ui-launcher" - chmod +x "$out/bin/comfy-ui-launcher" - - # Create a symlink to the launcher - ln -s "$out/bin/comfy-ui-launcher" "$out/bin/comfy-ui" + # Install the launcher script with wrapper for required tools + # curl and jq are needed for downloading workflow template input files + makeWrapper "$out/share/comfy-ui/scripts/launcher.sh" "$out/bin/comfy-ui" \ + --prefix PATH : "${ + pkgs.lib.makeBinPath [ + pkgs.curl + pkgs.jq + pkgs.git + pkgs.coreutils + ] + }" + + # Create alias for backwards compatibility + ln -s "$out/bin/comfy-ui" "$out/bin/comfy-ui-launcher" ''; meta = with pkgs.lib; { @@ -183,6 +197,7 @@ pkgs.netcat pkgs.git pkgs.curl + pkgs.jq pkgs.cacert pkgs.libGL pkgs.libGLU @@ -257,6 +272,7 @@ pkgs.netcat pkgs.git pkgs.curl + pkgs.jq pkgs.cacert pkgs.libGL pkgs.libGLU diff --git a/scripts/install.sh b/scripts/install.sh index ae332d5..c026cf0 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -284,11 +284,19 @@ install_all() { install_model_downloader setup_venv setup_persistence_scripts - + # Now set up the actual symlinks source "$SCRIPT_DIR/persistence.sh" setup_persistence - + + # Download workflow template input files (non-blocking) + source "$SCRIPT_DIR/template_inputs.sh" + if needs_template_inputs; then + download_template_inputs + else + log_debug "Template input files are up to date" + fi + log_section "Installation complete" log_info "ComfyUI $COMFY_VERSION has been successfully installed" } diff --git a/scripts/template_inputs.sh b/scripts/template_inputs.sh new file mode 100644 index 0000000..a12d776 --- /dev/null +++ b/scripts/template_inputs.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash +# template_inputs.sh: Download workflow template input files from GitHub + +# Guard against multiple sourcing +[[ -n "${_TEMPLATE_INPUTS_SH_SOURCED:-}" ]] && return +_TEMPLATE_INPUTS_SH_SOURCED=1 + +# Template input files manifest URL +TEMPLATE_MANIFEST_URL="https://raw.githubusercontent.com/Comfy-Org/workflow_templates/refs/heads/main/workflow_template_input_files.json" + +# Download template input files that don't exist locally +download_template_inputs() { + local input_dir="$BASE_DIR/input" + local manifest_cache="$input_dir/.template_manifest.json" + local download_count=0 + local skip_count=0 + local fail_count=0 + + log_info "Checking workflow template input files..." + + # Ensure input directory exists + mkdir -p "$input_dir" + + # Download manifest (with timeout to not block startup) + log_debug "Fetching template manifest from GitHub..." + if ! curl -fsSL --connect-timeout 5 --max-time 30 "$TEMPLATE_MANIFEST_URL" -o "$manifest_cache.tmp" 2>/dev/null; then + log_warn "Could not fetch template manifest (network unavailable or timeout)" + log_warn "Template input files may be missing - workflows might not work correctly" + return 0 # Don't fail startup + fi + + mv "$manifest_cache.tmp" "$manifest_cache" + + # Check if jq is available + if ! command -v jq &>/dev/null; then + log_warn "jq not found - cannot parse template manifest" + return 0 + fi + + # Count total files + local total_files + total_files=$(jq -r '.assets | length' "$manifest_cache" 2>/dev/null || echo "0") + + if [[ "$total_files" == "0" ]]; then + log_warn "No template input files found in manifest" + return 0 + fi + + log_info "Found $total_files template input files to check" + + # Process each asset in the manifest + local i=0 + while IFS= read -r line; do + local file_path url display_name + file_path=$(echo "$line" | jq -r '.file_path // empty') + url=$(echo "$line" | jq -r '.url // empty') + display_name=$(echo "$line" | jq -r '.display_name // empty') + + # Skip if missing required fields + if [[ -z "$file_path" || -z "$url" ]]; then + continue + fi + + # Extract just the filename from file_path (e.g., "input/foo.png" -> "foo.png") + local filename + filename=$(basename "$file_path") + local local_path="$input_dir/$filename" + + # Skip if file already exists + if [[ -f "$local_path" ]]; then + ((skip_count++)) + log_debug "Skipping existing: $filename" + continue + fi + + # Download the file + log_debug "Downloading: $filename" + if curl -fsSL --connect-timeout 5 --max-time 60 "$url" -o "$local_path.tmp" 2>/dev/null; then + mv "$local_path.tmp" "$local_path" + ((download_count++)) + # Show progress every 10 files + if ((download_count % 10 == 0)); then + log_info "Downloaded $download_count files..." + fi + else + ((fail_count++)) + log_debug "Failed to download: $filename" + rm -f "$local_path.tmp" + fi + + ((i++)) + done < <(jq -c '.assets[]' "$manifest_cache" 2>/dev/null) + + # Summary + if ((download_count > 0)); then + log_info "Downloaded $download_count new template input files" + fi + if ((skip_count > 0)); then + log_debug "Skipped $skip_count existing files" + fi + if ((fail_count > 0)); then + log_warn "Failed to download $fail_count files (will retry next startup)" + fi + + log_info "Template input files ready" +} + +# Quick check if template inputs need downloading (for faster startup) +needs_template_inputs() { + local input_dir="$BASE_DIR/input" + local manifest_cache="$input_dir/.template_manifest.json" + + # If no manifest cache, we need to download + if [[ ! -f "$manifest_cache" ]]; then + return 0 + fi + + # Check if manifest is older than 7 days (re-check periodically) + local manifest_age + manifest_age=$(( $(date +%s) - $(stat -c %Y "$manifest_cache" 2>/dev/null || echo "0") )) + if ((manifest_age > 604800)); then # 7 days in seconds + return 0 + fi + + # Quick check: if input dir has fewer than 20 files, probably needs download + local file_count + file_count=$(find "$input_dir" -maxdepth 1 -type f ! -name ".*" 2>/dev/null | wc -l) + if ((file_count < 20)); then + return 0 + fi + + return 1 +} From 72990279918cfe666491e591cc741d1a8037849c Mon Sep 17 00:00:00 2001 From: James Brink Date: Sun, 14 Dec 2025 21:29:48 -0700 Subject: [PATCH 2/2] fix: Address code review security and robustness concerns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Security improvements in template_inputs.sh: - Add path traversal validation to reject file paths containing ".." - Add file size validation (50MB limit) to prevent DoS attacks - Add file type validation using MIME type checking - Validate JSON manifest structure before processing Robustness improvements: - Capture and display actual curl error messages for debugging - Add atomic write pattern with empty file validation - Platform-agnostic file size detection (macOS/Linux compatible) - Remove unused display_name variable (shellcheck warning) CI/CD improvements in docker.yml: - Add manifest existence verification before creating multi-arch manifests - Use variables for architecture-specific tags for consistency - Add clear error messages when builds fail 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 --- .github/workflows/docker.yml | 70 ++++++++++++---------- scripts/template_inputs.sh | 109 ++++++++++++++++++++++++++++++----- 2 files changed, 136 insertions(+), 43 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ac7df57..c22b0b9 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -186,36 +186,45 @@ jobs: if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then MANIFEST_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest" VERSION_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }}" + AMD64_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-amd64" + ARM64_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-arm64" elif [[ "${{ github.ref }}" == refs/tags/v* ]]; then TAG_VERSION=${GITHUB_REF#refs/tags/v} MANIFEST_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${TAG_VERSION}" VERSION_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest" + AMD64_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${TAG_VERSION}-amd64" + ARM64_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${TAG_VERSION}-arm64" fi + # Verify both architecture images exist before creating manifest + echo "Verifying architecture-specific images exist..." + for arch_tag in "$AMD64_TAG" "$ARM64_TAG"; do + if ! docker manifest inspect "$arch_tag" &>/dev/null; then + echo "ERROR: Required image not found: $arch_tag" + echo "This likely means the build job for this architecture failed." + exit 1 + fi + echo "Found: $arch_tag" + done + # Create multi-arch manifest for CPU variant echo "Creating multi-arch manifest: $MANIFEST_TAG" docker manifest create "$MANIFEST_TAG" \ - "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-amd64" \ - "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-arm64" + "$AMD64_TAG" \ + "$ARM64_TAG" - docker manifest annotate "$MANIFEST_TAG" \ - "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-amd64" --arch amd64 - docker manifest annotate "$MANIFEST_TAG" \ - "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-arm64" --arch arm64 + docker manifest annotate "$MANIFEST_TAG" "$AMD64_TAG" --arch amd64 + docker manifest annotate "$MANIFEST_TAG" "$ARM64_TAG" --arch arm64 docker manifest push "$MANIFEST_TAG" # Also push version tag if on main if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then echo "Creating multi-arch manifest: $VERSION_TAG" - docker manifest create "$VERSION_TAG" \ - "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-amd64" \ - "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-arm64" + docker manifest create "$VERSION_TAG" "$AMD64_TAG" "$ARM64_TAG" - docker manifest annotate "$VERSION_TAG" \ - "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-amd64" --arch amd64 - docker manifest annotate "$VERSION_TAG" \ - "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-arm64" --arch arm64 + docker manifest annotate "$VERSION_TAG" "$AMD64_TAG" --arch amd64 + docker manifest annotate "$VERSION_TAG" "$ARM64_TAG" --arch arm64 docker manifest push "$VERSION_TAG" fi @@ -227,28 +236,31 @@ jobs: if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then CUDA_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-cuda" VERSION_CUDA_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }}-cuda" - - # Pull the amd64 cuda image and retag - docker pull "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-cuda-amd64" - - docker tag "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-cuda-amd64" "$CUDA_TAG" - docker push "$CUDA_TAG" - - docker tag "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-cuda-amd64" "$VERSION_CUDA_TAG" - docker push "$VERSION_CUDA_TAG" + CUDA_AMD64_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-cuda-amd64" elif [[ "${{ github.ref }}" == refs/tags/v* ]]; then TAG_VERSION=${GITHUB_REF#refs/tags/v} CUDA_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${TAG_VERSION}-cuda" - LATEST_CUDA_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-cuda" + VERSION_CUDA_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-cuda" + CUDA_AMD64_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${TAG_VERSION}-cuda-amd64" + fi + + # Verify CUDA image exists before creating alias tags + echo "Verifying CUDA image exists..." + if ! docker manifest inspect "$CUDA_AMD64_TAG" &>/dev/null; then + echo "ERROR: Required CUDA image not found: $CUDA_AMD64_TAG" + echo "This likely means the CUDA build job failed." + exit 1 + fi + echo "Found: $CUDA_AMD64_TAG" - docker pull "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${TAG_VERSION}-cuda-amd64" + # Pull and retag for alias + docker pull "$CUDA_AMD64_TAG" - docker tag "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${TAG_VERSION}-cuda-amd64" "$CUDA_TAG" - docker push "$CUDA_TAG" + docker tag "$CUDA_AMD64_TAG" "$CUDA_TAG" + docker push "$CUDA_TAG" - docker tag "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${TAG_VERSION}-cuda-amd64" "$LATEST_CUDA_TAG" - docker push "$LATEST_CUDA_TAG" - fi + docker tag "$CUDA_AMD64_TAG" "$VERSION_CUDA_TAG" + docker push "$VERSION_CUDA_TAG" - name: Generate manifest summary run: | diff --git a/scripts/template_inputs.sh b/scripts/template_inputs.sh index a12d776..746efa6 100644 --- a/scripts/template_inputs.sh +++ b/scripts/template_inputs.sh @@ -8,6 +8,21 @@ _TEMPLATE_INPUTS_SH_SOURCED=1 # Template input files manifest URL TEMPLATE_MANIFEST_URL="https://raw.githubusercontent.com/Comfy-Org/workflow_templates/refs/heads/main/workflow_template_input_files.json" +# Maximum file size for template inputs (50MB - prevents DoS via large files) +MAX_TEMPLATE_FILE_SIZE=$((50 * 1024 * 1024)) + +# Platform-agnostic file size function +get_file_size() { + local file=$1 + if [[ "$(uname)" == "Darwin" ]]; then + # macOS/BSD + stat -f%z "$file" 2>/dev/null || echo "0" + else + # Linux/GNU + stat -c%s "$file" 2>/dev/null || echo "0" + fi +} + # Download template input files that don't exist locally download_template_inputs() { local input_dir="$BASE_DIR/input" @@ -23,23 +38,42 @@ download_template_inputs() { # Download manifest (with timeout to not block startup) log_debug "Fetching template manifest from GitHub..." - if ! curl -fsSL --connect-timeout 5 --max-time 30 "$TEMPLATE_MANIFEST_URL" -o "$manifest_cache.tmp" 2>/dev/null; then - log_warn "Could not fetch template manifest (network unavailable or timeout)" + local curl_error + if ! curl_error=$(curl -fsSL --connect-timeout 5 --max-time 30 "$TEMPLATE_MANIFEST_URL" -o "$manifest_cache.tmp" 2>&1); then + log_warn "Could not fetch template manifest: ${curl_error:-network unavailable or timeout}" + log_debug "URL: $TEMPLATE_MANIFEST_URL" log_warn "Template input files may be missing - workflows might not work correctly" + rm -f "$manifest_cache.tmp" return 0 # Don't fail startup fi - mv "$manifest_cache.tmp" "$manifest_cache" + # Validate downloaded manifest is not empty and move atomically + if [[ ! -s "$manifest_cache.tmp" ]]; then + log_warn "Downloaded manifest is empty, keeping previous cache" + rm -f "$manifest_cache.tmp" + return 0 + fi # Check if jq is available if ! command -v jq &>/dev/null; then log_warn "jq not found - cannot parse template manifest" + rm -f "$manifest_cache.tmp" return 0 fi + # Validate JSON structure before processing + if ! jq -e '.assets' "$manifest_cache.tmp" &>/dev/null; then + log_warn "Invalid manifest format (missing 'assets' array), removing cache" + rm -f "$manifest_cache.tmp" + return 0 + fi + + # Atomically move the validated manifest + mv "$manifest_cache.tmp" "$manifest_cache" + # Count total files local total_files - total_files=$(jq -r '.assets | length' "$manifest_cache" 2>/dev/null || echo "0") + total_files=$(jq -r '.assets | length' "$manifest_cache") if [[ "$total_files" == "0" ]]; then log_warn "No template input files found in manifest" @@ -51,16 +85,22 @@ download_template_inputs() { # Process each asset in the manifest local i=0 while IFS= read -r line; do - local file_path url display_name + local file_path url file_path=$(echo "$line" | jq -r '.file_path // empty') url=$(echo "$line" | jq -r '.url // empty') - display_name=$(echo "$line" | jq -r '.display_name // empty') # Skip if missing required fields if [[ -z "$file_path" || -z "$url" ]]; then continue fi + # Security: Check for path traversal attempts + if [[ "$file_path" =~ \.\. ]]; then + log_warn "Skipping potentially malicious file_path: $file_path" + ((fail_count++)) + continue + fi + # Extract just the filename from file_path (e.g., "input/foo.png" -> "foo.png") local filename filename=$(basename "$file_path") @@ -75,7 +115,39 @@ download_template_inputs() { # Download the file log_debug "Downloading: $filename" - if curl -fsSL --connect-timeout 5 --max-time 60 "$url" -o "$local_path.tmp" 2>/dev/null; then + local download_error + if download_error=$(curl -fsSL --connect-timeout 5 --max-time 60 "$url" -o "$local_path.tmp" 2>&1); then + # Validate file size to prevent DoS + local file_size + file_size=$(get_file_size "$local_path.tmp") + if ((file_size > MAX_TEMPLATE_FILE_SIZE)); then + log_warn "File too large (${file_size} bytes, max ${MAX_TEMPLATE_FILE_SIZE}), skipping: $filename" + rm -f "$local_path.tmp" + ((fail_count++)) + continue + fi + + # Validate file is not empty + if [[ ! -s "$local_path.tmp" ]]; then + log_debug "Downloaded file is empty, skipping: $filename" + rm -f "$local_path.tmp" + ((fail_count++)) + continue + fi + + # Validate file type if 'file' command is available + if command -v file &>/dev/null; then + local file_type + file_type=$(file -b --mime-type "$local_path.tmp" 2>/dev/null || echo "unknown") + # Allow images, videos, and common asset types + if [[ ! $file_type =~ ^(image/|video/|audio/|application/octet-stream|text/) ]]; then + log_warn "Unexpected file type ($file_type), skipping: $filename" + rm -f "$local_path.tmp" + ((fail_count++)) + continue + fi + fi + mv "$local_path.tmp" "$local_path" ((download_count++)) # Show progress every 10 files @@ -84,12 +156,12 @@ download_template_inputs() { fi else ((fail_count++)) - log_debug "Failed to download: $filename" + log_debug "Failed to download $filename: ${download_error:-unknown error}" rm -f "$local_path.tmp" fi ((i++)) - done < <(jq -c '.assets[]' "$manifest_cache" 2>/dev/null) + done < <(jq -c '.assets[]' "$manifest_cache") # Summary if ((download_count > 0)); then @@ -106,6 +178,7 @@ download_template_inputs() { } # Quick check if template inputs need downloading (for faster startup) +# Returns 0 if download needed, 1 if up to date needs_template_inputs() { local input_dir="$BASE_DIR/input" local manifest_cache="$input_dir/.template_manifest.json" @@ -115,14 +188,22 @@ needs_template_inputs() { return 0 fi - # Check if manifest is older than 7 days (re-check periodically) - local manifest_age - manifest_age=$(( $(date +%s) - $(stat -c %Y "$manifest_cache" 2>/dev/null || echo "0") )) - if ((manifest_age > 604800)); then # 7 days in seconds + # Check if manifest is older than 7 days (604800 seconds) + # This ensures we periodically re-check for new template files + local manifest_age current_time file_mtime + current_time=$(date +%s) + if [[ "$(uname)" == "Darwin" ]]; then + file_mtime=$(stat -f%m "$manifest_cache" 2>/dev/null || echo "0") + else + file_mtime=$(stat -c %Y "$manifest_cache" 2>/dev/null || echo "0") + fi + manifest_age=$((current_time - file_mtime)) + if ((manifest_age > 604800)); then return 0 fi - # Quick check: if input dir has fewer than 20 files, probably needs download + # Quick heuristic: if input dir has fewer than 20 files, probably needs download + # This catches cases where the manifest exists but files were deleted local file_count file_count=$(find "$input_dir" -maxdepth 1 -type f ! -name ".*" 2>/dev/null | wc -l) if ((file_count < 20)); then