diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 3967725..c22b0b9 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -28,7 +28,7 @@ env: IMAGE_NAME: ${{ github.repository }} jobs: - build-and-push: + build: runs-on: ubuntu-latest permissions: contents: read @@ -36,13 +36,25 @@ jobs: id-token: write strategy: + fail-fast: false matrix: - variant: - - name: cpu + include: + # CPU variants for both architectures + - variant: cpu + arch: x86_64-linux nix_target: dockerImage + platform: linux/amd64 tag_suffix: '' - - name: cuda + - variant: cpu + arch: aarch64-linux + nix_target: dockerImage + platform: linux/arm64 + tag_suffix: '' + # CUDA only for x86_64 (NVIDIA CUDA not available for ARM) + - variant: cuda + arch: x86_64-linux nix_target: dockerImageCuda + platform: linux/amd64 tag_suffix: '-cuda' steps: @@ -51,6 +63,14 @@ jobs: with: fetch-depth: 0 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: arm64 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Install Nix uses: cachix/install-nix-action@v31 with: @@ -58,6 +78,7 @@ jobs: extra_nix_config: | experimental-features = nix-command flakes accept-flake-config = true + extra-platforms = aarch64-linux - name: Setup Cachix uses: cachix/cachix-action@v15 @@ -81,28 +102,29 @@ jobs: VERSION=$(grep -m1 'comfyuiVersion = ' flake.nix | sed 's/.*"\(.*\)".*/\1/') echo "version=$VERSION" >> $GITHUB_OUTPUT - # Generate Docker tags - TAGS="" + # Determine architecture suffix for intermediate tags + if [[ "${{ matrix.arch }}" == "aarch64-linux" ]]; then + ARCH_SUFFIX="-arm64" + else + ARCH_SUFFIX="-amd64" + fi + echo "arch_suffix=$ARCH_SUFFIX" >> $GITHUB_OUTPUT + + # Generate Docker tags for this specific architecture if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then - TAGS="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest${{ matrix.variant.tag_suffix }}" - TAGS="$TAGS,${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${VERSION}${{ matrix.variant.tag_suffix }}" + TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest${{ matrix.tag_suffix }}${ARCH_SUFFIX}" elif [[ "${{ github.ref }}" == refs/tags/v* ]]; then TAG_VERSION=${GITHUB_REF#refs/tags/v} - TAGS="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${TAG_VERSION}${{ matrix.variant.tag_suffix }}" - TAGS="$TAGS,${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest${{ matrix.variant.tag_suffix }}" + TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${TAG_VERSION}${{ matrix.tag_suffix }}${ARCH_SUFFIX}" else - TAGS="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:pr-${{ github.event.pull_request.number }}${{ matrix.variant.tag_suffix }}" + TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:pr-${{ github.event.pull_request.number }}${{ matrix.tag_suffix }}${ARCH_SUFFIX}" fi - echo "tags=$TAGS" >> $GITHUB_OUTPUT - - # Generate labels - echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT - echo "revision=${{ github.sha }}" >> $GITHUB_OUTPUT + echo "tag=$TAG" >> $GITHUB_OUTPUT - name: Build Docker image with Nix run: | - echo "Building ${{ matrix.variant.name }} variant..." - nix build .#${{ matrix.variant.nix_target }} --print-build-logs + echo "Building ${{ matrix.variant }} variant for ${{ matrix.arch }}..." + nix build .#packages.${{ matrix.arch }}.${{ matrix.nix_target }} --print-build-logs # Load the image into Docker docker load < result @@ -110,51 +132,163 @@ jobs: # Show loaded image docker images | grep comfy-ui - - name: Tag and push Docker images + - name: Tag and push Docker image if: github.event_name != 'pull_request' run: | # Get the image ID from the loaded image - if [[ "${{ matrix.variant.name }}" == "cuda" ]]; then + if [[ "${{ matrix.variant }}" == "cuda" ]]; then IMAGE_ID=$(docker images comfy-ui:cuda -q | head -n1) else IMAGE_ID=$(docker images comfy-ui:latest -q | head -n1) fi - # Tag and push each tag - IFS=',' read -ra TAG_ARRAY <<< "${{ steps.meta.outputs.tags }}" - for tag in "${TAG_ARRAY[@]}"; do - echo "Tagging and pushing: $tag" - docker tag "$IMAGE_ID" "$tag" - docker push "$tag" + echo "Tagging and pushing: ${{ steps.meta.outputs.tag }}" + docker tag "$IMAGE_ID" "${{ steps.meta.outputs.tag }}" + docker push "${{ steps.meta.outputs.tag }}" + + - name: Generate build summary + run: | + echo "## Docker Image Built (${{ matrix.variant }} - ${{ matrix.arch }})" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Version:** ${{ steps.meta.outputs.version }}" >> $GITHUB_STEP_SUMMARY + echo "**Variant:** ${{ matrix.variant }}" >> $GITHUB_STEP_SUMMARY + echo "**Architecture:** ${{ matrix.arch }}" >> $GITHUB_STEP_SUMMARY + echo "**Platform:** ${{ matrix.platform }}" >> $GITHUB_STEP_SUMMARY + + # Create and push multi-arch manifests after all builds complete + create-manifests: + needs: build + runs-on: ubuntu-latest + if: github.event_name != 'pull_request' + permissions: + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Log in to Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + run: | + VERSION=$(grep -m1 'comfyuiVersion = ' flake.nix | sed 's/.*"\(.*\)".*/\1/') + echo "version=$VERSION" >> $GITHUB_OUTPUT + + - name: Create and push multi-arch manifest (CPU) + run: | + # Determine tags based on ref + if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then + MANIFEST_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest" + VERSION_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }}" + AMD64_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-amd64" + ARM64_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-arm64" + elif [[ "${{ github.ref }}" == refs/tags/v* ]]; then + TAG_VERSION=${GITHUB_REF#refs/tags/v} + MANIFEST_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${TAG_VERSION}" + VERSION_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest" + AMD64_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${TAG_VERSION}-amd64" + ARM64_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${TAG_VERSION}-arm64" + fi + + # Verify both architecture images exist before creating manifest + echo "Verifying architecture-specific images exist..." + for arch_tag in "$AMD64_TAG" "$ARM64_TAG"; do + if ! docker manifest inspect "$arch_tag" &>/dev/null; then + echo "ERROR: Required image not found: $arch_tag" + echo "This likely means the build job for this architecture failed." + exit 1 + fi + echo "Found: $arch_tag" done - - name: Generate image summary - if: github.event_name != 'pull_request' + # Create multi-arch manifest for CPU variant + echo "Creating multi-arch manifest: $MANIFEST_TAG" + docker manifest create "$MANIFEST_TAG" \ + "$AMD64_TAG" \ + "$ARM64_TAG" + + docker manifest annotate "$MANIFEST_TAG" "$AMD64_TAG" --arch amd64 + docker manifest annotate "$MANIFEST_TAG" "$ARM64_TAG" --arch arm64 + + docker manifest push "$MANIFEST_TAG" + + # Also push version tag if on main + if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then + echo "Creating multi-arch manifest: $VERSION_TAG" + docker manifest create "$VERSION_TAG" "$AMD64_TAG" "$ARM64_TAG" + + docker manifest annotate "$VERSION_TAG" "$AMD64_TAG" --arch amd64 + docker manifest annotate "$VERSION_TAG" "$ARM64_TAG" --arch arm64 + + docker manifest push "$VERSION_TAG" + fi + + - name: Create and push CUDA manifest (x86_64 only) + run: | + # CUDA is only available for x86_64, so no multi-arch manifest needed + # Just create alias tags for consistency + if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then + CUDA_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-cuda" + VERSION_CUDA_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }}-cuda" + CUDA_AMD64_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-cuda-amd64" + elif [[ "${{ github.ref }}" == refs/tags/v* ]]; then + TAG_VERSION=${GITHUB_REF#refs/tags/v} + CUDA_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${TAG_VERSION}-cuda" + VERSION_CUDA_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-cuda" + CUDA_AMD64_TAG="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${TAG_VERSION}-cuda-amd64" + fi + + # Verify CUDA image exists before creating alias tags + echo "Verifying CUDA image exists..." + if ! docker manifest inspect "$CUDA_AMD64_TAG" &>/dev/null; then + echo "ERROR: Required CUDA image not found: $CUDA_AMD64_TAG" + echo "This likely means the CUDA build job failed." + exit 1 + fi + echo "Found: $CUDA_AMD64_TAG" + + # Pull and retag for alias + docker pull "$CUDA_AMD64_TAG" + + docker tag "$CUDA_AMD64_TAG" "$CUDA_TAG" + docker push "$CUDA_TAG" + + docker tag "$CUDA_AMD64_TAG" "$VERSION_CUDA_TAG" + docker push "$VERSION_CUDA_TAG" + + - name: Generate manifest summary run: | - echo "## Docker Image Published (${{ matrix.variant.name }})" >> $GITHUB_STEP_SUMMARY + echo "## Multi-Architecture Docker Images Published" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "**Version:** ${{ steps.meta.outputs.version }}" >> $GITHUB_STEP_SUMMARY - echo "**Variant:** ${{ matrix.variant.name }}" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - echo "### Pull Commands:" >> $GITHUB_STEP_SUMMARY + echo "### CPU (Multi-arch: amd64 + arm64)" >> $GITHUB_STEP_SUMMARY echo '```bash' >> $GITHUB_STEP_SUMMARY - IFS=',' read -ra TAG_ARRAY <<< "${{ steps.meta.outputs.tags }}" - for tag in "${TAG_ARRAY[@]}"; do - echo "docker pull $tag" >> $GITHUB_STEP_SUMMARY - done + echo "docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest" >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY - echo "### Run Command:" >> $GITHUB_STEP_SUMMARY + echo "### CUDA (amd64 only)" >> $GITHUB_STEP_SUMMARY echo '```bash' >> $GITHUB_STEP_SUMMARY - if [[ "${{ matrix.variant.name }}" == "cuda" ]]; then - echo "docker run --gpus all -p 8188:8188 -v \$PWD/data:/data ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-cuda" >> $GITHUB_STEP_SUMMARY - else - echo "docker run -p 8188:8188 -v \$PWD/data:/data ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest" >> $GITHUB_STEP_SUMMARY - fi + echo "docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-cuda" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Run Commands" >> $GITHUB_STEP_SUMMARY + echo '```bash' >> $GITHUB_STEP_SUMMARY + echo "# CPU (works on both amd64 and arm64)" >> $GITHUB_STEP_SUMMARY + echo "docker run -p 8188:8188 -v \$PWD/data:/data ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "# CUDA (amd64 only)" >> $GITHUB_STEP_SUMMARY + echo "docker run --gpus all -p 8188:8188 -v \$PWD/data:/data ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest-cuda" >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY update-description: - needs: build-and-push + needs: create-manifests runs-on: ubuntu-latest if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' permissions: diff --git a/CLAUDE.md b/CLAUDE.md index 51db5d3..d776fcd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -104,11 +104,13 @@ custom_nodes/ - Persistent custom node installations #### Docker Image Publishing (`.github/workflows/docker.yml`) - **Purpose**: Automatically build and publish Docker images to GitHub Container Registry - **Triggers**: Push to main, version tags (v*), pull requests -- **Build Matrix**: Both CPU and CUDA variants built in parallel +- **Multi-Architecture**: CPU images built for both amd64 and arm64 (via QEMU emulation) +- **Build Matrix**: CPU (multi-arch) and CUDA (x86_64 only) variants built in parallel - **Outputs**: Images published to `ghcr.io/utensils/comfyui-nix` - **Tags**: - Main branch: `latest`, `X.Y.Z` (from flake.nix) - Version tags: `vX.Y.Z`, `latest` + - Architecture-specific: `latest-amd64`, `latest-arm64` - Pull requests: `pr-N` (build only, no push) - **Caching**: Uses Cachix for Nix build caching (requires `CACHIX_AUTH_TOKEN` secret) @@ -126,7 +128,8 @@ custom_nodes/ - Persistent custom node installations - **Location**: GitHub Container Registry (ghcr.io) - **Public Access**: All images are publicly readable - **Namespace**: `ghcr.io/utensils/comfyui-nix` -- **Variants**: CPU (`:latest`) and CUDA (`:latest-cuda`) +- **Variants**: CPU (`:latest`, multi-arch) and CUDA (`:latest-cuda`, x86_64 only) +- **Architectures**: amd64 (x86_64) and arm64 (aarch64/Apple Silicon) for CPU images ## Code Style Guidelines diff --git a/README.md b/README.md index 80903ab..a257b9d 100644 --- a/README.md +++ b/README.md @@ -246,23 +246,23 @@ This structure ensures clear separation of concerns and makes the codebase easie ## Docker Support -This flake includes Docker support for running ComfyUI in a containerized environment while preserving all functionality. Both CPU and CUDA-enabled GPU images are available. +This flake includes Docker support for running ComfyUI in a containerized environment while preserving all functionality. Multi-architecture images are available for both x86_64 (amd64) and ARM64 (aarch64) platforms. ### Pre-built Images (GitHub Container Registry) Pre-built Docker images are automatically published to GitHub Container Registry on every release. This is the easiest way to get started: -#### Pull and Run CPU Version +#### Pull and Run CPU Version (Multi-arch: amd64 + arm64) ```bash -# Pull the latest CPU version +# Pull the latest CPU version (automatically selects correct architecture) docker pull ghcr.io/utensils/comfyui-nix:latest # Run the container docker run -p 8188:8188 -v "$PWD/data:/data" ghcr.io/utensils/comfyui-nix:latest ``` -#### Pull and Run CUDA (GPU) Version +#### Pull and Run CUDA (GPU) Version (x86_64 only) ```bash # Pull the latest CUDA version @@ -274,13 +274,31 @@ docker run --gpus all -p 8188:8188 -v "$PWD/data:/data" ghcr.io/utensils/comfyui #### Available Tags -- `latest` - Latest CPU version from main branch -- `latest-cuda` - Latest CUDA version from main branch -- `X.Y.Z` - Specific version (CPU) -- `X.Y.Z-cuda` - Specific version (CUDA) +- `latest` - Latest CPU version, multi-arch (amd64 + arm64) +- `latest-cuda` - Latest CUDA version (x86_64/amd64 only) +- `latest-amd64` - Latest CPU version for x86_64 +- `latest-arm64` - Latest CPU version for ARM64 +- `X.Y.Z` - Specific version (CPU, multi-arch) +- `X.Y.Z-cuda` - Specific version (CUDA, x86_64 only) Visit the [packages page](https://github.com/utensils/comfyui-nix/pkgs/container/comfyui-nix) to see all available versions. +### Apple Silicon (M1/M2/M3) Support + +The `latest` and `latest-arm64` tags work on Apple Silicon Macs via Docker Desktop: + +```bash +# Works on Apple Silicon Macs +docker run -p 8188:8188 -v "$PWD/data:/data" ghcr.io/utensils/comfyui-nix:latest +``` + +**Important**: Docker containers on macOS cannot access the Metal GPU (MPS). The Docker image runs **CPU-only** on Apple Silicon. For GPU acceleration on Apple Silicon, use `nix run` directly instead of Docker: + +```bash +# For GPU acceleration on Apple Silicon, use nix directly (not Docker) +nix run github:utensils/comfyui-nix +``` + ### Building the Docker Image Locally #### CPU Version @@ -374,11 +392,13 @@ The Docker image follows the same modular structure as the regular installation, Docker images are automatically built and published to GitHub Container Registry via GitHub Actions: - **Trigger events**: Push to main branch, version tags (v*), and pull requests -- **Build matrix**: Both CPU and CUDA variants are built in parallel +- **Multi-architecture**: CPU images built for both amd64 and arm64 (via QEMU emulation) +- **Build matrix**: CPU (multi-arch) and CUDA (x86_64 only) variants built in parallel - **Tagging strategy**: - Main branch pushes: `latest` and `X.Y.Z` (version from flake.nix) - Version tags: `vX.Y.Z` and `latest` - Pull requests: `pr-N` (for testing, not pushed to registry) + - Architecture-specific: `latest-amd64`, `latest-arm64` - **Registry**: All images are publicly accessible at `ghcr.io/utensils/comfyui-nix` - **Build cache**: Nix builds are cached using Cachix for faster CI runs diff --git a/flake.nix b/flake.nix index dfd3de7..9415eab 100644 --- a/flake.nix +++ b/flake.nix @@ -87,6 +87,11 @@ pythonEnv = pythonEnv; }; + templateInputsScript = pkgs.substituteAll { + src = ./scripts/template_inputs.sh; + pythonEnv = pythonEnv; + }; + # Main launcher script with substitutions launcherScript = pkgs.substituteAll { src = ./scripts/launcher.sh; @@ -106,6 +111,7 @@ cp ${installScript} $out/install.sh cp ${persistenceShScript} $out/persistence.sh cp ${runtimeScript} $out/runtime.sh + cp ${templateInputsScript} $out/template_inputs.sh cp ${launcherScript} $out/launcher.sh chmod +x $out/*.sh ''; @@ -152,12 +158,20 @@ # Copy all script files cp -r ${scriptDir}/* "$out/share/comfy-ui/scripts/" - # Install the launcher script - ln -s "$out/share/comfy-ui/scripts/launcher.sh" "$out/bin/comfy-ui-launcher" - chmod +x "$out/bin/comfy-ui-launcher" - - # Create a symlink to the launcher - ln -s "$out/bin/comfy-ui-launcher" "$out/bin/comfy-ui" + # Install the launcher script with wrapper for required tools + # curl and jq are needed for downloading workflow template input files + makeWrapper "$out/share/comfy-ui/scripts/launcher.sh" "$out/bin/comfy-ui" \ + --prefix PATH : "${ + pkgs.lib.makeBinPath [ + pkgs.curl + pkgs.jq + pkgs.git + pkgs.coreutils + ] + }" + + # Create alias for backwards compatibility + ln -s "$out/bin/comfy-ui" "$out/bin/comfy-ui-launcher" ''; meta = with pkgs.lib; { @@ -183,6 +197,7 @@ pkgs.netcat pkgs.git pkgs.curl + pkgs.jq pkgs.cacert pkgs.libGL pkgs.libGLU @@ -257,6 +272,7 @@ pkgs.netcat pkgs.git pkgs.curl + pkgs.jq pkgs.cacert pkgs.libGL pkgs.libGLU diff --git a/scripts/install.sh b/scripts/install.sh index ae332d5..c026cf0 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -284,11 +284,19 @@ install_all() { install_model_downloader setup_venv setup_persistence_scripts - + # Now set up the actual symlinks source "$SCRIPT_DIR/persistence.sh" setup_persistence - + + # Download workflow template input files (non-blocking) + source "$SCRIPT_DIR/template_inputs.sh" + if needs_template_inputs; then + download_template_inputs + else + log_debug "Template input files are up to date" + fi + log_section "Installation complete" log_info "ComfyUI $COMFY_VERSION has been successfully installed" } diff --git a/scripts/template_inputs.sh b/scripts/template_inputs.sh new file mode 100644 index 0000000..746efa6 --- /dev/null +++ b/scripts/template_inputs.sh @@ -0,0 +1,214 @@ +#!/usr/bin/env bash +# template_inputs.sh: Download workflow template input files from GitHub + +# Guard against multiple sourcing +[[ -n "${_TEMPLATE_INPUTS_SH_SOURCED:-}" ]] && return +_TEMPLATE_INPUTS_SH_SOURCED=1 + +# Template input files manifest URL +TEMPLATE_MANIFEST_URL="https://raw.githubusercontent.com/Comfy-Org/workflow_templates/refs/heads/main/workflow_template_input_files.json" + +# Maximum file size for template inputs (50MB - prevents DoS via large files) +MAX_TEMPLATE_FILE_SIZE=$((50 * 1024 * 1024)) + +# Platform-agnostic file size function +get_file_size() { + local file=$1 + if [[ "$(uname)" == "Darwin" ]]; then + # macOS/BSD + stat -f%z "$file" 2>/dev/null || echo "0" + else + # Linux/GNU + stat -c%s "$file" 2>/dev/null || echo "0" + fi +} + +# Download template input files that don't exist locally +download_template_inputs() { + local input_dir="$BASE_DIR/input" + local manifest_cache="$input_dir/.template_manifest.json" + local download_count=0 + local skip_count=0 + local fail_count=0 + + log_info "Checking workflow template input files..." + + # Ensure input directory exists + mkdir -p "$input_dir" + + # Download manifest (with timeout to not block startup) + log_debug "Fetching template manifest from GitHub..." + local curl_error + if ! curl_error=$(curl -fsSL --connect-timeout 5 --max-time 30 "$TEMPLATE_MANIFEST_URL" -o "$manifest_cache.tmp" 2>&1); then + log_warn "Could not fetch template manifest: ${curl_error:-network unavailable or timeout}" + log_debug "URL: $TEMPLATE_MANIFEST_URL" + log_warn "Template input files may be missing - workflows might not work correctly" + rm -f "$manifest_cache.tmp" + return 0 # Don't fail startup + fi + + # Validate downloaded manifest is not empty and move atomically + if [[ ! -s "$manifest_cache.tmp" ]]; then + log_warn "Downloaded manifest is empty, keeping previous cache" + rm -f "$manifest_cache.tmp" + return 0 + fi + + # Check if jq is available + if ! command -v jq &>/dev/null; then + log_warn "jq not found - cannot parse template manifest" + rm -f "$manifest_cache.tmp" + return 0 + fi + + # Validate JSON structure before processing + if ! jq -e '.assets' "$manifest_cache.tmp" &>/dev/null; then + log_warn "Invalid manifest format (missing 'assets' array), removing cache" + rm -f "$manifest_cache.tmp" + return 0 + fi + + # Atomically move the validated manifest + mv "$manifest_cache.tmp" "$manifest_cache" + + # Count total files + local total_files + total_files=$(jq -r '.assets | length' "$manifest_cache") + + if [[ "$total_files" == "0" ]]; then + log_warn "No template input files found in manifest" + return 0 + fi + + log_info "Found $total_files template input files to check" + + # Process each asset in the manifest + local i=0 + while IFS= read -r line; do + local file_path url + file_path=$(echo "$line" | jq -r '.file_path // empty') + url=$(echo "$line" | jq -r '.url // empty') + + # Skip if missing required fields + if [[ -z "$file_path" || -z "$url" ]]; then + continue + fi + + # Security: Check for path traversal attempts + if [[ "$file_path" =~ \.\. ]]; then + log_warn "Skipping potentially malicious file_path: $file_path" + ((fail_count++)) + continue + fi + + # Extract just the filename from file_path (e.g., "input/foo.png" -> "foo.png") + local filename + filename=$(basename "$file_path") + local local_path="$input_dir/$filename" + + # Skip if file already exists + if [[ -f "$local_path" ]]; then + ((skip_count++)) + log_debug "Skipping existing: $filename" + continue + fi + + # Download the file + log_debug "Downloading: $filename" + local download_error + if download_error=$(curl -fsSL --connect-timeout 5 --max-time 60 "$url" -o "$local_path.tmp" 2>&1); then + # Validate file size to prevent DoS + local file_size + file_size=$(get_file_size "$local_path.tmp") + if ((file_size > MAX_TEMPLATE_FILE_SIZE)); then + log_warn "File too large (${file_size} bytes, max ${MAX_TEMPLATE_FILE_SIZE}), skipping: $filename" + rm -f "$local_path.tmp" + ((fail_count++)) + continue + fi + + # Validate file is not empty + if [[ ! -s "$local_path.tmp" ]]; then + log_debug "Downloaded file is empty, skipping: $filename" + rm -f "$local_path.tmp" + ((fail_count++)) + continue + fi + + # Validate file type if 'file' command is available + if command -v file &>/dev/null; then + local file_type + file_type=$(file -b --mime-type "$local_path.tmp" 2>/dev/null || echo "unknown") + # Allow images, videos, and common asset types + if [[ ! $file_type =~ ^(image/|video/|audio/|application/octet-stream|text/) ]]; then + log_warn "Unexpected file type ($file_type), skipping: $filename" + rm -f "$local_path.tmp" + ((fail_count++)) + continue + fi + fi + + mv "$local_path.tmp" "$local_path" + ((download_count++)) + # Show progress every 10 files + if ((download_count % 10 == 0)); then + log_info "Downloaded $download_count files..." + fi + else + ((fail_count++)) + log_debug "Failed to download $filename: ${download_error:-unknown error}" + rm -f "$local_path.tmp" + fi + + ((i++)) + done < <(jq -c '.assets[]' "$manifest_cache") + + # Summary + if ((download_count > 0)); then + log_info "Downloaded $download_count new template input files" + fi + if ((skip_count > 0)); then + log_debug "Skipped $skip_count existing files" + fi + if ((fail_count > 0)); then + log_warn "Failed to download $fail_count files (will retry next startup)" + fi + + log_info "Template input files ready" +} + +# Quick check if template inputs need downloading (for faster startup) +# Returns 0 if download needed, 1 if up to date +needs_template_inputs() { + local input_dir="$BASE_DIR/input" + local manifest_cache="$input_dir/.template_manifest.json" + + # If no manifest cache, we need to download + if [[ ! -f "$manifest_cache" ]]; then + return 0 + fi + + # Check if manifest is older than 7 days (604800 seconds) + # This ensures we periodically re-check for new template files + local manifest_age current_time file_mtime + current_time=$(date +%s) + if [[ "$(uname)" == "Darwin" ]]; then + file_mtime=$(stat -f%m "$manifest_cache" 2>/dev/null || echo "0") + else + file_mtime=$(stat -c %Y "$manifest_cache" 2>/dev/null || echo "0") + fi + manifest_age=$((current_time - file_mtime)) + if ((manifest_age > 604800)); then + return 0 + fi + + # Quick heuristic: if input dir has fewer than 20 files, probably needs download + # This catches cases where the manifest exists but files were deleted + local file_count + file_count=$(find "$input_dir" -maxdepth 1 -type f ! -name ".*" 2>/dev/null | wc -l) + if ((file_count < 20)); then + return 0 + fi + + return 1 +}