(WIP) feat: split pkg build vs container

.github/workflows/build_push_pr_ecr.yml

@@ -4,59 +4,147 @@ on:
   pull_request:
     types: [opened, synchronize, reopened]
 
+env:
+  CONDA_CHANNEL: "s3://hls-conda-channels/hls-atmospheric-correction/pr-${{ github.event.pull_request.number }}"
+  CONDA_CHANNEL_HTTP: "https://hls-conda-channels.s3.amazonaws.com/hls-atmospheric-correction/pr-${{ github.event.pull_request.number }}"
+  # FIXME: for now set "--frozen" since we're having trouble with inconsistent lockfiles
+  PIXI_FROZEN: true
+
+permissions:
+  contents: write
+  id-token: write
+
 jobs:
-  build_push_to_ecr:
+  # ------------------------------------------------------------------
+  # JOB 1: Dynamic Analysis
+  # ------------------------------------------------------------------
+  analyze:
     runs-on: ubuntu-latest
+    outputs:
+      has_changes: ${{ steps.detect.outputs.has_changes }}
+      docker_changed: ${{ steps.detect.outputs.docker_changed }}
+      build_order: ${{ steps.detect.outputs.build_order }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: prefix-dev/setup-pixi@v0.9.2
+        with:
+          frozen: true
+          environments: build
+
+      - name: Run Change Detection
+        id: detect
+        run: |
+          pixi run -e build determine-targets
 
+  # ------------------------------------------------------------------
+  # JOB 2: Sequential Build & Publish
+  # ------------------------------------------------------------------
+  build-and-publish:
+    needs: analyze
+    if: needs.analyze.outputs.has_changes == 'true'
+    runs-on: ubuntu-latest
     steps:
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v5
+      - uses: actions/checkout@v4
+
+      - name: Free disk space
+        uses: jlumbroso/free-disk-space@main
+        with:
+          large-packages: false  # takes too long for ~4.7GB
+          docker-images: true
+
+      - uses: prefix-dev/setup-pixi@v0.9.2
+        with:
+          frozen: true
+          environments: build
+
+      - name: Install Tools
+        run: pixi global install rattler-build rattler-index awscli
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
         with:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: us-west-2
 
-      - name: Login to Amazon ECR
-        id: login-ecr
-        uses: aws-actions/amazon-ecr-login@v2
+      - name: Execute Build Sequence
+        run: |
+          pixi run -e build \
+            build-publish-packages \
+            "${{ needs.analyze.outputs.build_order }}"
 
-      - uses: actions/checkout@v2
+  # ------------------------------------------------------------------
+  # JOB 3: Docker Build & Lockfile Update
+  # ------------------------------------------------------------------
+  build-docker:
+    needs: [analyze, build-and-publish]
+    # Run if Docker changed OR if packages were rebuilt (and build job succeeded)
+    if: |
+      always() &&
+      (needs.analyze.outputs.docker_changed == 'true' || needs.analyze.outputs.has_changes == 'true') &&
+      (needs.build-and-publish.result == 'success' || needs.build-and-publish.result == 'skipped')
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.ref }}
 
-      # FIXME: run the tests in the container...
-      #    - that's really what matters for this project
-      #    - the C code deps are hard to build
-      # - name: Run tests
-      #   run: |
-      #     scripts/test
+      - name: Free disk space
+        uses: jlumbroso/free-disk-space@main
+        with:
+          large-packages: false  # takes too long for ~4.7GB
+          docker-images: true
 
-      # - name: Lint code
-      #   run: |
-      #     scripts/lint
-      #     scripts/typecheck
+      - uses: prefix-dev/setup-pixi@v0.9.2
+        with:
+          frozen: true
+          environments: build
 
-      - name: Cleanup space
-        # ref: https://github.com/actions/runner-images/issues/2840
+      - name: Update Lockfile
+        if: needs.analyze.outputs.has_changes == 'true'
         run: |
-            echo "Before:"
-            df -h /
-            rm -rf /usr/share/dotnet
-            rm -rf "$AGENT_TOOLSDIRECTORY"
-            echo "After:"
-            df -h /
-
-      - name: Build image
+          echo "Updating lockfile for: ${{ needs.analyze.outputs.build_order }}"
+
+          # Add local package builds to "pr" feature as top priority
+          pixi workspace channel add --no-install --prepend ${CONDA_CHANNEL_HTTP}
+
+          # Update for these new builds
+          pixi update --no-install ${{ needs.analyze.outputs.build_order }}
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v2
+
+      - name: Build and Push Docker Image
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          ECR_REPOSITORY: hls-atmospheric-correction
+          IMAGE_TAG: ${{ github.event.pull_request.number }}
         run: |
-          df -h
-          docker build -t hls-atmospheric-correction .
-
-      # FIXME: use Github Actions version,
-      # https://github.com/depot/build-push-action/blob/main/docs/build-and-push-ecr.md
-      - name: Push to latest ECR
-        id: push-latest-ecr
-        uses: jwalton/gh-ecr-push@v1
-        with:
-          access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          region: us-west-2
-          local-image: hls-atmospheric-correction
-          image: hls-atmospheric-correction:${{ github.event.pull_request.number }}
+          docker build \
+            -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG \
+            .
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
+
+#      - name: Push update to lockfile
+#        if: needs.analyze.outputs.has_changes == 'true'
+#        run: |
+#          if [[ -n $(git status -s pixi.lock) ]]; then
+#            git config --global user.name "GitHub Actions"
+#            git config --global user.email "actions@github.com"
+#            git add pixi.lock
+#            git commit -m "chore: update pixi.lock [skip ci]"
+#            git push
+#          else
+#            echo "No lockfile change detected".
+#          fi

Dockerfile

@@ -2,6 +2,8 @@
 ARG PLATFORM=linux/amd64
 FROM --platform=${PLATFORM} ghcr.io/prefix-dev/pixi:bookworm-slim AS build
 
+ARG ENVIRONMENT=default
+
 WORKDIR /app
 
 RUN apt-get update && \
@@ -11,15 +13,14 @@ RUN apt-get update && \
         && \
     rm -rf /var/lib/apt/lists/*
 
-COPY --parents pixi.toml pixi.lock packages /app/
+ENV PIXI_FROZEN=true
 
+COPY --parents pixi.toml pixi.lock /app/
 RUN --mount=type=cache,target=/root/.cache/rattler/cache,sharing=private \
-    pixi install --frozen
-
-ENV PREFIX=/app/.pixi/envs/default
+    pixi install -e $ENVIRONMENT
 
 RUN echo '#!/bin/bash' > /app/entrypoint.sh && \
-    pixi shell-hook --frozen -e default -s bash >> /app/entrypoint.sh && \
+    pixi shell-hook -e $ENVIRONMENT -s bash >> /app/entrypoint.sh && \
     echo 'exec "$@"' >> /app/entrypoint.sh
 
 # "Productionize" pixi install: https://pixi.sh/latest/deployment/container/

packages/espa-product-formatter/recipe.yml

@@ -97,3 +97,9 @@ tests:
         exists:
           - bin/convert_espa_to_hdf
           - schema/espa_internal_metadata_v2_2.xsd
+  - script:
+      - convert_espa_to_hdf --help | grep "convert_espa_to_hdf version"
+
+about:
+  homepage: https://github.com/NASA-IMPACT/espa-product-formatter
+  license: MIT

packages/espa-surface-reflectance/recipe.yml

@@ -19,10 +19,11 @@ requirements:
     - liblzma-devel
   run:
     - python
+    - espa-product-formatter
 
 build:
   number: 0
-  script : |
+  script: |
     export HDFINC="${PREFIX}/include"
     export HDFLIB="${PREFIX}/lib"
     export HDF5INC="${PREFIX}/include"
@@ -57,4 +58,8 @@ tests:
         exists:
           - bin/lasrc
   - script:
-      - lasrc --help
+      - lasrc --help | grep "LaSRC (Land Surface Reflectance Code)"
+
+about:
+  homepage: https://github.com/NASA-IMPACT/espa-surface-reflectance
+  license: MIT

packages/fmask4/build.sh

@@ -6,11 +6,10 @@ INSTALL_DIR="$PREFIX/bin/fmask4"
 
 INSTALLER_FILE="Fmask_Linux_mcr.install"
 
-wget -q -O ${INSTALLER_FILE} https://fmask4installer.s3.amazonaws.com/Fmask_4_7_issue40_Linux_mcr.install
-
 chmod +x $INSTALLER_FILE
 ./$INSTALLER_FILE -destinationFolder ${INSTALL_DIR} -agreeToLicense yes -mode silent
 
+# cleanup giant install file to keep disk utilization down
 rm $INSTALLER_FILE
 
 cp ${RECIPE_DIR}/run_Fmask.sh ${PREFIX}/bin/

packages/fmask4/recipe.yml

@@ -2,10 +2,10 @@ package:
   name: fmask4
   version: "4.7"
 
-# source:
-#   url: https://fmask4installer.s3.amazonaws.com/Fmask_4_7_issue40_Linux_mcr.install
-#   sha256: 12aa4604d5f40d2cb045d42eaa472b1091abb05f8ab15cb0078103f7c79bd9a0
-#   file_name: Fmask_Linux_mcr.install
+source:
+  url: https://fmask4installer.s3.amazonaws.com/Fmask_4_7_issue40_Linux_mcr.install
+  sha256: 12aa4604d5f40d2cb045d42eaa472b1091abb05f8ab15cb0078103f7c79bd9a0
+  file_name: Fmask_Linux_mcr.install
 
 build:
   number: 0
@@ -17,8 +17,10 @@ requirements:
   build:
     - curl
     - unzip
-    - wget
   run:
     - libgcc
     - libstdcxx
     - xorg-libxt
+
+about:
+  homepage: https://github.com/GERSL/Fmask4/

packages/hls-libs/recipe.yml

@@ -16,10 +16,12 @@ requirements:
     - hdfeos2
     - hdfeos5
     - liblzma-devel
+  run:
+    - espa-product-formatter
 
 build:
   number: 0
-  script : |
+  script: |
     export HDFINC="${PREFIX}/include"
     export HDFLIB="${PREFIX}/lib"
     export GCTPINC="${PREFIX}/include"
@@ -51,3 +53,9 @@ tests:
         exists:
           - bin/sentinel-add-fmask-sds
           - bin/landsat-add-fmask-sds
+  - script:
+      - sentinel-add-fmask-sds --help 2>&1 | grep "fmask"
+
+about:
+  summary: Utility programs for the HLS Landsat and Sentinel-2 pipelines.
+  license: MIT