NGWPC NWM PI-3 Formulation Selection Delivery 2025-11-21

.github/workflows/ci-cd.yml

@@ -0,0 +1,218 @@
+name: CI/CD Pipeline
+
+on:
+  pull_request:
+    branches: [main, nwm-main, development, release-candidate]
+  push:
+    branches: [main, nwm-main, development, release-candidate]
+  release:
+    types: [published]
+
+permissions:
+  contents: read
+  packages: write
+  security-events: write
+
+env:
+  REGISTRY: ghcr.io
+  PYTHON_VERSION: '3.11'
+
+jobs:
+  setup:
+    runs-on: ubuntu-latest
+    outputs:
+      image_base: ${{ steps.vars.outputs.image_base }}
+      pr_tag: ${{ steps.vars.outputs.pr_tag }}
+      commit_sha: ${{ steps.vars.outputs.commit_sha }}
+      commit_sha_short: ${{ steps.vars.outputs.commit_sha_short }}
+      test_image_tag: ${{ steps.vars.outputs.test_image_tag }}
+    steps:
+      - name: Compute image vars
+        id: vars
+        shell: bash
+        run: |
+          set -euo pipefail
+          ORG="$(echo "${GITHUB_REPOSITORY_OWNER}" | tr '[:upper:]' '[:lower:]')"
+          REPO="$(basename "${GITHUB_REPOSITORY}")"
+          IMAGE_BASE="${REGISTRY}/${ORG}/${REPO}"
+          echo "image_base=${IMAGE_BASE}" >> "$GITHUB_OUTPUT"
+
+          if [ "${GITHUB_EVENT_NAME}" = "pull_request" ]; then
+            PR_NUM="${{ github.event.pull_request.number }}"
+            PR_TAG="pr-${PR_NUM}-build"
+            echo "pr_tag=${PR_TAG}" >> "$GITHUB_OUTPUT"
+            echo "test_image_tag=${PR_TAG}" >> "$GITHUB_OUTPUT"
+          fi
+
+          if [ "${GITHUB_EVENT_NAME}" = "push" ]; then
+            COMMIT_SHA="${GITHUB_SHA}"
+            SHORT_SHA="${COMMIT_SHA:0:12}"
+            echo "commit_sha=${COMMIT_SHA}" >> "$GITHUB_OUTPUT"
+            echo "commit_sha_short=${SHORT_SHA}" >> "$GITHUB_OUTPUT"
+            echo "test_image_tag=${SHORT_SHA}" >> "$GITHUB_OUTPUT"
+          fi
+
+  build:
+    name: build
+    if: github.event_name == 'pull_request' || github.event_name == 'push'
+    runs-on: ubuntu-latest
+    needs: setup
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Log in to registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build & push image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          push: true
+          tags: ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }}
+          build-args: |
+            NGEN_IMAGE_TAG=${{ env.NGEN_IMAGE_TAG || 'latest' }}
+            CI_COMMIT_REF_NAME=${{ github.ref_name }}
+
+  unit-test:
+    name: unit-test
+    if: github.event_name == 'pull_request' || github.event_name == 'push'
+    runs-on: ubuntu-latest
+    needs: [setup, build]
+    container:
+      image: ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }}
+    steps:
+      - name: Run unit tests
+        run: |
+          echo "TODO: add unit tests here"
+
+  codeql-scan:
+    if: github.event_name == 'pull_request' || github.event_name == 'push'
+    runs-on: ubuntu-latest
+    needs: [setup, build]
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: python
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3
+
+  container-scanning:
+    if: github.event_name == 'pull_request' || github.event_name == 'push'
+    runs-on: ubuntu-latest
+    needs: [setup, build]
+    steps:
+      - name: Scan container with Trivy
+        uses: aquasecurity/trivy-action@0.20.0
+        with:
+          image-ref: ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }}
+          format: 'template'
+          template: '@/contrib/sarif.tpl'
+          output: 'trivy-results.sarif'
+          severity: 'CRITICAL,HIGH'
+      - name: Upload Trivy SARIF
+        uses: github/codeql-action/upload-sarif@v3
+        with:
+          sarif_file: 'trivy-results.sarif'
+
+  deploy-latest-on-development:
+    name: deploy-latest-on-development
+    if: github.event_name == 'push' && github.ref_name == 'development'
+    runs-on: ubuntu-latest
+    needs: [setup, build, unit-test, codeql-scan, container-scanning]
+    steps:
+      - name: Tag image with 'latest'
+        shell: bash
+        run: |
+          set -euo pipefail
+          IMAGE_BASE="${{ needs.setup.outputs.image_base }}"
+          SHORT_SHA="${{ needs.setup.outputs.commit_sha_short }}"
+
+          # ensure skopeo is available
+          if ! command -v skopeo >/dev/null 2>&1; then
+            sudo apt-get update -y
+            sudo apt-get install -y --no-install-recommends skopeo
+          fi
+
+          skopeo copy \
+            --src-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \
+            --dest-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \
+            docker://"${IMAGE_BASE}:${SHORT_SHA}" docker://"${IMAGE_BASE}:latest"
+
+  release:
+    name: release
+    if: github.event_name == 'release' && github.event.action == 'published'
+    runs-on: ubuntu-latest
+    needs: setup
+    steps:
+      - name: Get commit sha for the tag
+        id: rev
+        shell: bash
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -euo pipefail
+          TAG="${{ github.event.release.tag_name }}"
+          REPO="${{ github.repository }}"
+
+          # ensure jq is available
+          if ! command -v jq >/dev/null 2>&1; then
+            sudo apt-get update -y
+            sudo apt-get install -y --no-install-recommends jq
+          fi
+
+          # ensure gh cli is available
+          if ! command -v gh >/dev/null 2>&1; then
+            sudo apt-get update -y
+            sudo apt-get install -y --no-install-recommends gh
+          fi
+
+          REF_JSON="$(gh api "repos/${REPO}/git/refs/tags/${TAG}")"
+          OBJ_SHA="$(jq -r '.object.sha' <<<"$REF_JSON")"
+          OBJ_TYPE="$(jq -r '.object.type' <<<"$REF_JSON")"
+
+          if [ "$OBJ_TYPE" = "tag" ]; then
+            TAG_OBJ="$(gh api "repos/${REPO}/git/tags/${OBJ_SHA}")"
+            COMMIT_SHA="$(jq -r '.object.sha' <<<"$TAG_OBJ")"
+          else
+            COMMIT_SHA="$OBJ_SHA"
+          fi
+
+          SHORT_SHA="${COMMIT_SHA:0:12}"
+          echo "short_sha=${SHORT_SHA}" >> "$GITHUB_OUTPUT"
+
+      - name: Tag image with release tag
+        shell: bash
+        run: |
+          set -euo pipefail
+          IMAGE_BASE="${{ needs.setup.outputs.image_base }}"
+          SHORT_SHA="${{ steps.rev.outputs.short_sha }}"
+          RELEASE_TAG="${{ github.event.release.tag_name }}"
+
+          # ensure skopeo is available
+          if ! command -v skopeo >/dev/null 2>&1; then
+            sudo apt-get update -y
+            sudo apt-get install -y --no-install-recommends skopeo
+          fi
+
+          skopeo copy \
+            --src-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \
+            --dest-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \
+            docker://"${IMAGE_BASE}:${SHORT_SHA}" docker://"${IMAGE_BASE}:${RELEASE_TAG}"

.gitignore

@@ -1,2 +1,94 @@
+# Compiled source #
+###################
+*.com
+*.class
+*.dll
+*.exe
+*.o
+*.so
+_site/
+
+# Packages #
+############
+# it's better to unpack these files and commit the raw source
+# git has its own built in compression methods
+*.7z
+*.dmg
+*.gz
+*.iso
+*.jar
+*.rar
+*.tar
+*.zip
+
+# Logs and databases #
+######################
+*.log
+*.sql
+*.sqlite
+
+# OS generated files #
+######################
+.DS_Store
+.DS_Store?
+.Spotlight-V100
+.Trashes
+Icon?
+ehthumbs.db
+Thumbs.db
+
+# Vim swap files #
+##################
+*.swp
+
+# Python #
+#################
+*.pyc
+*.egg-info/
+__pycache__/
+*.py[cod]
+.env
+.python-version
+venv
+*.pytest_cache
+build
+
+# pyenv #
+#########
+.python-version
+
+# Django #
+#################
+*.egg-info
+.installed.cfg
+
+# Unit test / coverage reports
+#################
+htmlcov/
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Front-End #
+#############
+node_modules/
+bower_components/
+.grunt/
+src/vendor/
+dist/
+
+# Temporary Directories #
+*tmp/
+*temp/
+
+# Other
 .idea
 /git_info.json
+
+# Ngen FIles #
+#############
+cat*
+nex*
+troute*

Dockerfile

@@ -1,27 +1,39 @@
-ARG  NGEN_VERSION=latest
-FROM registry.sh.nextgenwaterprediction.com/ngwpc/nwm-ngen/ngen:${NGEN_VERSION}
+# syntax=docker/dockerfile:1.4
+ARG  NGEN_IMAGE_TAG=latest
+FROM ghcr.io/ngwpc/ngen:${NGEN_IMAGE_TAG}
 
-RUN set -eux; \
-    dnf install -y \
-        jq; \
-    dnf clean all
+# Uncomment when building ngen locally or if ngen-int image is available locally
+# modify to use image tag for local ngen image if needed
+#FROM ngen
 
-COPY requirements.txt .
-RUN set -eux; \
-	\
-    pip3 install -r requirements.txt ; \
-    pip3 cache purge ; \
-    rm --force requirements.txt ;
+# Activate the existing virtual environment
+ENV PATH="/ngen-app/ngen-python/bin:${PATH}"
 
+RUN set -eux; \
+    dnf install -y jq; \
+    dnf clean all
 
 COPY . /ngen-app/ngen-fcst/
 COPY ./docker/run-ngen-fcst.sh /ngen-app/bin/
+
 RUN set -eux; \
-	\
     chmod +x /ngen-app/bin/run-ngen-fcst.sh
 
 WORKDIR /ngen-app/ngen-fcst
 
+# Install missing dependencies that aren't in base image
+RUN --mount=type=cache,target=/root/.cache/pip,id=pip-cache \
+    set -eux; \
+    pip3 install \
+        "matplotlib~=3.10.6"; \
+        #"geopandas~=1.1.1"; \
+    pip3 cache purge
+
+# Install into the existing virtual environment without upgrading base packages
+RUN set -eux; \
+    pip3 install --no-deps . || pip3 install .; \
+    pip3 cache purge;
+
 ARG CI_COMMIT_REF_NAME
 
 RUN set -eux; \
@@ -47,4 +59,4 @@ RUN set -eux; \
 
 WORKDIR /
 
-ENTRYPOINT [ "/ngen-app/bin/run-ngen-fcst.sh" ] 
+ENTRYPOINT [ "/ngen-app/bin/run-ngen-fcst.sh" ]