build_test_fp_x86.yaml

name: Build, Test Forcing Processor Docker Containers on X86
on:
  workflow_dispatch:
  push:
    branches:
      - main
      - fp_workflow
      - trivy_scan
    paths:
      - 'docker/**' 
      - 'src/forcingprocessor/**'
      - '!src/forcingprocessor/README.md'
  pull_request:
    paths:
      - 'docker/**' 
      - 'src/forcingprocessor/**'
      - '!src/forcingprocessor/README.md'


permissions:
  contents: read
  security-events: write

jobs:
  build-x86:
    runs-on: ubuntu-latest
    steps:
      - name: Pre-cleanup workspace
        continue-on-error: true
        run: |
          sudo rm -rf ${{ github.workspace }}/* || true
          sudo rm -rf ${{ github.workspace }}/.* || true
          
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Configure AWS
        run: |
          aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws configure set region us-east-1

      - name: Build forcingprocessor-deps
        run: |
          cd docker
          TAG=latest-x86 docker compose -f docker-compose.yml build forcingprocessor-deps
            
      - name: Build forcingprocessor
        run: |
          cd docker
          TAG=latest-x86 docker compose -f docker-compose.yml build forcingprocessor
            
      - name: List built images
        run: docker images | grep forcingprocessor


      - name: Save Docker images
        run: |
          docker save awiciroh/forcingprocessor:latest-x86 | gzip > /tmp/forcingprocessor-x86.tar.gz
          
      - name: Upload forcingprocessor image
        uses: actions/upload-artifact@v4
        with:
          name: forcingprocessor-x86
          path: /tmp/forcingprocessor-x86.tar.gz
          retention-days: 1

      - name: Clean up local tar files
        if: always()
        run: rm -f /tmp/forcingprocessor-x86.tar.gz


  test-x86:
    needs: build-x86
    runs-on: ubuntu-latest
    timeout-minutes: 30
    strategy:
      fail-fast: false
      max-parallel: 10
      matrix:
        test-suite:
          - name: "NOMADS"
            image: "forcingprocessor"
            command: "pytest -q --tb=short --disable-warnings --maxfail=1 -k test_nomads"
          
          - name: "Google Cloud Storage"
            image: "forcingprocessor"
            command: "pytest -q --tb=short --disable-warnings --maxfail=1 -k 'test_nwm_google_apis or test_google or test_gs or test_gcs'"
          
          - name: "NOAA NWM PDS - Base HTTPS"
            image: "forcingprocessor"
            command: "pytest -q --tb=short --disable-warnings --maxfail=1 -k 'test_noaa_nwm_pds_https and not short_range and not medium_range and not analysis_assim'"
          
          - name: "NOAA NWM - Short Range"
            image: "forcingprocessor"
            command: "pytest -q --tb=short --disable-warnings --maxfail=1 -k 'test_noaa_nwm_pds_https_short_range'"
          
          - name: "NOAA NWM - Medium Range"
            image: "forcingprocessor"
            command: "pytest -q --tb=short --disable-warnings --maxfail=1 -k 'test_noaa_nwm_pds_https_medium_range'"
          
          - name: "NOAA NWM - Analysis Assim"
            image: "forcingprocessor"
            command: "pytest -q --tb=short --disable-warnings --maxfail=1 -k 'test_noaa_nwm_pds_https_analysis_assim and not extend'"
          
          - name: "NOAA NWM - Analysis Assim Extend"
            image: "forcingprocessor"
            command: "pytest -q --tb=short --disable-warnings --maxfail=1 -k 'test_noaa_nwm_pds_https_analysis_assim_extend'"
          
          - name: "NOAA NWM - S3 Access"
            image: "forcingprocessor"
            command: "pytest -q --tb=short --disable-warnings --maxfail=1 -k 'test_noaa_nwm_pds_s3'"
          
          - name: "Retrospective v2.1"
            image: "forcingprocessor"
            command: "pytest -q --tb=short --disable-warnings --maxfail=1 -k 'test_retro_2_1'"
          
          - name: "Retrospective v3.0"
            image: "forcingprocessor"
            command: "pytest -q --tb=short --disable-warnings --maxfail=1 -k 'test_retro_3_0'"
          
          - name: "Output Formats"
            image: "forcingprocessor"
            command: "pytest -q --tb=short --disable-warnings --maxfail=1 -k 'test_csv_output_type or test_parquet_output_type or test_tar_output_type or test_netcdf_output_type'"
          
          - name: "Plotting & S3 Output"
            image: "forcingprocessor"
            command: "pytest -q --tb=short --disable-warnings --maxfail=1 -k 'test_plotting or test_s3_output'"

          - name: "Weight Generation"
            image: "forcingprocessor"
            command: "pytest -q --tb=short --disable-warnings --maxfail=1 tests/test_hf2ds.py"            

    name: "${{ matrix.test-suite.name }}"
    
    steps:
      - name: Pre-cleanup workspace
        continue-on-error: true
        run: |
          sudo rm -rf ${{ github.workspace }}/* || true
          sudo rm -rf ${{ github.workspace }}/.* || true
          
      - name: Checkout code
        uses: actions/checkout@v4
        
      - name: Download Docker image
        uses: actions/download-artifact@v4
        with:
          name: forcingprocessor-x86  
          path: /tmp/artifacts


      - name: Verify artifact and load
        run: |
          set -euo pipefail
          ls -lh /tmp/artifacts
          
          # Load gzipped image
          for i in 1 2 3; do
            if gunzip -c /tmp/artifacts/forcingprocessor-x86.tar.gz | docker load; then
              echo "Successfully loaded image on attempt $i"
              docker images | grep forcingprocessor
              break
            else
              echo "Failed to load image on attempt $i"
              if [ $i -lt 3 ]; then
                echo "Retrying in 10 seconds..."
                sleep 10
              else
                exit 1
              fi
            fi
          done


      - name: Run ${{ matrix.test-suite.name }}
        run: |
          docker run --rm \
            -v $(pwd):/forcingprocessor \
            -w /forcingprocessor \
            -e AWS_REGION=us-east-1 \
            -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
            -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
            awiciroh/${{ matrix.test-suite.image }}:latest-x86 \
            ${{ matrix.test-suite.command }}
            
      # - name: Clean up Docker image
      #   if: always()
      #   run: |
      #     docker rmi awiciroh/${{ matrix.test-suite.image }}:latest-x86 || true
      #     docker system prune -f
          
  cleanup-artifacts:
    needs: test-x86
    runs-on: ubuntu-latest
    if: always()
    steps:
      - name: Delete artifacts
        uses: geekyeggo/delete-artifact@v5
        with:
          name: forcingprocessor-x86