Update workflow to use uv and Python 3.10

.github/workflows/build_test_create_ami.yml

@@ -2,8 +2,6 @@ name: Build, Test, and Push Datastream AMI
 on:
   workflow_dispatch:   
   push:
-    branches: 
-      - main
     paths:
       - 'ami_version.yml'
 
@@ -503,4 +501,4 @@ jobs:
             echo " OVERALL RESULT: All VPU tests passed successfully"
             echo " Custom AMI is validated and preserved"
             echo " Ready for production deployment"
-          fi
+          fi

.github/workflows/build_test_docker_x86.yaml

@@ -47,11 +47,18 @@ jobs:
 
     - name: Install packages for datastream
       run: |
-        curl -L -O https://github.com/lynker-spatial/hfsubsetCLI/releases/download/v1.1.0/hfsubset-v1.1.0-linux_amd64.tar.gz && tar -xzvf hfsubset-v1.1.0-linux_amd64.tar.gz && sudo mv ./hfsubset /usr/bin/hfsubset && sudo apt-get install git pip pigz -y        
+        # Install system packages
+        sudo apt-get update && sudo apt-get install -y git python3-pip pigz
+
+        # Download and install hfsubset
+        wget -O hfsubset-v1.1.0-linux_amd64.tar.gz https://github.com/lynker-spatial/hfsubsetCLI/releases/download/v1.1.0/hfsubset-v1.1.0-linux_amd64.tar.gz
+        tar -xzvf hfsubset-v1.1.0-linux_amd64.tar.gz
+        sudo mv ./hfsubset /usr/bin/hfsubset
+        sudo chmod +x /usr/bin/hfsubset        
 
     - name: Build docker containers
       run : |        
-        ./scripts/docker_builds.sh -f -d
+        ./scripts/docker_builds.sh -e -f -d
 
     - name: Test docker containers
       run : |

.github/workflows/build_test_push_docker_x86.yaml

@@ -40,7 +40,14 @@ jobs:
 
     - name: Install packages for datastream
       run: |
-        curl -L -O https://github.com/lynker-spatial/hfsubsetCLI/releases/download/v1.1.0/hfsubset-v1.1.0-linux_amd64.tar.gz && tar -xzvf hfsubset-v1.1.0-linux_amd64.tar.gz && sudo mv ./hfsubset /usr/bin/hfsubset && sudo apt-get install git pip pigz -y        
+        # Install system packages
+        sudo apt-get update && sudo apt-get install -y git python3-pip pigz
+
+        # Download and install hfsubset
+        wget -O hfsubset-v1.1.0-linux_amd64.tar.gz https://github.com/lynker-spatial/hfsubsetCLI/releases/download/v1.1.0/hfsubset-v1.1.0-linux_amd64.tar.gz
+        tar -xzvf hfsubset-v1.1.0-linux_amd64.tar.gz
+        sudo mv ./hfsubset /usr/bin/hfsubset
+        sudo chmod +x /usr/bin/hfsubset        
 
     - name: Build docker containers
       run : |        

.github/workflows/datastream_python.yaml

@@ -24,17 +24,21 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python 3.9
+    - name: Set up Python 3.10
       uses: actions/setup-python@v3
       with:
-        python-version: "3.9"
+        python-version: "3.10"
 
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -e ./python_tools
-        pip install pytest
+        pip install uv
+        uv venv
+        source .venv/bin/activate
+        uv pip install -e ./python_tools
+        uv pip install pytest
 
     - name: Test with pytest
       run: |
+        source .venv/bin/activate
         python -m pytest -vv python_tools/

.github/workflows/forcingprocessor_aws_sources.yaml

@@ -24,10 +24,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python 3.9
+    - name: Set up Python 3.10
       uses: actions/setup-python@v3
       with:
-        python-version: "3.9"
+        python-version: "3.10"
 
     - name: Configure AWS
       run: |
@@ -38,11 +38,15 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -e ./forcingprocessor
-        pip install pytest
+        pip install uv
+        uv venv
+        source .venv/bin/activate
+        uv pip install -e ./forcingprocessor
+        uv pip install pytest
 
     - name: Test with pytest
       run: |
+        source .venv/bin/activate
         cd forcingprocessor
         python -m pytest -vv -k test_noaa_nwm_pds_https 
         python -m pytest -vv -k test_noaa_nwm_pds_https_short_range 

.github/workflows/forcingprocessor_gcs_sources.yaml

@@ -24,10 +24,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python 3.9
+    - name: Set up Python 3.10
       uses: actions/setup-python@v3
       with:
-        python-version: "3.9"
+        python-version: "3.10"
 
     - name: Configure AWS
       run: |
@@ -38,11 +38,15 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -e ./forcingprocessor
-        pip install pytest
+        pip install uv
+        uv venv
+        source .venv/bin/activate
+        uv pip install -e ./forcingprocessor
+        uv pip install pytest
 
     - name: Test with pytest
       run: |
+        source .venv/bin/activate
         cd forcingprocessor
         python -m pytest -vv -k test_google_cloud_storage 
         python -m pytest -vv -k test_gs 

.github/workflows/forcingprocessor_output_opts.yaml

@@ -24,10 +24,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python 3.9
+    - name: Set up Python 3.10
       uses: actions/setup-python@v3
       with:
-        python-version: "3.9"
+        python-version: "3.10"
 
     - name: Configure AWS
       run: |
@@ -38,11 +38,15 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -e ./forcingprocessor
-        pip install pytest
+        pip install uv
+        uv venv
+        source .venv/bin/activate
+        uv pip install -e ./forcingprocessor
+        uv pip install pytest
 
     - name: Test with pytest
       run: |
+        source .venv/bin/activate
         cd forcingprocessor
         # netcdf already tested
         python -m pytest -vv -k test_csv_output_type

.github/workflows/forcingprocessor_plotting.yaml

@@ -24,10 +24,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python 3.9
+    - name: Set up Python 3.10
       uses: actions/setup-python@v3
       with:
-        python-version: "3.9"
+        python-version: "3.10"
 
     - name: Configure AWS
       run: |
@@ -38,11 +38,15 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -e ./forcingprocessor
-        pip install pytest
+        pip install uv
+        uv venv
+        source .venv/bin/activate
+        uv pip install -e ./forcingprocessor
+        uv pip install pytest
 
     - name: Test with pytest
       run: |
+        source .venv/bin/activate
         cd forcingprocessor
         python -m pytest -vv tests/test_plotter.py
         python -m pytest -vv -k test_plotting

.github/workflows/forcingprocessor_weights.yaml

@@ -24,10 +24,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python 3.9
+    - name: Set up Python 3.10
       uses: actions/setup-python@v3
       with:
-        python-version: "3.9"
+        python-version: "3.10"
 
     - name: Configure AWS
       run: |
@@ -38,10 +38,14 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -e ./forcingprocessor
-        pip install pytest
+        pip install uv
+        uv venv
+        source .venv/bin/activate
+        uv pip install -e ./forcingprocessor
+        uv pip install pytest
 
     - name: Test with hf2ds
       run: |
+        source .venv/bin/activate
         cd forcingprocessor
         python -m pytest -vv tests/test_hf2ds.py

.github/workflows/test_datastream_ngiab.yaml

@@ -60,7 +60,7 @@ jobs:
 
     - name: Build docker containers
       run : |        
-        ./scripts/docker_builds.sh -f -d  
+        ./scripts/docker_builds.sh -e -f -d  
 
     - name: Base test and NWM_RETRO_V3
       run: |        

.github/workflows/test_datastream_options.yaml

@@ -61,7 +61,7 @@ jobs:
 
     - name: Build docker containers
       run : |        
-        ./scripts/docker_builds.sh -f -d    
+        ./scripts/docker_builds.sh -e -f -d    
 
     - name: Get geopackage 
       run: |

.github/workflows/test_hfsubset_v2_1.yaml

@@ -32,11 +32,14 @@ jobs:
 
     - name: Build docker containers
       run : |        
-        ./scripts/docker_builds.sh -f -d    
+        ./scripts/docker_builds.sh -e -f -d    
 
     - name: Install packages for datastream
       run: |
-        curl -L -O https://github.com/lynker-spatial/hfsubsetCLI/releases/download/v1.1.0/hfsubset-v1.1.0-linux_amd64.tar.gz && tar -xzvf hfsubset-v1.1.0-linux_amd64.tar.gz && sudo mv ./hfsubset /usr/bin/hfsubset
+        wget -O hfsubset-v1.1.0-linux_amd64.tar.gz https://github.com/lynker-spatial/hfsubsetCLI/releases/download/v1.1.0/hfsubset-v1.1.0-linux_amd64.tar.gz
+        tar -xzvf hfsubset-v1.1.0-linux_amd64.tar.gz
+        sudo mv ./hfsubset /usr/bin/hfsubset
+        sudo chmod +x /usr/bin/hfsubset
 
     - name: Get geopackage v2.1
       if: always()

.github/workflows/test_hfsubset_v2_2.yaml

@@ -32,11 +32,14 @@ jobs:
 
     - name: Build docker containers
       run : |        
-        ./scripts/docker_builds.sh -f -d    
+        ./scripts/docker_builds.sh -e -f -d    
 
     - name: Install packages for datastream
       run: |
-        curl -L -O https://github.com/lynker-spatial/hfsubsetCLI/releases/download/v1.1.0/hfsubset-v1.1.0-linux_amd64.tar.gz && tar -xzvf hfsubset-v1.1.0-linux_amd64.tar.gz && sudo mv ./hfsubset /usr/bin/hfsubset
+        wget -O hfsubset-v1.1.0-linux_amd64.tar.gz https://github.com/lynker-spatial/hfsubsetCLI/releases/download/v1.1.0/hfsubset-v1.1.0-linux_amd64.tar.gz
+        tar -xzvf hfsubset-v1.1.0-linux_amd64.tar.gz
+        sudo mv ./hfsubset /usr/bin/hfsubset
+        sudo chmod +x /usr/bin/hfsubset
 
     - name: Get geopackage v2.2
       run: |

.github/workflows/test_teehr_integration.yaml

@@ -42,7 +42,7 @@ jobs:
 
     - name: Build docker containers
       run : |        
-        ./scripts/docker_builds.sh -f -d        
+        ./scripts/docker_builds.sh -e -f -d        
 
     - name: Get geopackage 
       run: |

docker/Dockerfile.datastream-deps

@@ -1,12 +1,23 @@
-FROM amazonlinux:2023 AS datastream-base
+FROM ubuntu:22.04 AS datastream-base
 
-RUN dnf -y install pigz tar git python3.9 python3-pip
+# Install basic tools and Python 3.10 (default in Ubuntu 22.04)
+RUN apt-get update && apt-get install -y \
+    pigz tar git python3 python3-pip python3-dev \
+    wget gcc g++ make cmake \
+    zlib1g-dev libssl-dev libffi-dev libsqlite3-dev \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Verify Python version
+RUN python3 --version && pip3 --version
 
 ARG ARCH="x86"
 RUN if [ "${ARCH}" = "aarch64" ]; then \
-        dnf -y install wget gcc-c++ cpp sqlite-devel libtiff cmake python3-devel openssl-devel tcl libtiff-devel libcurl-devel swig libpng-devel libjpeg-turbo-devel expat-devel && \
-        dnf clean all && \
-        pip3 install setuptools numpy; \
+        apt-get update && apt-get install -y \
+        wget g++ cpp libsqlite3-dev libtiff-dev cmake python3-dev \
+        libssl-dev tcl libtiff5-dev libcurl4-openssl-dev swig \
+        libpng-dev libjpeg-turbo8-dev libexpat1-dev \
+        && apt-get clean && rm -rf /var/lib/apt/lists/* \
+        && pip3 install setuptools numpy; \
     fi
 
 # --- PROJ Install ---
@@ -37,7 +48,7 @@ RUN if [ "${ARCH}" = "aarch64" ]; then \
 
 # --- HDF5 Install ---
 RUN if [ "${ARCH}" = "aarch64" ]; then \
-        dnf -y install python3.11-devel python3-Cython && \
+        apt-get update && apt-get install -y python3-dev cython3 && \
         wget https://hdf-wordpress-1.s3.amazonaws.com/wp-content/uploads/manual/HDF5/HDF5_1_14_3/src/hdf5-1.14.3.tar.gz && \
         tar zxvf hdf5-1.14.3.tar.gz && \
         cd hdf5-1.14.3 && \
@@ -52,7 +63,7 @@ RUN if [ "${ARCH}" = "aarch64" ]; then \
 
 # --- GEOS Install ---
 RUN if [ "${ARCH}" = "aarch64" ]; then \
-    dnf -y install wget gcc-c++ make cmake bzip2 && \
+    apt-get update && apt-get install -y wget g++ make cmake bzip2 && \
     wget https://download.osgeo.org/geos/geos-3.12.1.tar.bz2 && \
     tar xvjf geos-3.12.1.tar.bz2 && \
     cd geos-3.12.1 && \

forcingprocessor/setup.cfg

@@ -18,7 +18,7 @@ classifiers =
     Intended Audience :: Education
     Intended Audience :: Science/Research
     License :: Free To Use But Restricted
-    Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3.10
     Topic :: Scientific/Engineering :: Hydrology
     Operating System :: OS Independent
 
@@ -46,7 +46,7 @@ install_requires =
     scipy
     xarray
 
-python_requires = >=3.9
+python_requires = >=3.10
 include_package_data = True
 
 [options.packages.find]