diff --git a/.github/workflows/build_test_create_ami.yml b/.github/workflows/build_test_create_ami.yml index cf529d5b3..84315d997 100644 --- a/.github/workflows/build_test_create_ami.yml +++ b/.github/workflows/build_test_create_ami.yml @@ -2,8 +2,6 @@ name: Build, Test, and Push Datastream AMI on: workflow_dispatch: push: - branches: - - main paths: - 'ami_version.yml' @@ -503,4 +501,4 @@ jobs: echo " OVERALL RESULT: All VPU tests passed successfully" echo " Custom AMI is validated and preserved" echo " Ready for production deployment" - fi \ No newline at end of file + fi diff --git a/.github/workflows/build_test_docker_x86.yaml b/.github/workflows/build_test_docker_x86.yaml index 8b1e5f22e..927d1160c 100644 --- a/.github/workflows/build_test_docker_x86.yaml +++ b/.github/workflows/build_test_docker_x86.yaml @@ -51,11 +51,13 @@ jobs: - name: Build docker containers run : | - # export ARCH=$(uname -m) - # docker compose -f docker/docker-compose.yml build datastream-deps - docker compose -f docker/docker-compose.yml build datastream - docker compose -f docker/docker-compose.yml build forcingprocessor + docker system prune -a + TAG=latest-x86 docker compose -f docker/docker-compose.yml build datastream-deps --no-cache + TAG=latest-x86 docker compose -f docker/docker-compose.yml build forcingprocessor --no-cache + TAG=latest-x86 docker compose -f docker/docker-compose.yml build datastream --no-cache + docker images + - name: Test docker containers run : | curl -L -O https://ngen-datastream.s3.us-east-2.amazonaws.com/palisade.gpkg - ./scripts/datastream -s 202006200100 -e 202006200200 -C NWM_RETRO_V3 -d $(pwd)/data/datastream_test -g $(pwd)/palisade.gpkg -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json \ No newline at end of file + ./scripts/datastream -s 202006200100 -e 202006200200 -C NWM_RETRO_V3 -d $(pwd)/data/datastream_test -g $(pwd)/palisade.gpkg -R $(pwd)/configs/ngen/realization_sloth_nom_cfe_pet.json diff --git a/.github/workflows/build_test_push_docker_x86.yaml b/.github/workflows/build_test_push_docker_x86.yaml index 44e111b1b..ae665a10b 100644 --- a/.github/workflows/build_test_push_docker_x86.yaml +++ b/.github/workflows/build_test_push_docker_x86.yaml @@ -1,24 +1,11 @@ name: Build, Test, and Push Datastream Docker Containers on: - workflow_dispatch: + workflow_dispatch: push: - branches: - - main paths: - - 'docker/**' - - 'forcingprocessor/**' - - 'scripts/**' - - 'python_tools/**' - - '!docs/**' - - '!scripts/README.md' - - '!docker/README.md' - - '!forcingprocessor/README.md' - - '!python_tools/README.md' - -permissions: - contents: read - + - 'versions.yml' + jobs: build-test-push-docker-x86: runs-on: ubuntu-latest @@ -63,4 +50,4 @@ jobs: run: | TAG=latest-x86 docker compose -f docker/docker-compose.yml push datastream-deps TAG=latest-x86 docker compose -f docker/docker-compose.yml push datastream - TAG=latest-x86 docker compose -f docker/docker-compose.yml push forcingprocessor \ No newline at end of file + TAG=latest-x86 docker compose -f docker/docker-compose.yml push forcingprocessor diff --git a/.github/workflows/datastream_python.yaml b/.github/workflows/datastream_python.yaml index 7cb6ba4ef..7ca4cf8a8 100644 --- a/.github/workflows/datastream_python.yaml +++ b/.github/workflows/datastream_python.yaml @@ -24,17 +24,21 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Set up Python 3.9 + - name: Set up Python 3.10 uses: actions/setup-python@v3 with: - python-version: "3.9" + python-version: "3.10" - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -e ./python_tools - pip install pytest + pip install uv + uv venv + source .venv/bin/activate + uv pip install -e ./python_tools + uv pip install pytest - name: Test with pytest run: | + source .venv/bin/activate python -m pytest -vv python_tools/ diff --git a/.github/workflows/forcingprocessor_aws_sources.yaml b/.github/workflows/forcingprocessor_aws_sources.yaml index f1e372889..c4eeaa91c 100644 --- a/.github/workflows/forcingprocessor_aws_sources.yaml +++ b/.github/workflows/forcingprocessor_aws_sources.yaml @@ -24,10 +24,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Set up Python 3.9 + - name: Set up Python 3.10 uses: actions/setup-python@v3 with: - python-version: "3.9" + python-version: "3.10" - name: Configure AWS run: | @@ -38,11 +38,15 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -e ./forcingprocessor - pip install pytest + pip install uv + uv venv + source .venv/bin/activate + uv pip install -e ./forcingprocessor + uv pip install pytest - name: Test with pytest run: | + source .venv/bin/activate cd forcingprocessor python -m pytest -vv -k test_noaa_nwm_pds_https python -m pytest -vv -k test_noaa_nwm_pds_https_short_range diff --git a/.github/workflows/forcingprocessor_gcs_sources.yaml b/.github/workflows/forcingprocessor_gcs_sources.yaml index 1ecc8b680..ee69d29cd 100644 --- a/.github/workflows/forcingprocessor_gcs_sources.yaml +++ b/.github/workflows/forcingprocessor_gcs_sources.yaml @@ -24,10 +24,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Set up Python 3.9 + - name: Set up Python 3.10 uses: actions/setup-python@v3 with: - python-version: "3.9" + python-version: "3.10" - name: Configure AWS run: | @@ -38,11 +38,15 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -e ./forcingprocessor - pip install pytest + pip install uv + uv venv + source .venv/bin/activate + uv pip install -e ./forcingprocessor + uv pip install pytest - name: Test with pytest run: | + source .venv/bin/activate cd forcingprocessor python -m pytest -vv -k test_google_cloud_storage python -m pytest -vv -k test_gs diff --git a/.github/workflows/forcingprocessor_output_opts.yaml b/.github/workflows/forcingprocessor_output_opts.yaml index 7068c914c..a0c1d75a4 100644 --- a/.github/workflows/forcingprocessor_output_opts.yaml +++ b/.github/workflows/forcingprocessor_output_opts.yaml @@ -24,10 +24,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Set up Python 3.9 + - name: Set up Python 3.10 uses: actions/setup-python@v3 with: - python-version: "3.9" + python-version: "3.10" - name: Configure AWS run: | @@ -38,11 +38,15 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -e ./forcingprocessor - pip install pytest + pip install uv + uv venv + source .venv/bin/activate + uv pip install -e ./forcingprocessor + uv pip install pytest - name: Test with pytest run: | + source .venv/bin/activate cd forcingprocessor # netcdf already tested python -m pytest -vv -k test_csv_output_type diff --git a/.github/workflows/forcingprocessor_plotting.yaml b/.github/workflows/forcingprocessor_plotting.yaml index e327cf9b0..be3234c86 100644 --- a/.github/workflows/forcingprocessor_plotting.yaml +++ b/.github/workflows/forcingprocessor_plotting.yaml @@ -24,10 +24,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Set up Python 3.9 + - name: Set up Python 3.10 uses: actions/setup-python@v3 with: - python-version: "3.9" + python-version: "3.10" - name: Configure AWS run: | @@ -38,11 +38,15 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -e ./forcingprocessor - pip install pytest + pip install uv + uv venv + source .venv/bin/activate + uv pip install -e ./forcingprocessor + uv pip install pytest - name: Test with pytest run: | + source .venv/bin/activate cd forcingprocessor python -m pytest -vv tests/test_plotter.py python -m pytest -vv -k test_plotting diff --git a/.github/workflows/forcingprocessor_weights.yaml b/.github/workflows/forcingprocessor_weights.yaml index 17b4cde69..541bad603 100644 --- a/.github/workflows/forcingprocessor_weights.yaml +++ b/.github/workflows/forcingprocessor_weights.yaml @@ -24,10 +24,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Set up Python 3.9 + - name: Set up Python 3.10 uses: actions/setup-python@v3 with: - python-version: "3.9" + python-version: "3.10" - name: Configure AWS run: | @@ -38,10 +38,14 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -e ./forcingprocessor - pip install pytest + pip install uv + uv venv + source .venv/bin/activate + uv pip install -e ./forcingprocessor + uv pip install pytest - name: Test with hf2ds run: | + source .venv/bin/activate cd forcingprocessor python -m pytest -vv tests/test_hf2ds.py diff --git a/.github/workflows/test_hfsubset_v2_1.yaml b/.github/workflows/test_hfsubset_v2_1.yaml index ce97c087e..1a29cc80d 100644 --- a/.github/workflows/test_hfsubset_v2_1.yaml +++ b/.github/workflows/test_hfsubset_v2_1.yaml @@ -37,7 +37,10 @@ jobs: - name: Install packages for datastream run: | - curl -L -O https://github.com/lynker-spatial/hfsubsetCLI/releases/download/v1.1.0/hfsubset-v1.1.0-linux_amd64.tar.gz && tar -xzvf hfsubset-v1.1.0-linux_amd64.tar.gz && sudo mv ./hfsubset /usr/bin/hfsubset + wget -O hfsubset-v1.1.0-linux_amd64.tar.gz https://github.com/lynker-spatial/hfsubsetCLI/releases/download/v1.1.0/hfsubset-v1.1.0-linux_amd64.tar.gz + tar -xzvf hfsubset-v1.1.0-linux_amd64.tar.gz + sudo mv ./hfsubset /usr/bin/hfsubset + sudo chmod +x /usr/bin/hfsubset - name: Get geopackage v2.1 if: always() diff --git a/.github/workflows/test_hfsubset_v2_2.yaml b/.github/workflows/test_hfsubset_v2_2.yaml index fe8ac207f..946e612b6 100644 --- a/.github/workflows/test_hfsubset_v2_2.yaml +++ b/.github/workflows/test_hfsubset_v2_2.yaml @@ -37,7 +37,10 @@ jobs: - name: Install packages for datastream run: | - curl -L -O https://github.com/lynker-spatial/hfsubsetCLI/releases/download/v1.1.0/hfsubset-v1.1.0-linux_amd64.tar.gz && tar -xzvf hfsubset-v1.1.0-linux_amd64.tar.gz && sudo mv ./hfsubset /usr/bin/hfsubset + wget -O hfsubset-v1.1.0-linux_amd64.tar.gz https://github.com/lynker-spatial/hfsubsetCLI/releases/download/v1.1.0/hfsubset-v1.1.0-linux_amd64.tar.gz + tar -xzvf hfsubset-v1.1.0-linux_amd64.tar.gz + sudo mv ./hfsubset /usr/bin/hfsubset + sudo chmod +x /usr/bin/hfsubset - name: Get geopackage v2.2 run: | diff --git a/docker/Dockerfile.datastream-deps b/docker/Dockerfile.datastream-deps index 3dd503ca8..0f4d169fb 100644 --- a/docker/Dockerfile.datastream-deps +++ b/docker/Dockerfile.datastream-deps @@ -1,11 +1,24 @@ -FROM amazonlinux:2023 AS datastream-base +FROM ubuntu:22.04 AS datastream-base -RUN dnf -y install pigz tar git python3.9 python3-pip +# Install basic tools and Python 3.10 (default in Ubuntu 22.04) +RUN apt-get update && apt-get install -y \ + pigz tar git python3 python3-pip python3-dev \ + wget gcc g++ make cmake ninja-build \ + zlib1g-dev libssl-dev libffi-dev libsqlite3-dev \ + libgeos-dev \ + && apt-get clean && rm -rf /var/lib/apt/lists/* + +# Verify Python version +RUN python3 --version && pip3 --version ARG ARCH="x86" RUN if [ "${ARCH}" = "aarch64" ]; then \ - dnf -y install wget gcc-c++ cpp sqlite-devel libtiff cmake python3-devel openssl-devel tcl libtiff-devel libcurl-devel swig libpng-devel libjpeg-turbo-devel expat-devel && \ - pip3 install setuptools numpy; \ + apt-get update && apt-get install -y \ + wget g++ cpp libsqlite3-dev libtiff-dev cmake python3-dev \ + libssl-dev tcl libtiff5-dev libcurl4-openssl-dev swig \ + libpng-dev libjpeg-turbo8-dev libexpat1-dev \ + && apt-get clean && rm -rf /var/lib/apt/lists/* \ + && pip3 install setuptools numpy; \ fi # --- PROJ Install --- @@ -36,7 +49,7 @@ RUN if [ "${ARCH}" = "aarch64" ]; then \ # --- HDF5 Install --- RUN if [ "${ARCH}" = "aarch64" ]; then \ - dnf -y install python3.11-devel python3-Cython && \ + apt-get update && apt-get install -y python3-dev cython3 && \ wget https://hdf-wordpress-1.s3.amazonaws.com/wp-content/uploads/manual/HDF5/HDF5_1_14_3/src/hdf5-1.14.3.tar.gz && \ tar zxvf hdf5-1.14.3.tar.gz && \ cd hdf5-1.14.3 && \ @@ -51,7 +64,7 @@ RUN if [ "${ARCH}" = "aarch64" ]; then \ # --- GEOS Install --- RUN if [ "${ARCH}" = "aarch64" ]; then \ - dnf -y install wget gcc-c++ make cmake bzip2 && \ + apt-get update && apt-get install -y wget g++ make cmake bzip2 && \ wget https://download.osgeo.org/geos/geos-3.12.1.tar.bz2 && \ tar xvjf geos-3.12.1.tar.bz2 && \ cd geos-3.12.1 && \ diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 547edbbed..091f7794f 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -24,4 +24,6 @@ services: build: context: .. dockerfile: docker/Dockerfile.forcingprocessor + args: + TAG_NAME: ${TAG:-latest} image: awiciroh/forcingprocessor:${TAG:-latest} diff --git a/forcingprocessor/setup.cfg b/forcingprocessor/setup.cfg index 4947f9be3..6d8dfcd2b 100644 --- a/forcingprocessor/setup.cfg +++ b/forcingprocessor/setup.cfg @@ -18,7 +18,7 @@ classifiers = Intended Audience :: Education Intended Audience :: Science/Research License :: Free To Use But Restricted - Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 Topic :: Scientific/Engineering :: Hydrology Operating System :: OS Independent @@ -46,7 +46,7 @@ install_requires = scipy xarray -python_requires = >=3.9 +python_requires = >=3.10 include_package_data = True [options.packages.find] diff --git a/python_tools/setup.cfg b/python_tools/setup.cfg index 6ba282c9b..8be8df604 100644 --- a/python_tools/setup.cfg +++ b/python_tools/setup.cfg @@ -18,7 +18,7 @@ classifiers = Intended Audience :: Education Intended Audience :: Science/Research License :: Free To Use But Restricted - Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 Topic :: Scientific/Engineering :: Hydrology Operating System :: OS Independent @@ -38,7 +38,7 @@ install_requires = ngen_init_config[all] @ git+https://github.com/ciroh-ua/ngen-cal@master#egg=ngen_init_config&subdirectory=python/ngen_init_config ngen_config_gen[all] @ git+https://github.com/ciroh-ua/ngen-cal@master#egg=ngen_config_gen&subdirectory=python/ngen_config_gen -python_requires = >=3.9 +python_requires = >=3.10 include_package_data = True [options.packages.find] diff --git a/versions.yml b/versions.yml index e82eebc9c..49fbfaa34 100644 --- a/versions.yml +++ b/versions.yml @@ -1,3 +1,3 @@ datastream-deps: "2.2.3" -forcingprocessor: "1.0.1" +forcingprocessor: "test_python_version" datastream: "2.2.3"