diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml new file mode 100644 index 0000000..0409842 --- /dev/null +++ b/.github/workflows/ci-cd.yml @@ -0,0 +1,218 @@ +name: CI/CD Pipeline + +on: + pull_request: + branches: [main, nwm-main, development, release-candidate] + push: + branches: [main, nwm-main, development, release-candidate] + release: + types: [published] + +permissions: + contents: read + packages: write + security-events: write + +env: + REGISTRY: ghcr.io + PYTHON_VERSION: '3.11' + +jobs: + setup: + runs-on: ubuntu-latest + outputs: + image_base: ${{ steps.vars.outputs.image_base }} + pr_tag: ${{ steps.vars.outputs.pr_tag }} + commit_sha: ${{ steps.vars.outputs.commit_sha }} + commit_sha_short: ${{ steps.vars.outputs.commit_sha_short }} + test_image_tag: ${{ steps.vars.outputs.test_image_tag }} + steps: + - name: Compute image vars + id: vars + shell: bash + run: | + set -euo pipefail + ORG="$(echo "${GITHUB_REPOSITORY_OWNER}" | tr '[:upper:]' '[:lower:]')" + REPO="$(basename "${GITHUB_REPOSITORY}")" + IMAGE_BASE="${REGISTRY}/${ORG}/${REPO}" + echo "image_base=${IMAGE_BASE}" >> "$GITHUB_OUTPUT" + + if [ "${GITHUB_EVENT_NAME}" = "pull_request" ]; then + PR_NUM="${{ github.event.pull_request.number }}" + PR_TAG="pr-${PR_NUM}-build" + echo "pr_tag=${PR_TAG}" >> "$GITHUB_OUTPUT" + echo "test_image_tag=${PR_TAG}" >> "$GITHUB_OUTPUT" + fi + + if [ "${GITHUB_EVENT_NAME}" = "push" ]; then + COMMIT_SHA="${GITHUB_SHA}" + SHORT_SHA="${COMMIT_SHA:0:12}" + echo "commit_sha=${COMMIT_SHA}" >> "$GITHUB_OUTPUT" + echo "commit_sha_short=${SHORT_SHA}" >> "$GITHUB_OUTPUT" + echo "test_image_tag=${SHORT_SHA}" >> "$GITHUB_OUTPUT" + fi + + build: + name: build + if: github.event_name == 'pull_request' || github.event_name == 'push' + runs-on: ubuntu-latest + needs: setup + steps: + - uses: actions/checkout@v4 + + - name: Log in to registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build & push image + uses: docker/build-push-action@v6 + with: + context: . + push: true + tags: ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }} + build-args: | + NGEN_IMAGE_TAG=${{ env.NGEN_IMAGE_TAG || 'latest' }} + CI_COMMIT_REF_NAME=${{ github.ref_name }} + + unit-test: + name: unit-test + if: github.event_name == 'pull_request' || github.event_name == 'push' + runs-on: ubuntu-latest + needs: [setup, build] + container: + image: ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }} + steps: + - name: Run unit tests + run: | + echo "TODO: add unit tests here" + + codeql-scan: + if: github.event_name == 'pull_request' || github.event_name == 'push' + runs-on: ubuntu-latest + needs: [setup, build] + permissions: + actions: read + contents: read + security-events: write + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: python + - name: Install dependencies + run: | + python -m pip install --upgrade pip + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + + container-scanning: + if: github.event_name == 'pull_request' || github.event_name == 'push' + runs-on: ubuntu-latest + needs: [setup, build] + steps: + - name: Scan container with Trivy + uses: aquasecurity/trivy-action@0.20.0 + with: + image-ref: ${{ needs.setup.outputs.image_base }}:${{ needs.setup.outputs.test_image_tag }} + format: 'template' + template: '@/contrib/sarif.tpl' + output: 'trivy-results.sarif' + severity: 'CRITICAL,HIGH' + - name: Upload Trivy SARIF + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: 'trivy-results.sarif' + + deploy-latest-on-development: + name: deploy-latest-on-development + if: github.event_name == 'push' && github.ref_name == 'development' + runs-on: ubuntu-latest + needs: [setup, build, unit-test, codeql-scan, container-scanning] + steps: + - name: Tag image with 'latest' + shell: bash + run: | + set -euo pipefail + IMAGE_BASE="${{ needs.setup.outputs.image_base }}" + SHORT_SHA="${{ needs.setup.outputs.commit_sha_short }}" + + # ensure skopeo is available + if ! command -v skopeo >/dev/null 2>&1; then + sudo apt-get update -y + sudo apt-get install -y --no-install-recommends skopeo + fi + + skopeo copy \ + --src-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ + --dest-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ + docker://"${IMAGE_BASE}:${SHORT_SHA}" docker://"${IMAGE_BASE}:latest" + + release: + name: release + if: github.event_name == 'release' && github.event.action == 'published' + runs-on: ubuntu-latest + needs: setup + steps: + - name: Get commit sha for the tag + id: rev + shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + TAG="${{ github.event.release.tag_name }}" + REPO="${{ github.repository }}" + + # ensure jq is available + if ! command -v jq >/dev/null 2>&1; then + sudo apt-get update -y + sudo apt-get install -y --no-install-recommends jq + fi + + # ensure gh cli is available + if ! command -v gh >/dev/null 2>&1; then + sudo apt-get update -y + sudo apt-get install -y --no-install-recommends gh + fi + + REF_JSON="$(gh api "repos/${REPO}/git/refs/tags/${TAG}")" + OBJ_SHA="$(jq -r '.object.sha' <<<"$REF_JSON")" + OBJ_TYPE="$(jq -r '.object.type' <<<"$REF_JSON")" + + if [ "$OBJ_TYPE" = "tag" ]; then + TAG_OBJ="$(gh api "repos/${REPO}/git/tags/${OBJ_SHA}")" + COMMIT_SHA="$(jq -r '.object.sha' <<<"$TAG_OBJ")" + else + COMMIT_SHA="$OBJ_SHA" + fi + + SHORT_SHA="${COMMIT_SHA:0:12}" + echo "short_sha=${SHORT_SHA}" >> "$GITHUB_OUTPUT" + + - name: Tag image with release tag + shell: bash + run: | + set -euo pipefail + IMAGE_BASE="${{ needs.setup.outputs.image_base }}" + SHORT_SHA="${{ steps.rev.outputs.short_sha }}" + RELEASE_TAG="${{ github.event.release.tag_name }}" + + # ensure skopeo is available + if ! command -v skopeo >/dev/null 2>&1; then + sudo apt-get update -y + sudo apt-get install -y --no-install-recommends skopeo + fi + + skopeo copy \ + --src-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ + --dest-creds "${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}" \ + docker://"${IMAGE_BASE}:${SHORT_SHA}" docker://"${IMAGE_BASE}:${RELEASE_TAG}" diff --git a/.gitignore b/.gitignore index 2d5316f..4ebe459 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,94 @@ +# Compiled source # +################### +*.com +*.class +*.dll +*.exe +*.o +*.so +_site/ + +# Packages # +############ +# it's better to unpack these files and commit the raw source +# git has its own built in compression methods +*.7z +*.dmg +*.gz +*.iso +*.jar +*.rar +*.tar +*.zip + +# Logs and databases # +###################### +*.log +*.sql +*.sqlite + +# OS generated files # +###################### +.DS_Store +.DS_Store? +.Spotlight-V100 +.Trashes +Icon? +ehthumbs.db +Thumbs.db + +# Vim swap files # +################## +*.swp + +# Python # +################# +*.pyc +*.egg-info/ +__pycache__/ +*.py[cod] +.env +.python-version +venv +*.pytest_cache +build + +# pyenv # +######### +.python-version + +# Django # +################# +*.egg-info +.installed.cfg + +# Unit test / coverage reports +################# +htmlcov/ +.tox/ +.coverage +.cache +nosetests.xml +coverage.xml + +# Front-End # +############# +node_modules/ +bower_components/ +.grunt/ +src/vendor/ +dist/ + +# Temporary Directories # +*tmp/ +*temp/ + +# Other .idea /git_info.json + +# Ngen FIles # +############# +cat* +nex* +troute* diff --git a/Dockerfile b/Dockerfile index 21296c8..b224149 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,27 +1,39 @@ -ARG NGEN_VERSION=latest -FROM registry.sh.nextgenwaterprediction.com/ngwpc/nwm-ngen/ngen:${NGEN_VERSION} +# syntax=docker/dockerfile:1.4 +ARG NGEN_IMAGE_TAG=latest +FROM ghcr.io/ngwpc/ngen:${NGEN_IMAGE_TAG} -RUN set -eux; \ - dnf install -y \ - jq; \ - dnf clean all +# Uncomment when building ngen locally or if ngen-int image is available locally +# modify to use image tag for local ngen image if needed +#FROM ngen -COPY requirements.txt . -RUN set -eux; \ - \ - pip3 install -r requirements.txt ; \ - pip3 cache purge ; \ - rm --force requirements.txt ; +# Activate the existing virtual environment +ENV PATH="/ngen-app/ngen-python/bin:${PATH}" +RUN set -eux; \ + dnf install -y jq; \ + dnf clean all COPY . /ngen-app/ngen-fcst/ COPY ./docker/run-ngen-fcst.sh /ngen-app/bin/ + RUN set -eux; \ - \ chmod +x /ngen-app/bin/run-ngen-fcst.sh WORKDIR /ngen-app/ngen-fcst +# Install missing dependencies that aren't in base image +RUN --mount=type=cache,target=/root/.cache/pip,id=pip-cache \ + set -eux; \ + pip3 install \ + "matplotlib~=3.10.6"; \ + #"geopandas~=1.1.1"; \ + pip3 cache purge + +# Install into the existing virtual environment without upgrading base packages +RUN set -eux; \ + pip3 install --no-deps . || pip3 install .; \ + pip3 cache purge; + ARG CI_COMMIT_REF_NAME RUN set -eux; \ @@ -47,4 +59,4 @@ RUN set -eux; \ WORKDIR / -ENTRYPOINT [ "/ngen-app/bin/run-ngen-fcst.sh" ] +ENTRYPOINT [ "/ngen-app/bin/run-ngen-fcst.sh" ] diff --git a/README.md b/README.md index e2957f1..8e5f37b 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,13 @@ -# ngen-fcst - - - -## Name -ngen Forecast +# nwm-fcst-mgr ## Description -A program to run ngen given the forecast forcing provided via a .nc file and a configuration file from validation with ngen-cal +A program to execute ngen cold start and forecast runs provided a configuration file from validation with nwm-cal-mgr and realization files for the cold start and forecast periods ## Installation -### Clone ngen-fcst +### Clone nwm-fcst-mgr -git clone -b development --recurse-submodules https://gitlab.sh.nextgenwaterprediction.com/NGWPC/nwm-ngen/ngen-fcst.git +git clone -b development --recurse-submodules https://github.com/NGWPC/nwm-fcst-mgr.git ### Build the environment @@ -38,62 +33,48 @@ where [VENV_ROOT] and [NGEN_ROOT] refer to the directory to install the python v Follow the following steps to test the program: 1. source [VENV_ROOT]/env.ngen/bin/activate -2. cd [NGEN-FCST_ROOT]/ngen-fcst -3. python python/run_ngen_fcst.py test_data/forcing.nc test_data/valid_config.yaml fcst_run1 +2. cd [NWM-FCST-MGR_ROOT]/nwm-fcst-mgr +3. pip install . + +where [NWM-FCST-MGR_ROOT] is where nwm-fcst-mgr is installed. + +The program takes three arguments: +1) Path to the config yaml file for a validation run (from nwm-cal-mgr) +2) Path to a realization file for the forecast run (generated by nwm-msw-mgr) +3) Optional Path to a realization file for the cold start run in provided by user (generated by nwm-msw-mgr) -where [NGEN-FCST_ROOT] is where ngen-fcst is installed +Nwm-fcst-mgr can be run from the CLI or from Python code directly. -The program takes three command line arguments: -1) Path to the NetCDF forcing file -2) Path to the config yaml file for a validation run (from ngen-cal) -3) Path to the folder to be created for storing inputs/outputs from running ngen, relative to the Output directory of the calibration run as indicated in the config yaml file. For example, if "fcst_run1" is the 3rd argument, and "yaml_file" in the "general" section of the config file is '/home/yuqiong.liu/work/Gitlab/run/kge_DDS/noah_cfes/01123000/Output/Validation_Run/01123000_config_valid_best.yaml', then the new output directory to be created for the ngen-fcst run would be: +### Python +1. from nwm_fcst_mgr.forecast import run_fcst +2. valid_yaml = '~/ngwpc/run_ngen/kge_dds/noah_cfes/01123000/Output/Validation_Run/01123000_config_valid_best.yaml' +3. real_path = '~/ngwpc/run_ngen/kge_dds/noah_cfes/01123000/Output/Forecast_Run/fcst_run1/01123000_realization_config_bmi_fcst.json' +4. run_fcst(valid_yaml=valid_yaml, real_path=real_path) + +If the user wishes to run a cold_start_period, then real_path would be replaced by the path to the cold start realization file. + +### CLI +python -m nwm_fcst_mgr.forecast valid_yaml real_path + +where the arguments are replaced by the paths above. -/home/yuqiong.liu/work/Gitlab/run/kge_DDS/noah_cfes/01123000/Output/Forecast_Run/fcst_run1 ## Docker container ### Requirements -To build and run ngen-fcst, you will need the following software installed and running on your system: +To build and run nwm-fcst-mgr, you will need the following software installed and running on your system: - Docker Engine -You will also need the following data: -- a forcing file in NetCDF format -- a YAML-formatted configuration file - ### Build -To build the ngen-fcst container, execute the following command: +To build the nwm-fcst-mgr container, execute the following command: ``` -docker build --tag=ngen-fcst . +docker build --tag=nwm-fcst-mgr . ``` ### Running -To run the ngen-fcst applicaton, execute the following command: -``` -docker run ngen-fcst -``` - -This will print a usage statement for the container: -``` -Usage: run-ngen-fcst.sh [log_file] [venv_path] - -FORCING_FILE: Path to the NetCDF forcing file. -CONFIG_FILE: Path to the config yaml file for a validation run (from ngen-cal). -OUTPUT_PATH: Path to the folder to be created for storing inputs/outputs from running ngen. -LOG_FILE (optional): Path to the output file where the script's output will be saved. Used when running in LOCAL or DOCKER environment -VENV_PATH (optional): Path to the Python virtual environment. Used when running in the LOCAL environment. - -Examples: - run-ngen-fcst.sh test_data/forcing.nc test_data/valid_config.yaml fcst_run1 - run-ngen-fcst.sh test_data/forcing.nc test_data/valid_config.yaml fcst_run1 /path/to/output /path/to/venv -``` - -The path provided for any files should match the path within the container, as well as the paths insider your configuration file. So if `forcing.nc` is located at `~/ngencerf/data/ngen-cal-data/forcing/forcing.nc` and `valid_config.yaml` is located at `~/ngencerf/data/ngen-cal-data/configs/valid_config.yaml`, you should run the command: -``` -docker run -v ~/ngencerf/data/ngen-cal-data/forcing/:/ngencerf/data/forcing/ -v ~/ngencerf/data/ngen-cal-data/configs/:/ngencerf/data/configs/ ngen-fcst /ngencerf/data/forcing/forcing.nc /ngencerf/data/configs/valid_config.yaml fcst_run1 -``` ## Contributing State if you are open to contributions and what your requirements are for accepting them. diff --git a/docker/run-ngen-fcst.sh b/docker/run-ngen-fcst.sh index 73ff0d4..755a2ad 100755 --- a/docker/run-ngen-fcst.sh +++ b/docker/run-ngen-fcst.sh @@ -1,34 +1,37 @@ #!/bin/bash + # Define valid commands -VALID_COMMANDS=("forecast") +VALID_COMMANDS=("cold_start" "forecast") -# This shell script lives in the ngen-fcst repo. It is used by CerfServer when calling ngen-fcst +# This shell script lives in the nwm-fcst-mgr repo. It is used by CerfServer when calling nwm-fcst-mgr # # It is used by CerfServer directly when running in LOCAL mode. -# It is used by the ngen-fcst docker container when the server is running in DOCKER or PARALLEL_WORKS mode +# It is used by the nwm-fcst-mgr docker container when the server is running in DOCKER or PARALLEL_WORKS mode -FORECAST_SCRIPT=/ngen-app/ngen-fcst/python/run_ngen_fcst.py +FORECAST_SCRIPT=nwm_fcst_mgr.forecast +COLD_START_SCRIPT=nwm_fcst_mgr.forecast # Set the umask so files and directories are created with 777 permissions umask 000 # Function to display help message show_help() { - echo "Usage: $(basename "$0") [stdout_file] [venv_path]" - echo "" + echo "Usage: $(basename "$0") [stdout_file] [venv_path]" echo "" echo "COMMAND:" - echo " forecast Run forecast script." + echo " forecast Run forecast script (requires validation_yaml + forecast_realization)." + echo " cold_start Run cold start script (requires validation_yaml + cold_start_realization)." echo "" - echo "FORCING_FILE: Path to the NetCDF forcing file." - echo "CONFIG_FILE: Path to the config yaml file for a validation run (from ngen-cal)." - echo "FORECAST_DIR: Name of the folder to to store the forecast output." + echo "VALIDATION_YAML: Path to the config yaml file for a validation run (from MSWM)." + echo "FORECAST_REALIZATION: (Required for forecast) Path to the forecast realization file." + echo "COLD_START_REALIZATION: (Required for cold_start) Path to the cold start realization file." echo "STDOUT_FILE (optional): Path to the stdout file where the script's console output will be saved. Used when running in LOCAL or DOCKER environment" echo "VENV_PATH (optional): Path to the Python virtual environment. Used when running in the LOCAL environment." echo "" echo "Examples:" - echo " $(basename "$0") forecast test_data/forcing.nc test_data/valid_config.yaml fcst_run1" - echo " $(basename "$0") forecast test_data/forcing.nc test_data/valid_config.yaml fcst_run1 /path/to/output/ngen-fcst.log /path/to/venv" + echo " $(basename "$0") forecast validation.yaml realization.yaml" + echo " $(basename "$0") cold_start validation.yaml cold_start_realization.yaml" + echo " $(basename "$0") forecast validation.yaml realization.yaml cold_start_realization.yaml" echo "" exit 1 } @@ -40,7 +43,7 @@ fi # Check if the command for the script is provided as the first argument if [ -z "$1" ]; then - echo "Error: No script command provided. Allowable commands are: ${VALID_COMMANDS[*]}." + echo "[run-ngen-fcst.sh] Error: No script command provided. Allowable commands are: ${VALID_COMMANDS[*]}." show_help fi @@ -51,48 +54,51 @@ shift 1 case "$SCRIPT_COMMAND" in "forecast") SCRIPT_PATH=$FORECAST_SCRIPT - REQUIRED_ARGS=3 + REQUIRED_ARGS=2 + ;; + "cold_start") + SCRIPT_PATH=$COLD_START_SCRIPT + REQUIRED_ARGS=2 ;; *) - echo "Error: Invalid script command: '$SCRIPT_COMMAND'. Allowable commands are: ${VALID_COMMANDS[*]}." + echo "[run-ngen-fcst.sh] Error: Invalid script command: '$SCRIPT_COMMAND'. Allowable commands are: ${VALID_COMMANDS[*]}." show_help ;; esac -# Check if the selected script exists -if [ ! -f "$SCRIPT_PATH" ]; then - echo "Error: Script not found at $SCRIPT_PATH" - exit 1 -fi - # Check if the correct number of arguments are provided for the selected command if [ $# -lt $REQUIRED_ARGS ]; then - echo "Error: Insufficient arguments. $SCRIPT_COMMAND requires $REQUIRED_ARGS arguments." + echo "[run-ngen-fcst.sh] Error: Insufficient arguments. $SCRIPT_COMMAND requires $REQUIRED_ARGS arguments." show_help fi -FORCING_FILE=$1 -CONFIG_FILE=$2 -FORECAST_DIR=$3 -shift $REQUIRED_ARGS +VALIDATION_YAML=$1 +REALIZATION_FILE=$2 +shift 2 -echo "FORCING_FILE: ${FORCING_FILE}" -echo "CONFIG_FILE: ${CONFIG_FILE}" -echo "FORECAST_DIR: ${FORECAST_DIR}" +echo "[run-ngen-fcst.sh] VALIDATION_YAML: ${VALIDATION_YAML}" +echo "[run-ngen-fcst.sh] REALIZATION_FILE: ${REALIZATION_FILE}" -# Check if the forcing data exists -if [ ! -f "${FORCING_FILE}" ]; then - echo "Forcing data not found at ${FORCING_FILE}" +# File existence checks (fatal if missing) +if [[ ! -f "${VALIDATION_YAML}" && ! -d "${VALIDATION_YAML}" ]]; then + echo "[run-ngen-fcst.sh] Fatal: Config file not found at ${VALIDATION_YAML}" + exit 1 fi -# Check if the configuration file exists -if [ ! -f "${CONFIG_FILE}" ]; then - echo "Configuration file not found at ${CONFIG_FILE}" +if [ ! -f "${REALIZATION_FILE}" ]; then + if [ "$SCRIPT_COMMAND" == "forecast" ]; then + echo "[run-ngen-fcst.sh] Fatal: Forecast realization file not found at ${REALIZATION_FILE}" + else + echo "[run-ngen-fcst.sh] Fatal: Cold start realization file not found at ${REALIZATION_FILE}" + fi + exit 1 fi +# Handle optional stdout file +STDOUT_FILE="" if [ $# -ge 1 ]; then STDOUT_FILE=$1 - echo "Output file: $STDOUT_FILE" + echo "[run-ngen-fcst.sh] Output file: $STDOUT_FILE" # Create output directory if it doesn't exist STDOUT_DIR=$(dirname "$STDOUT_FILE") @@ -103,9 +109,11 @@ if [ $# -ge 1 ]; then shift 1 fi +# Handle optional virtual environment +VENV_PATH="" if [ $# -ge 1 ]; then VENV_PATH=$1 - echo "Virtual environment: $VENV_PATH" + echo "[run-ngen-fcst.sh] Virtual environment: $VENV_PATH" shift 1 fi @@ -114,34 +122,35 @@ if [ -n "$VENV_PATH" ]; then if [ -d "$VENV_PATH/bin" ]; then source "$VENV_PATH/bin/activate" else - echo "Error: Virtual environment path '$VENV_PATH' is invalid." + echo "[run-ngen-fcst.sh] Fatal: Virtual environment path '$VENV_PATH' is invalid." exit 1 fi else - echo "No virtual environment provided, running with default Python environment." + echo "[run-ngen-fcst.sh] No virtual environment provided, running with default Python environment." fi # Run the Python script, redirecting its output if an output file is provided -echo " Running $(basename "$SCRIPT_PATH") with input file: $CONFIG_FILE" +echo "[run-ngen-fcst.sh] Running $SCRIPT_PATH ($SCRIPT_COMMAND) with inputs: ${VALIDATION_YAML} ${REALIZATION_FILE}" + if [ -z "$STDOUT_FILE" ]; then - python "${SCRIPT_PATH}" "${FORCING_FILE}" "${CONFIG_FILE}" "${FORECAST_DIR}" + python -m $SCRIPT_PATH "$VALIDATION_YAML" "$REALIZATION_FILE" else - python "${SCRIPT_PATH}" "${FORCING_FILE}" "${CONFIG_FILE}" "${FORECAST_DIR}" &> "${STDOUT_FILE}" 2>&1 + python -m $SCRIPT_PATH "$VALIDATION_YAML" "$REALIZATION_FILE" &> "$STDOUT_FILE" 2>&1 fi python_exit_code=$? if [ $python_exit_code -ne 0 ]; then - echo "$(basename "$SCRIPT_PATH") exited with code $python_exit_code" + echo "[run-ngen-fcst.sh] $SCRIPT_PATH exited with code $python_exit_code" fi # Display output if redirected to a file if [ -n "$STDOUT_FILE" ]; then - echo "Output from running $(basename "$SCRIPT_PATH")" + echo "Output from running $SCRIPT_PATH" echo "-------------- start of $STDOUT_FILE -----------------------------" cat "$STDOUT_FILE" echo "---------------- end of $STDOUT_FILE -----------------------------" fi -echo "Done running $(basename "$SCRIPT_PATH")" +echo "[run-ngen-fcst.sh] Done running $SCRIPT_PATH" exit $python_exit_code diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..672e0cf --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,21 @@ +[build-system] +requires = ["setuptools>=65", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "nwm_fcst_mgr" +version = "0.1.1" # or mark as dynamic if you use setuptools-scm +requires-python = ">=3.10" +dependencies = [ + "geopandas~=1.1.1", + "pandas~=2.3.1", + "netCDF4==1.6.3", + "matplotlib~=3.10.6", + "PyYAML~=6.0.2", +] + +[tool.setuptools] +package-dir = {"" = "python"} + +[tool.setuptools.packages.find] +where = ["python"] diff --git a/python/nwm_fcst_mgr/__init__.py b/python/nwm_fcst_mgr/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/python/nwm_fcst_mgr/forecast.py b/python/nwm_fcst_mgr/forecast.py new file mode 100644 index 0000000..30fdd32 --- /dev/null +++ b/python/nwm_fcst_mgr/forecast.py @@ -0,0 +1,228 @@ +import glob +import json +import logging +import os +import shutil +import subprocess +from pathlib import Path +import geopandas as gpd +import pandas as pd +import netCDF4 + +import matplotlib.pyplot as plt +import yaml +import argparse + +from nwm_fcst_mgr.log_level import log_level_set +from nwm_fcst_mgr.git_util import print_git_info_all + +# setup the logger +log_level_set() +logger = logging.getLogger(__name__) + + +def run_fcst(valid_yaml: str, real_path: str): + """ + Execute ngen run for forecast period and cold start period (if provided) + valid_yaml: path to validation yaml file from past calibration run + real_path: path to realization file for a cold start or forecast period + """ + + # set environment variable for ngencerf backend + os.environ["NGEN_RESULTS_DIR"] = str(Path(real_path).parent) + logging.info( + f"Set environment variable NGEN_RESULTS_DIR to: {os.environ['NGEN_RESULTS_DIR']}" + ) + + # Read validation yaml file + valid_config = load_yaml(valid_yaml) + + logger.info(f'Validation file loaded from: {valid_yaml}') + + # Retrieve output_dir + real_file = Path(real_path) + out_dir = real_file.parent + + # Retrieve hydrofabric gpkg + gpkg_cats = valid_config['model']['catchments'] + gpkg_nexus = valid_config['model']['nexus'] + + # Retrieve ngen executable + ngen_exe = valid_config['model']['binary'] + + # get gage ID and make sure it is not empty + try: + gage0 = valid_config['model']['eval_params']['basinID'] + except ValueError as e: + logger.critical(f'Key model/eval_params/basinID not found in {valid_yaml}\n{e}') + raise + if gage0 == "": + try: + raise ValueError(f'basinID in {valid_yaml} cannot be empty') + except ValueError as e: + logger.critical(e) + raise + + # Execute ngen run for either cold-start or forecast period + cmd = f'{ngen_exe} {gpkg_cats} "all" {gpkg_nexus} "all" {real_path}' + + logger.info(f'Initializing NGEN run from: {real_path}') + + # kick off ngen run and save stdout & stderr to ngen_stdout_stderr.log + log_file = out_dir / "ngen_stdout_stderr.log" + try: + with open(log_file, 'a+') as log: + subprocess.check_call(cmd, stdout=log, stderr=log, shell=True, cwd=str(out_dir)) + except subprocess.CalledProcessError as e: + logger.critical(f'Ngen run failed with return code {e.returncode}. Command: {e.cmd}') + raise + except Exception as e: + logger.critical(f'Ngen run failed while running command: {e}') + raise + + logger.info('NGEN run completed successfully') + + # move output files to output directory + run_output_dir = out_dir / "output/" + run_output_dir.mkdir(parents=True, exist_ok=True) + for pat1 in ['cat*.csv', 'nex*.csv', 'troute*.nc']: + for f1 in glob.glob(f'{out_dir}/{pat1}'): + shutil.move(f1, Path(run_output_dir, os.path.basename(f1))) + + logger.info(f'NGEN outputs moved to: {run_output_dir}') + + # read troute output file + outfile = glob.glob(f'{run_output_dir}/troute*.nc')[0] + output = read_troute_output(gage0, valid_config['model']['crosswalk'], gpkg_cats, outfile) + + logger.info(f'Reading T-route output file: {outfile}') + + # plot the hydrograph + plot_path = Path(run_output_dir, gage0 + '_hydrograph.png') + output.plot(y='sim_flow', kind='line') + plt.xlabel('Time') + plt.ylabel('Streamflow (m^3/s)') + plt.savefig(plot_path, bbox_inches="tight") + + logger.info(f'Hydrograph plot saved to: {plot_path}') + + # save streamflow simulation to csv + output.to_csv(Path(run_output_dir, gage0 + '_output.csv')) + + logger.info(f'Fcst-mgr NGEN run outputs saved at: {run_output_dir}') + + +def load_yaml(file_path: str) -> dict: + """ + Read yaml-based configuration file from previous ngen calibration run + """ + # Confirm input file exists + file_path = Path(file_path).absolute() + if not file_path.exists(): + try: + raise FileNotFoundError(f'Input file not found: {file_path}') + except FileNotFoundError as e: + logger.critical(e) + raise + + # Read the yaml-based configuration file + try: + with open(file_path) as file: + yaml_dict = yaml.safe_load(file) + except FileNotFoundError as e: + logger.critical(f'Config valid yaml file does not exist: {file_path}\n{e}') + raise + except yaml.YAMLError as e: + logger.critical(f"YAML parsing error in valid config yaml file: {file_path}\n{e}") + raise + except Exception as e: + logger.critical(f"Unexpected error loading valid config yaml file at: {file_path}\n{e}") + raise + + return yaml_dict + + +def read_troute_output( + gage0: str, + cwt_file: Path, + gpkg_file: Path, + out_file: Path, +) -> pd.DataFrame: + + """ + Arguments: + --------- + gage0: gage ID to retrieve streamflow simulations + cwt_file: path to crosswalk file mapping gage to catchments + gpkg_file: path to geopackage file + out_file: path to t-route output file (in NetCDF format) + + Returns: + --------- + dataframe containing time and streamflow simulations + + """ + # Handle crosswalk file (in order to get the correct feature_id when reading t-route data) + x_walk = pd.Series(dtype=object) + try: + with open(cwt_file) as fp: + data = json.load(fp) + for id, values in data.items(): + gage = values.get('Gage_no') + if gage: + if not isinstance(gage, str): + gage = gage[0] + if gage == gage0: + x_walk[id] = gage + break + except FileNotFoundError as e: + logger.critical(f'Crosswalk file not found: {cwt_file}\n{e}') + raise + except json.JSONDecodeError as e: + logger.critical(f'Failed to parse JSON from crosswalk file: {cwt_file}\n{e}') + raise + + if x_walk.empty: + try: + raise Exception(f'{gage0} is not found in crosswalk file {cwt_file}') + except Exception as e: + logger.critical(e) + raise + + # get catchment at basin outlet for reading from t-route output + catchment_hydro_fabric = gpd.read_file(gpkg_file, layer='divides') + catchment_hydro_fabric.set_index('id', inplace=True) + nexus_id = catchment_hydro_fabric.loc[x_walk.index[0].replace('cat', 'wb')]['toid'] + wb_lst = [x.split('-')[1] for x in catchment_hydro_fabric.index[catchment_hydro_fabric['toid'] == nexus_id]] + + # read troute output + ncvar = netCDF4.Dataset(out_file, "r") + fid_index = [list(ncvar['feature_id'][0:]).index(int(fid)) for fid in wb_lst] + output = pd.DataFrame(data={'sim_flow': pd.DataFrame(ncvar['flow'][fid_index], index=fid_index).T.sum(axis=1)}) + t0 = pd.to_datetime(ncvar.file_reference_time, format="%Y-%m-%d_%H:%M:%S") + output.index = [t0 + pd.Timedelta(seconds=int(t1)) for t1 in ncvar['time']] + output.index.name = 'Time' + + return output + + +def parse_args(): + # Create command line parser + parser = argparse.ArgumentParser() + + # Add arguments + parser.add_argument('valid_yaml', type=str, help=('Path to validation yaml file from previous run of nwm-cal-mgr')) + parser.add_argument('real_path', type=str, help=('Path to cold start or forecast period realization file')) + + return parser.parse_args() + + +def main(): + args = parse_args() + + run_fcst(args.valid_yaml, args.real_path) + + +if __name__ == "__main__": + print_git_info_all() + main() diff --git a/python/git_util.py b/python/nwm_fcst_mgr/git_util.py similarity index 80% rename from python/git_util.py rename to python/nwm_fcst_mgr/git_util.py index 5e0a65d..f41d6d0 100644 --- a/python/git_util.py +++ b/python/nwm_fcst_mgr/git_util.py @@ -1,4 +1,7 @@ import json +import logging + +logger = logging.getLogger(__name__) def transform_component(component_git_info): @@ -50,17 +53,17 @@ def recursive_print(d: dict, indent: int = 0) -> None: """ for key, value in d.items(): if isinstance(value, dict): - print(" " * indent + f"{key}:") + logger.info(" " * indent + f"{key}:") recursive_print(value, indent + 2) elif isinstance(value, list): - print(" " * indent + f"{key}:") + logger.info(" " * indent + f"{key}:") for item in value: if isinstance(item, dict): recursive_print(item, indent + 2) else: - print(" " * (indent + 2) + str(item)) + logger.info(" " * (indent + 2) + str(item)) else: - print(" " * indent + f"{key}: {value}") + logger.info(" " * indent + f"{key}: {value}") def print_git_info(git_info_file: str): @@ -72,21 +75,23 @@ def print_git_info(git_info_file: str): :param git_info_file: Path to the JSON file containing Git information. """ try: - with open(git_info_file, 'r') as f: + with open(git_info_file, "r") as f: git_info = json.load(f) except FileNotFoundError: - print(f'{git_info_file} not found') + logger.warning(f"{git_info_file} not found") return except json.decoder.JSONDecodeError as e: - print(f"Error reading {git_info_file}: {e}") + logger.warning(f"Error reading {git_info_file}: {e}") return if not git_info: - print(f"Failed to retrieve git information from {git_info_file}.") + logger.error(f"Failed to retrieve git information from {git_info_file}.") return # Transform each top-level component without removing the keys. - transformed_git_info = {key: transform_component(value) for key, value in git_info.items()} + transformed_git_info = { + key: transform_component(value) for key, value in git_info.items() + } recursive_print(transformed_git_info) @@ -95,6 +100,7 @@ def print_git_info_all(): """ Convenience function to print Git information from multiple JSON files. """ - print_git_info('/ngen-app/ngen-fcst_git_info.json') - print_git_info('/ngen-app/ngen_git_info.json') - print() + print_git_info("/ngen-app/nwm-fcst-mgr_git_info.json") + print_git_info("/ngen-app/ngen-bmi-forcing_git_info.json") + print_git_info("/ngen-app/ngen_git_info.json") + logger.info(" ") diff --git a/python/nwm_fcst_mgr/log_level.py b/python/nwm_fcst_mgr/log_level.py new file mode 100644 index 0000000..e1196e7 --- /dev/null +++ b/python/nwm_fcst_mgr/log_level.py @@ -0,0 +1,97 @@ +import logging +import os +import sys +import time +from datetime import datetime, timezone +from pathlib import Path + +log_level = logging.INFO +MODULE_NAME = "FCST-MGR" +LOG_MODULE_NAME_LEN = 8 + + +class CustomFormatter(logging.Formatter): + LEVEL_NAME_MAP = { + logging.DEBUG: "DEBUG", + logging.INFO: "INFO", + logging.WARNING: "WARNING", + logging.ERROR: "SEVERE", + logging.CRITICAL: "FATAL" + } + + def format(self, record): + original_levelname = record.levelname + record.levelname = self.LEVEL_NAME_MAP.get(record.levelno, original_levelname) + record.levelname_padded = record.levelname.ljust(7)[:7] # Exactly 7 chars + formatted = super().format(record) + record.levelname = original_levelname # Restore original in case it's reused + return formatted + + +def create_timestamp(date_only: bool = False, iso: bool = False, append_ms: bool = False) -> str: + now = datetime.now(timezone.utc) + + if date_only: + ts_base = now.strftime("%Y%m%d") + elif iso: + ts_base = now.strftime("%Y-%m-%dT%H:%M:%S") + else: + ts_base = now.strftime("%Y%m%dT%H%M%S") + + if append_ms: + ms_str = f".{now.microsecond // 1000:03d}" + return ts_base + ms_str + else: + return ts_base + + +def log_level_set(): + ''' + Set logging level and specify logger configuration. + + Arguments + --------- + None + + Returns + ------- + None + + Notes + ----- + In the absense of user-specified logging level, level defaults to DEBUG + See also https://docs.python.org/3/library/logging.html + + ''' + + BASE_DIR = Path(__file__).resolve().parent.parent + + if Path("/ngencerf/data").exists(): + log_file_dir = Path('/ngencerf/data/run-logs/nwm-fcst-mgr/') + else: + log_file_dir = Path(BASE_DIR) / 'run-logs/nwm-fcst-mgr/' + + log_file_name = f"nwm-fcst-mgr_{create_timestamp()}.log" + os.makedirs(log_file_dir, exist_ok=True) + logFilePath = os.path.join(log_file_dir, log_file_name) + + formatted_module = MODULE_NAME.upper().ljust(LOG_MODULE_NAME_LEN)[:LOG_MODULE_NAME_LEN] + + try: + handler = logging.FileHandler(logFilePath, mode='a') + formatter = CustomFormatter( + fmt=f"%(asctime)s.%(msecs)03d {formatted_module} %(levelname_padded)s %(message)s", + datefmt="%Y-%m-%dT%H:%M:%S" + ) + handler.setFormatter(formatter) + + logging.Formatter.converter = time.gmtime + + logger = logging.getLogger() + logger.setLevel(log_level) + logger.handlers.clear() + logger.addHandler(handler) + + print(f"Logging into: {logFilePath}") + except OSError: + print(f"Can't Open local directory Log File: {logFilePath}", file=sys.stderr) diff --git a/python/run_ngen_fcst.py b/python/run_ngen_fcst.py deleted file mode 100644 index adb70fc..0000000 --- a/python/run_ngen_fcst.py +++ /dev/null @@ -1,302 +0,0 @@ -# python program to run ngen for realtime forecasting given the forcing data in .nc and -# validation configuration file in .yaml - -import glob -import json -import logging -import os -import shutil -import subprocess -import sys -import time -from datetime import datetime, timezone -from pathlib import Path - -import geopandas as gpd -import matplotlib.pyplot as plt -import netCDF4 -import pandas as pd -import yaml - -from git_util import print_git_info_all - -logger = logging.getLogger(__name__) - - -#logging.basicConfig(level=logging.INFO) - -#LOG = logging.getLogger(__name__) - -def create_timestamp() -> str: - now = datetime.now(timezone.utc) - return now.strftime("%Y-%m-%d") - - -def log_level_set(): - ''' - Set logging level and specify logger configuration. - - Arguments - --------- - input_parameters (dict): User input logging parameters - - Returns - ------- - None - - Notes - ----- - In the absense of user-specified logging level, level defaults to DEBUG - See also https://docs.python.org/3/library/logging.html - - ''' - - log_level = 'INFO' - if True: - BASE_DIR = Path(__file__).resolve().parent.parent - - if Path("/ngencerf/data").exists(): - log_file_dir = Path(f'/ngencerf/data/run-logs/ngen_fcst_{create_timestamp()}/') - else: - log_file_dir = Path(BASE_DIR) / f'run-logs/ngen_fcst_{create_timestamp()}/' - - log_file_name = "ngen_fcst.log" - os.makedirs(log_file_dir, exist_ok=True) - logFilePath = os.path.join(log_file_dir, log_file_name) - try: - logFile = open(logFilePath, "a") - print(f"Logging into: {logFilePath}") - except IOError: - print(f"Can't Open local directory Log File: {logFilePath}", file=sys.stderr) - - logging.Formatter.converter = time.gmtime - logging.basicConfig( - force=True, - level=log_level, - format='%(asctime)s.%(msecs)03d NGEN_FCST %(levelname)s %(message)s', - datefmt='%Y-%m-%dT%H:%M:%S', - handlers=[ - logging.FileHandler(logFilePath, mode='a'), # Log to a file - #logging.StreamHandler(sys.stdout) - ]) - else: - logging.basicConfig( - level=log_level, - format='%(asctime)s - %(name)s - %(levelname)s - [%(filename)s:%(lineno)s - %(funcName)s]: %(message)s', - stream=sys.stderr, - ) - - -import argparse - - -# setup the logger -log_level_set() - -print_git_info_all() - -# set environment variable for ngencerf backend - @TODO -#os.environ['NGEN_RESULTS_DIR'] = str(Path(agent.workdir).parent.parent) -#logging.info(f'Set environment variable NGEN_RESULTS_DIR to: {os.environ["NGEN_RESULTS_DIR"]}') - -# Create the parser -parser = argparse.ArgumentParser() - -# Add arguments -parser.add_argument('forcing_file', type=str, help='Path to the NetCDF forcing file') -parser.add_argument('config_file', type=str, help='Path to the config yaml file for a validation run (e.g., 01123000_config_valid_best.yaml from ngen-cal)') -parser.add_argument('output_folder', type=str, help='Path to the folder to be created for storing inputs/outputs from running ngen') - -# Parse the arguments -args = parser.parse_args() -logger.info(f"Forcing file to use: {args.forcing_file}") -logger.info(f"Validation config file to use: {args.config_file}") -logger.info(f"Relative folder path to outputs: {args.output_folder}") - -# define forcing file and valid_best config file -forcing_file = Path(args.forcing_file).resolve(strict=True) -config_file = Path(args.config_file).resolve(strict=True) - -# make sure forcing file exists as netcdf -if os.path.splitext(forcing_file)[1] != '.nc': - logger.warning(f'{forcing_file} does not have .nc extension. Assuming it is a netcdf file') - -# read forcing to get start and end times -ncvar = netCDF4.Dataset(forcing_file, "r") - -t0 = pd.to_datetime(ncvar.model_initialization_time, format="%Y-%m-%d_%H:%M:%S") -times = [t1 for t1 in ncvar['Time']] -start_time = t0 + pd.Timedelta(seconds=3600) -end_time = t0 + pd.Timedelta(seconds=(times[-1] - times[0] + 60) * 60) -logger.info(f'Start time: {start_time}') -logger.info(f'End time: {end_time}') - -# read yaml configuration file for best validation -with open(config_file) as file: - conf = yaml.safe_load(file) - -# read the realization file -real_file = Path(conf['model']['realization']) -real_file.resolve(strict=True) -with open(real_file) as fp: - real_config = json.load(fp) - -# update forcing in realization file -real_config['global']['forcing'] = dict([('path', str(forcing_file)), ('provider', 'NetCDF')]) - -# update time period in realization file -real_config['time']['start_time'] = str(start_time) -real_config['time']['end_time'] = str(end_time) - -# create output directory in Calibration Output directory -out_dir0 = Path(conf['general']['yaml_file']).parent.parent.resolve(strict=True) -out_dir = Path(out_dir0, 'Forecast_Run', args.output_folder) -out_dir.mkdir(parents=True, exist_ok=True) -out_dir = out_dir.resolve() -logger.info(f'New run directory created at: {out_dir}') - -# for noah-owp-modular & UEB, update path to BMI config files in realization file -# as well as start/end times in BMI config files -modules = real_config['global']['formulations'][0]['params']['modules'] -mod_dict = {'NoahOWP': 'noah-owp-modular', 'UEB': 'ueb'} - -startdate = start_time.strftime("%Y%m%d%H%M") -enddate = end_time.strftime("%Y%m%d%H%M") - -for i1, m1 in enumerate(modules): - if m1['params']['model_type_name'] in ['NoahOWP', 'UEB']: - - # read the BMI config files from the source directory in the realization file - src0 = real_config['global']['formulations'][0]['params']['modules'][i1]['params']['init_config'] - src = Path(src0.replace('{{id}}', '*')) - dst = Path(out_dir, mod_dict[m1['params']['model_type_name']] + '_input') - dst.mkdir(parents=True, exist_ok=True) - for f1 in glob.glob(f'{src}'): - with open(f1) as f: - lines = f.readlines() - - # update start/end times - for i2, l1 in enumerate(lines): - if m1['params']['model_type_name'] == 'NoahOWP': - if 'startdate' in l1: - lines[i2] = " " + "startdate".ljust(19) + "= " + "'" + startdate + "'" + " ! UTC time start of simulation (YYYYMMDDhhmm)\n" - elif 'enddate' in l1: - lines[i2] = " " + "enddate".ljust(19) + "= " + "'" + enddate + "'" + " ! UTC time end of simulation (YYYYMMDDhhmm)\n" - elif m1['params']['model_type_name'] == 'UEB': - lines[8] = f'{startdate[:4]} {startdate[4:6]} {startdate[6:8]} {startdate[8:10]}.0\n' - lines[9] = f'{enddate[:4]} {enddate[4:6]} {enddate[6:8]} {enddate[8:10]}.0\n' - - # write to new BMI config files - with open(Path(dst, os.path.basename(f1)), 'w') as outfile: - outfile.writelines(lines) - - # replace path to BMI config file in realization file - real_config['global']['formulations'][0]['params']['modules'][i1]['params']['init_config'] = str(Path(dst, os.path.basename(src0))) - -# For t-route, update path to config file as well as time-related info in the config file -src = Path(real_config['routing']['t_route_config_file_with_path']) -src.resolve(strict=True) -with open(src) as fp1: - rt_config = yaml.safe_load(fp1) - -# compute number of time steps and max_loop_size -nts = len(pd.date_range(start=start_time, end=end_time, freq='5min')) - 1 -max_loop_size = divmod(nts * 300, 3600)[0] + 1 -stream_output_time = divmod(nts * 300, 3600)[0] + 1 - -# update t-route config -rt_config['compute_parameters']['restart_parameters']['start_datetime'] = str(start_time) -rt_config['compute_parameters']['forcing_parameters']['nts'] = nts -rt_config['compute_parameters']['forcing_parameters']['max_loop_size'] = max_loop_size -rt_config['output_parameters']['stream_output']['stream_output_time'] = stream_output_time - -# write to new t-route config file -new_file = Path(out_dir, os.path.basename(src)) -with open(new_file, 'w') as file: - yaml.dump(rt_config, file, sort_keys=False, default_flow_style=False, indent=4) - -# update path to new t-route config in realization -real_config['routing']['t_route_config_file_with_path'] = str(new_file) - -# save the new realization file -new_real_file = Path(out_dir, os.path.basename(real_file)) -with open(new_real_file, 'w') as outfile: - json.dump(real_config, outfile, indent=4, separators=(", ", ": "), sort_keys=False) - -# hydrofabric gpkg -gpkg_cats = conf['model']['catchments'] -gpkg_nexus = conf['model']['nexus'] - -# ngen executable -ngen_exe = conf['model']['binary'] - -# run command -cmd = f'{ngen_exe} {gpkg_cats} "all" {gpkg_nexus} "all" {new_real_file}' - -# kick off ngen run and save stdout & stderr to ngen_stdout_stderr.log -log_file = Path(out_dir, 'ngen_stdout_stderr.log') -with open(log_file, 'a+') as log: - subprocess.check_call(cmd, stdout=log, stderr=log, shell=True, cwd=str(out_dir)) - -# move output files to output directory -output_dir = Path(out_dir, "output/") -output_dir.mkdir(parents=True, exist_ok=True) -for pat1 in ['cat*.csv', 'nex*.csv', 'troute*.nc']: - for f1 in glob.glob(f'{out_dir}/{pat1}'): - shutil.move(f1, Path(output_dir, os.path.basename(f1))) -logger.info(f'Outputs are saved at: {output_dir}') - -# get gage ID and make sure it is not empty -try: - gage0 = conf['model']['eval_params']['basinID'] -except: - raise ValueError(f'Key model/eval_params/basinID not found in {config_file}') -if gage0=="": - raise ValueError(f'basinID in {config_file} cannot be empty') - -# Handle crosswalk file (in order to get the correct feature_id when reading t-route data) -x_walk = pd.Series(dtype=object) -cwt_file = conf['model']['crosswalk'] -try: - with open(cwt_file) as fp: - data = json.load(fp) - for id, values in data.items(): - gage = values.get('Gage_no') - if gage: - if not isinstance(gage, str): - gage = gage[0] - if gage==gage0: - x_walk[id] = gage - break -except FileNotFoundError: - raise FileNotFoundError(f"Crosswalk file '{cwt_file}' not found.") -except json.JSONDecodeError: - raise ValueError(f"Failed to parse JSON from crosswalk file '{cwt_file}'.") - -if x_walk.empty: - raise Exception(f'{gage0} is not found in crosswalk file {cwt_file}') - -# get catchment at basin outlet for reading from t-route output -catchment_hydro_fabric = gpd.read_file(gpkg_cats, layer='divides') -catchment_hydro_fabric.set_index('id', inplace=True) -nexus_id = catchment_hydro_fabric.loc[x_walk.index[0].replace('cat', 'wb')]['toid'] -wb_lst = [x.split('-')[1] for x in list(catchment_hydro_fabric.query('toid==@nexus_id').index)] - -# read troute output -file1 = glob.glob(f'{output_dir}/troute*.nc')[0] -ncvar = netCDF4.Dataset(file1, "r") -fid_index = [list(ncvar['feature_id'][0:]).index(int(fid)) for fid in wb_lst] -output = pd.DataFrame(data={'sim_flow': pd.DataFrame(ncvar['flow'][fid_index], index=fid_index).T.sum(axis=1)}) -t0 = pd.to_datetime(ncvar.file_reference_time, format="%Y-%m-%d_%H:%M:%S") -output.index = [t0 + pd.Timedelta(seconds=int(t1)) for t1 in ncvar['time']] -output.index.name = 'Time' - -# plot the hydrograph -output.plot(y='sim_flow', kind='line') -plt.xlabel('Time') -plt.ylabel('Streamflow (m^3/s)') -plt.savefig(Path(output_dir, gage0 + '_hydrograph.png'), bbox_inches ="tight") - -# save streamflow simulation to csv -output.to_csv(Path(output_dir, gage0 + '_output.csv')) -logger.info('Run completed!') \ No newline at end of file diff --git a/test_data/01123000_config_valid_best.yaml b/test_data/01123000_config_valid_best.yaml new file mode 100644 index 0000000..a519f08 --- /dev/null +++ b/test_data/01123000_config_valid_best.yaml @@ -0,0 +1,117 @@ +general: + strategy: + type: estimation + algorithm: dds + name: valid_best + log: true + workdir: /home/jeff.wade/ngwpc/run_ngen/kge_DDS/noah_cfes/01123000 + yaml_file: /home/jeff.wade/ngwpc/run_ngen/kge_DDS/noah_cfes/01123000/Output/Validation_Run/01123000_config_valid_best.yaml + start_iteration: 0 + iterations: 2 + restart: 0 + calibration_run_id: 1 + ngen_cerf: false + auth_token: '' +CFE: &id001 +- name: b + min: 2.0 + max: 15.0 + init: 4.05 +- name: satpsi + min: 0.03 + max: 0.955 + init: 0.355 +- name: satdk + min: 1.0e-07 + max: 0.000726 + init: 3.38e-06 +- name: maxsmc + min: 0.16 + max: 0.59 + init: 0.439 +- name: refkdt + min: 0.1 + max: 4.0 + init: 1.0 +- name: slope + min: 0.0 + max: 1.0 + init: 0.1 +- name: max_gw_storage + min: 0.01 + max: 0.25 + init: 0.05 +- name: expon + min: 1.0 + max: 8.0 + init: 3.0 +- name: Cgw + min: 1.8e-06 + max: 0.0018 + init: 1.8e-05 +- name: Klf + min: 0.0 + max: 1.0 + init: 0.01 +- name: Kn + min: 0.0 + max: 1.0 + init: 0.03 +NoahOWP: &id002 +- name: RSURF_EXP + min: 1.0 + max: 6.0 + init: 5.0 +- name: CWP + min: 0.09 + max: 0.36 + init: 0.18 +- name: MP + min: 3.6 + max: 12.6 + init: 9.0 +- name: VCMX25 + min: 24.0 + max: 112.0 + init: 52.2 +- name: MFSNO + min: 0.5 + max: 4.0 + init: 2.0 +- name: RSURF_SNOW + min: 0.136 + max: 100.0 + init: 50.0 +- name: SCAMAX + min: 0.7 + max: 1.0 + init: 0.9 +model: + type: ngen + binary: /home/jeff.wade/ngwpc/run_ngen/kge_DDS/noah_cfes/01123000/Input/ngen + realization: /home/jeff.wade/ngwpc/run_ngen/kge_DDS/noah_cfes/01123000/Output/Validation_Run/01123000_realization_config_bmi_valid_best.json + catchments: /home/jeff.wade/ngwpc/run_ngen/kge_DDS/noah_cfes/01123000/Input/gauge_01123000.gpkg + nexus: /home/jeff.wade/ngwpc/run_ngen/kge_DDS/noah_cfes/01123000/Input/gauge_01123000.gpkg + crosswalk: /home/jeff.wade/ngwpc/run_ngen/kge_DDS/noah_cfes/01123000/Input/01123000_crosswalk.json + obsflow: null + strategy: uniform + params: + CFE: *id001 + NoahOWP: *id002 + eval_params: + objective: kge + evaluation_start: '2016-10-01 00:00:00' + evaluation_stop: '2017-09-30 23:00:00' + valid_start_time: '2014-10-01 00:00:00' + valid_end_time: '2017-09-30 23:00:00' + valid_eval_start_time: '2015-10-01 00:00:00' + valid_eval_end_time: '2016-09-30 23:00:00' + full_eval_start_time: '2015-10-01 00:00:00' + full_eval_end_time: '2017-09-30 23:00:00' + save_output_iteration: 0 + save_plot_iteration: 0 + save_plot_iter_freq: 1 + basinID: '01123000' + threshold: 3.88 + site_name: 'USGS 01123000: "Little River Near Hanover, CT"' + user: '' diff --git a/test_data/Gage_01123000_csv/cat-11466.csv b/test_data/Gage_01123000_csv/cat-11466.csv new file mode 100644 index 0000000..1e82171 --- /dev/null +++ b/test_data/Gage_01123000_csv/cat-11466.csv @@ -0,0 +1,19 @@ +Time,U2D,V2D,LWDOWN,RAINRATE,T2D,Q2D,PSFC,SWDOWN +2025-02-20 15:00:00,0.792,-2.814,191.328,0,266.9,0.001349,99149.5,539.232 +2025-02-20 16:00:00,0.109,-2.05,198.545,0,268.02,0.001483,98998.1,611.038 +2025-02-20 17:00:00,1.044,-2.056,202.387,0,269.13,0.001609,99029.5,650.483 +2025-02-20 18:00:00,0.702,-1.589,214.491,0,269.73,0.001823,98934.7,582.878 +2025-02-20 19:00:00,0.712,-2.757,239.591,0,270.03,0.002107,98758.1,441.591 +2025-02-20 20:00:00,0.699,-2.393,285.533,0,269.85,0.002417,98698.1,146.196 +2025-02-20 21:00:00,0.871,-2.21,285.905,1.99466558825634e-08,269.55,0.002408,98708.1,65.686 +2025-02-20 22:00:00,1.16,-1.65,285.135,1.56998922307139e-07,269.27,0.002408,98798.1,8.292 +2025-02-20 23:00:00,1.35,-1.556,282.701,1.37052268200932e-07,268.92,0.002354,98882.9,0 +2025-02-21 00:00:00,1.126,-1.475,281.208,4.10320035371114e-06,268.61,0.002307,98907.4,0 +2025-02-21 01:00:00,2.177,-1.808,281.035,8.98366215551505e-06,268.28,0.002286,98864,0 +2025-02-21 02:00:00,3.328,-2.844,280.598,7.67198162066052e-06,267.86,0.002214,98860.1,0 +2025-02-21 03:00:00,3.551,-2.953,274.01,3.49171727975772e-06,267.19,0.002025,98857.4,0 +2025-02-21 04:00:00,4.323,-3.511,237.324,1.60811384830595e-06,266.06,0.001849,98765.3,0 +2025-02-21 05:00:00,4.294,-3.625,222.341,5.74366549699334e-06,265.51,0.001769,98745.3,0 +2025-02-21 06:00:00,4.374,-4.081,260.361,1.01648311101599e-06,265.86,0.001773,98785.3,0 +2025-02-21 07:00:00,4.366,-4.145,266.781,3.52459181840459e-07,266.1,0.001782,98947.4,0 +2025-02-21 08:00:00,4.428,-2.926,247.543,0,265.87,0.001763,99026.7,0 diff --git a/test_data/Gage_01123000_csv/cat-11467.csv b/test_data/Gage_01123000_csv/cat-11467.csv new file mode 100644 index 0000000..29e8beb --- /dev/null +++ b/test_data/Gage_01123000_csv/cat-11467.csv @@ -0,0 +1,19 @@ +Time,U2D,V2D,LWDOWN,RAINRATE,T2D,Q2D,PSFC,SWDOWN +2025-02-20 15:00:00,0.758,-2.851,192.956,0,267.26,0.00141,99640.4,537.832 +2025-02-20 16:00:00,0.523,-2.426,197.546,0,268.37,0.00151,99484.1,627.121 +2025-02-20 17:00:00,0.855,-2.042,206.172,0,269.57,0.001621,99527.7,641.23 +2025-02-20 18:00:00,0.596,-1.747,208.462,0,270.1,0.001835,99427.7,616.255 +2025-02-20 19:00:00,0.784,-3.035,236.852,0,270.3,0.002082,99247.7,430.428 +2025-02-20 20:00:00,0.618,-2.488,284.795,0,270.25,0.0023,99197.7,172.695 +2025-02-20 21:00:00,0.26,-2.371,287.52,0,269.9,0.002393,99207.7,45.438 +2025-02-20 22:00:00,0.62,-2.113,287.299,1.17783451969444e-06,269.76,0.002391,99297.7,7.782 +2025-02-20 23:00:00,0.4,-1.709,285.371,3.56382588506676e-06,269.62,0.002412,99377.7,0 +2025-02-21 00:00:00,0.884,-1.32,282.467,1.2050659279339e-05,269.39,0.0024,99407.7,0 +2025-02-21 01:00:00,1.973,-1.658,282.189,9.8840391729027e-06,269.03,0.002232,99368.2,0 +2025-02-21 02:00:00,2.919,-2.037,282.106,9.38357879931573e-06,268.63,0.002231,99358.2,0 +2025-02-21 03:00:00,3.371,-2.767,278.065,4.51473715656903e-06,268.06,0.002022,99348.8,0 +2025-02-21 04:00:00,4.306,-3.252,239.9,8.95100868092413e-07,267.06,0.001841,99249.3,0 +2025-02-21 05:00:00,4.184,-3.59,240.403,3.11436951960786e-06,266.54,0.001761,99234.1,0 +2025-02-21 06:00:00,4.032,-4.083,264.086,1.25582755572395e-06,266.73,0.001722,99274.1,0 +2025-02-21 07:00:00,3.918,-4.279,266.154,1.76601261614451e-07,267.02,0.001732,99444.1,0 +2025-02-21 08:00:00,4.363,-2.819,233.758,0,266.63,0.001706,99528.2,0 diff --git a/test_data/Gage_01123000_csv/cat-11468.csv b/test_data/Gage_01123000_csv/cat-11468.csv new file mode 100644 index 0000000..5f69190 --- /dev/null +++ b/test_data/Gage_01123000_csv/cat-11468.csv @@ -0,0 +1,19 @@ +Time,U2D,V2D,LWDOWN,RAINRATE,T2D,Q2D,PSFC,SWDOWN +2025-02-20 15:00:00,1.049,-3.097,194.012,0,267.64,0.001388,99899.4,538.833 +2025-02-20 16:00:00,0.859,-2.472,197.599,0,268.77,0.00144,99761,629.704 +2025-02-20 17:00:00,1.042,-2.052,204.894,0,269.84,0.001545,99814.4,648.795 +2025-02-20 18:00:00,1.132,-1.756,206.135,0,270.41,0.001719,99699,620.776 +2025-02-20 19:00:00,1.363,-2.974,225.007,0,270.67,0.001942,99514.9,468.159 +2025-02-20 20:00:00,0.888,-2.582,274.586,0,270.66,0.002167,99474.4,228.357 +2025-02-20 21:00:00,0.415,-2.254,287.74,0,270.29,0.002314,99478.2,41.091 +2025-02-20 22:00:00,0.891,-2.019,287.287,9.33754620291438e-08,270.11,0.002297,99574.4,12.185 +2025-02-20 23:00:00,0.881,-1.731,285.052,1.20095103284257e-06,269.87,0.002256,99654.4,0 +2025-02-21 00:00:00,1.046,-1.468,282.259,4.83463645650772e-06,269.61,0.002251,99684.4,0 +2025-02-21 01:00:00,2.249,-1.431,282.579,7.62731997383526e-06,269.31,0.002213,99654.4,0 +2025-02-21 02:00:00,3.344,-1.753,282.767,1.77536730916472e-05,268.68,0.002173,99651.8,0 +2025-02-21 03:00:00,3.563,-3.079,278.598,4.73335740025504e-06,268.28,0.002007,99635.6,0 +2025-02-21 04:00:00,4.31,-3.274,252.871,1.34906053972372e-06,267.5,0.001835,99536.8,0 +2025-02-21 05:00:00,4.306,-3.639,243.142,2.44010925598559e-06,266.85,0.001743,99516.3,0 +2025-02-21 06:00:00,4.101,-4.076,255.891,1.66666666245874e-06,266.89,0.00169,99555.6,0 +2025-02-21 07:00:00,4.006,-4.283,250.621,1.26259971366949e-08,267.02,0.0017,99722.2,0 +2025-02-21 08:00:00,4.482,-3.241,242.697,0,266.98,0.00169,99816.3,0 diff --git a/test_data/Gage_01123000_csv/cat-11469.csv b/test_data/Gage_01123000_csv/cat-11469.csv new file mode 100644 index 0000000..9ee4a93 --- /dev/null +++ b/test_data/Gage_01123000_csv/cat-11469.csv @@ -0,0 +1,19 @@ +Time,U2D,V2D,LWDOWN,RAINRATE,T2D,Q2D,PSFC,SWDOWN +2025-02-20 15:00:00,0.924,-3.005,193.576,0,267.4,0.001388,99653.3,538.293 +2025-02-20 16:00:00,0.731,-2.498,196.844,0,268.52,0.001454,99502.8,628.996 +2025-02-20 17:00:00,0.987,-2.078,205.213,0,269.64,0.001556,99558.1,645.443 +2025-02-20 18:00:00,0.904,-1.78,205.45,0,270.21,0.001756,99451.3,619.84 +2025-02-20 19:00:00,1.132,-3.03,229.556,0,270.44,0.001994,99265.5,447.186 +2025-02-20 20:00:00,0.778,-2.555,282.015,0,270.41,0.002216,99221.3,198.286 +2025-02-20 21:00:00,0.24,-2.379,287.328,0,270.03,0.002355,99231.3,39.195 +2025-02-20 22:00:00,0.823,-2.131,286.732,3.82554389943834e-07,269.87,0.00233,99321.3,11.062 +2025-02-20 23:00:00,0.793,-1.794,284.898,2.17054912354797e-06,269.68,0.002299,99401.3,0 +2025-02-21 00:00:00,1.009,-1.535,282.171,7.37064465283765e-06,269.45,0.002301,99431.3,0 +2025-02-21 01:00:00,2.077,-1.578,282.171,8.38033156469464e-06,269.1,0.002207,99398.6,0 +2025-02-21 02:00:00,3.143,-1.811,281.867,1.37969791467185e-05,268.57,0.002198,99389.7,0 +2025-02-21 03:00:00,3.503,-2.945,277.479,4.60539285995765e-06,268.09,0.002014,99379.7,0 +2025-02-21 04:00:00,4.284,-3.296,249.937,1.11024598936638e-06,267.2,0.001835,99281.2,0 +2025-02-21 05:00:00,4.267,-3.672,247.189,2.77587491837039e-06,266.68,0.001754,99259.7,0 +2025-02-21 06:00:00,4.092,-4.133,257.009,1.59250953402079e-06,266.72,0.001697,99299.7,0 +2025-02-21 07:00:00,4.038,-4.407,256.044,4.35377138785498e-08,266.93,0.001706,99463.9,0 +2025-02-21 08:00:00,4.514,-3.146,230.612,0,266.7,0.001693,99559.7,0 diff --git a/test_data/Gage_01123000_csv/cat-11470.csv b/test_data/Gage_01123000_csv/cat-11470.csv new file mode 100644 index 0000000..44f1e92 --- /dev/null +++ b/test_data/Gage_01123000_csv/cat-11470.csv @@ -0,0 +1,19 @@ +Time,U2D,V2D,LWDOWN,RAINRATE,T2D,Q2D,PSFC,SWDOWN +2025-02-20 15:00:00,0.707,-2.723,193.036,0,267.33,0.001411,99739.6,536.866 +2025-02-20 16:00:00,0.307,-2.24,204.368,0,268.41,0.001516,99579.6,566.782 +2025-02-20 17:00:00,0.921,-2.019,205.77,0,269.65,0.001621,99619.6,642.067 +2025-02-20 18:00:00,0.585,-1.678,210.385,0,270.2,0.001834,99519.6,611.884 +2025-02-20 19:00:00,0.701,-2.871,233.496,0,270.41,0.002103,99339.6,442.381 +2025-02-20 20:00:00,0.592,-2.391,286.201,0,270.33,0.002333,99288.7,146.062 +2025-02-20 21:00:00,0.404,-2.275,287.296,6.18914741679077e-09,270.02,0.002393,99298.9,50.32 +2025-02-20 22:00:00,0.79,-1.966,287.09,4.48279422471387e-07,269.89,0.00237,99388.7,7.308 +2025-02-20 23:00:00,0.685,-1.595,284.74,1.31408648940123e-06,269.66,0.00237,99469.6,0 +2025-02-21 00:00:00,0.95,-1.269,282.154,6.69180235490785e-06,269.43,0.002339,99498.7,0 +2025-02-21 01:00:00,2.047,-1.555,282.202,9.21615173865575e-06,269.05,0.002248,99458.7,0 +2025-02-21 02:00:00,3.09,-2.368,282.459,7.94950392446481e-06,268.62,0.002222,99448.7,0 +2025-02-21 03:00:00,3.464,-2.756,277.82,4.18467425333802e-06,268.04,0.002017,99446.3,0 +2025-02-21 04:00:00,4.33,-3.31,234.327,1.25920075788599e-06,267.02,0.001836,99346.3,0 +2025-02-21 05:00:00,4.282,-3.555,227.052,4.33934701504768e-06,266.44,0.001756,99328.4,0 +2025-02-21 06:00:00,4.205,-4.002,264.252,1.30372109197197e-06,266.7,0.001736,99368.7,0 +2025-02-21 07:00:00,4.099,-4.111,266.466,2.77777786550359e-07,266.93,0.001746,99538.7,0 +2025-02-21 08:00:00,4.323,-2.705,237.109,0,266.64,0.001713,99618.7,0 diff --git a/test_data/Gage_01123000_csv/cat-11475.csv b/test_data/Gage_01123000_csv/cat-11475.csv new file mode 100644 index 0000000..101d16d --- /dev/null +++ b/test_data/Gage_01123000_csv/cat-11475.csv @@ -0,0 +1,19 @@ +Time,U2D,V2D,LWDOWN,RAINRATE,T2D,Q2D,PSFC,SWDOWN +2025-02-20 15:00:00,0.727,-2.901,191.31,0,266.83,0.001361,99158.5,540.038 +2025-02-20 16:00:00,0.032,-2.052,201.812,0,267.91,0.001507,99008.2,593.963 +2025-02-20 17:00:00,0.796,-2.064,203.199,0,269.11,0.001651,99038.5,649.874 +2025-02-20 18:00:00,0.419,-1.59,214.166,0,269.64,0.001888,98938.6,576.776 +2025-02-20 19:00:00,0.432,-2.824,264.45,0,269.84,0.002132,98768.2,364.464 +2025-02-20 20:00:00,0.456,-2.505,285.814,0,269.74,0.002435,98708.2,142.887 +2025-02-20 21:00:00,0.791,-2.059,286.74,1.97764705944792e-07,269.43,0.002438,98718.2,64.705 +2025-02-20 22:00:00,0.647,-1.498,286.084,8.07674553016113e-07,269.22,0.002464,98808.2,7.133 +2025-02-20 23:00:00,0.542,-1.207,283.479,6.09909818649612e-07,268.98,0.002473,98898.1,0 +2025-02-21 00:00:00,0.931,-1.071,281.361,7.30867941456381e-06,268.63,0.002382,98911.1,0 +2025-02-21 01:00:00,2.015,-1.972,281.028,1.04325326901744e-05,268.23,0.002281,98861.5,0 +2025-02-21 02:00:00,3.169,-2.644,280.75,7.09363666828722e-06,267.92,0.002233,98863.6,0 +2025-02-21 03:00:00,3.523,-3.003,265.495,3.01202931041189e-06,267.1,0.002014,98861.1,0 +2025-02-21 04:00:00,4.443,-3.453,238.954,1.46059414873889e-06,265.97,0.001841,98763.7,0 +2025-02-21 05:00:00,4.335,-3.64,220.694,5.01193380841869e-06,265.46,0.001761,98743.7,0 +2025-02-21 06:00:00,4.353,-4.134,266.53,8.44531996335718e-07,265.89,0.001771,98783.7,0 +2025-02-21 07:00:00,4.28,-4.271,268.488,3.46592202049578e-07,266.13,0.001773,98951.1,0 +2025-02-21 08:00:00,4.441,-2.871,230.399,0,265.69,0.001755,99024,0 diff --git a/test_data/Gage_01123000_csv/cat-11476.csv b/test_data/Gage_01123000_csv/cat-11476.csv new file mode 100644 index 0000000..6597f2b --- /dev/null +++ b/test_data/Gage_01123000_csv/cat-11476.csv @@ -0,0 +1,19 @@ +Time,U2D,V2D,LWDOWN,RAINRATE,T2D,Q2D,PSFC,SWDOWN +2025-02-20 15:00:00,0.665,-3.028,190.54,0,266.66,0.001391,99006.9,539.989 +2025-02-20 16:00:00,-0.152,-2.008,199.171,0,267.74,0.001547,98856.9,601.635 +2025-02-20 17:00:00,0.679,-2.01,201.387,0,268.94,0.00171,98886.9,654.517 +2025-02-20 18:00:00,0.311,-1.636,218.935,0,269.54,0.001956,98790.5,585.454 +2025-02-20 19:00:00,0.282,-2.702,272.56,0,269.74,0.002187,98620.5,333.321 +2025-02-20 20:00:00,0.379,-2.594,285.545,0,269.55,0.002433,98551.3,132.411 +2025-02-20 21:00:00,0.9,-2.006,286.027,1.41223821970016e-07,269.36,0.002422,98571.3,63.953 +2025-02-20 22:00:00,0.598,-1.122,285.98,7.68172071730078e-07,269.14,0.002485,98661.3,7.75 +2025-02-20 23:00:00,0.549,-0.949,283.509,7.61058743137255e-07,268.81,0.002472,98750.5,0 +2025-02-21 00:00:00,1.003,-1.141,281.179,7.73560896050185e-06,268.42,0.002333,98760.5,0 +2025-02-21 01:00:00,2.125,-2.375,280.881,9.92011882772204e-06,268.08,0.002262,98706.9,0 +2025-02-21 02:00:00,3.09,-2.571,280.187,8.32584919407964e-06,267.74,0.002211,98717.7,0 +2025-02-21 03:00:00,3.471,-3.085,270.524,2.11965652852086e-06,266.91,0.001996,98706.2,0 +2025-02-21 04:00:00,4.287,-3.264,237.939,1.68953647516901e-06,265.73,0.001823,98622,0 +2025-02-21 05:00:00,4.196,-3.469,252.282,5.35991739525343e-06,265.57,0.001764,98597.1,0 +2025-02-21 06:00:00,4.233,-4.135,265.965,8.16355111510347e-07,265.74,0.001763,98636.2,0 +2025-02-21 07:00:00,4.174,-4.205,269.347,4.37201549630117e-07,266,0.001769,98806.2,0 +2025-02-21 08:00:00,4.419,-2.897,236.034,0,265.58,0.00175,98876.2,0 diff --git a/version.txt b/version.txt deleted file mode 100644 index 7b0d70e..0000000 --- a/version.txt +++ /dev/null @@ -1,3 +0,0 @@ -version=1.2.0 -date=2025-03-19 -commit=bc9d7fe866e87bacb0626ba7f699961f65261ecc