update and fix surrogate

Author: iacopoff

docker/hydromt/requirements.txt

@@ -21,7 +21,7 @@ h5netcdf # to read netcd on S3
 openeo-pg-parser-networkx==2024.5.0
 hydromt @ git+https://github.com/interTwin-eu/hydromt.git@stac#egg=hydromt
 hydromt_wflow @ git+https://github.com/interTwin-eu/hydromt_wflow.git@stac#egg=hydromt_wflow
-raster2stac==0.0.7
+raster2stac==0.0.8
 #https://gitlab.inf.unibz.it/earth_observation_public/raster-to-stac/-/tree/update_url
 #raster2stac @ git+https://gitlab.inf.unibz.it/earth_observation_public/raster-to-stac.git@update_url#egg=raster2stac
 s3fs # to access aws s3 bucket

docker/surrogate/Dockerfile

@@ -1,13 +1,13 @@
 #FROM ghcr.io/intertwin-eu/itwinai:jlab-slim-latest
 #FROM ghcr.io/intertwin-eu/itwinai:0.2.2-torch2.1
 #FROM ghcr.io/intertwin-eu/itwinai:torch-slim-latest
-FROM ghcr.io/intertwin-eu/hython-itwinai-plugin:v0.2
+FROM ghcr.io/intertwin-eu/hython-itwinai-plugin:v0.4
 
 
 ENV HYDRA_FULL_ERROR=1
 
 # !! Workaraound for overriding the default image plugin which does not have latest developments
-RUN git clone -b main https://github.com/interTwin-eu/hython-itwinai-plugin.git
+RUN git clone -b iacopo https://github.com/interTwin-eu/hython-itwinai-plugin.git
 RUN cd /app/hython-itwinai-plugin && pip install . 
 RUN cd .
 
@@ -16,6 +16,11 @@ COPY requirements.txt ./requirements.txt
 # Breaks if not using --use-pep517 (https://github.com/pypa/pip/issues/6334) 
 RUN pip install --upgrade pip && pip install --no-cache-dir --use-pep517 -r requirements.txt 
 
+# !! Remove when fixing image plugin 
+RUN git clone -b main https://github.com/interTwin-eu/hython.git
+RUN cd /app/hython && pip install . 
+RUN cd .
+
 RUN mkdir -p ./use-case
 RUN mkdir -p ./model
 RUN mkdir -p /data

docker/surrogate/requirements.txt

@@ -2,4 +2,4 @@ s3fs==2025.5.0
 omegaconf==2.3.0
 
 # TODO ITWINAI PLUGIN INSTALLED IN DOCKER
-# itwinai-hython-plugin @ git+https://github.com/interTwin-eu/hython-itwinai-plugin.git@main#egg=hython-itwinai-plugin
+# itwinai-hython-plugin @ git+https://github.com/interTwin-eu/hython-itwinai-plugin.git@main#egg=hython-itwinai-plugin

docker/surrogate/src/training_local.yaml

@@ -120,17 +120,16 @@ data_source:
           client_kwargs:
               endpoint_url: "https://objectstore.eodc.eu:2222" 
   s3:
-    static_inputs: "s3://rucio/interTwin_EURAC/hydroform/wflow/be76bdf2-858c-4350-986c-40b57ab0247f_/wflow_output_statics.zarr"
-    dynamic_inputs: "s3://rucio/interTwin_EURAC/hydroform/wflow/be76bdf2-858c-4350-986c-40b57ab0247f_/wflow_output_dynamics.zarr"
-    target_variables: "s3://rucio/interTwin_EURAC/hydroform/wflow/be76bdf2-858c-4350-986c-40b57ab0247f_/wflow_output_dynamics.zarr"
+    static_inputs: "s3://rucio/interTwin_EURAC/hydroform/wflow/6af2cc6a-d2ba-4743-a528-9b49f79c439e_/wflow_output_statics.zarr"
+    dynamic_inputs: "s3://rucio/interTwin_EURAC/hydroform/wflow/6af2cc6a-d2ba-4743-a528-9b49f79c439e_/wflow_output_dynamics.zarr"
+    target_variables: "s3://rucio/interTwin_EURAC/hydroform/wflow/6af2cc6a-d2ba-4743-a528-9b49f79c439e_/wflow_output_dynamics.zarr"
 
 
 static_categorical_inputs:
    - wflow_landuse
    - wflow_soil
 
 static_inputs:
-  - thetaS
   - thetaR
   - KsatVer
   - c
@@ -147,10 +146,60 @@ mask_variables:
   - mask_missing
   # - mask_lake
 
+
+# Preprocess
+
+preprocessor: null
+    # static_inputs:
+    #     lazy: False
+    #     variant:
+    #       - _target_: hython.preprocessor.Log10p
+    #         variable:
+    #             - "KsatVer"
+    # dynamic_inputs:
+    #     lazy: False
+    #     variant:
+    #       - _target_: hython.preprocessor.Log10p
+    #         variable:
+    #             - "precip"
+
 # Scaling
 
-scaling_variant: minmax
-scaling_use_cached: True
+scaling_use_cached: False
+scaling_static_range:
+    thetaS: [0.25, 0.95] 
+    thetaR: [0.01, 0.35] # it was 0.25 in table
+    KsatVer: [1, 10000]
+    c: [1, 20]
+    f: [0.00001, 0.1] # from training data (it is fitted)
+    M: [1, 3000]
+    RootingDepth: [100, 2000 ] #5000] # (mm)
+    Swood: [0.0, 0.5] # -
+    Sl: [0.02, 0.2] #(mm)
+    Kext: [0.48, 0.96] # 
+
+scaler:
+    static_inputs:
+        lazy: False
+        variant:
+          - _target_: hython.scaler.BoundedScaler
+            variable:
+              thetaR: ${scaling_static_range.thetaR}
+              KsatVer: ${scaling_static_range.KsatVer}
+              c: ${scaling_static_range.c}
+    dynamic_inputs:
+        lazy: False
+        variant:
+          - _target_: hython.scaler.MinMax01Scaler
+            variable:
+              - "precip"
+              - "pet"
+              - "temp"
+    target_variables:
+        lazy: False
+        variant:
+          - _target_: hython.scaler.MinMax01Scaler
+            variable: ${target_variables}
 
 # MLFLow
 tracking_uri: null # http://mlflow.intertwin.fedcloud.eu/
@@ -178,10 +227,10 @@ training_pipeline:
       dataset: ${dataset}
       data_lazy_load: ${data_lazy_load}
       data_source: ${data_source}
-      scaling_variant: ${scaling_variant}
+      preprocessor: ${preprocessor}
+      scaler: ${scaler}
       scaling_use_cached: ${scaling_use_cached}
       experiment_name: ${experiment_name}
-      run_id: ${run_id}
       work_dir: ${work_dir}
       train_downsampler: ${train_downsampler}
       valid_downsampler: ${valid_downsampler}
@@ -191,7 +240,6 @@ training_pipeline:
       config:
         experiment: ${experiment_name}/${run_id}
         experiment_name: ${experiment_name}
-        run_id: ${run_id}
         work_dir: ${work_dir}
         batch_size: ${batch}
         learning_rate: ${learning_rate}
@@ -229,12 +277,12 @@ training_pipeline:
         # model logger
         model_logger: ${model_logger}
 
+      run_id: $(run_id)
       strategy: ${strategy}
       epochs: ${epochs}
       measure_gpu_data: True
-      measure_communication_overhead: True
       measure_epoch_time: True
-      run_id: $(run_id)
+      torch_profiling: False
       random_seed: ${seed}
       profiling_wait_epochs: 1
       profiling_warmup_epochs: 1
@@ -263,25 +311,26 @@ hpo:
       dataset: ${dataset}
       data_lazy_load: ${data_lazy_load}
       data_source: ${data_source}
-      scaling_variant: ${scaling_variant}
+      preprocessor: ${preprocessor}
+      scaler: ${scaler}
       scaling_use_cached: ${scaling_use_cached}
       experiment_name: ${experiment_name}
-      run_id: ${run_id}
       work_dir: ${work_dir}
       train_downsampler: ${train_downsampler}
       valid_downsampler: ${valid_downsampler}
+      seq_length: ${seq_length}
     - _target_: itwinai.plugins.hython.trainer.RNNDistributedTrainer
       model: ${model}
       config:
         experiment: ${experiment_name}/${run_id}
         experiment_name: ${experiment_name}
-        run_id: ${run_id}
         work_dir: ${work_dir}
         batch_size: ${batch}
         learning_rate: ${learning_rate}
         num_workers_dataloader: ${num_workers_dataloader}
         pin_gpu_memory: True
         hython_trainer: ${hython_trainer}
+        dynamic_downsampler: ${dynamic_downsampler}
         seq_length: ${seq_length}
         target_variables: ${target_variables}
         dynamic_inputs: ${dynamic_inputs}
@@ -314,13 +363,13 @@ hpo:
         # model logger
         model_logger: ${model_logger}
 
+      run_id: $(run_id)
       time_ray: ${time_ray}
       strategy: ${strategy}
       epochs: ${epochs}
       measure_gpu_data: True
-      measure_communication_overhead: True
       measure_epoch_time: True
-      run_id: $(run_id)
+      torch_profiling: False
       random_seed: ${seed}
       profiling_wait_epochs: 1
       profiling_warmup_epochs: 1
@@ -354,7 +403,7 @@ hpo:
         #   reduction_factor: 6
         #   brackets: 1
       ray_run_config:
-        _target_: ray.train.RunConfig
+        _target_: ray.tune.RunConfig
         # storage_path must be an absolute path. Defaulting to the directory from which the
         # job is launched using the itwinai custom OmegaConf resolver ${itwinai.cwd:}
         storage_path: ${itwinai.cwd:}/ray_checkpoints
@@ -380,8 +429,4 @@ hpo:
           type: qrandint
           lower: 90
           upper: 365
-          q: 30
-        interval_value:
-          type: choice
-          categories: [3, 5, 10]
-        
+          q: 30

docker/wflow/requirements.txt

@@ -4,9 +4,7 @@ urllib3
 zarr<3.0.0
 cf-xarray==0.10.4
 dask
-# replace r2s with https://gitlab.inf.unibz.it/earth_observation_public/raster-to-stac/-/tree/update_url
-raster2stac @ git+https://gitlab.inf.unibz.it/earth_observation_public/raster-to-stac.git@update_url#egg=raster2stac
-#raster2stac==0.0.7
+raster2stac==0.0.8
 s3fs # to access aws s3 bucket
 rioxarray==0.17.0
 xarray==2024.3.0

tests/test_hydromt.sh

@@ -5,7 +5,7 @@ echo $PWD
 HYDROMT_DIR=$PWD/docker/hydromt
 
 # Source credentials for AWS S3 bucket
-source $PWD/tests/.env_s3
+source $PWD/tests/.env_s3_intertwin
 echo $AWS_ACCESS_KEY
 echo $AWS_SECRET_KEY
 

tests/test_surrogate_local.sh

@@ -12,7 +12,7 @@ fi
 source $PWD/tests/.env_mlflow
 
 # Source credentials for AWS S3 bucket
-source $PWD/tests/.env_s3
+source $PWD/tests/.env_s3_intertwin
 
 docker build --no-cache -f $SURR_DIR/Dockerfile -t surrogate-test $SURR_DIR
 
@@ -30,9 +30,9 @@ if [ -z "$1" ]; then
 	surrogate-test:latest /bin/bash -c "python ./use-case/gen_surr_config.py \
 		--config './use-case/training_local.yaml' \
 		--output './use-case/training_local_updated.yaml' \
-		--cp 'train_temporal_range=[\"2015-01-01\", \"2016-12-01\"]' \
-		--cp 'valid_temporal_range=[\"2015-01-01\", \"2016-12-01\"]' \
-		--cp 'test_temporal_range=[\"2015-01-01\", \"2016-12-01\"]' \
+		--cp 'train_temporal_range=[\"2001-01-01\", \"2001-03-31\"]' \
+		--cp 'valid_temporal_range=[\"2001-01-01\", \"2001-03-31\"]' \
+		--cp 'test_temporal_range=[\"2001-01-01\", \"2001-03-31\"]' \
 		&& cat ./use-case/training_local_updated.yaml \
 		&& itwinai exec-pipeline --config-dir ./use-case --config-name training_local_updated"
 

tests/test_wflow.sh

@@ -22,6 +22,6 @@ docker container run \
     -v $PWD/tests/tmp/:/data \
     -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_KEY \
     -it --rm wflow-test /bin/bash -c \
-    'python3 /app/src/read_stac.py "https://stac.intertwin.fedcloud.eu/collections/8db57c23-4013-45d3-a2f5-a73abf64adc4_WFLOW_FORCINGS_STATICMAPS" "data" \
+    'python3 /app/src/read_stac.py "https://stac.intertwin.fedcloud.eu/collections/6cd6cf68-4858-43b8-8569-f81970ddf6f2_WFLOW_FORCINGS_STATICMAPS" "data" \
     && run_wflow "/data/wflow_sbm.toml" \
     && python3 /app/src/to_zarr.py "/data/forcings.nc" "/data/staticmaps.nc" "/data/run_default/output.nc" "/data"'