Skip to content

Commit 821a332

Browse files
HansVRPHansVRP
authored
Merge pull request #101 from ESA-APEx/hv_notebook_fix
Hv notebook fix
2 parents 4885ff4 + bae14c5 commit 821a332

File tree

7 files changed

+270
-501
lines changed

7 files changed

+270
-501
lines changed

‎algorithm_catalog/vito/eurac_pv_farm_detection/benchmark_scenarios/eurac_pv_farm_detection.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,9 @@
55
"description": "ML photovoltaic farm detection, developed by EURAC",
66
"backend": "openeofed.dataspace.copernicus.eu",
77
"process_graph": {
8-
"maxndvi1": {
8+
"pvfarm": {
99
"process_id": "eurac_pv_farm_detection",
10-
"namespace": "https://raw.githubusercontent.com/ESA-APEx/apex_algorithms/4003046e3b79ec3ab8dace888a231655db389d66/openeo_udp/eurac_pv_farm_detection/eurac_pv_farm_detection.json",
10+
"namespace": "https://raw.githubusercontent.com/ESA-APEx/apex_algorithms/refs/heads/main/algorithm_catalog/vito/eurac_pv_farm_detection/openeo_udp/eurac_pv_farm_detection.json",
1111
"arguments": {
1212
"bbox": {
1313
"east": 16.414,

‎algorithm_catalog/vito/eurac_pv_farm_detection/openeo_udp/eurac_pv_farm_detection.json‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,7 @@
172172
"from_parameter": "data"
173173
},
174174
"runtime": "Python",
175-
"udf": "import functools\nimport logging\nimport os\nimport sys\nimport zipfile\nfrom typing import Dict\n\nimport numpy as np\nimport requests\nimport xarray as xr\n\nfrom openeo.udf import inspect\n\n\n\n# TODO move standard code to UDF repo\n\n# Fixed directories for dependencies and model files\nDEPENDENCIES_DIR = \"onnx_dependencies\"\nMODEL_DIR = \"model_files\"\n\n\ndef download_file(url, path):\n \"\"\"\n Downloads a file from the given URL to the specified path.\n \"\"\"\n response = requests.get(url, stream=True)\n with open(path, \"wb\") as file:\n file.write(response.content)\n\n\ndef extract_zip(zip_path, extract_to):\n \"\"\"\n Extracts a zip file from zip_path to the specified extract_to directory.\n \"\"\"\n with zipfile.ZipFile(zip_path, \"r\") as zip_ref:\n zip_ref.extractall(extract_to)\n os.remove(zip_path) # Clean up the zip file after extraction\n\n\ndef add_directory_to_sys_path(directory):\n \"\"\"\n Adds a directory to the Python sys.path if it's not already present.\n \"\"\"\n if directory not in sys.path:\n sys.path.append(directory)\n\n@functools.lru_cache(maxsize=5)\ndef setup_model_and_dependencies(model_url, dependencies_url):\n \"\"\"\n Main function to set up the model and dependencies by downloading, extracting,\n and adding necessary directories to sys.path.\n \"\"\"\n\n inspect(message=\"Create directories\")\n # Ensure base directories exist\n os.makedirs(DEPENDENCIES_DIR, exist_ok=True)\n os.makedirs(MODEL_DIR, exist_ok=True)\n\n # Download and extract dependencies if not already present\n if not os.listdir(DEPENDENCIES_DIR):\n\n inspect(message=\"Extract dependencies\")\n zip_path = os.path.join(DEPENDENCIES_DIR, \"temp.zip\")\n download_file(dependencies_url, zip_path)\n extract_zip(zip_path, DEPENDENCIES_DIR)\n\n # Add the extracted dependencies directory to sys.path\n add_directory_to_sys_path(DEPENDENCIES_DIR)\n\n # Download and extract model if not already present\n if not os.listdir(MODEL_DIR):\n\n inspect(message=\"Extract model\")\n zip_path = os.path.join(MODEL_DIR, \"temp.zip\")\n download_file(model_url, zip_path)\n extract_zip(zip_path, MODEL_DIR)\n\n\nsetup_model_and_dependencies(\n model_url=\"https://s3.waw3-1.cloudferro.com/swift/v1/project_dependencies/EURAC_pvfarm_rf_1_median_depth_15.zip\",\n dependencies_url=\"https://s3.waw3-1.cloudferro.com/swift/v1/project_dependencies/onnx_dependencies_1.16.3.zip\",\n)\n\n# Add dependencies to the Python path\nimport onnxruntime as ort # Import after downloading dependencies\n\n\n@functools.lru_cache(maxsize=5)\ndef load_onnx_model(model_name: str) -> ort.InferenceSession:\n \"\"\"\n Loads an ONNX model from the onnx_models folder and returns an ONNX runtime session.\n\n \"\"\"\n # The onnx_models folder contains the content of the model archive provided in the job options\n return ort.InferenceSession(\n f\"{MODEL_DIR}/{model_name}\", providers=[\"CPUExecutionProvider\"]\n )\n\n\ndef preprocess_input(\n input_xr: xr.DataArray, ort_session: ort.InferenceSession\n) -> tuple:\n \"\"\"\n Preprocess the input DataArray by ensuring the dimensions are in the correct order,\n reshaping it, and returning the reshaped numpy array and the original shape.\n \"\"\"\n input_xr = input_xr.transpose(\"y\", \"x\", \"bands\")\n input_shape = input_xr.shape\n input_np = input_xr.values.reshape(-1, ort_session.get_inputs()[0].shape[1])\n input_np = input_np.astype(np.float32)\n return input_np, input_shape\n\n\ndef run_inference(input_np: np.ndarray, ort_session: ort.InferenceSession) -> tuple:\n \"\"\"\n Run inference using the ONNX runtime session and return predicted labels and probabilities.\n \"\"\"\n ort_inputs = {ort_session.get_inputs()[0].name: input_np}\n ort_outputs = ort_session.run(None, ort_inputs)\n predicted_labels = ort_outputs[0]\n return predicted_labels\n\n\ndef postprocess_output(predicted_labels: np.ndarray, input_shape: tuple) -> tuple:\n \"\"\"\n Postprocess the output by reshaping the predicted labels and probabilities into the original spatial structure.\n \"\"\"\n predicted_labels = predicted_labels.reshape(input_shape[0], input_shape[1])\n\n return predicted_labels\n\n\ndef create_output_xarray(\n predicted_labels: np.ndarray, input_xr: xr.DataArray\n) -> xr.DataArray:\n \"\"\"\n Create an xarray DataArray with predicted labels and probabilities stacked along the bands dimension.\n \"\"\"\n\n return xr.DataArray(\n predicted_labels,\n dims=[\"y\", \"x\"],\n coords={\"y\": input_xr.coords[\"y\"], \"x\": input_xr.coords[\"x\"]},\n )\n\n\ndef apply_model(input_xr: xr.DataArray) -> xr.DataArray:\n \"\"\"\n Run inference on the given input data using the provided ONNX runtime session.\n This method is called for each timestep in the chunk received by apply_datacube.\n \"\"\"\n\n # Step 1: Load the ONNX model\n inspect(message=\"load onnx model\")\n ort_session = load_onnx_model(\"EURAC_pvfarm_rf_1_median_depth_15.onnx\")\n\n # Step 2: Preprocess the input\n inspect(message=\"preprocess input\")\n input_np, input_shape = preprocess_input(input_xr, ort_session)\n\n # Step 3: Perform inference\n inspect(message=\"run model inference\")\n predicted_labels = run_inference(input_np, ort_session)\n\n # Step 4: Postprocess the output\n inspect(message=\"post process output\")\n predicted_labels = postprocess_output(predicted_labels, input_shape)\n\n # Step 5: Create the output xarray\n inspect(message=\"create output xarray\")\n return create_output_xarray(predicted_labels, input_xr)\n\n\ndef apply_datacube(cube: xr.DataArray, context: Dict) -> xr.DataArray:\n \"\"\"\n Function that is called for each chunk of data that is processed.\n The function name and arguments are defined by the UDF API.\n \"\"\"\n # Define how you want to handle nan values\n cube = cube.fillna(-999999)\n\n # Apply the model for each timestep in the chunk\n output_data = apply_model(cube)\n\n return output_data\n"
175+
"udf": "\nimport functools\nfrom typing import Dict\nimport sys\nimport numpy as np\nimport xarray as xr\nfrom openeo.udf import inspect\n\nsys.path.append(\"onnx_deps\") \nimport onnxruntime as ort\n\n\n\n@functools.lru_cache(maxsize=1)\ndef load_onnx_model(model_name: str) -> ort.InferenceSession:\n \"\"\"\n Loads an ONNX model from the onnx_models folder and returns an ONNX runtime session.\n\n \"\"\"\n # The onnx_models folder contains the content of the model archive provided in the job options\n return ort.InferenceSession(f\"onnx_models/{model_name}\")\n\n\n\ndef preprocess_input(\n input_xr: xr.DataArray, ort_session: ort.InferenceSession\n) -> tuple:\n \"\"\"\n Preprocess the input DataArray by ensuring the dimensions are in the correct order,\n reshaping it, and returning the reshaped numpy array and the original shape.\n \"\"\"\n input_xr = input_xr.transpose(\"y\", \"x\", \"bands\")\n input_shape = input_xr.shape\n input_np = input_xr.values.reshape(-1, ort_session.get_inputs()[0].shape[1])\n input_np = input_np.astype(np.float32)\n return input_np, input_shape\n\n\ndef run_inference(input_np: np.ndarray, ort_session: ort.InferenceSession) -> tuple:\n \"\"\"\n Run inference using the ONNX runtime session and return predicted labels and probabilities.\n \"\"\"\n ort_inputs = {ort_session.get_inputs()[0].name: input_np}\n ort_outputs = ort_session.run(None, ort_inputs)\n predicted_labels = ort_outputs[0]\n return predicted_labels\n\n\ndef postprocess_output(predicted_labels: np.ndarray, input_shape: tuple) -> tuple:\n \"\"\"\n Postprocess the output by reshaping the predicted labels and probabilities into the original spatial structure.\n \"\"\"\n predicted_labels = predicted_labels.reshape(input_shape[0], input_shape[1])\n\n return predicted_labels\n\n\ndef create_output_xarray(\n predicted_labels: np.ndarray, input_xr: xr.DataArray\n) -> xr.DataArray:\n \"\"\"\n Create an xarray DataArray with predicted labels and probabilities stacked along the bands dimension.\n \"\"\"\n\n return xr.DataArray(\n predicted_labels,\n dims=[\"y\", \"x\"],\n coords={\"y\": input_xr.coords[\"y\"], \"x\": input_xr.coords[\"x\"]},\n )\n\n\ndef apply_model(input_xr: xr.DataArray) -> xr.DataArray:\n \"\"\"\n Run inference on the given input data using the provided ONNX runtime session.\n This method is called for each timestep in the chunk received by apply_datacube.\n \"\"\"\n\n # Step 1: Load the ONNX model\n inspect(message=\"load onnx model\")\n ort_session = load_onnx_model(\"EURAC_pvfarm_rf_1_median_depth_15.onnx\")\n\n # Step 2: Preprocess the input\n inspect(message=\"preprocess input\")\n input_np, input_shape = preprocess_input(input_xr, ort_session)\n\n # Step 3: Perform inference\n inspect(message=\"run model inference\")\n predicted_labels = run_inference(input_np, ort_session)\n\n # Step 4: Postprocess the output\n inspect(message=\"post process output\")\n predicted_labels = postprocess_output(predicted_labels, input_shape)\n\n # Step 5: Create the output xarray\n inspect(message=\"create output xarray\")\n return create_output_xarray(predicted_labels, input_xr)\n\n\ndef apply_datacube(cube: xr.DataArray, context: Dict) -> xr.DataArray:\n \"\"\"\n Function that is called for each chunk of data that is processed.\n The function name and arguments are defined by the UDF API.\n \"\"\"\n # Define how you want to handle nan values\n cube = cube.fillna(-999999)\n\n # Apply the model for each timestep in the chunk\n output_data = apply_model(cube)\n\n return output_data\n"
176176
},
177177
"result": true
178178
}
@@ -393,5 +393,13 @@
393393
],
394394
"optional": true
395395
}
396-
]
396+
],
397+
"default_job_options": {
398+
"python-memory": "4g",
399+
"udf-dependency-archives": [
400+
"https://s3.waw3-1.cloudferro.com/swift/v1/project_dependencies/EURAC_pvfarm_rf_1_median_depth_15.zip#onnx_models",
401+
"https://s3.waw3-1.cloudferro.com/swift/v1/project_dependencies/onnx_dependencies_1.16.3.zip#onnx_deps"
402+
]
403+
404+
}
397405
}

‎algorithm_catalog/vito/eurac_pv_farm_detection/openeo_udp/generate.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
11
# %%
2+
3+
4+
25
import json
36
from pathlib import Path
47

@@ -23,7 +26,7 @@ def generate() -> dict:
2326

2427
# load the input data
2528
conn = openeo.connect(
26-
"https://openeofed.dataspace.copernicus.eu/"
29+
"https://openeo.dataspace.copernicus.eu/"
2730
).authenticate_oidc()
2831

2932
s2_cube = conn.load_collection(
@@ -57,6 +60,7 @@ def generate() -> dict:
5760
# Run ML inference to get the classification output
5861
udf = openeo.UDF.from_file(
5962
Path(__file__).parent / "udf_eurac_pvfarm_onnx.py",
63+
6064
)
6165

6266
prediction = s2_cube.reduce_bands(reducer=udf)
@@ -88,3 +92,6 @@ def generate() -> dict:
8892
# Save the generated process to a file
8993
with open(output_path / "eurac_pv_farm_detection.json", "w") as f:
9094
json.dump(generate(), f, indent=2)
95+
96+
97+
#%%

‎algorithm_catalog/vito/eurac_pv_farm_detection/openeo_udp/udf_eurac_pvfarm_onnx.py‎

Lines changed: 6 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -1,102 +1,24 @@
1+
12
import functools
2-
import logging
3-
import os
4-
import sys
5-
import zipfile
63
from typing import Dict
7-
4+
import sys
85
import numpy as np
9-
import requests
106
import xarray as xr
11-
127
from openeo.udf import inspect
138

9+
sys.path.append("onnx_deps")
10+
import onnxruntime as ort
1411

1512

16-
# TODO move standard code to UDF repo
1713

18-
# Fixed directories for dependencies and model files
19-
DEPENDENCIES_DIR = "onnx_dependencies"
20-
MODEL_DIR = "model_files"
21-
22-
23-
def download_file(url, path):
24-
"""
25-
Downloads a file from the given URL to the specified path.
26-
"""
27-
response = requests.get(url, stream=True)
28-
with open(path, "wb") as file:
29-
file.write(response.content)
30-
31-
32-
def extract_zip(zip_path, extract_to):
33-
"""
34-
Extracts a zip file from zip_path to the specified extract_to directory.
35-
"""
36-
with zipfile.ZipFile(zip_path, "r") as zip_ref:
37-
zip_ref.extractall(extract_to)
38-
os.remove(zip_path) # Clean up the zip file after extraction
39-
40-
41-
def add_directory_to_sys_path(directory):
42-
"""
43-
Adds a directory to the Python sys.path if it's not already present.
44-
"""
45-
if directory not in sys.path:
46-
sys.path.append(directory)
47-
48-
@functools.lru_cache(maxsize=5)
49-
def setup_model_and_dependencies(model_url, dependencies_url):
50-
"""
51-
Main function to set up the model and dependencies by downloading, extracting,
52-
and adding necessary directories to sys.path.
53-
"""
54-
55-
inspect(message="Create directories")
56-
# Ensure base directories exist
57-
os.makedirs(DEPENDENCIES_DIR, exist_ok=True)
58-
os.makedirs(MODEL_DIR, exist_ok=True)
59-
60-
# Download and extract dependencies if not already present
61-
if not os.listdir(DEPENDENCIES_DIR):
62-
63-
inspect(message="Extract dependencies")
64-
zip_path = os.path.join(DEPENDENCIES_DIR, "temp.zip")
65-
download_file(dependencies_url, zip_path)
66-
extract_zip(zip_path, DEPENDENCIES_DIR)
67-
68-
# Add the extracted dependencies directory to sys.path
69-
add_directory_to_sys_path(DEPENDENCIES_DIR)
70-
71-
# Download and extract model if not already present
72-
if not os.listdir(MODEL_DIR):
73-
74-
inspect(message="Extract model")
75-
zip_path = os.path.join(MODEL_DIR, "temp.zip")
76-
download_file(model_url, zip_path)
77-
extract_zip(zip_path, MODEL_DIR)
78-
79-
80-
setup_model_and_dependencies(
81-
model_url="https://s3.waw3-1.cloudferro.com/swift/v1/project_dependencies/EURAC_pvfarm_rf_1_median_depth_15.zip",
82-
dependencies_url="https://s3.waw3-1.cloudferro.com/swift/v1/project_dependencies/onnx_dependencies_1.16.3.zip",
83-
)
84-
85-
# Add dependencies to the Python path
86-
import onnxruntime as ort # Import after downloading dependencies
87-
88-
89-
@functools.lru_cache(maxsize=5)
14+
@functools.lru_cache(maxsize=1)
9015
def load_onnx_model(model_name: str) -> ort.InferenceSession:
9116
"""
9217
Loads an ONNX model from the onnx_models folder and returns an ONNX runtime session.
9318
9419
"""
9520
# The onnx_models folder contains the content of the model archive provided in the job options
96-
return ort.InferenceSession(
97-
f"{MODEL_DIR}/{model_name}", providers=["CPUExecutionProvider"]
98-
)
99-
21+
return ort.InferenceSession(f"onnx_models/{model_name}")
10022

10123
def preprocess_input(
10224
input_xr: xr.DataArray, ort_session: ort.InferenceSession

‎utils/openeo_udp_cost_profiling/max_ndvi_composite_profiler.csv‎

Lines changed: 0 additions & 9 deletions
This file was deleted.

0 commit comments

Comments
 (0)