Scenario Failure: ramona_benin #285
Description
Benchmark scenario ID: ramona_benin
Benchmark scenario definition: https://github.com/ESA-APEx/apex_algorithms/blob/44c2d7db930923caefd0accd98ebc4276b992b75/algorithm_catalog/dhi/RAMONA-herbaceous_rangeland_biomass-country-mosaick/benchmark_scenarios/RAMONA-herbaceous_rangeland_biomass-country-mosaick.json
openEO backend: openeo.dataspace.copernicus.eu
GitHub Actions workflow run: https://github.com/ESA-APEx/apex_algorithms/actions/runs/20004630144
Workflow artifacts: https://github.com/ESA-APEx/apex_algorithms/actions/runs/20004630144#artifacts
Test start: 2025-12-07 13:04:03.456903+00:00
Test duration: 0:07:54.072228
Test outcome: ❌ failed
Last successful test phase: download-reference
Failure in test phase: compare:derived_from-change
Contact Information
| Name | Organization | Contact |
|---|---|---|
| The RAMONA project consortium: University of Aarhus, DHI, Lund University and GeoVille. | University of Aarhus, DHI, Lund University and GeoVille. | Contact via DHI (Ramona Website, GitHub) |
Process Graph
{
"RAMONA-herbaceous_rangeland_biomass-country-mosaick": {
"process_id": "RAMONA-herbaceous_rangeland_biomass-country-mosaick",
"arguments": {
"year": "2022",
"month": "06",
"country": "benin"
},
"result": true,
"namespace": "https://raw.githubusercontent.com/ESA-APEx/apex_algorithms/8efbe040499e0108a6ed7fce5ca92a3f08115a7f/algorithm_catalog/dhi/RAMONA-herbaceous_rangeland_biomass-country-mosaick/openeo_udp/RAMONA-herbaceous_rangeland_biomass-country-mosaick.json"
}
}Error Logs
scenario = BenchmarkScenario(id='ramona_benin', description='extract for Benin', backend='openeo.dataspace.copernicus.eu', proces...ceous_rangeland_biomass-country-mosaick/benchmark_scenarios/RAMONA-herbaceous_rangeland_biomass-country-mosaick.json'))
connection_factory = <function connection_factory.<locals>.get_connection at 0x7f23fe479080>
tmp_path = PosixPath('/home/runner/work/apex_algorithms/apex_algorithms/qa/benchmarks/tmp_path_root/test_run_benchmark_ramona_beni0')
track_metric = <function track_metric.<locals>.track at 0x7f23fe4791c0>
track_phase = <function track_phase.<locals>.track at 0x7f23fe479300>
upload_assets_on_fail = <function upload_assets_on_fail.<locals>.collect at 0x7f23fe4793a0>
request = <FixtureRequest for <Function test_run_benchmark[ramona_benin]>>
@pytest.mark.parametrize(
"scenario",
[
# Use scenario id as parameterization id to give nicer test names.
pytest.param(uc, id=uc.id)
for uc in get_benchmark_scenarios()
],
)
def test_run_benchmark(
scenario: BenchmarkScenario,
connection_factory,
tmp_path: Path,
track_metric,
track_phase,
upload_assets_on_fail,
request,
):
track_metric("scenario_id", scenario.id)
with track_phase(phase="connect"):
# Check if a backend override has been provided via cli options.
override_backend = request.config.getoption("--override-backend")
backend_filter = request.config.getoption("--backend-filter")
if backend_filter and not re.match(backend_filter, scenario.backend):
# TODO apply filter during scenario retrieval, but seems to be hard to retrieve cli param
pytest.skip(
f"skipping scenario {scenario.id} because backend {scenario.backend} does not match filter {backend_filter!r}"
)
backend = scenario.backend
if override_backend:
_log.info(f"Overriding backend URL with {override_backend!r}")
backend = override_backend
connection: openeo.Connection = connection_factory(url=backend)
with track_phase(phase="create-job"):
# TODO #14 scenario option to use synchronous instead of batch job mode?
job = connection.create_job(
process_graph=scenario.process_graph,
title=f"APEx benchmark {scenario.id}",
additional=scenario.job_options,
)
track_metric("job_id", job.job_id)
with track_phase(phase="run-job"):
# TODO: monitor timing and progress
# TODO: abort excessively long batch jobs? https://github.com/Open-EO/openeo-python-client/issues/589
job.start_and_wait()
# TODO: separate "job started" and run phases?
with track_phase(phase="collect-metadata"):
collect_metrics_from_job_metadata(job, track_metric=track_metric)
results = job.get_results()
collect_metrics_from_results_metadata(results, track_metric=track_metric)
with track_phase(phase="download-actual"):
# Download actual results
actual_dir = tmp_path / "actual"
paths = results.download_files(target=actual_dir, include_stac_metadata=True)
# Upload assets on failure
upload_assets_on_fail(*paths)
with track_phase(phase="download-reference"):
reference_dir = download_reference_data(
scenario=scenario, reference_dir=tmp_path / "reference"
)
with track_phase(
phase="compare", describe_exception=analyse_results_comparison_exception
):
# Compare actual results with reference data
> assert_job_results_allclose(
actual=actual_dir,
expected=reference_dir,
tmp_path=tmp_path,
rtol=scenario.reference_options.get("rtol", 1e-6),
atol=scenario.reference_options.get("atol", 1e-6),
pixel_tolerance=scenario.reference_options.get("pixel_tolerance", 0.0),
)
tests/test_benchmarks.py:95:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
actual = PosixPath('/home/runner/work/apex_algorithms/apex_algorithms/qa/benchmarks/tmp_path_root/test_run_benchmark_ramona_beni0/actual')
expected = PosixPath('/home/runner/work/apex_algorithms/apex_algorithms/qa/benchmarks/tmp_path_root/test_run_benchmark_ramona_beni0/reference')
def assert_job_results_allclose(
actual: Union[BatchJob, JobResults, str, Path],
expected: Union[BatchJob, JobResults, str, Path],
*,
rtol: float = _DEFAULT_RTOL,
atol: float = _DEFAULT_ATOL,
pixel_tolerance: float = _DEFAULT_PIXELTOL,
tmp_path: Optional[Path] = None,
):
"""
Assert that two job results sets are equal (with tolerance).
:param actual: actual job results, provided as :py:class:`~openeo.rest.job.BatchJob` object,
:py:meth:`~openeo.rest.job.JobResults` object or path to directory with downloaded assets.
:param expected: expected job results, provided as :py:class:`~openeo.rest.job.BatchJob` object,
:py:meth:`~openeo.rest.job.JobResults` object or path to directory with downloaded assets.
:param rtol: relative tolerance
:param atol: absolute tolerance
:param pixel_tolerance: maximum fraction of pixels (in percent)
that is allowed to be significantly different (considering ``atol`` and ``rtol``)
:param tmp_path: root temp path to download results if needed.
It's recommended to pass pytest's `tmp_path` fixture here
:raises AssertionError: if not equal within the given tolerance
.. versionadded:: 0.31.0
.. warning::
This function is experimental and subject to change.
"""
issues = _compare_job_results(
actual, expected, rtol=rtol, atol=atol, pixel_tolerance=pixel_tolerance, tmp_path=tmp_path
)
if issues:
> raise AssertionError("\n".join(issues))
E AssertionError: Issues for metadata file 'job-results.json':
E Differing 'derived_from' links (0 common, 31 only in actual, 24 only in expected):
E only in actual: {'Masked_RAMONA_HRB_gDWm2Month_202206_ECLUE_MSIL2A_31NDJ_10m_v02', 'Masked_RAMONA_HRB_gDWm2Month_202206_ECLUE_MSIL2A_31NBJ_10m_v02', 'Masked_RAMONA_HRB_gDWm2Month_202206_ECLUE_MSIL2A_31NDH_10m_v02', 'Masked_RAMONA_HRB_gDWm2Month_202206_ECLUE_MSIL2A_31PDM_10m_v02', 'Masked_RAMONA_HRB_gDWm2Month_202206_ECLUE_MSIL2A_31PBM_10m_v02', 'Masked_RAMONA_HRB_gDWm2Month_202206_ECLUE_MSIL2A_31PCK_10m_v02', 'Masked_RAMONA_HRB_gDWm2Month_202206_ECLUE_MSIL2A_31NBH_10m_v02', 'Masked_RAMONA_HRB_gDWm2Month_202206_ECLUE_MSIL2A_31PCN_10m_v02', 'Masked_RAMONA_HRB_gDWm2Month_202206_ECLUE_MSIL2A_31NCG_10m_v02', 'Masked_RAMONA_HRB_gDWm2Month_202206_ECLUE_MSIL2A_31NCJ_10m_v02', 'Masked_RAMONA_HRB_gDWm2Month_202206_ECLUE_MSIL2A_31NBG_10m_v02', 'Masked_RAMONA_HRB_gDWm2Month_202206_ECLUE_MSIL2A_31PEL_10m_v02', 'Masked_RAMONA_HRB_gDWm2Month_202206_ECLUE_MSIL2A_31PBK_10m_v02', 'Masked_RAMONA_HRB_gDWm2Month_202206_ECLUE_MSIL2A_31PDN_10m_v02', 'Masked_RAMONA_HRB_gDWm2Month_202206_ECLUE_MSIL2A_31PDP_10m_v02', 'Maske...
E only in expected: {'/eodata/Sentinel-2/MSI/L2A_N0500/2023/08/15/S2B_MSIL2A_20230815T103629_N0510_R008_T31UFS_20241015T231831.SAFE', '/eodata/Sentinel-2/MSI/L2A_N0500/2023/08/25/S2B_MSIL2A_20230825T103629_N0510_R008_T31UFS_20241015T164304.SAFE', '/eodata/Sentinel-2/MSI/L2A_N0500/2023/09/02/S2A_MSIL2A_20230902T104631_N0510_R051_T31UFS_20241025T212417.SAFE', '/eodata/Sentinel-2/MSI/L2A_N0500/2023/08/13/S2A_MSIL2A_20230813T105031_N0510_R051_T31UFS_20241021T053115.SAFE', '/eodata/Sentinel-2/MSI/L2A_N0500/2023/08/05/S2B_MSIL2A_20230805T103629_N0510_R008_T31UFS_20241027T005114.SAFE', '/eodata/Sentinel-2/MSI/L2A_N0500/2023/09/22/S2A_MSIL2A_20230922T104741_N0510_R051_T31UFS_20241027T084641.SAFE', '/eodata/Sentinel-2/MSI/L2A_N0500/2023/09/12/S2A_MSIL2A_20230912T104631_N0510_R051_T31UFS_20241025T023820.SAFE', '/eodata/Sentinel-2/MSI/L2A_N0500/2023/09/04/S2B_MSIL2A_20230904T103629_N0510_R008_T31UFS_20241029T101947.SAFE', '/eodata/Sentinel-2/MSI/L2A_N0500/2023/08/20/S2A_MSIL2A_20230820T103631_N0510_R008_T31UFS_20....
E Issues for file 'ramona_hrb.tif':
E Coordinates mismatch for dimension 'x': [0.77801453 0.77810485 0.77819518 ... 3.84018629 3.84027662 3.84036695] != [644525. 644535. 644545. 644555. 644565. 644575. 644585. 644595. 644605.
E 644615. 644625. 644635. 644645. 644655. 644665. 644675. 644685. 644695.
E 644705. 644715. 644725. 644735. 644745. 644755. 644765. 644775. 644785.
E 644795. 644805. 644815. 644825. 644835. 644845. 644855. 644865. 644875.
E 644885. 644895. 644905. 644915. 644925. 644935. 644945. 644955. 644965.
E 644975. 644985. 644995. 645005. 645015. 645025. 645035. 645045. 645055.
E 645065. 645075. 645085. 645095. 645105. 645115. 645125. 645135. 645145.
E 645155. 645165. 645175. 645185. 645195. 645205. 645215. 645225. 645235.
E 645245. 645255. 645265. 645275. 645285. 645295. 645305. 645315. 645325.
E 645335. 645345. 645355. 645365. 645375. 645385. 645395. 645405. 645415.
E 645425. 645435. 645445. 645455. 645465. 645475. 645485. 645495. 645505.
E 645515. 645525. 645535. 645545. 645555. 645565. 645575. 645585. 645595.
E 645605. 645615. 645625. 645635. 645645. 645655. 645665. 645675. 645685.
E 645695. 645705. 645715. 645725. 645735. 645745. 645755. 645765. 645775.
E 645785. 645795. 645805. 645815. 645825. 645835. 645845. 645855. 645865.
E 645875. 645885. 645895. 645905. 645915. 645925. 645935. 645945. 645955.
E 645965. 645975. 645985.]
E Coordinates mismatch for dimension 'y': [12.40769684 12.40760651 12.40751618 ... 6.23358156 6.23349123
E 6.2334009 ] != [5677475. 5677465. 5677455. 5677445. 5677435. 5677425. 5677415. 5677405.
E 5677395. 5677385. 5677375. 5677365. 5677355. 5677345. 5677335. 5677325.
E 5677315. 5677305. 5677295. 5677285. 5677275. 5677265. 5677255. 5677245.
E 5677235. 5677225. 5677215. 5677205. 5677195. 5677185. 5677175. 5677165.
E 5677155. 5677145. 5677135. 5677125. 5677115. 5677105. 5677095. 5677085.
E 5677075. 5677065. 5677055. 5677045. 5677035. 5677025. 5677015. 5677005.
E 5676995. 5676985. 5676975. 5676965. 5676955. 5676945. 5676935. 5676925.
E 5676915. 5676905. 5676895. 5676885. 5676875. 5676865. 5676855. 5676845.
E 5676835. 5676825. 5676815. 5676805. 5676795. 5676785. 5676775. 5676765.
E 5676755. 5676745. 5676735. 5676725. 5676715. 5676705. 5676695. 5676685.
E 5676675. 5676665. 5676655. 5676645. 5676635. 5676625. 5676615. 5676605.
E 5676595. 5676585. 5676575. 5676565. 5676555. 5676545. 5676535. 5676525.
E 5676515. 5676505. 5676495. 5676485. 5676475. 5676465. 5676455. 5676445.
E 5676435. 5676425. 5676415. 5676405. 5676395. 5676385. 5676375. 5676365.
E 5676355. 5676345. 5676335. 5676325. 5676315. 5676305. 5676295. 5676285.
E 5676275. 5676265. 5676255. 5676245. 5676235. 5676225. 5676215. 5676205.
E 5676195. 5676185. 5676175. 5676165. 5676155. 5676145. 5676135. 5676125.
E 5676115. 5676105. 5676095. 5676085. 5676075. 5676065. 5676055. 5676045.
E 5676035. 5676025. 5676015. 5676005. 5675995. 5675985. 5675975. 5675965.
E 5675955. 5675945. 5675935. 5675925. 5675915. 5675905. 5675895. 5675885.
E 5675875. 5675865. 5675855. 5675845. 5675835. 5675825. 5675815. 5675805.
E 5675795. 5675785. 5675775. 5675765. 5675755. 5675745. 5675735. 5675725.
E 5675715. 5675705. 5675695. 5675685. 5675675. 5675665. 5675655. 5675645.
E 5675635. 5675625. 5675615. 5675605. 5675595. 5675585. 5675575. 5675565.
E 5675555. 5675545. 5675535. 5675525. 5675515. 5675505. 5675495. 5675485.
E 5675475. 5675465. 5675455. 5675445. 5675435. 5675425. 5675415. 5675405.
E 5675395. 5675385. 5675375. 5675365. 5675355. 5675345. 5675335. 5675325.
E 5675315. 5675305. 5675295. 5675285. 5675275. 5675265. 5675255. 5675245.
E 5675235. 5675225. 5675215.]
E Shape mismatch: (1, 68354, 33903) != (1, 227, 147)
E Left and right DataArray objects are not close
E Differing dimensions:
E (band: 1, y: 68354, x: 33903) != (band: 1, y: 227, x: 147)
E Differing values:
E L
E array([[[-32768, -32768, ..., -32768, -32768],
E [-32768, -32768, ..., -32768, -32768],
E ...,
E [-32768, -32768, ..., -32768, -32768],
E [-32768, -32768, ..., -32768, -32768]]],
E shape=(1, 68354, 33903), dtype=int16)
E R
E array([[[0.702807, 0.713594, ..., 0.66436 , 0.797876],
E [0.2981 , 0.275221, ..., 0.726756, 0.838676],
E ...,
E [0.816577, 0.828269, ..., 0.89057 , 0.879856],
E [0.825291, 0.823701, ..., 0.885344, 0.876766]]],
E shape=(1, 227, 147), dtype=float32)
E Differing coordinates:
E L * x (x) float64 271kB 0.778 0.7781 0.7782 0.7783 ... 3.84 3.84 3.84
E R * x (x) float64 1kB 6.445e+05 6.445e+05 ... 6.46e+05 6.46e+05
E L * y (y) float64 547kB 12.41 12.41 12.41 12.41 ... 6.234 6.233 6.233
E R * y (y) float64 2kB 5.677e+06 5.677e+06 ... 5.675e+06 5.675e+06
/opt/hostedtoolcache/Python/3.12.12/x64/lib/python3.12/site-packages/openeo/testing/results.py:515: AssertionError
----------------------------- Captured stdout call -----------------------------
0:00:00 Job 'j-25120713040843b7b1866af919c365e5': send 'start'
0:00:17 Job 'j-25120713040843b7b1866af919c365e5': created (progress 0%)
0:00:22 Job 'j-25120713040843b7b1866af919c365e5': created (progress 0%)
0:00:29 Job 'j-25120713040843b7b1866af919c365e5': created (progress 0%)
0:00:37 Job 'j-25120713040843b7b1866af919c365e5': created (progress 0%)
0:00:47 Job 'j-25120713040843b7b1866af919c365e5': created (progress 0%)
0:00:59 Job 'j-25120713040843b7b1866af919c365e5': created (progress 0%)
0:01:15 Job 'j-25120713040843b7b1866af919c365e5': running (progress N/A)
0:01:34 Job 'j-25120713040843b7b1866af919c365e5': running (progress N/A)
0:01:58 Job 'j-25120713040843b7b1866af919c365e5': running (progress N/A)
0:02:29 Job 'j-25120713040843b7b1866af919c365e5': running (progress N/A)
0:03:06 Job 'j-25120713040843b7b1866af919c365e5': running (progress N/A)
0:03:53 Job 'j-25120713040843b7b1866af919c365e5': running (progress N/A)
0:04:52 Job 'j-25120713040843b7b1866af919c365e5': running (progress N/A)
0:05:52 Job 'j-25120713040843b7b1866af919c365e5': finished (progress 100%)
------------------------------ Captured log call -------------------------------
INFO conftest:conftest.py:131 Connecting to 'openeo.dataspace.copernicus.eu'
INFO openeo.config:config.py:193 Loaded openEO client config from sources: []
INFO conftest:conftest.py:144 Checking for auth_env_var='OPENEO_AUTH_CLIENT_CREDENTIALS_CDSEFED' to drive auth against url='openeo.dataspace.copernicus.eu'.
INFO conftest:conftest.py:148 Extracted provider_id='CDSE' client_id='openeo-apex-benchmarks-service-account' from auth_env_var='OPENEO_AUTH_CLIENT_CREDENTIALS_CDSEFED'
INFO openeo.rest.connection:connection.py:255 Found OIDC providers: ['CDSE']
INFO openeo.rest.auth.oidc:oidc.py:404 Doing 'client_credentials' token request 'https://identity.dataspace.copernicus.eu/auth/realms/CDSE/protocol/openid-connect/token' with post data fields ['grant_type', 'client_id', 'client_secret', 'scope'] (client_id 'openeo-apex-benchmarks-service-account')
INFO openeo.rest.connection:connection.py:354 Obtained tokens: ['access_token', 'id_token']
INFO openeo.rest.job:job.py:436 Downloading Job result asset 'ramona_hrb.tif' from https://s3.waw3-1.openeo.v1.dataspace.copernicus.eu/openeo-data-prod-waw4-1/batch_jobs/j-25120713040843b7b1866af919c365e5/ramona_hrb.tif?X-Proxy-Head-As-Get=true&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=d3e0a56e35114cc7b8656595ac59d6be%2F20251207%2Fwaw4-1%2Fs3%2Faws4_request&X-Amz-Date=20251207T131002Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Security-Token=eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlX2FybiI6ImFybjpvcGVuZW93czppYW06Ojpyb2xlL29wZW5lby1kYXRhLXByb2Qtd2F3NC0xLXdvcmtzcGFjZSIsImluaXRpYWxfaXNzdWVyIjoib3BlbmVvLnByb2Qud2F3My0xLm9wZW5lby1pbnQudjEuZGF0YXNwYWNlLmNvcGVybmljdXMuZXUiLCJodHRwczovL2F3cy5hbWF6b24uY29tL3RhZ3MiOnsicHJpbmNpcGFsX3RhZ3MiOnsiam9iX2lkIjpbImotMjUxMjA3MTMwNDA4NDNiN2IxODY2YWY5MTljMzY1ZTUiXSwidXNlcl9pZCI6WyI2YTc3ZmNkMS05YzA4LTQ2ZTktYjg3NS01NGZiOTk5YWIyMDAiXX0sInRyYW5zaXRpdmVfdGFnX2tleXMiOlsidXNlcl9pZCIsImpvYl9pZCJdfSwiaXNzIjoic3RzLndhdzMtMS5vcGVuZW8udjEuZGF0YXNwYWNlLmNvcGVybmljdXMuZXUiLCJzdWIiOiJvcGVuZW8tZHJpdmVyIiwiZXhwIjoxNzY1MTE2NjAyLCJuYmYiOjE3NjUxMTMwMDIsImlhdCI6MTc2NTExMzAwMiwianRpIjoiNzAwZTUwMjItMWQyZC00YzJkLTlmN2ItYjc4Yjc4OThiOTJjIiwiYWNjZXNzX2tleV9pZCI6ImQzZTBhNTZlMzUxMTRjYzdiODY1NjU5NWFjNTlkNmJlIn0.EJJ7QdKVzKUpYHMDYHtDpFAn9yTOGMJ-3iZA5okssvAcn33cCORNmeLoJ0Af1dmX4V4Slyldq8gmSH13Fsq3_98EFqTbm6048nayXjzlyiSDJVrOYdp2t-hNrvcd8maZlCos8dzFKdGx09N_I5SAQKEOpY8TDY8eOEu_D9UtinZwR-UTcXfNd5GAEYj2rlKDT2ng_pxi1VpPTbZoSDoRzv-_LXnklRTCNsxIsEtftN0lP9nyc4bjdtsYNr9FniiaCwOrOQxkTW1Pl6HK5KZdPY1VUDcPN0J9dgh1E6HUAuxrFDWH8HbdJ07DwyASf99E-yfPMnDvIlemAiFiseeu_w&X-Amz-Signature=7c146ae9c682af7e443202e46010ee8266c1684362748344006b8a60ef728e67 to /home/runner/work/apex_algorithms/apex_algorithms/qa/benchmarks/tmp_path_root/test_run_benchmark_ramona_beni0/actual/ramona_hrb.tif
INFO apex_algorithm_qa_tools.scenarios:util.py:345 Downloading reference data for scenario.id='ramona_benin' to reference_dir=PosixPath('/home/runner/work/apex_algorithms/apex_algorithms/qa/benchmarks/tmp_path_root/test_run_benchmark_ramona_beni0/reference'): start 2025-12-07 13:11:40.687143
INFO apex_algorithm_qa_tools.scenarios:util.py:345 Downloading source='https://s3.waw3-1.cloudferro.com/apex-benchmarks/gh-15066001243!tests_test_benchmarks.py__test_run_benchmark_max_ndvi_!actual/job-results.json' to path=PosixPath('/home/runner/work/apex_algorithms/apex_algorithms/qa/benchmarks/tmp_path_root/test_run_benchmark_ramona_beni0/reference/job-results.json'): start 2025-12-07 13:11:40.687463
INFO apex_algorithm_qa_tools.scenarios:util.py:351 Downloading source='https://s3.waw3-1.cloudferro.com/apex-benchmarks/gh-15066001243!tests_test_benchmarks.py__test_run_benchmark_max_ndvi_!actual/job-results.json' to path=PosixPath('/home/runner/work/apex_algorithms/apex_algorithms/qa/benchmarks/tmp_path_root/test_run_benchmark_ramona_beni0/reference/job-results.json'): end 2025-12-07 13:11:41.557401, elapsed 0:00:00.869938
INFO apex_algorithm_qa_tools.scenarios:util.py:345 Downloading source='https://s3.waw3-1.cloudferro.com/apex-benchmarks/gh-15066001243!tests_test_benchmarks.py__test_run_benchmark_max_ndvi_!actual/openEO.tif' to path=PosixPath('/home/runner/work/apex_algorithms/apex_algorithms/qa/benchmarks/tmp_path_root/test_run_benchmark_ramona_beni0/reference/ramona_hrb.tif'): start 2025-12-07 13:11:41.557761
INFO apex_algorithm_qa_tools.scenarios:util.py:351 Downloading source='https://s3.waw3-1.cloudferro.com/apex-benchmarks/gh-15066001243!tests_test_benchmarks.py__test_run_benchmark_max_ndvi_!actual/openEO.tif' to path=PosixPath('/home/runner/work/apex_algorithms/apex_algorithms/qa/benchmarks/tmp_path_root/test_run_benchmark_ramona_beni0/reference/ramona_hrb.tif'): end 2025-12-07 13:11:42.734229, elapsed 0:00:01.176468
INFO apex_algorithm_qa_tools.scenarios:util.py:351 Downloading reference data for scenario.id='ramona_benin' to reference_dir=PosixPath('/home/runner/work/apex_algorithms/apex_algorithms/qa/benchmarks/tmp_path_root/test_run_benchmark_ramona_beni0/reference'): end 2025-12-07 13:11:42.734460, elapsed 0:00:02.047317
INFO openeo.testing.results:results.py:418 Comparing job results: PosixPath('/home/runner/work/apex_algorithms/apex_algorithms/qa/benchmarks/tmp_path_root/test_run_benchmark_ramona_beni0/actual') vs PosixPath('/home/runner/work/apex_algorithms/apex_algorithms/qa/benchmarks/tmp_path_root/test_run_benchmark_ramona_beni0/reference')