Skip to content

Commit 926afe1

Browse files
iulusoyMaHaWo
iulusoy
and
MaHaWo
authored
Update jmodel: model cards, invalid value masking (#48)
* feat & fix: include model readmes, apply value to masked jmodel values, no rand in test * maintain: switch to ruff as linter and formatter * fix: fix the tests with mixed-up lat/long and incorrect geometry cropping * update dependencies (temp) * remove test for obsolte functions * remove nuts level, coordinate system selection from JModel * fix test exclusion from coverage report * remove nuts from jmodel config * change indicator for missing R0 to nan * change test to expect Nan for missing R0 * remove invalid resolution option from test data fixture: not responsibility of model backend * adjust jmodel example to nuts removal * adjust read_data function: exception handling, removal of explicit compute. adjust type hints * remove accidental uv.lock * restructure tests for reading, init * fix: remove empty card --------- Co-authored-by: Harald Mack <harald.mack@protonmail.com>
1 parent 11bf4c8 commit 926afe1

19 files changed

+691
-6241
lines changed

‎.pre-commit-config.yaml‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
repos:
2-
- repo: https://github.com/psf/black-pre-commit-mirror
3-
rev: 25.12.0
2+
- repo: https://github.com/astral-sh/ruff-pre-commit
3+
# Ruff version.
4+
rev: v0.14.10
45
hooks:
5-
- id: black
6-
- repo: https://github.com/pycqa/flake8
7-
rev: 7.3.0
8-
hooks:
9-
- id: flake8
6+
# Run the linter.
7+
- id: ruff-check
8+
# Run the formatter.
9+
- id: ruff-format
1010
- repo: https://github.com/kynan/nbstripout
1111
rev: 0.8.2
1212
hooks:

‎docs/sources/config_jmodel.json‎

Lines changed: 15 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,47 +1,45 @@
11
{
22
"graph": {
33
"setup_modeldata": {
4-
"function": "setup_modeldata",
4+
"function": "setup_modeldata",
55
"module": "./Jmodel",
6-
"input": [],
6+
"input": [],
77
"args": [],
88
"kwargs": {
9-
"input": "/path/to/input_data.nc",
10-
"output": "/path/to/output_data.nc",
9+
"input": "/path/to/input_data.nc",
10+
"output": "/path/to/output_data.nc",
1111
"r0_path": "/path/to/R0_data.csv",
1212
"run_mode": "forbidden",
1313
"grid_data_baseurl": "https://gisco-services.ec.europa.eu/distribution/v2/nuts",
14-
"nuts_level": "3",
15-
"resolution": "10M",
1614
"year": "2024",
1715
"temp_colname": "t2m",
1816
"out_colname": "R0"
1917
}
20-
},
18+
},
2119
"read_input_data": {
22-
"function": "read_input_data",
20+
"function": "read_input_data",
2321
"module": "./Jmodel",
24-
"input": ["setup_modeldata"],
22+
"input": ["setup_modeldata"],
2523
"args": [],
2624
"kwargs": {}
27-
},
25+
},
2826
"run_model": {
29-
"function": "run_model",
27+
"function": "run_model",
3028
"module": "./Jmodel",
31-
"input": ["setup_modeldata", "read_input_data"],
29+
"input": ["setup_modeldata", "read_input_data"],
3230
"args": [],
3331
"kwargs": {}
34-
},
32+
},
3533
"store_output_data": {
36-
"function": "store_output_data",
34+
"function": "store_output_data",
3735
"module": "./Jmodel",
38-
"input": ["setup_modeldata", "run_model"],
36+
"input": ["setup_modeldata", "run_model"],
3937
"args": [],
4038
"kwargs": {}
4139
}
42-
},
40+
},
4341
"execution": {
44-
"scheduler": "synchronous",
42+
"scheduler": "synchronous",
4543
"log_level": "INFO"
4644
}
4745
}

‎docs/sources/model_jmodel_python.ipynb‎

Lines changed: 52 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,6 @@
1818
"import heiplanet_models as mb\n",
1919
"from pathlib import Path\n",
2020
"import xarray as xr\n",
21-
"import rioxarray as rioxr\n",
22-
"import numpy as np\n",
2321
"from copy import deepcopy\n",
2422
"from datetime import datetime\n",
2523
"import json"
@@ -63,7 +61,7 @@
6361
"metadata": {},
6462
"outputs": [],
6563
"source": [
66-
"data_file = \"era5_data_2016-2017_allm_2t_tp_monthly_unicoords_adjlon_celsius_mm_05deg_trim_ts20250923-065745_hssc-laptop01.nc\""
64+
"data_file = \"era5_data_2016-2017_allm_2t_tp_monthly_unicoords_adjlon_celsius_mm_05deg_trim_ts20250930-105159_hssc10.nc\""
6765
]
6866
},
6967
{
@@ -99,7 +97,9 @@
9997
"metadata": {},
10098
"outputs": [],
10199
"source": [
102-
"outpath = Path.cwd().parent.parent / \"data\" / \"out\" / datetime.today().strftime(\"%Y-%m-%d\")\n",
100+
"outpath = (\n",
101+
" Path.cwd().parent.parent / \"data\" / \"out\" / datetime.today().strftime(\"%Y-%m-%d\")\n",
102+
")\n",
103103
"outpath.mkdir(parents=True, exist_ok=True)"
104104
]
105105
},
@@ -159,8 +159,8 @@
159159
"config[\"graph\"][\"setup_modeldata\"][\"kwargs\"][\"input\"] = str(path / data_file)\n",
160160
"config[\"graph\"][\"setup_modeldata\"][\"kwargs\"][\"r0_path\"] = str(path / ro_file)\n",
161161
"config[\"graph\"][\"setup_modeldata\"][\"kwargs\"][\"output\"] = str(\n",
162-
" outpath / f\"output_jmodel_europe.nc\"\n",
163-
")\n"
162+
" outpath / \"output_jmodel_europe.nc\"\n",
163+
")"
164164
]
165165
},
166166
{
@@ -178,7 +178,6 @@
178178
"metadata": {},
179179
"outputs": [],
180180
"source": [
181-
"\n",
182181
"with open(outpath / \"config_europe.json\", \"w\") as f:\n",
183182
" json.dump(config, f)"
184183
]
@@ -235,6 +234,7 @@
235234
"metadata": {},
236235
"outputs": [],
237236
"source": [
237+
"%%time\n",
238238
"computation.execute()"
239239
]
240240
},
@@ -302,7 +302,7 @@
302302
"metadata": {},
303303
"outputs": [],
304304
"source": [
305-
"data = xr.open_dataset(outpath / \"output_jmodel_europe.nc\", engine=\"rasterio\")"
305+
"data = xr.open_dataset(outpath / \"output_jmodel_europe.nc\", engine=\"netcdf4\")"
306306
]
307307
},
308308
{
@@ -336,17 +336,27 @@
336336
]
337337
},
338338
{
339-
"cell_type": "markdown",
339+
"cell_type": "code",
340+
"execution_count": null,
340341
"id": "34",
341342
"metadata": {},
343+
"outputs": [],
344+
"source": [
345+
"data.close()"
346+
]
347+
},
348+
{
349+
"cell_type": "markdown",
350+
"id": "35",
351+
"metadata": {},
342352
"source": [
343353
"## Run model in parallel mode on a global map"
344354
]
345355
},
346356
{
347357
"cell_type": "code",
348358
"execution_count": null,
349-
"id": "35",
359+
"id": "36",
350360
"metadata": {},
351361
"outputs": [],
352362
"source": [
@@ -355,20 +365,20 @@
355365
"grid_data_baseurl = None\n",
356366
"nuts_level = None\n",
357367
"resolution = None\n",
358-
"year = None\n"
368+
"year = None"
359369
]
360370
},
361371
{
362372
"cell_type": "markdown",
363-
"id": "36",
373+
"id": "37",
364374
"metadata": {},
365375
"source": [
366376
"When we pass `None` for the `nuts_level` and `resolution`, the global map data 'as is' will be used for the Jmodel"
367377
]
368378
},
369379
{
370380
"cell_type": "markdown",
371-
"id": "37",
381+
"id": "38",
372382
"metadata": {},
373383
"source": [
374384
"update the config file"
@@ -377,7 +387,7 @@
377387
{
378388
"cell_type": "code",
379389
"execution_count": null,
380-
"id": "38",
390+
"id": "39",
381391
"metadata": {},
382392
"outputs": [],
383393
"source": [
@@ -389,15 +399,13 @@
389399
"global_config[\"graph\"][\"setup_modeldata\"][\"kwargs\"][\"grid_data_baseurl\"] = (\n",
390400
" grid_data_baseurl\n",
391401
")\n",
392-
"global_config[\"graph\"][\"setup_modeldata\"][\"kwargs\"][\"nuts_level\"] = nuts_level\n",
393-
"global_config[\"graph\"][\"setup_modeldata\"][\"kwargs\"][\"resolution\"] = resolution\n",
394402
"global_config[\"graph\"][\"setup_modeldata\"][\"kwargs\"][\"year\"] = year"
395403
]
396404
},
397405
{
398406
"cell_type": "code",
399407
"execution_count": null,
400-
"id": "39",
408+
"id": "40",
401409
"metadata": {},
402410
"outputs": [],
403411
"source": [
@@ -406,7 +414,7 @@
406414
},
407415
{
408416
"cell_type": "markdown",
409-
"id": "40",
417+
"id": "41",
410418
"metadata": {},
411419
"source": [
412420
"again, save the config with the data, build a computation graph and run it. We leave out the visualization here, because we have not changed anything in that regard. We made no new output directory, so the data will land in the same output we had before. "
@@ -415,7 +423,7 @@
415423
{
416424
"cell_type": "code",
417425
"execution_count": null,
418-
"id": "41",
426+
"id": "42",
419427
"metadata": {},
420428
"outputs": [],
421429
"source": [
@@ -426,7 +434,7 @@
426434
{
427435
"cell_type": "code",
428436
"execution_count": null,
429-
"id": "42",
437+
"id": "43",
430438
"metadata": {},
431439
"outputs": [],
432440
"source": [
@@ -436,16 +444,17 @@
436444
{
437445
"cell_type": "code",
438446
"execution_count": null,
439-
"id": "43",
447+
"id": "44",
440448
"metadata": {},
441449
"outputs": [],
442450
"source": [
451+
"%%time\n",
443452
"computation_global.execute()"
444453
]
445454
},
446455
{
447456
"cell_type": "markdown",
448-
"id": "44",
457+
"id": "45",
449458
"metadata": {},
450459
"source": [
451460
"Check out the input map"
@@ -454,7 +463,7 @@
454463
{
455464
"cell_type": "code",
456465
"execution_count": null,
457-
"id": "45",
466+
"id": "46",
458467
"metadata": {},
459468
"outputs": [],
460469
"source": [
@@ -464,7 +473,7 @@
464473
{
465474
"cell_type": "code",
466475
"execution_count": null,
467-
"id": "46",
476+
"id": "47",
468477
"metadata": {},
469478
"outputs": [],
470479
"source": [
@@ -473,7 +482,7 @@
473482
},
474483
{
475484
"cell_type": "markdown",
476-
"id": "47",
485+
"id": "48",
477486
"metadata": {},
478487
"source": [
479488
"Look at the output data"
@@ -482,17 +491,17 @@
482491
{
483492
"cell_type": "code",
484493
"execution_count": null,
485-
"id": "48",
494+
"id": "49",
486495
"metadata": {},
487496
"outputs": [],
488497
"source": [
489-
"data = xr.open_dataset(output, engine=\"rasterio\", chunks=None).compute()"
498+
"data = xr.open_dataset(output, engine=\"netcdf4\", chunks=None).compute()"
490499
]
491500
},
492501
{
493502
"cell_type": "code",
494503
"execution_count": null,
495-
"id": "49",
504+
"id": "50",
496505
"metadata": {},
497506
"outputs": [],
498507
"source": [
@@ -502,16 +511,26 @@
502511
{
503512
"cell_type": "code",
504513
"execution_count": null,
505-
"id": "50",
514+
"id": "51",
506515
"metadata": {},
507516
"outputs": [],
508517
"source": [
509518
"data[\"R0\"].sel(time=\"2016-08-01\").plot(cmap=\"viridis\")"
510519
]
511520
},
521+
{
522+
"cell_type": "code",
523+
"execution_count": null,
524+
"id": "52",
525+
"metadata": {},
526+
"outputs": [],
527+
"source": [
528+
"data.close()"
529+
]
530+
},
512531
{
513532
"cell_type": "markdown",
514-
"id": "51",
533+
"id": "53",
515534
"metadata": {},
516535
"source": [
517536
"## Concluding remarks\n",
@@ -533,15 +552,15 @@
533552
{
534553
"cell_type": "code",
535554
"execution_count": null,
536-
"id": "52",
555+
"id": "54",
537556
"metadata": {},
538557
"outputs": [],
539558
"source": []
540559
}
541560
],
542561
"metadata": {
543562
"kernelspec": {
544-
"display_name": "model-be",
563+
"display_name": "onehealth-model-backend",
545564
"language": "python",
546565
"name": "python3"
547566
},
@@ -555,7 +574,7 @@
555574
"name": "python",
556575
"nbconvert_exporter": "python",
557576
"pygments_lexer": "ipython3",
558-
"version": "3.11.11"
577+
"version": "3.12.3"
559578
}
560579
},
561580
"nbformat": 4,

0 commit comments

Comments
 (0)