Skip to content

Commit c53399d

Browse files
bolliger32bolliger32
authored
Merge pull request #8 from ClimateImpactLab/feature/upload-nc-sliiders
Add netcdf versions of SLIIDERS and pyCIAM results to zenodo
2 parents dd53baa + bda96c5 commit c53399d

File tree

3 files changed

+75
-36
lines changed

3 files changed

+75
-36
lines changed

‎HISTORY.rst‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
History
22
=======
33

4-
v1.1.3 (unreleased)
4+
v1.2 (unreleased)
55
-------------------
66
* Point `data-acquisition.ipynb` to updated Zenodo deposit that fixes the dtype of `subsets` variable in `diaz2016_inputs_raw.zarr.zip` to be bool rather than int8
77
* Variable name bugfix in `data-acquisition.ipynb`
8+
* Add netcdf versions of SLIIDERS and the pyCIAM results to `upload-zenodo.ipynb`
89

910
v1.1.2
1011
------

‎notebooks/post-processing/zenodo-upload.ipynb‎

Lines changed: 66 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,17 @@
33
{
44
"cell_type": "code",
55
"execution_count": 1,
6+
"id": "42ac0087-fe1b-4088-919e-ce007e6dea8c",
7+
"metadata": {},
8+
"outputs": [],
9+
"source": [
10+
"%load_ext autoreload\n",
11+
"%autoreload 2"
12+
]
13+
},
14+
{
15+
"cell_type": "code",
16+
"execution_count": 2,
617
"id": "706cee66-6557-402d-ae97-679ed202a9fe",
718
"metadata": {
819
"tags": []
@@ -16,7 +27,7 @@
1627
},
1728
{
1829
"cell_type": "code",
19-
"execution_count": 4,
30+
"execution_count": 3,
2031
"id": "29a54dac-abdc-4a95-967f-2289fc1c9ecb",
2132
"metadata": {
2233
"tags": []
@@ -28,18 +39,33 @@
2839
"from pathlib import Path\n",
2940
"from shutil import make_archive\n",
3041
"from tempfile import TemporaryDirectory\n",
31-
"from pathlib import Path\n",
3242
"\n",
3343
"import dask.config\n",
3444
"import requests\n",
3545
"import shared\n",
46+
"import xarray as xr\n",
3647
"from sliiders import settings as sset\n",
48+
"from sliiders.io import open_zarr\n",
3749
"from zarr import ZipStore"
3850
]
3951
},
4052
{
4153
"cell_type": "code",
42-
"execution_count": 3,
54+
"execution_count": 4,
55+
"id": "f804bd08-6edb-43f6-8bbd-7a2b98281c29",
56+
"metadata": {},
57+
"outputs": [],
58+
"source": [
59+
"PATH_SLIIDERS_NC = sset.PATH_SLIIDERS.parent / (sset.PATH_SLIIDERS.stem + \".nc\")\n",
60+
"PATH_OUTPUTS_NC = shared.PATH_OUTPUTS.parent / (shared.PATH_OUTPUTS.stem + \".nc\")\n",
61+
"\n",
62+
"shared.save_dataset(shared.open_zarr(sset.PATH_SLIIDERS).load(), PATH_SLIIDERS_NC)\n",
63+
"shared.save_dataset(shared.open_zarr(sset.PATH_OUTPUTS).load(), PATH_OUTPUTS_NC)"
64+
]
65+
},
66+
{
67+
"cell_type": "code",
68+
"execution_count": 4,
4369
"id": "d37ca26f-091d-4cb6-be02-385464e0b687",
4470
"metadata": {
4571
"tags": []
@@ -48,10 +74,10 @@
4874
{
4975
"data": {
5076
"text/plain": [
51-
"<dask.config.set at 0x112a25b70>"
77+
"<dask.config.set at 0x7cfe44438890>"
5278
]
5379
},
54-
"execution_count": 3,
80+
"execution_count": 4,
5581
"metadata": {},
5682
"output_type": "execute_result"
5783
}
@@ -70,7 +96,7 @@
7096
},
7197
{
7298
"cell_type": "code",
73-
"execution_count": 6,
99+
"execution_count": 21,
74100
"id": "55a3e849-a554-49fa-882c-846acf76f3b8",
75101
"metadata": {
76102
"tags": [
@@ -79,19 +105,19 @@
79105
},
80106
"outputs": [],
81107
"source": [
82-
"ACCESS_TOKEN = environ[\"ACCESS_TOKEN\"]\n",
83-
"VERSION = \"1.1.0\"\n",
108+
"ACCESS_TOKEN = \"Q5z5IQ1m5Z9l1QS7ZYeV78IS5bqPmhzcFVo0KSNLoh2p39HRMPgFoJsCyQt5\"\n",
109+
"VERSION = \"1.2.0\"\n",
84110
"TITLES = {\n",
85111
" \"SLIIDERS\": \"SLIIDERS: Sea Level Impacts Input Dataset by Elevation, Region, and Scenario\",\n",
86-
" \"pyCIAM\": \"Estimates of Global Coastal Losses Under Multiple Sea Level Rise Scenarios\",\n",
112+
" # \"pyCIAM\": \"Estimates of Global Coastal Losses Under Multiple Sea Level Rise Scenarios\",\n",
87113
"}\n",
88-
"PYCIAM_CODE_PATH=Path(\"pyCIAM-1.1.2.zip\")\n",
89-
"SLIIDERS_CODE_PATH=Path(\"sliiders-1.1.1.zip\")"
114+
"PYCIAM_CODE_PATH = Path(\"pyCIAM-1.1.2.zip\")\n",
115+
"SLIIDERS_CODE_PATH = Path(\"/tmp/sliiders-1.2.zip\")"
90116
]
91117
},
92118
{
93119
"cell_type": "code",
94-
"execution_count": 7,
120+
"execution_count": 22,
95121
"id": "8725b84c-8a7f-4d88-96ec-849e963ef8ce",
96122
"metadata": {},
97123
"outputs": [],
@@ -101,7 +127,7 @@
101127
},
102128
{
103129
"cell_type": "code",
104-
"execution_count": 8,
130+
"execution_count": 23,
105131
"id": "956c40ee-e3e8-4257-bd0d-2c938546604d",
106132
"metadata": {},
107133
"outputs": [],
@@ -136,7 +162,7 @@
136162
},
137163
{
138164
"cell_type": "code",
139-
"execution_count": 9,
165+
"execution_count": 24,
140166
"id": "b5fcb2b8-3fb4-4789-bae0-18f7a66b7c51",
141167
"metadata": {},
142168
"outputs": [],
@@ -206,15 +232,15 @@
206232
},
207233
{
208234
"cell_type": "code",
209-
"execution_count": 10,
235+
"execution_count": 25,
210236
"id": "55188020-c02f-4fa6-bf4c-cf1cd87e1754",
211237
"metadata": {},
212238
"outputs": [],
213239
"source": [
214240
"# Files and paths\n",
215241
"ORIGINAL_PATHS = {\n",
216242
" \"SLIIDERS\": {\n",
217-
" \"products\": [shared.PATH_SLIIDERS],\n",
243+
" \"products\": [sset.PATH_SLIIDERS, PATH_SLIIDERS_NC],\n",
218244
" \"inputs\": [\n",
219245
" sset.PATH_GEOG_GTSM_SNAPPED,\n",
220246
" sset.PATH_GEOG_GTSM_STATIONS_TOTHIN,\n",
@@ -224,6 +250,7 @@
224250
" \"pyCIAM\": {\n",
225251
" \"products\": [\n",
226252
" shared.PATH_OUTPUTS,\n",
253+
" PATH_OUTPUTS_NC,\n",
227254
" shared.PATH_DIAZ_RES,\n",
228255
" shared.PATH_MOVEFACTOR_DATA,\n",
229256
" ],\n",
@@ -240,7 +267,7 @@
240267
"if PYCIAM_CODE_PATH is not None:\n",
241268
" ORIGINAL_PATHS[\"pyCIAM\"][\"source\"] = [PYCIAM_CODE_PATH]\n",
242269
"if SLIIDERS_CODE_PATH is not None:\n",
243-
" ORIGINAL_PATHS[\"SLIIDERS\"][\"source\"] = [SLIIDERS_CODE_PATH"
270+
" ORIGINAL_PATHS[\"SLIIDERS\"][\"source\"] = [SLIIDERS_CODE_PATH]"
244271
]
245272
},
246273
{
@@ -255,7 +282,7 @@
255282
},
256283
{
257284
"cell_type": "code",
258-
"execution_count": 11,
285+
"execution_count": 40,
259286
"id": "50c10e4c-b3eb-4714-8d32-486ac878d041",
260287
"metadata": {},
261288
"outputs": [],
@@ -264,7 +291,7 @@
264291
" dep = EXISTING_DEPOSITS[name]\n",
265292
" # create new deposit\n",
266293
" deposition_id = dep[\"id\"]\n",
267-
" if deposition_id == int(dep[\"links\"][\"latest\"].split(\"/\")[-1]):\n",
294+
" if \"latest_draft\" not in dep[\"links\"]:\n",
268295
" url = f\"{Z_URL}/{deposition_id}/actions/newversion\"\n",
269296
" r = requests.post(url, params=PARAMS)\n",
270297
" if r.status_code not in [200, 201]:\n",
@@ -390,7 +417,7 @@
390417
},
391418
{
392419
"cell_type": "code",
393-
"execution_count": 12,
420+
"execution_count": 41,
394421
"id": "868b1d78-5eb4-4fa6-bcfc-0dbd8c88d39b",
395422
"metadata": {},
396423
"outputs": [],
@@ -408,23 +435,28 @@
408435
},
409436
{
410437
"cell_type": "code",
411-
"execution_count": 14,
438+
"execution_count": 44,
412439
"id": "ff8703bb-c630-418e-9f2f-c62b116ee7e8",
413440
"metadata": {},
414441
"outputs": [
415442
{
416443
"name": "stdout",
417444
"output_type": "stream",
418445
"text": [
419-
"pyCIAM\n",
420-
"Uploading: gs://rhg-data/impactlab-rhg/coastal/sliiders/raw/CIAM_2016/diaz2016_inputs_raw.zarr\n",
421-
"Uploading: gs://rhg-data/impactlab-rhg/coastal/ciam_paper/data/int/slr/ar5/ar5-msl-rel-2005-quantiles.zarr\n",
422-
"Uploading: gs://rhg-data/impactlab-rhg/coastal/sliiders/int/exposure/ypk/finalized/ypk_2000_2100_20221122.zarr\n",
423-
"Uploading: gs://rhg-data/impactlab-rhg/coastal/ciam_paper/data/int/surge-lookup-v1.1-seg.zarr\n",
424-
"Uploading: gs://rhg-data/impactlab-rhg/coastal/ciam_paper/data/int/surge-lookup-v1.1-seg_adm.zarr\n",
425-
"Uploading: gs://rhg-data/impactlab-rhg/coastal/ciam_paper/results-v1.1/pyCIAM_outputs.zarr\n",
426-
"Uploading: gs://rhg-data/impactlab-rhg/coastal/ciam_paper/results-v1.1/diaz2016_outputs.zarr\n",
427-
"Uploading: gs://rhg-data/impactlab-rhg/coastal/ciam_paper/results-v1.1/suboptimal_capital_by_movefactor.zarr\n"
446+
"SLIIDERS\n"
447+
]
448+
},
449+
{
450+
"ename": "TypeError",
451+
"evalue": "string indices must be integers, not 'str'",
452+
"output_type": "error",
453+
"traceback": [
454+
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
455+
"\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)",
456+
"Cell \u001b[0;32mIn[44], line 6\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[38;5;28mprint\u001b[39m(name)\n\u001b[1;32m 4\u001b[0m this_dep \u001b[38;5;241m=\u001b[39m draft_deps[name]\n\u001b[0;32m----> 6\u001b[0m uploads[name] \u001b[38;5;241m=\u001b[39m \u001b[43mupload_file_list\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 7\u001b[0m \u001b[43m \u001b[49m\u001b[43mthis_dep\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkind\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43minputs\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43minputs/\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moverwrite\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\n\u001b[1;32m 8\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 9\u001b[0m uploads[name] \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m upload_file_list(\n\u001b[1;32m 10\u001b[0m this_dep, kind[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mproducts\u001b[39m\u001b[38;5;124m\"\u001b[39m], \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mproducts/\u001b[39m\u001b[38;5;124m\"\u001b[39m, overwrite\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m 11\u001b[0m )\n\u001b[1;32m 12\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msource\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01min\u001b[39;00m kind\u001b[38;5;241m.\u001b[39mkeys():\n",
457+
"Cell \u001b[0;32mIn[40], line 83\u001b[0m, in \u001b[0;36mupload_file_list\u001b[0;34m(deposit, flist, root, overwrite)\u001b[0m\n\u001b[1;32m 81\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mupload_file_list\u001b[39m(deposit, flist, root, overwrite\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m):\n\u001b[1;32m 82\u001b[0m out \u001b[38;5;241m=\u001b[39m []\n\u001b[0;32m---> 83\u001b[0m existing_files \u001b[38;5;241m=\u001b[39m \u001b[43m{\u001b[49m\n\u001b[1;32m 84\u001b[0m \u001b[43m \u001b[49m\u001b[43mf\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfilename\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mf\u001b[49m\n\u001b[1;32m 85\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mf\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mrequests\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdeposit\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mlinks\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfiles\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mparams\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mPARAMS\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mjson\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 86\u001b[0m \u001b[43m \u001b[49m\u001b[43m}\u001b[49m\n\u001b[1;32m 87\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m f \u001b[38;5;129;01min\u001b[39;00m flist:\n\u001b[1;32m 88\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mUploading: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mstr\u001b[39m(f)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n",
458+
"Cell \u001b[0;32mIn[40], line 84\u001b[0m, in \u001b[0;36m<dictcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 81\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mupload_file_list\u001b[39m(deposit, flist, root, overwrite\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m):\n\u001b[1;32m 82\u001b[0m out \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m 83\u001b[0m existing_files \u001b[38;5;241m=\u001b[39m {\n\u001b[0;32m---> 84\u001b[0m \u001b[43mf\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mfilename\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m]\u001b[49m: f\n\u001b[1;32m 85\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m f \u001b[38;5;129;01min\u001b[39;00m requests\u001b[38;5;241m.\u001b[39mget(deposit[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mlinks\u001b[39m\u001b[38;5;124m\"\u001b[39m][\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mfiles\u001b[39m\u001b[38;5;124m\"\u001b[39m], params\u001b[38;5;241m=\u001b[39mPARAMS)\u001b[38;5;241m.\u001b[39mjson()\n\u001b[1;32m 86\u001b[0m }\n\u001b[1;32m 87\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m f \u001b[38;5;129;01min\u001b[39;00m flist:\n\u001b[1;32m 88\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mUploading: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mstr\u001b[39m(f)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n",
459+
"\u001b[0;31mTypeError\u001b[0m: string indices must be integers, not 'str'"
428460
]
429461
}
430462
],
@@ -440,7 +472,7 @@
440472
" uploads[name] += upload_file_list(\n",
441473
" this_dep, kind[\"products\"], \"products/\", overwrite=False\n",
442474
" )\n",
443-
" if \"source\" in kind.keys(): \n",
475+
" if \"source\" in kind.keys():\n",
444476
" uploads[name] += upload_file_list(\n",
445477
" this_dep, kind[\"source\"], \"source/\", overwrite=False\n",
446478
" )"
@@ -449,9 +481,9 @@
449481
],
450482
"metadata": {
451483
"kernelspec": {
452-
"display_name": "Python [conda env:sliiders]",
484+
"display_name": "Python 3 (ipykernel)",
453485
"language": "python",
454-
"name": "conda-env-sliiders-py"
486+
"name": "python3"
455487
},
456488
"language_info": {
457489
"codemirror_mode": {
@@ -463,7 +495,7 @@
463495
"name": "python",
464496
"nbconvert_exporter": "python",
465497
"pygments_lexer": "ipython3",
466-
"version": "3.10.9"
498+
"version": "3.11.8"
467499
},
468500
"widgets": {
469501
"application/vnd.jupyter.widget-state+json": {

‎notebooks/shared.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99

1010
DIR_SCRATCH = Path("/tmp/ciam-scratch")
1111

12-
SLIIDERS_VERS = "v1.1"
12+
SLIIDERS_VERS = "v1.2"
1313
RES_VERS = "v1.1"
1414

1515
# Cloud Storage tools (will work with local storage as well but may need to be specifiec
@@ -223,6 +223,12 @@ def open_dataset(path, **kwargs):
223223
return xr.open_dataset(_path, **kwargs)
224224

225225

226+
def save_dataset(ds, path, **kwargs):
227+
_path = str(_to_fuse(path))
228+
_generate_parent_fuse_dirs(_path)
229+
return ds.to_netcdf(_path, **kwargs)
230+
231+
226232
def open_dataarray(path, **kwargs):
227233
_path = str(_to_fuse(path))
228234
_generate_parent_fuse_dirs(_path)

0 commit comments

Comments
 (0)