Skip to content

Commit 088367c

Browse files
wwiederwwieder
committed
notebooks to modify regional CRUJRA forcing
1 parent 90bbcdb commit 088367c

File tree

2 files changed

+2265
-0
lines changed

2 files changed

+2265
-0
lines changed

‎tools/contrib/CRUJRA_antarctica.ipynb‎

Lines changed: 307 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,307 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "markdown",
5+
"id": "42eb1117-39e4-401c-83f6-1d91ed78fbee",
6+
"metadata": {},
7+
"source": [
8+
"## Fill CRUJRA weather over Antartica with data from GSWP3\n",
9+
"\n",
10+
"Will Wieder\n",
11+
"\n",
12+
"Sept 2024"
13+
]
14+
},
15+
{
16+
"cell_type": "code",
17+
"execution_count": 1,
18+
"id": "ea159d7e-5965-478c-886a-e05288af4c24",
19+
"metadata": {},
20+
"outputs": [],
21+
"source": [
22+
"%load_ext autoreload\n",
23+
"%autoreload 2\n",
24+
"\n",
25+
"import os\n",
26+
"from glob import glob\n",
27+
"from os.path import join\n",
28+
"\n",
29+
"import calendar\n",
30+
"\n",
31+
"import tqdm\n",
32+
"import cftime\n",
33+
"from datetime import datetime\n",
34+
"import dask\n",
35+
"\n",
36+
"import numpy as np\n",
37+
"import pandas as pd\n",
38+
"import xarray as xr\n",
39+
"\n",
40+
"import matplotlib\n",
41+
"import matplotlib.pyplot as plt\n",
42+
"import matplotlib.dates as mdates\n",
43+
"from matplotlib.pyplot import cm\n",
44+
"\n",
45+
"from distributed import wait\n",
46+
"\n",
47+
"import warnings\n",
48+
"warnings.simplefilter(\"ignore\", category=FutureWarning)\n",
49+
"\n",
50+
"%matplotlib inline"
51+
]
52+
},
53+
{
54+
"cell_type": "raw",
55+
"id": "83623a03-7463-42c9-b37e-a60c7d6e0cf2",
56+
"metadata": {},
57+
"source": [
58+
"## Dask was really slow for some reason?\n",
59+
"\n",
60+
"import dask\n",
61+
"from dask_jobqueue import PBSCluster \n",
62+
"\n",
63+
"# Create a PBS cluster object\n",
64+
"cluster = PBSCluster(\n",
65+
" job_name = 'dask-worker',\n",
66+
" cores = 1,\n",
67+
" memory = '10GiB',\n",
68+
" processes = 1,\n",
69+
" local_directory = '/local_scratch/pbs.$PBS_JOBID/dask/spill',\n",
70+
" log_directory ='/glade/derecho/scratch/wwieder/temp/dask-scratch-space',\n",
71+
" resource_spec = 'select=1:ncpus=1:mem=4GB',\n",
72+
" queue = 'casper',\n",
73+
" walltime = '1:30:00',\n",
74+
" interface = 'ext',\n",
75+
" job_extra_directives=['-m n']\n",
76+
")\n",
77+
"\n",
78+
"# Next, print the job script for our debugging: \n",
79+
"print(cluster.job_script()) "
80+
]
81+
},
82+
{
83+
"cell_type": "raw",
84+
"id": "a993e461-4d3f-4434-bb61-9c338975aa0d",
85+
"metadata": {},
86+
"source": [
87+
"# Add a client and scale up:\n",
88+
"from dask.distributed import Client\n",
89+
"client = Client(cluster)\n",
90+
"client "
91+
]
92+
},
93+
{
94+
"cell_type": "raw",
95+
"id": "503c7a0d-077f-43b0-9bc0-a7ac86b912bb",
96+
"metadata": {},
97+
"source": [
98+
"# Scale the cluster to n workers\n",
99+
"cluster.scale(2)"
100+
]
101+
},
102+
{
103+
"cell_type": "code",
104+
"execution_count": 2,
105+
"id": "dc364e98-858c-435c-ba72-7aa48a488b20",
106+
"metadata": {},
107+
"outputs": [],
108+
"source": [
109+
"# CRUJRA data\n",
110+
"Cfin = '/glade/campaign/cgd/tss/projects/TRENDY2024/inputs/three_stream/'\n",
111+
"Cftypes = ['Prec','Solr','TPQWL']\n",
112+
"Cdir_in = 'clmforc.CRUJRAv2.5_0.5x0.5.'\n",
113+
"\n",
114+
"# GSWP3 DATA\n",
115+
"Gfin = '/glade/campaign/cesm/cesmdata/inputdata/atm/datm7/atm_forcing.datm7.GSWP3.0.5d.v1.c200929/'\n",
116+
"Gftypes = ['Precip','Solar','TPHWL']\n",
117+
"Gdir_in = 'clmforc.GSWP3.c2011.0.5x0.5.'\n",
118+
"\n",
119+
"vars = ['PRECTmms','FSDS',['TBOT','PSRF','QBOT','WIND','FLDS']]\n",
120+
"debug = False\n",
121+
"\n",
122+
"out_dir = '/glade/derecho/scratch/wwieder/TRENDY2024/inputs/three_stream/'\n",
123+
"if not os.path.isdir(out_dir):\n",
124+
" os.makedirs(out_dir, exist_ok=True)"
125+
]
126+
},
127+
{
128+
"cell_type": "code",
129+
"execution_count": 21,
130+
"id": "738627a8-b69c-4f4d-bb5c-6d427fa08ebb",
131+
"metadata": {},
132+
"outputs": [
133+
{
134+
"name": "stdout",
135+
"output_type": "stream",
136+
"text": [
137+
"starting Prec\n",
138+
"crujra and GSWP3 years = 2014, 2014\n",
139+
"wrote /glade/derecho/scratch/wwieder/TRENDY2024/inputs/three_stream/clmforc.CRUJRAv2.5_0.5x0.5.Prec.2014.nc\n",
140+
"crujra and GSWP3 years = 2015, 2005\n",
141+
"wrote /glade/derecho/scratch/wwieder/TRENDY2024/inputs/three_stream/clmforc.CRUJRAv2.5_0.5x0.5.Prec.2015.nc\n",
142+
"crujra and GSWP3 years = 2016, 2006\n",
143+
"wrote /glade/derecho/scratch/wwieder/TRENDY2024/inputs/three_stream/clmforc.CRUJRAv2.5_0.5x0.5.Prec.2016.nc\n",
144+
"crujra and GSWP3 years = 2017, 2007\n",
145+
"wrote /glade/derecho/scratch/wwieder/TRENDY2024/inputs/three_stream/clmforc.CRUJRAv2.5_0.5x0.5.Prec.2017.nc\n",
146+
"crujra and GSWP3 years = 2018, 2008\n",
147+
"wrote /glade/derecho/scratch/wwieder/TRENDY2024/inputs/three_stream/clmforc.CRUJRAv2.5_0.5x0.5.Prec.2018.nc\n",
148+
"crujra and GSWP3 years = 2019, 2009\n",
149+
"wrote /glade/derecho/scratch/wwieder/TRENDY2024/inputs/three_stream/clmforc.CRUJRAv2.5_0.5x0.5.Prec.2019.nc\n",
150+
"crujra and GSWP3 years = 2020, 2010\n",
151+
"wrote /glade/derecho/scratch/wwieder/TRENDY2024/inputs/three_stream/clmforc.CRUJRAv2.5_0.5x0.5.Prec.2020.nc\n",
152+
"crujra and GSWP3 years = 2021, 2011\n",
153+
"wrote /glade/derecho/scratch/wwieder/TRENDY2024/inputs/three_stream/clmforc.CRUJRAv2.5_0.5x0.5.Prec.2021.nc\n",
154+
"crujra and GSWP3 years = 2022, 2012\n",
155+
"wrote /glade/derecho/scratch/wwieder/TRENDY2024/inputs/three_stream/clmforc.CRUJRAv2.5_0.5x0.5.Solr.2016.nc\n",
156+
"crujra and GSWP3 years = 2017, 2007\n",
157+
"wrote /glade/derecho/scratch/wwieder/TRENDY2024/inputs/three_stream/clmforc.CRUJRAv2.5_0.5x0.5.Solr.2017.nc\n",
158+
"crujra and GSWP3 years = 2018, 2008\n",
159+
"wrote /glade/derecho/scratch/wwieder/TRENDY2024/inputs/three_stream/clmforc.CRUJRAv2.5_0.5x0.5.Solr.2018.nc\n",
160+
"crujra and GSWP3 years = 2019, 2009\n",
161+
"wrote /glade/derecho/scratch/wwieder/TRENDY2024/inputs/three_stream/clmforc.CRUJRAv2.5_0.5x0.5.Solr.2019.nc\n",
162+
"crujra and GSWP3 years = 2020, 2010\n",
163+
"wrote /glade/derecho/scratch/wwieder/TRENDY2024/inputs/three_stream/clmforc.CRUJRAv2.5_0.5x0.5.Solr.2020.nc\n",
164+
"crujra and GSWP3 years = 2021, 2011\n",
165+
"wrote /glade/derecho/scratch/wwieder/TRENDY2024/inputs/three_stream/clmforc.CRUJRAv2.5_0.5x0.5.Solr.2021.nc\n",
166+
"crujra and GSWP3 years = 2022, 2012\n",
167+
"wrote /glade/derecho/scratch/wwieder/TRENDY2024/inputs/three_stream/clmforc.CRUJRAv2.5_0.5x0.5.Solr.2022.nc\n",
168+
"crujra and GSWP3 years = 2023, 2013\n",
169+
"wrote /glade/derecho/scratch/wwieder/TRENDY2024/inputs/three_stream/clmforc.CRUJRAv2.5_0.5x0.5.Solr.2023.nc\n",
170+
"starting TPQWL\n",
171+
"crujra and GSWP3 years = 2014, 2014\n",
172+
"wrote /glade/derecho/scratch/wwieder/TRENDY2024/inputs/three_stream/clmforc.CRUJRAv2.5_0.5x0.5.TPQWL.2014.nc\n",
173+
"crujra and GSWP3 years = 2015, 2005\n",
174+
"wrote /glade/derecho/scratch/wwieder/TRENDY2024/inputs/three_stream/clmforc.CRUJRAv2.5_0.5x0.5.TPQWL.2015.nc\n",
175+
"crujra and GSWP3 years = 2016, 2006\n",
176+
"wrote /glade/derecho/scratch/wwieder/TRENDY2024/inputs/three_stream/clmforc.CRUJRAv2.5_0.5x0.5.TPQWL.2016.nc\n",
177+
"crujra and GSWP3 years = 2017, 2007\n",
178+
"wrote /glade/derecho/scratch/wwieder/TRENDY2024/inputs/three_stream/clmforc.CRUJRAv2.5_0.5x0.5.TPQWL.2017.nc\n",
179+
"crujra and GSWP3 years = 2018, 2008\n",
180+
"wrote /glade/derecho/scratch/wwieder/TRENDY2024/inputs/three_stream/clmforc.CRUJRAv2.5_0.5x0.5.TPQWL.2018.nc\n",
181+
"crujra and GSWP3 years = 2019, 2009\n",
182+
"wrote /glade/derecho/scratch/wwieder/TRENDY2024/inputs/three_stream/clmforc.CRUJRAv2.5_0.5x0.5.TPQWL.2020.nc\n",
183+
"crujra and GSWP3 years = 2021, 2011\n",
184+
"wrote /glade/derecho/scratch/wwieder/TRENDY2024/inputs/three_stream/clmforc.CRUJRAv2.5_0.5x0.5.TPQWL.2021.nc\n",
185+
"crujra and GSWP3 years = 2022, 2012\n",
186+
"wrote /glade/derecho/scratch/wwieder/TRENDY2024/inputs/three_stream/clmforc.CRUJRAv2.5_0.5x0.5.TPQWL.2022.nc\n",
187+
"crujra and GSWP3 years = 2023, 2013\n",
188+
"wrote /glade/derecho/scratch/wwieder/TRENDY2024/inputs/three_stream/clmforc.CRUJRAv2.5_0.5x0.5.TPQWL.2023.nc\n"
189+
]
190+
}
191+
],
192+
"source": [
193+
"v = 0\n",
194+
"for Cftype in Cftypes:\n",
195+
" print('starting '+Cftype)\n",
196+
" Gftype = Gftypes[v]\n",
197+
" for year in range(2014, 2024):\n",
198+
" # cycle over GSWP3 years 2005-2014 to fill out end of TRENDY time series 2023\n",
199+
" if year < 2015:\n",
200+
" gyear = year\n",
201+
" ajdTime = False\n",
202+
" else: \n",
203+
" gyear = year - 10\n",
204+
" ajdTime = True\n",
205+
" year = str(year)\n",
206+
" gyear = str(gyear)\n",
207+
" print('crujra and GSWP3 years = '+year+', '+gyear)\n",
208+
" # list of files\n",
209+
" Cfiles = []\n",
210+
" Gfiles = []\n",
211+
"\n",
212+
" Cfiles.extend(sorted(glob(join(Cfin, '*' + Cftype + \".\" + year + \"*.nc\"))))\n",
213+
" Gfiles.extend(sorted(glob(join(Gfin, Gftype, '*' + Cftype + \".\" + gyear + \"*.nc\"))))\n",
214+
"\n",
215+
" Cds = xr.open_mfdataset(Cfiles, decode_times=True, combine='by_coords', parallel=True)\n",
216+
" Gds = xr.open_mfdataset(Gfiles, decode_times=True, combine='by_coords',parallel=True)\n",
217+
"\n",
218+
" # check that lat-lon dimensions are identical\n",
219+
" if debug == True:\n",
220+
" print(Cfiles)\n",
221+
" print((Gds.LONGXY.isel(time=0) - Cds.LONGXY).min().values)\n",
222+
" print((Gds.LONGXY.isel(time=0) - Cds.LONGXY).max().values)\n",
223+
" print((Gds.LATIXY.isel(time=0) - Cds.LATIXY).min().values)\n",
224+
" print((Gds.LATIXY.isel(time=0) - Cds.LATIXY).max().values)\n",
225+
"\n",
226+
" # assign missing coords to GSWP3 (this may not be necessary?)\n",
227+
" Gds = Gds.assign_coords({'lon': Cds.lon})\n",
228+
" Gds = Gds.assign_coords({'lat': Cds.lat})\n",
229+
"\n",
230+
" # Couldn't get interp_like to work as intended, using resample and coarsen instead?\n",
231+
" # just select the nearest values for solar, time mean for others\n",
232+
" if Cftype == 'Solr':\n",
233+
" with dask.config.set(**{'array.slicing.split_large_chunks': False}):\n",
234+
" x = Gds[vars[v]].resample(time=\"6h\").nearest()\n",
235+
" else:\n",
236+
" with dask.config.set(**{'array.slicing.split_large_chunks': False}):\n",
237+
" x = Gds[vars[v]].coarsen(time=2, boundary=\"trim\").mean()\n",
238+
" \n",
239+
" if ajdTime == True :\n",
240+
" x = x.assign_coords({'time': Cds.time})\n",
241+
"\n",
242+
" #with dask.config.set(**{'array.slicing.split_large_chunks': True}):\n",
243+
" # x = Gds.PRECTmms.interp_like(Cds.PRECTmms,method='linear',assume_sorted=False)\n",
244+
"\n",
245+
" # Create the output dataset where GSWP3 is used south of 60S\n",
246+
" ds_out = Cds.copy('deep')\n",
247+
" ds_out[vars[v]] = xr.where(ds_out.lat < -60, x, ds_out[vars[v]])\n",
248+
"\n",
249+
" # transpose coords to get time first and add attributes\n",
250+
" ds_out = ds_out.transpose(\"time\", \"lat\", \"lon\")\n",
251+
" ds_out['lat'].attrs = Cds['lat'].attrs\n",
252+
" ds_out['lon'].attrs = Cds['lon'].attrs\n",
253+
" ds_out['time'].attrs = Cds['time'].attrs\n",
254+
" ds_out['lat'].attrs['_FillValue'] = 1.e36\n",
255+
" ds_out['lon'].attrs['_FillValue'] = 1.e36\n",
256+
" ds_out['time'].attrs['_FillValue'] = 1.e36\n",
257+
" ds_out['LATIXY'].attrs['_FillValue'] = 1.e36\n",
258+
" ds_out['LONGXY'].attrs['_FillValue'] = 1.e36\n",
259+
" ds_out.attrs['creation_date'] = datetime.today().strftime('%Y-%m-%d')\n",
260+
" ds_out.attrs['case_title2'] = 'merged with GSWP3 data over Antarctica' \n",
261+
" if v < 2 :\n",
262+
" ds_out[vars[v]].attrs = Cds[vars[v]].attrs\n",
263+
" ds_out[vars[v]].attrs['_FillValue'] = 1.e36\n",
264+
" else:\n",
265+
" for var in vars[v]:\n",
266+
" ds_out[var].attrs = Cds[var].attrs\n",
267+
" ds_out[var].attrs['_FillValue'] = 1.e36\n",
268+
"\n",
269+
" # Write out the new file\n",
270+
" fout = out_dir + Cdir_in + Cftype+\".\"+year+\".nc\"\n",
271+
" ds_out.to_netcdf(fout, format=\"NETCDF4\")\n",
272+
" print('wrote '+fout)\n",
273+
"\n",
274+
" v = v+1"
275+
]
276+
},
277+
{
278+
"cell_type": "code",
279+
"execution_count": null,
280+
"id": "0c893493-ff53-4b88-b08d-2bfe307f934f",
281+
"metadata": {},
282+
"outputs": [],
283+
"source": []
284+
}
285+
],
286+
"metadata": {
287+
"kernelspec": {
288+
"display_name": "NPL 2024b",
289+
"language": "python",
290+
"name": "npl-2024b"
291+
},
292+
"language_info": {
293+
"codemirror_mode": {
294+
"name": "ipython",
295+
"version": 3
296+
},
297+
"file_extension": ".py",
298+
"mimetype": "text/x-python",
299+
"name": "python",
300+
"nbconvert_exporter": "python",
301+
"pygments_lexer": "ipython3",
302+
"version": "3.11.9"
303+
}
304+
},
305+
"nbformat": 4,
306+
"nbformat_minor": 5
307+
}

0 commit comments

Comments
 (0)