Skip to content

Commit 4bcad3e

Browse files
JohnnyRyan1JohnnyRyan1
committed
update
1 parent 590f59a commit 4bcad3e

32 files changed

+3760
-1309
lines changed

‎.ipynb_checkpoints/01-download-merra-longwave-checkpoint.ipynb‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": 46,
6+
"id": "8950e9c4-396b-4191-8007-dc95b33615d5",
7+
"metadata": {},
8+
"outputs": [],
9+
"source": [
10+
"\"\"\"\n",
11+
"\n",
12+
"1. Download MERRA2 data at daily interval\n",
13+
"\n",
14+
"2. Save as NetCDF4\n",
15+
" - Downward shortwave radiation (allsky) \n",
16+
" - Downward shortwave radiation (clearsky)\n",
17+
"\n",
18+
"\"\"\"\n",
19+
"\n",
20+
"# Import libraries\n",
21+
"import glob\n",
22+
"import os\n",
23+
"from datetime import datetime\n",
24+
"import numpy as np\n",
25+
"import xarray as xr\n",
26+
"import pandas as pd\n",
27+
"import netCDF4\n",
28+
"import matplotlib.pyplot as plt\n",
29+
"\n",
30+
"# Define user\n",
31+
"user = 'johnnyryan'\n",
32+
"\n",
33+
"# Define base path\n",
34+
"path = '/Users/' + user + '/Dropbox (University of Oregon)/research/feedbacks/'\n",
35+
"\n",
36+
"# Define path to links\n",
37+
"links = pd.read_csv(path + 'data/links/subset_M2T1NXRAD_5.12.4_20231217_213425_.txt', \n",
38+
" skiprows=1, sep='\\t', header=None)"
39+
]
40+
},
41+
{
42+
"cell_type": "code",
43+
"execution_count": 53,
44+
"id": "63e9c232",
45+
"metadata": {},
46+
"outputs": [],
47+
"source": [
48+
"links.rename(columns={0: \"link\"}, inplace=True)\n",
49+
"links['year'] = links['link'].str[119:123]\n",
50+
"links['month'] = links['link'].str[162:164].astype(int)\n",
51+
"\n",
52+
"# Define years\n",
53+
"years = np.arange(2000, 2024)\n",
54+
"\n",
55+
"# Filter June\n",
56+
"links_summer = links[(links['month'] > 5) & (links['month'] < 9)]"
57+
]
58+
},
59+
{
60+
"cell_type": "code",
61+
"execution_count": 54,
62+
"id": "5ecde3ed-968a-4d34-b8cd-d3978e1819ee",
63+
"metadata": {},
64+
"outputs": [],
65+
"source": [
66+
"%%capture\n",
67+
"# Loop over every link, resample to daily, and save as NetCDF\n",
68+
"for year in years:\n",
69+
" \n",
70+
" if os.path.exists(path + 'data/merra-swd/swd_' + str(year) + '.nc'):\n",
71+
" print(f'Skipping...{str(year)}')\n",
72+
" else:\n",
73+
" print(f'Processing...{str(year)}')\n",
74+
"\n",
75+
" # Make a new DataFrame\n",
76+
" link_year = links_summer[links_summer['year'] == str(year)]\n",
77+
"\n",
78+
" t = []\n",
79+
" swd_allsky = []\n",
80+
" swd_clrsky = []\n",
81+
" for j in range(len(link_year)):\n",
82+
"\n",
83+
" # Index link\n",
84+
" link = '\"' + str(link_year.iloc[j].values[0]) + '\"'\n",
85+
"\n",
86+
" # Download MERRA2 using WGET\n",
87+
" !wget --load-cookies ~/.urs_cookies --save-cookies ~/.urs_cookies --auth-no-challenge=on --keep-session-cookies --no-check-certificate --content-disposition $link --directory-prefix=temp-files -nd\n",
88+
"\n",
89+
" # Import temporary file\n",
90+
" merra = xr.open_dataset(sorted(glob.glob(path + 'repo/temp-files/*.nc'))[0])\n",
91+
" \n",
92+
" # Clean up temporary files\n",
93+
" files = glob.glob(path + 'repo/temp-files/*.nc')\n",
94+
" for f in files:\n",
95+
" os.remove(f)\n",
96+
"\n",
97+
" # Calculate daily mean\n",
98+
" swd_resample = np.mean(merra['SWGDN'], axis=0).values\n",
99+
" clrsky_resample = np.mean(merra['SWGDNCLR'], axis=0).values\n",
100+
"\n",
101+
" # Append to list\n",
102+
" swd_allsky.append(swd_resample)\n",
103+
" swd_clrsky.append(clrsky_resample)\n",
104+
" t.append(merra['time'].values[0].astype('datetime64[D]'))\n",
105+
"\n",
106+
" # Save as NetCDF\n",
107+
" ds_data = xr.Dataset(\n",
108+
" data_vars={\n",
109+
" \"swd_allsky\": ((\"time\", \"lat\", \"lon\"), np.array(swd_allsky).astype('float32')),\n",
110+
" \"swd_clrsky\": ((\"time\", \"lat\", \"lon\"), np.array(swd_clrsky).astype('float32')),\n",
111+
" },\n",
112+
"\n",
113+
" coords={\n",
114+
" \"time\": pd.DatetimeIndex(np.array(t), freq='D'),\n",
115+
" \"longitude\": (('lon',), merra['lon'].values),\n",
116+
" \"latitude\": (('lat',), merra['lat'].values), \n",
117+
" },\n",
118+
"\n",
119+
" attrs={\n",
120+
" \"Produced\": datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\"),\n",
121+
" \"Units\": 'Wm-2',\n",
122+
" \"Author\":'Johnny Ryan', \n",
123+
" \"Email\":'jryan4@uoregon.edu'\n",
124+
" },\n",
125+
" )\n",
126+
"\n",
127+
" # Save\n",
128+
" ds_data.to_netcdf(path + 'data/merra-swd/swd_' + str(year) + '.nc')\n",
129+
" "
130+
]
131+
},
132+
{
133+
"cell_type": "code",
134+
"execution_count": null,
135+
"id": "a9a40f9e",
136+
"metadata": {},
137+
"outputs": [],
138+
"source": []
139+
}
140+
],
141+
"metadata": {
142+
"kernelspec": {
143+
"display_name": "Python 3 (ipykernel)",
144+
"language": "python",
145+
"name": "python3"
146+
},
147+
"language_info": {
148+
"codemirror_mode": {
149+
"name": "ipython",
150+
"version": 3
151+
},
152+
"file_extension": ".py",
153+
"mimetype": "text/x-python",
154+
"name": "python",
155+
"nbconvert_exporter": "python",
156+
"pygments_lexer": "ipython3",
157+
"version": "3.10.4"
158+
},
159+
"vscode": {
160+
"interpreter": {
161+
"hash": "e0eb5a6df1d7c6bae117cfee1babba0e4754d93c015e52cba518523ef19a3034"
162+
}
163+
}
164+
},
165+
"nbformat": 4,
166+
"nbformat_minor": 5
167+
}

‎01-download-merra-longwave.ipynb‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": 1,
6+
"id": "8950e9c4-396b-4191-8007-dc95b33615d5",
7+
"metadata": {},
8+
"outputs": [],
9+
"source": [
10+
"\"\"\"\n",
11+
"\n",
12+
"1. Download MERRA2 data at daily interval\n",
13+
"\n",
14+
"2. Save as NetCDF4\n",
15+
" - Downward longwave radiation - LWGAB (allsky) \n",
16+
" - Downward longwave radiation - LWGABCLR (clearsky)\n",
17+
"\n",
18+
"\"\"\"\n",
19+
"\n",
20+
"# Import libraries\n",
21+
"import glob\n",
22+
"import os\n",
23+
"from datetime import datetime\n",
24+
"import numpy as np\n",
25+
"import xarray as xr\n",
26+
"import pandas as pd\n",
27+
"import netCDF4\n",
28+
"import matplotlib.pyplot as plt\n",
29+
"\n",
30+
"# Define user\n",
31+
"user = 'jryan4'\n",
32+
"\n",
33+
"# Define base path\n",
34+
"path = '/Users/' + user + '/Dropbox (University of Oregon)/research/feedbacks/'\n",
35+
"\n",
36+
"# Define path to links\n",
37+
"links = pd.read_csv(path + 'data/links/subset_M2T1NXRAD_5.12.4_20240228_175520_.txt', \n",
38+
" skiprows=1, sep='\\t', header=None)"
39+
]
40+
},
41+
{
42+
"cell_type": "code",
43+
"execution_count": 2,
44+
"id": "63e9c232",
45+
"metadata": {},
46+
"outputs": [],
47+
"source": [
48+
"links.rename(columns={0: \"link\"}, inplace=True)\n",
49+
"links['year'] = links['link'].str[119:123]\n",
50+
"links['month'] = links['link'].str[162:164].astype(int)\n",
51+
"\n",
52+
"# Define years\n",
53+
"years = np.arange(2002, 2024)\n",
54+
"\n",
55+
"# Filter June\n",
56+
"links_summer = links[(links['month'] > 5) & (links['month'] < 9)]"
57+
]
58+
},
59+
{
60+
"cell_type": "code",
61+
"execution_count": 22,
62+
"id": "5ecde3ed-968a-4d34-b8cd-d3978e1819ee",
63+
"metadata": {},
64+
"outputs": [],
65+
"source": [
66+
"%%capture\n",
67+
"# Loop over every link, resample to daily, and save as NetCDF\n",
68+
"for year in years:\n",
69+
" \n",
70+
" if os.path.exists(path + 'data/merra-lwd/lwd_' + str(year) + '.nc'):\n",
71+
" print(f'Skipping...{str(year)}')\n",
72+
" else:\n",
73+
" print(f'Processing...{str(year)}')\n",
74+
"\n",
75+
" # Make a new DataFrame\n",
76+
" link_year = links_summer[links_summer['year'] == str(year)]\n",
77+
"\n",
78+
" t = []\n",
79+
" lwd_allsky = []\n",
80+
" lwd_clrsky = []\n",
81+
" for j in range(len(link_year)):\n",
82+
"\n",
83+
" # Index link\n",
84+
" link = '\"' + str(link_year.iloc[j].values[0]) + '\"'\n",
85+
"\n",
86+
" # Download MERRA2 using WGET\n",
87+
" !wget --load-cookies ~/.urs_cookies --save-cookies ~/.urs_cookies --auth-no-challenge=on --keep-session-cookies --no-check-certificate --content-disposition $link --directory-prefix=temp-files -nd\n",
88+
"\n",
89+
" # Import temporary file\n",
90+
" merra = xr.open_dataset(sorted(glob.glob(path + 'repo/temp-files/*.nc'))[0])\n",
91+
" \n",
92+
" # Clean up temporary files\n",
93+
" files = glob.glob(path + 'repo/temp-files/*.nc')\n",
94+
" for f in files:\n",
95+
" os.remove(f)\n",
96+
"\n",
97+
" # Calculate daily mean\n",
98+
" lwd_resample = np.mean(merra['LWGAB'], axis=0).values\n",
99+
" clrsky_resample = np.mean(merra['LWGABCLR'], axis=0).values\n",
100+
"\n",
101+
" # Append to list\n",
102+
" lwd_allsky.append(lwd_resample)\n",
103+
" lwd_clrsky.append(clrsky_resample)\n",
104+
" t.append(merra['time'].values[0].astype('datetime64[D]'))\n",
105+
"\n",
106+
" # Save as NetCDF\n",
107+
" ds_data = xr.Dataset(\n",
108+
" data_vars={\n",
109+
" \"lwd_allsky\": ((\"time\", \"lat\", \"lon\"), np.array(lwd_allsky).astype('float32')),\n",
110+
" \"lwd_clrsky\": ((\"time\", \"lat\", \"lon\"), np.array(lwd_clrsky).astype('float32')),\n",
111+
" },\n",
112+
"\n",
113+
" coords={\n",
114+
" \"time\": pd.DatetimeIndex(np.array(t), freq='D'),\n",
115+
" \"longitude\": (('lon',), merra['lon'].values),\n",
116+
" \"latitude\": (('lat',), merra['lat'].values), \n",
117+
" },\n",
118+
"\n",
119+
" attrs={\n",
120+
" \"Produced\": datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\"),\n",
121+
" \"Units\": 'Wm-2',\n",
122+
" \"Author\":'Johnny Ryan', \n",
123+
" \"Email\":'jryan4@uoregon.edu'\n",
124+
" },\n",
125+
" )\n",
126+
"\n",
127+
" # Save\n",
128+
" ds_data.to_netcdf(path + 'data/merra-lwd/lwd_' + str(year) + '.nc')\n",
129+
" "
130+
]
131+
},
132+
{
133+
"cell_type": "code",
134+
"execution_count": null,
135+
"id": "88301d2a",
136+
"metadata": {},
137+
"outputs": [],
138+
"source": []
139+
}
140+
],
141+
"metadata": {
142+
"kernelspec": {
143+
"display_name": "Python 3 (ipykernel)",
144+
"language": "python",
145+
"name": "python3"
146+
},
147+
"language_info": {
148+
"codemirror_mode": {
149+
"name": "ipython",
150+
"version": 3
151+
},
152+
"file_extension": ".py",
153+
"mimetype": "text/x-python",
154+
"name": "python",
155+
"nbconvert_exporter": "python",
156+
"pygments_lexer": "ipython3",
157+
"version": "3.10.4"
158+
},
159+
"vscode": {
160+
"interpreter": {
161+
"hash": "e0eb5a6df1d7c6bae117cfee1babba0e4754d93c015e52cba518523ef19a3034"
162+
}
163+
}
164+
},
165+
"nbformat": 4,
166+
"nbformat_minor": 5
167+
}

‎01-download-mod10a1.txt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,12 @@
33
# Configure your username and password for authentication using a .netrc file
44
cd ~
55
touch .netrc
6-
echo "machine urs.earthdata.nasa.gov login johnnyryan password Northbar39" >> .netrc
6+
echo "machine urs.earthdata.nasa.gov login johnnyryan1607 password Elephant12#!" >> .netrc
77
chmod 0600 .netrc
88

99
# Create a cookie file
1010
cd ~
1111
touch .urs_cookies
1212

1313
# Use a WGET command to download your data.
14-
wget -nc --load-cookies ~/.urs_cookies --save-cookies ~/.urs_cookies --keep-session-cookies --no-check-certificate --auth-no-challenge=on -r --reject "index.html*" -np -e robots=off -i /Users/johnnyryan/Dropbox\ \(University\ of\ Oregon\)/research/feedbacks/data/links/h17v02.txt --directory-prefix=/Users/johnnyryan/Dropbox\ \(University\ of\ Oregon\)/research/feedbacks/data/mod10a1/ -nd
14+
wget -nc --load-cookies ~/.urs_cookies --save-cookies ~/.urs_cookies --keep-session-cookies --no-check-certificate --auth-no-challenge=on -r --reject "index.html*" -np -e robots=off -i /Users/jryan4/Dropbox\ \(University\ of\ Oregon\)/research/antarctica/data/links/3964461709-download.txt --directory-prefix=/Users/jryan4/Dropbox\ \(University\ of\ Oregon\)/research/antarctica/data/mod10a2 -nd

0 commit comments

Comments
 (0)