Skip to content

Commit 5762a5f

Browse files
cd3593cd3593
authored
Merge pull request #33 from advanced-computing/Lab5Feb27
adding ruff
2 parents 286098e + ee62ec5 commit 5762a5f

File tree

6 files changed

+211
-132
lines changed

6 files changed

+211
-132
lines changed

‎app.py‎

Lines changed: 99 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -1,70 +1,99 @@
1-
import streamlit as st
2-
import pandas as pd
3-
import plotly.express as px
4-
5-
from data_utils import convert_units, parse_period_and_value, top_n_by_total
6-
from eia_api import fetch_all_pages
7-
8-
st.set_page_config(page_title="EIA Fuel Type Demand", layout="wide")
9-
st.title("U.S. Electricity Demand by Fuel Type (Eastern Time)")
10-
11-
# API Key Retrieval
12-
api_key = st.secrets.get("EIA_API_KEY", None)
13-
14-
# Predefine time and unit values
15-
start = st.sidebar.text_input("Start date (YYYY-MM-DD)", value="2026-02-09")
16-
end = st.sidebar.text_input("End date (YYYY-MM-DD)", value="2026-02-16")
17-
units = st.sidebar.radio("Units", ["MWh", "GWh"], horizontal=True)
18-
19-
# Adding in filters for user to choose for display
20-
top_n = st.sidebar.slider("Show top N fuel types (by total)", 1, 15, 10)
21-
filter_eastern = st.sidebar.checkbox("Filter to Eastern timezone only", value=True)
22-
23-
BASE_URL = "https://api.eia.gov/v2/electricity/rto/daily-fuel-type-data/data/"
24-
25-
@st.cache_data(show_spinner=False)
26-
def load_fuel_data(api_key: str, start: str, end: str) -> pd.DataFrame:
27-
params = {
28-
"api_key": api_key,
29-
"frequency": "daily",
30-
"data[0]": "value",
31-
"start": start,
32-
"end": end,
33-
"sort[0][column]": "period",
34-
"sort[0][direction]": "asc",
35-
"offset": 0,
36-
"length": 5000,
37-
}
38-
rows = fetch_all_pages(BASE_URL, params)
39-
return pd.json_normalize(rows)
40-
41-
with st.spinner("Loading data..."):
42-
df = load_fuel_data(api_key, start, end)
43-
44-
if df.empty:
45-
st.warning("No data returned. Check dates/API key.")
46-
st.stop()
47-
48-
df = parse_period_and_value(df)
49-
df, ycol, ylabel = convert_units(df, units)
50-
51-
# Aggregation by date and fuel type
52-
agg = (
53-
df.groupby(["period", "type-name"], as_index=False)[ycol]
54-
.sum()
55-
.rename(columns={ycol: "Demand"})
56-
) # type: ignore
57-
58-
# Keep top N fuel types by total
59-
agg = top_n_by_total(agg, "type-name", "Demand", top_n=top_n)
60-
61-
# Plot Graph
62-
fig = px.line(
63-
agg.sort_values("period"),
64-
x="period",
65-
y="Demand",
66-
color="type-name",
67-
title=f"Electricity demand by fuel type ({start} to {end})",
68-
labels={"period": "Date", "Demand": ylabel, "type-name": "Fuel type"},
69-
)
70-
st.plotly_chart(fig, use_container_width=True)
1+
import streamlit as st
2+
import pandas as pd
3+
import requests
4+
import plotly.express as px
5+
6+
st.set_page_config(page_title="EIA Fuel Type Demand", layout="wide")
7+
st.title("U.S. Electricity Demand by Fuel Type (Eastern Time)")
8+
9+
# API Key Retrieval
10+
api_key = st.secrets.get("EIA_API_KEY", None)
11+
12+
# Predefine time and unit values
13+
start = st.sidebar.text_input("Start date (YYYY-MM-DD)", value="2026-02-09")
14+
end = st.sidebar.text_input("End date (YYYY-MM-DD)", value="2026-02-16")
15+
units = st.sidebar.radio("Units", ["MWh", "GWh"], horizontal=True)
16+
17+
# Adding in filters for user to choose for display
18+
top_n = st.sidebar.slider("Show top N fuel types (by total)", 1, 15, 10)
19+
filter_eastern = st.sidebar.checkbox("Filter to Eastern timezone only", value=True)
20+
21+
BASE_URL = "https://api.eia.gov/v2/electricity/rto/daily-fuel-type-data/data/"
22+
23+
24+
def fetch_all_pages(base_url: str, params: dict) -> list:
25+
all_rows = []
26+
offset = 0
27+
length = params.get("length", 5000)
28+
29+
while True:
30+
params["offset"] = offset
31+
r = requests.get(base_url, params=params, timeout=60)
32+
r.raise_for_status()
33+
payload = r.json()
34+
rows = payload.get("response", {}).get("data", [])
35+
all_rows.extend(rows)
36+
if len(rows) < length:
37+
break
38+
offset += length
39+
40+
return all_rows
41+
42+
43+
@st.cache_data(show_spinner=False)
44+
def load_fuel_data(api_key: str, start: str, end: str) -> pd.DataFrame:
45+
params = {
46+
"api_key": api_key,
47+
"frequency": "daily",
48+
"data[0]": "value",
49+
"start": start,
50+
"end": end,
51+
"sort[0][column]": "period",
52+
"sort[0][direction]": "asc",
53+
"offset": 0,
54+
"length": 5000,
55+
}
56+
rows = fetch_all_pages(BASE_URL, params)
57+
return pd.json_normalize(rows)
58+
59+
60+
with st.spinner("Loading data..."):
61+
df = load_fuel_data(api_key, start, end)
62+
63+
if df.empty:
64+
st.warning("No data returned. Check dates/API key.")
65+
st.stop()
66+
67+
# Ignoring Na values
68+
df["period"] = pd.to_datetime(df["period"], errors="coerce")
69+
df["value"] = pd.to_numeric(df["value"], errors="coerce")
70+
71+
# Convert units
72+
ycol = "value"
73+
ylabel = "Demand (MWh)"
74+
if units == "GWh":
75+
df["value_gwh"] = df["value"] / 1000.0
76+
ycol = "value_gwh"
77+
ylabel = "Demand (GWh)"
78+
79+
# Aggregation by date and fuel type
80+
agg = (
81+
df.groupby(["period", "type-name"], as_index=False)[ycol]
82+
.sum()
83+
.rename(columns={ycol: "Demand"})
84+
)
85+
86+
# Keep top N fuel types by total
87+
top_fuels = agg.groupby("type-name")["Demand"].sum().nlargest(top_n).index
88+
agg = agg[agg["type-name"].isin(top_fuels)].copy()
89+
90+
# Plot Graph
91+
fig = px.line(
92+
agg.sort_values("period"),
93+
x="period",
94+
y="Demand",
95+
color="type-name",
96+
title=f"Electricity demand by fuel type ({start} to {end})",
97+
labels={"period": "Date", "Demand": ylabel, "type-name": "Fuel type"},
98+
)
99+
st.plotly_chart(fig, use_container_width=True)

‎eia_api.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,9 @@
33
import requests
44

55

6-
def fetch_all_pages(base_url: str, params: dict[str, Any], timeout: int = 60) -> list[dict[str, Any]]:
6+
def fetch_all_pages(
7+
base_url: str, params: dict[str, Any], timeout: int = 60
8+
) -> list[dict[str, Any]]:
79
all_rows: list[dict[str, Any]] = []
810
offset = 0
911
length = params.get("length", 5000)

‎mainPage.py‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
1-
import streamlit as st
2-
3-
# Define the pages
4-
main_page = st.Page("app.py", title="EIA Fuel Type Demand")
5-
page_2 = st.Page("region.py", title="EIA Region Demand")
6-
7-
# Set up navigation
8-
pg = st.navigation([main_page, page_2])
9-
10-
# Run the selected page
11-
pg.run()
1+
import streamlit as st
2+
3+
# Define the pages
4+
main_page = st.Page("app.py", title="EIA Fuel Type Demand")
5+
page_2 = st.Page("region.py", title="EIA Region Demand")
6+
7+
# Set up navigation
8+
pg = st.navigation([main_page, page_2])
9+
10+
# Run the selected page
11+
pg.run()

‎region.py‎

Lines changed: 88 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1,64 +1,104 @@
11
import streamlit as st
22
import pandas as pd
33
import plotly.express as px
4+
import requests
5+
6+
from data_utils import (
7+
parse_period_and_value,
8+
top_n_by_total,
9+
)
10+
# from eia_api import fetch_all_pages
11+
12+
st.set_page_config(page_title="EIA Demand by Region (ET)", layout="wide")
13+
st.title("U.S. Electricity Demand by Region (Eastern Time)")
14+
15+
# API Key Retrieval
16+
api_key = st.secrets.get("EIA_API_KEY", None)
17+
18+
# Predefine time and unit values
19+
start = st.sidebar.text_input("Start date (YYYY-MM-DD)", value="2026-02-09")
20+
end = st.sidebar.text_input("End date (YYYY-MM-DD)", value="2026-02-16")
21+
units = st.sidebar.radio("Units", ["MWh", "GWh"], horizontal=True)
422

5-
from data_utils import convert_units, filter_to_timezone, parse_period_and_value, top_n_by_total
6-
from eia_api import fetch_all_pages
7-
8-
st.set_page_config(page_title="EIA Demand by Region (ET)", layout="wide")
9-
st.title("U.S. Electricity Demand by Region (Eastern Time)")
10-
11-
# API Key Retrieval
12-
api_key = st.secrets.get("EIA_API_KEY", None)
13-
14-
# Predefine time and unit values
15-
start = st.sidebar.text_input("Start date (YYYY-MM-DD)", value="2026-02-09")
16-
end = st.sidebar.text_input("End date (YYYY-MM-DD)", value="2026-02-16")
17-
units = st.sidebar.radio("Units", ["MWh", "GWh"], horizontal=True)
18-
1923
BASE_URL = "https://api.eia.gov/v2/electricity/rto/daily-region-data/data/"
20-
21-
@st.cache_data(show_spinner=False)
22-
def load_region_data(api_key: str, start: str, end: str) -> pd.DataFrame:
23-
params = {
24-
"api_key": api_key,
25-
"frequency": "daily",
26-
"data[0]": "value",
27-
"start": start,
28-
"end": end,
29-
"sort[0][column]": "period",
30-
"sort[0][direction]": "asc",
31-
"offset": 0,
32-
"length": 5000,
33-
}
34-
rows = fetch_all_pages(BASE_URL, params)
35-
df = pd.json_normalize(rows)
36-
return df
37-
38-
with st.spinner("Loading data from EIA..."):
39-
df = load_region_data(api_key, start, end)
40-
24+
25+
26+
def fetch_all_pages(base_url: str, params: dict) -> list:
27+
# Pagination
28+
all_rows = []
29+
offset = 0
30+
length = params.get("length", 5000)
31+
32+
while True:
33+
params["offset"] = offset
34+
r = requests.get(base_url, params=params, timeout=60)
35+
r.raise_for_status()
36+
payload = r.json()
37+
rows = payload.get("response", {}).get("data", [])
38+
all_rows.extend(rows)
39+
if len(rows) < length:
40+
break
41+
offset += length
42+
43+
return all_rows
44+
45+
46+
@st.cache_data(show_spinner=False)
47+
def load_region_data(api_key: str, start: str, end: str) -> pd.DataFrame:
48+
params = {
49+
"api_key": api_key,
50+
"frequency": "daily",
51+
"data[0]": "value",
52+
"start": start,
53+
"end": end,
54+
"sort[0][column]": "period",
55+
"sort[0][direction]": "asc",
56+
"offset": 0,
57+
"length": 5000,
58+
}
59+
rows = fetch_all_pages(BASE_URL, params)
60+
df = pd.json_normalize(rows)
61+
return df
62+
63+
64+
with st.spinner("Loading data from EIA..."):
65+
df = load_region_data(api_key, start, end)
66+
4167
if df.empty:
4268
st.warning("No data returned. Double-check your dates and API key.")
4369
st.stop()
4470

4571
df = parse_period_and_value(df)
4672

47-
# Fix to Eastern Time
48-
df = filter_to_timezone(df, "eastern")
73+
# Fix to Eastern Time
74+
if "timezone" in df.columns:
75+
df = df[df["timezone"].str.lower().eq("eastern")]
76+
77+
# Convert units if needed
78+
ycol = "value"
79+
ylabel = "Demand (MWh)"
80+
if units == "GWh":
81+
df["value_gwh"] = df["value"] / 1000.0
82+
ycol = "value_gwh"
83+
ylabel = "Demand (GWh)"
4984

50-
df, ycol, ylabel = convert_units(df, units)
85+
# Choose region label
86+
region_col = (
87+
"region-name"
88+
if "region-name" in df.columns
89+
else ("region" if "region" in df.columns else None)
90+
)
5191

5292
# Plot Graph
5393
df = top_n_by_total(df, "respondent", "value", top_n=10)
5494
region_col = "respondent"
55-
56-
fig = px.line(
57-
df.sort_values("period"),
58-
x="period",
59-
y=ycol,
60-
color=region_col,
61-
title=f"U.S. electricity demand by region ({start} to {end}), Eastern Time",
62-
labels={"period": "Date", ycol: ylabel, region_col: "Region"},
63-
)
64-
st.plotly_chart(fig, use_container_width=True)
95+
96+
fig = px.line(
97+
df.sort_values("period"),
98+
x="period",
99+
y=ycol,
100+
color=region_col,
101+
title=f"U.S. electricity demand by region ({start} to {end}), Eastern Time",
102+
labels={"period": "Date", ycol: ylabel, region_col: "Region"},
103+
)
104+
st.plotly_chart(fig, use_container_width=True)

‎requirements.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,4 @@ plotly
55
numpy
66
requests
77
pytest
8+
pytest-cov

‎tests/test_data_utils.py‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,11 @@
11
import pandas as pd
22

3-
from data_utils import convert_units, filter_to_timezone, parse_period_and_value, top_n_by_total
3+
from data_utils import (
4+
convert_units,
5+
filter_to_timezone,
6+
parse_period_and_value,
7+
top_n_by_total,
8+
)
49

510

611
def test_convert_units_to_gwh_creates_scaled_column():
@@ -49,7 +54,9 @@ def test_top_n_by_total_keeps_only_largest_categories():
4954

5055

5156
def test_parse_period_and_value_converts_types():
52-
df = pd.DataFrame({"period": ["2026-02-01", "not-a-date"], "value": ["12.5", "oops"]})
57+
df = pd.DataFrame(
58+
{"period": ["2026-02-01", "not-a-date"], "value": ["12.5", "oops"]}
59+
)
5360
parsed = parse_period_and_value(df)
5461
assert parsed["period"].notna().sum() == 1
5562
assert parsed["value"].tolist()[0] == 12.5

0 commit comments

Comments
 (0)