diff --git a/AGENTS.md b/AGENTS.md index 297ec687..2e263805 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -130,6 +130,12 @@ Match existing style: Ruff for formatting/lint, **ty** for type checking, dprint **LaTeX in markdown:** GitHub's MathJax renderer does not support escaped underscores inside `\text{}` (e.g. `\text{avg\_mc\_peak}` will fail). Use proper math symbols instead: `\overline{MC}_{\text{peak}}`, `MC_h`, `L_h`, etc. Bare subscripts and `\text{}` with simple words (no underscores) are fine. +## Git commits + +- **Never write commit messages via a temp file** (e.g. `/tmp/commit_msg.txt`). Pass the message directly with `-m "..."` or let the user commit manually. +- **Never add co-author trailers** (`Co-authored-by: ...`) or any other generated-by attribution to commit messages or PR bodies. +- **For `gh pr create` body**: use `--body-file -` with a shell heredoc (stdin) to avoid attribution injection — do NOT use `--body "..."` with multi-line strings or `--body-file /tmp/...`. Example: `gh pr create --body-file - <<'PRBODY'\n...\nPRBODY` + ## Code Quality (required before every commit) - Run `just check` — no linter errors, no type errors, no warnings diff --git a/data/resstock/Justfile b/data/resstock/Justfile index 1f35a8d3..6efe709b 100644 --- a/data/resstock/Justfile +++ b/data/resstock/Justfile @@ -304,3 +304,31 @@ create-sb-release-for-upgrade-02-RI: sudo aws s3 sync s3://data.sb/nrel/resstock/res_2024_amy2018_2_sb/ /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/ just add-monthly-loads RI "00 02" just upload-monthly-loads RI "00 02" + +# ============================================================================= +# Adoption trajectory upgrades (01, 04, 05): convenience shortcuts and end-to-end recipes +# ============================================================================= + +adjust-mf-electricity-NY-upgrade-01: + just adjust-mf-electricity NY res_2024_amy2018_2 res_2024_amy2018_2_sb "01" + +adjust-mf-electricity-NY-upgrade-04: + just adjust-mf-electricity NY res_2024_amy2018_2 res_2024_amy2018_2_sb "04" + +adjust-mf-electricity-NY-upgrade-05: + just adjust-mf-electricity NY res_2024_amy2018_2 res_2024_amy2018_2_sb "05" + +# Copy, adjust loads, and sync upgrades 01, 04, 05 into the _sb release for NY. +# Assumes prepare-metadata-ny has already been run (it processes all upgrades 00-05). +# metadata_utility (utility assignment) is upgrade-independent and is not re-copied here. + +# We are NOT running approximate-non-hp-load for upgrades 4 and 5 because they strictly only apply to certain building types. +create-sb-release-for-adoption-upgrades-NY: + just copy-resstock-data-2024-amy2018-2-NY "04 05" "metadata load_curve_hourly" + just approximate-non-hp-load NY 01 res_2024_amy2018_2 res_2024_amy2018_2_sb 15 True True + just adjust-mf-electricity-NY-upgrade-01 + just adjust-mf-electricity-NY-upgrade-04 + just adjust-mf-electricity-NY-upgrade-05 + sudo aws s3 sync s3://data.sb/nrel/resstock/res_2024_amy2018_2_sb/ /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/ + just add-monthly-loads NY "01 04 05" + just upload-monthly-loads NY "01 04 05" diff --git a/rate_design/hp_rates/Justfile b/rate_design/hp_rates/Justfile index f1d230d3..de433279 100644 --- a/rate_design/hp_rates/Justfile +++ b/rate_design/hp_rates/Justfile @@ -79,12 +79,15 @@ path_bulk_tx_mc := env_var_or_default('BULK_TX_MC', "") path_supply_energy_mc := env_var_or_default('SUPPLY_ENERGY_MC', "s3://data.sb/switchbox/marginal_costs/" + state + "/supply/energy/utility=" + utility + "/year=" + mc_year + "/data.parquet") path_supply_capacity_mc := env_var_or_default('SUPPLY_CAPACITY_MC', "s3://data.sb/switchbox/marginal_costs/" + state + "/supply/capacity/utility=" + utility + "/year=" + mc_year + "/data.parquet") path_supply_ancillary_mc := env_var_or_default('SUPPLY_ANCILLARY_MC', "") +path_resstock_root := "/ebs/data/nrel/resstock" +resstock_release_key := "res_2024_amy2018_2" path_resstock_release := "/ebs/data/nrel/resstock/res_2024_amy2018_2_sb" path_resstock_metadata := path_resstock_release + "/metadata" path_utility_assignment := path_resstock_release + "/metadata_utility" path_resstock_loads_00 := path_resstock_release + "/load_curve_hourly/state=" + state_upper + "/upgrade=" + upgrade path_mc_table := env_var_or_default('MC_TABLE', path_config / "marginal_costs" / state + "_marginal_costs_" + year + ".csv") path_s3_mc_output := "s3://data.sb/switchbox/marginal_costs/" + state + "/dist_and_sub_tx/" +path_s3_mc_output_cambium := "s3://data.sb/switchbox/marginal_costs/" + state + "/cambium_dist_and_sub_tx/" path_s3_utility_loads := "s3://data.sb/eia/hourly_demand/utilities/" path_electric_utility_stats := "s3://data.sb/eia/861/electric_utility_stats/year=2024/state=" + state_upper + "/data.parquet" path_genability := path_rev_requirement / "top-ups" @@ -337,6 +340,37 @@ create-dist-and-sub-tx-mc-data-all: UTILITY="$util" just create-dist-and-sub-tx-mc-data done +# Distribution marginal costs using Cambium busbar_load as the PoP allocation load shape. +# Writes to the cambium_dist_and_sub_tx/ prefix to avoid overwriting EIA-based dist MCs. +# +# Example: + +# just s ny create-dist-mc-cambium 2030 +create-dist-mc-cambium year_mc=year: + uv run python {{ path_repo }}/utils/pre/marginal_costs/generate_utility_tx_dx_mc.py \ + --state {{ state_upper }} \ + --utility {{ utility }} \ + --year {{ year_mc }} \ + --load-source cambium \ + --cambium-path "s3://data.sb/nrel/cambium/2024/scenario=MidCase/t={{ year_mc }}/gea=NYISO/r={{ cambium_ba }}/data.parquet" \ + --mc-table-path {{ path_mc_table }} \ + --output-s3-base {{ path_s3_mc_output_cambium }} \ + --n-hours {{ upstream_hours }} \ + --upload + +# Generate Cambium dist MCs for all adoption trajectory years (2025–2050). +# +# Example: + +# just s ny create-cambium-dist-mc-all-years +create-cambium-dist-mc-all-years: + #!/usr/bin/env bash + set -euo pipefail + for yr in 2025 2030 2035 2040 2045 2050; do + echo ">> Generating Cambium dist MC for year=${yr}" >&2 + just create-dist-mc-cambium "${yr}" + done + # ============================================================================= # MID-CONFIG: generate between runs (using outputs from earlier runs) # ============================================================================= @@ -677,6 +711,185 @@ run-subset runs: just "run-${num}" done +# ============================================================================= +# ADOPTION TRAJECTORY (mixed-upgrade) +# ============================================================================= + +path_adoption_config_dir := path_config / "adoption" + +# Fit logistic S-curves to NYISO Gold Book 2025 digitized data and write the +# adoption config YAML + a curve-fit diagnostic plot. +# +# Example: + +# just s ny fit-adoption-config nyca_electrification +fit-adoption-config config_name="nyca_electrification": + uv run python {{ path_repo }}/utils/pre/fit_adoption_config.py \ + --output "{{ path_adoption_config_dir }}/{{ config_name }}.yaml" \ + --plot-output "{{ path_adoption_config_dir }}/{{ config_name }}_curves.png" \ + --stacked-plot-output "{{ path_adoption_config_dir }}/{{ config_name }}_stacked.png" + +# Materialize per-year ResStock data for a mixed-upgrade adoption trajectory. +# Reads the adoption config YAML, assigns buildings to upgrades per year, and +# writes year=/ directories under the adoption output path. +# +# Example: + +# just s ny materialize-adoption nyca_electrification +materialize-adoption config_name="default": + uv run python {{ path_repo }}/utils/pre/materialize_mixed_upgrade.py \ + --state "{{ state }}" \ + --utility "{{ utility }}" \ + --adoption-config "{{ path_adoption_config_dir }}/{{ config_name }}.yaml" \ + --path-resstock-release "{{ path_resstock_root }}" \ + --release "{{ resstock_release_key }}" \ + --output-dir "{{ path_resstock_release }}/adoption/{{ config_name }}" + +# Generate per-year scenario YAML entries for adoption runs. +# Uses Cambium supply MCs (energy_cost_enduse / capacity_cost_enduse), Cambium +# busbar_load dist MCs, and 0% residual cost so revenue requirement = total MC. +# Output: config/scenarios/scenarios__adoption.yaml +# +# When config_name is set, --adoption-tariff-dir is passed so that runs 5/6 +# in the YAML reference per-year seasonal tariff files instead of the shared +# static tariff. Those files are written by run-adoption-all between run-2 +# and run-5 for each year. +# +# Example: + +# just s ny generate-adoption-scenarios nyca_electrification 1,2,5,6 +generate-adoption-scenarios config_name="default" runs="1,2,5,6": + uv run python {{ path_repo }}/utils/pre/generate_adoption_scenario_yamls.py \ + --base-scenario "{{ path_scenario_config }}" \ + --runs "{{ runs }}" \ + --adoption-config "{{ path_adoption_config_dir }}/{{ config_name }}.yaml" \ + --materialized-dir "{{ path_resstock_release }}/adoption/{{ config_name }}" \ + --output "{{ path_scenarios }}/scenarios_{{ utility }}_adoption.yaml" \ + --residual-cost-frac 0.0 \ + --cambium-supply \ + --cambium-gea NYISO \ + --cambium-ba {{ cambium_ba }} \ + --cambium-dist-mc-base "s3://data.sb/switchbox/marginal_costs/{{ state }}/cambium_dist_and_sub_tx" \ + --adoption-tariff-dir "{{ path_tariffs_electric }}/adoption/{{ config_name }}" + +# Run a single adoption scenario by (year-indexed) run number. +# Run keys use the scheme (year_index + 1) * 100 + base_run_num, matching the +# output of generate_adoption_scenario_yamls.py (e.g. 101, 102, 201, 202, ...). +# +# Example: + +# just s ny run-adoption-scenario 101 +run-adoption-scenario run_num: + #!/usr/bin/env bash + set -euo pipefail + : "${RDP_BATCH:?Set RDP_BATCH before running}" + export RDP_BATCH + log_dir="${HOME}/rdp_run_logs" + mkdir -p "${log_dir}" + log_file="${log_dir}/{{ utility }}_adoption_run{{ run_num }}_${RDP_BATCH}.log" + echo ">> run-adoption-scenario {{ run_num }}: logging to ${log_file}" >&2 + uv run python {{ path_repo }}/rate_design/hp_rates/run_scenario.py \ + --state "{{ state }}" \ + --scenario-config "{{ path_scenarios }}/scenarios_{{ utility }}_adoption.yaml" \ + --run-num "{{ run_num }}" \ + --output-dir "{{ path_outputs_base }}/${RDP_BATCH}" \ + 2>&1 | tee "${log_file}" + +# Orchestrate the full adoption pipeline: materialize → Cambium dist MCs → generate scenarios → run all. +# All runs use Cambium supply MCs, Cambium busbar_load dist MCs, and 0% residual cost. +# Iterates over all (year × run) combinations using the key scheme +# (year_index + 1) * 100 + base_run_num produced by generate_adoption_scenario_yamls.py. +# +# For each year the loop: +# 1. Runs precalc-flat runs (1, 2) and copies their calibrated tariffs to +# tariffs/electric/adoption//year=/. +# 2. Derives per-year seasonal tariffs from run-1/2 outputs + that year's +# mixed-upgrade loads (hive-partitioned under the materialized dir). +# 3. Runs seasonal-precalc runs (5, 6) and copies their calibrated tariffs. +# +# Example: + +# RDP_BATCH=ny_20260320_adoption just s ny run-adoption-all nyca_electrification 1,2,5,6 +run-adoption-all config_name="default" runs="1,2,5,6": + #!/usr/bin/env bash + set -euo pipefail + : "${RDP_BATCH:?Set RDP_BATCH before running}" + export RDP_BATCH + just materialize-adoption "{{ config_name }}" + just create-cambium-dist-mc-all-years + just generate-adoption-scenarios "{{ config_name }}" "{{ runs }}" + adoption_yaml="{{ path_scenarios }}/scenarios_{{ utility }}_adoption.yaml" + adoption_base="{{ path_resstock_release }}/adoption/{{ config_name }}" + tariffs_adoption_base="{{ path_tariffs_electric }}/adoption/{{ config_name }}" + # Read calendar years from the adoption config (one per line). + mapfile -t year_list < <(uv run python "{{ path_repo }}/utils/pre/list_adoption_years.py" \ + "{{ path_adoption_config_dir }}/{{ config_name }}.yaml") + IFS=',' read -ra all_runs <<< "{{ runs }}" + for yi in "${!year_list[@]}"; do + year="${year_list[$yi]}" + key_prefix=$(( (yi + 1) * 100 )) + tariff_dir="${tariffs_adoption_base}/year=${year}" + mkdir -p "${tariff_dir}" + loads_base="${adoption_base}/year=${year}" + echo ">> run-adoption-all: year=${year} (yi=${yi}, key_prefix=${key_prefix})" >&2 + # --- Runs 1 and 2: precalc flat --- + for base_run in "${all_runs[@]}"; do + [[ "${base_run}" == "1" || "${base_run}" == "2" ]] || continue + key=$(( key_prefix + base_run )) + echo ">> run-adoption-all: run-${key} (year=${year}, base_run=${base_run})" >&2 + just run-adoption-scenario "${key}" + run_dir=$(bash "{{ latest_output }}" "${adoption_yaml}" "${key}") + just copy-calibrated-tariff-from-run "${run_dir}" "${tariff_dir}" + done + # --- Derive per-year seasonal tariffs (only when runs 1,2 and 5 or 6 are present) --- + has_run1=false; has_run2=false; has_run5=false; has_run6=false + for r in "${all_runs[@]}"; do + [[ "$r" == "1" ]] && has_run1=true + [[ "$r" == "2" ]] && has_run2=true + [[ "$r" == "5" ]] && has_run5=true + [[ "$r" == "6" ]] && has_run6=true + done + if $has_run1 && $has_run5; then + run1_dir=$(bash "{{ latest_output }}" "${adoption_yaml}" $(( key_prefix + 1 ))) + echo ">> run-adoption-all: deriving seasonal (delivery) from ${run1_dir}" >&2 + just compute-seasonal-discount-inputs \ + "${run1_dir}" "${loads_base}" "{{ state_upper }}" "{{ upgrade }}" + just create-seasonal-discount-tariff \ + "${tariff_dir}/{{ utility }}_flat_calibrated.json" \ + "${run1_dir}/seasonal_discount_rate_inputs.csv" \ + "{{ utility }}_hp_seasonal" \ + "${tariff_dir}/{{ utility }}_hp_seasonal.json" + fi + if $has_run2 && $has_run6; then + run2_dir=$(bash "{{ latest_output }}" "${adoption_yaml}" $(( key_prefix + 2 ))) + echo ">> run-adoption-all: deriving seasonal (supply) from ${run2_dir}" >&2 + just compute-seasonal-discount-inputs \ + "${run2_dir}" "${loads_base}" "{{ state_upper }}" "{{ upgrade }}" + just create-seasonal-discount-tariff \ + "${tariff_dir}/{{ utility }}_flat_supply_calibrated.json" \ + "${run2_dir}/seasonal_discount_rate_inputs.csv" \ + "{{ utility }}_hp_seasonal_supply" \ + "${tariff_dir}/{{ utility }}_hp_seasonal_supply.json" + fi + # --- Runs 5 and 6: precalc seasonal --- + for base_run in "${all_runs[@]}"; do + [[ "${base_run}" == "5" || "${base_run}" == "6" ]] || continue + key=$(( key_prefix + base_run )) + echo ">> run-adoption-all: run-${key} (year=${year}, base_run=${base_run})" >&2 + just run-adoption-scenario "${key}" + run_dir=$(bash "{{ latest_output }}" "${adoption_yaml}" "${key}") + just copy-calibrated-tariff-from-run "${run_dir}" "${tariff_dir}" + done + # --- Any remaining runs (not 1,2,5,6) --- + for base_run in "${all_runs[@]}"; do + [[ "${base_run}" == "1" || "${base_run}" == "2" ]] && continue + [[ "${base_run}" == "5" || "${base_run}" == "6" ]] && continue + key=$(( key_prefix + base_run )) + echo ">> run-adoption-all: run-${key} (year=${year}, base_run=${base_run})" >&2 + just run-adoption-scenario "${key}" + done + done + # ============================================================================= # HELPERS # ============================================================================= diff --git a/rate_design/hp_rates/ny/config/adoption/nyca_electrification.yaml b/rate_design/hp_rates/ny/config/adoption/nyca_electrification.yaml new file mode 100644 index 00000000..e3eaba98 --- /dev/null +++ b/rate_design/hp_rates/ny/config/adoption/nyca_electrification.yaml @@ -0,0 +1,35 @@ +# NYCA building electrification adoption trajectory (NYISO Gold Book 2025). +# Generated by utils/pre/fit_adoption_config.py — do not edit by hand. +# +# Fractions represent the share of total NYCA buildings assigned to each +# ResStock upgrade at each year. Remaining buildings stay at upgrade 0 (baseline). +# Year indices map to calendar years via year_labels. +# +# Technology → ResStock upgrade mapping: +# ASHP Full Capacity → 2 (cold-climate ASHP, 90% capacity @ 5F, elec backup) +# ASHP Dual Fuel → 4 (ENERGY STAR ASHP + existing fossil backup) +# Ground Source HP → 5 (geothermal heat pump) +# Supplemental Heat → 1 (ENERGY STAR ASHP, 50% capacity @ 5F, elec backup) +# Electric Resistance → baseline upgrade 0, already captured there +# +# Methodology: logistic S-curves f(t) = L / (1 + exp(-k * (t - t0))) fit +# (scipy curve_fit) to housing-unit counts digitized from the NYISO Gold +# Book 2025 NYCA stacked-area chart. Denominator: 7,900,000 total NYCA +# occupied housing units (Census ACS / NYISO estimate). 2025 forced to 0.0 +# (all buildings at upgrade-0 baseline). +# +# Fitted parameters: +# upgrade 2 (ASHP full capacity): L=0.2168 k=0.2169 t0=2042.8 +# upgrade 4 (ASHP dual fuel): L=0.1087 k=0.2290 t0=2040.0 +# upgrade 5 (ground source HP): L=0.0115 k=0.2633 t0=2039.3 +# upgrade 1 (supplemental heat): L=0.1281 k=0.3098 t0=2040.2 +scenario_name: nyca_electrification +random_seed: 42 +scenario: + 2: [0.0000, 0.0128, 0.0339, 0.0767, 0.1340, 0.1794] # ASHP full capacity + 4: [0.0000, 0.0100, 0.0263, 0.0544, 0.0825, 0.0987] # ASHP dual fuel + 5: [0.0000, 0.0009, 0.0028, 0.0063, 0.0094, 0.0109] # ground source HP + 1: [0.0000, 0.0052, 0.0211, 0.0617, 0.1043, 0.1222] # supplemental heat +# Calendar years for each scenario index (= run years). +# Aligns with Cambium 5-year MC intervals; 2025 is baseline. +year_labels: [2025, 2030, 2035, 2040, 2045, 2050] diff --git a/rate_design/hp_rates/ny/config/scenarios/scenarios_nyseg_adoption.yaml b/rate_design/hp_rates/ny/config/scenarios/scenarios_nyseg_adoption.yaml new file mode 100644 index 00000000..dac336e5 --- /dev/null +++ b/rate_design/hp_rates/ny/config/scenarios/scenarios_nyseg_adoption.yaml @@ -0,0 +1,732 @@ +runs: + 101: + run_name: ny_nyseg_run1_up00_precalc_y2025_mixed__flat + state: NY + utility: nyseg + run_type: precalc + upgrade: '0' + path_tariff_maps_electric: tariff_maps/electric/nyseg_flat.csv + path_tariff_maps_gas: tariff_maps/gas/nyseg_u00.csv + path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2025/metadata-sb.parquet + path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2025/load_curve_hourly/state=NY/upgrade=00 + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ny/cambium_dist_and_sub_tx/utility=nyseg/year=2025/data.parquet + path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=NY/utility_assignment.parquet + path_tariffs_gas: tariffs/gas + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ny/nyseg//ny_nyseg_run1_up00_precalc__flat + path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ny/supply/energy/utility=nyseg/year=2025/zero.parquet + path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ny/supply/capacity/utility=nyseg/year=2025/zero.parquet + path_tariffs_electric: + all: tariffs/electric/nyseg_flat.json + utility_revenue_requirement: null + run_includes_supply: false + run_includes_subclasses: false + path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=NY/data.parquet + path_bulk_tx_mc: '' + solar_pv_compensation: net_metering + year_run: 2025 + year_dollar_conversion: 2025 + process_workers: 8 + elasticity: 0.0 + residual_cost_frac: 0.0 + + 102: + run_name: ny_nyseg_run2_up00_precalc_supply_y2025_mixed__flat + state: NY + utility: nyseg + run_type: precalc + upgrade: '0' + path_tariff_maps_electric: tariff_maps/electric/nyseg_flat_supply.csv + path_tariff_maps_gas: tariff_maps/gas/nyseg_u00.csv + path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2025/metadata-sb.parquet + path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2025/load_curve_hourly/state=NY/upgrade=00 + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ny/cambium_dist_and_sub_tx/utility=nyseg/year=2025/data.parquet + path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=NY/utility_assignment.parquet + path_tariffs_gas: tariffs/gas + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ny/nyseg//ny_nyseg_run2_up00_precalc_supply__flat + path_supply_energy_mc: s3://data.sb/nrel/cambium/2024/scenario=MidCase/t=2025/gea=NYISO/r=p127/data.parquet + path_supply_capacity_mc: s3://data.sb/nrel/cambium/2024/scenario=MidCase/t=2025/gea=NYISO/r=p127/data.parquet + path_tariffs_electric: + all: tariffs/electric/nyseg_flat_supply.json + utility_revenue_requirement: null + run_includes_supply: true + run_includes_subclasses: false + path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=NY/data.parquet + path_bulk_tx_mc: '' + solar_pv_compensation: net_metering + year_run: 2025 + year_dollar_conversion: 2025 + process_workers: 8 + elasticity: 0.0 + residual_cost_frac: 0.0 + + 105: + run_name: ny_nyseg_run5_up00_precalc_y2025_mixed__hp_seasonal_vs_flat + state: NY + utility: nyseg + run_type: precalc + upgrade: '0' + path_tariff_maps_electric: tariff_maps/electric/nyseg_hp_seasonal_vs_flat.csv + path_tariff_maps_gas: tariff_maps/gas/nyseg_u00.csv + path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2025/metadata-sb.parquet + path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2025/load_curve_hourly/state=NY/upgrade=00 + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ny/cambium_dist_and_sub_tx/utility=nyseg/year=2025/data.parquet + path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=NY/utility_assignment.parquet + path_tariffs_gas: tariffs/gas + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ny/nyseg//ny_nyseg_run5_up00_precalc__hp_seasonal_vs_flat + path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ny/supply/energy/utility=nyseg/year=2025/zero.parquet + path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ny/supply/capacity/utility=nyseg/year=2025/zero.parquet + path_tariffs_electric: + hp: /ebs/home/sherry_switch_box/rate-design-platform/rate_design/hp_rates/ny/config/tariffs/electric/adoption/nyca_electrification/year=2025/nyseg_hp_seasonal.json + non-hp: /ebs/home/sherry_switch_box/rate-design-platform/rate_design/hp_rates/ny/config/tariffs/electric/adoption/nyca_electrification/year=2025/nyseg_nonhp_flat.json + utility_revenue_requirement: null + run_includes_supply: false + run_includes_subclasses: true + path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=NY/data.parquet + path_bulk_tx_mc: '' + solar_pv_compensation: net_metering + year_run: 2025 + year_dollar_conversion: 2025 + process_workers: 8 + elasticity: 0.0 + residual_cost_frac: 0.0 + + 106: + run_name: ny_nyseg_run6_up00_precalc_supply_y2025_mixed__hp_seasonal_vs_flat + state: NY + utility: nyseg + run_type: precalc + upgrade: '0' + path_tariff_maps_electric: tariff_maps/electric/nyseg_hp_seasonal_vs_flat_supply.csv + path_tariff_maps_gas: tariff_maps/gas/nyseg_u00.csv + path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2025/metadata-sb.parquet + path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2025/load_curve_hourly/state=NY/upgrade=00 + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ny/cambium_dist_and_sub_tx/utility=nyseg/year=2025/data.parquet + path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=NY/utility_assignment.parquet + path_tariffs_gas: tariffs/gas + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ny/nyseg//ny_nyseg_run6_up00_precalc_supply__hp_seasonal_vs_flat + path_supply_energy_mc: s3://data.sb/nrel/cambium/2024/scenario=MidCase/t=2025/gea=NYISO/r=p127/data.parquet + path_supply_capacity_mc: s3://data.sb/nrel/cambium/2024/scenario=MidCase/t=2025/gea=NYISO/r=p127/data.parquet + path_tariffs_electric: + hp: /ebs/home/sherry_switch_box/rate-design-platform/rate_design/hp_rates/ny/config/tariffs/electric/adoption/nyca_electrification/year=2025/nyseg_hp_seasonal.json + non-hp: /ebs/home/sherry_switch_box/rate-design-platform/rate_design/hp_rates/ny/config/tariffs/electric/adoption/nyca_electrification/year=2025/nyseg_nonhp_flat.json + utility_revenue_requirement: null + run_includes_supply: true + run_includes_subclasses: true + path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=NY/data.parquet + path_bulk_tx_mc: '' + solar_pv_compensation: net_metering + year_run: 2025 + year_dollar_conversion: 2025 + process_workers: 8 + elasticity: 0.0 + residual_cost_frac: 0.0 + + 201: + run_name: ny_nyseg_run1_up00_precalc_y2030_mixed__flat + state: NY + utility: nyseg + run_type: precalc + upgrade: '0' + path_tariff_maps_electric: tariff_maps/electric/nyseg_flat.csv + path_tariff_maps_gas: tariff_maps/gas/nyseg_u00.csv + path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2030/metadata-sb.parquet + path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2030/load_curve_hourly/state=NY/upgrade=00 + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ny/cambium_dist_and_sub_tx/utility=nyseg/year=2030/data.parquet + path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=NY/utility_assignment.parquet + path_tariffs_gas: tariffs/gas + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ny/nyseg//ny_nyseg_run1_up00_precalc__flat + path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ny/supply/energy/utility=nyseg/year=2030/zero.parquet + path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ny/supply/capacity/utility=nyseg/year=2030/zero.parquet + path_tariffs_electric: + all: tariffs/electric/nyseg_flat.json + utility_revenue_requirement: null + run_includes_supply: false + run_includes_subclasses: false + path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=NY/data.parquet + path_bulk_tx_mc: '' + solar_pv_compensation: net_metering + year_run: 2030 + year_dollar_conversion: 2025 + process_workers: 8 + elasticity: 0.0 + residual_cost_frac: 0.0 + + 202: + run_name: ny_nyseg_run2_up00_precalc_supply_y2030_mixed__flat + state: NY + utility: nyseg + run_type: precalc + upgrade: '0' + path_tariff_maps_electric: tariff_maps/electric/nyseg_flat_supply.csv + path_tariff_maps_gas: tariff_maps/gas/nyseg_u00.csv + path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2030/metadata-sb.parquet + path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2030/load_curve_hourly/state=NY/upgrade=00 + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ny/cambium_dist_and_sub_tx/utility=nyseg/year=2030/data.parquet + path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=NY/utility_assignment.parquet + path_tariffs_gas: tariffs/gas + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ny/nyseg//ny_nyseg_run2_up00_precalc_supply__flat + path_supply_energy_mc: s3://data.sb/nrel/cambium/2024/scenario=MidCase/t=2030/gea=NYISO/r=p127/data.parquet + path_supply_capacity_mc: s3://data.sb/nrel/cambium/2024/scenario=MidCase/t=2030/gea=NYISO/r=p127/data.parquet + path_tariffs_electric: + all: tariffs/electric/nyseg_flat_supply.json + utility_revenue_requirement: null + run_includes_supply: true + run_includes_subclasses: false + path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=NY/data.parquet + path_bulk_tx_mc: '' + solar_pv_compensation: net_metering + year_run: 2030 + year_dollar_conversion: 2025 + process_workers: 8 + elasticity: 0.0 + residual_cost_frac: 0.0 + + 205: + run_name: ny_nyseg_run5_up00_precalc_y2030_mixed__hp_seasonal_vs_flat + state: NY + utility: nyseg + run_type: precalc + upgrade: '0' + path_tariff_maps_electric: tariff_maps/electric/nyseg_hp_seasonal_vs_flat.csv + path_tariff_maps_gas: tariff_maps/gas/nyseg_u00.csv + path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2030/metadata-sb.parquet + path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2030/load_curve_hourly/state=NY/upgrade=00 + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ny/cambium_dist_and_sub_tx/utility=nyseg/year=2030/data.parquet + path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=NY/utility_assignment.parquet + path_tariffs_gas: tariffs/gas + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ny/nyseg//ny_nyseg_run5_up00_precalc__hp_seasonal_vs_flat + path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ny/supply/energy/utility=nyseg/year=2030/zero.parquet + path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ny/supply/capacity/utility=nyseg/year=2030/zero.parquet + path_tariffs_electric: + hp: /ebs/home/sherry_switch_box/rate-design-platform/rate_design/hp_rates/ny/config/tariffs/electric/adoption/nyca_electrification/year=2030/nyseg_hp_seasonal.json + non-hp: /ebs/home/sherry_switch_box/rate-design-platform/rate_design/hp_rates/ny/config/tariffs/electric/adoption/nyca_electrification/year=2030/nyseg_nonhp_flat.json + utility_revenue_requirement: null + run_includes_supply: false + run_includes_subclasses: true + path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=NY/data.parquet + path_bulk_tx_mc: '' + solar_pv_compensation: net_metering + year_run: 2030 + year_dollar_conversion: 2025 + process_workers: 8 + elasticity: 0.0 + residual_cost_frac: 0.0 + + 206: + run_name: ny_nyseg_run6_up00_precalc_supply_y2030_mixed__hp_seasonal_vs_flat + state: NY + utility: nyseg + run_type: precalc + upgrade: '0' + path_tariff_maps_electric: tariff_maps/electric/nyseg_hp_seasonal_vs_flat_supply.csv + path_tariff_maps_gas: tariff_maps/gas/nyseg_u00.csv + path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2030/metadata-sb.parquet + path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2030/load_curve_hourly/state=NY/upgrade=00 + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ny/cambium_dist_and_sub_tx/utility=nyseg/year=2030/data.parquet + path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=NY/utility_assignment.parquet + path_tariffs_gas: tariffs/gas + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ny/nyseg//ny_nyseg_run6_up00_precalc_supply__hp_seasonal_vs_flat + path_supply_energy_mc: s3://data.sb/nrel/cambium/2024/scenario=MidCase/t=2030/gea=NYISO/r=p127/data.parquet + path_supply_capacity_mc: s3://data.sb/nrel/cambium/2024/scenario=MidCase/t=2030/gea=NYISO/r=p127/data.parquet + path_tariffs_electric: + hp: /ebs/home/sherry_switch_box/rate-design-platform/rate_design/hp_rates/ny/config/tariffs/electric/adoption/nyca_electrification/year=2030/nyseg_hp_seasonal.json + non-hp: /ebs/home/sherry_switch_box/rate-design-platform/rate_design/hp_rates/ny/config/tariffs/electric/adoption/nyca_electrification/year=2030/nyseg_nonhp_flat.json + utility_revenue_requirement: null + run_includes_supply: true + run_includes_subclasses: true + path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=NY/data.parquet + path_bulk_tx_mc: '' + solar_pv_compensation: net_metering + year_run: 2030 + year_dollar_conversion: 2025 + process_workers: 8 + elasticity: 0.0 + residual_cost_frac: 0.0 + + 301: + run_name: ny_nyseg_run1_up00_precalc_y2035_mixed__flat + state: NY + utility: nyseg + run_type: precalc + upgrade: '0' + path_tariff_maps_electric: tariff_maps/electric/nyseg_flat.csv + path_tariff_maps_gas: tariff_maps/gas/nyseg_u00.csv + path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2035/metadata-sb.parquet + path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2035/load_curve_hourly/state=NY/upgrade=00 + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ny/cambium_dist_and_sub_tx/utility=nyseg/year=2035/data.parquet + path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=NY/utility_assignment.parquet + path_tariffs_gas: tariffs/gas + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ny/nyseg//ny_nyseg_run1_up00_precalc__flat + path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ny/supply/energy/utility=nyseg/year=2035/zero.parquet + path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ny/supply/capacity/utility=nyseg/year=2035/zero.parquet + path_tariffs_electric: + all: tariffs/electric/nyseg_flat.json + utility_revenue_requirement: null + run_includes_supply: false + run_includes_subclasses: false + path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=NY/data.parquet + path_bulk_tx_mc: '' + solar_pv_compensation: net_metering + year_run: 2035 + year_dollar_conversion: 2025 + process_workers: 8 + elasticity: 0.0 + residual_cost_frac: 0.0 + + 302: + run_name: ny_nyseg_run2_up00_precalc_supply_y2035_mixed__flat + state: NY + utility: nyseg + run_type: precalc + upgrade: '0' + path_tariff_maps_electric: tariff_maps/electric/nyseg_flat_supply.csv + path_tariff_maps_gas: tariff_maps/gas/nyseg_u00.csv + path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2035/metadata-sb.parquet + path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2035/load_curve_hourly/state=NY/upgrade=00 + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ny/cambium_dist_and_sub_tx/utility=nyseg/year=2035/data.parquet + path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=NY/utility_assignment.parquet + path_tariffs_gas: tariffs/gas + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ny/nyseg//ny_nyseg_run2_up00_precalc_supply__flat + path_supply_energy_mc: s3://data.sb/nrel/cambium/2024/scenario=MidCase/t=2035/gea=NYISO/r=p127/data.parquet + path_supply_capacity_mc: s3://data.sb/nrel/cambium/2024/scenario=MidCase/t=2035/gea=NYISO/r=p127/data.parquet + path_tariffs_electric: + all: tariffs/electric/nyseg_flat_supply.json + utility_revenue_requirement: null + run_includes_supply: true + run_includes_subclasses: false + path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=NY/data.parquet + path_bulk_tx_mc: '' + solar_pv_compensation: net_metering + year_run: 2035 + year_dollar_conversion: 2025 + process_workers: 8 + elasticity: 0.0 + residual_cost_frac: 0.0 + + 305: + run_name: ny_nyseg_run5_up00_precalc_y2035_mixed__hp_seasonal_vs_flat + state: NY + utility: nyseg + run_type: precalc + upgrade: '0' + path_tariff_maps_electric: tariff_maps/electric/nyseg_hp_seasonal_vs_flat.csv + path_tariff_maps_gas: tariff_maps/gas/nyseg_u00.csv + path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2035/metadata-sb.parquet + path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2035/load_curve_hourly/state=NY/upgrade=00 + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ny/cambium_dist_and_sub_tx/utility=nyseg/year=2035/data.parquet + path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=NY/utility_assignment.parquet + path_tariffs_gas: tariffs/gas + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ny/nyseg//ny_nyseg_run5_up00_precalc__hp_seasonal_vs_flat + path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ny/supply/energy/utility=nyseg/year=2035/zero.parquet + path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ny/supply/capacity/utility=nyseg/year=2035/zero.parquet + path_tariffs_electric: + hp: /ebs/home/sherry_switch_box/rate-design-platform/rate_design/hp_rates/ny/config/tariffs/electric/adoption/nyca_electrification/year=2035/nyseg_hp_seasonal.json + non-hp: /ebs/home/sherry_switch_box/rate-design-platform/rate_design/hp_rates/ny/config/tariffs/electric/adoption/nyca_electrification/year=2035/nyseg_nonhp_flat.json + utility_revenue_requirement: null + run_includes_supply: false + run_includes_subclasses: true + path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=NY/data.parquet + path_bulk_tx_mc: '' + solar_pv_compensation: net_metering + year_run: 2035 + year_dollar_conversion: 2025 + process_workers: 8 + elasticity: 0.0 + residual_cost_frac: 0.0 + + 306: + run_name: ny_nyseg_run6_up00_precalc_supply_y2035_mixed__hp_seasonal_vs_flat + state: NY + utility: nyseg + run_type: precalc + upgrade: '0' + path_tariff_maps_electric: tariff_maps/electric/nyseg_hp_seasonal_vs_flat_supply.csv + path_tariff_maps_gas: tariff_maps/gas/nyseg_u00.csv + path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2035/metadata-sb.parquet + path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2035/load_curve_hourly/state=NY/upgrade=00 + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ny/cambium_dist_and_sub_tx/utility=nyseg/year=2035/data.parquet + path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=NY/utility_assignment.parquet + path_tariffs_gas: tariffs/gas + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ny/nyseg//ny_nyseg_run6_up00_precalc_supply__hp_seasonal_vs_flat + path_supply_energy_mc: s3://data.sb/nrel/cambium/2024/scenario=MidCase/t=2035/gea=NYISO/r=p127/data.parquet + path_supply_capacity_mc: s3://data.sb/nrel/cambium/2024/scenario=MidCase/t=2035/gea=NYISO/r=p127/data.parquet + path_tariffs_electric: + hp: /ebs/home/sherry_switch_box/rate-design-platform/rate_design/hp_rates/ny/config/tariffs/electric/adoption/nyca_electrification/year=2035/nyseg_hp_seasonal.json + non-hp: /ebs/home/sherry_switch_box/rate-design-platform/rate_design/hp_rates/ny/config/tariffs/electric/adoption/nyca_electrification/year=2035/nyseg_nonhp_flat.json + utility_revenue_requirement: null + run_includes_supply: true + run_includes_subclasses: true + path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=NY/data.parquet + path_bulk_tx_mc: '' + solar_pv_compensation: net_metering + year_run: 2035 + year_dollar_conversion: 2025 + process_workers: 8 + elasticity: 0.0 + residual_cost_frac: 0.0 + + 401: + run_name: ny_nyseg_run1_up00_precalc_y2040_mixed__flat + state: NY + utility: nyseg + run_type: precalc + upgrade: '0' + path_tariff_maps_electric: tariff_maps/electric/nyseg_flat.csv + path_tariff_maps_gas: tariff_maps/gas/nyseg_u00.csv + path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2040/metadata-sb.parquet + path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2040/load_curve_hourly/state=NY/upgrade=00 + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ny/cambium_dist_and_sub_tx/utility=nyseg/year=2040/data.parquet + path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=NY/utility_assignment.parquet + path_tariffs_gas: tariffs/gas + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ny/nyseg//ny_nyseg_run1_up00_precalc__flat + path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ny/supply/energy/utility=nyseg/year=2040/zero.parquet + path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ny/supply/capacity/utility=nyseg/year=2040/zero.parquet + path_tariffs_electric: + all: tariffs/electric/nyseg_flat.json + utility_revenue_requirement: null + run_includes_supply: false + run_includes_subclasses: false + path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=NY/data.parquet + path_bulk_tx_mc: '' + solar_pv_compensation: net_metering + year_run: 2040 + year_dollar_conversion: 2025 + process_workers: 8 + elasticity: 0.0 + residual_cost_frac: 0.0 + + 402: + run_name: ny_nyseg_run2_up00_precalc_supply_y2040_mixed__flat + state: NY + utility: nyseg + run_type: precalc + upgrade: '0' + path_tariff_maps_electric: tariff_maps/electric/nyseg_flat_supply.csv + path_tariff_maps_gas: tariff_maps/gas/nyseg_u00.csv + path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2040/metadata-sb.parquet + path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2040/load_curve_hourly/state=NY/upgrade=00 + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ny/cambium_dist_and_sub_tx/utility=nyseg/year=2040/data.parquet + path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=NY/utility_assignment.parquet + path_tariffs_gas: tariffs/gas + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ny/nyseg//ny_nyseg_run2_up00_precalc_supply__flat + path_supply_energy_mc: s3://data.sb/nrel/cambium/2024/scenario=MidCase/t=2040/gea=NYISO/r=p127/data.parquet + path_supply_capacity_mc: s3://data.sb/nrel/cambium/2024/scenario=MidCase/t=2040/gea=NYISO/r=p127/data.parquet + path_tariffs_electric: + all: tariffs/electric/nyseg_flat_supply.json + utility_revenue_requirement: null + run_includes_supply: true + run_includes_subclasses: false + path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=NY/data.parquet + path_bulk_tx_mc: '' + solar_pv_compensation: net_metering + year_run: 2040 + year_dollar_conversion: 2025 + process_workers: 8 + elasticity: 0.0 + residual_cost_frac: 0.0 + + 405: + run_name: ny_nyseg_run5_up00_precalc_y2040_mixed__hp_seasonal_vs_flat + state: NY + utility: nyseg + run_type: precalc + upgrade: '0' + path_tariff_maps_electric: tariff_maps/electric/nyseg_hp_seasonal_vs_flat.csv + path_tariff_maps_gas: tariff_maps/gas/nyseg_u00.csv + path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2040/metadata-sb.parquet + path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2040/load_curve_hourly/state=NY/upgrade=00 + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ny/cambium_dist_and_sub_tx/utility=nyseg/year=2040/data.parquet + path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=NY/utility_assignment.parquet + path_tariffs_gas: tariffs/gas + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ny/nyseg//ny_nyseg_run5_up00_precalc__hp_seasonal_vs_flat + path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ny/supply/energy/utility=nyseg/year=2040/zero.parquet + path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ny/supply/capacity/utility=nyseg/year=2040/zero.parquet + path_tariffs_electric: + hp: /ebs/home/sherry_switch_box/rate-design-platform/rate_design/hp_rates/ny/config/tariffs/electric/adoption/nyca_electrification/year=2040/nyseg_hp_seasonal.json + non-hp: /ebs/home/sherry_switch_box/rate-design-platform/rate_design/hp_rates/ny/config/tariffs/electric/adoption/nyca_electrification/year=2040/nyseg_nonhp_flat.json + utility_revenue_requirement: null + run_includes_supply: false + run_includes_subclasses: true + path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=NY/data.parquet + path_bulk_tx_mc: '' + solar_pv_compensation: net_metering + year_run: 2040 + year_dollar_conversion: 2025 + process_workers: 8 + elasticity: 0.0 + residual_cost_frac: 0.0 + + 406: + run_name: ny_nyseg_run6_up00_precalc_supply_y2040_mixed__hp_seasonal_vs_flat + state: NY + utility: nyseg + run_type: precalc + upgrade: '0' + path_tariff_maps_electric: tariff_maps/electric/nyseg_hp_seasonal_vs_flat_supply.csv + path_tariff_maps_gas: tariff_maps/gas/nyseg_u00.csv + path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2040/metadata-sb.parquet + path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2040/load_curve_hourly/state=NY/upgrade=00 + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ny/cambium_dist_and_sub_tx/utility=nyseg/year=2040/data.parquet + path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=NY/utility_assignment.parquet + path_tariffs_gas: tariffs/gas + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ny/nyseg//ny_nyseg_run6_up00_precalc_supply__hp_seasonal_vs_flat + path_supply_energy_mc: s3://data.sb/nrel/cambium/2024/scenario=MidCase/t=2040/gea=NYISO/r=p127/data.parquet + path_supply_capacity_mc: s3://data.sb/nrel/cambium/2024/scenario=MidCase/t=2040/gea=NYISO/r=p127/data.parquet + path_tariffs_electric: + hp: /ebs/home/sherry_switch_box/rate-design-platform/rate_design/hp_rates/ny/config/tariffs/electric/adoption/nyca_electrification/year=2040/nyseg_hp_seasonal.json + non-hp: /ebs/home/sherry_switch_box/rate-design-platform/rate_design/hp_rates/ny/config/tariffs/electric/adoption/nyca_electrification/year=2040/nyseg_nonhp_flat.json + utility_revenue_requirement: null + run_includes_supply: true + run_includes_subclasses: true + path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=NY/data.parquet + path_bulk_tx_mc: '' + solar_pv_compensation: net_metering + year_run: 2040 + year_dollar_conversion: 2025 + process_workers: 8 + elasticity: 0.0 + residual_cost_frac: 0.0 + + 501: + run_name: ny_nyseg_run1_up00_precalc_y2045_mixed__flat + state: NY + utility: nyseg + run_type: precalc + upgrade: '0' + path_tariff_maps_electric: tariff_maps/electric/nyseg_flat.csv + path_tariff_maps_gas: tariff_maps/gas/nyseg_u00.csv + path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2045/metadata-sb.parquet + path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2045/load_curve_hourly/state=NY/upgrade=00 + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ny/cambium_dist_and_sub_tx/utility=nyseg/year=2045/data.parquet + path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=NY/utility_assignment.parquet + path_tariffs_gas: tariffs/gas + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ny/nyseg//ny_nyseg_run1_up00_precalc__flat + path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ny/supply/energy/utility=nyseg/year=2045/zero.parquet + path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ny/supply/capacity/utility=nyseg/year=2045/zero.parquet + path_tariffs_electric: + all: tariffs/electric/nyseg_flat.json + utility_revenue_requirement: null + run_includes_supply: false + run_includes_subclasses: false + path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=NY/data.parquet + path_bulk_tx_mc: '' + solar_pv_compensation: net_metering + year_run: 2045 + year_dollar_conversion: 2025 + process_workers: 8 + elasticity: 0.0 + residual_cost_frac: 0.0 + + 502: + run_name: ny_nyseg_run2_up00_precalc_supply_y2045_mixed__flat + state: NY + utility: nyseg + run_type: precalc + upgrade: '0' + path_tariff_maps_electric: tariff_maps/electric/nyseg_flat_supply.csv + path_tariff_maps_gas: tariff_maps/gas/nyseg_u00.csv + path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2045/metadata-sb.parquet + path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2045/load_curve_hourly/state=NY/upgrade=00 + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ny/cambium_dist_and_sub_tx/utility=nyseg/year=2045/data.parquet + path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=NY/utility_assignment.parquet + path_tariffs_gas: tariffs/gas + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ny/nyseg//ny_nyseg_run2_up00_precalc_supply__flat + path_supply_energy_mc: s3://data.sb/nrel/cambium/2024/scenario=MidCase/t=2045/gea=NYISO/r=p127/data.parquet + path_supply_capacity_mc: s3://data.sb/nrel/cambium/2024/scenario=MidCase/t=2045/gea=NYISO/r=p127/data.parquet + path_tariffs_electric: + all: tariffs/electric/nyseg_flat_supply.json + utility_revenue_requirement: null + run_includes_supply: true + run_includes_subclasses: false + path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=NY/data.parquet + path_bulk_tx_mc: '' + solar_pv_compensation: net_metering + year_run: 2045 + year_dollar_conversion: 2025 + process_workers: 8 + elasticity: 0.0 + residual_cost_frac: 0.0 + + 505: + run_name: ny_nyseg_run5_up00_precalc_y2045_mixed__hp_seasonal_vs_flat + state: NY + utility: nyseg + run_type: precalc + upgrade: '0' + path_tariff_maps_electric: tariff_maps/electric/nyseg_hp_seasonal_vs_flat.csv + path_tariff_maps_gas: tariff_maps/gas/nyseg_u00.csv + path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2045/metadata-sb.parquet + path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2045/load_curve_hourly/state=NY/upgrade=00 + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ny/cambium_dist_and_sub_tx/utility=nyseg/year=2045/data.parquet + path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=NY/utility_assignment.parquet + path_tariffs_gas: tariffs/gas + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ny/nyseg//ny_nyseg_run5_up00_precalc__hp_seasonal_vs_flat + path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ny/supply/energy/utility=nyseg/year=2045/zero.parquet + path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ny/supply/capacity/utility=nyseg/year=2045/zero.parquet + path_tariffs_electric: + hp: /ebs/home/sherry_switch_box/rate-design-platform/rate_design/hp_rates/ny/config/tariffs/electric/adoption/nyca_electrification/year=2045/nyseg_hp_seasonal.json + non-hp: /ebs/home/sherry_switch_box/rate-design-platform/rate_design/hp_rates/ny/config/tariffs/electric/adoption/nyca_electrification/year=2045/nyseg_nonhp_flat.json + utility_revenue_requirement: null + run_includes_supply: false + run_includes_subclasses: true + path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=NY/data.parquet + path_bulk_tx_mc: '' + solar_pv_compensation: net_metering + year_run: 2045 + year_dollar_conversion: 2025 + process_workers: 8 + elasticity: 0.0 + residual_cost_frac: 0.0 + + 506: + run_name: ny_nyseg_run6_up00_precalc_supply_y2045_mixed__hp_seasonal_vs_flat + state: NY + utility: nyseg + run_type: precalc + upgrade: '0' + path_tariff_maps_electric: tariff_maps/electric/nyseg_hp_seasonal_vs_flat_supply.csv + path_tariff_maps_gas: tariff_maps/gas/nyseg_u00.csv + path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2045/metadata-sb.parquet + path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2045/load_curve_hourly/state=NY/upgrade=00 + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ny/cambium_dist_and_sub_tx/utility=nyseg/year=2045/data.parquet + path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=NY/utility_assignment.parquet + path_tariffs_gas: tariffs/gas + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ny/nyseg//ny_nyseg_run6_up00_precalc_supply__hp_seasonal_vs_flat + path_supply_energy_mc: s3://data.sb/nrel/cambium/2024/scenario=MidCase/t=2045/gea=NYISO/r=p127/data.parquet + path_supply_capacity_mc: s3://data.sb/nrel/cambium/2024/scenario=MidCase/t=2045/gea=NYISO/r=p127/data.parquet + path_tariffs_electric: + hp: /ebs/home/sherry_switch_box/rate-design-platform/rate_design/hp_rates/ny/config/tariffs/electric/adoption/nyca_electrification/year=2045/nyseg_hp_seasonal.json + non-hp: /ebs/home/sherry_switch_box/rate-design-platform/rate_design/hp_rates/ny/config/tariffs/electric/adoption/nyca_electrification/year=2045/nyseg_nonhp_flat.json + utility_revenue_requirement: null + run_includes_supply: true + run_includes_subclasses: true + path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=NY/data.parquet + path_bulk_tx_mc: '' + solar_pv_compensation: net_metering + year_run: 2045 + year_dollar_conversion: 2025 + process_workers: 8 + elasticity: 0.0 + residual_cost_frac: 0.0 + + 601: + run_name: ny_nyseg_run1_up00_precalc_y2050_mixed__flat + state: NY + utility: nyseg + run_type: precalc + upgrade: '0' + path_tariff_maps_electric: tariff_maps/electric/nyseg_flat.csv + path_tariff_maps_gas: tariff_maps/gas/nyseg_u00.csv + path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2050/metadata-sb.parquet + path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2050/load_curve_hourly/state=NY/upgrade=00 + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ny/cambium_dist_and_sub_tx/utility=nyseg/year=2050/data.parquet + path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=NY/utility_assignment.parquet + path_tariffs_gas: tariffs/gas + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ny/nyseg//ny_nyseg_run1_up00_precalc__flat + path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ny/supply/energy/utility=nyseg/year=2050/zero.parquet + path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ny/supply/capacity/utility=nyseg/year=2050/zero.parquet + path_tariffs_electric: + all: tariffs/electric/nyseg_flat.json + utility_revenue_requirement: null + run_includes_supply: false + run_includes_subclasses: false + path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=NY/data.parquet + path_bulk_tx_mc: '' + solar_pv_compensation: net_metering + year_run: 2050 + year_dollar_conversion: 2025 + process_workers: 8 + elasticity: 0.0 + residual_cost_frac: 0.0 + + 602: + run_name: ny_nyseg_run2_up00_precalc_supply_y2050_mixed__flat + state: NY + utility: nyseg + run_type: precalc + upgrade: '0' + path_tariff_maps_electric: tariff_maps/electric/nyseg_flat_supply.csv + path_tariff_maps_gas: tariff_maps/gas/nyseg_u00.csv + path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2050/metadata-sb.parquet + path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2050/load_curve_hourly/state=NY/upgrade=00 + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ny/cambium_dist_and_sub_tx/utility=nyseg/year=2050/data.parquet + path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=NY/utility_assignment.parquet + path_tariffs_gas: tariffs/gas + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ny/nyseg//ny_nyseg_run2_up00_precalc_supply__flat + path_supply_energy_mc: s3://data.sb/nrel/cambium/2024/scenario=MidCase/t=2050/gea=NYISO/r=p127/data.parquet + path_supply_capacity_mc: s3://data.sb/nrel/cambium/2024/scenario=MidCase/t=2050/gea=NYISO/r=p127/data.parquet + path_tariffs_electric: + all: tariffs/electric/nyseg_flat_supply.json + utility_revenue_requirement: null + run_includes_supply: true + run_includes_subclasses: false + path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=NY/data.parquet + path_bulk_tx_mc: '' + solar_pv_compensation: net_metering + year_run: 2050 + year_dollar_conversion: 2025 + process_workers: 8 + elasticity: 0.0 + residual_cost_frac: 0.0 + + 605: + run_name: ny_nyseg_run5_up00_precalc_y2050_mixed__hp_seasonal_vs_flat + state: NY + utility: nyseg + run_type: precalc + upgrade: '0' + path_tariff_maps_electric: tariff_maps/electric/nyseg_hp_seasonal_vs_flat.csv + path_tariff_maps_gas: tariff_maps/gas/nyseg_u00.csv + path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2050/metadata-sb.parquet + path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2050/load_curve_hourly/state=NY/upgrade=00 + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ny/cambium_dist_and_sub_tx/utility=nyseg/year=2050/data.parquet + path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=NY/utility_assignment.parquet + path_tariffs_gas: tariffs/gas + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ny/nyseg//ny_nyseg_run5_up00_precalc__hp_seasonal_vs_flat + path_supply_energy_mc: s3://data.sb/switchbox/marginal_costs/ny/supply/energy/utility=nyseg/year=2050/zero.parquet + path_supply_capacity_mc: s3://data.sb/switchbox/marginal_costs/ny/supply/capacity/utility=nyseg/year=2050/zero.parquet + path_tariffs_electric: + hp: /ebs/home/sherry_switch_box/rate-design-platform/rate_design/hp_rates/ny/config/tariffs/electric/adoption/nyca_electrification/year=2050/nyseg_hp_seasonal.json + non-hp: /ebs/home/sherry_switch_box/rate-design-platform/rate_design/hp_rates/ny/config/tariffs/electric/adoption/nyca_electrification/year=2050/nyseg_nonhp_flat.json + utility_revenue_requirement: null + run_includes_supply: false + run_includes_subclasses: true + path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=NY/data.parquet + path_bulk_tx_mc: '' + solar_pv_compensation: net_metering + year_run: 2050 + year_dollar_conversion: 2025 + process_workers: 8 + elasticity: 0.0 + residual_cost_frac: 0.0 + + 606: + run_name: ny_nyseg_run6_up00_precalc_supply_y2050_mixed__hp_seasonal_vs_flat + state: NY + utility: nyseg + run_type: precalc + upgrade: '0' + path_tariff_maps_electric: tariff_maps/electric/nyseg_hp_seasonal_vs_flat_supply.csv + path_tariff_maps_gas: tariff_maps/gas/nyseg_u00.csv + path_resstock_metadata: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2050/metadata-sb.parquet + path_resstock_loads: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification/year=2050/load_curve_hourly/state=NY/upgrade=00 + path_dist_and_sub_tx_mc: s3://data.sb/switchbox/marginal_costs/ny/cambium_dist_and_sub_tx/utility=nyseg/year=2050/data.parquet + path_utility_assignment: /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/metadata_utility/state=NY/utility_assignment.parquet + path_tariffs_gas: tariffs/gas + path_outputs: /data.sb/switchbox/cairo/outputs/hp_rates/ny/nyseg//ny_nyseg_run6_up00_precalc_supply__hp_seasonal_vs_flat + path_supply_energy_mc: s3://data.sb/nrel/cambium/2024/scenario=MidCase/t=2050/gea=NYISO/r=p127/data.parquet + path_supply_capacity_mc: s3://data.sb/nrel/cambium/2024/scenario=MidCase/t=2050/gea=NYISO/r=p127/data.parquet + path_tariffs_electric: + hp: /ebs/home/sherry_switch_box/rate-design-platform/rate_design/hp_rates/ny/config/tariffs/electric/adoption/nyca_electrification/year=2050/nyseg_hp_seasonal.json + non-hp: /ebs/home/sherry_switch_box/rate-design-platform/rate_design/hp_rates/ny/config/tariffs/electric/adoption/nyca_electrification/year=2050/nyseg_nonhp_flat.json + utility_revenue_requirement: null + run_includes_supply: true + run_includes_subclasses: true + path_electric_utility_stats: s3://data.sb/eia/861/electric_utility_stats/year=2024/state=NY/data.parquet + path_bulk_tx_mc: '' + solar_pv_compensation: net_metering + year_run: 2050 + year_dollar_conversion: 2025 + process_workers: 8 + elasticity: 0.0 + residual_cost_frac: 0.0 diff --git a/rate_design/hp_rates/run_scenario.py b/rate_design/hp_rates/run_scenario.py index d5249b1c..38516d0a 100644 --- a/rate_design/hp_rates/run_scenario.py +++ b/rate_design/hp_rates/run_scenario.py @@ -117,6 +117,7 @@ class ScenarioSettings: path_tou_supply_energy_mc: str | Path | None = None path_tou_supply_capacity_mc: str | Path | None = None path_supply_ancillary_mc: str | Path | None = None + residual_cost_frac: float | None = None def apply_prototype_sample( @@ -241,13 +242,38 @@ def _build_settings_from_yaml_run( _require_value(run, "run_includes_subclasses"), "run_includes_subclasses", ) - rr_config: RevenueRequirementConfig = _parse_utility_revenue_requirement( - _require_value(run, "utility_revenue_requirement"), - path_config, - raw_path_tariffs_electric, - add_supply=run_includes_supply, - run_includes_subclasses=run_includes_subclasses, - ) + residual_cost_frac_raw = run.get("residual_cost_frac") + residual_cost_frac: float | None = None + if residual_cost_frac_raw is not None: + residual_cost_frac = _parse_float(residual_cost_frac_raw, "residual_cost_frac") + urr_raw = run.get("utility_revenue_requirement") + urr_present = urr_raw is not None and str(urr_raw).strip() not in ( + "", + "none", + "null", + ) + if residual_cost_frac is not None and urr_present: + raise ValueError( + "Specify exactly one of 'residual_cost_frac' or 'utility_revenue_requirement', " + "not both. Set 'utility_revenue_requirement: none' (or omit it) when using " + "residual_cost_frac." + ) + if residual_cost_frac is not None: + # When residual_cost_frac is set, revenue requirement is derived at runtime + # from ResStock loads × MC prices; no YAML RR file is needed. + rr_config = RevenueRequirementConfig( + rr_total=0.0, + subclass_rr=None, + run_includes_subclasses=run_includes_subclasses, + ) + else: + rr_config = _parse_utility_revenue_requirement( + _require_value(run, "utility_revenue_requirement"), + path_config, + raw_path_tariffs_electric, + add_supply=run_includes_supply, + run_includes_subclasses=run_includes_subclasses, + ) path_tariff_maps_gas = _resolve_path( str(_require_value(run, "path_tariff_maps_gas")), path_config, @@ -322,6 +348,7 @@ def _build_settings_from_yaml_run( path_supply_ancillary_mc=path_supply_ancillary_mc if run_includes_supply else None, + residual_cost_frac=residual_cost_frac, ) @@ -674,13 +701,24 @@ def run(settings: ScenarioSettings, num_workers: int | None = None) -> None: ) = _return_revenue_requirement_target( building_load=raw_load_elec, sample_weight=customer_metadata[["bldg_id", "weight"]], - revenue_requirement_target=settings.rr_total, + revenue_requirement_target=settings.rr_total + if settings.residual_cost_frac is None + else None, residual_cost=None, - residual_cost_frac=None, + residual_cost_frac=settings.residual_cost_frac, bulk_marginal_costs=bulk_marginal_costs, distribution_marginal_costs=dist_and_sub_tx_marginal_costs, low_income_strategy=None, ) + if revenue_requirement is None: + # residual_cost_frac was set: RR = Total MC (0% residual) + revenue_requirement = float(costs_by_type["Total System Costs ($)"]) + log.info( + "residual_cost_frac=%.4f: revenue_requirement derived from " + "Total System Costs = $%.0f", + settings.residual_cost_frac, + revenue_requirement, + ) effective_load_elec = raw_load_elec elasticity_tracker = pd.DataFrame() if settings.run_includes_subclasses: diff --git a/tests/pre/__init__.py b/tests/pre/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/pre/test_generate_adoption_scenario_yamls.py b/tests/pre/test_generate_adoption_scenario_yamls.py new file mode 100644 index 00000000..e977e8b4 --- /dev/null +++ b/tests/pre/test_generate_adoption_scenario_yamls.py @@ -0,0 +1,605 @@ +"""Tests for utils/pre/generate_adoption_scenario_yamls.py.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import pytest +import yaml + +from utils.pre.generate_adoption_scenario_yamls import ( + _replace_year_in_value, + _update_run_name, + main, +) + +ADOPTION_CONFIG: dict[str, Any] = { + "scenario_name": "test_scenario", + "year_labels": [2025, 2030], + "run_years": [2025, 2030], + "upgrades": {2: {"label": "hp", "fractions": [0.1, 0.2]}}, +} + +BASE_RUNS: dict[str, Any] = { + "runs": { + 1: { + "run_name": "ny_nyseg_run1_up00_precalc__flat", + "state": "NY", + "utility": "nyseg", + "run_type": "precalc", + "upgrade": "0", + "path_resstock_metadata": "/old/metadata-sb.parquet", + "path_resstock_loads": "/old/loads/", + "path_dist_and_sub_tx_mc": "s3://data.sb/switchbox/marginal_costs/ny/dist_and_sub_tx/utility=nyseg/year=2025/data.parquet", + "path_supply_energy_mc": "s3://data.sb/switchbox/marginal_costs/ny/supply/energy/utility=nyseg/year=2025/zero.parquet", + "path_supply_capacity_mc": "s3://data.sb/switchbox/marginal_costs/ny/supply/capacity/utility=nyseg/year=2025/zero.parquet", + "path_bulk_tx_mc": "s3://data.sb/switchbox/marginal_costs/ny/bulk_tx/utility=nyseg/year=2025/data.parquet", + "utility_revenue_requirement": "rev_requirement/nyseg.yaml", + "run_includes_supply": False, + "year_run": 2025, + }, + 2: { + "run_name": "ny_nyseg_run2_up00_precalc_supply__flat", + "state": "NY", + "utility": "nyseg", + "run_type": "precalc", + "upgrade": "0", + "path_resstock_metadata": "/old/metadata-sb.parquet", + "path_resstock_loads": "/old/loads/", + "path_dist_and_sub_tx_mc": "s3://data.sb/switchbox/marginal_costs/ny/dist_and_sub_tx/utility=nyseg/year=2025/data.parquet", + "path_supply_energy_mc": "s3://data.sb/switchbox/marginal_costs/ny/supply/energy/utility=nyseg/year=2025/data.parquet", + "path_supply_capacity_mc": "s3://data.sb/switchbox/marginal_costs/ny/supply/capacity/utility=nyseg/year=2025/data.parquet", + "path_bulk_tx_mc": "s3://data.sb/switchbox/marginal_costs/ny/bulk_tx/utility=nyseg/year=2025/data.parquet", + "utility_revenue_requirement": "rev_requirement/nyseg.yaml", + "run_includes_supply": True, + "year_run": 2025, + }, + } +} + +BASE_RUNS_WITH_CAMBIUM_T: dict[str, Any] = { + "runs": { + 1: { + "run_name": "ny_nyseg_run1_y2025_mixed__flat", + "state": "NY", + "utility": "nyseg", + "run_type": "precalc", + "upgrade": "0", + "path_resstock_metadata": "/old/metadata-sb.parquet", + "path_resstock_loads": "/old/loads/", + "path_dist_and_sub_tx_mc": "s3://data.sb/switchbox/marginal_costs/ny/dist_and_sub_tx/utility=nyseg/year=2025/data.parquet", + "path_supply_energy_mc": "s3://data.sb/nrel/cambium/2024/scenario=MidCase/t=2025/gea=NYISO/r=p127/data.parquet", + "path_supply_capacity_mc": "s3://data.sb/nrel/cambium/2024/scenario=MidCase/t=2025/gea=NYISO/r=p127/data.parquet", + "path_bulk_tx_mc": "", + "utility_revenue_requirement": None, + "run_includes_supply": True, + "year_run": 2025, + "residual_cost_frac": 0.0, + }, + } +} + + +def _write_yaml(path: Path, data: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(yaml.dump(data, default_flow_style=False)) + + +def _make_test_inputs( + tmp_path: Path, + base_runs: dict[str, Any] | None = None, + adoption_config: dict[str, Any] | None = None, +) -> tuple[Path, Path, Path, Path]: + if base_runs is None: + base_runs = BASE_RUNS + if adoption_config is None: + adoption_config = ADOPTION_CONFIG + path_base = tmp_path / "scenarios_nyseg.yaml" + _write_yaml(path_base, base_runs) + path_adopt = tmp_path / "adoption.yaml" + _write_yaml(path_adopt, adoption_config) + path_mat = tmp_path / "materialized" + for yr in [2025, 2030]: + (path_mat / f"year={yr}").mkdir(parents=True) + path_out = tmp_path / "scenarios_nyseg_adoption.yaml" + return path_base, path_adopt, path_mat, path_out + + +class TestReplaceYearInValue: + """_replace_year_in_value handles both year= (Hive) and t= (Cambium) tokens.""" + + def test_replaces_year_token(self) -> None: + assert ( + _replace_year_in_value("path/year=2025/data.parquet", 2025, 2030) + == "path/year=2030/data.parquet" + ) + + def test_replaces_t_token(self) -> None: + result = _replace_year_in_value( + "s3://cambium/t=2025/gea=NYISO/data.parquet", 2025, 2030 + ) + assert "t=2030" in result + assert "t=2025" not in result + + def test_replaces_both_tokens(self) -> None: + assert ( + _replace_year_in_value("year=2025/t=2025/x", 2025, 2030) + == "year=2030/t=2030/x" + ) + + def test_no_replacement_when_year_absent(self) -> None: + assert ( + _replace_year_in_value("year=2024/data.parquet", 2025, 2030) + == "year=2024/data.parquet" + ) + + def test_t_token_not_replaced_when_year_mismatch(self) -> None: + assert ( + _replace_year_in_value("t=2024/data.parquet", 2025, 2030) + == "t=2024/data.parquet" + ) + + def test_replaces_in_dict(self) -> None: + d = {"a": "year=2025/a.parquet", "b": "t=2025/b.parquet"} + result = _replace_year_in_value(d, 2025, 2030) + assert result == {"a": "year=2030/a.parquet", "b": "t=2030/b.parquet"} + + def test_replaces_in_nested_dict(self) -> None: + result = _replace_year_in_value( + {"inner": {"path": "year=2025/x.parquet"}}, 2025, 2030 + ) + assert result["inner"]["path"] == "year=2030/x.parquet" + + def test_replaces_in_list(self) -> None: + assert _replace_year_in_value( + ["year=2025/a.parquet", "t=2025/b.parquet"], 2025, 2030 + ) == ["year=2030/a.parquet", "t=2030/b.parquet"] + + def test_non_string_unchanged(self) -> None: + assert _replace_year_in_value(42, 2025, 2030) == 42 + assert _replace_year_in_value(None, 2025, 2030) is None + + +class TestUpdateRunName: + def test_no_double_underscore_suffix(self) -> None: + assert _update_run_name("ny_nyseg_run1", 2030) == "ny_nyseg_run1_y2030_mixed" + + def test_with_double_underscore_suffix(self) -> None: + assert ( + _update_run_name("ny_nyseg_run1_up00_precalc__flat", 2030) + == "ny_nyseg_run1_up00_precalc_y2030_mixed__flat" + ) + + +class TestMainBaseline: + def test_generates_correct_count(self, tmp_path: Path) -> None: + path_base, path_adopt, path_mat, path_out = _make_test_inputs(tmp_path) + main( + [ + "--base-scenario", + str(path_base), + "--runs", + "1,2", + "--adoption-config", + str(path_adopt), + "--materialized-dir", + str(path_mat), + "--output", + str(path_out), + ] + ) + result = yaml.safe_load(path_out.read_text()) + assert len(result["runs"]) == 4 # 2 years x 2 runs + + def test_output_keys(self, tmp_path: Path) -> None: + path_base, path_adopt, path_mat, path_out = _make_test_inputs(tmp_path) + main( + [ + "--base-scenario", + str(path_base), + "--runs", + "1,2", + "--adoption-config", + str(path_adopt), + "--materialized-dir", + str(path_mat), + "--output", + str(path_out), + ] + ) + result = yaml.safe_load(path_out.read_text()) + assert set(result["runs"].keys()) == {101, 102, 201, 202} + + def test_year_token_replaced_in_dist_mc_path(self, tmp_path: Path) -> None: + path_base, path_adopt, path_mat, path_out = _make_test_inputs(tmp_path) + main( + [ + "--base-scenario", + str(path_base), + "--runs", + "1", + "--adoption-config", + str(path_adopt), + "--materialized-dir", + str(path_mat), + "--output", + str(path_out), + ] + ) + result = yaml.safe_load(path_out.read_text()) + assert "year=2030" in result["runs"][201]["path_dist_and_sub_tx_mc"] + assert "year=2025" not in result["runs"][201]["path_dist_and_sub_tx_mc"] + + +class TestMainResidualCostFrac: + def _run_with_frac(self, tmp_path: Path, frac: str = "0.0") -> dict[str, Any]: + path_base, path_adopt, path_mat, path_out = _make_test_inputs(tmp_path) + main( + [ + "--base-scenario", + str(path_base), + "--runs", + "1,2", + "--adoption-config", + str(path_adopt), + "--materialized-dir", + str(path_mat), + "--output", + str(path_out), + "--residual-cost-frac", + frac, + ] + ) + return yaml.safe_load(path_out.read_text()) + + def test_frac_present_in_all_entries(self, tmp_path: Path) -> None: + for entry in self._run_with_frac(tmp_path)["runs"].values(): + assert entry.get("residual_cost_frac") == pytest.approx(0.0) + + def test_utility_revenue_requirement_none(self, tmp_path: Path) -> None: + for entry in self._run_with_frac(tmp_path)["runs"].values(): + assert entry.get("utility_revenue_requirement") is None + + def test_custom_frac_value(self, tmp_path: Path) -> None: + for entry in self._run_with_frac(tmp_path, frac="0.1")["runs"].values(): + assert entry.get("residual_cost_frac") == pytest.approx(0.1) + + def test_no_flag_leaves_field_absent(self, tmp_path: Path) -> None: + path_base, path_adopt, path_mat, path_out = _make_test_inputs(tmp_path) + main( + [ + "--base-scenario", + str(path_base), + "--runs", + "1", + "--adoption-config", + str(path_adopt), + "--materialized-dir", + str(path_mat), + "--output", + str(path_out), + ] + ) + result = yaml.safe_load(path_out.read_text()) + for entry in result["runs"].values(): + assert "residual_cost_frac" not in entry + + +class TestMainCambiumSupply: + def _run(self, tmp_path: Path, runs: str = "1,2") -> dict[str, Any]: + path_base, path_adopt, path_mat, path_out = _make_test_inputs(tmp_path) + main( + [ + "--base-scenario", + str(path_base), + "--runs", + runs, + "--adoption-config", + str(path_adopt), + "--materialized-dir", + str(path_mat), + "--output", + str(path_out), + "--cambium-supply", + "--cambium-gea", + "NYISO", + "--cambium-ba", + "p127", + ] + ) + return yaml.safe_load(path_out.read_text()) + + def test_supply_run_gets_cambium_energy_path(self, tmp_path: Path) -> None: + run_102 = self._run(tmp_path)["runs"][102] + assert "data.sb/nrel/cambium" in run_102["path_supply_energy_mc"] + assert "MidCase" in run_102["path_supply_energy_mc"] + + def test_supply_run_gets_cambium_capacity_path(self, tmp_path: Path) -> None: + assert ( + "data.sb/nrel/cambium" + in self._run(tmp_path)["runs"][102]["path_supply_capacity_mc"] + ) + + def test_delivery_run_supply_paths_not_overwritten(self, tmp_path: Path) -> None: + run_101 = self._run(tmp_path)["runs"][101] + assert "data.sb/nrel/cambium" not in run_101["path_supply_energy_mc"] + assert "data.sb/nrel/cambium" not in run_101["path_supply_capacity_mc"] + + def test_bulk_tx_cleared_for_all_runs(self, tmp_path: Path) -> None: + for entry in self._run(tmp_path)["runs"].values(): + assert entry.get("path_bulk_tx_mc") == "" + + def test_cambium_path_uses_correct_year(self, tmp_path: Path) -> None: + run_202 = self._run(tmp_path)["runs"][202] + assert "t=2030" in run_202["path_supply_energy_mc"] + assert "t=2025" not in run_202["path_supply_energy_mc"] + + def test_cambium_path_includes_gea_and_ba(self, tmp_path: Path) -> None: + path = self._run(tmp_path)["runs"][102]["path_supply_energy_mc"] + assert "gea=NYISO" in path + assert "r=p127" in path + + def test_cambium_supply_without_ba_raises(self, tmp_path: Path) -> None: + path_base, path_adopt, path_mat, path_out = _make_test_inputs(tmp_path) + with pytest.raises(ValueError, match="--cambium-ba"): + main( + [ + "--base-scenario", + str(path_base), + "--runs", + "1", + "--adoption-config", + str(path_adopt), + "--materialized-dir", + str(path_mat), + "--output", + str(path_out), + "--cambium-supply", + ] + ) + + +class TestMainCambiumDistMcBase: + CAMBIUM_BASE = "s3://data.sb/switchbox/marginal_costs/ny/cambium_dist_and_sub_tx" + + def _run(self, tmp_path: Path) -> dict[str, Any]: + path_base, path_adopt, path_mat, path_out = _make_test_inputs(tmp_path) + main( + [ + "--base-scenario", + str(path_base), + "--runs", + "1", + "--adoption-config", + str(path_adopt), + "--materialized-dir", + str(path_mat), + "--output", + str(path_out), + "--cambium-dist-mc-base", + self.CAMBIUM_BASE, + ] + ) + return yaml.safe_load(path_out.read_text()) + + def test_dist_mc_path_uses_cambium_base(self, tmp_path: Path) -> None: + for entry in self._run(tmp_path)["runs"].values(): + assert "cambium_dist_and_sub_tx" in entry["path_dist_and_sub_tx_mc"] + + def test_dist_mc_path_contains_utility_partition(self, tmp_path: Path) -> None: + for entry in self._run(tmp_path)["runs"].values(): + assert "utility=nyseg" in entry["path_dist_and_sub_tx_mc"] + + def test_dist_mc_path_year_matches_run_year(self, tmp_path: Path) -> None: + result = self._run(tmp_path) + assert "year=2025" in result["runs"][101]["path_dist_and_sub_tx_mc"] + assert "year=2030" in result["runs"][201]["path_dist_and_sub_tx_mc"] + + def test_dist_mc_path_ends_with_data_parquet(self, tmp_path: Path) -> None: + for entry in self._run(tmp_path)["runs"].values(): + assert entry["path_dist_and_sub_tx_mc"].endswith("/data.parquet") + + +class TestMainTTokenReplacement: + """End-to-end: base runs with Cambium t= paths get tokens updated per year.""" + + def test_t_token_replaced_in_supply_mc_paths(self, tmp_path: Path) -> None: + path_base = tmp_path / "scenarios_nyseg.yaml" + _write_yaml(path_base, BASE_RUNS_WITH_CAMBIUM_T) + path_adopt = tmp_path / "adoption.yaml" + _write_yaml(path_adopt, ADOPTION_CONFIG) + path_mat = tmp_path / "materialized" + for yr in [2025, 2030]: + (path_mat / f"year={yr}").mkdir(parents=True) + path_out = tmp_path / "out.yaml" + main( + [ + "--base-scenario", + str(path_base), + "--runs", + "1", + "--adoption-config", + str(path_adopt), + "--materialized-dir", + str(path_mat), + "--output", + str(path_out), + ] + ) + result = yaml.safe_load(path_out.read_text()) + # key 101: year_index 0 (2025) - t= should stay t=2025 + assert "t=2025" in result["runs"][101]["path_supply_energy_mc"] + # key 201: year_index 1 (2030) - t= should be updated to t=2030 + assert "t=2030" in result["runs"][201]["path_supply_energy_mc"] + assert "t=2025" not in result["runs"][201]["path_supply_energy_mc"] + + +class TestHiveLoadsPath: + """path_resstock_loads is the hive-leaf upgrade=00/ dir, not a flat loads/ dir.""" + + def _run(self, tmp_path: Path) -> dict[str, Any]: + path_base, path_adopt, path_mat, path_out = _make_test_inputs(tmp_path) + main( + [ + "--base-scenario", + str(path_base), + "--runs", + "1", + "--adoption-config", + str(path_adopt), + "--materialized-dir", + str(path_mat), + "--output", + str(path_out), + ] + ) + return yaml.safe_load(path_out.read_text()) + + def test_loads_path_contains_load_curve_hourly(self, tmp_path: Path) -> None: + result = self._run(tmp_path) + for entry in result["runs"].values(): + assert "load_curve_hourly" in entry["path_resstock_loads"] + + def test_loads_path_contains_state_partition(self, tmp_path: Path) -> None: + result = self._run(tmp_path) + for entry in result["runs"].values(): + assert "state=NY" in entry["path_resstock_loads"] + + def test_loads_path_contains_upgrade_partition(self, tmp_path: Path) -> None: + result = self._run(tmp_path) + for entry in result["runs"].values(): + assert "upgrade=00" in entry["path_resstock_loads"] + + def test_loads_path_does_not_contain_flat_loads(self, tmp_path: Path) -> None: + result = self._run(tmp_path) + for entry in result["runs"].values(): + assert "/loads/" not in entry["path_resstock_loads"] + + def test_loads_path_year_matches_run_year(self, tmp_path: Path) -> None: + result = self._run(tmp_path) + assert "year=2025" in result["runs"][101]["path_resstock_loads"] + assert "year=2030" in result["runs"][201]["path_resstock_loads"] + + +# Base run with run_includes_subclasses=True for adoption-tariff-dir tests. +BASE_RUNS_SUBCLASS: dict[str, Any] = { + "runs": { + 5: { + "run_name": "ny_nyseg_run5_up00_precalc__hp_seasonal_vs_flat", + "state": "NY", + "utility": "nyseg", + "run_type": "precalc", + "upgrade": "0", + "path_resstock_metadata": "/old/metadata-sb.parquet", + "path_resstock_loads": "/old/loads/", + "path_dist_and_sub_tx_mc": "s3://dist/year=2025/data.parquet", + "path_supply_energy_mc": "s3://supply/year=2025/zero.parquet", + "path_supply_capacity_mc": "s3://supply/year=2025/zero.parquet", + "path_bulk_tx_mc": "", + "utility_revenue_requirement": None, + "run_includes_supply": False, + "run_includes_subclasses": True, + "year_run": 2025, + "path_tariffs_electric": { + "hp": "tariffs/electric/nyseg_hp_seasonal.json", + "non-hp": "tariffs/electric/nyseg_nonhp_flat.json", + }, + }, + } +} + + +class TestAdoptionTariffDir: + """--adoption-tariff-dir rewrites hp/non-hp paths for subclass runs only.""" + + def _run( + self, + tmp_path: Path, + adoption_tariff_dir: str | None = None, + runs: str = "5", + ) -> dict[str, Any]: + path_base = tmp_path / "scenarios_nyseg.yaml" + _write_yaml(path_base, BASE_RUNS_SUBCLASS) + path_adopt = tmp_path / "adoption.yaml" + _write_yaml(path_adopt, ADOPTION_CONFIG) + path_mat = tmp_path / "materialized" + for yr in [2025, 2030]: + (path_mat / f"year={yr}").mkdir(parents=True) + path_out = tmp_path / "out.yaml" + args = [ + "--base-scenario", + str(path_base), + "--runs", + runs, + "--adoption-config", + str(path_adopt), + "--materialized-dir", + str(path_mat), + "--output", + str(path_out), + ] + if adoption_tariff_dir is not None: + args += ["--adoption-tariff-dir", adoption_tariff_dir] + main(args) + return yaml.safe_load(path_out.read_text()) + + def test_hp_path_rewritten_with_tariff_dir(self, tmp_path: Path) -> None: + result = self._run(tmp_path, adoption_tariff_dir="/tariffs/adoption/cfg") + entry = result["runs"][105] + assert "/tariffs/adoption/cfg/year=2025" in entry["path_tariffs_electric"]["hp"] + assert "nyseg_hp_seasonal.json" in entry["path_tariffs_electric"]["hp"] + + def test_non_hp_path_rewritten_with_tariff_dir(self, tmp_path: Path) -> None: + result = self._run(tmp_path, adoption_tariff_dir="/tariffs/adoption/cfg") + entry = result["runs"][105] + assert ( + "/tariffs/adoption/cfg/year=2025" + in entry["path_tariffs_electric"]["non-hp"] + ) + assert "nyseg_nonhp_flat.json" in entry["path_tariffs_electric"]["non-hp"] + + def test_tariff_dir_includes_correct_year(self, tmp_path: Path) -> None: + result = self._run(tmp_path, adoption_tariff_dir="/tariffs/adoption/cfg") + entry_2030 = result["runs"][205] + assert "year=2030" in entry_2030["path_tariffs_electric"]["hp"] + assert "year=2025" not in entry_2030["path_tariffs_electric"]["hp"] + + def test_without_tariff_dir_paths_unchanged(self, tmp_path: Path) -> None: + result = self._run(tmp_path, adoption_tariff_dir=None) + entry = result["runs"][105] + assert ( + entry["path_tariffs_electric"]["hp"] + == "tariffs/electric/nyseg_hp_seasonal.json" + ) + assert ( + entry["path_tariffs_electric"]["non-hp"] + == "tariffs/electric/nyseg_nonhp_flat.json" + ) + + def test_non_subclass_run_not_affected(self, tmp_path: Path) -> None: + """Runs without run_includes_subclasses keep their original tariff paths.""" + path_base, path_adopt, path_mat, path_out = _make_test_inputs(tmp_path) + main( + [ + "--base-scenario", + str(path_base), + "--runs", + "1", + "--adoption-config", + str(path_adopt), + "--materialized-dir", + str(path_mat), + "--output", + str(path_out), + "--adoption-tariff-dir", + "/tariffs/adoption/cfg", + ] + ) + result = yaml.safe_load(path_out.read_text()) + # Run 1 uses path_tariffs_electric.all (not hp/non-hp); must be untouched. + for entry in result["runs"].values(): + tariff_elec = entry.get("path_tariffs_electric", {}) + assert "hp" not in tariff_elec or "/tariffs/adoption/cfg" not in str( + tariff_elec.get("hp", "") + ) diff --git a/tests/pre/test_list_adoption_years.py b/tests/pre/test_list_adoption_years.py new file mode 100644 index 00000000..46d76753 --- /dev/null +++ b/tests/pre/test_list_adoption_years.py @@ -0,0 +1,71 @@ +"""Tests for utils/pre/list_adoption_years.py.""" + +from __future__ import annotations + +import pytest + +from utils.pre.list_adoption_years import list_run_years, main + + +class TestListRunYears: + def test_all_labels_when_run_years_omitted(self) -> None: + config = { + "scenario_name": "test", + "year_labels": [2025, 2030, 2035], + } + assert list_run_years(config) == [2025, 2030, 2035] + + def test_run_years_subset(self) -> None: + config = { + "year_labels": [2025, 2030, 2035, 2040], + "run_years": [2025, 2040], + } + assert list_run_years(config) == [2025, 2040] + + def test_run_years_snaps_to_nearest(self) -> None: + config = { + "year_labels": [2025, 2030, 2035], + "run_years": [2027], + } + with pytest.warns(UserWarning, match="snapping"): + result = list_run_years(config) + assert result == [2025] + + def test_empty_year_labels(self) -> None: + config: dict = {"year_labels": []} + assert list_run_years(config) == [] + + def test_single_year(self) -> None: + config = {"year_labels": [2030]} + assert list_run_years(config) == [2030] + + def test_string_year_labels_normalised(self) -> None: + config = {"year_labels": ["2025", "2030"]} + assert list_run_years(config) == [2025, 2030] + + +class TestMain: + def test_prints_one_year_per_line(self, tmp_path, capsys) -> None: + p = tmp_path / "adoption.yaml" + p.write_text( + "scenario_name: test\nyear_labels: [2025, 2030, 2035]\n", + encoding="utf-8", + ) + main([str(p)]) + out = capsys.readouterr().out.strip().splitlines() + assert out == ["2025", "2030", "2035"] + + def test_respects_run_years(self, tmp_path, capsys) -> None: + p = tmp_path / "adoption.yaml" + p.write_text( + "scenario_name: test\nyear_labels: [2025, 2030, 2035]\nrun_years: [2030]\n", + encoding="utf-8", + ) + main([str(p)]) + out = capsys.readouterr().out.strip().splitlines() + assert out == ["2030"] + + def test_missing_arg_exits_nonzero(self, capsys) -> None: + with pytest.raises(SystemExit) as exc: + main([]) + assert exc.value.code != 0 diff --git a/tests/pre/test_materialize_mixed_upgrade.py b/tests/pre/test_materialize_mixed_upgrade.py new file mode 100644 index 00000000..59e2cce6 --- /dev/null +++ b/tests/pre/test_materialize_mixed_upgrade.py @@ -0,0 +1,1062 @@ +"""Tests for utils/pre/materialize_mixed_upgrade.py.""" + +from __future__ import annotations + +import csv +from pathlib import Path + +import polars as pl +import pytest + +from buildstock_fetch.scenarios import InvalidScenarioError, validate_scenario +from utils.buildstock import SbMixedUpgradeScenario +from utils.pre.materialize_mixed_upgrade import ( + _build_load_file_map, + _parse_adoption_config, + assign_buildings, + main, +) + +# --------------------------------------------------------------------------- +# Fixtures: minimal in-memory data +# --------------------------------------------------------------------------- + +N_BLDGS = 100 + +# Simple two-upgrade scenario with 3 years. +SCENARIO_2UP = { + 2: [0.10, 0.20, 0.30], + 4: [0.05, 0.10, 0.15], +} +RUN_YEAR_INDICES = [0, 1, 2] + + +def _bldg_ids(n: int = N_BLDGS) -> list[int]: + return list(range(1, n + 1)) + + +def _make_metadata_df( + bldg_ids: list[int], + has_hp: list[bool] | None = None, +) -> pl.DataFrame: + """Return a minimal metadata DataFrame with the columns that main() uses.""" + n = len(bldg_ids) + if has_hp is None: + has_hp = [False] * n + return pl.DataFrame( + { + "bldg_id": bldg_ids, + "postprocess_group.has_hp": has_hp, + "postprocess_group.heating_type": ["Gas"] * n, + "in.vintage_acs": ["2000s"] * n, + "applicability": [True] * n, + } + ) + + +def _write_metadata(path: Path, df: pl.DataFrame) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + df.write_parquet(path) + + +def _touch_load_file(loads_dir: Path, bldg_id: int, upgrade_id: int) -> Path: + loads_dir.mkdir(parents=True, exist_ok=True) + p = loads_dir / f"{bldg_id}-{upgrade_id:02d}.parquet" + p.touch() + return p + + +# --------------------------------------------------------------------------- +# 1. Building assignment: fractions +# --------------------------------------------------------------------------- + + +class TestAssignBuildingsFractions: + """assign_buildings() assigns approximately the right fraction to each upgrade.""" + + def test_correct_fraction_single_upgrade(self) -> None: + bldg_ids = _bldg_ids(200) + scenario = {2: [0.10, 0.25, 0.50]} + assignments = assign_buildings(bldg_ids, scenario, [0, 1, 2], random_seed=0) + + for t, expected_frac in zip([0, 1, 2], [0.10, 0.25, 0.50]): + assigned_to_2 = sum(1 for v in assignments[t].values() if v == 2) + assert assigned_to_2 == int(200 * expected_frac), ( + f"year index {t}: expected {int(200 * expected_frac)} buildings " + f"assigned to upgrade 2, got {assigned_to_2}" + ) + + def test_correct_fractions_two_upgrades(self) -> None: + bldg_ids = _bldg_ids(N_BLDGS) + assignments = assign_buildings( + bldg_ids, SCENARIO_2UP, RUN_YEAR_INDICES, random_seed=42 + ) + + for t in RUN_YEAR_INDICES: + for u, fracs in SCENARIO_2UP.items(): + expected = int(N_BLDGS * fracs[t]) + actual = sum(1 for v in assignments[t].values() if v == u) + assert actual == expected, ( + f"upgrade={u} year={t}: expected {expected}, got {actual}" + ) + + def test_all_buildings_covered(self) -> None: + bldg_ids = _bldg_ids(N_BLDGS) + assignments = assign_buildings( + bldg_ids, SCENARIO_2UP, RUN_YEAR_INDICES, random_seed=42 + ) + for t in RUN_YEAR_INDICES: + assert set(assignments[t].keys()) == set(bldg_ids) + + def test_remaining_buildings_stay_at_baseline(self) -> None: + """Buildings not yet assigned to any upgrade must be upgrade 0.""" + bldg_ids = _bldg_ids(N_BLDGS) + assignments = assign_buildings( + bldg_ids, SCENARIO_2UP, RUN_YEAR_INDICES, random_seed=42 + ) + for t in RUN_YEAR_INDICES: + total_hp = sum( + 1 for v in assignments[t].values() if v in SCENARIO_2UP.keys() + ) + total_baseline = sum(1 for v in assignments[t].values() if v == 0) + assert total_hp + total_baseline == N_BLDGS + + def test_empty_bldg_list_returns_empty(self) -> None: + result = assign_buildings([], SCENARIO_2UP, RUN_YEAR_INDICES, random_seed=0) + for t in RUN_YEAR_INDICES: + assert result[t] == {} + + def test_zero_fractions_all_baseline(self) -> None: + bldg_ids = _bldg_ids(50) + scenario = {2: [0.0, 0.0, 0.0]} + assignments = assign_buildings(bldg_ids, scenario, [0, 1, 2], random_seed=0) + for t in [0, 1, 2]: + assert all(v == 0 for v in assignments[t].values()) + + +# --------------------------------------------------------------------------- +# 2. Building assignment: monotonicity +# --------------------------------------------------------------------------- + + +class TestAssignBuildingsMonotonic: + """Buildings that adopt in year N must keep their upgrade in year N+1.""" + + def test_monotonic_adoption(self) -> None: + bldg_ids = _bldg_ids(N_BLDGS) + # Fractions increase over time — monotonic adoption. + scenario = {2: [0.10, 0.20, 0.30]} + assignments = assign_buildings(bldg_ids, scenario, [0, 1, 2], random_seed=1) + + adopted_t0 = {bid for bid, u in assignments[0].items() if u == 2} + adopted_t1 = {bid for bid, u in assignments[1].items() if u == 2} + adopted_t2 = {bid for bid, u in assignments[2].items() if u == 2} + + # Every building that adopted in year t must still have the same upgrade in t+1. + assert adopted_t0.issubset(adopted_t1), "Some t0-adopters reverted in t1" + assert adopted_t1.issubset(adopted_t2), "Some t1-adopters reverted in t2" + + def test_no_building_assigned_two_upgrades(self) -> None: + bldg_ids = _bldg_ids(N_BLDGS) + assignments = assign_buildings( + bldg_ids, SCENARIO_2UP, RUN_YEAR_INDICES, random_seed=7 + ) + for t in RUN_YEAR_INDICES: + for bid, uid in assignments[t].items(): + # At any given year each building has exactly one upgrade. + assert uid in {0} | set(SCENARIO_2UP.keys()) + + def test_reproducible_with_same_seed(self) -> None: + bldg_ids = _bldg_ids(N_BLDGS) + a1 = assign_buildings(bldg_ids, SCENARIO_2UP, RUN_YEAR_INDICES, random_seed=99) + a2 = assign_buildings(bldg_ids, SCENARIO_2UP, RUN_YEAR_INDICES, random_seed=99) + assert a1 == a2 + + def test_different_seeds_differ(self) -> None: + bldg_ids = _bldg_ids(200) + a1 = assign_buildings(bldg_ids, SCENARIO_2UP, RUN_YEAR_INDICES, random_seed=1) + a2 = assign_buildings(bldg_ids, SCENARIO_2UP, RUN_YEAR_INDICES, random_seed=2) + # With 200 buildings and non-trivial fractions it is astronomically unlikely + # for both seeds to produce identical assignments. + assert a1[0] != a2[0] + + +# --------------------------------------------------------------------------- +# 3. Applicability-restricted assignment +# --------------------------------------------------------------------------- + + +class TestAssignBuildingsApplicability: + """assign_buildings() with applicable_bldg_ids_per_upgrade restricts pools.""" + + def test_only_applicable_buildings_assigned(self) -> None: + """No building outside its upgrade's applicable set should be assigned to it.""" + bldg_ids = _bldg_ids(100) + # upgrade 2 applicable to first 60; upgrade 4 applicable to last 40 + applicable = {2: set(range(1, 61)), 4: set(range(61, 101))} + scenario = {2: [0.10, 0.20], 4: [0.05, 0.10]} + assignments = assign_buildings( + bldg_ids, + scenario, + [0, 1], + random_seed=0, + applicable_bldg_ids_per_upgrade=applicable, + ) + for t in [0, 1]: + for bid, uid in assignments[t].items(): + if uid == 2: + assert bid in applicable[2], ( + f"bldg {bid} assigned to upgrade 2 but not applicable" + ) + if uid == 4: + assert bid in applicable[4], ( + f"bldg {bid} assigned to upgrade 4 but not applicable" + ) + + def test_non_overlapping_applicable_sets(self) -> None: + """Buildings in only one applicable set are assigned to that upgrade.""" + bldg_ids = _bldg_ids(100) + # Disjoint sets: upgrade 2 → 1-50, upgrade 4 → 51-100 + applicable = {2: set(range(1, 51)), 4: set(range(51, 101))} + scenario = {2: [0.30], 4: [0.20]} + assignments = assign_buildings( + bldg_ids, + scenario, + [0], + random_seed=0, + applicable_bldg_ids_per_upgrade=applicable, + ) + assigned_to_2 = {bid for bid, u in assignments[0].items() if u == 2} + assigned_to_4 = {bid for bid, u in assignments[0].items() if u == 4} + assert assigned_to_2.issubset(applicable[2]) + assert assigned_to_4.issubset(applicable[4]) + assert assigned_to_2.isdisjoint(assigned_to_4) + + def test_applicable_smaller_than_target_warns_and_caps(self) -> None: + """When applicable pool is smaller than target count, a warning is emitted.""" + bldg_ids = _bldg_ids(100) + # upgrade 5 only applicable to 5 buildings but scenario requests 30% + applicable = {5: set(range(1, 6))} + scenario = {5: [0.30]} + with pytest.warns(UserWarning, match="Upgrade 5"): + assignments = assign_buildings( + bldg_ids, + scenario, + [0], + random_seed=0, + applicable_bldg_ids_per_upgrade=applicable, + ) + assigned_to_5 = sum(1 for u in assignments[0].values() if u == 5) + assert assigned_to_5 == 5 # capped at pool size + + def test_overlapping_sets_no_double_assignment(self) -> None: + """When applicable sets overlap, each building gets at most one upgrade.""" + bldg_ids = _bldg_ids(100) + # Both upgrades applicable to all 100 buildings. + applicable = {2: set(range(1, 101)), 4: set(range(1, 101))} + scenario = {2: [0.20], 4: [0.15]} + assignments = assign_buildings( + bldg_ids, + scenario, + [0], + random_seed=0, + applicable_bldg_ids_per_upgrade=applicable, + ) + for bid, uid in assignments[0].items(): + assert uid in {0, 2, 4}, f"bldg {bid} assigned unknown upgrade {uid}" + # No building should be in two upgrade pools + assigned_to_2 = {bid for bid, u in assignments[0].items() if u == 2} + assigned_to_4 = {bid for bid, u in assignments[0].items() if u == 4} + assert assigned_to_2.isdisjoint(assigned_to_4) + # Later upgrades overwrite overlapping assignments from earlier upgrades. + assert len(assigned_to_4) == 15 + assert len(assigned_to_2) + len(assigned_to_4) == 20 + + def test_monotonicity_preserved_with_applicability(self) -> None: + """Monotonic adoption holds when using applicable_bldg_ids_per_upgrade.""" + bldg_ids = _bldg_ids(N_BLDGS) + applicable = {2: set(range(1, 81))} # 80 buildings applicable for upgrade 2 + scenario = {2: [0.10, 0.20, 0.30]} + assignments = assign_buildings( + bldg_ids, + scenario, + [0, 1, 2], + random_seed=3, + applicable_bldg_ids_per_upgrade=applicable, + ) + adopted_t0 = {bid for bid, u in assignments[0].items() if u == 2} + adopted_t1 = {bid for bid, u in assignments[1].items() if u == 2} + adopted_t2 = {bid for bid, u in assignments[2].items() if u == 2} + assert adopted_t0.issubset(adopted_t1) + assert adopted_t1.issubset(adopted_t2) + + def test_none_applicable_fallback_identical_to_unrestricted(self) -> None: + """Passing applicable_bldg_ids_per_upgrade=None gives same result as omitting it.""" + bldg_ids = _bldg_ids(N_BLDGS) + a_restricted = assign_buildings( + bldg_ids, + SCENARIO_2UP, + RUN_YEAR_INDICES, + random_seed=42, + applicable_bldg_ids_per_upgrade=None, + ) + a_unrestricted = assign_buildings( + bldg_ids, + SCENARIO_2UP, + RUN_YEAR_INDICES, + random_seed=42, + ) + assert a_restricted == a_unrestricted + + +# --------------------------------------------------------------------------- +# 4. Metadata combination — unit tests via main() +# --------------------------------------------------------------------------- + + +class TestMetadataCombination: + """Combined metadata parquet has correct columns and row count.""" + + @pytest.fixture() + def fs(self, tmp_path: Path) -> Path: + """Build a minimal on-disk fixture and return the release root.""" + release = tmp_path / "release" + bldg_ids = list(range(1, 11)) # 10 buildings + # 3 buildings already have HPs in baseline; remaining 7 are eligible. + has_hp = [False] * 7 + [True] * 3 + + for uid in [0, 2]: + meta_path = ( + release + / "metadata" + / "state=NY" + / f"upgrade={uid:02d}" + / "metadata-sb.parquet" + ) + # upgrade=02: eligible buildings (first 7) have has_hp=True (upgrade applied); + # already-HP buildings (last 3) also keep has_hp=True. + df = _make_metadata_df(bldg_ids, has_hp if uid == 0 else [True] * 10) + _write_metadata(meta_path, df) + loads_dir = ( + release / "load_curve_hourly" / "state=NY" / f"upgrade={uid:02d}" + ) + for bid in bldg_ids: + _touch_load_file(loads_dir, bid, uid) + + return release + + @pytest.fixture() + def adoption_yaml(self, tmp_path: Path) -> Path: + content = ( + "scenario_name: test_scenario\n" + "random_seed: 0\n" + "scenario:\n" + " 2: [0.20, 0.40]\n" + "year_labels: [2025, 2030]\n" + ) + p = tmp_path / "adoption.yaml" + p.write_text(content, encoding="utf-8") + return p + + def test_all_required_columns_present( + self, fs: Path, adoption_yaml: Path, tmp_path: Path + ) -> None: + out_dir = tmp_path / "out" + main( + [ + "--state", + "ny", + "--utility", + "test", + "--adoption-config", + str(adoption_yaml), + "--path-resstock-release", + str(fs), + "--output-dir", + str(out_dir), + ] + ) + for year in [2025, 2030]: + df = pl.read_parquet(out_dir / f"year={year}" / "metadata-sb.parquet") + for col in [ + "bldg_id", + "postprocess_group.has_hp", + "postprocess_group.heating_type", + "in.vintage_acs", + "applicability", + ]: + assert col in df.columns, f"Missing column '{col}' for year={year}" + + def test_each_building_appears_exactly_once( + self, fs: Path, adoption_yaml: Path, tmp_path: Path + ) -> None: + out_dir = tmp_path / "out" + main( + [ + "--state", + "ny", + "--utility", + "test", + "--adoption-config", + str(adoption_yaml), + "--path-resstock-release", + str(fs), + "--output-dir", + str(out_dir), + ] + ) + for year in [2025, 2030]: + df = pl.read_parquet(out_dir / f"year={year}" / "metadata-sb.parquet") + assert df.shape[0] == 10, ( + f"year={year}: expected 10 rows, got {df.shape[0]}" + ) + assert df["bldg_id"].n_unique() == 10, f"year={year}: duplicate bldg_ids" + + def test_already_hp_buildings_pinned_to_baseline( + self, fs: Path, adoption_yaml: Path, tmp_path: Path + ) -> None: + """The 3 buildings that already have HP stay at upgrade-0 metadata in all years.""" + out_dir = tmp_path / "out" + main( + [ + "--state", + "ny", + "--utility", + "test", + "--adoption-config", + str(adoption_yaml), + "--path-resstock-release", + str(fs), + "--output-dir", + str(out_dir), + ] + ) + for year in [2025, 2030]: + df = pl.read_parquet(out_dir / f"year={year}" / "metadata-sb.parquet") + # Buildings 8, 9, 10 have has_hp=True in baseline (indices 7-9, bldg_ids 8-10). + already_hp_ids = [8, 9, 10] + already_hp_df = df.filter(pl.col("bldg_id").is_in(already_hp_ids)) + assert already_hp_df["postprocess_group.has_hp"].to_list() == [True] * 3 + + def test_more_hp_buildings_at_later_year( + self, fs: Path, adoption_yaml: Path, tmp_path: Path + ) -> None: + """Later years should have a higher fraction of HP buildings.""" + out_dir = tmp_path / "out" + main( + [ + "--state", + "ny", + "--utility", + "test", + "--adoption-config", + str(adoption_yaml), + "--path-resstock-release", + str(fs), + "--output-dir", + str(out_dir), + ] + ) + df_early = pl.read_parquet(out_dir / "year=2025" / "metadata-sb.parquet") + df_late = pl.read_parquet(out_dir / "year=2030" / "metadata-sb.parquet") + + # Upgrade-2 metadata has has_hp=None (defaulted to False in fixture). + # Count rows with has_hp True: comes from already-HP + newly assigned. + hp_early = df_early["postprocess_group.has_hp"].sum() + hp_late = df_late["postprocess_group.has_hp"].sum() + assert hp_late >= hp_early + + +# --------------------------------------------------------------------------- +# 5. Symlink creation +# --------------------------------------------------------------------------- + + +class TestSymlinkCreation: + """loads/ directory contains correctly targeted symlinks.""" + + @pytest.fixture() + def fs_and_out(self, tmp_path: Path) -> tuple[Path, Path]: + """Fixture: 5 buildings, upgrades 0 and 2.""" + release = tmp_path / "release" + bldg_ids = list(range(1, 6)) + + for uid in [0, 2]: + meta = ( + release + / "metadata" + / "state=RI" + / f"upgrade={uid:02d}" + / "metadata-sb.parquet" + ) + has_hp = [False] * len(bldg_ids) if uid == 0 else [True] * len(bldg_ids) + _write_metadata(meta, _make_metadata_df(bldg_ids, has_hp)) + loads_dir = ( + release / "load_curve_hourly" / "state=RI" / f"upgrade={uid:02d}" + ) + for bid in bldg_ids: + _touch_load_file(loads_dir, bid, uid) + + adoption_yaml = tmp_path / "adoption.yaml" + adoption_yaml.write_text( + "scenario_name: test\nrandom_seed: 0\nscenario:\n 2: [0.40]\n" + "year_labels: [2025]\n", + encoding="utf-8", + ) + out_dir = tmp_path / "out" + main( + [ + "--state", + "ri", + "--utility", + "test", + "--adoption-config", + str(adoption_yaml), + "--path-resstock-release", + str(release), + "--output-dir", + str(out_dir), + ] + ) + return release, out_dir + + def _loads_dir(self, out_dir: Path, year: int = 2025, state: str = "RI") -> Path: + return ( + out_dir + / f"year={year}" + / "load_curve_hourly" + / f"state={state}" + / "upgrade=00" + ) + + def test_loads_dir_exists(self, fs_and_out: tuple[Path, Path]) -> None: + _, out_dir = fs_and_out + assert self._loads_dir(out_dir).is_dir() + + def test_symlink_count_equals_building_count( + self, fs_and_out: tuple[Path, Path] + ) -> None: + _, out_dir = fs_and_out + links = list(self._loads_dir(out_dir).iterdir()) + assert len(links) == 5 + + def test_symlinks_are_actual_symlinks(self, fs_and_out: tuple[Path, Path]) -> None: + _, out_dir = fs_and_out + for p in self._loads_dir(out_dir).iterdir(): + assert p.is_symlink(), f"{p} is not a symlink" + + def test_symlink_targets_exist(self, fs_and_out: tuple[Path, Path]) -> None: + release, out_dir = fs_and_out + for p in self._loads_dir(out_dir).iterdir(): + assert p.resolve().exists(), f"Dangling symlink: {p}" + + def test_symlink_filename_convention(self, fs_and_out: tuple[Path, Path]) -> None: + """All symlink names follow {bldg_id}-{upgrade_id}.parquet.""" + _, out_dir = fs_and_out + for p in self._loads_dir(out_dir).iterdir(): + stem = p.stem # e.g. "3-02" + parts = stem.split("-", maxsplit=1) + assert len(parts) == 2, f"Unexpected filename: {p.name}" + bldg_id_str, upgrade_str = parts + assert bldg_id_str.isdigit(), f"Non-numeric bldg_id in {p.name}" + assert upgrade_str.isdigit(), f"Non-numeric upgrade_id in {p.name}" + + def test_assigned_buildings_link_to_correct_upgrade( + self, fs_and_out: tuple[Path, Path] + ) -> None: + """Buildings assigned to upgrade 2 must symlink to upgrade=02 load files.""" + release, out_dir = fs_and_out + for link in self._loads_dir(out_dir).iterdir(): + target = link.resolve() + # The upgrade_id is encoded in the filename (e.g. "3-02.parquet"). + stem = link.stem + upgrade_str = stem.split("-", maxsplit=1)[1] + expected_upgrade_dir = f"upgrade={int(upgrade_str):02d}" + assert expected_upgrade_dir in str(target), ( + f"Symlink {link.name} → {target} does not point into {expected_upgrade_dir}" + ) + + def test_hive_partition_path(self, fs_and_out: tuple[Path, Path]) -> None: + """Symlinks live under load_curve_hourly/state=RI/upgrade=00/ (hive layout).""" + _, out_dir = fs_and_out + hive_dir = self._loads_dir(out_dir) + assert hive_dir.is_dir() + # No flat 'loads/' directory should exist. + assert not (out_dir / "year=2025" / "loads").exists() + + +# --------------------------------------------------------------------------- +# 6. Scenario CSV output +# --------------------------------------------------------------------------- + + +class TestScenarioCsv: + """Scenario CSV is written with the correct structure.""" + + @pytest.fixture() + def out_dir(self, tmp_path: Path) -> Path: + release = tmp_path / "release" + bldg_ids = list(range(1, 9)) + + for uid in [0, 2]: + meta = ( + release + / "metadata" + / "state=NY" + / f"upgrade={uid:02d}" + / "metadata-sb.parquet" + ) + has_hp = [False] * len(bldg_ids) if uid == 0 else [True] * len(bldg_ids) + _write_metadata(meta, _make_metadata_df(bldg_ids, has_hp)) + ld = release / "load_curve_hourly" / "state=NY" / f"upgrade={uid:02d}" + for bid in bldg_ids: + _touch_load_file(ld, bid, uid) + + adoption_yaml = tmp_path / "adoption.yaml" + adoption_yaml.write_text( + "scenario_name: test\nrandom_seed: 0\nscenario:\n" + " 2: [0.25, 0.50]\nyear_labels: [2025, 2030]\n", + encoding="utf-8", + ) + out = tmp_path / "out" + main( + [ + "--state", + "ny", + "--utility", + "test", + "--adoption-config", + str(adoption_yaml), + "--path-resstock-release", + str(release), + "--output-dir", + str(out), + ] + ) + return out + + def test_csv_exists(self, out_dir: Path) -> None: + assert (out_dir / "scenario_assignments.csv").exists() + + def test_csv_columns(self, out_dir: Path) -> None: + with open( + out_dir / "scenario_assignments.csv", newline="", encoding="utf-8" + ) as f: + reader = csv.reader(f) + header = next(reader) + assert header[0] == "bldg_id" + assert "year_2025" in header + assert "year_2030" in header + + def test_csv_row_count(self, out_dir: Path) -> None: + with open( + out_dir / "scenario_assignments.csv", newline="", encoding="utf-8" + ) as f: + rows = list(csv.reader(f)) + # header + 8 buildings + assert len(rows) == 9 + + def test_csv_values_are_valid_upgrade_ids(self, out_dir: Path) -> None: + with open( + out_dir / "scenario_assignments.csv", newline="", encoding="utf-8" + ) as f: + reader = csv.DictReader(f) + for row in reader: + for col in ["year_2025", "year_2030"]: + assert int(row[col]) in {0, 2}, ( + f"Unexpected upgrade id in CSV: {row[col]}" + ) + + def test_csv_later_year_has_more_or_equal_adopters(self, out_dir: Path) -> None: + with open( + out_dir / "scenario_assignments.csv", newline="", encoding="utf-8" + ) as f: + reader = csv.DictReader(f) + rows = list(reader) + adopters_2025 = sum(1 for r in rows if int(r["year_2025"]) != 0) + adopters_2030 = sum(1 for r in rows if int(r["year_2030"]) != 0) + assert adopters_2030 >= adopters_2025 + + +# --------------------------------------------------------------------------- +# 7. Validation error paths +# --------------------------------------------------------------------------- + + +class TestValidationErrors: + """Error conditions: missing upgrade dirs, invalid fractions, etc.""" + + def test_missing_upgrade_metadata_raises(self, tmp_path: Path) -> None: + release = tmp_path / "release" + # Only create upgrade=00 metadata; upgrade=02 is missing. + meta = release / "metadata" / "state=NY" / "upgrade=00" / "metadata-sb.parquet" + _write_metadata(meta, _make_metadata_df([1, 2])) + + adoption_yaml = tmp_path / "adoption.yaml" + adoption_yaml.write_text( + "scenario_name: t\nrandom_seed: 0\nscenario:\n 2: [1.00]\n" + "year_labels: [2025]\n", + encoding="utf-8", + ) + with pytest.raises(FileNotFoundError, match="upgrade=02"): + main( + [ + "--state", + "ny", + "--utility", + "test", + "--adoption-config", + str(adoption_yaml), + "--path-resstock-release", + str(release), + "--output-dir", + str(tmp_path / "out"), + ] + ) + + def test_missing_loads_dir_raises(self, tmp_path: Path) -> None: + release = tmp_path / "release" + # Create metadata for both upgrades but omit loads dir for upgrade=02. + bldg_ids = [1, 2] + for uid in [0, 2]: + meta = ( + release + / "metadata" + / "state=NY" + / f"upgrade={uid:02d}" + / "metadata-sb.parquet" + ) + has_hp = [False] * len(bldg_ids) if uid == 0 else [True] * len(bldg_ids) + _write_metadata(meta, _make_metadata_df(bldg_ids, has_hp)) + loads_dir_0 = release / "load_curve_hourly" / "state=NY" / "upgrade=00" + loads_dir_0.mkdir(parents=True) + for bid in bldg_ids: + _touch_load_file(loads_dir_0, bid, 0) + + adoption_yaml = tmp_path / "adoption.yaml" + adoption_yaml.write_text( + "scenario_name: t\nrandom_seed: 0\nscenario:\n 2: [1.00]\n" + "year_labels: [2025]\n", + encoding="utf-8", + ) + with pytest.raises(FileNotFoundError, match="upgrade=02"): + main( + [ + "--state", + "ny", + "--utility", + "test", + "--adoption-config", + str(adoption_yaml), + "--path-resstock-release", + str(release), + "--output-dir", + str(tmp_path / "out"), + ] + ) + + def test_fractions_outside_range_raise(self) -> None: + with pytest.raises(InvalidScenarioError): + validate_scenario({2: [0.05, 1.10]}) # 1.10 > 1.0 + + def test_negative_fraction_raises(self) -> None: + with pytest.raises(InvalidScenarioError): + validate_scenario({2: [-0.01, 0.05]}) + + def test_non_monotonic_fractions_raise(self) -> None: + with pytest.raises(InvalidScenarioError): + validate_scenario({2: [0.30, 0.10]}) # decreasing + + def test_total_exceeds_one_raises(self) -> None: + with pytest.raises(InvalidScenarioError): + validate_scenario( + {2: [0.60, 0.70], 4: [0.50, 0.40]} + ) # sums to >1.0 in year 0 + + def test_missing_load_file_for_building_raises(self, tmp_path: Path) -> None: + """FileNotFoundError when a building's load file is absent from the loads dir.""" + release = tmp_path / "release" + bldg_ids = list(range(1, 4)) + + for uid in [0, 2]: + meta = ( + release + / "metadata" + / "state=NY" + / f"upgrade={uid:02d}" + / "metadata-sb.parquet" + ) + # upgrade=02 metadata must have has_hp=True so buildings are applicable. + has_hp = [False] * len(bldg_ids) if uid == 0 else [True] * len(bldg_ids) + _write_metadata(meta, _make_metadata_df(bldg_ids, has_hp)) + ld = release / "load_curve_hourly" / "state=NY" / f"upgrade={uid:02d}" + for bid in bldg_ids: + _touch_load_file(ld, bid, uid) + + # Remove load file for one building in upgrade=02. + missing = ( + release / "load_curve_hourly" / "state=NY" / "upgrade=02" / "2-02.parquet" + ) + missing.unlink() + + adoption_yaml = tmp_path / "adoption.yaml" + # Assign enough buildings to upgrade 2 so bldg_id=2 gets assigned. + adoption_yaml.write_text( + "scenario_name: t\nrandom_seed: 0\nscenario:\n 2: [0.67]\n" + "year_labels: [2025]\n", + encoding="utf-8", + ) + # The exact building assigned depends on shuffling; we accept either a + # successful run (if bldg_id=2 was not assigned to upgrade 2) or an error. + # To force the error deterministically, assign all buildings to upgrade 2. + adoption_yaml.write_text( + "scenario_name: t\nrandom_seed: 0\nscenario:\n 2: [1.00]\n" + "year_labels: [2025]\n", + encoding="utf-8", + ) + with pytest.raises(FileNotFoundError): + main( + [ + "--state", + "ny", + "--utility", + "test", + "--adoption-config", + str(adoption_yaml), + "--path-resstock-release", + str(release), + "--output-dir", + str(tmp_path / "out"), + ] + ) + + +# --------------------------------------------------------------------------- +# 8. Config parsing: run_years snap + year indices +# --------------------------------------------------------------------------- + + +class TestParseAdoptionConfig: + """_parse_adoption_config correctly handles run_years and year snapping.""" + + def test_all_years_when_run_years_omitted(self) -> None: + config = { + "scenario_name": "test", + "random_seed": 1, + "scenario": {2: [0.1, 0.2, 0.3]}, + "year_labels": [2025, 2030, 2035], + } + _, _, _, year_labels, run_year_indices = _parse_adoption_config(config) + assert run_year_indices == [0, 1, 2] + assert year_labels == [2025, 2030, 2035] + + def test_run_years_subset_selects_correct_indices(self) -> None: + config = { + "scenario_name": "test", + "random_seed": 1, + "scenario": {2: [0.1, 0.2, 0.3]}, + "year_labels": [2025, 2030, 2035], + "run_years": [2025, 2035], + } + _, _, _, _, run_year_indices = _parse_adoption_config(config) + assert run_year_indices == [0, 2] + + def test_run_years_snaps_to_nearest(self) -> None: + config = { + "scenario_name": "test", + "random_seed": 1, + "scenario": {2: [0.1, 0.2, 0.3]}, + "year_labels": [2025, 2030, 2035], + "run_years": [2028], + } + with pytest.warns(UserWarning, match="snapping"): + _, _, _, _, run_year_indices = _parse_adoption_config(config) + assert run_year_indices == [1] # snaps to 2030 + + def test_string_keys_normalised_to_int(self) -> None: + config = { + "scenario_name": "test", + "random_seed": 1, + "scenario": {"2": [0.1, 0.2], "4": [0.05, 0.10]}, + "year_labels": [2025, 2030], + } + _, _, scenario, _, _ = _parse_adoption_config(config) + assert 2 in scenario + assert 4 in scenario + assert "2" not in scenario + + +# --------------------------------------------------------------------------- +# 9. _build_load_file_map +# --------------------------------------------------------------------------- + + +class TestBuildLoadFileMap: + """_build_load_file_map scans a directory and returns {bldg_id: path}.""" + + def test_finds_matching_files(self, tmp_path: Path) -> None: + d = tmp_path / "loads" + d.mkdir() + (d / "1-02.parquet").touch() + (d / "3-02.parquet").touch() + (d / "99-02.parquet").touch() + + result = _build_load_file_map(d, {1, 3, 99}) + assert set(result.keys()) == {1, 3, 99} + + def test_filters_to_requested_bldg_ids(self, tmp_path: Path) -> None: + d = tmp_path / "loads" + d.mkdir() + (d / "1-02.parquet").touch() + (d / "2-02.parquet").touch() + (d / "3-02.parquet").touch() + + result = _build_load_file_map(d, {1, 3}) + assert set(result.keys()) == {1, 3} + + def test_ignores_non_parquet_files(self, tmp_path: Path) -> None: + d = tmp_path / "loads" + d.mkdir() + (d / "1-02.parquet").touch() + (d / "readme.txt").touch() + + result = _build_load_file_map(d, {1}) + assert set(result.keys()) == {1} + + def test_empty_directory_returns_empty(self, tmp_path: Path) -> None: + d = tmp_path / "loads" + d.mkdir() + result = _build_load_file_map(d, {1, 2}) + assert result == {} + + +# --------------------------------------------------------------------------- +# 10. SbMixedUpgradeScenario wrapper behavior +# --------------------------------------------------------------------------- + + +class TestSbMixedUpgradeScenario: + def test_release_mapping_via_main_uses_subdir(self, tmp_path: Path) -> None: + root = tmp_path / "resstock_root" + release_name = "res_2024_amy2018_2_sb" + release = root / release_name + bldg_ids = [1, 2, 3, 4] + + for uid in [0, 2]: + meta = ( + release + / "metadata" + / "state=NY" + / f"upgrade={uid:02d}" + / "metadata-sb.parquet" + ) + has_hp = [False] * len(bldg_ids) if uid == 0 else [True] * len(bldg_ids) + _write_metadata(meta, _make_metadata_df(bldg_ids, has_hp)) + loads = release / "load_curve_hourly" / "state=NY" / f"upgrade={uid:02d}" + for bid in bldg_ids: + _touch_load_file(loads, bid, uid) + + adoption_yaml = tmp_path / "adoption.yaml" + adoption_yaml.write_text( + "scenario_name: test\nrandom_seed: 0\nscenario:\n 2: [0.5]\n" + "year_labels: [2025]\n", + encoding="utf-8", + ) + + out_dir = tmp_path / "out" + main( + [ + "--state", + "ny", + "--utility", + "test", + "--adoption-config", + str(adoption_yaml), + "--path-resstock-release", + str(root), + "--release", + release_name, + "--output-dir", + str(out_dir), + ] + ) + assert (out_dir / "year=2025" / "metadata-sb.parquet").exists() + + def test_uses_metadata_sb_variant(self, tmp_path: Path) -> None: + release = tmp_path / "release" + bldg_ids = [1, 2, 3] + + for uid in [0, 2]: + meta = ( + release + / "metadata" + / "state=RI" + / f"upgrade={uid:02d}" + / "metadata-sb.parquet" + ) + has_hp = [False] * len(bldg_ids) if uid == 0 else [True] * len(bldg_ids) + _write_metadata(meta, _make_metadata_df(bldg_ids, has_hp)) + loads = release / "load_curve_hourly" / "state=RI" / f"upgrade={uid:02d}" + for bid in bldg_ids: + _touch_load_file(loads, bid, uid) + + adoption_yaml = tmp_path / "adoption.yaml" + adoption_yaml.write_text( + "scenario_name: test\nrandom_seed: 0\nscenario:\n 2: [0.34]\n" + "year_labels: [2025]\n", + encoding="utf-8", + ) + + out_dir = tmp_path / "out" + main( + [ + "--state", + "ri", + "--utility", + "test", + "--adoption-config", + str(adoption_yaml), + "--path-resstock-release", + str(release), + "--output-dir", + str(out_dir), + ] + ) + assert (out_dir / "year=2025" / "metadata-sb.parquet").exists() + + def test_hp_filtering_integration_baseline_pin_and_applicability( + self, tmp_path: Path + ) -> None: + release = tmp_path / "release" + bldg_ids = [1, 2, 3, 4, 5, 6] + baseline_has_hp = [False, False, False, True, True, True] + upgrade_2_has_hp = [True, True, False, False, False, False] + + _write_metadata( + release / "metadata" / "state=NY" / "upgrade=00" / "metadata-sb.parquet", + _make_metadata_df(bldg_ids, baseline_has_hp), + ) + _write_metadata( + release / "metadata" / "state=NY" / "upgrade=02" / "metadata-sb.parquet", + _make_metadata_df(bldg_ids, upgrade_2_has_hp), + ) + + mixed = SbMixedUpgradeScenario( + path_resstock_release=release, + state="ny", + scenario_name="test", + scenario={2: [0.5]}, + random_seed=0, + year_labels=[2025], + run_year_indices=[0], + ) + assignments = mixed.build_assignments(assign_buildings) + year0 = assignments[0] + + # Baseline-HP buildings must stay on upgrade 0. + for bldg_id in [4, 5, 6]: + assert year0[bldg_id] == 0 + + # Newly assigned HP buildings must come from upgrade-specific applicability. + assigned_to_2 = {bid for bid, uid in year0.items() if uid == 2} + assert assigned_to_2.issubset({1, 2}) diff --git a/tests/test_cambium_dist_mc.py b/tests/test_cambium_dist_mc.py new file mode 100644 index 00000000..8a18eb28 --- /dev/null +++ b/tests/test_cambium_dist_mc.py @@ -0,0 +1,171 @@ +"""Tests for Cambium busbar_load dist MC loading and empty bulk TX handling. + +Covers: +- load_cambium_load_profile: busbar_load → load_mw rename, utility column +- Missing required columns raise a clear error +- add_bulk_tx_and_dist_and_sub_tx_marginal_cost with empty/None path_bulk_tx_mc + returns dist-only MC (no bulk TX added) +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from pathlib import Path + +import pandas as pd +import polars as pl +import pytest + +from utils.cairo import add_bulk_tx_and_dist_and_sub_tx_marginal_cost +from utils.pre.marginal_costs.generate_utility_tx_dx_mc import ( + load_cambium_load_profile, +) + + +def _make_cambium_parquet(tmp_path: Path, *, rows: int = 8760) -> Path: + """Write a minimal Cambium-style parquet with timestamp + busbar_load.""" + timestamps = pl.datetime_range( + datetime(2025, 1, 1, 0, 0, 0), + datetime(2025, 12, 31, 23, 0, 0), + interval="1h", + eager=True, + )[:rows] + df = pl.DataFrame( + { + "timestamp": timestamps, + "busbar_load": [1000.0 + i * 0.1 for i in range(rows)], + "energy_cost_enduse": [0.05] * rows, + } + ) + path = tmp_path / "cambium.parquet" + df.write_parquet(path) + return path + + +def _make_dist_mc_parquet(tmp_path: Path, *, rows: int = 8760) -> Path: + """Write a minimal dist+sub-tx MC parquet in the format cairo expects. + + The loader (load_dist_and_sub_tx_marginal_costs) expects columns: + timestamp, mc_total_per_kwh (and optionally mc_upstream_per_kwh, mc_dist_per_kwh). + Timestamps must be tz-naive (the loader tz-localizes to EST). + """ + timestamps = pd.date_range("2025-01-01", periods=rows, freq="h") + df = pd.DataFrame( + { + "timestamp": timestamps, + "mc_total_per_kwh": [0.001] * rows, + "mc_upstream_per_kwh": [0.0005] * rows, + "mc_dist_per_kwh": [0.0005] * rows, + } + ) + path = tmp_path / "dist_mc.parquet" + df.to_parquet(path, index=False) + return path + + +class TestLoadCambiumLoadProfile: + def test_renames_busbar_load_to_load_mw(self, tmp_path: Path) -> None: + path = _make_cambium_parquet(tmp_path) + result = load_cambium_load_profile(str(path), "nyseg", {}) + assert "load_mw" in result.columns + assert "busbar_load" not in result.columns + + def test_adds_utility_column(self, tmp_path: Path) -> None: + path = _make_cambium_parquet(tmp_path) + result = load_cambium_load_profile(str(path), "nyseg", {}) + assert "utility" in result.columns + assert result["utility"].unique().to_list() == ["nyseg"] + + def test_preserves_row_count(self, tmp_path: Path) -> None: + path = _make_cambium_parquet(tmp_path) + result = load_cambium_load_profile(str(path), "nyseg", {}) + assert len(result) == 8760 + + def test_timestamp_column_preserved(self, tmp_path: Path) -> None: + path = _make_cambium_parquet(tmp_path) + result = load_cambium_load_profile(str(path), "nyseg", {}) + assert "timestamp" in result.columns + + def test_missing_busbar_load_raises(self, tmp_path: Path) -> None: + df = pl.DataFrame( + { + "timestamp": [datetime(2025, 1, 1)] * 10, + "energy_cost_enduse": [0.05] * 10, + } + ) + path = tmp_path / "bad.parquet" + df.write_parquet(path) + with pytest.raises(ValueError, match="busbar_load"): + load_cambium_load_profile(str(path), "nyseg", {}) + + def test_missing_timestamp_raises(self, tmp_path: Path) -> None: + df = pl.DataFrame({"busbar_load": [1000.0] * 10}) + path = tmp_path / "bad.parquet" + df.write_parquet(path) + with pytest.raises(ValueError, match="timestamp"): + load_cambium_load_profile(str(path), "nyseg", {}) + + def test_load_mw_values_match_busbar_load(self, tmp_path: Path) -> None: + path = _make_cambium_parquet(tmp_path) + result = load_cambium_load_profile(str(path), "nyseg", {}) + expected = [1000.0 + i * 0.1 for i in range(8760)] + actual = result["load_mw"].to_list() + assert actual == pytest.approx(expected, rel=1e-6) + + +class TestAddBulkTxEmptyPath: + """add_bulk_tx_and_dist_and_sub_tx_marginal_cost with empty/None bulk TX path + returns dist-only MC (no bulk TX contribution).""" + + def _make_target_index(self) -> pd.DatetimeIndex: + return pd.date_range("2025-01-01", periods=8760, freq="h", tz="EST") + + def test_empty_string_path_skips_bulk_tx(self, tmp_path: Path) -> None: + path_dist = _make_dist_mc_parquet(tmp_path) + target_idx = self._make_target_index() + + result = add_bulk_tx_and_dist_and_sub_tx_marginal_cost( + path_dist_and_sub_tx_mc=path_dist, + path_bulk_tx_mc="", + target_index=target_idx, + ) + + assert result is not None + assert len(result) == 8760 + assert result.sum() == pytest.approx(0.001 * 8760, rel=1e-4) + + def test_none_path_skips_bulk_tx(self, tmp_path: Path) -> None: + path_dist = _make_dist_mc_parquet(tmp_path) + target_idx = self._make_target_index() + + result = add_bulk_tx_and_dist_and_sub_tx_marginal_cost( + path_dist_and_sub_tx_mc=path_dist, + path_bulk_tx_mc=None, + target_index=target_idx, + ) + + assert result.sum() == pytest.approx(0.001 * 8760, rel=1e-4) + + def test_whitespace_only_path_skips_bulk_tx(self, tmp_path: Path) -> None: + path_dist = _make_dist_mc_parquet(tmp_path) + target_idx = self._make_target_index() + + result = add_bulk_tx_and_dist_and_sub_tx_marginal_cost( + path_dist_and_sub_tx_mc=path_dist, + path_bulk_tx_mc=" ", + target_index=target_idx, + ) + + assert result.sum() == pytest.approx(0.001 * 8760, rel=1e-4) + + def test_result_name_is_delivery_mc(self, tmp_path: Path) -> None: + path_dist = _make_dist_mc_parquet(tmp_path) + target_idx = self._make_target_index() + + result = add_bulk_tx_and_dist_and_sub_tx_marginal_cost( + path_dist_and_sub_tx_mc=path_dist, + path_bulk_tx_mc=None, + target_index=target_idx, + ) + + assert result.name == "Marginal Distribution Costs ($/kWh)" diff --git a/tests/test_create_scenario_yamls.py b/tests/test_create_scenario_yamls.py index 72a285e7..1aac8f46 100644 --- a/tests/test_create_scenario_yamls.py +++ b/tests/test_create_scenario_yamls.py @@ -74,3 +74,47 @@ def test_row_to_run_omits_path_tou_supply_mc_when_blank() -> None: run = _row_to_run(row, headers) assert "path_tou_supply_mc" not in run + + +def test_row_to_run_includes_residual_cost_frac_when_set() -> None: + """residual_cost_frac cell value is parsed as float and included.""" + row = _base_row() + row["residual_cost_frac"] = "0.0" + headers = list(row.keys()) + + run = _row_to_run(row, headers) + + assert run.get("residual_cost_frac") == 0.0 + + +def test_row_to_run_omits_residual_cost_frac_when_blank() -> None: + """Empty residual_cost_frac cell is omitted from output (existing runs unaffected).""" + row = _base_row() + row["residual_cost_frac"] = "" + headers = list(row.keys()) + + run = _row_to_run(row, headers) + + assert "residual_cost_frac" not in run + + +def test_row_to_run_omits_residual_cost_frac_when_column_absent() -> None: + """Column absence (older sheets) does not break parsing.""" + row = _base_row() + headers = [k for k in row if k != "residual_cost_frac"] + + run = _row_to_run(row, headers) + + assert "residual_cost_frac" not in run + + +def test_row_to_run_residual_cost_frac_invalid_raises() -> None: + """Non-numeric residual_cost_frac raises a clear error.""" + row = _base_row() + row["residual_cost_frac"] = "not-a-number" + headers = list(row.keys()) + + import pytest + + with pytest.raises(ValueError, match="residual_cost_frac"): + _row_to_run(row, headers) diff --git a/tests/test_scenario_config.py b/tests/test_scenario_config.py index 05f83b5e..083ceb30 100644 --- a/tests/test_scenario_config.py +++ b/tests/test_scenario_config.py @@ -7,6 +7,7 @@ from __future__ import annotations from pathlib import Path +from typing import Any import pytest import yaml @@ -193,3 +194,100 @@ def test_nested_subclass_format(self, tmp_path: Path) -> None: "hp_supply": 500.0, "nonhp_supply": 1000.0, } + + +class TestResidualCostFracYamlGuard: + """_build_settings_from_yaml_run: residual_cost_frac vs utility_revenue_requirement guard. + + Uses actual NYSEG scenario run 1 as the base (all file paths are real), then + overrides only the revenue-requirement-related fields to test the guard. + """ + + _SCENARIOS_NYSEG = ( + Path(__file__).resolve().parents[1] + / "rate_design" + / "hp_rates" + / "ny" + / "config" + / "scenarios" + / "scenarios_nyseg.yaml" + ) + + def _load_base_run(self) -> dict[str, Any]: + """Load run 1 from the real NYSEG scenarios YAML.""" + with self._SCENARIOS_NYSEG.open(encoding="utf-8") as f: + data = yaml.safe_load(f) + return dict(data["runs"][1]) + + def test_residual_cost_frac_and_urr_both_set_raises(self, tmp_path: Path) -> None: + """Setting both residual_cost_frac and utility_revenue_requirement raises.""" + from rate_design.hp_rates.run_scenario import _build_settings_from_yaml_run + + run = self._load_base_run() + run["residual_cost_frac"] = 0.0 + # utility_revenue_requirement already set in run 1; keep it → should raise + + with pytest.raises(ValueError, match="residual_cost_frac"): + _build_settings_from_yaml_run( + run=run, + run_num=1, + state="NY", + output_dir_override=tmp_path, + run_name_override="test", + ) + + def test_residual_cost_frac_with_urr_none_is_allowed(self, tmp_path: Path) -> None: + """residual_cost_frac=0.0 with utility_revenue_requirement: none is valid.""" + from rate_design.hp_rates.run_scenario import _build_settings_from_yaml_run + + run = self._load_base_run() + run["residual_cost_frac"] = 0.0 + run["utility_revenue_requirement"] = None + + settings = _build_settings_from_yaml_run( + run=run, + run_num=1, + state="NY", + output_dir_override=tmp_path, + run_name_override="test", + ) + + assert settings.residual_cost_frac == pytest.approx(0.0) + assert settings.rr_total == pytest.approx(0.0) + + def test_residual_cost_frac_with_urr_absent_is_allowed( + self, tmp_path: Path + ) -> None: + """residual_cost_frac=0.0 with utility_revenue_requirement absent is valid.""" + from rate_design.hp_rates.run_scenario import _build_settings_from_yaml_run + + run = self._load_base_run() + run["residual_cost_frac"] = 0.0 + run.pop("utility_revenue_requirement", None) + + settings = _build_settings_from_yaml_run( + run=run, + run_num=1, + state="NY", + output_dir_override=tmp_path, + run_name_override="test", + ) + + assert settings.residual_cost_frac == pytest.approx(0.0) + + def test_residual_cost_frac_absent_defaults_to_none(self, tmp_path: Path) -> None: + """When residual_cost_frac is not in the run dict, it defaults to None.""" + from rate_design.hp_rates.run_scenario import _build_settings_from_yaml_run + + run = self._load_base_run() + # residual_cost_frac absent; utility_revenue_requirement already set in run 1 + + settings = _build_settings_from_yaml_run( + run=run, + run_num=1, + state="NY", + output_dir_override=tmp_path, + run_name_override="test", + ) + + assert settings.residual_cost_frac is None diff --git a/utils/buildstock.py b/utils/buildstock.py new file mode 100644 index 00000000..53b41652 --- /dev/null +++ b/utils/buildstock.py @@ -0,0 +1,244 @@ +"""Buildstock integration for mixed-upgrade adoption materialization. + +This module keeps CAIRO-facing materialization logic in RDP while re-exporting +scenario helpers from `buildstock-fetch`. +""" + +from __future__ import annotations + +import csv +import os +import warnings +from pathlib import Path +from typing import Callable + +import polars as pl +from buildstock_fetch.scenarios import uniform_adoption, validate_scenario + +HAS_HP_COL = "postprocess_group.has_hp" +__all__ = ["SbMixedUpgradeScenario", "validate_scenario", "uniform_adoption"] + + +def _build_load_file_map(loads_dir: Path, bldg_ids: set[int]) -> dict[int, Path]: + """Return `{bldg_id: parquet_path}` for files present in `loads_dir`.""" + file_map: dict[int, Path] = {} + if not loads_dir.exists(): + return file_map + for path in loads_dir.glob("*.parquet"): + parts = path.stem.split("-", maxsplit=1) + if len(parts) != 2: + continue + if not parts[0].isdigit(): + continue + bldg_id = int(parts[0]) + if bldg_id in bldg_ids: + file_map[bldg_id] = path + return file_map + + +class SbMixedUpgradeScenario: + """Materialize mixed-upgrade scenario assignments for CAIRO input layout.""" + + def __init__( + self, + *, + path_resstock_release: Path, + state: str, + scenario_name: str, + scenario: dict[int, list[float]], + random_seed: int, + year_labels: list[int], + run_year_indices: list[int], + ) -> None: + self.path_resstock_release = path_resstock_release + self.state = state.upper() + self.scenario_name = scenario_name + self.scenario = scenario + self.random_seed = random_seed + self.year_labels = year_labels + self.run_year_indices = run_year_indices + self._metadata_cache: dict[int, pl.DataFrame] = {} + + validate_scenario(self.scenario) + + def _path_metadata(self, upgrade_id: int) -> Path: + return ( + self.path_resstock_release + / "metadata" + / f"state={self.state}" + / f"upgrade={upgrade_id:02d}" + / "metadata-sb.parquet" + ) + + def _path_loads(self, upgrade_id: int) -> Path: + return ( + self.path_resstock_release + / "load_curve_hourly" + / f"state={self.state}" + / f"upgrade={upgrade_id:02d}" + ) + + def _read_metadata(self, upgrade_id: int) -> pl.DataFrame: + cached = self._metadata_cache.get(upgrade_id) + if cached is not None: + return cached + + path = self._path_metadata(upgrade_id) + if not path.exists(): + raise FileNotFoundError( + f"Missing metadata file for upgrade={upgrade_id:02d}: {path}" + ) + df = pl.read_parquet(path) + self._metadata_cache[upgrade_id] = df + return df + + def compute_hp_pools( + self, + ) -> tuple[frozenset[int], frozenset[int], dict[int, set[int]]]: + """Return `(all_ids, eligible_ids, applicable_by_upgrade)`.""" + baseline_df = self._read_metadata(0) + all_ids = frozenset(baseline_df["bldg_id"].to_list()) + + if HAS_HP_COL in baseline_df.columns: + eligible = frozenset( + baseline_df.filter(pl.col(HAS_HP_COL) != True)["bldg_id"].to_list() # noqa: E712 + ) + else: + eligible = all_ids + + applicable_by_upgrade: dict[int, set[int]] = {} + for upgrade_id in sorted(self.scenario.keys()): + upgrade_df = self._read_metadata(upgrade_id) + if HAS_HP_COL not in upgrade_df.columns: + warnings.warn( + f"Upgrade {upgrade_id}: '{HAS_HP_COL}' missing; using full eligible pool.", + stacklevel=2, + ) + applicable_by_upgrade[upgrade_id] = set(eligible) + continue + applicable_ids = set( + upgrade_df.filter(pl.col(HAS_HP_COL) == True)["bldg_id"].to_list() # noqa: E712 + ) + applicable_by_upgrade[upgrade_id] = applicable_ids & set(eligible) + + return all_ids, eligible, applicable_by_upgrade + + def build_assignments( + self, + assign_buildings: Callable[ + [ + list[int], + dict[int, list[float]], + list[int], + int, + dict[int, set[int]] | None, + ], + dict[int, dict[int, int]], + ], + ) -> dict[int, dict[int, int]]: + """Build year-wise assignments including baseline-pinned buildings.""" + all_ids, eligible_ids, applicable_by_upgrade = self.compute_hp_pools() + + year_indices = list(range(len(self.year_labels))) + eligible_assignments = assign_buildings( + sorted(eligible_ids), + self.scenario, + year_indices, + self.random_seed, + applicable_by_upgrade, + ) + + full_assignments: dict[int, dict[int, int]] = {} + for year_idx in year_indices: + full_year = {bldg_id: 0 for bldg_id in all_ids} + full_year.update(eligible_assignments[year_idx]) + full_assignments[year_idx] = full_year + return full_assignments + + def materialize( + self, *, path_output_dir: Path, assignments: dict[int, dict[int, int]] + ) -> None: + """Write `metadata-sb.parquet` and load symlinks per run year.""" + path_output_dir.mkdir(parents=True, exist_ok=True) + + upgrades = [0, *sorted(self.scenario.keys())] + for upgrade_id in upgrades: + self._read_metadata(upgrade_id) + + for year_idx in self.run_year_indices: + calendar_year = self.year_labels[year_idx] + year_dir = path_output_dir / f"year={calendar_year}" + year_dir.mkdir(parents=True, exist_ok=True) + # All buildings land in a single upgrade=00 partition so that + # scan_resstock_loads (hive-partitioned) and CAIRO can share the + # same base path. The symlink targets still point at the correct + # per-building upgrade source files. + loads_out_dir = ( + year_dir / "load_curve_hourly" / f"state={self.state}" / "upgrade=00" + ) + loads_out_dir.mkdir(parents=True, exist_ok=True) + year_map = assignments[year_idx] + + bldgs_by_upgrade: dict[int, set[int]] = { + upgrade_id: set() for upgrade_id in upgrades + } + for bldg_id, upgrade_id in year_map.items(): + bldgs_by_upgrade.setdefault(upgrade_id, set()).add(bldg_id) + + metadata_parts: list[pl.DataFrame] = [] + for upgrade_id in upgrades: + bldg_ids = bldgs_by_upgrade[upgrade_id] + if not bldg_ids: + continue + + metadata_parts.append( + self._metadata_cache[upgrade_id].filter( + pl.col("bldg_id").is_in(sorted(bldg_ids)) + ) + ) + + loads_dir = self._path_loads(upgrade_id) + if not loads_dir.exists(): + raise FileNotFoundError( + f"Missing loads directory for upgrade={upgrade_id:02d}: {loads_dir}" + ) + load_map = _build_load_file_map(loads_dir, bldg_ids) + missing = sorted(bldg_ids - set(load_map.keys())) + if missing: + raise FileNotFoundError( + f"Missing load parquet(s) for upgrade={upgrade_id:02d}; " + f"first missing bldg_id={missing[0]}" + ) + for src in load_map.values(): + dst = loads_out_dir / src.name + if dst.exists() or dst.is_symlink(): + dst.unlink() + os.symlink(src.resolve(), dst) + + if not metadata_parts: + raise ValueError(f"No metadata rows found for year index {year_idx}") + + metadata_df = pl.concat(metadata_parts, how="diagonal_relaxed").sort("bldg_id") + metadata_df.write_parquet(year_dir / "metadata-sb.parquet") + + def export_scenario_csv( + self, *, path_output_dir: Path, assignments: dict[int, dict[int, int]] + ) -> None: + """Write `scenario_assignments.csv` in the existing format.""" + path_output_dir.mkdir(parents=True, exist_ok=True) + bldg_ids = sorted(assignments[0].keys()) + csv_path = path_output_dir / "scenario_assignments.csv" + + with open(csv_path, "w", encoding="utf-8", newline="") as f: + writer = csv.writer(f) + writer.writerow(["bldg_id", *[f"year_{y}" for y in self.year_labels]]) + for bldg_id in bldg_ids: + writer.writerow( + [ + bldg_id, + *[ + assignments[year_idx][bldg_id] + for year_idx in range(len(self.year_labels)) + ], + ] + ) diff --git a/utils/mid/patches.py b/utils/mid/patches.py index bab0e2b9..ab230947 100644 --- a/utils/mid/patches.py +++ b/utils/mid/patches.py @@ -94,12 +94,14 @@ def _return_loads_combined( # 2. Read timestamps from the first file only — all ResStock buildings share # the same 8760-hour series, so one file is sufficient. - first_table = pq.read_table(paths[0], columns=["timestamp"]) - ts_first = first_table.column("timestamp").to_numpy() + # Use ParquetFile to avoid dataset/Hive-partition inference that can + # fail when the parent path contains partition keys (e.g. year=2025). + pf = pq.ParquetFile(paths[0]) + ts_first = pf.read(columns=["timestamp"]).column("timestamp").to_numpy() + del pf assert len(ts_first) == 8760, ( f"Expected 8760 rows in first file, got {len(ts_first)}" ) - del first_table # 3. Compute timeshift parameters from source timestamps source_year = int(pd.Timestamp(ts_first[0]).year) @@ -118,14 +120,15 @@ def _return_loads_combined( unique_times = unique_times.tz_localize(force_tz) # 5. Read only the 3 data columns from all files (skip bldg_id and timestamp). - # Use the first file's schema — ResStock load files share identical schemas. - schema = pq.read_schema(paths[0]) + # Don't force a unified schema — mixed-upgrade symlinked files may have + # different encodings for columns we don't need (e.g. dictionary-encoded + # vs plain int32 for `year`). Omitting `schema` lets PyArrow unify per-file. _DATA_COLS = [ "out.electricity.total.energy_consumption", "out.electricity.pv.energy_consumption", "out.natural_gas.total.energy_consumption", ] - ds = pad.dataset(paths, format="parquet", schema=schema) + ds = pad.dataset(paths, format="parquet") table = ds.to_table(columns=_DATA_COLS) _log_mem("after to_table (3 data cols, arrow)") diff --git a/utils/pre/create_scenario_yamls.py b/utils/pre/create_scenario_yamls.py index d6058daf..37e6e135 100644 --- a/utils/pre/create_scenario_yamls.py +++ b/utils/pre/create_scenario_yamls.py @@ -327,6 +327,15 @@ def parse_required_float(key: str) -> float: run["elasticity"] = parse_required_float("elasticity") + residual_cost_frac_raw = get_optional("residual_cost_frac") + if residual_cost_frac_raw: + try: + run["residual_cost_frac"] = float(residual_cost_frac_raw) + except ValueError as exc: + raise ValueError( + f"residual_cost_frac must be a float, got {residual_cost_frac_raw!r}" + ) from exc + return run diff --git a/utils/pre/fit_adoption_config.py b/utils/pre/fit_adoption_config.py new file mode 100644 index 00000000..593229b7 --- /dev/null +++ b/utils/pre/fit_adoption_config.py @@ -0,0 +1,617 @@ +"""Fit logistic S-curves to digitized NYISO data and write adoption config YAML. + +Source: NYISO Gold Book 2025, "Number of Residential Households Converted to +Electric Heating By Technology (NYCA)" stacked-area chart. + +Parametric form: f(t) = L / (1 + exp(-k * (t - t0))) + L = long-run saturation fraction of all NYCA housing units + k = growth rate + t0 = inflection year + +Fractions are normalized by the total NY residential electric customer count +derived from EIA-861 (PUDL), summing bundled and delivery-only service types +for the most recent available year (excludes "energy" rows which duplicate +delivery-only customers who switched to an ESCO). 2025 is forced to 0.0 — all +buildings remain at upgrade-0 baseline — regardless of the logistic value. + +Technology → ResStock upgrade mapping: + ASHP Full Capacity → 2 (cold-climate ASHP, 90% capacity @ 5F, elec backup) + ASHP Dual Fuel → 4 (ENERGY STAR ASHP + existing fossil backup) + Ground Source HP → 5 (geothermal heat pump) + Supplemental Heat → 1 (ENERGY STAR ASHP, 50% capacity @ 5F, elec backup) + Electric Resistance → baseline upgrade 0, already captured there + +Usage:: + + uv run python utils/pre/fit_adoption_config.py \\ + --output rate_design/hp_rates/ny/config/adoption/nyca_electrification.yaml \\ + --plot-output rate_design/hp_rates/ny/config/adoption/nyca_electrification_curves.png +""" + +from __future__ import annotations + +import argparse +import logging +from pathlib import Path + +import numpy as np +import polars as pl +from cloudpathlib import S3Path +from plotnine import ( + aes, + element_line, + element_text, + geom_area, + geom_line, + geom_point, + geom_vline, + ggplot, + labs, + scale_color_manual, + scale_fill_manual, + scale_x_continuous, + scale_y_continuous, + theme, + theme_minimal, +) +from scipy.optimize import curve_fit + +from buildstock_fetch.scenarios import validate_scenario + +logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") +log = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Source data (NYISO Gold Book 2025) +# --------------------------------------------------------------------------- + +SCENARIO_NAME = "nyca_electrification" +RANDOM_SEED = 42 + +# S3 path to EIA-861 electric utility stats (Hive-partitioned by year and state). +_EIA861_S3_BASE = "s3://data.sb/eia/861/electric_utility_stats/" + +# --------------------------------------------------------------------------- +# EIA-861 residential customer count +# --------------------------------------------------------------------------- + + +def load_total_hu(state: str = "NY", max_year: int | None = None) -> tuple[float, int]: + """Return (total_residential_customers, year) from EIA-861 on S3. + + Sums bundled and delivery service types only — excludes "energy" rows + which duplicate delivery-only customers who switched to an ESCO supplier. + Uses the most recent partition year <= ``max_year`` (defaults to the + latest available year). + """ + from utils import get_aws_region + + storage_options = {"aws_region": get_aws_region()} + base = S3Path(_EIA861_S3_BASE) + year_dirs = sorted( + int(p.name.split("=")[1]) for p in base.iterdir() if p.name.startswith("year=") + ) + if not year_dirs: + raise FileNotFoundError( + f"No EIA-861 year partitions found at {_EIA861_S3_BASE}" + ) + if max_year is not None: + year_dirs = [y for y in year_dirs if y <= max_year] + if not year_dirs: + raise ValueError(f"No EIA-861 year partitions found <= {max_year}") + year = year_dirs[-1] + + path = f"{_EIA861_S3_BASE}year={year}/state={state}/data.parquet" + df = pl.scan_parquet(path, storage_options=storage_options) + + # The pre-aggregated parquet sums all service types (bundled + delivery + energy). + # We need bundled + delivery only, so we fall back to the PUDL source. + # Check whether a service_type column is present; if not, re-derive from PUDL. + schema = df.schema + if "service_type" in schema: + total = ( + df.filter(pl.col("service_type").is_in(["bundled", "delivery"])) + .select(pl.col("residential_customers").sum()) + .collect()["residential_customers"][0] + ) + else: + # Pre-aggregated file has no service_type; it already summed all types. + # Re-derive from PUDL directly to exclude the "energy" double-count. + pudl_version = "v2026.2.0" + pudl_url = ( + f"https://s3.us-west-2.amazonaws.com/pudl.catalyst.coop" + f"/{pudl_version}/core_eia861__yearly_sales.parquet" + ) + total = ( + pl.scan_parquet(pudl_url) + .filter( + (pl.col("state") == state) + & (pl.col("customer_class") == "residential") + & (pl.col("service_type").is_in(["bundled", "delivery"])) + & (pl.col("report_date").dt.year() == year) + ) + .select(pl.col("customers").sum()) + .collect()["customers"][0] + ) + + return float(total), year + + +# Digitized from the NYISO Gold Book 2025 NYCA stacked-area chart. +# Each entry: (calendar_year, individual_technology_housing_units_in_thousands). +# 2025 is forced to 0.0 by the evaluation logic below. +_RAW_DATA: dict[int, list[tuple[int, float]]] = { + 2: [ # ASHP Full Capacity + (2030, 75), + (2035, 250), + (2040, 640), + (2045, 1050), + (2050, 1400), + (2057, 1650), + ], + 4: [ # ASHP Dual Fuel + (2030, 65), + (2035, 205), + (2040, 440), + (2045, 660), + (2050, 750), + (2057, 850), + ], + 5: [ # Ground Source HP + (2030, 10), + (2035, 20), + (2040, 50), + (2045, 75), + (2050, 85), + (2057, 90), + ], + 1: [ # Supplemental Heat + (2030, 35), + (2035, 210), + (2040, 440), + (2045, 870), + (2050, 950), + (2057, 1000), + ], +} + +_UPGRADE_LABELS: dict[int, str] = { + 2: "ASHP full capacity", + 4: "ASHP dual fuel", + 5: "ground source HP", + 1: "supplemental heat", +} + +# Plot labels include the ResStock upgrade code so charts are self-documenting. +_UPGRADE_PLOT_LABELS: dict[int, str] = { + uid: f"upgrade {uid} — {label}" for uid, label in _UPGRADE_LABELS.items() +} + +# Wong colorblind-friendly palette matched to NYISO chart hues. +_UPGRADE_COLORS: dict[int, str] = { + 2: "#D55E00", # vermillion / orange + 4: "#999999", # gray + 5: "#0072B2", # blue + 1: "#009E73", # green +} + +_DEFAULT_RUN_YEARS: list[int] = [2025, 2030, 2035, 2040, 2045, 2050] + +# --------------------------------------------------------------------------- +# Logistic model +# --------------------------------------------------------------------------- + + +def _logistic(t: np.ndarray, L: float, k: float, t0: float) -> np.ndarray: + return L / (1.0 + np.exp(-k * (t - t0))) + + +def _fit_logistic(years: np.ndarray, fracs: np.ndarray) -> tuple[float, float, float]: + """Fit logistic to (years, fracs); return (L, k, t0).""" + L_min = float(fracs.max()) * 1.01 + p0 = [fracs.max() * 1.5, 0.10, 2045.0] + bounds = ([L_min, 0.001, 2020.0], [1.0, 1.0, 2080.0]) + popt, _ = curve_fit(_logistic, years, fracs, p0=p0, bounds=bounds, maxfev=20_000) + return float(popt[0]), float(popt[1]), float(popt[2]) + + +# --------------------------------------------------------------------------- +# Fit +# --------------------------------------------------------------------------- + + +def fit_all( + run_years: list[int], + total_hu: float, +) -> tuple[dict[int, list[float]], dict[int, tuple[float, float, float]]]: + """Fit logistic curves; return ``(scenario_fracs, params)``. + + ``scenario_fracs[upgrade_id][i]`` is the adoption fraction at + ``run_years[i]``. 2025 is forced to ``0.0``. + + Args: + run_years: Calendar years to evaluate. + total_hu: Total residential customer count used as the fraction denominator. + """ + scenario: dict[int, list[float]] = {} + params: dict[int, tuple[float, float, float]] = {} + + for upgrade_id, pts in _RAW_DATA.items(): + years_arr = np.array([y for y, _ in pts], dtype=float) + fracs_arr = np.array([hu * 1_000 / total_hu for _, hu in pts]) + + L, k, t0 = _fit_logistic(years_arr, fracs_arr) + params[upgrade_id] = (L, k, t0) + log.info( + "upgrade %d (%s): L=%.4f k=%.4f t0=%.1f", + upgrade_id, + _UPGRADE_LABELS[upgrade_id], + L, + k, + t0, + ) + + year_fracs: list[float] = [] + for yr in run_years: + if yr <= 2025: + year_fracs.append(0.0) + else: + val = float(_logistic(np.array([float(yr)]), L, k, t0)[0]) + year_fracs.append(round(val, 4)) + scenario[upgrade_id] = year_fracs + + return scenario, params + + +# --------------------------------------------------------------------------- +# YAML writer +# --------------------------------------------------------------------------- + + +def write_yaml( + path_output: Path, + scenario: dict[int, list[float]], + params: dict[int, tuple[float, float, float]], + run_years: list[int], + total_hu: float, + eia_year: int, +) -> None: + """Write adoption config YAML with full methodology commentary.""" + param_block = "\n".join( + f"# upgrade {uid} ({_UPGRADE_LABELS[uid]}): " + f"L={params[uid][0]:.4f} k={params[uid][1]:.4f} t0={params[uid][2]:.1f}" + for uid in [2, 4, 5, 1] + ) + + scenario_block = "\n".join( + f" {uid}: [{', '.join(f'{v:.4f}' for v in scenario[uid])}]" + f" # {_UPGRADE_LABELS[uid]}" + for uid in [2, 4, 5, 1] + ) + + year_labels_str = "[" + ", ".join(str(y) for y in run_years) + "]" + + lines = [ + "# NYCA building electrification adoption trajectory (NYISO Gold Book 2025).", + "# Generated by utils/pre/fit_adoption_config.py — do not edit by hand.", + "#", + "# Fractions represent the share of total NYCA buildings assigned to each", + "# ResStock upgrade at each year. Remaining buildings stay at upgrade 0 (baseline).", + "# Year indices map to calendar years via year_labels.", + "#", + "# Technology → ResStock upgrade mapping:", + "# ASHP Full Capacity → 2 (cold-climate ASHP, 90% capacity @ 5F, elec backup)", + "# ASHP Dual Fuel → 4 (ENERGY STAR ASHP + existing fossil backup)", + "# Ground Source HP → 5 (geothermal heat pump)", + "# Supplemental Heat → 1 (ENERGY STAR ASHP, 50% capacity @ 5F, elec backup)", + "# Electric Resistance → baseline upgrade 0, already captured there", + "#", + "# Methodology: logistic S-curves f(t) = L / (1 + exp(-k * (t - t0))) fit", + "# (scipy curve_fit) to housing-unit counts digitized from the NYISO Gold", + f"# Book 2025 NYCA stacked-area chart. Denominator: {total_hu:,.0f} NY residential", + f"# electric customers (EIA-861 {eia_year}, bundled + delivery service types,", + "# excluding ESCO energy-only rows which duplicate delivery customers).", + "# 2025 forced to 0.0 (all buildings at upgrade-0 baseline).", + "#", + "# Fitted parameters:", + param_block, + "", + f"scenario_name: {SCENARIO_NAME}", + f"random_seed: {RANDOM_SEED}", + "", + "scenario:", + scenario_block, + "", + "# Calendar years for each scenario index (= run years).", + "# Aligns with Cambium 5-year MC intervals; 2025 is baseline.", + f"year_labels: {year_labels_str}", + "", + ] + + path_output.parent.mkdir(parents=True, exist_ok=True) + path_output.write_text("\n".join(lines), encoding="utf-8") + log.info("wrote %s", path_output) + + +# --------------------------------------------------------------------------- +# Plot +# --------------------------------------------------------------------------- + +_PLOT_YEARS_DENSE = np.linspace(2024, 2058, 400) + + +def make_plot( + params: dict[int, tuple[float, float, float]], + run_years: list[int], + path_plot: Path, + total_hu: float, + eia_year: int, +) -> None: + """Save a plotnine figure: continuous logistic curves + digitized points.""" + # Build long-format DataFrame for fitted curves. + curve_rows: list[dict] = [] + for uid, (L, k, t0) in params.items(): + fracs = _logistic(_PLOT_YEARS_DENSE, L, k, t0) + for yr, frac in zip(_PLOT_YEARS_DENSE, fracs): + pct = float(frac) * 100.0 + # Clip negative values that arise from logistic tails near 2024. + curve_rows.append( + { + "year": float(yr), + "technology": _UPGRADE_PLOT_LABELS[uid], + "pct": max(pct, 0.0), + } + ) + + curves_df = pl.DataFrame(curve_rows) + + # Build long-format DataFrame for digitized source points (excluding 2025=0). + point_rows: list[dict] = [] + for uid, pts in _RAW_DATA.items(): + for yr, hu_k in pts: + point_rows.append( + { + "year": float(yr), + "technology": _UPGRADE_PLOT_LABELS[uid], + "pct": hu_k * 1_000 / total_hu * 100.0, + } + ) + + points_df = pl.DataFrame(point_rows) + + # Ordered technology names for the legend (matches NYISO chart order, bottom→top). + tech_order = [_UPGRADE_PLOT_LABELS[uid] for uid in [2, 4, 5, 1]] + color_map = { + _UPGRADE_PLOT_LABELS[uid]: _UPGRADE_COLORS[uid] for uid in [2, 4, 5, 1] + } + + # Convert to pandas for plotnine; use pandas Categorical for legend order. + import pandas as pd # noqa: PLC0415 + + curves_pd = curves_df.to_pandas() + curves_pd["technology"] = pd.Categorical( + curves_pd["technology"], categories=tech_order + ) + + points_pd = points_df.to_pandas() + points_pd["technology"] = pd.Categorical( + points_pd["technology"], categories=tech_order + ) + + vlines_df = pl.DataFrame( + {"year": [float(y) for y in run_years if y > 2025]} + ).to_pandas() + + p = ( + ggplot() + + geom_vline( + data=vlines_df, + mapping=aes(xintercept="year"), + color="#cccccc", + linetype="dashed", + size=0.5, + ) + + geom_line( + data=curves_pd, + mapping=aes(x="year", y="pct", color="technology"), + size=1.0, + ) + + geom_point( + data=points_pd, + mapping=aes(x="year", y="pct", color="technology"), + size=2.5, + shape="o", + fill="white", + stroke=1.2, + ) + + scale_color_manual(values=color_map, breaks=tech_order) + + scale_x_continuous( + breaks=list(range(2025, 2060, 5)), + limits=(2024, 2058), + ) + + scale_y_continuous( + labels=lambda x: [f"{v:.0f}%" for v in x], + ) + + labs( + title="NYCA HP adoption trajectory — NYISO Gold Book 2025 logistic fit", + subtitle=( + f"Denominator: {total_hu:,.0f} NY residential electric customers" + f" (EIA-861 {eia_year}, bundled + delivery)" + ), + x="Year", + y="Share of NY residential electric customers", + color="Technology (ResStock upgrade)", + ) + + theme_minimal() + + theme( + plot_title=element_text(size=11), + plot_subtitle=element_text(size=9), + axis_title=element_text(size=10), + legend_title=element_text(size=9), + legend_text=element_text(size=9), + panel_grid_minor=element_line(size=0), + ) + ) + + path_plot.parent.mkdir(parents=True, exist_ok=True) + p.save(str(path_plot), dpi=150, width=9, height=5) + log.info("wrote %s", path_plot) + + +def make_stacked_plot( + params: dict[int, tuple[float, float, float]], + run_years: list[int], + path_plot: Path, + total_hu: float, + eia_year: int, +) -> None: + """Save a stacked area chart matching the NYISO Gold Book visual style.""" + import pandas as pd # noqa: PLC0415 + + # Stacking order bottom→top mirrors the NYISO chart. + stack_order = [_UPGRADE_PLOT_LABELS[uid] for uid in [2, 4, 5, 1]] + fill_map = {_UPGRADE_PLOT_LABELS[uid]: _UPGRADE_COLORS[uid] for uid in [2, 4, 5, 1]} + + curve_rows: list[dict] = [] + for uid, (L, k, t0) in params.items(): + fracs = _logistic(_PLOT_YEARS_DENSE, L, k, t0) + for yr, frac in zip(_PLOT_YEARS_DENSE, fracs): + curve_rows.append( + { + "year": float(yr), + "technology": _UPGRADE_PLOT_LABELS[uid], + "pct": max(float(frac) * 100.0, 0.0), + } + ) + + curves_pd = pl.DataFrame(curve_rows).to_pandas() + # Reverse stack order so the first level sits at the bottom in geom_area. + curves_pd["technology"] = pd.Categorical( + curves_pd["technology"], categories=stack_order + ) + + vlines_df = pl.DataFrame( + {"year": [float(y) for y in run_years if y > 2025]} + ).to_pandas() + + p = ( + ggplot(curves_pd, aes(x="year", y="pct", fill="technology")) + + geom_area(position="stack", alpha=0.9) + + geom_vline( + data=vlines_df, + mapping=aes(xintercept="year"), + color="white", + linetype="dashed", + size=0.5, + alpha=0.7, + ) + + scale_fill_manual(values=fill_map, breaks=stack_order) + + scale_x_continuous( + breaks=list(range(2025, 2060, 5)), + limits=(2024, 2058), + ) + + scale_y_continuous( + labels=lambda x: [f"{v:.0f}%" for v in x], + ) + + labs( + title="NYCA HP adoption trajectory — NYISO Gold Book 2025 logistic fit (stacked)", + subtitle=( + f"Denominator: {total_hu:,.0f} NY residential electric customers" + f" (EIA-861 {eia_year}, bundled + delivery)" + ), + x="Year", + y="Share of NY residential electric customers", + fill="Technology (ResStock upgrade)", + ) + + theme_minimal() + + theme( + plot_title=element_text(size=11), + plot_subtitle=element_text(size=9), + axis_title=element_text(size=10), + legend_title=element_text(size=9), + legend_text=element_text(size=9), + panel_grid_minor=element_line(size=0), + ) + ) + + path_plot.parent.mkdir(parents=True, exist_ok=True) + p.save(str(path_plot), dpi=150, width=9, height=5) + log.info("wrote %s", path_plot) + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Fit NYISO adoption S-curves and write adoption config YAML.", + ) + parser.add_argument( + "--output", + required=True, + metavar="PATH", + dest="path_output", + help="Destination YAML path (e.g. rate_design/hp_rates/ny/config/adoption/nyca_electrification.yaml).", + ) + parser.add_argument( + "--plot-output", + metavar="PATH", + dest="path_plot", + default=None, + help="Optional path for the curve-fit line plot (.png).", + ) + parser.add_argument( + "--stacked-plot-output", + metavar="PATH", + dest="path_stacked_plot", + default=None, + help="Optional path for the stacked area plot (.png).", + ) + parser.add_argument( + "--run-years", + metavar="YEARS", + default=",".join(str(y) for y in _DEFAULT_RUN_YEARS), + help=( + "Comma-separated calendar years to evaluate and write to the YAML. " + f"Default: {','.join(str(y) for y in _DEFAULT_RUN_YEARS)}" + ), + ) + return parser + + +def main() -> None: + args = build_parser().parse_args() + run_years = [int(y.strip()) for y in args.run_years.split(",")] + + log.info("loading NY residential customer count from EIA-861 (S3)…") + total_hu, eia_year = load_total_hu(state="NY") + log.info( + "EIA-861 %d: %,.0f NY residential electric customers (bundled + delivery)", + eia_year, + total_hu, + ) + + scenario, params = fit_all(run_years, total_hu) + + validate_scenario({uid: scenario[uid] for uid in scenario}) + + # Log per-year totals. + for i, yr in enumerate(run_years): + total = sum(scenario[uid][i] for uid in scenario) + log.info("year %d: total fraction = %.4f", yr, total) + + write_yaml(Path(args.path_output), scenario, params, run_years, total_hu, eia_year) + + if args.path_plot: + make_plot(params, run_years, Path(args.path_plot), total_hu, eia_year) + + if args.path_stacked_plot: + make_stacked_plot( + params, run_years, Path(args.path_stacked_plot), total_hu, eia_year + ) + + +if __name__ == "__main__": + main() diff --git a/utils/pre/generate_adoption_scenario_yamls.py b/utils/pre/generate_adoption_scenario_yamls.py new file mode 100644 index 00000000..5379efbc --- /dev/null +++ b/utils/pre/generate_adoption_scenario_yamls.py @@ -0,0 +1,420 @@ +"""Generate per-year scenario YAML entries for mixed-upgrade adoption runs. + +Reads a base scenario YAML, extracts selected run configs, and emits a new +YAML file (``scenarios__adoption.yaml``) with one entry per +(year × run) combination. The per-year ``path_resstock_metadata`` and +``path_resstock_loads`` are rewritten to point at the materialized data +produced by ``materialize_mixed_upgrade.py``. ``year_run`` and all path +strings containing ``year={old_year_run}`` are also updated to the calendar +year for each generated entry. + +Run keys in the output YAML use the scheme ``(year_index + 1) * 100 + run_num``: + +- Year index 0 (first run year), base run 1 → key 101 +- Year index 0, base run 2 → key 102 +- Year index 1 (second run year), base run 1 → key 201 +- Year index 1, base run 2 → key 202 +- … + +This ensures run keys are unique across (year, run) combinations and +memorable when passed to ``run-adoption-scenario``. + +Usage +----- +:: + + uv run python utils/pre/generate_adoption_scenario_yamls.py \\ + --base-scenario rate_design/hp_rates/ri/config/scenarios/scenarios_rie.yaml \\ + --runs 1,2,5,6 \\ + --adoption-config rate_design/hp_rates/ny/config/adoption/nyca_electrification.yaml \\ + --materialized-dir /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/adoption/nyca_electrification \\ + --output rate_design/hp_rates/ri/config/scenarios/scenarios_rie_adoption.yaml +""" + +from __future__ import annotations + +import argparse +import copy +import sys +import warnings +from pathlib import Path +from typing import Any + +import numpy as np +import yaml + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser( + description="Generate per-year scenario YAMLs for mixed-upgrade adoption runs.", + ) + p.add_argument( + "--base-scenario", + required=True, + metavar="PATH", + dest="path_base_scenario", + help="Existing scenario YAML to use as the run config template.", + ) + p.add_argument( + "--runs", + required=True, + help="Comma-separated run numbers to include (e.g. 1,2,5,6).", + ) + p.add_argument( + "--adoption-config", + required=True, + metavar="PATH", + dest="path_adoption_config", + help="Path to adoption trajectory YAML (for year_labels and scenario_name).", + ) + p.add_argument( + "--materialized-dir", + required=True, + metavar="PATH", + dest="path_materialized_dir", + help="Root of materialized per-year data (output of materialize_mixed_upgrade).", + ) + p.add_argument( + "--output", + required=True, + metavar="PATH", + dest="path_output", + help="Path to write the generated adoption scenario YAML.", + ) + p.add_argument( + "--residual-cost-frac", + type=float, + default=None, + dest="residual_cost_frac", + help=( + "When set, adds residual_cost_frac to every generated run entry and " + "sets utility_revenue_requirement: none. Use 0.0 for a 0%% residual " + "(revenue requirement = total marginal costs only)." + ), + ) + p.add_argument( + "--cambium-supply", + action="store_true", + dest="cambium_supply", + help=( + "When set, rewrites supply MC paths to Cambium for runs with " + "run_includes_supply: true, and clears path_bulk_tx_mc for all runs " + "(bulk TX is already embedded in Cambium enduse costs)." + ), + ) + p.add_argument( + "--cambium-gea", + type=str, + default="NYISO", + dest="cambium_gea", + help="Cambium grid emission area (GEA) code, e.g. NYISO (default: NYISO).", + ) + p.add_argument( + "--cambium-ba", + type=str, + default=None, + dest="cambium_ba", + help="Cambium balancing area code, e.g. p127. Required when --cambium-supply is set.", + ) + p.add_argument( + "--cambium-dist-mc-base", + type=str, + default=None, + dest="cambium_dist_mc_base", + help=( + "S3 base path for Cambium-based dist MCs. When set, " + "path_dist_and_sub_tx_mc is replaced with " + "{base}/utility={utility}/year={calendar_year}/data.parquet." + ), + ) + p.add_argument( + "--adoption-tariff-dir", + type=str, + default=None, + dest="adoption_tariff_dir", + help=( + "Base directory template for per-year adoption tariffs. When set, " + "for run entries with run_includes_subclasses=true (runs 5/6), " + "path_tariffs_electric.hp is rewritten to " + "/year=/_hp_seasonal.json and " + ".non-hp to /year=/_nonhp_flat.json. " + "The directory must exist before the run (created by run-adoption-all)." + ), + ) + return p + + +# --------------------------------------------------------------------------- +# Adoption config helpers (mirrors materialize_mixed_upgrade logic) +# --------------------------------------------------------------------------- + + +def _load_yaml(path: Path) -> dict[str, Any]: + with open(path, encoding="utf-8") as f: + return yaml.safe_load(f) + + +def _resolve_run_years(config: dict[str, Any]) -> list[tuple[int, int]]: + """Return ``[(year_index, calendar_year), ...]`` to generate entries for. + + Uses ``run_years`` from the config when present; otherwise uses all + ``year_labels``. Snaps run_years entries to the nearest year_label when + an exact match is not found. + """ + year_labels: list[int] = [int(y) for y in config["year_labels"]] + run_years_raw: list[int] | None = config.get("run_years") + + if run_years_raw is None: + return list(enumerate(year_labels)) + + result: list[tuple[int, int]] = [] + for yr in run_years_raw: + distances = [abs(yl - int(yr)) for yl in year_labels] + nearest_idx = int(np.argmin(distances)) + nearest_year = year_labels[nearest_idx] + if nearest_year != int(yr): + warnings.warn( + f"run_years entry {yr} not in year_labels; " + f"snapping to {nearest_year} (index {nearest_idx})", + stacklevel=2, + ) + result.append((nearest_idx, nearest_year)) + return result + + +# --------------------------------------------------------------------------- +# Config transformation helpers +# --------------------------------------------------------------------------- + + +def _replace_year_in_value(value: Any, old_year: int, new_year: int) -> Any: + """Recursively replace year tokens in strings. + + Handles both ``year={old_year}`` (Hive partition keys) and + ``t={old_year}`` (Cambium path segment) patterns. + """ + if isinstance(value, str): + value = value.replace(f"year={old_year}", f"year={new_year}") + value = value.replace(f"t={old_year}", f"t={new_year}") + return value + if isinstance(value, dict): + return { + k: _replace_year_in_value(v, old_year, new_year) for k, v in value.items() + } + if isinstance(value, list): + return [_replace_year_in_value(item, old_year, new_year) for item in value] + return value + + +def _insert_blank_lines_between_runs(yaml_str: str) -> str: + """Insert a blank line before run keys 2+, not before the first run key.""" + lines = yaml_str.splitlines() + out: list[str] = [] + seen_run_key = False + for line in lines: + stripped = line.strip() + is_run_key = ( + line.startswith(" ") and stripped.endswith(":") and stripped[:-1].isdigit() + ) + if is_run_key and seen_run_key and (not out or out[-1] != ""): + out.append("") + if is_run_key: + seen_run_key = True + out.append(line) + return "\n".join(out) + ("\n" if yaml_str.endswith("\n") else "") + + +def _update_run_name(run_name: str, calendar_year: int) -> str: + """Append ``_y{year}_mixed`` to a run name (before any trailing double-underscore suffix). + + Examples: + ``ri_rie_run1_up00_precalc__flat`` → ``ri_rie_run1_y2025_mixed_precalc__flat`` + ``ny_nyseg_run5_up02_default__tou`` → ``ny_nyseg_run5_y2025_mixed_default__tou`` + """ + # Locate the first double-underscore which separates the "stem" from the tariff suffix. + double_us = run_name.find("__") + year_tag = f"_y{calendar_year}_mixed" + if double_us == -1: + return run_name + year_tag + return run_name[:double_us] + year_tag + run_name[double_us:] + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +def main(argv: list[str] | None = None) -> None: + args = build_parser().parse_args(argv) + + path_base_scenario = Path(args.path_base_scenario) + path_adoption_config = Path(args.path_adoption_config) + path_materialized_dir = Path(args.path_materialized_dir) + path_output = Path(args.path_output) + + # Validate Cambium arg combinations. + if args.cambium_supply and not args.cambium_ba: + raise ValueError("--cambium-ba is required when --cambium-supply is set.") + + # Parse run numbers. + try: + run_nums = [int(r.strip()) for r in args.runs.split(",") if r.strip()] + except ValueError as exc: + raise ValueError( + f"--runs must be comma-separated integers, got: {args.runs!r}" + ) from exc + if not run_nums: + raise ValueError("--runs is empty; at least one run number is required.") + + # 1. Load adoption config for year info. + adoption_config = _load_yaml(path_adoption_config) + scenario_name: str = adoption_config["scenario_name"] + year_run_pairs = _resolve_run_years(adoption_config) + + # 2. Load base scenario YAML and extract requested run configs. + base_doc = _load_yaml(path_base_scenario) + base_runs: dict[int, dict[str, Any]] = { + int(k): v for k, v in base_doc.get("runs", {}).items() + } + + missing_runs = [r for r in run_nums if r not in base_runs] + if missing_runs: + available = sorted(base_runs.keys()) + raise KeyError( + f"Run(s) {missing_runs} not found in {path_base_scenario}. " + f"Available runs: {available}" + ) + + print( + f"Generating adoption scenario YAML for '{scenario_name}': " + f"{len(year_run_pairs)} year(s) × {len(run_nums)} run(s) = " + f"{len(year_run_pairs) * len(run_nums)} entries" + ) + if args.residual_cost_frac is not None: + print( + f" residual_cost_frac={args.residual_cost_frac} (utility_revenue_requirement: none)" + ) + if args.cambium_supply: + print(f" Cambium supply MCs: gea={args.cambium_gea}, ba={args.cambium_ba}") + if args.cambium_dist_mc_base: + print(f" Cambium dist MC base: {args.cambium_dist_mc_base}") + if args.adoption_tariff_dir: + print(f" Adoption tariff dir: {args.adoption_tariff_dir}/year=/") + + # 3. Build generated run entries. + output_runs: dict[int, dict[str, Any]] = {} + + for year_index, calendar_year in year_run_pairs: + meta_path = str( + path_materialized_dir / f"year={calendar_year}" / "metadata-sb.parquet" + ) + for run_num in run_nums: + base_run = base_runs[run_num] + old_year_run = int(base_run.get("year_run", calendar_year)) + + # Deep-copy so base configs remain unmodified. + run_entry: dict[str, Any] = copy.deepcopy(base_run) + + # Build the hive-leaf loads path for this run's state. + # materialize() writes symlinks under: + # year=YYYY/load_curve_hourly/state=/upgrade=00/ + # CAIRO's build_bldg_id_to_load_filepath does a flat glob("*.parquet") + # on path_resstock_loads, so it must point at the upgrade=00 leaf. + # scan_resstock_loads receives the year=YYYY/ base via the Justfile. + run_state: str = str(run_entry.get("state", "")).upper() + loads_path = str( + path_materialized_dir + / f"year={calendar_year}" + / "load_curve_hourly" + / f"state={run_state}" + / "upgrade=00" + / "" + ) + + # Replace ResStock data paths. + run_entry["path_resstock_metadata"] = meta_path + run_entry["path_resstock_loads"] = loads_path + + # Update year_run to the calendar year for this adoption cohort. + run_entry["year_run"] = calendar_year + + # Replace year= and t= tokens in all string path values so MC data + # resolves to the correct year. + run_entry = _replace_year_in_value(run_entry, old_year_run, calendar_year) + + # Update run_name to include year and mixed tag. + run_entry["run_name"] = _update_run_name( + str(base_run.get("run_name", f"run{run_num}")), + calendar_year, + ) + + # Apply Cambium-specific path overrides. + if args.cambium_supply: + cambium_path = ( + f"s3://data.sb/nrel/cambium/2024/scenario=MidCase" + f"/t={calendar_year}/gea={args.cambium_gea}/r={args.cambium_ba}/data.parquet" + ) + run_includes_supply = bool(run_entry.get("run_includes_supply", False)) + if run_includes_supply: + run_entry["path_supply_energy_mc"] = cambium_path + run_entry["path_supply_capacity_mc"] = cambium_path + # Clear bulk TX for all runs: Cambium enduse costs already include it. + run_entry["path_bulk_tx_mc"] = "" + + if args.cambium_dist_mc_base: + utility_val = str(run_entry.get("utility", "")) + base = args.cambium_dist_mc_base.rstrip("/") + run_entry["path_dist_and_sub_tx_mc"] = ( + f"{base}/utility={utility_val}/year={calendar_year}/data.parquet" + ) + + # Apply residual cost fraction override. + if args.residual_cost_frac is not None: + run_entry["residual_cost_frac"] = args.residual_cost_frac + run_entry["utility_revenue_requirement"] = None + + # Rewrite seasonal tariff paths for subclass runs (5/6) to + # point at per-year files written by run-adoption-all. + if args.adoption_tariff_dir and bool( + run_entry.get("run_includes_subclasses", False) + ): + utility_val = str(run_entry.get("utility", "")) + tariff_dir = ( + f"{args.adoption_tariff_dir.rstrip('/')}/year={calendar_year}" + ) + tariff_elec = run_entry.get("path_tariffs_electric", {}) + if isinstance(tariff_elec, dict): + if "hp" in tariff_elec: + tariff_elec["hp"] = ( + f"{tariff_dir}/{utility_val}_hp_seasonal.json" + ) + if "non-hp" in tariff_elec: + tariff_elec["non-hp"] = ( + f"{tariff_dir}/{utility_val}_nonhp_flat.json" + ) + run_entry["path_tariffs_electric"] = tariff_elec + + output_key = (year_index + 1) * 100 + run_num + output_runs[output_key] = run_entry + print( + f" [{output_key}] year={calendar_year}, " + f"base_run={run_num}: {run_entry['run_name']}" + ) + + # 4. Write combined YAML. + path_output.parent.mkdir(parents=True, exist_ok=True) + payload: dict[str, Any] = {"runs": output_runs} + yaml_str = yaml.dump( + payload, + default_flow_style=False, + sort_keys=False, + allow_unicode=True, + ) + yaml_str = _insert_blank_lines_between_runs(yaml_str) + path_output.write_text(yaml_str, encoding="utf-8") + + print(f"Wrote {len(output_runs)} run entries to {path_output}") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/utils/pre/list_adoption_years.py b/utils/pre/list_adoption_years.py new file mode 100644 index 00000000..6720fef0 --- /dev/null +++ b/utils/pre/list_adoption_years.py @@ -0,0 +1,62 @@ +"""Print one calendar year per line for each run year in an adoption config YAML. + +Respects ``run_years`` when present; otherwise uses all ``year_labels``. +Snaps ``run_years`` entries to the nearest ``year_label`` (matching the logic +in ``materialize_mixed_upgrade`` and ``generate_adoption_scenario_yamls``). + +Usage:: + + uv run python utils/pre/list_adoption_years.py path/to/config.yaml + +Exit code 0; prints one integer per line to stdout. Supersedes +``count_adoption_years.py`` — callers get the actual years and the count +implicitly via ``${#year_list[@]}`` in bash. +""" + +from __future__ import annotations + +import sys +import warnings +from pathlib import Path + +import numpy as np +import yaml + + +def list_run_years(config: dict) -> list[int]: + """Return the ordered list of calendar years to run, honouring ``run_years``.""" + year_labels: list[int] = [int(y) for y in config.get("year_labels", [])] + run_years_raw: list[int] | None = config.get("run_years") + + if run_years_raw is None: + return year_labels + + result: list[int] = [] + for yr in run_years_raw: + distances = [abs(yl - int(yr)) for yl in year_labels] + nearest_idx = int(np.argmin(distances)) + nearest_year = year_labels[nearest_idx] + if nearest_year != int(yr): + warnings.warn( + f"run_years entry {yr} not in year_labels; " + f"snapping to {nearest_year} (index {nearest_idx})", + stacklevel=2, + ) + result.append(nearest_year) + return result + + +def main(args: list[str] | None = None) -> None: + argv = args if args is not None else sys.argv[1:] + if not argv: + print("usage: list_adoption_years.py ", file=sys.stderr) + sys.exit(1) + path = Path(argv[0]) + with path.open(encoding="utf-8") as f: + cfg = yaml.safe_load(f) + for year in list_run_years(cfg): + print(year) + + +if __name__ == "__main__": + main() diff --git a/utils/pre/marginal_costs/generate_utility_tx_dx_mc.py b/utils/pre/marginal_costs/generate_utility_tx_dx_mc.py index 4f35fb73..c1fe364f 100644 --- a/utils/pre/marginal_costs/generate_utility_tx_dx_mc.py +++ b/utils/pre/marginal_costs/generate_utility_tx_dx_mc.py @@ -5,7 +5,8 @@ price signals. Input: - - Utility hourly load profile: s3://data.sb/eia/hourly_demand/utilities/region=/utility=X/year=YYYY/month=M/data.parquet + - Utility hourly load profile (EIA): s3://data.sb/eia/hourly_demand/utilities/region=/utility=X/year=YYYY/month=M/data.parquet + - OR Cambium busbar_load (--load-source cambium): s3://data.sb/nrel/cambium/2024/scenario=MidCase/t=YYYY/gea=GEA/r=BA/data.parquet - Marginal cost table CSV with columns: utility, sub_tx_and_dist_mc_kw_yr - Load year (determines which load profile year to use) @@ -15,21 +16,30 @@ - Partition path: utility=X/year=YYYY/data.parquet Usage: - # Inspect results (no upload) - uses 2025 loads + # EIA load source (default) - Inspect results (no upload) - uses 2025 loads python generate_utility_tx_dx_mc.py --state RI --utility rie --load-year 2025 \ --mc-table-path rate_design/hp_rates/ri/config/marginal_costs/ri_marginal_costs_2025.csv \ --utility-load-s3-base s3://data.sb/eia/hourly_demand/utilities/ \ --output-s3-base s3://data.sb/switchbox/marginal_costs/ri/dist_and_sub_tx/ - # Upload to S3 + # EIA load source - Upload to S3 python generate_utility_tx_dx_mc.py --state NY --utility nyseg --load-year 2024 \ --mc-table-path rate_design/hp_rates/ny/config/marginal_costs/ny_sub_tx_and_dist_mc_levelized.csv \ --utility-load-s3-base s3://data.sb/eia/hourly_demand/utilities/ \ --output-s3-base s3://data.sb/switchbox/marginal_costs/ny/dist_and_sub_tx/ \ --upload + + # Cambium busbar_load source - Upload to S3 + python generate_utility_tx_dx_mc.py --state NY --utility nyseg --year 2030 \ + --load-source cambium \ + --cambium-path s3://data.sb/nrel/cambium/2024/scenario=MidCase/t=2030/gea=NYISO/r=p127/data.parquet \ + --mc-table-path rate_design/hp_rates/ny/config/marginal_costs/ny_sub_tx_and_dist_mc_levelized.csv \ + --output-s3-base s3://data.sb/switchbox/marginal_costs/ny/cambium_dist_and_sub_tx/ \ + --upload """ import argparse +import calendar import io from datetime import datetime, timedelta from pathlib import Path @@ -94,6 +104,57 @@ def load_utility_load_profile( return df +def load_cambium_load_profile( + cambium_path: str, + utility: str, + storage_options: dict[str, str], +) -> pl.DataFrame: + """Load Cambium busbar_load as a utility load profile for PoP allocation. + + Reads the ``busbar_load`` column (MWh busbar-equivalent) from a Cambium + parquet file and returns it in the same format expected by + :func:`normalize_load_to_cairo_8760`. + + Distribution infrastructure capacity is sized to busbar-level peaks (before + distribution losses), so ``busbar_load`` is the appropriate PoP allocation + load shape — analogous to the EIA utility ``load_mw`` column. + + Args: + cambium_path: S3 or local path to a Cambium data.parquet file with + columns ``timestamp`` and ``busbar_load``. + utility: Utility short code (e.g. ``nyseg``) — written into the + ``utility`` column of the returned DataFrame. + storage_options: Polars S3 storage options with AWS bucket region. + + Returns: + DataFrame with columns: timestamp, utility, load_mw (8760 rows expected + after normalization; ``busbar_load`` is renamed to ``load_mw``). + """ + print(f"Loading Cambium busbar_load from: {cambium_path}") + if cambium_path.startswith("s3://"): + df = pl.read_parquet(cambium_path, storage_options=storage_options) + else: + df = pl.read_parquet(cambium_path) + + required = {"timestamp", "busbar_load"} + missing = required - set(df.columns) + if missing: + raise ValueError( + f"Cambium parquet is missing expected columns {missing}. " + f"Available: {df.columns}" + ) + + df = df.select( + [ + pl.col("timestamp"), + pl.col("busbar_load").alias("load_mw"), + ] + ).with_columns(pl.lit(utility).alias("utility")) + + print(f"Loaded {len(df):,} Cambium rows (busbar_load → load_mw) for {utility}") + return df + + def normalize_load_to_cairo_8760( load_df: pl.DataFrame, utility: str, year_load: int ) -> pl.DataFrame: @@ -136,13 +197,16 @@ def normalize_load_to_cairo_8760( if df.is_empty(): raise ValueError(f"No load rows found for load_year={year_load}") - # Cairo leap-year rule: if Feb 29 exists, drop Dec 31. - has_feb29 = df.select( + # Cairo leap-year rule: drop Dec 31 so the year has exactly 8760 hours. + # Check the calendar year (not just data) because some sources (e.g. Cambium) + # already omit Feb 29 for leap years, so the data won't contain it. + is_leap = calendar.isleap(year_load) + data_has_feb29 = df.select( ((pl.col("timestamp").dt.month() == 2) & (pl.col("timestamp").dt.day() == 29)) .any() .alias("has_feb29") ).item() - if has_feb29: + if data_has_feb29: print( " Leap-year pattern detected (Feb 29 present); dropping Dec 31 to match Cairo." ) @@ -164,7 +228,7 @@ def normalize_load_to_cairo_8760( agg_exprs.append(pl.col("utility").first().alias("utility")) df = df.group_by("timestamp").agg(agg_exprs).sort("timestamp") - # Build expected 8760 index for this year. + # Build expected 8760 index for this year (drop Dec 31 for leap years). start = datetime(year_load, 1, 1, 0, 0, 0) end = datetime(year_load, 12, 31, 23, 0, 0) expected = [] @@ -172,10 +236,14 @@ def normalize_load_to_cairo_8760( while cur <= end: expected.append(cur) cur += timedelta(hours=1) - if has_feb29: + if is_leap: expected = [t for t in expected if not (t.month == 12 and t.day == 31)] expected_df = pl.DataFrame({"timestamp": expected}) + # Align timestamp precision so the join key dtypes match. + if df["timestamp"].dtype != expected_df["timestamp"].dtype: + expected_df = expected_df.cast({"timestamp": df["timestamp"].dtype}) + # Reindex and fill any missing hours. df = expected_df.join(df, on="timestamp", how="left").sort("timestamp") missing_before_fill = df.filter(pl.col("load_mw").is_null()).height @@ -479,15 +547,38 @@ def main(): required=True, help="Path to marginal cost table CSV (local or s3://)", ) + parser.add_argument( + "--load-source", + type=str, + choices=["eia", "cambium"], + default="eia", + help=( + "Load profile source for PoP allocation. " + "'eia' uses EIA/NYISO utility hourly loads (default). " + "'cambium' uses Cambium busbar_load -- requires --cambium-path." + ), + ) + parser.add_argument( + "--cambium-path", + type=str, + default=None, + dest="cambium_path", + help=( + "Path to Cambium data.parquet (S3 or local). Required when " + "--load-source cambium. " + "E.g. s3://data.sb/nrel/cambium/2024/scenario=MidCase/t=2030/gea=NYISO/r=p127/data.parquet" + ), + ) parser.add_argument( "--utility-load-s3-base", "--nyiso-s3-base", dest="utility_load_s3_base", type=str, - required=True, + default=None, help=( "Base S3 path for utility loads " - "(e.g. s3://data.sb/eia/hourly_demand/utilities/)" + "(e.g. s3://data.sb/eia/hourly_demand/utilities/). " + "Required when --load-source eia (the default)." ), ) parser.add_argument( @@ -511,6 +602,15 @@ def main(): args = parser.parse_args() validate_mc_table_path(args.mc_table_path) + + # Validate load-source-specific required args. + if args.load_source == "eia" and not args.utility_load_s3_base: + parser.error( + "--utility-load-s3-base is required when --load-source eia (the default)." + ) + if args.load_source == "cambium" and not args.cambium_path: + parser.error("--cambium-path is required when --load-source cambium.") + load_dotenv() config = get_state_config(args.state) storage_options = get_aws_storage_options() @@ -518,17 +618,20 @@ def main(): output_year = args.year load_year = args.load_year if args.load_year else output_year - # Detect whether load path uses EIA layout (with region partition) or - # NYISO layout (no region partition) based on path prefix. - s3_base = args.utility_load_s3_base - iso_region: str | None = config.iso_region - if "nyiso/hourly_demand" in s3_base: - iso_region = None - print("=" * 60) print("MARGINAL COST ALLOCATION") print(f"State: {config.state}") - print(f"ISO region partition: {iso_region or '(none — NYISO native path)'}") + print(f"Load source: {args.load_source}") + if args.load_source == "cambium": + print(f"Cambium path: {args.cambium_path}") + else: + # Detect whether load path uses EIA layout (with region partition) or + # NYISO layout (no region partition) based on path prefix. + s3_base = args.utility_load_s3_base + iso_region: str | None = config.iso_region + if "nyiso/hourly_demand" in s3_base: + iso_region = None + print(f"ISO region partition: {iso_region or '(none — NYISO native path)'}") print(f"AWS bucket region: {storage_options.get('region')}") print("=" * 60) print(f"Utility: {args.utility}") @@ -538,14 +641,22 @@ def main(): print(f"Upload to S3: {'Yes' if args.upload else 'No (inspection only)'}") print("=" * 60) - load_df = load_utility_load_profile( - s3_base, - iso_region, - load_year, - args.utility, - storage_options, - ) - load_df = normalize_load_to_cairo_8760(load_df, args.utility, load_year) + if args.load_source == "cambium": + load_df = load_cambium_load_profile( + args.cambium_path, + args.utility, + storage_options, + ) + load_df = normalize_load_to_cairo_8760(load_df, args.utility, load_year) + else: + load_df = load_utility_load_profile( + s3_base, + iso_region, + load_year, + args.utility, + storage_options, + ) + load_df = normalize_load_to_cairo_8760(load_df, args.utility, load_year) if load_year != output_year: print(f"\n Remapping load timestamps: {load_year} → {output_year}") diff --git a/utils/pre/materialize_mixed_upgrade.py b/utils/pre/materialize_mixed_upgrade.py new file mode 100644 index 00000000..a69a12f1 --- /dev/null +++ b/utils/pre/materialize_mixed_upgrade.py @@ -0,0 +1,240 @@ +"""Materialize per-year ResStock data for mixed-upgrade HP adoption trajectories.""" + +from __future__ import annotations + +import argparse +import sys +import warnings +from pathlib import Path +from typing import Any + +import numpy as np +import yaml + +from utils.buildstock import ( + SbMixedUpgradeScenario, + _build_load_file_map as _buildstock_load_file_map, +) + + +def build_parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser( + description="Materialize per-year mixed-upgrade ResStock data for adoption trajectories.", + ) + p.add_argument( + "--state", required=True, help="Two-letter state abbreviation (e.g. ny, ri)." + ) + p.add_argument("--utility", required=True, help="Utility slug (e.g. rie, nyseg).") + p.add_argument( + "--adoption-config", + required=True, + metavar="PATH", + dest="path_adoption_config", + help="Path to adoption trajectory YAML.", + ) + p.add_argument( + "--path-resstock-release", + required=True, + help="ResStock release path or root path containing the release.", + ) + p.add_argument( + "--release", + required=False, + help="Optional release directory name under --path-resstock-release.", + ) + p.add_argument( + "--output-dir", + required=True, + metavar="PATH", + dest="path_output_dir", + help="Directory to write per-year materialized data.", + ) + return p + + +def _load_adoption_config(path: Path) -> dict[str, Any]: + with open(path, encoding="utf-8") as f: + return yaml.safe_load(f) + + +def _parse_adoption_config( + config: dict[str, Any], +) -> tuple[str, int, dict[int, list[float]], list[int], list[int]]: + """Parse and return core fields from the adoption config.""" + scenario_name: str = config["scenario_name"] + random_seed: int = int(config.get("random_seed", 42)) + + scenario_raw: dict[Any, list[float]] = config["scenario"] + scenario: dict[int, list[float]] = { + int(k): [float(v) for v in vals] for k, vals in scenario_raw.items() + } + + year_labels: list[int] = [int(y) for y in config["year_labels"]] + + run_years_raw: list[int] | None = config.get("run_years") + if run_years_raw is None: + run_year_indices = list(range(len(year_labels))) + else: + run_year_indices = [] + for yr in run_years_raw: + distances = [abs(yl - int(yr)) for yl in year_labels] + nearest_idx = int(np.argmin(distances)) + nearest_year = year_labels[nearest_idx] + if nearest_year != int(yr): + warnings.warn( + f"run_years entry {yr} not in year_labels; " + f"snapping to {nearest_year} (index {nearest_idx})", + stacklevel=2, + ) + run_year_indices.append(nearest_idx) + + return scenario_name, random_seed, scenario, year_labels, run_year_indices + + +def _build_load_file_map(loads_dir: Path, bldg_ids: set[int]) -> dict[int, Path]: + """Compatibility shim re-exported for existing tests/imports.""" + return _buildstock_load_file_map(loads_dir, bldg_ids) + + +def assign_buildings( + bldg_ids: list[int], + scenario: dict[int, list[float]], + run_year_indices: list[int], + random_seed: int, + applicable_bldg_ids_per_upgrade: dict[int, set[int]] | None = None, +) -> dict[int, dict[int, int]]: + """Assign buildings to upgrades by year, preserving monotonic adoption.""" + assignments = {t: {} for t in run_year_indices} + if not bldg_ids: + return assignments + + shuffled = np.array(sorted(bldg_ids), dtype=int) + np.random.default_rng(random_seed).shuffle(shuffled) + shuffled_ids = shuffled.tolist() + + upgrade_order = list(scenario.keys()) + upgrade_allocations: dict[int, list[int]] = {uid: [] for uid in upgrade_order} + + if applicable_bldg_ids_per_upgrade is None: + for t in run_year_indices: + assigned_any = set().union( + *( + set(upgrade_allocations[uid]) + for uid in upgrade_order + if upgrade_allocations[uid] + ) + ) + for uid in upgrade_order: + target = int(len(bldg_ids) * scenario[uid][t]) + current = len(upgrade_allocations[uid]) + needed = max(0, target - current) + if needed == 0: + continue + available = [bid for bid in shuffled_ids if bid not in assigned_any] + take = available[:needed] + upgrade_allocations[uid].extend(take) + assigned_any.update(take) + if len(take) < needed: + warnings.warn( + f"Upgrade {uid}: target {target} buildings but only " + f"{len(available)} available; capping at {current + len(take)}.", + stacklevel=2, + ) + else: + filtered_pools: dict[int, list[int]] = {} + for uid in upgrade_order: + applicable = applicable_bldg_ids_per_upgrade.get(uid, set()) + # Keep per-upgrade pools independent. If pools overlap, final + # assignment below resolves conflicts by iteration order. + filtered_pools[uid] = [bid for bid in shuffled_ids if bid in applicable] + + for t in run_year_indices: + for uid in upgrade_order: + target = int(len(bldg_ids) * scenario[uid][t]) + current = len(upgrade_allocations[uid]) + if target <= current: + continue + pool = filtered_pools[uid] + if target > len(pool): + warnings.warn( + f"Upgrade {uid}: target {target} buildings but only " + f"{len(pool)} are applicable; capping at {len(pool)}.", + stacklevel=2, + ) + target = len(pool) + upgrade_allocations[uid].extend(pool[current:target]) + + all_ids = set(bldg_ids) + for t in run_year_indices: + year_map = {bid: 0 for bid in bldg_ids} + for uid in upgrade_order: + target = int(len(bldg_ids) * scenario[uid][t]) + for bid in upgrade_allocations[uid][:target]: + year_map[bid] = uid + assigned = set(bid for bid, uid in year_map.items() if uid != 0) + if not assigned.issubset(all_ids): + raise ValueError( + "Internal assignment bug: assigned building outside input set" + ) + assignments[t] = year_map + return assignments + + +def _resolve_release_path(path_resstock_release: Path, release: str | None) -> Path: + """Resolve the on-disk release directory, preferring `_sb` when available.""" + if release: + if path_resstock_release.name in {release, f"{release}_sb"}: + return path_resstock_release + candidate_sb = path_resstock_release / f"{release}_sb" + if candidate_sb.exists(): + return candidate_sb + candidate = path_resstock_release / release + if candidate.exists(): + return candidate + return candidate_sb + + # No explicit release: use the provided path as-is. + return path_resstock_release + + +def main(argv: list[str] | None = None) -> None: + args = build_parser().parse_args(argv) + + path_adoption_config = Path(args.path_adoption_config) + path_output_dir = Path(args.path_output_dir) + state = args.state.lower() + path_release = _resolve_release_path( + Path(args.path_resstock_release), getattr(args, "release", None) + ) + + config = _load_adoption_config(path_adoption_config) + scenario_name, random_seed, scenario, year_labels, run_year_indices = ( + _parse_adoption_config(config) + ) + + print( + f"Materialising '{scenario_name}' for state={state.upper()}, utility={args.utility}" + ) + print( + f" upgrades: {[0, *sorted(scenario.keys())]} | " + f"years: {[year_labels[t] for t in run_year_indices]}" + ) + + mixed = SbMixedUpgradeScenario( + path_resstock_release=path_release, + state=state, + scenario_name=scenario_name, + scenario=scenario, + random_seed=random_seed, + year_labels=year_labels, + run_year_indices=run_year_indices, + ) + assignments = mixed.build_assignments(assign_buildings) + mixed.materialize(path_output_dir=path_output_dir, assignments=assignments) + mixed.export_scenario_csv(path_output_dir=path_output_dir, assignments=assignments) + + print(f"Done. Materialised {len(run_year_indices)} year(s) to {path_output_dir}") + + +if __name__ == "__main__": + sys.exit(main())