Add mixed-upgrade HP adoption trajectory pipeline (skeleton)

AGENTS.md

@@ -130,6 +130,12 @@ Match existing style: Ruff for formatting/lint, **ty** for type checking, dprint
 
 **LaTeX in markdown:** GitHub's MathJax renderer does not support escaped underscores inside `\text{}` (e.g. `\text{avg\_mc\_peak}` will fail). Use proper math symbols instead: `\overline{MC}_{\text{peak}}`, `MC_h`, `L_h`, etc. Bare subscripts and `\text{}` with simple words (no underscores) are fine.
 
+## Git commits
+
+- **Never write commit messages via a temp file** (e.g. `/tmp/commit_msg.txt`). Pass the message directly with `-m "..."` or let the user commit manually.
+- **Never add co-author trailers** (`Co-authored-by: ...`) or any other generated-by attribution to commit messages or PR bodies.
+- **For `gh pr create` body**: use `--body-file -` with a shell heredoc (stdin) to avoid attribution injection — do NOT use `--body "..."` with multi-line strings or `--body-file /tmp/...`. Example: `gh pr create --body-file - <<'PRBODY'\n...\nPRBODY`
+
 ## Code Quality (required before every commit)
 
 - Run `just check` — no linter errors, no type errors, no warnings

data/resstock/Justfile

@@ -304,3 +304,31 @@ create-sb-release-for-upgrade-02-RI:
     sudo aws s3 sync s3://data.sb/nrel/resstock/res_2024_amy2018_2_sb/ /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/
     just add-monthly-loads RI "00 02"
     just upload-monthly-loads RI "00 02"
+
+# =============================================================================
+# Adoption trajectory upgrades (01, 04, 05): convenience shortcuts and end-to-end recipes
+# =============================================================================
+
+adjust-mf-electricity-NY-upgrade-01:
+    just adjust-mf-electricity NY res_2024_amy2018_2 res_2024_amy2018_2_sb "01"
+
+adjust-mf-electricity-NY-upgrade-04:
+    just adjust-mf-electricity NY res_2024_amy2018_2 res_2024_amy2018_2_sb "04"
+
+adjust-mf-electricity-NY-upgrade-05:
+    just adjust-mf-electricity NY res_2024_amy2018_2 res_2024_amy2018_2_sb "05"
+
+# Copy, adjust loads, and sync upgrades 01, 04, 05 into the _sb release for NY.
+# Assumes prepare-metadata-ny has already been run (it processes all upgrades 00-05).
+# metadata_utility (utility assignment) is upgrade-independent and is not re-copied here.
+
+# We are NOT running approximate-non-hp-load for upgrades 4 and 5 because they strictly only apply to certain building types.
+create-sb-release-for-adoption-upgrades-NY:
+    just copy-resstock-data-2024-amy2018-2-NY "04 05" "metadata load_curve_hourly"
+    just approximate-non-hp-load NY 01 res_2024_amy2018_2 res_2024_amy2018_2_sb 15 True True
+    just adjust-mf-electricity-NY-upgrade-01
+    just adjust-mf-electricity-NY-upgrade-04
+    just adjust-mf-electricity-NY-upgrade-05
+    sudo aws s3 sync s3://data.sb/nrel/resstock/res_2024_amy2018_2_sb/ /ebs/data/nrel/resstock/res_2024_amy2018_2_sb/
+    just add-monthly-loads NY "01 04 05"
+    just upload-monthly-loads NY "01 04 05"

rate_design/hp_rates/Justfile

@@ -79,12 +79,15 @@ path_bulk_tx_mc := env_var_or_default('BULK_TX_MC', "")
 path_supply_energy_mc := env_var_or_default('SUPPLY_ENERGY_MC', "s3://data.sb/switchbox/marginal_costs/" + state + "/supply/energy/utility=" + utility + "/year=" + mc_year + "/data.parquet")
 path_supply_capacity_mc := env_var_or_default('SUPPLY_CAPACITY_MC', "s3://data.sb/switchbox/marginal_costs/" + state + "/supply/capacity/utility=" + utility + "/year=" + mc_year + "/data.parquet")
 path_supply_ancillary_mc := env_var_or_default('SUPPLY_ANCILLARY_MC', "")
+path_resstock_root := "/ebs/data/nrel/resstock"
+resstock_release_key := "res_2024_amy2018_2"
 path_resstock_release := "/ebs/data/nrel/resstock/res_2024_amy2018_2_sb"
 path_resstock_metadata := path_resstock_release + "/metadata"
 path_utility_assignment := path_resstock_release + "/metadata_utility"
 path_resstock_loads_00 := path_resstock_release + "/load_curve_hourly/state=" + state_upper + "/upgrade=" + upgrade
 path_mc_table := env_var_or_default('MC_TABLE', path_config / "marginal_costs" / state + "_marginal_costs_" + year + ".csv")
 path_s3_mc_output := "s3://data.sb/switchbox/marginal_costs/" + state + "/dist_and_sub_tx/"
+path_s3_mc_output_cambium := "s3://data.sb/switchbox/marginal_costs/" + state + "/cambium_dist_and_sub_tx/"
 path_s3_utility_loads := "s3://data.sb/eia/hourly_demand/utilities/"
 path_electric_utility_stats := "s3://data.sb/eia/861/electric_utility_stats/year=2024/state=" + state_upper + "/data.parquet"
 path_genability := path_rev_requirement / "top-ups"
@@ -337,6 +340,37 @@ create-dist-and-sub-tx-mc-data-all:
         UTILITY="$util" just create-dist-and-sub-tx-mc-data
     done
 
+# Distribution marginal costs using Cambium busbar_load as the PoP allocation load shape.
+# Writes to the cambium_dist_and_sub_tx/ prefix to avoid overwriting EIA-based dist MCs.
+#
+# Example:
+
+# just s ny create-dist-mc-cambium 2030
+create-dist-mc-cambium year_mc=year:
+    uv run python {{ path_repo }}/utils/pre/marginal_costs/generate_utility_tx_dx_mc.py \
+      --state {{ state_upper }} \
+      --utility {{ utility }} \
+      --year {{ year_mc }} \
+      --load-source cambium \
+      --cambium-path "s3://data.sb/nrel/cambium/2024/scenario=MidCase/t={{ year_mc }}/gea=NYISO/r={{ cambium_ba }}/data.parquet" \
+      --mc-table-path {{ path_mc_table }} \
+      --output-s3-base {{ path_s3_mc_output_cambium }} \
+      --n-hours {{ upstream_hours }} \
+      --upload
+
+# Generate Cambium dist MCs for all adoption trajectory years (2025–2050).
+#
+# Example:
+
+# just s ny create-cambium-dist-mc-all-years
+create-cambium-dist-mc-all-years:
+    #!/usr/bin/env bash
+    set -euo pipefail
+    for yr in 2025 2030 2035 2040 2045 2050; do
+        echo ">> Generating Cambium dist MC for year=${yr}" >&2
+        just create-dist-mc-cambium "${yr}"
+    done
+
 # =============================================================================
 # MID-CONFIG: generate between runs (using outputs from earlier runs)
 # =============================================================================
@@ -677,6 +711,185 @@ run-subset runs:
         just "run-${num}"
     done
 
+# =============================================================================
+# ADOPTION TRAJECTORY (mixed-upgrade)
+# =============================================================================
+
+path_adoption_config_dir := path_config / "adoption"
+
+# Fit logistic S-curves to NYISO Gold Book 2025 digitized data and write the
+# adoption config YAML + a curve-fit diagnostic plot.
+#
+# Example:
+
+# just s ny fit-adoption-config nyca_electrification
+fit-adoption-config config_name="nyca_electrification":
+    uv run python {{ path_repo }}/utils/pre/fit_adoption_config.py \
+        --output "{{ path_adoption_config_dir }}/{{ config_name }}.yaml" \
+        --plot-output "{{ path_adoption_config_dir }}/{{ config_name }}_curves.png" \
+        --stacked-plot-output "{{ path_adoption_config_dir }}/{{ config_name }}_stacked.png"
+
+# Materialize per-year ResStock data for a mixed-upgrade adoption trajectory.
+# Reads the adoption config YAML, assigns buildings to upgrades per year, and
+# writes year=<N>/ directories under the adoption output path.
+#
+# Example:
+
+# just s ny materialize-adoption nyca_electrification
+materialize-adoption config_name="default":
+    uv run python {{ path_repo }}/utils/pre/materialize_mixed_upgrade.py \
+        --state "{{ state }}" \
+        --utility "{{ utility }}" \
+        --adoption-config "{{ path_adoption_config_dir }}/{{ config_name }}.yaml" \
+        --path-resstock-release "{{ path_resstock_root }}" \
+        --release "{{ resstock_release_key }}" \
+        --output-dir "{{ path_resstock_release }}/adoption/{{ config_name }}"
+
+# Generate per-year scenario YAML entries for adoption runs.
+# Uses Cambium supply MCs (energy_cost_enduse / capacity_cost_enduse), Cambium
+# busbar_load dist MCs, and 0% residual cost so revenue requirement = total MC.
+# Output: config/scenarios/scenarios_<utility>_adoption.yaml
+#
+# When config_name is set, --adoption-tariff-dir is passed so that runs 5/6
+# in the YAML reference per-year seasonal tariff files instead of the shared
+# static tariff.  Those files are written by run-adoption-all between run-2
+# and run-5 for each year.
+#
+# Example:
+
+# just s ny generate-adoption-scenarios nyca_electrification 1,2,5,6
+generate-adoption-scenarios config_name="default" runs="1,2,5,6":
+    uv run python {{ path_repo }}/utils/pre/generate_adoption_scenario_yamls.py \
+        --base-scenario "{{ path_scenario_config }}" \
+        --runs "{{ runs }}" \
+        --adoption-config "{{ path_adoption_config_dir }}/{{ config_name }}.yaml" \
+        --materialized-dir "{{ path_resstock_release }}/adoption/{{ config_name }}" \
+        --output "{{ path_scenarios }}/scenarios_{{ utility }}_adoption.yaml" \
+        --residual-cost-frac 0.0 \
+        --cambium-supply \
+        --cambium-gea NYISO \
+        --cambium-ba {{ cambium_ba }} \
+        --cambium-dist-mc-base "s3://data.sb/switchbox/marginal_costs/{{ state }}/cambium_dist_and_sub_tx" \
+        --adoption-tariff-dir "{{ path_tariffs_electric }}/adoption/{{ config_name }}"
+
+# Run a single adoption scenario by (year-indexed) run number.
+# Run keys use the scheme (year_index + 1) * 100 + base_run_num, matching the
+# output of generate_adoption_scenario_yamls.py (e.g. 101, 102, 201, 202, ...).
+#
+# Example:
+
+# just s ny run-adoption-scenario 101
+run-adoption-scenario run_num:
+    #!/usr/bin/env bash
+    set -euo pipefail
+    : "${RDP_BATCH:?Set RDP_BATCH before running}"
+    export RDP_BATCH
+    log_dir="${HOME}/rdp_run_logs"
+    mkdir -p "${log_dir}"
+    log_file="${log_dir}/{{ utility }}_adoption_run{{ run_num }}_${RDP_BATCH}.log"
+    echo ">> run-adoption-scenario {{ run_num }}: logging to ${log_file}" >&2
+    uv run python {{ path_repo }}/rate_design/hp_rates/run_scenario.py \
+        --state "{{ state }}" \
+        --scenario-config "{{ path_scenarios }}/scenarios_{{ utility }}_adoption.yaml" \
+        --run-num "{{ run_num }}" \
+        --output-dir "{{ path_outputs_base }}/${RDP_BATCH}" \
+        2>&1 | tee "${log_file}"
+
+# Orchestrate the full adoption pipeline: materialize → Cambium dist MCs → generate scenarios → run all.
+# All runs use Cambium supply MCs, Cambium busbar_load dist MCs, and 0% residual cost.
+# Iterates over all (year × run) combinations using the key scheme
+# (year_index + 1) * 100 + base_run_num produced by generate_adoption_scenario_yamls.py.
+#
+# For each year the loop:
+#   1. Runs precalc-flat runs (1, 2) and copies their calibrated tariffs to
+#      tariffs/electric/adoption/<config_name>/year=<YYYY>/.
+#   2. Derives per-year seasonal tariffs from run-1/2 outputs + that year's
+#      mixed-upgrade loads (hive-partitioned under the materialized dir).
+#   3. Runs seasonal-precalc runs (5, 6) and copies their calibrated tariffs.
+#
+# Example:
+
+# RDP_BATCH=ny_20260320_adoption just s ny run-adoption-all nyca_electrification 1,2,5,6
+run-adoption-all config_name="default" runs="1,2,5,6":
+    #!/usr/bin/env bash
+    set -euo pipefail
+    : "${RDP_BATCH:?Set RDP_BATCH before running}"
+    export RDP_BATCH
+    just materialize-adoption "{{ config_name }}"
+    just create-cambium-dist-mc-all-years
+    just generate-adoption-scenarios "{{ config_name }}" "{{ runs }}"
+    adoption_yaml="{{ path_scenarios }}/scenarios_{{ utility }}_adoption.yaml"
+    adoption_base="{{ path_resstock_release }}/adoption/{{ config_name }}"
+    tariffs_adoption_base="{{ path_tariffs_electric }}/adoption/{{ config_name }}"
+    # Read calendar years from the adoption config (one per line).
+    mapfile -t year_list < <(uv run python "{{ path_repo }}/utils/pre/list_adoption_years.py" \
+        "{{ path_adoption_config_dir }}/{{ config_name }}.yaml")
+    IFS=',' read -ra all_runs <<< "{{ runs }}"
+    for yi in "${!year_list[@]}"; do
+        year="${year_list[$yi]}"
+        key_prefix=$(( (yi + 1) * 100 ))
+        tariff_dir="${tariffs_adoption_base}/year=${year}"
+        mkdir -p "${tariff_dir}"
+        loads_base="${adoption_base}/year=${year}"
+        echo ">> run-adoption-all: year=${year} (yi=${yi}, key_prefix=${key_prefix})" >&2
+        # --- Runs 1 and 2: precalc flat ---
+        for base_run in "${all_runs[@]}"; do
+            [[ "${base_run}" == "1" || "${base_run}" == "2" ]] || continue
+            key=$(( key_prefix + base_run ))
+            echo ">> run-adoption-all: run-${key} (year=${year}, base_run=${base_run})" >&2
+            just run-adoption-scenario "${key}"
+            run_dir=$(bash "{{ latest_output }}" "${adoption_yaml}" "${key}")
+            just copy-calibrated-tariff-from-run "${run_dir}" "${tariff_dir}"
+        done
+        # --- Derive per-year seasonal tariffs (only when runs 1,2 and 5 or 6 are present) ---
+        has_run1=false; has_run2=false; has_run5=false; has_run6=false
+        for r in "${all_runs[@]}"; do
+            [[ "$r" == "1" ]] && has_run1=true
+            [[ "$r" == "2" ]] && has_run2=true
+            [[ "$r" == "5" ]] && has_run5=true
+            [[ "$r" == "6" ]] && has_run6=true
+        done
+        if $has_run1 && $has_run5; then
+            run1_dir=$(bash "{{ latest_output }}" "${adoption_yaml}" $(( key_prefix + 1 )))
+            echo ">> run-adoption-all: deriving seasonal (delivery) from ${run1_dir}" >&2
+            just compute-seasonal-discount-inputs \
+                "${run1_dir}" "${loads_base}" "{{ state_upper }}" "{{ upgrade }}"
+            just create-seasonal-discount-tariff \
+                "${tariff_dir}/{{ utility }}_flat_calibrated.json" \
+                "${run1_dir}/seasonal_discount_rate_inputs.csv" \
+                "{{ utility }}_hp_seasonal" \
+                "${tariff_dir}/{{ utility }}_hp_seasonal.json"
+        fi
+        if $has_run2 && $has_run6; then
+            run2_dir=$(bash "{{ latest_output }}" "${adoption_yaml}" $(( key_prefix + 2 )))
+            echo ">> run-adoption-all: deriving seasonal (supply) from ${run2_dir}" >&2
+            just compute-seasonal-discount-inputs \
+                "${run2_dir}" "${loads_base}" "{{ state_upper }}" "{{ upgrade }}"
+            just create-seasonal-discount-tariff \
+                "${tariff_dir}/{{ utility }}_flat_supply_calibrated.json" \
+                "${run2_dir}/seasonal_discount_rate_inputs.csv" \
+                "{{ utility }}_hp_seasonal_supply" \
+                "${tariff_dir}/{{ utility }}_hp_seasonal_supply.json"
+        fi
+        # --- Runs 5 and 6: precalc seasonal ---
+        for base_run in "${all_runs[@]}"; do
+            [[ "${base_run}" == "5" || "${base_run}" == "6" ]] || continue
+            key=$(( key_prefix + base_run ))
+            echo ">> run-adoption-all: run-${key} (year=${year}, base_run=${base_run})" >&2
+            just run-adoption-scenario "${key}"
+            run_dir=$(bash "{{ latest_output }}" "${adoption_yaml}" "${key}")
+            just copy-calibrated-tariff-from-run "${run_dir}" "${tariff_dir}"
+        done
+        # --- Any remaining runs (not 1,2,5,6) ---
+        for base_run in "${all_runs[@]}"; do
+            [[ "${base_run}" == "1" || "${base_run}" == "2" ]] && continue
+            [[ "${base_run}" == "5" || "${base_run}" == "6" ]] && continue
+            key=$(( key_prefix + base_run ))
+            echo ">> run-adoption-all: run-${key} (year=${year}, base_run=${base_run})" >&2
+            just run-adoption-scenario "${key}"
+        done
+    done
+
 # =============================================================================
 # HELPERS
 # =============================================================================

rate_design/hp_rates/ny/config/adoption/nyca_electrification.yaml

@@ -0,0 +1,35 @@
+# NYCA building electrification adoption trajectory (NYISO Gold Book 2025).
+# Generated by utils/pre/fit_adoption_config.py — do not edit by hand.
+#
+# Fractions represent the share of total NYCA buildings assigned to each
+# ResStock upgrade at each year. Remaining buildings stay at upgrade 0 (baseline).
+# Year indices map to calendar years via year_labels.
+#
+# Technology → ResStock upgrade mapping:
+#   ASHP Full Capacity  → 2  (cold-climate ASHP, 90% capacity @ 5F, elec backup)
+#   ASHP Dual Fuel      → 4  (ENERGY STAR ASHP + existing fossil backup)
+#   Ground Source HP    → 5  (geothermal heat pump)
+#   Supplemental Heat   → 1  (ENERGY STAR ASHP, 50% capacity @ 5F, elec backup)
+#   Electric Resistance → baseline upgrade 0, already captured there
+#
+# Methodology: logistic S-curves  f(t) = L / (1 + exp(-k * (t - t0)))  fit
+# (scipy curve_fit) to housing-unit counts digitized from the NYISO Gold
+# Book 2025 NYCA stacked-area chart. Denominator: 7,900,000 total NYCA
+# occupied housing units (Census ACS / NYISO estimate). 2025 forced to 0.0
+# (all buildings at upgrade-0 baseline).
+#
+# Fitted parameters:
+#   upgrade 2 (ASHP full capacity):  L=0.2168  k=0.2169  t0=2042.8
+#   upgrade 4 (ASHP dual fuel):  L=0.1087  k=0.2290  t0=2040.0
+#   upgrade 5 (ground source HP):  L=0.0115  k=0.2633  t0=2039.3
+#   upgrade 1 (supplemental heat):  L=0.1281  k=0.3098  t0=2040.2
+scenario_name: nyca_electrification
+random_seed: 42
+scenario:
+  2: [0.0000, 0.0128, 0.0339, 0.0767, 0.1340, 0.1794] # ASHP full capacity
+  4: [0.0000, 0.0100, 0.0263, 0.0544, 0.0825, 0.0987] # ASHP dual fuel
+  5: [0.0000, 0.0009, 0.0028, 0.0063, 0.0094, 0.0109] # ground source HP
+  1: [0.0000, 0.0052, 0.0211, 0.0617, 0.1043, 0.1222] # supplemental heat
+# Calendar years for each scenario index (= run years).
+# Aligns with Cambium 5-year MC intervals; 2025 is baseline.
+year_labels: [2025, 2030, 2035, 2040, 2045, 2050]