Add demand-flex analytical validation and elasticity calibration

context/README.md

@@ -56,10 +56,11 @@ BAT framework, residual allocation, and how they connect to the literature.
 
 Cost-reflective TOU design: theory and window optimization.
 
-| File                               | Purpose                                                                                                                                                                                                         |
-| ---------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| cost_reflective_tou_rate_design.md | Theory and practice of cost-reflective TOU rate design: demand-weighted MC averages, cost-causation ratios, period selection, assumptions, demand flexibility implications, and partial vs. general equilibrium |
-| tou_window_optimization.md         | TOU window width ($N$) sweep: welfare-loss metric derivation, HP-demand weighting proof, sweep algorithm, CLI/Justfile, NY results                                                                              |
+| File                                  | Purpose                                                                                                                                                                                                         |
+| ------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| cost_reflective_tou_rate_design.md    | Theory and practice of cost-reflective TOU rate design: demand-weighted MC averages, cost-causation ratios, period selection, assumptions, demand flexibility implications, and partial vs. general equilibrium |
+| tou_window_optimization.md            | TOU window width ($N$) sweep: welfare-loss metric derivation, HP-demand weighting proof, sweep algorithm, CLI/Justfile, NY results                                                                              |
+| demand_flex_elasticity_calibration.md | Per-utility elasticity calibration: Arcturus 2.0 meta-analysis anchor, diagnostic methodology, results table (ε = -0.10 or -0.12 per utility), two savings mechanisms, known limitations                        |
 
 ### methods/marginal_costs/
 

rate_design/hp_rates/Justfile

@@ -689,8 +689,8 @@ run-all-sequential:
     : "${RDP_BATCH:?Set RDP_BATCH before running (e.g. ny_20260305c_r1-8)}"
     export RDP_BATCH
     echo ">> run-all-sequential [${RDP_BATCH}]" >&2
-    # just run-1
-    # just run-2
+    just run-1
+    just run-2
     just compute-rev-requirements
     just run-3
     just run-4

rate_design/hp_rates/ny/Justfile

@@ -252,3 +252,58 @@ validate-all-runs-ny timestamp="":
     just validate-or "{{ timestamp }}"
     just validate-psegli "{{ timestamp }}"
     just validate-rge "{{ timestamp }}"
+
+# =============================================================================
+# NY-only: demand-flex diagnostics
+# =============================================================================
+
+path_flex_plots := path_repo / "dev_plots/flex"
+
+# Validate demand-flex load shifting for one utility: reproduces the shift
+# analytically, checks energy conservation and direction, cross-checks against
+# CAIRO's elasticity tracker, and generates diagnostic plots.
+#   just -f ny/Justfile validate-demand-flex coned
+
+# just -f ny/Justfile validate-demand-flex coned -0.12 ny_20260325b_r1-16
+validate-demand-flex utility_arg elasticity="-0.10" batch="":
+    cd {{ project_root }} && uv run python -m utils.post.validate_demand_flex_shift \
+      --utility {{ utility_arg }} \
+      --elasticity {{ elasticity }} \
+      --output-dir "{{ path_flex_plots }}/{{ utility_arg }}" \
+      {{ if batch != "" { "--batch " + batch } else { "" } }}
+
+validate-demand-flex-all elasticity="-0.10" batch="":
+    just validate-demand-flex cenhud {{ elasticity }} "{{ batch }}"
+    just validate-demand-flex coned {{ elasticity }} "{{ batch }}"
+    just validate-demand-flex nimo {{ elasticity }} "{{ batch }}"
+    just validate-demand-flex nyseg {{ elasticity }} "{{ batch }}"
+    just validate-demand-flex or {{ elasticity }} "{{ batch }}"
+    just validate-demand-flex psegli {{ elasticity }} "{{ batch }}"
+    just validate-demand-flex rge {{ elasticity }} "{{ batch }}"
+
+# Run the elasticity calibration sweep: tests a range of elasticity values per
+# utility and finds the one matching the Arcturus 2.0 empirical target.
+#
+# Sweep all utilities (default range -0.04 to -0.50, step -0.02):
+#   just -f ny/Justfile calibrate-demand-flex-elasticity
+#
+# Subset of utilities:
+#   just -f ny/Justfile calibrate-demand-flex-elasticity --utilities=coned,nimo
+#
+# Custom range:
+#   just -f ny/Justfile calibrate-demand-flex-elasticity --epsilon-start -0.04 --epsilon-end -0.30 --epsilon-step -0.01
+#
+# Write no-tech + with-tech seasonal elasticities to config/periods/*.yaml:
+#   just -f ny/Justfile calibrate-demand-flex-elasticity --write-periods
+#
+# Compare analytical predictions against a CAIRO batch (no-tech):
+#   just -f ny/Justfile calibrate-demand-flex-elasticity --utilities=cenhud --compare-batch ny_20260326_batch
+#
+# Compare against a CAIRO batch that used with-tech elasticities:
+
+# just -f ny/Justfile calibrate-demand-flex-elasticity --utilities=cenhud --compare-batch ny_20260326_batch --with-tech
+calibrate-demand-flex-elasticity *args:
+    cd {{ project_root }} && uv run python -m utils.pre.calibrate_demand_flex_elasticity \
+      --state ny \
+      --output-dir "{{ path_flex_plots }}/diagnostic" \
+      {{ args }}

rate_design/hp_rates/ny/config/periods/cenhud.yaml

@@ -1,2 +1,14 @@
-winter_months: [10, 11, 12, 1, 2, 3]
+winter_months:
+  - 10
+  - 11
+  - 12
+  - 1
+  - 2
+  - 3
 tou_window_hours: 5
+elasticity:
+  summer: -0.14
+  winter: -0.12
+elasticity_with_tech:
+  summer: -0.22
+  winter: -0.18

rate_design/hp_rates/ny/config/periods/coned.yaml

@@ -1,2 +1,14 @@
-winter_months: [10, 11, 12, 1, 2, 3]
+winter_months:
+  - 10
+  - 11
+  - 12
+  - 1
+  - 2
+  - 3
 tou_window_hours: 3
+elasticity:
+  summer: -0.12
+  winter: -0.1
+elasticity_with_tech:
+  summer: -0.18
+  winter: -0.16

rate_design/hp_rates/ny/config/periods/nimo.yaml

@@ -1,2 +1,14 @@
-winter_months: [10, 11, 12, 1, 2, 3]
+winter_months:
+  - 10
+  - 11
+  - 12
+  - 1
+  - 2
+  - 3
 tou_window_hours: 3
+elasticity:
+  summer: -0.1
+  winter: -0.1
+elasticity_with_tech:
+  summer: -0.18
+  winter: -0.16

rate_design/hp_rates/ny/config/periods/nyseg.yaml

@@ -1,2 +1,14 @@
-winter_months: [10, 11, 12, 1, 2, 3]
+winter_months:
+  - 10
+  - 11
+  - 12
+  - 1
+  - 2
+  - 3
 tou_window_hours: 3
+elasticity:
+  summer: -0.1
+  winter: -0.1
+elasticity_with_tech:
+  summer: -0.18
+  winter: -0.16

rate_design/hp_rates/ny/config/periods/or.yaml

@@ -1,2 +1,14 @@
-winter_months: [10, 11, 12, 1, 2, 3]
+winter_months:
+  - 10
+  - 11
+  - 12
+  - 1
+  - 2
+  - 3
 tou_window_hours: 5
+elasticity:
+  summer: -0.14
+  winter: -0.12
+elasticity_with_tech:
+  summer: -0.24
+  winter: -0.2

rate_design/hp_rates/ny/config/periods/psegli.yaml

@@ -1,2 +1,14 @@
-winter_months: [10, 11, 12, 1, 2, 3]
+winter_months:
+  - 10
+  - 11
+  - 12
+  - 1
+  - 2
+  - 3
 tou_window_hours: 5
+elasticity:
+  summer: -0.14
+  winter: -0.12
+elasticity_with_tech:
+  summer: -0.22
+  winter: -0.18

rate_design/hp_rates/ny/config/periods/rge.yaml

@@ -1,2 +1,14 @@
-winter_months: [10, 11, 12, 1, 2, 3]
+winter_months:
+  - 10
+  - 11
+  - 12
+  - 1
+  - 2
+  - 3
 tou_window_hours: 3
+elasticity:
+  summer: -0.1
+  winter: -0.1
+elasticity_with_tech:
+  summer: -0.18
+  winter: -0.16

rate_design/hp_rates/ny/config/rev_requirement/cenhud_hp_vs_nonhp.yaml

@@ -1,15 +1,15 @@
 utility: cenhud
 group_col: has_hp
 cross_subsidy_col: BAT_percustomer
-source_run_dir: s3://data.sb/switchbox/cairo/outputs/hp_rates/ny/cenhud/ny_20260325b_r1-16/20260325_143137_ny_cenhud_run1_up00_precalc__default
+source_run_dir: s3://data.sb/switchbox/cairo/outputs/hp_rates/ny/cenhud/ny_20260326_elast_seasonal_tech/20260326_194621_ny_cenhud_run1_up00_precalc__default
 total_delivery_revenue_requirement: 418151641.73
 total_delivery_and_supply_revenue_requirement: 646688375.32
 subclass_revenue_requirements:
   non-hp:
-    delivery: 401392812.5614301
-    supply: 209823391.6038903
-    total: 611216204.1653204
+    delivery: 401392812.56143004
+    supply: 209823391.60389048
+    total: 611216204.1653205
   hp:
-    delivery: 16758829.168569982
-    supply: 18713341.98610972
-    total: 35472171.1546797
+    delivery: 16758829.16856999
+    supply: 18713341.986109704
+    total: 35472171.15467969

rate_design/hp_rates/ny/config/scenarios/scenarios_cenhud.yaml

@@ -367,7 +367,9 @@ runs:
     year_run: 2025
     year_dollar_conversion: 2025
     process_workers: 8
-    elasticity: -0.1
+    elasticity:
+      summer: -0.22
+      winter: -0.18
   14:
     run_name: ny_cenhud_run14_up00_precalc_supply__hp_seasonalTOU_flex_vs_default
     state: NY
@@ -396,7 +398,9 @@ runs:
     year_run: 2025
     year_dollar_conversion: 2025
     process_workers: 8
-    elasticity: -0.1
+    elasticity:
+      summer: -0.22
+      winter: -0.18
   15:
     run_name: ny_cenhud_run15_up02_default__hp_seasonalTOU_flex
     state: NY
@@ -424,7 +428,9 @@ runs:
     year_run: 2025
     year_dollar_conversion: 2025
     process_workers: 8
-    elasticity: -0.1
+    elasticity:
+      summer: -0.22
+      winter: -0.18
   16:
     run_name: ny_cenhud_run16_up02_default_supply__hp_seasonalTOU_flex
     state: NY
@@ -452,4 +458,6 @@ runs:
     year_run: 2025
     year_dollar_conversion: 2025
     process_workers: 8
-    elasticity: -0.1
+    elasticity:
+      summer: -0.22
+      winter: -0.18

rate_design/hp_rates/ny/config/scenarios/scenarios_coned.yaml

@@ -367,7 +367,9 @@ runs:
     year_run: 2025
     year_dollar_conversion: 2025
     process_workers: 8
-    elasticity: -0.1
+    elasticity:
+      summer: -0.18
+      winter: -0.16
   14:
     run_name: ny_coned_run14_up00_precalc_supply__hp_seasonalTOU_flex_vs_default
     state: NY
@@ -396,7 +398,9 @@ runs:
     year_run: 2025
     year_dollar_conversion: 2025
     process_workers: 8
-    elasticity: -0.1
+    elasticity:
+      summer: -0.18
+      winter: -0.16
   15:
     run_name: ny_coned_run15_up02_default__hp_seasonalTOU_flex
     state: NY
@@ -424,7 +428,9 @@ runs:
     year_run: 2025
     year_dollar_conversion: 2025
     process_workers: 8
-    elasticity: -0.1
+    elasticity:
+      summer: -0.18
+      winter: -0.16
   16:
     run_name: ny_coned_run16_up02_default_supply__hp_seasonalTOU_flex
     state: NY
@@ -452,4 +458,6 @@ runs:
     year_run: 2025
     year_dollar_conversion: 2025
     process_workers: 8
-    elasticity: -0.1
+    elasticity:
+      summer: -0.18
+      winter: -0.16

rate_design/hp_rates/ny/config/scenarios/scenarios_nimo.yaml

@@ -367,7 +367,9 @@ runs:
     year_run: 2025
     year_dollar_conversion: 2025
     process_workers: 8
-    elasticity: -0.1
+    elasticity:
+      summer: -0.18
+      winter: -0.16
   14:
     run_name: ny_nimo_run14_up00_precalc_supply__hp_seasonalTOU_flex_vs_default
     state: NY
@@ -396,7 +398,9 @@ runs:
     year_run: 2025
     year_dollar_conversion: 2025
     process_workers: 8
-    elasticity: -0.1
+    elasticity:
+      summer: -0.18
+      winter: -0.16
   15:
     run_name: ny_nimo_run15_up02_default__hp_seasonalTOU_flex
     state: NY
@@ -424,7 +428,9 @@ runs:
     year_run: 2025
     year_dollar_conversion: 2025
     process_workers: 8
-    elasticity: -0.1
+    elasticity:
+      summer: -0.18
+      winter: -0.16
   16:
     run_name: ny_nimo_run16_up02_default_supply__hp_seasonalTOU_flex
     state: NY
@@ -452,4 +458,6 @@ runs:
     year_run: 2025
     year_dollar_conversion: 2025
     process_workers: 8
-    elasticity: -0.1
+    elasticity:
+      summer: -0.18
+      winter: -0.16

rate_design/hp_rates/ny/config/scenarios/scenarios_nyseg.yaml

@@ -367,7 +367,9 @@ runs:
     year_run: 2025
     year_dollar_conversion: 2025
     process_workers: 8
-    elasticity: -0.1
+    elasticity:
+      summer: -0.18
+      winter: -0.16
   14:
     run_name: ny_nyseg_run14_up00_precalc_supply__hp_seasonalTOU_flex_vs_default
     state: NY
@@ -396,7 +398,9 @@ runs:
     year_run: 2025
     year_dollar_conversion: 2025
     process_workers: 8
-    elasticity: -0.1
+    elasticity:
+      summer: -0.18
+      winter: -0.16
   15:
     run_name: ny_nyseg_run15_up02_default__hp_seasonalTOU_flex
     state: NY
@@ -424,7 +428,9 @@ runs:
     year_run: 2025
     year_dollar_conversion: 2025
     process_workers: 8
-    elasticity: -0.1
+    elasticity:
+      summer: -0.18
+      winter: -0.16
   16:
     run_name: ny_nyseg_run16_up02_default_supply__hp_seasonalTOU_flex
     state: NY
@@ -452,4 +458,6 @@ runs:
     year_run: 2025
     year_dollar_conversion: 2025
     process_workers: 8
-    elasticity: -0.1
+    elasticity:
+      summer: -0.18
+      winter: -0.16