Merge branch 'main' into 41-revenue-sharing-mechanism

Author: alxsmith

docs/index.md

@@ -10,11 +10,27 @@ package for ca_npa_howtopay
 ## Overview
 The `npa-howtopay` package provides functionality for analyzing energy costs and project economics under different expense scenarios.
 
+## Scenario Definitions
+| Scenario Name      | Description                                                                                      |
+|--------------------|--------------------------------------------------------------------------------------------------|
+| bau                | Business-as-usual (BAU): No NPA projects, baseline utility costs and spending.                   |
+| taxpayer           | All NPA costs are paid by taxpayers, not by utility customers.                                   |
+| gas_capex          | Gas utility pays for NPA projects as capital expenditures (added to gas ratebase).               |
+| gas_opex           | Gas utility pays for NPA projects as operating expenses (expensed in year incurred).             |
+| electric_capex     | Electric utility pays for NPA projects as capital expenditures (added to electric ratebase).     |
+| electric_opex      | Electric utility pays for NPA projects as operating expenses (expensed in year incurred).        |
+
+Each scenario specifies who pays for NPA projects (gas utility, electric utility, or taxpayers) and whether costs are treated as capital (capex) or operating (opex) expenses.
+
+
+
 ## Core Modules
 
 ### Main Package (`npa_howtopay`)
 - **`run_model`** - Main function to execute the cost analysis model for a single scenario
-- **`analyze_scenarios`** - Execute run_model for all scenarios and return results and delta from BAU (no NPAs) dfs.
+- **`run_all_scenarios`** - Execute run_model for all scenarios and return all results
+- **`create_delta_df`** - Selects columns of interest from all results and calculates difference from BAU (expect for converter bills which are compared to non-converter bill in each scenario)
+- **`return_absolute_values_df`** - Concats dfs from `run_all_scenarios` and filters to selected columns
 
 ### Initialize Model
 If running locally:
@@ -54,5 +70,10 @@ from npa_howtopay import run_model, load_scenario_from_yaml
   scenario_runs = create_scenario_runs(2025, 2050, ["gas", "electric"], ["capex", "opex"])
   input_params = load_scenario_from_yaml(run_name)
   ts_params = load_time_series_params_from_yaml(run_name)
-  results_df, delta_bau_df = analyze_scenarios(scenario_runs, input_params, ts_params)
+  # Run model for all scenarios
+  results_df_all = run_all_scenarios(scenario_runs, input_params, ts_params)
+  # Single df with delta values
+  delta_df = create_delta_df(results_df_all, COMPARE_COLS)
+  # Single df with absolute values
+  results_df = return_absolute_values_df(results_df_all, COMPARE_COLS)
 ```

run_example.py

@@ -1,8 +1,7 @@
 # run_example.py
 
-import polars as pl
 
-from npa_howtopay.model import analyze_scenarios, create_scenario_runs
+from npa_howtopay.model import create_delta_df, create_scenario_runs, return_absolute_values_df, run_all_scenarios
 from npa_howtopay.params import (
     COMPARE_COLS,
     load_scenario_from_yaml,
@@ -25,30 +24,25 @@
     scenario_runs = create_scenario_runs(2025, 2050, ["gas", "electric"], ["capex", "opex"])
     input_params = load_scenario_from_yaml("sample")
     ts_params = load_time_series_params_from_yaml("sample")
-    results_df, delta_bau_df = analyze_scenarios(scenario_runs, input_params, ts_params)
+    results_df_all = run_all_scenarios(scenario_runs, input_params, ts_params)
+
+    delta_df = create_delta_df(results_df_all, COMPARE_COLS)
+    results_df = return_absolute_values_df(results_df_all, COMPARE_COLS)
+
     # EDA plots
     # For delta values (original behavior)
-    plt_df_delta = transform_to_long_format(delta_bau_df)
+    plt_df_delta = transform_to_long_format(delta_df)
     plot_revenue_requirements(plt_df_delta, show_absolute=False, save_dir="plots")
     plot_volumetric_tariff(plt_df_delta, show_absolute=False, save_dir="plots")
     plot_ratebase(plt_df_delta, show_absolute=False, save_dir="plots")
     plot_return_on_ratebase_pct(plt_df_delta, show_absolute=False, save_dir="plots")
     plot_depreciation_accruals(plt_df_delta, show_absolute=False, save_dir="plots")
     plot_user_bills_converts(plt_df_delta, show_absolute=False, save_dir="plots")
     plot_user_bills_nonconverts(plt_df_delta, show_absolute=False, save_dir="plots")
-    # For absolute values - filter results_df to COMPARE_COLS and transform
-    filtered_results = {}
-    for scenario_name, scenario_df in results_df.items():
-        if scenario_name == "bau":
-            continue  # Skip BAU for absolute value plotting
-        filtered_results[scenario_name] = scenario_df.select(["year", *COMPARE_COLS])
-
-    # Concatenate and transform to long format
-    combined_df = pl.concat(
-        [df.with_columns(pl.lit(scenario_id).alias("scenario_id")) for scenario_id, df in filtered_results.items()],
-        how="vertical",
-    )
-    plt_df_absolute = transform_to_long_format(combined_df)
+    plot_total_bills(plt_df_delta, show_absolute=False, save_dir="plots")
+    plot_return_on_ratebase_pct(plt_df_delta, show_absolute=False, save_dir="plots")
+    # For absolute values
+    plt_df_absolute = transform_to_long_format(results_df)
     plot_revenue_requirements(plt_df_absolute, show_absolute=True, save_dir="plots")
     plot_volumetric_tariff(plt_df_absolute, show_absolute=True, save_dir="plots")
     plot_ratebase(plt_df_absolute, show_absolute=True, save_dir="plots")
@@ -57,7 +51,8 @@
     plot_user_bills_converts(plt_df_absolute, show_absolute=True, save_dir="plots")
     plot_user_bills_nonconverts(plt_df_absolute, show_absolute=True, save_dir="plots")
 
-    plot_total_bills(delta_bau_df, save_dir="plots")
+    plot_total_bills(results_df, show_absolute=True, save_dir="plots")
+    plot_return_on_ratebase_pct(results_df, show_absolute=True, save_dir="plots")
 
 # Method 2: Using web parameters (scalar values)
 web_params = {
@@ -72,10 +67,14 @@
     "gas_fixed_overhead_costs": 100.0,
     "electric_fixed_overhead_costs": 100.0,
     "gas_bau_lpp_costs_per_year": 100.0,
+    "npa_year_start": 2025,
+    "npa_year_end": 2030,
     "is_scattershot": False,
 }
 
 input_params2 = load_scenario_from_yaml("sample")  # Still load base params from YAML
 ts_params2 = load_time_series_params_from_web_params(web_params, 2025, 2050)
 
-out2 = analyze_scenarios(scenario_runs, input_params2, ts_params2)
+out2 = run_all_scenarios(scenario_runs, input_params2, ts_params2)
+delta_df2 = create_delta_df(out2, COMPARE_COLS)
+results_df2 = return_absolute_values_df(out2, COMPARE_COLS)

src/npa_howtopay/model.py

@@ -702,58 +702,91 @@ def run_model(scenario_params: ScenarioParams, input_params: InputParams, ts_par
     return results_df
 
 
-def create_delta_bau_df(results_df: dict[str, pl.DataFrame], compare_cols_all: list[str]) -> pl.DataFrame:
-    bau_df = results_df["bau"].select(["year"] + compare_cols_all)
+def create_delta_df(results_dfs: dict[str, pl.DataFrame], compare_cols_all: list[str]) -> pl.DataFrame:
+    bau_df = results_dfs["bau"].select(["year"] + compare_cols_all)
 
     # Default mapping: most columns compare against themselves in BAU
-    column_mappings = {col: col for col in compare_cols_all}
+    column_mappings = {col: (col, "bau") for col in compare_cols_all}
 
-    # Override special cases: converts columns compare against nonconverts in BAU
+    # Override special cases: converts columns compare against nonconverts in same scenario
     special_cases = {
-        "converts_total_bill_per_user": "nonconverts_total_bill_per_user",
-        "electric_converts_bill_per_user": "electric_nonconverts_bill_per_user",
-        "gas_converts_bill_per_user": "gas_nonconverts_bill_per_user",
+        "converts_total_bill_per_user": ("nonconverts_total_bill_per_user", "self"),
+        "electric_converts_bill_per_user": ("electric_nonconverts_bill_per_user", "self"),
+        "gas_converts_bill_per_user": ("gas_nonconverts_bill_per_user", "self"),
     }
     column_mappings.update(special_cases)
 
+    # Determine what BAU columns we need
+    bau_cols_needed = set()
+    for scenario_col, (baseline_col, baseline_df_name) in column_mappings.items():
+        if baseline_df_name == "bau" and scenario_col in compare_cols_all:
+            bau_cols_needed.add(baseline_col)
+
     # Create comparison DataFrames for each scenario
     comparison_dfs = {}
-    for scenario_name, scenario_df in results_df.items():
+    for scenario_name, scenario_df in results_dfs.items():
         if scenario_name == "bau":
-            continue  # Skip BAU itself
-
-        # Join with BAU columns
-        bau_cols_to_join = ["year"] + list(set(column_mappings.values()))
-        bau_renames = {col: f"bau_{col}" for col in set(column_mappings.values())}
-
-        comparison_df = scenario_df.join(
-            bau_df.select(bau_cols_to_join).rename(bau_renames),
-            on="year",
-        )
+            continue
+
+        # Do one join with all needed BAU columns
+        if bau_cols_needed:
+            bau_cols_to_join = ["year"] + list(bau_cols_needed)
+            bau_renames = {col: f"bau_{col}" for col in bau_cols_needed}
+            working_df = scenario_df.join(
+                bau_df.select(bau_cols_to_join).rename(bau_renames),
+                on="year",
+            )
+        else:
+            working_df = scenario_df
 
-        # Create comparison columns
+        # Create all comparison expressions
         comparison_cols = []
-        for scenario_col, bau_col in column_mappings.items():
+        for scenario_col, (baseline_col, baseline_df_name) in column_mappings.items():
             if scenario_col in compare_cols_all:
-                comparison_cols.append(pl.col(scenario_col).sub(pl.col(f"bau_{bau_col}")))
+                if baseline_df_name == "self":
+                    if baseline_col in scenario_df.columns:
+                        comparison_cols.append(pl.col(scenario_col).sub(pl.col(baseline_col)))
+                else:  # bau
+                    if baseline_col in bau_df.columns:
+                        comparison_cols.append(pl.col(scenario_col).sub(pl.col(f"bau_{baseline_col}")))
 
         # Select final columns
-        comparison_df = comparison_df.select(["year", *comparison_cols])
-        comparison_dfs[scenario_name] = comparison_df
+        if comparison_cols:
+            final_df = working_df.select(["year", *comparison_cols])
+        else:
+            final_df = working_df.select(["year"])
+
+        comparison_dfs[scenario_name] = final_df
 
-    delta_bau_df = pl.concat(
+    delta_df = pl.concat(
         [df.with_columns(pl.lit(scenario_id).alias("scenario_id")) for scenario_id, df in comparison_dfs.items()],
         how="vertical",
     )
-    return delta_bau_df
+    return delta_df
 
 
-def analyze_scenarios(
+def run_all_scenarios(
     scenario_runs: dict[str, ScenarioParams], input_params: InputParams, ts_params: TimeSeriesParams
-) -> tuple[dict[str, pl.DataFrame], pl.DataFrame]:
-    results_df = {}
+) -> dict[str, pl.DataFrame]:
+    results_dfs = {}
     for scenario_name, scenario_params in scenario_runs.items():
         logger.info(f"Running scenario: {scenario_name}")
-        results_df[scenario_name] = run_model(scenario_params, input_params, ts_params)
+        results_dfs[scenario_name] = run_model(scenario_params, input_params, ts_params)
+
+    return results_dfs
+
+
+def return_absolute_values_df(results_dfs: dict[str, pl.DataFrame], compare_cols_all: list[str]) -> pl.DataFrame:
+    filtered_results = {}
+    for scenario_name, scenario_df in results_dfs.items():
+        filtered_results[scenario_name] = scenario_df.select(["year", *compare_cols_all])
+        print(f"Added {scenario_name} with shape {scenario_df.shape}")
+
+    print(f"filtered_results keys: {filtered_results.keys()}")
+    # Concatenate and transform to long format
+    combined_df = pl.concat(
+        [df.with_columns(pl.lit(scenario_id).alias("scenario_id")) for scenario_id, df in filtered_results.items()],
+        how="vertical",
+    )
 
-    return results_df, create_delta_bau_df(results_df, COMPARE_COLS)
+    return combined_df

src/npa_howtopay/web_params.py

@@ -17,6 +17,9 @@ class WebParams:
     electric_fixed_overhead_costs: float
     gas_bau_lpp_costs_per_year: float
     is_scattershot: bool
+    npa_year_start: Optional[int] = None
+    npa_year_end: Optional[int] = None
+
     # stuff_for_producing_ratebase_baseline
 
 
@@ -101,8 +104,16 @@ def create_time_series_from_web_params(
     web_params: WebParams, start_year: int, end_year: int, cost_inflation_rate: float = 0.0
 ) -> dict[str, pl.DataFrame]:
     """Create all time series DataFrames from web parameters"""
+    npa_year_end = web_params.npa_year_end if web_params.npa_year_end is not None else end_year
+    npa_year_start = web_params.npa_year_start if web_params.npa_year_start is not None else start_year
+
+    if npa_year_start < start_year:
+        raise ValueError("npa_year_start must be greater than or equal to SharedParams.start_year")
+    if npa_year_end > end_year:
+        raise ValueError("npa_year_end must be less than or equal to npa_end_year")
+
     return {
-        "npa_projects": create_npa_projects(web_params, start_year, end_year),
+        "npa_projects": create_npa_projects(web_params, npa_year_start, npa_year_end),
         "scattershot_electrification_users_per_year": create_scattershot_electrification_df(
             web_params, start_year, end_year
         ),