remove partial actual yrs from drf

Author: ge-tbs

notebooks/drf-yrs.ipynb

@@ -51,112 +51,14 @@
     "si = pd.read_csv(si_path, keep_default_na=False, na_values='', delimiter=';', engine='python', skipfooter=2)"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "4071d216",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "drf = load_csv('rbpo.csv', config, False)\n",
-    "drf = standardize_column_names(drf)\n",
-    "drf['fiscal_yr'] = drf['fiscal_yr'].apply(clean_fiscal_yr)\n",
-    "\n",
-    "# Define columns related to planned and actual measures: spending and FTEs \n",
-    "fte_spend_cols = [\n",
-    "    'planned_spending_1', \n",
-    "    'actual_spending', \n",
-    "    'planned_spending_2', \n",
-    "    'planned_spending_3',\n",
-    "    'planned_ftes_1', \n",
-    "    'actual_ftes', \n",
-    "    'planned_ftes_2', \n",
-    "    'planned_ftes_3'\n",
-    "]\n",
-    "\n",
-    "# Melt (unpivot) the DataFrame to long format\n",
-    "drf = pd.melt(\n",
-    "    drf, \n",
-    "    id_vars=['fiscal_yr', 'org_id', 'program_id'], \n",
-    "    value_vars=fte_spend_cols, \n",
-    "    var_name='plan_actual_spendfte_yr', \n",
-    "    value_name='measure'\n",
-    ")\n",
-    "\n",
-    "# Split 'plan_actual_yr' into separate columns for planned/actual, spending/FTEs, and year adjustment\n",
-    "drf[['planned_actual', 'spending_fte', 'yr_adjust']] = drf['plan_actual_spendfte_yr'].str.split('_', expand=True)\n",
-    "drf['yr_adjust'] = drf['yr_adjust'].fillna('1').astype(int) - 1\n",
-    "\n",
-    "\n",
-    "# Calculate 4-digit 'measure_yr' and 'report_yr' from 'fiscal_yr' and 'yr_adjust'\n",
-    "drf['measure_yr'] = drf['fiscal_yr'].str.split('-').str[1].astype(int) + drf['yr_adjust']\n",
-    "drf['report_yr'] = drf['fiscal_yr'].str.split('-').str[1].astype(int)\n",
-    "\n",
-    "latest_si_fy_by_org = si.groupby('org_id')['fiscal_yr'].max()\n",
-    "\n",
-    "drf = pd.merge(drf, latest_si_fy_by_org, on='org_id', how='left', suffixes=['', '_si'])\n",
-    "drf['fiscal_yr_si'] = drf['fiscal_yr_si'].fillna('0-0')\n",
-    "drf['latest_si_yr'] = drf['fiscal_yr_si'].str.split('-').str[1].astype(int)\n",
-    "\n",
-    "drf_actuals = drf[\n",
-    "    (drf['planned_actual'] == 'actual')\n",
-    "].dropna()\n",
-    "\n",
-    "drf_planned = drf[\n",
-    "    (drf['planned_actual'] == 'planned')\n",
-    "].dropna()\n",
-    "\n",
-    "# Determine the highest measure year for actuals\n",
-    "latest_actuals = drf_actuals.groupby(['org_id', 'program_id', 'spending_fte'])['report_yr'].max().reset_index()\n",
-    "\n",
-    "# Merge in the highest measure year for actuals in the planned table\n",
-    "drf_planned = pd.merge(left=drf_planned, right=latest_actuals, how='left', on=['org_id', 'program_id', 'spending_fte'], suffixes=['', '_actuals']) \n",
-    "\n",
-    "# Only keep planned years that are greater than the latest actual report year\n",
-    "drf_planned = drf_planned[drf_planned['measure_yr']>drf_planned['report_yr_actuals']]\n",
-    "\n",
-    "# # # Each report year has 3 measure years for planned values.\n",
-    "# # Only keep records that have the highest report year for that given program, measure type, and measure year\n",
-    "idx = drf_planned.groupby(['org_id', 'program_id', 'spending_fte', 'measure_yr'])['report_yr'].idxmax()\n",
-    "drf_planned = drf_planned.loc[idx]\n",
-    "\n",
-    "# # # Concatenate actuals and planned entries\n",
-    "drf = pd.concat([drf_actuals, drf_planned])\n",
-    "\n",
-    "drf = drf[[\n",
-    "    'org_id', \n",
-    "    'latest_si_yr', \n",
-    "    'program_id', \n",
-    "    'report_yr', \n",
-    "    'measure_yr', \n",
-    "    'planned_actual', \n",
-    "    'spending_fte',\n",
-    "    'measure']].reset_index(drop=True)\n",
-    "\n",
-    "# # Set up si_link_yr: a fiscal year column to be able to include years \n",
-    "# # beyond the service inventory when joining by service id and fy.\n",
-    "# # if measure year > latest service fy, = latest service fy, else use measure_yr\n",
-    "drf.loc[drf['measure_yr']>drf['latest_si_yr'], 'si_link_yr'] = drf['latest_si_yr']\n",
-    "drf.loc[drf['measure_yr']<=drf['latest_si_yr'], 'si_link_yr'] = drf['measure_yr']\n",
-    "drf['si_link_yr'] = drf['si_link_yr'].astype(int)\n",
-    "\n",
-    "# # # Return years to fiscal year YYYY-YYYY format\n",
-    "drf['report_yr'] = (drf['report_yr']-1).apply(str) +\"-\"+ (drf['report_yr']).apply(str)\n",
-    "drf['measure_yr'] = (drf['measure_yr']-1).apply(str) +\"-\"+ (drf['measure_yr']).apply(str)\n",
-    "drf['si_link_yr'] = (drf['si_link_yr']-1).apply(str) +\"-\"+ (drf['si_link_yr']).apply(str)\n",
-    "drf['latest_si_yr'] = (drf['latest_si_yr']-1).apply(str) +\"-\"+ (drf['latest_si_yr']).apply(str)\n",
-    "\n",
-    "\n"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
    "id": "2373cc17",
    "metadata": {},
    "outputs": [],
    "source": [
-    "drf.loc[(drf['org_id']==130)&(drf['program_id']=='ISS02')]\n",
+    "drf.loc[(drf['program_id']=='BRB01')]\n",
     "# drf['valid_plan'] = (drf['report_yr'] > latest_si_fy)\n",
     "\n",
     "# drf[(drf['org_id']==130) & (drf['program_id']=='ISS02') & (drf['planned_actual']=='planned') & (drf['report_yr'] > latest_si_fy)]\n"
@@ -219,6 +121,11 @@
     "drf_actuals = drf[drf['planned_actual']=='actual'].dropna(subset=['measure']).copy()\n",
     "drf_planned = drf[drf['planned_actual']=='planned'].dropna(subset=['measure']).copy()\n",
     "\n",
+    "# Drop any actuals from the fiscal year in progress\n",
+    "# TODO: Turn this into a function that looks at the current datetime\n",
+    "current_yr = 2026\n",
+    "drf_actuals = drf_actuals[drf_actuals['measure_yr']<current_yr]\n",
+    "\n",
     "# Determine the highest measure year for actuals\n",
     "latest_actuals = (drf_actuals\n",
     "                  .groupby(['org_id', 'program_id', 'spending_fte'], as_index=False)['report_yr']\n",

src/utils.py

@@ -155,6 +155,11 @@ def build_drf(si, config, snapshot=False):
         drf_actuals = drf[drf['planned_actual']=='actual'].dropna(subset=['measure']).copy()
         drf_planned = drf[drf['planned_actual']=='planned'].dropna(subset=['measure']).copy()
 
+        # Drop any actuals from the fiscal year in progress
+        # TODO: Turn this into a function that looks at the current datetime
+        current_yr = 2026
+        drf_actuals = drf_actuals[drf_actuals['measure_yr']<current_yr]
+
         # Determine the highest measure year for actuals
         latest_actuals = (drf_actuals
                         .groupby(['org_id', 'program_id', 'spending_fte'], as_index=False)['report_yr']