updated sid-list to display latest and first fy reported

Author: ge-tbs

inputs/si_2024.csv

@@ -0,0 +1,214 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "f9a37f13-0b46-42f5-b8fe-99ff5099f890",
+   "metadata": {},
+   "source": [
+    "# Figuring out the connection to DRF/DRR/DP figures"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 42,
+   "id": "a65fa6c7-24f9-4fb6-8382-096f3c487d10",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "import re\n",
+    "import pytz\n",
+    "import os\n",
+    "from pathlib import Path\n",
+    "import sys\n",
+    "sys.path.append(\"/home/jovyan/shared/service-data\")\n",
+    "\n",
+    "from src.clean import clean_percentage, normalize_string, standardize_column_names, clean_fiscal_yr\n",
+    "from src.load import load_csv_from_raw\n",
+    "from src.export import export_to_csv\n",
+    "from src.merge import merge_si, merge_ss"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 43,
+   "id": "048e8db3-f8e2-45f0-a9de-5ca5bc0fe351",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Exported dept.csv to /home/jovyan/shared/service-data/outputs/utils\n",
+      "Exported si.csv to /home/jovyan/shared/service-data/outputs\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Define the base directory\n",
+    "base_dir = Path.cwd()\n",
+    "parent_dir = base_dir.parent\n",
+    "\n",
+    "# File paths for outputs\n",
+    "data_files = {\n",
+    "    \"rbpo\": parent_dir / \"inputs\" / \"rbpo.csv\",\n",
+    "    \"org_var\": parent_dir / \"inputs\" / \"org_var.csv\",\n",
+    "    \"serv_prog\": parent_dir / \"inputs\" / \"serv_prog.csv\"\n",
+    "}\n",
+    "\n",
+    "si = merge_si()\n",
+    "rbpo = pd.read_csv(data_files[\"rbpo\"])\n",
+    "serv_prog = pd.read_csv(data_files[\"serv_prog\"])\n",
+    "\n",
+    "rbpo = standardize_column_names(rbpo)\n",
+    "rbpo['fiscal_yr'] = rbpo['fiscal_yr'].apply(clean_fiscal_yr)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "77e12a73-f357-4834-a99f-eab56fcc87dd",
+   "metadata": {},
+   "source": [
+    "# Define columns related to measures: spending and FTEs (planned and actual)\n",
+    "fte_spend_cols = [\n",
+    "    'planned_spending_1', 'actual_spending', 'planned_spending_2', 'planned_spending_3',\n",
+    "    'planned_ftes_1', 'actual_ftes', 'planned_ftes_2', 'planned_ftes_3'\n",
+    "]\n",
+    "\n",
+    "# Melt (unpivot) the DataFrame to long format\n",
+    "drf = pd.melt(\n",
+    "    rbpo, \n",
+    "    id_vars=['fiscal_yr', 'org_id', 'program_id'], \n",
+    "    value_vars=fte_spend_cols, \n",
+    "    var_name='plan_actual_spendfte_yr', \n",
+    "    value_name='measure'\n",
+    ")\n",
+    "\n",
+    "# Split 'plan_actual_yr' into separate columns for planned/actual, spending/FTEs, and year adjustment\n",
+    "drf[['planned_actual', 'spending_fte', 'yr_adjust']] = drf['plan_actual_spendfte_yr'].str.split('_', expand=True)\n",
+    "drf['yr_adjust'] = drf['yr_adjust'].fillna('1').astype(int) - 1\n",
+    "\n",
+    "# Calculate 4-digit 'measure_yr' and 'report_yr' from 'fiscal_yr' and 'yr_adjust'\n",
+    "drf['measure_yr'] = drf['fiscal_yr'].str.split('-').str[1].astype(int) + drf['yr_adjust']\n",
+    "drf['report_yr'] = drf['fiscal_yr'].str.split('-').str[1].astype(int)\n",
+    "\n",
+    "# Get the latest fiscal year from the Service inventory (four digit fy, year of end of fy)\n",
+    "# latest_si_fy = si['fiscal_yr'].str.split('-').str[1].astype(int).max()\n",
+    "latest_si_fy = 2024\n",
+    "\n",
+    "# Separate actuals and future planned data\n",
+    "drf_actuals = drf[\n",
+    "    (drf['planned_actual'] == 'actual') & \n",
+    "    (drf['report_yr'] <= latest_si_fy)\n",
+    "].dropna()\n",
+    "\n",
+    "drf_planned = drf[\n",
+    "    (drf['planned_actual'] == 'planned') &\n",
+    "    (drf['report_yr'] > latest_si_fy) \n",
+    "].dropna()\n",
+    "\n",
+    "# Each report year has 3 measure years for planned values.\n",
+    "# Only keep records that have the highest report year for that given program, measure type, and measure year\n",
+    "idx = drf_planned.groupby(['program_id', 'spending_fte', 'measure_yr'])['report_yr'].idxmax()\n",
+    "drf_planned = drf_planned.loc[idx]\n",
+    "\n",
+    "drf_actuals_checksum = drf_actuals['measure'].sum()\n",
+    "drf_planned_checksum = drf_planned['measure'].sum()\n",
+    "\n",
+    "print(\"drf_actuals.shape:\", drf_actuals.shape)\n",
+    "print(\"checksum:\", drf_actuals_checksum)\n",
+    "print(\"drf_planned.shape:\", drf_planned.shape)\n",
+    "print(\"checksum:\", drf_planned_checksum)\n",
+    "\n",
+    "# Concatenate actuals and planned entries\n",
+    "drf = pd.concat([drf_actuals, drf_planned])\n",
+    "drf_checksum = drf['measure'].sum()\n",
+    "\n",
+    "print(\"drf.shape:\", drf.shape)\n",
+    "print(\"checksum:\", drf_checksum)\n",
+    "print(\"checksum difference:\", drf_checksum - (drf_planned_checksum+drf_actuals_checksum))\n",
+    "print(drf.info())\n",
+    "\n",
+    "# Pivot to get a wide format table with spending/FTE columns\n",
+    "print(\"pivoting drf\")\n",
+    "drf = drf.pivot_table(\n",
+    "    index=['org_id', 'program_id', 'report_yr', 'measure_yr', 'planned_actual'], \n",
+    "    columns=['spending_fte'], \n",
+    "    values='measure'\n",
+    ").sort_values(\n",
+    "    by=['org_id', 'program_id', 'report_yr','measure_yr']\n",
+    ").reset_index()\n",
+    "\n",
+    "print(\"drf.shape:\", drf.shape)\n",
+    "\n",
+    "ftes_checksum = drf['ftes'].sum()\n",
+    "print('ftes_checksum:', ftes_checksum)\n",
+    "spending_checksum = drf['spending'].sum()\n",
+    "print('spending_checksum:', spending_checksum)\n",
+    "print(\"checksum difference:\", drf_checksum - (ftes_checksum+spending_checksum))\n",
+    "print(drf.info())\n",
+    "\n",
+    "# Set up si_link_yr: a fiscal year column to be able to include years \n",
+    "# beyond the service inventory when joining by service id and fy.\n",
+    "# if measure year > latest service fy, = latest service fy, else use measure_yr\n",
+    "drf.loc[drf['measure_yr']>latest_si_fy, 'si_link_yr'] = latest_si_fy\n",
+    "drf.loc[drf['measure_yr']<=latest_si_fy, 'si_link_yr'] = drf['measure_yr']\n",
+    "drf['si_link_yr'] = drf['si_link_yr'].astype(int) \n",
+    "\n",
+    "\n",
+    "drf_files = {\n",
+    "    \"drf_actuals\":drf_actuals,\n",
+    "    \"drf_planned\": drf_planned,\n",
+    "    \"drf\": drf\n",
+    "}\n",
+    "\n",
+    "\n",
+    "#export_to_csv(drf_files, Path.cwd())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4259c781-eaa3-4967-8839-8b78fbca60fe",
+   "metadata": {},
+   "source": [
+    "si_drf = si.loc[:, ['service_id', 'fiscal_yr', 'program_id']]\n",
+    "si_drf = si_drf.explode('program_id')\n",
+    "si_drf['si_yr'] = si_drf['fiscal_yr'].str.split('-').str[1].astype(int)\n",
+    "si_drf = si_drf[si_drf['program_id'].notna()]\n",
+    "\n",
+    "service_fte_spending = pd.merge(\n",
+    "    si_drf, \n",
+    "    drf, \n",
+    "    how='left', \n",
+    "    left_on=['si_yr', 'program_id'], \n",
+    "    right_on=['si_link_yr', 'program_id']\n",
+    ")\n",
+    "\n",
+    "print(service_fte_spending.info())\n",
+    "service_fte_spending\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}