updated drr indicators, program code file download

Author: ge-tbs

README.md

@@ -93,7 +93,7 @@ The [Policy on Service and Digital](https://www.tbs-sct.canada.ca/pol/doc-eng.as
 
 ### Summary Files for Visualization and Review (outputs/indicators/)
 *All tables were built with `service_scope` containing `EXTERN` or `ENTERPRISE`*
-- `drr_all.csv`: a concatenated table with all the drr indicator columns and scores (dr_2467: percentage of high-volume external services (>=45k applications) that are delivered online end-to-end, dr_2468: percentage of high-volume external services (>=45k applications and telephone enquiries) that met at least one service standard, dr_2469: percentage of applications for high-volume external services (>45k applications) that used the online channel)
+- `drr_all.csv`: a concatenated table with all the drr indicator columns and scores (dr_2467: percentage of high-volume external services (>=45k applications) that are delivered online end-to-end, dr_2468: percentage of high-volume external services (>=45k applications and telephone enquiries) that met at least one service standard, dr_2469: percentage of applications for high-volume external services (>45k applications) that used the online channel), service_with_feedback_percentage: percentage of all services that have indicated something other than 'NON' in the client feedback channel field.
 - `maf_all.csv`: a concatenated table with all the maf columns and scores (maf1: percentage of services that have service standards, maf2: percentage of service standards met, maf5: percentage of applicable services that can be completed online end-to-end, maf6: percentage of client interaction points that are available online, maf8: percentage of services which have used client feedback to improve services in the year prior to reporting)
 - `service_fte_spending.csv`: FTEs and spending for programs delivering services.
 - `si_oip.csv`: Online interaction points activation status by service for the latest available fiscal year.

notebooks/experiment-template.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "id": "fd2d413a-666f-49d4-b86c-7d6898a496a5",
    "metadata": {},
    "outputs": [],
@@ -42,28 +42,50 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "config['input_dir']"
+    "# si = pd.read_csv(\"https://github.com/gcperformance/service-data/releases/latest/download/si.csv\", \n",
+    "#                  keep_default_na=False, \n",
+    "#                  na_values='', \n",
+    "#                  delimiter=';'\n",
+    "#                  )\n",
+    "\n",
+    "# ss = pd.read_csv(\"https://github.com/gcperformance/service-data/releases/latest/download/ss.csv\", \n",
+    "#                  keep_default_na=False, \n",
+    "#                  na_values='', \n",
+    "#                  delimiter=';'\n",
+    "#                  )\n",
+    "\n",
+    "\n",
+    "config=get_config()\n",
+    "si_path = parent_dir / 'outputs' / 'si.csv'\n",
+    "si = pd.read_csv(si_path, keep_default_na=False, na_values='', delimiter=';', engine='python', skipfooter=2)"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 10,
    "id": "be959c79",
    "metadata": {},
    "outputs": [],
    "source": [
-    "si = pd.read_csv(\"https://github.com/gcperformance/service-data/releases/latest/download/si.csv\", \n",
-    "                 keep_default_na=False, \n",
-    "                 na_values='', \n",
-    "                 delimiter=';'\n",
-    "                 )\n",
+    "dr_client_feedback_pc = si[['service_id', 'fiscal_yr', 'fy_org_id_service_id', 'client_feedback_channel']].copy()\n",
+    "dr_client_feedback_pc['client_feedback_collected'] = (dr_client_feedback_pc['client_feedback_channel']=='NON')\n",
+    "\n",
+    "# Determine fy-level counts for services that accept client feedback\n",
+    "dr_client_feedback_pc = dr_client_feedback_pc.groupby(['fiscal_yr'], as_index=False).agg(\n",
+    "    service_count=('fy_org_id_service_id', 'nunique'),\n",
+    "    service_with_feedback_count=('client_feedback_collected', 'sum')\n",
+    ")\n",
     "\n",
-    "ss = pd.read_csv(\"https://github.com/gcperformance/service-data/releases/latest/download/ss.csv\", \n",
-    "                 keep_default_na=False, \n",
-    "                 na_values='', \n",
-    "                 delimiter=';'\n",
-    "                 )"
+    "dr_client_feedback_pc['service_with_feedback_percentage'] = dr_client_feedback_pc['service_with_feedback_count']/dr_client_feedback_pc['service_count']\n"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "75409d49",
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {

src/load.py

@@ -43,8 +43,12 @@ def download_program_csv_files(config):
         file_path =  INPUT_DIR / f"{filename}.csv"
 
         try:
+            # Set up a session
+            s = requests.Session()
+
             # Fetch the file from the URL
-            response = requests.get(url)
+            # verify is set to false on advice from PSPC, who acknowledged they will not be fixing the SSL certificate issue
+            response = s.get(url, verify=False)
             response.raise_for_status()  # Raise an error for bad responses (4xx, 5xx)
 
             # Save the content to a file
@@ -67,8 +71,12 @@ def download_program_csv_files(config):
         file_path =  INPUT_DIR / f"{filename}.csv"
 
         try:
+            # Set up a session
+            s = requests.Session()
+
             # Fetch the file from the URL
-            response = requests.get(url)
+            # verify is set to false on advice from PSPC, who acknowledged they will not be fixing the SSL certificate issue
+            response = s.get(url, verify=False)
             response.raise_for_status()  # Raise an error for bad responses (4xx, 5xx)
 
             # Save the content to a file

src/process.py

@@ -566,8 +566,31 @@ def drr(si, ss, config, snapshot=False):
         # Determine score
         dr2469['dr2469_score'] = (dr2469['hv_online_applications']/dr2469['hv_total_applications'])*100
 
+        # =================================
+        # DRR Indicator: Fraction of services that accept client feedback:
+        # Number of entries in the service inventory that are identified as having client
+        # feedback (client_feedback_channel not equal to 'NON') divided by the total number of service inventory entries times 100
+
+        # (# of entries with client feedback / total number of entries) * 100
+        
+        # Select relevant columns from service inventory
+        dr_client_feedback_pc = si[['service_id', 'fiscal_yr', 'fy_org_id_service_id', 'client_feedback_channel']].copy()
+
+        # Identify which services indicated something other than 'NON' in the client feedback channel
+        dr_client_feedback_pc['client_feedback_collected'] = (dr_client_feedback_pc['client_feedback_channel'] != 'NON')
+
+        # Determine fy-level counts for services that accept client feedback
+        dr_client_feedback_pc = dr_client_feedback_pc.groupby(['fiscal_yr'], as_index=False).agg(
+            service_with_feedback_count=('client_feedback_collected', 'sum'),
+            service_count=('fy_org_id_service_id', 'nunique')
+        )
+
+        # Determine score
+        dr_client_feedback_pc['service_with_feedback_percentage'] = (dr_client_feedback_pc['service_with_feedback_count']/dr_client_feedback_pc['service_count'])*100
+
+
         # === SUMMARY DRR TABLE === 
-        drr_dfs = [dr2467, dr2468, dr2469]
+        drr_dfs = [dr2467, dr2468, dr2469, dr_client_feedback_pc]
         index_cols = ['fiscal_yr']
 
         # Set index for each DataFrame in the list
@@ -581,6 +604,7 @@ def drr(si, ss, config, snapshot=False):
             # "dr2467": dr2467,
             # "dr2468": dr2468,
             # "dr2469": dr2469,
+            # "dr_client_feedback_pc": dr_client_feedback_pc,
             "drr_all": drr_all
         }