Add support for non-billed service unit types

Support non-billed SU types from pi.yaml
Apply credit code 0005 and zero balances for matching rows
Update tests for the new YAML behavior

Author: jimmysway

process_report/loader.py

@@ -112,9 +112,42 @@ def get_alias_map(self) -> dict:
     def load_dataframe(self, filepath: str) -> pandas.DataFrame:
         return pandas.read_csv(filepath)
 
+    @functools.lru_cache
+    def _load_pi_config(self, filepath: str) -> tuple[list[str], dict[str, list[str]]]:
+        with open(filepath) as file:
+            parsed = yaml.safe_load(file) or []
+
+        if not isinstance(parsed, list):
+            raise ValueError("pi.yaml must contain a YAML list")
+
+        nonbillable_pis = []
+        pi_non_billed_su_types = {}
+        for entry in parsed:
+            if not isinstance(entry, dict):
+                continue
+
+            username = entry.get("username")
+            if not username:
+                continue
+
+            su_types = entry.get("non_billed_su_types")
+            if su_types:
+                pi_non_billed_su_types[username] = [
+                    str(item.get("name")).strip()
+                    for item in su_types
+                    if isinstance(item, dict) and item.get("name")
+                ]
+            else:
+                nonbillable_pis.append(username)
+
+        return nonbillable_pis, pi_non_billed_su_types
+
     def get_nonbillable_pis(self) -> list[str]:
-        with open(invoice_settings.nonbillable_pis_filepath) as file:
-            return [line.rstrip() for line in file]
+        return self._load_pi_config(invoice_settings.nonbillable_pis_filepath)[0]
+
+    def get_pi_non_billed_su_types(self) -> dict[str, list[str]]:
+        """PI usernames -> list of SU types that receive credit (zeroed out)."""
+        return self._load_pi_config(invoice_settings.nonbillable_pis_filepath)[1]
 
     @functools.lru_cache
     def get_nonbillable_projects(self) -> pandas.DataFrame:

process_report/processors/validate_billable_pi_processor.py

@@ -13,6 +13,7 @@
 
 
 NONBILLABLE_CLUSTERS = ["ocp-test", "barcelona"]
+_CREDIT_CODE_NON_BILLED_SU = "0005"
 
 
 def find_billable_projects(
@@ -131,8 +132,35 @@ def _get_billables(
 
         return pi_mask & project_mask & courses_mask
 
+    def _apply_non_billed_su_type_credits(self):
+        """
+        For PIs with non-billed-su-types in pi.yaml: Credit = Cost, Credit code 0005,
+        zero PI Balance and Balance for matching (email, SU type) rows.
+        """
+        pi_non_billed_su_types = loader.get_pi_non_billed_su_types()
+        if not pi_non_billed_su_types:
+            return
+
+        email = self.data[invoice.INVOICE_EMAIL_FIELD].fillna("").str.strip()
+        su_type = self.data[invoice.SU_TYPE_FIELD].fillna("").str.strip()
+
+        for username, su_types in pi_non_billed_su_types.items():
+            if not su_types:
+                continue
+            rule_mask = (email == username.strip()) & (su_type.isin(su_types))
+
+            self.data.loc[rule_mask, invoice.CREDIT_CODE_FIELD] = (
+                _CREDIT_CODE_NON_BILLED_SU
+            )
+            self.data.loc[rule_mask, invoice.CREDIT_FIELD] = self.data.loc[
+                rule_mask, invoice.COST_FIELD
+            ]
+            self.data.loc[rule_mask, invoice.PI_BALANCE_FIELD] = 0
+            self.data.loc[rule_mask, invoice.BALANCE_FIELD] = 0
+
     def _process(self):
         self.data[invoice.IS_BILLABLE_FIELD] = self._get_billables(
             self.data, self.nonbillable_pis, self.nonbillable_projects
         )
         self.data[invoice.MISSING_PI_FIELD] = self._validate_pi_names(self.data)
+        self._apply_non_billed_su_type_credits()

process_report/tests/e2e/README.md

@@ -41,7 +41,7 @@ The `test_data/` directory contains the minimal dataset required by the pipeline
 
 - **Invoice Files**: `test_nerc-ocp-test 2025-04.csv`, `test_NERC OpenShift 2025-04.csv`
 - **Configuration Files**:
-  - PI lists: `test_pi.txt`
+  - PI lists: `test_pi.yaml`
   - Project lists: `test_projects.txt`, `test_timed_projects.txt`
   - Prepay configurations: `test_prepay_debits.csv`, `test_prepay_credits.csv`, `test_prepay_projects.csv`, `test_prepay_contacts.csv`
 - **Historical Data**: `test_PI.csv` (old PI file), `test_alias.csv`

process_report/tests/e2e/test_data/test_pi.txt

@@ -0,0 +1 @@
+- username: PI9

process_report/tests/e2e/test_data/test_pi.yaml

@@ -124,7 +124,7 @@ def _prepare_pipeline_execution(
     env["PREPAY_CREDITS_FILEPATH"] = str(test_files["test_prepay_credits.csv"])
     env["PREPAY_PROJECTS_FILEPATH"] = str(test_files["test_prepay_projects.csv"])
     env["PREPAY_CONTACTS_FILEPATH"] = str(test_files["test_prepay_contacts.csv"])
-    env["nonbillable_pis_filepath"] = str(test_files["test_pi.txt"])
+    env["nonbillable_pis_filepath"] = str(test_files["test_pi.yaml"])
     env["nonbillable_projects_filepath"] = str(test_files["test_projects.yaml"])
 
     # Fallback ensures test works even when CI environment doesn't set Chrome path

process_report/tests/e2e/test_e2e_pipeline.py

@@ -1,12 +1,22 @@
 from unittest import TestCase
+from unittest.mock import patch
 import pandas
 import uuid
 import math
 
+from process_report.invoices import invoice
 from process_report.tests import util as test_utils
 
 
 class TestValidateBillablePIProcessor(TestCase):
+    def setUp(self):
+        self.get_pi_non_billed_su_types_patcher = patch(
+            "process_report.processors.validate_billable_pi_processor.loader.get_pi_non_billed_su_types",
+            return_value={},
+        )
+        self.get_pi_non_billed_su_types_patcher.start()
+        self.addCleanup(self.get_pi_non_billed_su_types_patcher.stop)
+
     def test_remove_nonbillables(self):
         pis = [uuid.uuid4().hex for _ in range(10)]
         projects = [
@@ -149,3 +159,36 @@ def test_is_course_marks_nonbillable(self):
         expected_billable = [False, True, True, True]
         actual_billable = output["Is Billable"].tolist()
         assert actual_billable == expected_billable
+
+    def test_applies_credit_code_0005_for_matching_non_billed_su_type(self):
+        test_invoice = pandas.DataFrame(
+            {
+                invoice.PROJECT_FIELD: ["p1"],
+                invoice.INVOICE_EMAIL_FIELD: ["emre_keskin@harvard.edu"],
+                invoice.SU_TYPE_FIELD: ["Openstack Storage"],
+                invoice.COST_FIELD: [100.0],
+                invoice.CREDIT_CODE_FIELD: ["0001"],
+                invoice.CREDIT_FIELD: [10.0],
+                invoice.PI_BALANCE_FIELD: [90.0],
+                invoice.BALANCE_FIELD: [80.0],
+                invoice.PI_FIELD: ["emre"],
+                "Cluster Name": ["c1"],
+                "Is Course": [False],
+                "Institution": ["Harvard"],
+            }
+        )
+
+        with patch(
+            "process_report.processors.validate_billable_pi_processor.loader.get_pi_non_billed_su_types",
+            return_value={"emre_keskin@harvard.edu": ["Openstack Storage"]},
+        ):
+            proc = test_utils.new_validate_billable_pi_processor(data=test_invoice)
+            proc.process()
+
+        assert proc.data.loc[0, invoice.CREDIT_CODE_FIELD] == "0005"
+        assert (
+            proc.data.loc[0, invoice.CREDIT_FIELD]
+            == proc.data.loc[0, invoice.COST_FIELD]
+        )
+        assert proc.data.loc[0, invoice.PI_BALANCE_FIELD] == 0
+        assert proc.data.loc[0, invoice.BALANCE_FIELD] == 0