Merge pull request #196 from datakind/feat/EdviseSchema

Feat: Add Edvise Schema Validation Support with Human-Readable Error Messages

Author: chapmanhk

.gitignore

@@ -114,3 +114,6 @@ dmypy.json
 # terraform
 **/.terraform/*
 **/terraform.tfvars
+
+# Cursor rule files
+.cursor/

pyproject.toml

@@ -86,8 +86,11 @@ lines-after-imports = 1
 [tool.pytest.ini_options]
 minversion = "8.0"
 addopts = ["--verbose", "--import-mode=importlib"]
-filterwarnings = ["ignore::DeprecationWarning"]
-testpaths = ["tests"]
+filterwarnings = [
+    "ignore::DeprecationWarning",
+    "ignore::FutureWarning:pandera",
+]
+testpaths = ["src"]
 
 [tool.mypy]
 files = ["src"]

src/webapp/database.py

@@ -22,6 +22,7 @@
     Integer,
     BigInteger,
     Index,
+    CheckConstraint,
     event,
 )
 from sqlalchemy.orm import (
@@ -273,6 +274,10 @@ class InstTable(Base):
     state: Mapped[str | None] = mapped_column(String(VAR_CHAR_LENGTH), nullable=True)
     # Only populated for PDP schools.
     pdp_id: Mapped[str | None] = mapped_column(String(VAR_CHAR_LENGTH), nullable=True)
+    # Only populated for Edvise schools.
+    edvise_id: Mapped[str | None] = mapped_column(
+        String(VAR_CHAR_LENGTH), nullable=True
+    )
     created_at: Mapped[datetime.datetime] = mapped_column(
         DateTime(timezone=True),
         server_default=func.now(),
@@ -672,9 +677,10 @@ class DocType(enum.Enum):
 class SchemaRegistryTable(Base):
     """
     Stores versioned schema documents:
-      - Base schema (doc_type=base, is_pdp=False, inst_id NULL)
-      - PDP shared extension (doc_type=extension, is_pdp=True, inst_id NULL)
-      - Custom institution extension (doc_type=extension, is_pdp=False, inst_id=<UUID>)
+      - Base schema (doc_type=base, is_pdp=False, is_edvise=False, inst_id NULL)
+      - PDP shared extension (doc_type=extension, is_pdp=True, is_edvise=False, inst_id NULL)
+      - Edvise shared extension (doc_type=extension, is_pdp=False, is_edvise=True, inst_id NULL)
+      - Custom institution extension (doc_type=extension, is_pdp=False, is_edvise=False, inst_id=<UUID>)
     Layers can reference a parent (extends_schema_id) that they extend.
     """
 
@@ -685,11 +691,12 @@ class SchemaRegistryTable(Base):
     doc_type: Mapped[DocType] = mapped_column(
         Enum(DocType, native_enum=False), nullable=False
     )
-    # Nullable: NULL for base and PDP shared extension
+    # Nullable: NULL for base, PDP shared extension, and Edvise shared extension
     inst_id: Mapped[uuid.UUID | None] = mapped_column(
         ForeignKey("inst.id", ondelete="RESTRICT", onupdate="CASCADE"), nullable=True
     )
     is_pdp: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+    is_edvise: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
     version_label: Mapped[str] = mapped_column(
         String(VAR_CHAR_STANDARD_LENGTH), nullable=False
     )
@@ -734,8 +741,12 @@ class SchemaRegistryTable(Base):
         UniqueConstraint("doc_type", "version_label", name="uq_base_version"),
         UniqueConstraint("is_pdp", "version_label", name="uq_pdp_version"),
         UniqueConstraint("inst_id", "version_label", name="uq_inst_version"),
+        CheckConstraint(
+            "NOT (is_pdp = 1 AND is_edvise = 1)", name="ck_no_pdp_and_edvise"
+        ),
         Index("idx_schema_active_base", "doc_type", "is_active"),
         Index("idx_schema_active_pdp", "is_pdp", "is_active"),
+        Index("idx_schema_active_edvise", "is_edvise", "is_active"),
         Index("idx_schema_active_inst", "inst_id", "is_active"),
     )
 
@@ -746,6 +757,8 @@ def namespace(self) -> str:
             return "base"
         if self.is_pdp:
             return "pdp"
+        if self.is_edvise:
+            return "edvise"
         if self.inst_id:
             return f"inst:{self.inst_id}"
         return "unknown"

src/webapp/databricks.py

@@ -16,7 +16,7 @@
 from .validation_extension import generate_extension_schema
 from .config import databricks_vars, gcs_vars
 from .utilities import databricksify_inst_name, SchemaType
-from typing import List, Any, Dict, IO, cast, Optional
+from typing import List, Any, Dict, Optional
 from fastapi import HTTPException
 import requests
 import hashlib
@@ -25,13 +25,7 @@
 from cachetools import TTLCache
 import threading
 import re
-
-try:
-    import tomllib as _toml  # Py 3.11+
-except ModuleNotFoundError:
-    import tomli as _toml  # Py ≤ 3.10
 import pandas as pd
-import re
 
 # Setting up logger
 LOGGER = logging.getLogger(__name__)
@@ -567,7 +561,7 @@ def get_key_for_file(
         Case-insensitive match of file_name against mapping values.
         Values may be:
         - str literal (e.g., "student.csv") → allow optional base suffixes before the ext.
-        - str regex (e.g., r"^course_.*\.csv$") → re.IGNORECASE fullmatch.
+        - str regex (e.g., r"^course_.*\\.csv$") → re.IGNORECASE fullmatch.
         - compiled regex (re.Pattern) → fullmatch, adding IGNORECASE if missing.
         - list of any of the above.
         """

src/webapp/fixtures/validation_error_snapshots/complete_error_flow.txt

@@ -0,0 +1,9 @@
+Missing required columns: 'Student ID' (Student identifier). These columns must be present in your file.
+
+Unexpected columns found: 'Extra Col'. Please remove these columns or rename them to match the expected schema.
+
+Column 'Age' has validation errors:
+  • Row 2: Value validation failed. Current value: found 'None'
+
+Column 'Grade' has validation errors:
+  • Row 1: Value must be one of: A, B, C. Current value: found 'X'

src/webapp/fixtures/validation_error_snapshots/extra_columns_ambiguous.txt

@@ -0,0 +1 @@
+Unexpected columns found: 'Student ID (and 2 similar)'. Please remove these columns or rename them to match the expected schema.

src/webapp/fixtures/validation_error_snapshots/missing_required_columns.txt

@@ -0,0 +1 @@
+Missing required columns: 'Student ID' (Unique student identifier), 'Grade' (Student grade (A-F)), 'Age' (Student age). These columns must be present in your file.

src/webapp/fixtures/validation_error_snapshots/mixed_types_multiple_columns.txt

@@ -0,0 +1,11 @@
+Column 'Age' has validation errors:
+  • Row 2: Value validation failed. Current value: found 'None'
+
+Column 'Grade' has validation errors:
+  • Row 1: Value must be one of: A, B, C, D, F. Current value: found 'X'
+
+Column 'Score' has validation errors:
+  • Row 3: Validation failed for greater_than(0) check. Current value: found '-5'
+
+Column 'Student ID' has validation errors:
+  • Row 1: Value must be at least 3 characters long. Current value: found '****' (value masked for privacy)

src/webapp/fixtures/validation_error_snapshots/multiple_rows_same_column.txt

@@ -0,0 +1,5 @@
+Column 'Grade' has validation errors:
+  • Row 1: Value must be one of: A, B, C, D, F. Current value: found 'X'
+  • Row 2: Value must be one of: A, B, C, D, F. Current value: found 'Y'
+  • Row 3: Value must be one of: A, B, C, D, F. Current value: found 'Z'
+  • Row 4: Value must be one of: A, B, C, D, F. Current value: found 'W'

src/webapp/fixtures/validation_error_snapshots/pii_masking_mixed.txt

@@ -0,0 +1,14 @@
+Column 'Course Name' has validation errors:
+  • Row 1: Value must be at least 3 characters long. Current value: found 'XY'
+
+Column 'Email' has validation errors:
+  • Row 2: Value must be at least 5 characters long. Current value: found 'ca******om' (value masked for privacy)
+
+Column 'Grade' has validation errors:
+  • Row 2: Value must be one of: A, B, C. Current value: found 'X'
+
+Column 'SSN' has validation errors:
+  • Row 3: Value must be at least 9 characters long. Current value: found 'CA******89' (value masked for privacy)
+
+Column 'Student Name' has validation errors:
+  • Row 1: Value must be at least 3 characters long. Current value: found 'CA******23' (value masked for privacy)