Amend GX test for people data load column ordering; add error handling for main apply script

Author: annajgibson

scripts/jobs/housing/housing_nec_migration_apply_gx_dq_tests.py

@@ -133,9 +133,17 @@ def main():
 
             table_results_df["unexpected_id_list"] = pd.Series(dtype="object")
             for i, row in query_df.iterrows():
-                table_results_df.loc[i, "unexpected_id_list"] = str(
+                try:
                     list(df[id_field].iloc[row["result.unexpected_index_list"]])
-                )
+                except Exception as e:
+                    print(
+                        f"Problem found with {table}: {e}, skipping making unexpected_id_list."
+                    )
+                    continue
+                else:
+                    table_results_df.loc[i, "unexpected_id_list"] = str(
+                        list(df[id_field].iloc[row["result.unexpected_index_list"]])
+                    )
 
             # drop columns not needed in metatdata
             cols_to_drop_meta = [

scripts/jobs/housing/housing_nec_migration_people_data_load_gx_suite.py

@@ -41,35 +41,35 @@ class ExpectTitleToBeInSet(gxe.ExpectColumnValuesToBeInSet):
 
 class ExpectPeopleColumnsToMatchOrderedList(gxe.ExpectTableColumnsToMatchOrderedList):
     column_list = [
-        "LPAR_TIN_HRV_TIR_CODE",
-        "LPAR_TIN_STAT_SUCCESSOR_IND",
-        "LPAR_TIN_START_DATE",
-        "LPAR_TIN_MAIN_TENANT_IND",
-        "LPAR_TIN_END_DATE",
-        "LPAR_TCY_IND",
+        "LPAR_HOP_START_DATE",
+        "LPAR_PER_SURNAME",
         "LPAR_TCY_ALT_REF",
-        "LPAR_PHONE",
+        "LPAR_PER_FORENAME",
+        "LPAR_HOP_HPSR_CODE",
         "LPAR_PER_TITLE",
-        "LPAR_PER_SURNAME",
-        "LPAR_PER_OTHER_NAME",
-        "LPAR_PER_NI_NO",
         "LPAR_PER_INITIALS",
-        "LPAR_PER_HOU_OAP_IND",
-        "LPAR_PER_HOU_HRV_HMS_CODE",
-        "LPAR_PER_HOU_EMPLOYER",
+        "LPAR_PER_DATE_OF_BIRTH",
         "LPAR_PER_HOU_DISABLED_IND",
-        "LPAR_PER_FRV_HGO_CODE",
-        "LPAR_PER_FRV_FNL_CODE",
+        "LPAR_PER_HOU_OAP_IND",
         "LPAR_PER_FRV_FGE_CODE",
-        "LPAR_PER_FRV_FEO_CODE",
-        "LPAR_PER_FORENAME",
-        "LPAR_PER_DATE_OF_BIRTH",
-        "LPAR_PER_ALT_REF",
-        "LPAR_HOP_START_DATE",
         "LPAR_HOP_HRV_REL_CODE",
-        "LPAR_HOP_HPSR_CODE",
-        "LPAR_HOP_HPER_CODE",
+        "LPAR_PER_HOU_EMPLOYER",
+        "LPAR_PER_HOU_HRV_HMS_CODE",
+        "LPAR_PHONE",
         "LPAR_HOP_END_DATE",
+        "LPAR_HOP_HPER_CODE",
+        "LPAR_TCY_IND",
+        "LPAR_TIN_MAIN_TENANT_IND",
+        "LPAR_TIN_START_DATE",
+        "LPAR_TIN_END_DATE",
+        "PAR_TIN_HRV_TIR_CODE",
+        "LPAR_TIN_STAT_SUCCESSOR_IND",
+        "LPAR_PER_ALT_REF",
+        "LPAR_PER_FRV_FEO_CODE",
+        "LPAR_PER_NI_NO",
+        "LPAR_PER_FRV_HGO_CODE",
+        "LPAR_PER_FRV_FNL_CODE",
+        "LPAR_PER_OTHER_NAME",
     ]
     description: str = "Expect people load columns to match ordered list exactly"