handling invalid matches

Author: ElJocho

mapswipe_workers/mapswipe_workers/firebase_to_postgres/transfer_results.py

@@ -54,7 +54,7 @@ def transfer_results(project_id_list=None):
     return project_id_list_transfered
 
 
-def transfer_results_for_project(project_id, results):
+def transfer_results_for_project(project_id, results, filter_mode: bool = False):
     """Transfer the results for a specific project.
     Save results into an in-memory file.
     Copy the results to postgres.
@@ -90,18 +90,20 @@ def transfer_results_for_project(project_id, results):
         # results at relatively high speed.
         results_file = results_to_file(results, project_id)
         truncate_temp_results()
-        save_results_to_postgres(results_file)
+        save_results_to_postgres(results_file, project_id, filter_mode=filter_mode)
     except psycopg2.errors.ForeignKeyViolation as e:
         sentry.capture_exception(e)
         sentry.capture_message(
             "could not transfer results to postgres due to ForeignKeyViolation: "
-            f"{project_id}"
+            f"{project_id}; filter_mode={filter_mode}"
         )
         logger.exception(e)
         logger.warning(
             "could not transfer results to postgres due to ForeignKeyViolation: "
-            f"{project_id}"
+            f"{project_id}; filter_mode={filter_mode}"
         )
+        if not filter_mode:
+            transfer_results_for_project(project_id, results, filter_mode=True)
     except Exception as e:
         sentry.capture_exception(e)
         sentry.capture_message(f"could not transfer results to postgres: {project_id}")
@@ -259,14 +261,16 @@ def results_to_file(results, projectId):
     return results_file
 
 
-def save_results_to_postgres(results_file):
+def save_results_to_postgres(results_file, project_id, filter_mode: bool):
     """
     Saves results to a temporary table in postgres
     using the COPY Statement of Postgres
     for a more efficient import into the database.
     Parameters
     ----------
     results_file: io.StringIO
+    filter_mode: boolean
+        If true, try to filter out invalid results.
     """
 
     p_con = auth.postgresDB()
@@ -283,6 +287,23 @@ def save_results_to_postgres(results_file):
     p_con.copy_from(results_file, "results_temp", columns)
     results_file.close()
 
+    if filter_mode:
+        logger.warn(f"trying to remove invalid tasks from {project_id}.")
+
+        filter_query = f"""
+            DELETE FROM results_temp
+            where task_id in (
+                select task_id from results_temp where task_id not in (
+                    select r.task_id from results_temp r join (
+                        select * from tasks where project_id = '{project_id}'
+                    ) as t
+                    on r.group_id = t.group_id
+                    and r.task_id = t.task_id
+                )
+            )
+        """
+        p_con.query(filter_query)
+
     query_insert_results = """
         INSERT INTO results
             SELECT * FROM results_temp

mapswipe_workers/tests/integration/test_transfer_results_invalid_results.py

@@ -46,7 +46,7 @@ def test_invalid_task_in_result(self):
         ref = fb_db.reference(
             f"v2/results/{self.project_id}/g115/test_build_area/results"
         )
-        self.assertEqual(len(ref.get(shallow=True)), 253)
+        self.assertIsNone(ref.get(shallow=True))
 
 
 if __name__ == "__main__":