NRL-1268 auto trigger crawler

Author: jackleary

terraform/account-wide-infrastructure/modules/glue/src/main.py

@@ -19,6 +19,7 @@
     source_path=args["source_path"],
     target_path=args["target_path"],
     schema=logSchema,
+    job_name=args["job_name"],
     partition_cols=partition_cols,
     transformations=[flatten_df, dtype_conversion],
 )

terraform/account-wide-infrastructure/modules/glue/src/pipeline.py

@@ -1,3 +1,4 @@
+import boto3
 from instances import GlueContextSingleton, LoggerSingleton
 
 
@@ -8,6 +9,7 @@ def __init__(
         source_path,
         target_path,
         schema,
+        job_name,
         partition_cols=[],
         transformations=[],
     ):
@@ -20,6 +22,12 @@ def __init__(
         self.schema = schema
         self.partition_cols = partition_cols
         self.transformations = transformations
+        self.glue = boto3.client(
+            service_name="glue",
+            region_name="eu-west-2",
+            endpoint_url="your-endpoint-url",
+        )
+        self.name_prefix = "-".join(job_name.split("-")[:3])
 
     def run(self):
         """Runs ETL"""
@@ -31,6 +39,8 @@ def run(self):
             self.logger.info("Data transformed successfully.")
             self.load(df)
             self.logger.info(f"Data loaded into {self.target_path}.")
+            self.logger.info("Trigger glue crawler")
+            self.trigger_crawler()
         except Exception as e:
             self.logger.error(f"ETL process failed: {e}")
             raise e
@@ -57,3 +67,9 @@ def load(self, dataframe):
         dataframe.write.mode("append").partitionBy(*self.partition_cols).parquet(
             self.target_path
         )
+
+    def trigger_crawler(self):
+        try:
+            self.glue.start_crawler(Name=f"{self.name_prefix}-log-crawler")
+        except Exception as e:
+            raise e