Add Sentry integration (#163)

Author: di

main.py

@@ -1,6 +1,5 @@
 import cattr
 
-import base64
 import datetime
 import os
 import json
@@ -11,10 +10,15 @@
 
 from linehaul.events.parser import parse, Download, Simple
 
+import sentry_sdk
+from sentry_sdk.integrations.serverless import serverless_function
 from google.api_core import exceptions
 from google.api_core.retry import Retry
 from google.cloud import bigquery, storage, pubsub_v1
 
+if dsn := os.environ.get("SENTRY_DSN"):
+    sentry_sdk.init(dsn=dsn, enable_tracing=True)
+
 _cattr = cattr.Converter()
 _cattr.register_unstructure_hook(
     datetime.datetime, lambda o: o.strftime("%Y-%m-%d %H:%M:%S +00:00")
@@ -39,6 +43,7 @@
 prefix = {Simple.__name__: "simple_requests", Download.__name__: "file_downloads"}
 
 
+@serverless_function
 def process_fastly_log(data, context):
     storage_client = storage.Client()
     file_name = os.path.basename(data["name"]).rstrip(".log.gz")
@@ -162,6 +167,7 @@ def _fetch_blobs(bucket, blob_type="downloads", past_partition=None, partition=N
     return (source_blobs, prefix)
 
 
+@serverless_function
 def load_processed_files_into_bigquery(event, context):
     continue_publishing = False
     if "attributes" in event and "partition" in event["attributes"]:

requirements.in

@@ -4,3 +4,4 @@ pyparsing
 google-cloud-storage
 google-cloud-bigquery
 google-cloud-pubsub
+sentry-sdk

requirements.txt

@@ -19,7 +19,9 @@ cattrs==23.2.3 \
 certifi==2022.12.7 \
     --hash=sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3 \
     --hash=sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18
-    # via requests
+    # via
+    #   requests
+    #   sentry-sdk
 charset-normalizer==3.1.0 \
     --hash=sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6 \
     --hash=sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1 \
@@ -97,12 +99,6 @@ charset-normalizer==3.1.0 \
     --hash=sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df \
     --hash=sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab
     # via requests
-# This requirement duplicates the line below, without an extra. We need this to
-# work around https://github.com/pypa/pip/issues/9644 because we can't use
-# --no-deps when this gets installed by GCF
-google-api-core==2.17.1 \
-    --hash=sha256:610c5b90092c360736baccf17bd3efbcb30dd380e7a6dc28a71059edb8bd0d8e \
-    --hash=sha256:9df18a1f87ee0df0bc4eea2770ebc4228392d8cc4066655b320e2cfccb15db95
 google-api-core[grpc]==2.17.1 \
     --hash=sha256:610c5b90092c360736baccf17bd3efbcb30dd380e7a6dc28a71059edb8bd0d8e \
     --hash=sha256:9df18a1f87ee0df0bc4eea2770ebc4228392d8cc4066655b320e2cfccb15db95
@@ -215,12 +211,6 @@ google-resumable-media==2.7.0 \
     # via
     #   google-cloud-bigquery
     #   google-cloud-storage
-# This requirement duplicates the line below, without an extra. We need this to
-# work around https://github.com/pypa/pip/issues/9644 because we can't use
-# --no-deps when this gets installed by GCF
-googleapis-common-protos==1.59.0 \
-    --hash=sha256:4168fcb568a826a52f23510412da405abd93f4d23ba544bb68d943b14ba3cb44 \
-    --hash=sha256:b287dc48449d1d41af0c69f4ea26242b5ae4c3d7249a38b0984c86a4caffff1f
 googleapis-common-protos[grpc]==1.59.0 \
     --hash=sha256:4168fcb568a826a52f23510412da405abd93f4d23ba544bb68d943b14ba3cb44 \
     --hash=sha256:b287dc48449d1d41af0c69f4ea26242b5ae4c3d7249a38b0984c86a4caffff1f
@@ -354,11 +344,17 @@ rsa==4.9 \
     --hash=sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7 \
     --hash=sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21
     # via google-auth
+sentry-sdk==1.43.0 \
+    --hash=sha256:41df73af89d22921d8733714fb0fc5586c3461907e06688e6537d01a27e0e0f6 \
+    --hash=sha256:8d768724839ca18d7b4c7463ef7528c40b7aa2bfbf7fe554d5f9a7c044acfd36
+    # via -r requirements.in
 six==1.16.0 \
     --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \
     --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254
     # via python-dateutil
 urllib3==1.26.15 \
     --hash=sha256:8a388717b9476f934a21484e8c8e61875ab60644d29b9b39e11e4b9dc1c6b305 \
     --hash=sha256:aa751d169e23c7479ce47a0cb0da579e3ede798f994f5816a74e4f4500dcea42
-    # via requests
+    # via
+    #   requests
+    #   sentry-sdk