feat(firestore-bigquery-export): ignore document deletion

Author: mmiro1

firestore-bigquery-export/extension.yaml

@@ -647,6 +647,21 @@ params:
     default: info
     required: true
 
+  - param: IGNORE_DOCUMENT_DELETION
+    label: Ignore Document Deletion
+    description: >-
+      If enabled, Firestore deleted document will not be exported (the Firestore
+      onDocumentDeleted event will be ignored). The reduction in data should be 
+      more performant, and avoid potential resource limitations.
+    type: select
+    required: false
+    default: no
+    options:
+      - label: Yes
+        value: yes
+      - label: No
+        value: no
+
 events:
   # OLD event types for backward compatibility
   - type: firebase.extensions.firestore-counter.v1.onStart

firestore-bigquery-export/functions/src/config.ts

@@ -77,4 +77,5 @@ export default {
     process.env.BACKUP_GCS_BUCKET || `${process.env.PROJECT_ID}.appspot.com`,
   backupDir: `_${process.env.INSTANCE_ID || "firestore-bigquery-export"}`,
   logLevel: process.env.LOG_LEVEL || LogLevel.INFO,
+  ignoreDocumentDeletion: process.env.IGNORE_DOCUMENT_DELETION === "yes" ? true : false,
 };

firestore-bigquery-export/functions/src/index.ts

@@ -173,7 +173,16 @@ export const fsexportbigquery = onDocumentWritten(
     const fullResourceName = `projects/${projectId}/databases/${config.databaseId}/documents/${relativeName}`;
     const eventId = context.id;
     const operation = changeType;
-
+    
+    if (config.ignoreDocumentDeletion && isDeleted) {
+      logs.logEventAction(
+        "Firestore event received and ignored by onDocumentWritten trigger",
+        fullResourceName,
+        eventId,
+        operation
+      );
+      return;
+    }
     logs.logEventAction(
       "Firestore event received by onDocumentWritten trigger",
       fullResourceName,