Skip to content

Commit b610dc5

Browse files
committed
GCSDeleteObjectsOperator: Add flag to ignore errors on blob deletion
Makes the `GCSHook.delete` `ignore_error` boolean available to ignore errors when a blob doesn't exist.
1 parent 82ab7f5 commit b610dc5

File tree

2 files changed

+12
-9
lines changed
  • providers/google
    • src/airflow/providers/google/cloud/operators
    • tests/unit/google/cloud/operators

2 files changed

+12
-9
lines changed

providers/google/src/airflow/providers/google/cloud/operators/gcs.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -279,6 +279,7 @@ class GCSDeleteObjectsOperator(GoogleCloudBaseOperator):
279279
of objects in the bucket, not including gs://bucket/
280280
:param prefix: String or list of strings, which filter objects whose name begin with
281281
it/them. (templated)
282+
:param ignore_error: (Optional) whether to ignore NotFound exceptions. Default: False
282283
:param gcp_conn_id: (Optional) The connection ID used to connect to Google Cloud.
283284
:param impersonation_chain: Optional service account to impersonate using short-term
284285
credentials, or chained list of accounts required to get the access_token
@@ -304,13 +305,15 @@ def __init__(
304305
bucket_name: str,
305306
objects: list[str] | None = None,
306307
prefix: str | list[str] | None = None,
308+
ignore_error: bool = False,
307309
gcp_conn_id: str = "google_cloud_default",
308310
impersonation_chain: str | Sequence[str] | None = None,
309311
**kwargs,
310312
) -> None:
311313
self.bucket_name = bucket_name
312314
self.objects = objects
313315
self.prefix = prefix
316+
self.ignore_error = ignore_error
314317
self.gcp_conn_id = gcp_conn_id
315318
self.impersonation_chain = impersonation_chain
316319

@@ -337,7 +340,7 @@ def execute(self, context: Context) -> None:
337340
objects = hook.list(bucket_name=self.bucket_name, prefix=self.prefix)
338341
self.log.info("Deleting %s objects from %s", len(objects), self.bucket_name)
339342
for object_name in objects:
340-
hook.delete(bucket_name=self.bucket_name, object_name=object_name)
343+
hook.delete(bucket_name=self.bucket_name, object_name=object_name, ignore_error=self.ignore_error)
341344

342345
def get_openlineage_facets_on_start(self):
343346
from airflow.providers.common.compat.openlineage.facet import (

providers/google/tests/unit/google/cloud/operators/test_gcs.py

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -126,8 +126,8 @@ def test_delete_objects(self, mock_hook):
126126
mock_hook.return_value.list.assert_not_called()
127127
mock_hook.return_value.delete.assert_has_calls(
128128
calls=[
129-
mock.call(bucket_name=TEST_BUCKET, object_name=MOCK_FILES[0]),
130-
mock.call(bucket_name=TEST_BUCKET, object_name=MOCK_FILES[1]),
129+
mock.call(bucket_name=TEST_BUCKET, object_name=MOCK_FILES[0], ignore_error=False),
130+
mock.call(bucket_name=TEST_BUCKET, object_name=MOCK_FILES[1], ignore_error=False),
131131
],
132132
any_order=True,
133133
)
@@ -149,8 +149,8 @@ def test_delete_prefix(self, mock_hook):
149149
mock_hook.return_value.list.assert_called_once_with(bucket_name=TEST_BUCKET, prefix=PREFIX)
150150
mock_hook.return_value.delete.assert_has_calls(
151151
calls=[
152-
mock.call(bucket_name=TEST_BUCKET, object_name=MOCK_FILES[1]),
153-
mock.call(bucket_name=TEST_BUCKET, object_name=MOCK_FILES[2]),
152+
mock.call(bucket_name=TEST_BUCKET, object_name=MOCK_FILES[1], ignore_error=False),
153+
mock.call(bucket_name=TEST_BUCKET, object_name=MOCK_FILES[2], ignore_error=False),
154154
],
155155
any_order=True,
156156
)
@@ -164,10 +164,10 @@ def test_delete_prefix_as_empty_string(self, mock_hook):
164164
mock_hook.return_value.list.assert_called_once_with(bucket_name=TEST_BUCKET, prefix="")
165165
mock_hook.return_value.delete.assert_has_calls(
166166
calls=[
167-
mock.call(bucket_name=TEST_BUCKET, object_name=MOCK_FILES[0]),
168-
mock.call(bucket_name=TEST_BUCKET, object_name=MOCK_FILES[1]),
169-
mock.call(bucket_name=TEST_BUCKET, object_name=MOCK_FILES[2]),
170-
mock.call(bucket_name=TEST_BUCKET, object_name=MOCK_FILES[3]),
167+
mock.call(bucket_name=TEST_BUCKET, object_name=MOCK_FILES[0], ignore_error=False),
168+
mock.call(bucket_name=TEST_BUCKET, object_name=MOCK_FILES[1], ignore_error=False),
169+
mock.call(bucket_name=TEST_BUCKET, object_name=MOCK_FILES[2], ignore_error=False),
170+
mock.call(bucket_name=TEST_BUCKET, object_name=MOCK_FILES[3], ignore_error=False),
171171
],
172172
any_order=True,
173173
)

0 commit comments

Comments
 (0)