Skip to content

Commit c2df299

Browse files
devin-ai-integration[bot]agarctfioctavia-squidington-iii
authored
fix(source-twilio): Handle 404 errors gracefully for date ranges with no data (#68680)
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Alfredo Garcia <[email protected]> Co-authored-by: [email protected] <[email protected]> Co-authored-by: Octavia Squidington III <[email protected]>
1 parent 28e92e1 commit c2df299

File tree

8 files changed

+527
-337
lines changed

8 files changed

+527
-337
lines changed

‎airbyte-integrations/connectors/source-twilio/manifest.yaml‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,12 @@ definitions:
2727
- 429
2828
error_message: >-
2929
Twilio rate limit reached (429). Backoff based on 'retry-after' header, then exponential backoff fallback.
30-
30+
- type: HttpResponseFilter
31+
action: IGNORE
32+
http_codes:
33+
- 404
34+
error_message: >-
35+
Skipping this slice—data may be available in later slices or for other accounts/subaccounts.
3136
base_stream:
3237
type: DeclarativeStream
3338
primary_key:

‎airbyte-integrations/connectors/source-twilio/metadata.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ data:
1313
connectorSubtype: api
1414
connectorType: source
1515
definitionId: b9dc6155-672e-42ea-b10d-9f1f1fb95ab1
16-
dockerImageTag: 0.17.2
16+
dockerImageTag: 0.17.3
1717
dockerRepository: airbyte/source-twilio
1818
documentationUrl: https://docs.airbyte.com/integrations/sources/twilio
1919
githubIssueLabel: source-twilio

‎airbyte-integrations/connectors/source-twilio/unit_tests/conftest.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
import sys
44
from pathlib import Path
55

6+
import pytest
7+
68
from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
79
from airbyte_cdk.test.catalog_builder import CatalogBuilder
810
from airbyte_cdk.test.state_builder import StateBuilder
@@ -17,6 +19,11 @@
1719
}
1820

1921

22+
@pytest.fixture
23+
def http_mocker() -> None:
24+
"""This fixture is needed to pass http_mocker parameter from the @HttpMocker decorator to a test"""
25+
26+
2027
def _get_manifest_path() -> Path:
2128
source_declarative_manifest_path = Path("/airbyte/integration_code/source_declarative_manifest")
2229
if source_declarative_manifest_path.exists():

‎airbyte-integrations/connectors/source-twilio/unit_tests/poetry.lock‎

Lines changed: 291 additions & 319 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎airbyte-integrations/connectors/source-twilio/unit_tests/pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ description = "Unit tests for source-twilio"
88
authors = ["Airbyte <[email protected]>"]
99
[tool.poetry.dependencies]
1010
python = "^3.10,<3.13"
11-
airbyte-cdk = "^6"
11+
airbyte-cdk = "^7"
1212
pytest-mock = "^3.12.0"
1313
freezegun = "^1.4.0"
1414
requests-mock = "^1.9.3"

‎airbyte-integrations/connectors/source-twilio/unit_tests/test_source.py‎

Lines changed: 17 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2,42 +2,44 @@
22
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
33
#
44

5-
from unittest.mock import Mock
5+
import logging
66

77
import pytest
88
import requests
99
from conftest import TEST_CONFIG, get_source
1010

11+
from airbyte_cdk.models import Status
1112
from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException
1213

1314

14-
TEST_INSTANCE = get_source(TEST_CONFIG)
15-
16-
1715
@pytest.mark.parametrize(
18-
"exception, expected_error_msg",
16+
"exception, expected_error_fragment",
1917
(
2018
(
2119
ConnectionError("Connection aborted"),
22-
"Encountered an error while checking availability of stream accounts. Error: Connection aborted",
20+
"Connection aborted",
2321
),
2422
(
2523
TimeoutError("Socket timed out"),
26-
"Encountered an error while checking availability of stream accounts. Error: Socket timed out",
24+
"Socket timed out",
2725
),
2826
(
2927
DefaultBackoffException(
3028
None, None, "Unexpected exception in error handler: 401 Client Error: Unauthorized for url: https://api.twilio.com/"
3129
),
32-
"Encountered an error while checking availability of stream accounts. "
33-
"Error: DefaultBackoffException: Unexpected exception in error handler: 401 Client Error: Unauthorized for url: https://api.twilio.com/",
30+
"401 Client Error: Unauthorized",
3431
),
3532
),
3633
)
37-
def test_check_connection_handles_exceptions(mocker, exception, expected_error_msg):
34+
def test_check_connection_handles_exceptions(mocker, exception, expected_error_fragment):
35+
"""Test that check connection properly handles network-level exceptions."""
3836
mocker.patch("time.sleep")
39-
mocker.patch.object(requests.Session, "send", Mock(side_effect=exception))
40-
logger_mock = Mock()
41-
status_ok, error = TEST_INSTANCE.check_connection(logger=logger_mock, config=TEST_CONFIG)
42-
assert not status_ok
43-
assert error == expected_error_msg
37+
mocker.patch.object(requests.Session, "send", side_effect=exception)
38+
39+
source = get_source(TEST_CONFIG)
40+
logger = logging.getLogger("airbyte")
41+
42+
connection_status = source.check(logger=logger, config=TEST_CONFIG)
43+
44+
assert connection_status.status == Status.FAILED
45+
assert expected_error_fragment in str(connection_status.message)

‎airbyte-integrations/connectors/source-twilio/unit_tests/test_usage_records_404_handling.py‎

Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
#
2+
# Copyright (c) 2025 Airbyte, Inc., all rights reserved.
3+
#
4+
5+
import json
6+
7+
from conftest import TEST_CONFIG, get_source
8+
from freezegun import freeze_time
9+
10+
from airbyte_cdk.models import SyncMode
11+
from airbyte_cdk.test.catalog_builder import CatalogBuilder
12+
from airbyte_cdk.test.entrypoint_wrapper import read
13+
from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse
14+
from airbyte_cdk.test.state_builder import StateBuilder
15+
16+
17+
BASE = "https://api.twilio.com/2010-04-01"
18+
19+
ACCOUNTS_JSON = {
20+
"accounts": [
21+
{
22+
"sid": "AC123",
23+
"date_created": "2022-01-01T00:00:00Z",
24+
"subresource_uris": {
25+
"usage": "/2010-04-01/Accounts/AC123/Usage.json",
26+
},
27+
}
28+
],
29+
}
30+
31+
32+
class TestUsageRecords404Handling:
33+
"""Test that usage_records stream handles 404 errors gracefully."""
34+
35+
@HttpMocker()
36+
@freeze_time("2022-11-16 12:03:11+00:00")
37+
def test_usage_records_ignores_404_responses(self, http_mocker: HttpMocker, caplog):
38+
"""Test that the sync ignores 404 responses and logs the appropriate message."""
39+
http_mocker.get(
40+
HttpRequest(url=f"{BASE}/Accounts.json", query_params={"PageSize": "1000"}),
41+
HttpResponse(body=json.dumps(ACCOUNTS_JSON), status_code=200),
42+
)
43+
44+
http_mocker.get(
45+
HttpRequest(
46+
url=f"{BASE}/Accounts/AC123/Usage/Records/Daily.json",
47+
query_params={"PageSize": "1000", "StartDate": "2022-11-15", "EndDate": "2022-11-16"},
48+
),
49+
HttpResponse(
50+
body=json.dumps({"code": 20404, "message": "The requested resource was not found"}),
51+
status_code=404,
52+
),
53+
)
54+
55+
catalog = CatalogBuilder().with_stream("usage_records", SyncMode.full_refresh).build()
56+
config = {**TEST_CONFIG, "start_date": "2022-11-15T00:00:00Z"}
57+
58+
output = read(get_source(config), config, catalog)
59+
60+
assert len(output.records) == 0, "Expected no records when 404 is returned"
61+
62+
expected_message = "Skipping this slice"
63+
log_messages = [record.message for record in caplog.records]
64+
assert any(
65+
expected_message in msg for msg in log_messages
66+
), f"Expected log message containing '{expected_message}' not found in logs: {log_messages}"
67+
68+
@HttpMocker()
69+
@freeze_time("2022-11-16 12:03:11+00:00")
70+
def test_usage_records_completes_with_mixed_responses(self, http_mocker: HttpMocker):
71+
"""Test that sync completes successfully with a sequence of 200, 404, 404, 200 responses."""
72+
accounts_json = {
73+
"accounts": [
74+
{"sid": "AC001", "date_created": "2022-01-01T00:00:00Z", "subresource_uris": {}},
75+
{"sid": "AC002", "date_created": "2022-01-02T00:00:00Z", "subresource_uris": {}},
76+
{"sid": "AC003", "date_created": "2022-01-03T00:00:00Z", "subresource_uris": {}},
77+
{"sid": "AC004", "date_created": "2022-01-04T00:00:00Z", "subresource_uris": {}},
78+
],
79+
}
80+
http_mocker.get(
81+
HttpRequest(url=f"{BASE}/Accounts.json", query_params={"PageSize": "1000"}),
82+
HttpResponse(body=json.dumps(accounts_json), status_code=200),
83+
)
84+
85+
http_mocker.get(
86+
HttpRequest(
87+
url=f"{BASE}/Accounts/AC001/Usage/Records/Daily.json",
88+
query_params={"PageSize": "1000", "StartDate": "2022-11-15", "EndDate": "2022-11-16"},
89+
),
90+
HttpResponse(
91+
body=json.dumps(
92+
{
93+
"usage_records": [
94+
{
95+
"account_sid": "AC001",
96+
"category": "calls",
97+
"start_date": "2022-11-15",
98+
"end_date": "2022-11-16",
99+
"count": "10",
100+
"usage": "100",
101+
}
102+
]
103+
}
104+
),
105+
status_code=200,
106+
),
107+
)
108+
109+
http_mocker.get(
110+
HttpRequest(
111+
url=f"{BASE}/Accounts/AC002/Usage/Records/Daily.json",
112+
query_params={"PageSize": "1000", "StartDate": "2022-11-15", "EndDate": "2022-11-16"},
113+
),
114+
HttpResponse(
115+
body=json.dumps({"code": 20404, "message": "The requested resource was not found"}),
116+
status_code=404,
117+
),
118+
)
119+
120+
http_mocker.get(
121+
HttpRequest(
122+
url=f"{BASE}/Accounts/AC003/Usage/Records/Daily.json",
123+
query_params={"PageSize": "1000", "StartDate": "2022-11-15", "EndDate": "2022-11-16"},
124+
),
125+
HttpResponse(
126+
body=json.dumps({"code": 20404, "message": "The requested resource was not found"}),
127+
status_code=404,
128+
),
129+
)
130+
131+
http_mocker.get(
132+
HttpRequest(
133+
url=f"{BASE}/Accounts/AC004/Usage/Records/Daily.json",
134+
query_params={"PageSize": "1000", "StartDate": "2022-11-15", "EndDate": "2022-11-16"},
135+
),
136+
HttpResponse(
137+
body=json.dumps(
138+
{
139+
"usage_records": [
140+
{
141+
"account_sid": "AC004",
142+
"category": "sms",
143+
"start_date": "2022-11-15",
144+
"end_date": "2022-11-16",
145+
"count": "5",
146+
"usage": "50",
147+
}
148+
]
149+
}
150+
),
151+
status_code=200,
152+
),
153+
)
154+
155+
catalog = CatalogBuilder().with_stream("usage_records", SyncMode.full_refresh).build()
156+
config = {**TEST_CONFIG, "start_date": "2022-11-15T00:00:00Z"}
157+
158+
output = read(get_source(config), config, catalog)
159+
160+
assert len(output.records) == 2, f"Expected 2 records from successful responses, got {len(output.records)}"
161+
162+
account_sids = [record.record.data["account_sid"] for record in output.records]
163+
assert "AC001" in account_sids, "Expected record from AC001"
164+
assert "AC004" in account_sids, "Expected record from AC004"
165+
166+
assert output.errors == [], f"Expected no errors, but got: {output.errors}"
167+
168+
@HttpMocker()
169+
@freeze_time("2022-11-16 12:03:11+00:00")
170+
def test_usage_records_incremental_with_404_handling(self, http_mocker: HttpMocker):
171+
"""Test that incremental sync handles 404 responses correctly."""
172+
http_mocker.get(
173+
HttpRequest(url=f"{BASE}/Accounts.json", query_params={"PageSize": "1000"}),
174+
HttpResponse(body=json.dumps(ACCOUNTS_JSON), status_code=200),
175+
)
176+
177+
http_mocker.get(
178+
HttpRequest(
179+
url=f"{BASE}/Accounts/AC123/Usage/Records/Daily.json",
180+
query_params={"PageSize": "1000", "StartDate": "2022-11-15", "EndDate": "2022-11-16"},
181+
),
182+
HttpResponse(
183+
body=json.dumps({"code": 20404, "message": "The requested resource was not found"}),
184+
status_code=404,
185+
),
186+
)
187+
188+
catalog = CatalogBuilder().with_stream("usage_records", SyncMode.incremental).build()
189+
config = {**TEST_CONFIG, "start_date": "2022-11-15T00:00:00Z"}
190+
191+
state = (
192+
StateBuilder()
193+
.with_stream_state(
194+
"usage_records", {"states": [{"partition": {"account_sid": "AC123"}, "cursor": {"start_date": "2022-11-13"}}]}
195+
)
196+
.build()
197+
)
198+
199+
output = read(get_source(config, state), config, catalog, state)
200+
201+
assert len(output.records) == 0, "Expected no records when 404 is returned in incremental sync"
202+
203+
assert output.errors == [], f"Expected no errors, but got: {output.errors}"

‎docs/integrations/sources/twilio.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ For more information, see [the Twilio docs for rate limitations](https://support
100100

101101
| Version | Date | Pull Request | Subject |
102102
|:------------|:-----------| :------------------------------------------------------- |:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------|
103+
| 0.17.3 | 2025-11-06 | [68680](https://github.com/airbytehq/airbyte/pull/68680) | Handle 404 errors gracefully for date ranges with no data |
103104
| 0.17.2 | 2025-10-22 | [68591](https://github.com/airbytehq/airbyte/pull/68591) | Add `suggestedStreams` |
104105
| 0.17.1 | 2025-09-11 | [66090](https://github.com/airbytehq/airbyte/pull/66090) | Update to CDK v7 |
105106
| 0.17.0 | 2025-09-05 | [65955](https://github.com/airbytehq/airbyte/pull/65955) | Promoting release candidate 0.17.0-rc.2 to a main version. |

0 commit comments

Comments
 (0)