Skip to content

Commit 1537dc5

Browse files
committed
fix(manifest-server): reset global record counter before test reads
1 parent 784bdb3 commit 1537dc5

File tree

3 files changed

+61
-0
lines changed

3 files changed

+61
-0
lines changed

airbyte_cdk/.DS_Store

6 KB
Binary file not shown.

airbyte_cdk/manifest_server/command_processor/processor.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
)
88
from fastapi import HTTPException
99

10+
import airbyte_cdk.sources.declarative.stream_slicers.declarative_partition_generator
1011
from airbyte_cdk.connector_builder.models import StreamRead
1112
from airbyte_cdk.connector_builder.test_reader import TestReader
1213
from airbyte_cdk.entrypoint import AirbyteEntrypoint
@@ -42,6 +43,12 @@ def test_read(
4243
Test the read method of the source.
4344
"""
4445

46+
# HACK: reset total_record_counter
47+
# DeclarativePartition defines total_record_counter as a global variable, which keeps around the record count
48+
# across multiple test_read calls, even if the source is different. This is a hack to reset the counter for
49+
# each test_read call.
50+
airbyte_cdk.sources.declarative.stream_slicers.declarative_partition_generator.total_record_counter = 0
51+
4552
test_read_handler = TestReader(
4653
max_pages_per_slice=page_limit,
4754
max_slices=slice_limit,

unit_tests/manifest_server/command_processor/test_processor.py

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -308,3 +308,57 @@ def test_discover_with_trace_error(self, command_processor, sample_config):
308308
# Verify exception is raised
309309
with pytest.raises(HTTPException):
310310
command_processor.discover(sample_config)
311+
312+
def test_test_read_resets_global_record_counter(
313+
self, command_processor, sample_config, sample_catalog
314+
):
315+
"""Test that test_read resets the global total_record_counter between calls."""
316+
from airbyte_cdk.sources.declarative import stream_slicers
317+
318+
# Mock the TestReader
319+
with patch(
320+
"airbyte_cdk.manifest_server.command_processor.processor.TestReader"
321+
) as mock_test_reader_class:
322+
mock_test_reader_instance = Mock()
323+
mock_test_reader_class.return_value = mock_test_reader_instance
324+
mock_stream_read = Mock()
325+
mock_test_reader_instance.run_test_read.return_value = mock_stream_read
326+
327+
# Set initial counter value to simulate previous test_read execution
328+
stream_slicers.declarative_partition_generator.total_record_counter = 100
329+
330+
# Execute test_read
331+
result1 = command_processor.test_read(
332+
config=sample_config,
333+
catalog=sample_catalog,
334+
state=[],
335+
record_limit=50,
336+
page_limit=3,
337+
slice_limit=7,
338+
)
339+
340+
# Verify counter was reset to 0 before the test_read
341+
assert stream_slicers.declarative_partition_generator.total_record_counter == 0
342+
343+
# Set counter again to simulate state from first call
344+
stream_slicers.declarative_partition_generator.total_record_counter = 200
345+
346+
# Execute another test_read
347+
result2 = command_processor.test_read(
348+
config=sample_config,
349+
catalog=sample_catalog,
350+
state=[],
351+
record_limit=25,
352+
page_limit=2,
353+
slice_limit=5,
354+
)
355+
356+
# Verify counter was reset again
357+
assert stream_slicers.declarative_partition_generator.total_record_counter == 0
358+
359+
# Verify both calls returned the expected results
360+
assert result1 == mock_stream_read
361+
assert result2 == mock_stream_read
362+
363+
# Verify TestReader was called twice
364+
assert mock_test_reader_class.call_count == 2

0 commit comments

Comments
 (0)