Skip to content
Merged
Show file tree
Hide file tree
Changes from 9 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
'boto3==1.39.8',
'urllib3==2.6.3',
'singer-encodings==0.3.0',
'singer-python==5.14.3',
'singer-python==5.19.0',
'voluptuous==0.15.2',
's3fs==2025.9.0'
],
Expand Down
2 changes: 2 additions & 0 deletions tap_s3_csv/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ def do_sync(config, catalog, state, sync_start_time):
LOGGER.info("%s: Skipping - not selected", stream_name)
continue

bookmark = singer.get_bookmark(state, stream_name, 'modified_since') or config['start_date']
state = singer.set_bookmark(state, stream_name, 'modified_since', bookmark)
singer.write_state(state)
key_properties = metadata.get(mdata, (), 'table-key-properties')
singer.write_schema(stream_name, stream['schema'], key_properties)
Expand Down
52 changes: 52 additions & 0 deletions tests/test_bookmarks.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
from datetime import datetime, date, timedelta, time
from tap_tester import connections, menagerie, runner
from functools import reduce
from singer import metadata
Expand Down Expand Up @@ -102,3 +103,54 @@ def test_run(self):
records = runner.get_records_from_target_output()
messages = records.get('chickens', {}).get('messages', [])
self.assertEqual(len(messages), 0, msg="Sync'd incorrect count of messages: {}".format(len(messages)))

class S3BookmarksStartDateSucceedsModifiedDate(S3CSVBaseTest):

table_entry = [{'table_name': 'skipped', 'search_prefix': 'tap_tester', 'search_pattern': 'tap_tester/bookmarks.*', 'key_properties': 'name'}]

def setUp(self):
self.conn_id = connections.ensure_connection(self)

def resource_name(self):
return "bookmarks.csv"

def name(self):
return "tap_tester_s3_csv_bookmarks"

def expected_check_streams(self):
return {
'skipped'
}

def expected_sync_streams(self):
return {
'skipped'
}

def expected_pks(self):
return {
'skipped': {"name"}
}

def get_properties(self, original: bool = True):
tomorrow = date.today() + timedelta(days=1)
self.start_date = datetime.combine(tomorrow, time.min).strftime("%Y-%m-%dT%H:%M:%SZ")

return super().get_properties(original) | {'start_date': self.start_date}

def test_run(self):
found_catalogs = self.run_and_verify_check_mode(self.conn_id)

# Select our catalogs
our_catalogs = [c for c in found_catalogs if c.get('tap_stream_id') in self.expected_sync_streams()]

self.perform_and_verify_table_and_field_selection(self.conn_id, our_catalogs)

# Clear state before our run
menagerie.set_state(self.conn_id, {})

# Sync 0 records because start date is after file modified_date
self.run_and_verify_sync(self.conn_id, True)

expected_state = {'bookmarks': {'skipped': {'modified_since': self.start_date}}}
self.assertEqual(menagerie.get_state(self.conn_id), expected_state)
Loading