Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
'boto3==1.39.8',
'urllib3==2.6.3',
'singer-encodings==0.3.0',
'singer-python==5.14.3',
'singer-python==5.19.0',
'voluptuous==0.15.2',
's3fs==2025.9.0'
],
Expand Down
2 changes: 2 additions & 0 deletions tap_s3_csv/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ def do_sync(config, catalog, state, sync_start_time):
LOGGER.info("%s: Skipping - not selected", stream_name)
continue

bookmark = singer.get_bookmark(state, stream_name, 'modified_since') or config['start_date']
state = singer.set_bookmark(state, stream_name, 'modified_since', bookmark)
singer.write_state(state)
key_properties = metadata.get(mdata, (), 'table-key-properties')
singer.write_schema(stream_name, stream['schema'], key_properties)
Expand Down
52 changes: 52 additions & 0 deletions tests/test_bookmarks.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
from datetime import datetime, date, timedelta, time
from tap_tester import connections, menagerie, runner
from functools import reduce
from singer import metadata
Expand Down Expand Up @@ -102,3 +103,54 @@ def test_run(self):
records = runner.get_records_from_target_output()
messages = records.get('chickens', {}).get('messages', [])
self.assertEqual(len(messages), 0, msg="Sync'd incorrect count of messages: {}".format(len(messages)))

class S3BookmarksStartDateSucceedsModifiedDate(S3CSVBaseTest):

table_entry = [{'table_name': 'skipped', 'search_prefix': 'tap_tester', 'search_pattern': 'tap_tester/bookmarks.*', 'key_properties': 'name'}]

def setUp(self):
self.conn_id = connections.ensure_connection(self)

def resource_name(self):
return "bookmarks.csv"

def name(self):
return "tap_tester_s3_csv_bookmarks"

def expected_check_streams(self):
return {
'skipped'
}

def expected_sync_streams(self):
return {
'skipped'
}

def expected_pks(self):
return {
'skipped': {"name"}
}

def get_properties(self, original: bool = True):
tomorrow = date.today() + timedelta(days=1)
self.start_date = datetime.combine(tomorrow, time.min).strftime("%Y-%m-%dT%H:%M:%SZ")

return super().get_properties(original) | {'start_date': self.start_date}

def test_run(self):
found_catalogs = self.run_and_verify_check_mode(self.conn_id)

# Select our catalogs
our_catalogs = [c for c in found_catalogs if c.get('tap_stream_id') in self.expected_sync_streams()]

self.perform_and_verify_table_and_field_selection(self.conn_id, our_catalogs)

# Clear state before our run
menagerie.set_state(self.conn_id, {})

# Sync 0 records because start date is after file modified_date
self.run_and_verify_sync(self.conn_id, True)

expected_state = {'bookmarks': {'skipped': {'modified_since': self.start_date}}}
self.assertEqual(menagerie.get_state(self.conn_id), expected_state)
Loading