Skip to content

Commit 9c9bd21

Browse files
committed
fix "starting_block" in base class
1 parent 7354e3a commit 9c9bd21

File tree

1 file changed

+13
-2
lines changed

1 file changed

+13
-2
lines changed

scraper_service/shared/shovel_base_class.py

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,18 @@ def _buffer_flush_done(self, tables, rows):
141141

142142
def get_checkpoint(self):
143143
if not table_exists("shovel_checkpoints"):
144-
return self.starting_block - 1
144+
return max(0, self.starting_block - 1)
145+
146+
# First check if our shovel has any entries
147+
query = f"""
148+
SELECT count(*)
149+
FROM shovel_checkpoints
150+
WHERE shovel_name = '{self.name}'
151+
"""
152+
count = get_clickhouse_client().execute(query)[0][0]
153+
if count == 0:
154+
return max(0, self.starting_block - 1)
155+
145156
query = f"""
146157
SELECT block_number
147158
FROM shovel_checkpoints
@@ -153,4 +164,4 @@ def get_checkpoint(self):
153164
if res:
154165
return res[0][0]
155166
else:
156-
return self.starting_block - 1
167+
return max(0, self.starting_block - 1) # This case shouldn't happen due to count check above

0 commit comments

Comments
 (0)