feat(schema): rename paging_token columns to TOID

aolieman · aolieman · commit 3793f417056d · 2025-11-11T13:39:28.000+01:00
diff --git a/sc_audit/db_schema/distribution.py b/sc_audit/db_schema/distribution.py
@@ -18,7 +18,7 @@ class DistributionTxBase(MappedAsDataclass):
     sender: Mapped[strkey]
     recipient: Mapped[strkey]
     carbon_amount: Mapped[kgdecimal]
-    paging_token: Mapped[bigintpk]
+    toid: Mapped[bigintpk]
 
 
 class DistributionTx(DistributionTxBase, ScBase):
diff --git a/sc_audit/db_schema/mint.py b/sc_audit/db_schema/mint.py
@@ -46,7 +46,7 @@ class MintedBlockBase(MappedAsDataclass, kw_only=True):
     sub_account_name: Mapped[VerraSubAccountName]
     vintage_start: Mapped[dt.date]
     vintage_end: Mapped[dt.date]
-    paging_token: Mapped[bigint]
+    toid: Mapped[bigint]
 
 
 class MintedBlock(MintedBlockBase, ScBase):
@@ -102,4 +102,4 @@ def _credits_remaining_on_date_expression(cls, on_date: dt.date) -> SQLColumnExp
 
 
 idx_created_at = Index("idx_block_created_at", MintedBlock.created_at)
-idx_toid = Index("idx_block_toid", MintedBlock.paging_token.desc(), unique=True)
+idx_toid = Index("idx_block_toid", MintedBlock.toid.desc(), unique=True)
diff --git a/sc_audit/db_schema/sink.py b/sc_audit/db_schema/sink.py
@@ -39,7 +39,7 @@ class SinkingTxBase(MappedAsDataclass, kw_only=True):
     vcs_project_id: Mapped[int] = mapped_column(ForeignKey("vcs_projects.id"))
     memo_type: Mapped[MemoType]
     memo_value: Mapped[str | None] = mapped_column(String(64))
-    paging_token: Mapped[bigint]
+    toid: Mapped[bigint]
 
 
 class SinkingTx(SinkingTxBase, ScBase):
@@ -73,6 +73,6 @@ def as_dict(self):
     
 
 idx_created_at = Index("idx_stx_created_at", SinkingTx.created_at.desc())
-idx_toid = Index("idx_stx_toid", SinkingTx.paging_token.desc(), unique=True)
+idx_toid = Index("idx_stx_toid", SinkingTx.toid.desc(), unique=True)
 idx_funder = Index("idx_stx_funder", SinkingTx.funder)
 idx_recipient = Index("idx_stx_recipient", SinkingTx.recipient)
diff --git a/sc_audit/loader/distribution_outflows.py b/sc_audit/loader/distribution_outflows.py
@@ -31,7 +31,7 @@ def load_distribution_txs(cursor: int=settings.FIRST_DIST_CURSOR) -> int:
                     sender=payment['from'],
                     recipient=payment['to'],
                     carbon_amount=Decimal(payment['amount']),
-                    paging_token=payment['paging_token'],
+                    toid=payment['paging_token'],
                 )
             )
         
diff --git a/sc_audit/loader/get_latest.py b/sc_audit/loader/get_latest.py
@@ -38,36 +38,36 @@ def get_latest_attr(*models: CoreModelName) -> LatestAttr | list[LatestAttr]:
                 latest_attrs[i] = latest_retirement_date
             elif model == 'sink_tx':
                 latest_sink_tx_cursor = session.scalar(
-                    select(SinkingTx.paging_token)
+                    select(SinkingTx.toid)
                     .where(SinkingTx.contract_id.is_(None))
-                    .order_by(SinkingTx.paging_token.desc())
+                    .order_by(SinkingTx.toid.desc())
                 )
-                latest_attrs[i] = increment_paging_token(latest_sink_tx_cursor) or settings.FIRST_SINK_CURSOR
+                latest_attrs[i] = increment_toid(latest_sink_tx_cursor) or settings.FIRST_SINK_CURSOR
             elif model == 'sink_call':
                 latest_sink_call_cursor = session.scalar(
-                    select(SinkingTx.paging_token)
+                    select(SinkingTx.toid)
                     .where(SinkingTx.contract_id.is_not(None))
-                    .order_by(SinkingTx.paging_token.desc())
+                    .order_by(SinkingTx.toid.desc())
                 )
                 latest_attrs[i] = latest_sink_call_cursor or settings.FIRST_SINK_CURSOR
             elif model == 'mint_tx':
                 latest_mint_tx_cursor = session.scalar(
-                    select(MintedBlock.paging_token).order_by(MintedBlock.paging_token.desc())
+                    select(MintedBlock.toid).order_by(MintedBlock.toid.desc())
                 )
-                latest_attrs[i] = increment_paging_token(latest_mint_tx_cursor) or settings.FIRST_MINT_CURSOR
+                latest_attrs[i] = increment_toid(latest_mint_tx_cursor) or settings.FIRST_MINT_CURSOR
             elif model == 'dist_tx':
                 latest_dist_tx_cursor = session.scalar(
-                    select(DistributionTx.paging_token).order_by(DistributionTx.paging_token.desc())
+                    select(DistributionTx.toid).order_by(DistributionTx.toid.desc())
                 )
-                latest_attrs[i] = increment_paging_token(latest_dist_tx_cursor) or settings.FIRST_DIST_CURSOR
+                latest_attrs[i] = increment_toid(latest_dist_tx_cursor) or settings.FIRST_DIST_CURSOR
 
     if len(latest_attrs) == 1:
         return latest_attrs[0]
     else:
         return latest_attrs
 
 
-def increment_paging_token(cursor: int | None) -> int | None:
+def increment_toid(cursor: int | None) -> int | None:
     if cursor:
         toid = TOID.from_int64(cursor)
         toid.increment_operation_order()
diff --git a/sc_audit/loader/minted_blocks.py b/sc_audit/loader/minted_blocks.py
@@ -62,15 +62,15 @@ def load_minted_blocks(cursor: int=settings.FIRST_MINT_CURSOR) -> int:
                     sub_account_name=carbon_pool[serial_hash]['sub_account_name'], # type: ignore
                     vintage_start=serial_number.vintage_start_date,
                     vintage_end=serial_number.vintage_end_date,
-                    paging_token=mint_tx['paging_token']
+                    toid=mint_tx['paging_token']
                 )
                 pristine_blocks.append(block)
             else:
                 retired_blocks.append(mint_tx)
 
         if retired_blocks:
             latest_loaded_block: MintedBlock | None = session.scalars(
-                select(MintedBlock).order_by(MintedBlock.paging_token.desc())
+                select(MintedBlock).order_by(MintedBlock.toid.desc())
             ).first()
             # select only retirements without retired_from_block relations
             retirements: Sequence[Retirement] = session.scalars(
@@ -150,7 +150,7 @@ def reconstruct_blocks(
             sub_account_name=sub_account_name,
             vintage_start=reconstructed_serial.vintage_start_date,
             vintage_end=reconstructed_serial.vintage_end_date,
-            paging_token=mint_tx['paging_token']
+            toid=mint_tx['paging_token']
         )
         latest_block = block
         reconstructed_blocks.append(block)
diff --git a/sc_audit/loader/sink_events.py b/sc_audit/loader/sink_events.py
@@ -62,7 +62,7 @@ def load_sink_events(cursor: int=settings.FIRST_SINK_CURSOR) -> int:
                     vcs_project_id=vcs_project_id,
                     memo_type='text' if memo else 'none',
                     memo_value=memo,
-                    paging_token=event_toid.to_int64(),
+                    toid=event_toid.to_int64(),
                 )
             )
         
diff --git a/sc_audit/loader/sink_invocations.py b/sc_audit/loader/sink_invocations.py
@@ -41,7 +41,7 @@ def load_sink_invocations(cursor: int=settings.FIRST_SINK_CURSOR) -> int:
                     vcs_project_id=vcs_project_id,
                     memo_type='text' if memo else 'none',
                     memo_value=memo,
-                    paging_token=sink_invoke.toid,
+                    toid=sink_invoke.toid,
                 )
             )
         
diff --git a/sc_audit/loader/sinking_txs.py b/sc_audit/loader/sinking_txs.py
@@ -60,7 +60,7 @@ def load_sinking_txs(cursor: int=settings.FIRST_SINK_CURSOR) -> int:
                     vcs_project_id=vcs_project_id,
                     memo_type=memo_type,
                     memo_value=memo_value,
-                    paging_token=sink_tx['paging_token'],
+                    toid=sink_tx['paging_token'],
                 )
             )
         
diff --git a/sc_audit/migrations/versions/371cdad348d3_rename_paging_token_columns_to_toid.py b/sc_audit/migrations/versions/371cdad348d3_rename_paging_token_columns_to_toid.py
@@ -0,0 +1,85 @@
+"""rename paging_token columns to TOID
+
+Revision ID: 371cdad348d3
+Revises: 52e5e68af772
+Create Date: 2025-11-11 12:18:35.680184
+
+"""
+from typing import Sequence
+
+from alembic import context, op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '371cdad348d3'
+down_revision: str | None = '52e5e68af772'
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    # turn off FK constraints for SQLite to allow table recreation
+    dialect_name = context.get_context().dialect.name
+    if dialect_name == 'sqlite':
+        op.execute("PRAGMA foreign_keys=OFF")
+
+    with op.batch_alter_table('distribution_txs', schema=None) as batch_op:
+        batch_op.alter_column('paging_token', new_column_name='toid')
+
+    with op.batch_alter_table('minted_blocks', schema=None) as batch_op:
+        batch_op.alter_column('paging_token', new_column_name='toid')
+        batch_op.drop_index(batch_op.f('idx_block_toid'))
+        
+    op.create_index('idx_block_toid', 'minted_blocks', [sa.literal_column('toid DESC')], unique=True)
+
+    with op.batch_alter_table('sinking_txs', schema=None) as batch_op:
+        batch_op.alter_column('paging_token', new_column_name='toid')
+        batch_op.drop_index(batch_op.f('idx_stx_toid'))
+        
+    op.create_index('idx_stx_toid', 'sinking_txs', [sa.literal_column('toid DESC')], unique=True)
+
+    with op.batch_alter_table('test_distribution_txs', schema=None) as batch_op:
+        batch_op.alter_column('paging_token', new_column_name='toid')
+
+    with op.batch_alter_table('test_minted_blocks', schema=None) as batch_op:
+        batch_op.alter_column('paging_token', new_column_name='toid')
+
+    with op.batch_alter_table('test_sinking_txs', schema=None) as batch_op:
+        batch_op.alter_column('paging_token', new_column_name='toid')
+
+    if dialect_name == 'sqlite':
+        op.execute("PRAGMA foreign_keys=ON")
+
+
+def downgrade() -> None:
+    dialect_name = context.get_context().dialect.name
+    if dialect_name == 'sqlite':
+        op.execute("PRAGMA foreign_keys=OFF")
+
+    with op.batch_alter_table('test_sinking_txs', schema=None) as batch_op:
+        batch_op.alter_column('toid', new_column_name='paging_token')
+
+    with op.batch_alter_table('test_minted_blocks', schema=None) as batch_op:
+        batch_op.alter_column('toid', new_column_name='paging_token')
+
+    with op.batch_alter_table('test_distribution_txs', schema=None) as batch_op:
+        batch_op.alter_column('toid', new_column_name='paging_token')
+
+    with op.batch_alter_table('sinking_txs', schema=None) as batch_op:
+        batch_op.alter_column('toid', new_column_name='paging_token')
+        batch_op.drop_index('idx_stx_toid')
+        
+    op.create_index(op.f('idx_stx_toid'), 'sinking_txs', [sa.literal_column('paging_token DESC')], unique=True)
+
+    with op.batch_alter_table('minted_blocks', schema=None) as batch_op:
+        batch_op.alter_column('toid', new_column_name='paging_token')
+        batch_op.drop_index('idx_block_toid')
+        
+    op.create_index(op.f('idx_block_toid'), 'minted_blocks', [sa.literal_column('paging_token DESC')], unique=True)
+
+    with op.batch_alter_table('distribution_txs', schema=None) as batch_op:
+        batch_op.alter_column('toid', new_column_name='paging_token')
+
+    if dialect_name == 'sqlite':
+        op.execute("PRAGMA foreign_keys=ON")
diff --git a/sc_audit/views/inventory.py b/sc_audit/views/inventory.py
@@ -40,7 +40,7 @@ def view_inventory(omit_empty: bool = False, until_date: dt.date | None = None)
         VcsProject.category,
     ]    
     columns = block_columns + vcs_project_columns
-    q_blocks = select(*columns).join(MintedBlock.vcs_project).order_by(MintedBlock.paging_token)
+    q_blocks = select(*columns).join(MintedBlock.vcs_project).order_by(MintedBlock.toid)
     if omit_empty:
         q_blocks = q_blocks.where(credits_remaining_col > 0)
 
diff --git a/sc_audit/views/sink_status.py b/sc_audit/views/sink_status.py
@@ -31,35 +31,35 @@ def view_sinking_txs(
     base_query = construct_stx_query(
         for_funder, for_recipient, from_date, before_date, finalized, contract_call
     ).subquery()
-    subq_paging_token = base_query.c.paging_token
+    subq_toid = base_query.c.toid
     # Determine order and cursor filter
     if order == 'asc':
-        subq_order = subq_paging_token.asc()
-        stx_order = SinkingTx.paging_token.asc()
-        cursor_filter = subq_paging_token > (cursor or 0)
+        subq_order = subq_toid.asc()
+        stx_order = SinkingTx.toid.asc()
+        cursor_filter = subq_toid > (cursor or 0)
     elif order == 'desc':
-        subq_order = subq_paging_token.desc()
-        stx_order = SinkingTx.paging_token.desc()
-        cursor_filter = subq_paging_token < (cursor or 0)
+        subq_order = subq_toid.desc()
+        stx_order = SinkingTx.toid.desc()
+        cursor_filter = subq_toid < (cursor or 0)
 
     with Session.begin() as session:
-        # Step 1: Get unique paging_tokens for the page
-        token_query = select(subq_paging_token).select_from(base_query)
+        # Step 1: Get unique TOIDs for the page
+        token_query = select(subq_toid).select_from(base_query)
         if cursor and cursor > 0:
             token_query = token_query.where(cursor_filter)
         
         token_query = token_query.order_by(subq_order).distinct().limit(limit)
-        paging_tokens = session.scalars(token_query).all()
+        toids = session.scalars(token_query).all()
 
-        if not paging_tokens:
+        if not toids:
             return pd.DataFrame()
 
-        # Step 2: Fetch SinkingTxs for those paging_tokens, eager load statuses
+        # Step 2: Fetch SinkingTxs for those TOIDs, eager load statuses
         stx_query = (
             select(SinkingTx)
             .outerjoin(SinkStatus)
             .options(contains_eager(SinkingTx.statuses))
-            .where(SinkingTx.paging_token.in_(paging_tokens))
+            .where(SinkingTx.toid.in_(toids))
             .order_by(stx_order)
         )
         stx_records = session.scalars(stx_query).unique().all()
diff --git a/tests/test_event.py b/tests/test_event.py
@@ -83,7 +83,7 @@ def test_load_sink_events(self, mock_session):
 
         with mock_session.begin() as session:
             loaded_events = session.scalars(
-                select(SinkingTx).order_by(SinkingTx.paging_token.asc())
+                select(SinkingTx).order_by(SinkingTx.toid.asc())
                 .options(joinedload(SinkingTx.vcs_project))
             ).all()
 
@@ -128,7 +128,7 @@ def test_load_sink_txs_and_events(self, mock_http, mock_session):
 
         with mock_session.begin() as session:
             loaded_transactions = session.scalars(
-                select(SinkingTx).order_by(SinkingTx.paging_token.asc())
+                select(SinkingTx).order_by(SinkingTx.toid.asc())
             ).all()
             assert len(loaded_transactions) == 30
             
@@ -140,7 +140,7 @@ def test_load_sink_txs_and_events(self, mock_http, mock_session):
                     # is the created_at time in accordance with the paging token?
                     assert tx.created_at >= prev_tx.created_at
                     # is each paging token unique?
-                    assert tx.paging_token != prev_tx.paging_token
+                    assert tx.toid != prev_tx.toid
 
                 prev_tx = tx
                 cumulative_amount += tx.carbon_amount
diff --git a/tests/test_invocation.py b/tests/test_invocation.py
@@ -116,7 +116,7 @@ def test_load_sink_invocations(self, mock_session):
 
         with mock_session.begin() as session:
             loaded_invocations = session.scalars(
-                select(SinkingTx).order_by(SinkingTx.paging_token.asc())
+                select(SinkingTx).order_by(SinkingTx.toid.asc())
                 .options(joinedload(SinkingTx.vcs_project))
             ).all()
 
@@ -146,7 +146,7 @@ def test_load_sink_txs_and_invocations(self, mock_http, mock_session):
 
         with mock_session.begin() as session:
             loaded_transactions = session.scalars(
-                select(SinkingTx).order_by(SinkingTx.paging_token.asc())
+                select(SinkingTx).order_by(SinkingTx.toid.asc())
             ).all()
             assert len(loaded_transactions) == 23
             
@@ -158,7 +158,7 @@ def test_load_sink_txs_and_invocations(self, mock_http, mock_session):
                     # is the created_at time in accordance with the paging token?
                     assert tx.created_at >= prev_tx.created_at
                     # is each paging token unique?
-                    assert tx.paging_token != prev_tx.paging_token
+                    assert tx.toid != prev_tx.toid
 
                 prev_tx = tx
                 cumulative_amount += tx.carbon_amount
diff --git a/tests/test_mint.py b/tests/test_mint.py
@@ -162,7 +162,7 @@ def test_load_minted_blocks_verify(self, mock_session, mock_http):
 
         with mock_session.begin() as session:
             loaded_blocks = session.scalars(
-                select(MintedBlock).order_by(MintedBlock.paging_token.asc())
+                select(MintedBlock).order_by(MintedBlock.toid.asc())
             ).all()
             assert len(loaded_blocks) == 9
             for block in loaded_blocks:
@@ -174,7 +174,7 @@ def test_load_distribution_outflows(self, mock_session, mock_http):
 
         with mock_session.begin() as session:
             loaded_outflows = session.scalars(
-                select(DistributionTx).order_by(DistributionTx.paging_token.asc())
+                select(DistributionTx).order_by(DistributionTx.toid.asc())
             ).all()
             assert len(loaded_outflows) == 5
             total_outflow = sum(
@@ -212,7 +212,7 @@ def first_block(first_block_data) -> MintedBlock:
         sub_account_name=verra_carbon_pool.name, 
         vintage_start=serial_number.vintage_start_date, 
         vintage_end=serial_number.vintage_end_date,
-        paging_token=first_block_data['paging_token']
+        toid=first_block_data['paging_token']
     )
 
 @pytest.fixture
diff --git a/tests/test_sink.py b/tests/test_sink.py
@@ -88,7 +88,7 @@ def test_load_sinking_transactions(self, mock_http, mock_session):
 
         with mock_session.begin() as session:
             loaded_transactions = session.scalars(
-                select(SinkingTx).order_by(SinkingTx.paging_token.asc())
+                select(SinkingTx).order_by(SinkingTx.toid.asc())
             ).all()
             assert len(loaded_transactions) == 20
 
diff --git a/tests/test_views.py b/tests/test_views.py

Original file line number	Diff line number	Diff line change
`@@ -31,7 +31,7 @@ def load_distribution_txs(cursor: int=settings.FIRST_DIST_CURSOR) -> int:`
`31`	`31`	`sender=payment['from'],`
`32`	`32`	`recipient=payment['to'],`
`33`	`33`	`carbon_amount=Decimal(payment['amount']),`
`34`		`- paging_token=payment['paging_token'],`
	`34`	`+ toid=payment['paging_token'],`
`35`	`35`	`)`
`36`	`36`	`)`
`37`	`37`
Original file line number	Diff line number	Diff line change
`@@ -62,7 +62,7 @@ def load_sink_events(cursor: int=settings.FIRST_SINK_CURSOR) -> int:`
`62`	`62`	`vcs_project_id=vcs_project_id,`
`63`	`63`	`memo_type='text' if memo else 'none',`
`64`	`64`	`memo_value=memo,`
`65`		`- paging_token=event_toid.to_int64(),`
	`65`	`+ toid=event_toid.to_int64(),`
`66`	`66`	`)`
`67`	`67`	`)`
`68`	`68`
Original file line number	Diff line number	Diff line change
`@@ -41,7 +41,7 @@ def load_sink_invocations(cursor: int=settings.FIRST_SINK_CURSOR) -> int:`
`41`	`41`	`vcs_project_id=vcs_project_id,`
`42`	`42`	`memo_type='text' if memo else 'none',`
`43`	`43`	`memo_value=memo,`
`44`		`- paging_token=sink_invoke.toid,`
	`44`	`+ toid=sink_invoke.toid,`
`45`	`45`	`)`
`46`	`46`	`)`
`47`	`47`
Original file line number	Diff line number	Diff line change
`@@ -60,7 +60,7 @@ def load_sinking_txs(cursor: int=settings.FIRST_SINK_CURSOR) -> int:`
`60`	`60`	`vcs_project_id=vcs_project_id,`
`61`	`61`	`memo_type=memo_type,`
`62`	`62`	`memo_value=memo_value,`
`63`		`- paging_token=sink_tx['paging_token'],`
	`63`	`+ toid=sink_tx['paging_token'],`
`64`	`64`	`)`
`65`	`65`	`)`
`66`	`66`