Fixed node log request, decimals in uniswap demo (#854)

Author: droserasprout

CHANGELOG.md

@@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog], and this project adheres to [Semantic Versioning].
 
+## [Unreleased]
+
+### Fixed
+
+- demos: Fixed decimal overflow in `demo_uniswap` project.
+- evm.node: Fixed incorrect log request parameters.
+
 ## [7.0.0] - 2023-09-25
 
 ### Fixed

src/demo_uniswap/models/__init__.py

@@ -12,23 +12,23 @@ class Factory(CachedModel):
     # amount of transactions all time
     tx_count = fields.BigIntField(default=0)
     # total volume all time in derived USD
-    total_volume_usd = fields.DecimalField(decimal_places=2, max_digits=16, default=0)
+    total_volume_usd = fields.DecimalField(decimal_places=2, max_digits=32, default=0)
     # total volume all time in derived ETH
-    total_volume_eth = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    total_volume_eth = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # total swap fees all time in USD
-    total_fees_usd = fields.DecimalField(decimal_places=2, max_digits=16, default=0)
+    total_fees_usd = fields.DecimalField(decimal_places=2, max_digits=32, default=0)
     # total swap fees all time in USD
-    total_fees_eth = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    total_fees_eth = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # all volume even through less reliable USD values
-    untracked_volume_usd = fields.DecimalField(decimal_places=2, max_digits=16, default=0)
+    untracked_volume_usd = fields.DecimalField(decimal_places=2, max_digits=32, default=0)
     # TVL derived in USD
-    total_value_locked_usd = fields.DecimalField(decimal_places=2, max_digits=16, default=0)
+    total_value_locked_usd = fields.DecimalField(decimal_places=2, max_digits=32, default=0)
     # TVL derived in ETH
-    total_value_locked_eth = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    total_value_locked_eth = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # TVL derived in USD untracked
-    total_value_locked_usd_untracked = fields.DecimalField(decimal_places=2, max_digits=16, default=0)
+    total_value_locked_usd_untracked = fields.DecimalField(decimal_places=2, max_digits=32, default=0)
     # TVL derived in ETH untracked
-    total_value_locked_eth_untracked = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    total_value_locked_eth_untracked = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # current owner of the factory
     owner = fields.TextField(default=ADDRESS_ZERO)
 
@@ -44,25 +44,25 @@ class Token(CachedModel):
     # token total supply
     total_supply = fields.BigIntField()
     # volume in token units
-    volume = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    volume = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # volume in derived USD
-    volume_usd = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    volume_usd = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # volume in USD even on pools with less reliable USD values
-    untracked_volume_usd = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    untracked_volume_usd = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # fees in USD
-    fees_usd = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    fees_usd = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # transactions across all pools that include this token
     tx_count = fields.BigIntField(default=0)
     # number of pools containing this token
     pool_count = fields.BigIntField(default=0)
     # liquidity across all pools in token units
-    total_value_locked = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    total_value_locked = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # liquidity across all pools in derived USD
-    total_value_locked_usd = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    total_value_locked_usd = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # TVL derived in USD untracked
-    total_value_locked_usd_untracked = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    total_value_locked_usd_untracked = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # derived price in ETH
-    derived_eth = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    derived_eth = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # pools token is in that are whitelisted for USD pricing
     whitelist_pools = fields.ArrayField(default=[])
 
@@ -89,41 +89,41 @@ class Pool(CachedModel):
     # tracker for global fee growth
     # fee_growth_global_1x128 = fields.BigIntField(default=0)
     # token0 per token1
-    token0_price = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    token0_price = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # token1 per token0
-    token1_price = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    token1_price = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # current tick
     tick = fields.BigIntField(null=True)
     # current observation index
     observation_index = fields.BigIntField(default=0)
     # all time token0 swapped
-    volume_token0 = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    volume_token0 = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # all time token1 swapped
-    volume_token1 = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    volume_token1 = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # all time USD swapped
-    volume_usd = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    volume_usd = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # all time USD swapped, unfiltered for unreliable USD pools
-    untracked_volume_usd = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    untracked_volume_usd = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # fees in USD
-    fees_usd = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    fees_usd = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # all time number of transactions
     tx_count = fields.BigIntField(default=0)
     # all time fees collected token0
-    collected_fees_token0 = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    collected_fees_token0 = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # all time fees collected token1
-    collected_fees_token1 = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    collected_fees_token1 = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # all time fees collected derived USD
-    collected_fees_usd = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    collected_fees_usd = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # total token 0 across all ticks
-    total_value_locked_token0 = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    total_value_locked_token0 = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # total token 1 across all ticks
-    total_value_locked_token1 = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    total_value_locked_token1 = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # tvl derived ETH
-    total_value_locked_eth = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    total_value_locked_eth = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # tvl USD
-    total_value_locked_usd = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    total_value_locked_usd = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # TVL derived in USD untracked
-    total_value_locked_usd_untracked = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    total_value_locked_usd_untracked = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # Fields used to help derived relationship
     liquidity_provider_count = fields.BigIntField(default=0)  # used to detect new exchanges
 
@@ -142,25 +142,25 @@ class Tick(Model):
     # how much liquidity changes when tick crossed
     liquidity_net = fields.DecimalField(decimal_places=0, max_digits=76, default=0)
     # calculated price of token0 of tick within this pool - constant
-    price0 = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    price0 = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # calculated price of token1 of tick within this pool - constant
-    price1 = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    price1 = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # lifetime volume of token0 with this tick in range
-    volume_token0 = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    volume_token0 = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # lifetime volume of token1 with this tick in range
-    volume_token1 = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    volume_token1 = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # lifetime volume in derived USD with this tick in range
-    volume_usd = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    volume_usd = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # lifetime volume in untracked USD with this tick in range
-    untracked_volume_usd = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    untracked_volume_usd = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # fees in USD
-    fees_usd = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    fees_usd = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # all time collected fees in token0
-    collected_fees_token0 = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    collected_fees_token0 = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # all time collected fees in token1
-    collected_fees_token1 = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    collected_fees_token1 = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # all time collected fees in USD
-    collected_fees_usd = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    collected_fees_usd = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # created time
     created_at_timestamp = fields.BigIntField()
     # created block
@@ -199,17 +199,17 @@ class Position(Model):
     # total position liquidity
     liquidity = fields.DecimalField(max_digits=76, decimal_places=0, default=0)
     # amount of token 0 ever deposited to position
-    deposited_token0 = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    deposited_token0 = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # amount of token 1 ever deposited to position
-    deposited_token1 = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    deposited_token1 = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # amount of token 0 ever withdrawn from position (without fees)
-    withdrawn_token0 = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    withdrawn_token0 = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # amount of token 1 ever withdrawn from position (without fees)
-    withdrawn_token1 = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    withdrawn_token1 = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # all time collected fees in token0
-    collected_fees_token0 = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    collected_fees_token0 = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # all time collected fees in token1
-    collected_fees_token1 = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    collected_fees_token1 = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # vars needed for fee computation
     # fee_growth_inside_0_last_x128 = fields.BigIntField(default=0)
     # fee_growth_inside_1_last_x128 = fields.BigIntField(default=0)
@@ -249,17 +249,17 @@ class PositionSnapshot(Model):
     # total position liquidity
     liquidity = fields.DecimalField(max_digits=76, decimal_places=0, default=0)
     # amount of token 0 ever deposited to position
-    deposited_token0 = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    deposited_token0 = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # amount of token 1 ever deposited to position
-    deposited_token1 = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    deposited_token1 = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # amount of token 0 ever withdrawn from position (without fees)
-    withdrawn_token0 = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    withdrawn_token0 = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # amount of token 1 ever withdrawn from position (without fees)
-    withdrawn_token1 = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    withdrawn_token1 = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # all time collected fees in token0
-    collected_fees_token0 = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    collected_fees_token0 = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # all time collected fees in token1
-    collected_fees_token1 = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    collected_fees_token1 = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # internal vars needed for fee computation
     # fee_growth_inside_0_last_x128 = fields.BigIntField()
     # fee_growth_inside_1_last_x128 = fields.BigIntField()
@@ -286,9 +286,9 @@ class Mint(Model):
     # amount of liquidity minted
     amount = fields.DecimalField(decimal_places=0, max_digits=76, default=0)
     # amount of token 0 minted
-    amount0 = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    amount0 = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # amount of token 1 minted
-    amount1 = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    amount1 = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # derived amount based on available prices of tokens
     amount_usd = fields.DecimalField(decimal_places=18, max_digits=76, null=True)
     # lower tick of the position
@@ -318,11 +318,11 @@ class Burn(Model):
     # amount of liquidity burned
     amount = fields.DecimalField(decimal_places=0, max_digits=76, default=0)
     # amount of token 0 burned
-    amount0 = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    amount0 = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # amount of token 1 burned
-    amount1 = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    amount1 = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # derived amount based on available prices of tokens
-    amount_usd = fields.DecimalField(decimal_places=2, max_digits=16, default=0)
+    amount_usd = fields.DecimalField(decimal_places=2, max_digits=32, default=0)
     # lower tick of position
     tick_lower = fields.BigIntField()
     # upper tick of position
@@ -350,11 +350,11 @@ class Swap(Model):
     # txn origin
     origin = fields.CharField(max_length=42)  # the EOA that initiated the txn
     # delta of token0 swapped
-    amount0 = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    amount0 = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # delta of token1 swapped
-    amount1 = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    amount1 = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # derived info
-    amount_usd = fields.DecimalField(decimal_places=2, max_digits=16, default=0)
+    amount_usd = fields.DecimalField(decimal_places=2, max_digits=32, default=0)
     # The sqrt(price) of the pool after the swap, as a Q64.96
     sqrt_price_x96 = fields.DecimalField(decimal_places=0, max_digits=76, default=0)
     # the tick after the swap
@@ -374,9 +374,9 @@ class Collect(Model):
     # owner of position collect was performed on
     owner = fields.CharField(max_length=42, null=True)
     # amount of token0 collected
-    amount0 = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    amount0 = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # amount of token1 collected
-    amount1 = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    amount1 = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # derived amount based on available prices of tokens
     amount_usd = fields.DecimalField(decimal_places=18, max_digits=76, null=True)
     # lower tick of position
@@ -400,14 +400,14 @@ class Flash(Model):
     # recipient of the flash
     recipient = fields.CharField(max_length=42)
     # amount of token0 flashed
-    amount0 = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    amount0 = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # amount of token1 flashed
-    amount1 = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    amount1 = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # derived amount based on available prices of tokens
-    amount_usd = fields.DecimalField(decimal_places=2, max_digits=16, default=0)
+    amount_usd = fields.DecimalField(decimal_places=2, max_digits=32, default=0)
     # amount token0 paid for flash
-    amount0_paid = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    amount0_paid = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # amount token1 paid for flash
-    amount1_paid = fields.DecimalField(decimal_places=18, max_digits=76, default=0)
+    amount1_paid = fields.DecimalField(decimal_places=18, max_digits=96, default=0)
     # index within the txn
     log_index = fields.BigIntField()

src/dipdup/indexes/evm_subsquid_events/index.py

@@ -155,13 +155,13 @@ async def _synchronize(self, sync_level: int) -> None:
             for handler in self._config.handlers:
                 typename = handler.contract.module_name
                 topics.add(self.topics[typename][handler.name])
+            # FIXME: This is terribly inefficient (but okay for the last mile); see advanced example in web3.py docs.
             for level in range(first_level, sync_level):
                 # NOTE: Get random one every time
                 level_logs = await self.random_node.get_logs(
                     {
                         'fromBlock': hex(level),
                         'toBlock': hex(level),
-                        'topics': tuple(topics),
                     }
                 )
                 block = await self.random_node.get_block_by_level(level)