Fix Subsquid workers (#811)

droserasprout · web-flow · commit d4f5b1430ada · 2023-08-31T13:39:50.000-03:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog], and this project adheres to [Semantic Versioning].
 
+## [Unreleased]
+
+### Fixed
+
+- evm.subsquid: Create a separate aiohttp session for each worker.
+- evm.node: Set `timestamp` field to the block timestamp.
+
 ## [7.0.0rc4] - 2023-08-23
 
 ### Added
diff --git a/pdm.lock b/pdm.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -147,7 +147,7 @@ ignore = [
 target-version = "py311"
 extend-select = ["B", "C4", "FA", "G", "I", "PTH", "Q", "RET", "RUF", "TCH", "UP"]
 flake8-quotes = { inline-quotes = "single", multiline-quotes = "double" }
-isort = { force-single-line = true, known-first-party = ["dipdup"] }
+isort = { force-single-line = true }
 
 [tool.mypy]
 python_version = "3.11"
diff --git a/requirements.dev.txt b/requirements.dev.txt
@@ -28,7 +28,7 @@ dnspython==2.4.2
 docker==6.1.3
 docutils==0.20.1
 email-validator==2.0.0.post2
-eth-abi==4.1.0
+eth-abi==4.2.0
 eth-account==0.9.0
 eth-hash==0.5.2
 eth-keyfile==0.6.1
@@ -94,21 +94,21 @@ rfc3339-validator==0.1.4
 rlp==3.0.0
 ruamel-yaml==0.17.32
 ruamel-yaml-clib==0.2.7
-ruff==0.0.285
-sentry-sdk==1.29.2
+ruff==0.0.286
+sentry-sdk==1.30.0
 setuptools==68.1.2
 six==1.16.0
 sniffio==1.3.0
 snowballstemmer==2.2.0
-sphinx==7.2.2
+sphinx==7.2.4
 sphinx-click==5.0.1
 sphinx-markdown-builder==0.6.5
 sphinxcontrib-applehelp==1.0.7
 sphinxcontrib-devhelp==1.0.5
 sphinxcontrib-htmlhelp==2.0.4
 sphinxcontrib-jsmath==1.0.1
 sphinxcontrib-qthelp==1.0.6
-sphinxcontrib-serializinghtml==1.1.8
+sphinxcontrib-serializinghtml==1.1.9
 sqlparse==0.4.4
 survey==4.5.4
 tabulate==0.9.0
@@ -121,7 +121,7 @@ typing-extensions==4.7.1
 tzlocal==5.0.1
 urllib3==2.0.4
 watchdog==3.0.0
-web3==6.8.0
+web3==6.9.0
 websocket-client==1.6.1
 websockets==10.4
 yarl==1.9.2
diff --git a/requirements.txt b/requirements.txt
@@ -23,7 +23,7 @@ cytoolz==0.12.2
 datamodel-code-generator==0.21.4
 dnspython==2.4.2
 email-validator==2.0.0.post2
-eth-abi==4.1.0
+eth-abi==4.2.0
 eth-account==0.9.0
 eth-hash==0.5.2
 eth-keyfile==0.6.1
@@ -77,7 +77,7 @@ rfc3339-validator==0.1.4
 rlp==3.0.0
 ruamel-yaml==0.17.32
 ruamel-yaml-clib==0.2.7
-sentry-sdk==1.29.2
+sentry-sdk==1.30.0
 setuptools==68.1.2
 six==1.16.0
 sniffio==1.3.0
@@ -90,6 +90,6 @@ tortoise-orm==0.19.3
 typing-extensions==4.7.1
 tzlocal==5.0.1
 urllib3==2.0.4
-web3==6.8.0
+web3==6.9.0
 websockets==10.4
 yarl==1.9.2
diff --git a/src/demo_auction/models/__init__.py b/src/demo_auction/models/__init__.py
@@ -1,6 +1,5 @@
 from enum import IntEnum
 
-
 from dipdup import fields
 from dipdup.models import Model
 
diff --git a/src/demo_big_maps/models/__init__.py b/src/demo_big_maps/models/__init__.py
@@ -1,6 +1,5 @@
 
 from dipdup import fields
-
 from dipdup.models import Model
 
 
diff --git a/src/demo_dao/models/__init__.py b/src/demo_dao/models/__init__.py
@@ -1,6 +1,5 @@
 
 from dipdup import fields
-
 from dipdup.models import Model
 
 
diff --git a/src/demo_dex/models/__init__.py b/src/demo_dex/models/__init__.py
@@ -1,7 +1,6 @@
 from enum import IntEnum
 
 from dipdup import fields
-
 from dipdup.models import Model
 
 
diff --git a/src/demo_domains/models/__init__.py b/src/demo_domains/models/__init__.py
@@ -1,6 +1,5 @@
 
 from dipdup import fields
-
 from dipdup.models import Model
 
 
diff --git a/src/demo_evm_events/handlers/on_transfer.py b/src/demo_evm_events/handlers/on_transfer.py
@@ -26,6 +26,7 @@ async def on_transfer(
         balance_update=amount,
         level=event.data.level,
     )
+    
 
 async def on_balance_update(
     address: str,
@@ -47,4 +48,4 @@ async def on_balance_update(
     holder.turnover += abs(balance_update)
     holder.tx_count += 1
     holder.last_seen = level
-    await holder.save()
+    await holder.save()
diff --git a/src/demo_factories/models/__init__.py b/src/demo_factories/models/__init__.py
@@ -1,6 +1,5 @@
 
 from dipdup import fields
-
 from dipdup.models import Model
 
 
diff --git a/src/demo_nft_marketplace/models/__init__.py b/src/demo_nft_marketplace/models/__init__.py
@@ -1,8 +1,6 @@
 from enum import IntEnum
 
-
 from dipdup import fields
-
 from dipdup.models import Model
 
 
diff --git a/src/demo_raw/models/__init__.py b/src/demo_raw/models/__init__.py
@@ -1,7 +1,6 @@
 from dipdup import fields
-
-from dipdup.models.tezos_tzkt import TzktOperationType
 from dipdup.models import Model
+from dipdup.models.tezos_tzkt import TzktOperationType
 
 
 class Operation(Model):
diff --git a/src/demo_token_transfers/models/__init__.py b/src/demo_token_transfers/models/__init__.py
@@ -1,5 +1,4 @@
 from dipdup import fields
-
 from dipdup.models import Model
 
 
diff --git a/src/demo_uniswap/abi/erc20/ERC20.json b/src/demo_uniswap/abi/erc20/ERC20.json
@@ -219,4 +219,4 @@
     "name": "Transfer",
     "type": "event"
   }
-]
+]
diff --git a/src/demo_uniswap/abi/erc20/ERC20NameBytes.json b/src/demo_uniswap/abi/erc20/ERC20NameBytes.json
@@ -14,4 +14,4 @@
     "stateMutability": "view",
     "type": "function"
   }
-]
+]
diff --git a/src/demo_uniswap/abi/erc20/ERC20SymbolBytes.json b/src/demo_uniswap/abi/erc20/ERC20SymbolBytes.json
@@ -14,4 +14,4 @@
     "stateMutability": "view",
     "type": "function"
   }
-]
+]
diff --git a/src/dipdup/datasources/evm_node.py b/src/dipdup/datasources/evm_node.py
@@ -283,7 +283,9 @@ async def _handle_subscription(self, subscription: EvmNodeSubscription, data: An
             self._head_events[level].set()
 
         elif isinstance(subscription, EvmNodeLogsSubscription):
-            logs = EvmNodeLogData.from_json(data)
+            # FIXME: Take from the head instead.
+            timestamp = int(time.time())
+            logs = EvmNodeLogData.from_json(data, timestamp)
             level = int(logs.block_number, 16)
 
             await self._head_events[level].wait()
diff --git a/src/dipdup/datasources/evm_subsquid.py b/src/dipdup/datasources/evm_subsquid.py
diff --git a/src/dipdup/indexes/evm_subsquid_events/index.py b/src/dipdup/indexes/evm_subsquid_events/index.py
diff --git a/src/dipdup/models/evm_node.py b/src/dipdup/models/evm_node.py
diff --git a/src/dipdup/models/evm_subsquid.py b/src/dipdup/models/evm_subsquid.py
diff --git a/tests/test_demos.py b/tests/test_demos.py
diff --git a/tests/test_models.py b/tests/test_models.py
diff --git a/tests/test_rollback.py b/tests/test_rollback.py

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,5 @@`
`1`	`1`
`2`	`2`	`from dipdup import fields`
`3`		`-`
`4`	`3`	`from dipdup.models import Model`
`5`	`4`
`6`	`5`