Update edo2net demos, improve CI (#202)

droserasprout · web-flow · commit 1f42e3788d8e · 2022-01-12T14:17:21.000Z
* Update edonet demos * lint * xdist? * bump? * Update registrydao * Cleanup * Try to disable hasura test in docker * ok, just skip it * wrong test 👀 * 👀 * typo * drop case? * cache venv * Revert "drop case?" This reverts commit e51efbc. * Do not discover hasura * and utils? * dokole * subprocess.call? * Disable capture? * @ci_skip * simplier detect * cleanup aaand it's done * Fix hashFiles * typo * agen * ageen * CI * Reenable test
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -12,6 +12,14 @@ jobs:
           python-version: "3.8.x"
       - uses: snok/install-poetry@v1.1.7
 
+      - uses: actions/cache@v2
+        id: cache-venv
+        with:
+          path: /home/runner/.cache/pypoetry/virtualenvs
+          key: ${{ runner.os }}-venv-${{ hashFiles('**/poetry.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-venv-
+
       - name: Install project
         run: make install
       - name: Run lint
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,18 @@
 
 Please use [this](https://docs.gitlab.com/ee/development/changelog.html) document as guidelines to keep a changelog.
 
+## [unreleased]
+
+### Fixed
+
+* demos: Tezos Domains and Homebase DAO demos were updated from edo2net to mainnet contracts.
+
+### Other
+
+* ci: Cache virtual environment in GitHub Actions.
+* ci: Detect CI environment and skip tests that fail in GitHub Actions.
+* ci: Execute tests in parallel with `pytest-xdist` when possible.
+
 ## 4.0.3 - 2022-01-09
 
 ### Fixed
diff --git a/Makefile b/Makefile
@@ -30,7 +30,7 @@ mypy:
 	poetry run mypy src tests
 
 test:
-	poetry run pytest --cov-report=term-missing --cov=dipdup --cov-report=xml -v tests
+	poetry run pytest --cov-report=term-missing --cov=dipdup --cov-report=xml -n auto --dist loadscope -s -v tests
 
 cover:
 	poetry run diff-cover coverage.xml
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -57,6 +57,7 @@ testcontainers = "^3.4.1"
 types-pytz = "^2021.1.2"
 types-tabulate = "^0.8.2"
 pprofile = "^2.1.0"
+pytest-xdist = "^2.5.0"
 
 [tool.poetry.extras]
 pytezos = ["pytezos"]
diff --git a/src/demo_registrydao/dipdup.yml b/src/demo_registrydao/dipdup.yml
@@ -7,13 +7,13 @@ database:
 
 contracts:
   registry:
-    address: KT1R8AZn5KG7mkbnJ5bzMuUw2isL8tMYkDVD
+    address: KT1JytEHBVBg27n9w85tNhSWUpbTvETD6rx6
     typename: registry
 
 datasources:
   tzkt:
     kind: tzkt
-    url: ${TZKT_URL:-https://api.edo2net.tzkt.io}
+    url: ${TZKT_URL:-https://api.tzkt.io}
 
 templates:
 
diff --git a/src/demo_registrydao/docker/dipdup.env.example b/src/demo_registrydao/docker/dipdup.env.example
@@ -8,5 +8,5 @@ HASURA_GRAPHQL_ADMIN_INTERNAL_ERRORS=true
 HASURA_GRAPHQL_ENABLED_LOG_TYPES=startup, http-log, webhook-log, websocket-log, query-log
 HASURA_GRAPHQL_ADMIN_SECRET=changeme
 HASURA_GRAPHQL_UNAUTHORIZED_ROLE=user
-TZKT_URL=https://api.edo2net.tzkt.io
+TZKT_URL=https://api.tzkt.io
 ADMIN_SECRET=changeme
diff --git a/src/demo_registrydao/types/registry/storage.py b/src/demo_registrydao/types/registry/storage.py
@@ -11,110 +11,86 @@
 from pydantic import Extra
 
 
-class FreezeHistory(BaseModel):
-    class Config:
-        extra = Extra.forbid
-
-    current_period_num: str
-    current_unstaked: str
-    past_unstaked: str
-    staked: str
-
-
-class LastPeriodChange(BaseModel):
-    class Config:
-        extra = Extra.forbid
-
-    changed_on: str
-    period_num: str
-
-
 class Key(BaseModel):
     class Config:
         extra = Extra.forbid
 
-    address: str
-    nat: str
+    delegate: str
+    owner: str
 
 
-class LedgerItem(BaseModel):
+class Delegate(BaseModel):
     class Config:
         extra = Extra.forbid
 
     key: Key
-    value: str
-
-
-class MaxQuorumThreshold(BaseModel):
-    class Config:
-        extra = Extra.forbid
-
-    denominator: str
-    numerator: str
+    value: Dict[str, Any]
 
 
-class MinQuorumThreshold(BaseModel):
+class FreezeHistory(BaseModel):
     class Config:
         extra = Extra.forbid
 
-    denominator: str
-    numerator: str
+    current_stage_num: str
+    current_unstaked: str
+    past_unstaked: str
+    staked: str
 
 
-class Key1(BaseModel):
+class GovernanceToken(BaseModel):
     class Config:
         extra = Extra.forbid
 
-    operator: str
-    owner: str
+    address: str
+    token_id: str
 
 
-class Operator(BaseModel):
+class ProposalKeyListSortByLevelItem(BaseModel):
     class Config:
         extra = Extra.forbid
 
-    key: Key1
-    value: Dict[str, Any]
+    bytes: str
+    nat: str
 
 
-class ProposalKeyListSortByDateItem(BaseModel):
+class Key1(BaseModel):
     class Config:
         extra = Extra.forbid
 
-    bytes: str
-    timestamp: str
+    address: str
+    bool: bool
 
 
 class Voter(BaseModel):
     class Config:
         extra = Extra.forbid
 
-    vote_amount: str
-    vote_type: bool
-    voter_address: str
+    key: Key1
+    value: str
 
 
 class Proposals(BaseModel):
     class Config:
         extra = Extra.forbid
 
     downvotes: str
-    metadata: Dict[str, str]
-    period_num: str
+    metadata: str
     proposer: str
-    proposer_fixed_fee_in_token: str
     proposer_frozen_token: str
-    start_date: str
+    quorum_threshold: str
+    start_level: str
     upvotes: str
     voters: List[Voter]
+    voting_stage_num: str
 
 
-class QuorumThreshold(BaseModel):
+class QuorumThresholdAtCycle(BaseModel):
     class Config:
         extra = Extra.forbid
 
-    denominator: str
-    numerator: str
+    last_updated_cycle: str
+    quorum_threshold: str
+    staked: str
 
 
 class RegistryStorage(BaseModel):
@@ -124,28 +100,30 @@ class Config:
     admin: str
     custom_entrypoints: Dict[str, str]
     decision_lambda: str
+    delegates: List[Delegate]
     extra: Dict[str, str]
     fixed_proposal_fee_in_token: str
     freeze_history: Dict[str, FreezeHistory]
     frozen_token_id: str
-    last_period_change: LastPeriodChange
-    ledger: List[LedgerItem]
+    frozen_total_supply: str
+    governance_token: GovernanceToken
+    governance_total_supply: str
+    guardian: str
     max_proposals: str
-    max_quorum_threshold: MaxQuorumThreshold
-    max_votes: str
-    max_voting_period: str
+    max_quorum_change: str
+    max_quorum_threshold: str
+    max_voters: str
     metadata: Dict[str, str]
-    min_quorum_threshold: MinQuorumThreshold
-    min_voting_period: str
-    operators: List[Operator]
+    min_quorum_threshold: str
     pending_owner: str
+    period: str
     permits_counter: str
     proposal_check: str
-    proposal_key_list_sort_by_date: List[ProposalKeyListSortByDateItem]
+    proposal_expired_level: str
+    proposal_flush_level: str
+    proposal_key_list_sort_by_level: List[ProposalKeyListSortByLevelItem]
     proposals: Dict[str, Proposals]
-    quorum_threshold: QuorumThreshold
-    rejected_proposal_return_value: str
-    token_address: str
-    total_supply: Dict[str, str]
-    unfrozen_token_id: str
-    voting_period: str
+    quorum_change: str
+    quorum_threshold_at_cycle: QuorumThresholdAtCycle
+    rejected_proposal_slash_value: str
+    start_level: str
diff --git a/src/demo_tezos_domains_big_map/dipdup.yml b/src/demo_tezos_domains_big_map/dipdup.yml
@@ -6,17 +6,17 @@ database:
   path: tezos_domains_big_map.sqlite3
 
 contracts:
-  edo_name_registry:
-    address: KT1JJbWfW8CHUY95hG9iq2CEMma1RiKhMHDR
+  name_registry:
+    address: KT1GBZmSxmnKJXGMdMLbugPfLyUPmuLSMwKS
     typename: name_registry
   mainnet_name_registry:
     address: KT1GBZmSxmnKJXGMdMLbugPfLyUPmuLSMwKS
     typename: name_registry
 
 datasources:
-  tzkt_edo:
+  tzkt:
     kind: tzkt
-    url: ${TZKT_URL:-https://api.edo2net.tzkt.io}
+    url: ${TZKT_URL:-https://api.tzkt.io}
 
   tzkt_mainnet:
     kind: tzkt
@@ -35,11 +35,11 @@ templates:
         path: store.expiry_map
 
 indexes:
-  tezos_domains_big_map_edo:
+  tezos_domains_big_map:
     template: tezos_domains_big_map
     values:
-      datasource: tzkt_edo
-      name_registry: edo_name_registry
+      datasource: tzkt
+      name_registry: name_registry
 
   tezos_domains_big_map_mainnet:
     template: tezos_domains_big_map
diff --git a/src/dipdup/utils/__init__.py b/src/dipdup/utils/__init__.py
@@ -1,6 +1,7 @@
 import asyncio
 import importlib
 import logging
+import os
 import pkgutil
 import time
 import types
@@ -25,6 +26,7 @@
 from typing import Sequence
 from typing import TextIO
 from typing import TypeVar
+from unittest import skip
 
 import humps  # type: ignore
 from genericpath import isdir
@@ -167,3 +169,9 @@ def remove_prefix(text: str, prefix: str) -> str:
     if text.startswith(prefix):
         text = text[len(prefix) :]
     return text.strip('_')
+
+
+def skip_ci(fn):
+    if os.environ.get('CI'):
+        return skip('CI environment, skipping')(fn)
+    return fn
diff --git a/tests/integration_tests/registrydao.yml b/tests/integration_tests/registrydao.yml
@@ -13,7 +13,7 @@ contracts:
 datasources:
   tzkt:
     kind: tzkt
-    url: ${TZKT_URL:-https://api.edo2net.tzkt.io}
+    url: ${TZKT_URL:-https://api.tzkt.io}
 
 templates:
 
diff --git a/tests/integration_tests/test_codegen.py b/tests/integration_tests/test_codegen.py
@@ -4,14 +4,14 @@
 from os.path import join
 from shutil import rmtree
 from unittest import IsolatedAsyncioTestCase
-from unittest import skip
 
 from dipdup import __version__
 from dipdup.config import DipDupConfig
 from dipdup.dipdup import DipDup
+from dipdup.utils import skip_ci
 
 
-@skip('FIXME: GitHub')
+@skip_ci
 class CodegenTest(IsolatedAsyncioTestCase):
     async def test_codegen(self):
         for name in [
diff --git a/tests/integration_tests/test_demos.py b/tests/integration_tests/test_demos.py
@@ -5,7 +5,6 @@
 from os.path import join
 from shutil import rmtree
 from unittest import IsolatedAsyncioTestCase
-from unittest import skip
 
 from tortoise.transactions import in_transaction
 
@@ -14,10 +13,11 @@
 import demo_tezos_domains.models
 import demo_tezos_domains_big_map.models
 import demo_tzcolors.models
+from dipdup.utils import skip_ci
 from dipdup.utils.database import tortoise_wrapper
 
 
-@skip('FIXME: GitHub')
+@skip_ci
 class DemosTest(IsolatedAsyncioTestCase):
     # TODO: store cache in xdg_cache_home, keep databases and logs after last run
     def setUp(self):
@@ -79,24 +79,22 @@ async def test_tzcolors(self):
             self.assertEqual(14, auctions)
             self.assertEqual(44, bids)
 
-    @skip('edonet is dead')
     async def test_tezos_domains(self):
         self.run_dipdup('tezos_domains.yml')
 
         async with tortoise_wrapper('sqlite:///tmp/dipdup/db.sqlite3', 'demo_tezos_domains.models'):
             tlds = await demo_tezos_domains.models.TLD.filter().count()
             domains = await demo_tezos_domains.models.Domain.filter().count()
 
-            self.assertEqual(5, tlds)
-            self.assertEqual(237, domains)
+            self.assertEqual(1, tlds)
+            self.assertEqual(145, domains)
 
-    @skip('edonet is dead')
     async def test_tezos_domains_big_map(self):
         self.run_dipdup('tezos_domains_big_map.yml')
 
         async with tortoise_wrapper('sqlite:///tmp/dipdup/db.sqlite3', 'demo_tezos_domains_big_map.models'):
             tlds = await demo_tezos_domains_big_map.models.TLD.filter().count()
             domains = await demo_tezos_domains_big_map.models.Domain.filter().count()
 
-            self.assertEqual(5, tlds)
-            self.assertEqual(237, domains)
+            self.assertEqual(1, tlds)
+            self.assertEqual(145, domains)
diff --git a/tests/integration_tests/test_hasura.py b/tests/integration_tests/test_hasura.py
diff --git a/tests/integration_tests/tezos_domains.yml b/tests/integration_tests/tezos_domains.yml
diff --git a/tests/integration_tests/tezos_domains_big_map.yml b/tests/integration_tests/tezos_domains_big_map.yml
