Add Codspeed performance benchmarks (#1831)

* Add Codspeed

* Add create and get tests with bigger model

* Use Python 3.12

* Skip benchmarks during regular tests

Author: henadzit

.github/workflows/codspeed.yml

@@ -0,0 +1,35 @@
+name: CodSpeed
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+  # `workflow_dispatch` allows CodSpeed to trigger backtest
+  # performance analysis in order to generate initial data.
+  workflow_dispatch:
+
+jobs:
+  benchmarks:
+    name: Run benchmarks
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          # 3.12 is the minimum reqquired version for profiling enabled
+          python-version: "3.12"
+
+      - name: Install and configure Poetry
+        run: |
+          pip install -U pip poetry
+          poetry config virtualenvs.create false
+
+      - name: Install dependencies
+        run: make build
+
+      - name: Run benchmarks
+        uses: CodSpeedHQ/action@v3
+        with:
+          token: ${{ secrets.CODSPEED_TOKEN }}
+          run: pytest tests/benchmarks --codspeed

poetry.lock

@@ -64,6 +64,7 @@ coveralls = "*"
 pytest = "*"
 pytest-xdist = "*"
 pytest-cov = "*"
+pytest-codspeed = { version = "*", python = "^3.9" }
 # Pypi
 twine = "*"
 # Sample integration - Quart

pyproject.toml

@@ -0,0 +1,83 @@
+from __future__ import annotations
+
+import asyncio
+from decimal import Decimal
+import random
+
+import pytest
+
+from tests.testmodels import BenchmarkFewFields, BenchmarkManyFields
+from tortoise.contrib.test import _restore_default, truncate_all_models
+
+
+@pytest.fixture(scope="function", autouse=True)
+def setup_database():
+    _restore_default()
+    yield
+    asyncio.get_event_loop().run_until_complete(truncate_all_models())
+
+
+@pytest.fixture(scope="module", autouse=True)
+def skip_if_codspeed_not_enabled(request):
+    if not request.config.getoption("--codspeed", default=None):
+        pytest.skip("codspeed is not enabled")
+
+
+@pytest.fixture
+def few_fields_benchmark_dataset() -> list[BenchmarkFewFields]:
+    async def _create() -> list[BenchmarkFewFields]:
+        res = []
+        for _ in range(100):
+            level = random.randint(0, 100)  # nosec
+            res.append(await BenchmarkFewFields.create(level=level, text="test"))
+        return res
+
+    return asyncio.get_event_loop().run_until_complete(_create())
+
+
+@pytest.fixture
+def many_fields_benchmark_dataset() -> list[BenchmarkManyFields]:
+    async def _create() -> list[BenchmarkManyFields]:
+        res = []
+        for _ in range(100):
+            res.append(
+                await BenchmarkManyFields.create(
+                    level=random.randint(0, 100),  # nosec
+                    text="test",
+                    col_float1=2.2,
+                    col_smallint1=2,
+                    col_int1=2000000,
+                    col_bigint1=99999999,
+                    col_char1="value1",
+                    col_text1="Moo,Foo,Baa,Waa,Moo,Foo,Baa,Waa,Moo,Foo,Baa,Waa",
+                    col_decimal1=Decimal("2.2"),
+                    col_json1={"a": 1, "b": "b", "c": [2], "d": {"e": 3}, "f": True},
+                    col_float2=0.2,
+                    col_smallint2=None,
+                    col_int2=22,
+                    col_bigint2=None,
+                    col_char2=None,
+                    col_text2=None,
+                    col_decimal2=None,
+                    col_json2=None,
+                    col_float3=2.2,
+                    col_smallint3=2,
+                    col_int3=2000000,
+                    col_bigint3=99999999,
+                    col_char3="value1",
+                    col_text3="Moo,Foo,Baa,Waa,Moo,Foo,Baa,Waa,Moo,Foo,Baa,Waa",
+                    col_decimal3=Decimal("2.2"),
+                    col_json3={"a": 1, "b": 2, "c": [2]},
+                    col_float4=0.00004,
+                    col_smallint4=None,
+                    col_int4=4,
+                    col_bigint4=99999999000000,
+                    col_char4="value4",
+                    col_text4="AAAAAAAA",
+                    col_decimal4=None,
+                    col_json4=None,
+                )
+            )
+        return res
+
+    return asyncio.get_event_loop().run_until_complete(_create())

tests/benchmarks/conftest.py

@@ -0,0 +1,63 @@
+import asyncio
+from decimal import Decimal
+import random
+
+from tests.testmodels import BenchmarkFewFields, BenchmarkManyFields
+
+
+def test_create_few_fields(benchmark):
+    loop = asyncio.get_event_loop()
+
+    @benchmark
+    def bench():
+        async def _bench():
+            level = random.randint(0, 100)  # nosec
+            await BenchmarkFewFields.create(level=level, text="test")
+
+        loop.run_until_complete(_bench())
+
+
+def test_create_many_fields(benchmark):
+    loop = asyncio.get_event_loop()
+
+    @benchmark
+    def bench():
+        async def _bench():
+            await BenchmarkManyFields.create(
+                level=random.randint(0, 100),  # nosec
+                text="test",
+                col_float1=2.2,
+                col_smallint1=2,
+                col_int1=2000000,
+                col_bigint1=99999999,
+                col_char1="value1",
+                col_text1="Moo,Foo,Baa,Waa,Moo,Foo,Baa,Waa,Moo,Foo,Baa,Waa",
+                col_decimal1=Decimal("2.2"),
+                col_json1={"a": 1, "b": "b", "c": [2], "d": {"e": 3}, "f": True},
+                col_float2=0.2,
+                col_smallint2=None,
+                col_int2=22,
+                col_bigint2=None,
+                col_char2=None,
+                col_text2=None,
+                col_decimal2=None,
+                col_json2=None,
+                col_float3=2.2,
+                col_smallint3=2,
+                col_int3=2000000,
+                col_bigint3=99999999,
+                col_char3="value1",
+                col_text3="Moo,Foo,Baa,Waa,Moo,Foo,Baa,Waa,Moo,Foo,Baa,Waa",
+                col_decimal3=Decimal("2.2"),
+                col_json3={"a": 1, "b": 2, "c": [2]},
+                col_float4=0.00004,
+                col_smallint4=None,
+                col_int4=4,
+                col_bigint4=99999999000000,
+                col_char4="value4",
+                col_text4="AAAAAAAA",
+                col_decimal4=None,
+                col_json4=None,
+            )
+
+        loop.run_until_complete(_bench())

tests/benchmarks/test_create.py

@@ -0,0 +1,16 @@
+import asyncio
+import random
+
+from tests.testmodels import BenchmarkFewFields
+
+
+def test_filter_few_fields(benchmark, few_fields_benchmark_dataset):
+    loop = asyncio.get_event_loop()
+    levels = list(set([o.level for o in few_fields_benchmark_dataset]))
+
+    @benchmark
+    def bench():
+        async def _bench():
+            await BenchmarkFewFields.filter(level__in=random.sample(levels, 5)).limit(5)
+
+        loop.run_until_complete(_bench())

tests/benchmarks/test_filter.py

@@ -0,0 +1,32 @@
+import asyncio
+import random
+
+from tests.testmodels import BenchmarkFewFields, BenchmarkManyFields
+
+
+def test_get_few_fields(benchmark, few_fields_benchmark_dataset):
+    loop = asyncio.get_event_loop()
+    minid = min(o.id for o in few_fields_benchmark_dataset)
+    maxid = max(o.id for o in few_fields_benchmark_dataset)
+
+    @benchmark
+    def bench():
+        async def _bench():
+            randid = random.randint(minid, maxid)  # nosec
+            await BenchmarkFewFields.get(id=randid)
+
+        loop.run_until_complete(_bench())
+
+
+def test_get_many_fields(benchmark, many_fields_benchmark_dataset):
+    loop = asyncio.get_event_loop()
+    minid = min(o.id for o in many_fields_benchmark_dataset)
+    maxid = max(o.id for o in many_fields_benchmark_dataset)
+
+    @benchmark
+    def bench():
+        async def _bench():
+            randid = random.randint(minid, maxid)  # nosec
+            await BenchmarkManyFields.get(id=randid)
+
+        loop.run_until_complete(_bench())

tests/benchmarks/test_get.py

@@ -998,3 +998,55 @@ class CallableDefault(Model):
     id = fields.IntField(primary_key=True)
     callable_default = fields.CharField(max_length=32, default=callable_default)
     async_default = fields.CharField(max_length=32, default=async_callable_default)
+
+
+class BenchmarkFewFields(Model):
+    timestamp = fields.DatetimeField(auto_now_add=True)
+    level = fields.SmallIntField(index=True)
+    text = fields.CharField(max_length=255)
+
+
+class BenchmarkManyFields(Model):
+    timestamp = fields.DatetimeField(auto_now_add=True)
+    level = fields.SmallIntField(index=True)
+    text = fields.CharField(max_length=255)
+
+    col_float1 = fields.FloatField(default=2.2)
+    col_smallint1 = fields.SmallIntField(default=2)
+    col_int1 = fields.IntField(default=2000000)
+    col_bigint1 = fields.BigIntField(default=99999999)
+    col_char1 = fields.CharField(max_length=255, default="value1")
+    col_text1 = fields.TextField(default="Moo,Foo,Baa,Waa,Moo,Foo,Baa,Waa,Moo,Foo,Baa,Waa")
+    col_decimal1 = fields.DecimalField(12, 8, default=Decimal("2.2"))
+    col_json1 = fields.JSONField[dict](
+        default={"a": 1, "b": "b", "c": [2], "d": {"e": 3}, "f": True}
+    )
+
+    col_float2 = fields.FloatField(null=True)
+    col_smallint2 = fields.SmallIntField(null=True)
+    col_int2 = fields.IntField(null=True)
+    col_bigint2 = fields.BigIntField(null=True)
+    col_char2 = fields.CharField(max_length=255, null=True)
+    col_text2 = fields.TextField(null=True)
+    col_decimal2 = fields.DecimalField(12, 8, null=True)
+    col_json2 = fields.JSONField[dict](null=True)
+
+    col_float3 = fields.FloatField(default=2.2)
+    col_smallint3 = fields.SmallIntField(default=2)
+    col_int3 = fields.IntField(default=2000000)
+    col_bigint3 = fields.BigIntField(default=99999999)
+    col_char3 = fields.CharField(max_length=255, default="value1")
+    col_text3 = fields.TextField(default="Moo,Foo,Baa,Waa,Moo,Foo,Baa,Waa,Moo,Foo,Baa,Waa")
+    col_decimal3 = fields.DecimalField(12, 8, default=Decimal("2.2"))
+    col_json3 = fields.JSONField[dict](
+        default={"a": 1, "b": "b", "c": [2], "d": {"e": 3}, "f": True}
+    )
+
+    col_float4 = fields.FloatField(null=True)
+    col_smallint4 = fields.SmallIntField(null=True)
+    col_int4 = fields.IntField(null=True)
+    col_bigint4 = fields.BigIntField(null=True)
+    col_char4 = fields.CharField(max_length=255, null=True)
+    col_text4 = fields.TextField(null=True)
+    col_decimal4 = fields.DecimalField(12, 8, null=True)
+    col_json4 = fields.JSONField[dict](null=True)

tests/testmodels.py

@@ -100,6 +100,16 @@ def _restore_default() -> None:
     Tortoise._inited = True
 
 
+async def truncate_all_models() -> None:
+    # TODO: This is a naive implementation: Will fail to clear M2M and non-cascade foreign keys
+    for app in Tortoise.apps.values():
+        for model in app.values():
+            quote_char = model._meta.db.query_class._builder().QUOTE_CHAR
+            await model._meta.db.execute_script(
+                f"DELETE FROM {quote_char}{model._meta.db_table}{quote_char}"  # nosec
+            )
+
+
 def initializer(
     modules: Iterable[Union[str, ModuleType]],
     db_url: Optional[str] = None,
@@ -287,13 +297,7 @@ async def _setUpDB(self) -> None:
 
     async def _tearDownDB(self) -> None:
         _restore_default()
-        # TODO: This is a naive implementation: Will fail to clear M2M and non-cascade foreign keys
-        for app in Tortoise.apps.values():
-            for model in app.values():
-                quote_char = model._meta.db.query_class._builder().QUOTE_CHAR
-                await model._meta.db.execute_script(
-                    f"DELETE FROM {quote_char}{model._meta.db_table}{quote_char}"  # nosec
-                )
+        await truncate_all_models()
         await super()._tearDownDB()