Merge pull request #4412 from opsmill/dga-20240921-rm-index-value

Remove the database index on AttributeValue value to allow attribute value to be larger than 8167 bytes

Author: dgarros

backend/infrahub/core/graph/__init__.py

@@ -1 +1 @@
-GRAPH_VERSION = 13
+GRAPH_VERSION = 14

backend/infrahub/core/graph/index.py

@@ -8,7 +8,6 @@
     IndexItem(name="node_kind", label="Node", properties=["kind"], type=IndexType.RANGE),
     IndexItem(name="attr_name", label="Attribute", properties=["name"], type=IndexType.RANGE),
     IndexItem(name="attr_uuid", label="Attribute", properties=["uuid"], type=IndexType.RANGE),
-    IndexItem(name="attr_value", label="AttributeValue", properties=["value"], type=IndexType.RANGE),
     IndexItem(name="attr_ipnet_bin", label="AttributeIPNetwork", properties=["binary_address"], type=IndexType.RANGE),
     IndexItem(name="attr_iphost_bin", label="AttributeIPHost", properties=["binary_address"], type=IndexType.RANGE),
     IndexItem(name="rel_uuid", label="Relationship", properties=["uuid"], type=IndexType.RANGE),

backend/infrahub/core/migrations/graph/__init__.py

@@ -15,6 +15,7 @@
 from .m011_remove_profile_relationship_schema import Migration011
 from .m012_convert_account_generic import Migration012
 from .m013_convert_git_password_credential import Migration013
+from .m014_remove_index_attr_value import Migration014
 
 if TYPE_CHECKING:
     from infrahub.core.root import Root
@@ -35,6 +36,7 @@
     Migration011,
     Migration012,
     Migration013,
+    Migration014,
 ]
 
 

backend/infrahub/core/migrations/graph/m014_remove_index_attr_value.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Sequence
+
+from infrahub.core.migrations.shared import MigrationResult
+from infrahub.core.query import Query  # noqa: TCH001
+from infrahub.database import DatabaseType
+from infrahub.database.constants import IndexType
+from infrahub.database.index import IndexItem
+
+from ..shared import GraphMigration
+
+if TYPE_CHECKING:
+    from infrahub.database import InfrahubDatabase
+
+
+INDEX_TO_DELETE = IndexItem(name="attr_value", label="AttributeValue", properties=["value"], type=IndexType.RANGE)
+
+
+class Migration014(GraphMigration):
+    name: str = "014_remove_index_attr_value"
+    queries: Sequence[type[Query]] = []
+    minimum_version: int = 13
+
+    async def execute(self, db: InfrahubDatabase) -> MigrationResult:
+        result = MigrationResult()
+
+        # Only execute this migration for Neo4j
+        if db.db_type != DatabaseType.NEO4J:
+            return result
+
+        async with db.start_transaction() as ts:
+            try:
+                ts.manager.index.init(nodes=[INDEX_TO_DELETE], rels=[])
+                await ts.manager.index.drop()
+            except Exception as exc:  # pylint: disable=broad-exception-caught
+                result.errors.append(str(exc))
+                return result
+
+        return result
+
+    async def validate_migration(self, db: InfrahubDatabase) -> MigrationResult:
+        result = MigrationResult()
+        return result

backend/infrahub/core/node/resource_manager/number_pool.py

@@ -42,7 +42,6 @@ async def get_resource(
 
     async def get_next(self, db: InfrahubDatabase, branch: Branch) -> int:
         query = await NumberPoolGetUsed.init(db=db, branch=branch, pool=self, branch_agnostic=True)
-
         await query.execute(db=db)
         taken = [result.get_as_optional_type("av.value", return_type=int) for result in query.results]
         next_number = find_next_free(

backend/infrahub/core/query/resource_manager.py

@@ -211,7 +211,7 @@ async def query_init(self, db: InfrahubDatabase, **kwargs: dict[str, Any]) -> No
         query = """
         MATCH (pool:%(number_pool)s { uuid: $pool_id })-[r:IS_RESERVED]->(av:AttributeValue )
         WHERE
-            av.value >= $start_range and av.value <= $end_range
+            toInteger(av.value) >= $start_range and toInteger(av.value) <= $end_range
             AND
             %(branch_filter)s
         """ % {"branch_filter": branch_filter, "number_pool": InfrahubKind.NUMBERPOOL}

backend/tests/unit/core/migrations/graph/test_014.py

@@ -0,0 +1,36 @@
+from infrahub.core.migrations.graph.m014_remove_index_attr_value import (
+    Migration014,
+)
+from infrahub.database import DatabaseType, InfrahubDatabase
+from infrahub.database.constants import IndexType
+from infrahub.database.index import IndexItem
+
+
+async def test_migration_014(
+    db: InfrahubDatabase,
+    reset_registry,
+    default_branch,
+    delete_all_nodes_in_db,
+):
+    indexes = [
+        IndexItem(name="node_uuid", label="Node", properties=["uuid"], type=IndexType.RANGE),
+        IndexItem(name="attr_value", label="AttributeValue", properties=["value"], type=IndexType.RANGE),
+    ]
+
+    db.manager.index.init(nodes=indexes, rels=[])
+    await db.manager.index.add()
+    nbr_indexes_before = len(await db.manager.index.list())
+
+    async with db.start_session() as dbs:
+        migration = Migration014()
+        execution_result = await migration.execute(db=dbs)
+        assert not execution_result.errors
+
+        validation_result = await migration.validate_migration(db=dbs)
+        assert not validation_result.errors
+
+    nbr_indexes_after = len(await db.manager.index.list())
+    if db.db_type == DatabaseType.NEO4J:
+        assert nbr_indexes_before - nbr_indexes_after == 1
+    else:
+        assert nbr_indexes_before - nbr_indexes_after == 0

backend/tests/unit/core/resource_manager/test_number_pool.py

@@ -0,0 +1,26 @@
+from infrahub.core.branch import Branch
+from infrahub.core.initialization import initialize_registry
+from infrahub.core.node import Node
+from infrahub.core.schema import SchemaRoot
+from infrahub.database import InfrahubDatabase
+from tests.helpers.schema import TICKET, load_schema
+
+
+async def test_allocate_from_number_pool(db: InfrahubDatabase, default_branch: Branch, register_core_models_schema):
+    await load_schema(db=db, schema=SchemaRoot(nodes=[TICKET]))
+    await initialize_registry(db=db)
+
+    np1 = await Node.init(db=db, schema="CoreNumberPool")
+    await np1.new(db=db, name="pool1", node="TestingTicket", node_attribute="ticket_id", start_range=1, end_range=10)
+    await np1.save(db=db)
+
+    ticket1 = await Node.init(db=db, schema=TICKET.kind)
+    await ticket1.new(db=db, title="ticket1", ticket_id={"from_pool": {"id": np1.id}})
+    await ticket1.save(db=db)
+
+    ticket2 = await Node.init(db=db, schema=TICKET.kind)
+    await ticket2.new(db=db, title="ticket2", ticket_id={"from_pool": {"id": np1.id}})
+    await ticket2.save(db=db)
+
+    assert ticket1.ticket_id.value == 1
+    assert ticket2.ticket_id.value == 2

backend/tests/unit/graphql/queries/test_resource_pool.py

@@ -626,6 +626,10 @@ async def test_number_pool_utilization(db: InfrahubDatabase, default_branch: Bra
         },
     )
 
+    assert not first.errors
+    assert not second.errors
+    assert not third.errors
+
     assert first.data
     assert second.data
     assert third.data

changelog/4399.fixed.md

@@ -0,0 +1 @@
+Removed database index in Attribute Value to attribute larger than 8167 bytes