Merge pull request #727 from Path-of-Modifiers/27-add-support-for-finding-unidentified-uniques

27 add support for finding unidentified uniques

Author: Ivareh

src/.env

@@ -36,7 +36,7 @@ SMTP_PASSWORD=changethis
 TURNSTILE_SECRET_KEY=1x0000000000000000000000000000000AA
 
 # League
-CURRENT_SOFTCORE_LEAGUE="Phrecia"
+CURRENT_SOFTCORE_LEAGUE="Standard"
 CURRENT_HARDCORE_LEAGUE="Hardcore ${CURRENT_SOFTCORE_LEAGUE}"
 
 LEAGUE_LAUNCH_TIME=2024-07-26T19:00:00Z # ISO 8601 format. Round backwards to whole hour number

src/backend_api/app/alembic/env.py

@@ -36,6 +36,10 @@ def get_url():
 # ... etc.
 
 
+def include_object(_, name: str | None, type_, *args, **kwargs) -> bool:
+    return not (type_ == "table" and name.startswith("_"))
+
+
 def run_migrations_offline() -> None:
     """Run migrations in 'offline' mode.
 
@@ -54,7 +58,8 @@ def run_migrations_offline() -> None:
         target_metadata=target_metadata,
         literal_binds=True,
         dialect_opts={"paramstyle": "named"},
-        compare_type=False
+        compare_type=False,
+        include_object=include_object,
     )
 
     with context.begin_transaction():
@@ -77,7 +82,9 @@ def run_migrations_online() -> None:
     )
 
     with connectable.connect() as connection:
-        context.configure(connection=connection, target_metadata=target_metadata, compare_type=False)
+        context.configure(
+            connection=connection, target_metadata=target_metadata, compare_type=False
+        )
 
         with context.begin_transaction():
             context.run_migrations()

src/backend_api/app/alembic/replaceable_objects/main.py

@@ -0,0 +1,116 @@
+from typing import Generic, TypeVar
+from alembic.operations import Operations, MigrateOperation
+
+# From Alembic cookbook, with personal additions for typing:
+# https://alembic.sqlalchemy.org/en/latest/cookbook.html#replaceable-objects
+
+
+class ReplaceableObject:
+    def __init__(self, name, sqltext):
+        self.name = name
+        self.sqltext = sqltext
+
+
+class ReplaceableTrigger(ReplaceableObject):
+    def __init__(self, name, table, function, trigger):
+        self.name = name
+        self.table = table
+        self.function = function
+        self.trigger = trigger
+
+
+ObjectType = TypeVar("ObjectType", bound=ReplaceableObject)
+
+
+class ReversibleOp(MigrateOperation, Generic[ObjectType]):
+    def __init__(self, target: ObjectType):
+        self.target = target
+
+    @classmethod
+    def invoke_for_target(cls, operations: Operations, target: ObjectType):
+        op = cls(target)
+        return operations.invoke(op)
+
+    def reverse(self):
+        raise NotImplementedError()
+
+    @classmethod
+    def _get_object_from_version(cls, operations: Operations, ident):
+        version, objname = ident.split(".")
+
+        module = operations.get_context().script.get_revision(version).module
+        obj = getattr(module, objname)
+        return obj
+
+    @classmethod
+    def replace(cls, operations: Operations, target, replaces=None, replace_with=None):
+
+        if replaces:
+            old_obj = cls._get_object_from_version(operations, replaces)
+            drop_old = cls(old_obj).reverse()
+            create_new = cls(target)
+        elif replace_with:
+            old_obj = cls._get_object_from_version(operations, replace_with)
+            drop_old = cls(target).reverse()
+            create_new = cls(old_obj)
+        else:
+            raise TypeError("replaces or replace_with is required")
+
+        operations.invoke(drop_old)
+        operations.invoke(create_new)
+
+
+@Operations.register_operation("create_view", "invoke_for_target")
+@Operations.register_operation("replace_view", "replace")
+class CreateViewOp(ReversibleOp[ReplaceableObject]):
+    def reverse(self):
+        return DropViewOp(self.target)
+
+
+@Operations.register_operation("drop_view", "invoke_for_target")
+class DropViewOp(ReversibleOp[ReplaceableObject]):
+    def reverse(self):
+        return CreateViewOp(self.target)
+
+
+@Operations.register_operation("create_trigger", "invoke_for_target")
+@Operations.register_operation("replace_trigger", "replace")
+class CreateTriggerOp(ReversibleOp[ReplaceableTrigger]):
+    def reverse(self):
+        return DropTriggerOp(self.target)
+
+
+@Operations.register_operation("drop_trigger", "invoke_for_target")
+class DropTriggerOp(ReversibleOp[ReplaceableTrigger]):
+    def reverse(self):
+        return CreateTriggerOp(self.target)
+
+
+@Operations.implementation_for(CreateViewOp)
+def create_view(operations: Operations, operation: CreateViewOp):
+    operations.execute(
+        "CREATE VIEW %s AS %s" % (operation.target.name, operation.target.sqltext)
+    )
+
+
+@Operations.implementation_for(DropViewOp)
+def drop_view(operations: Operations, operation: DropViewOp):
+    operations.execute("DROP VIEW %s" % operation.target.name)
+
+
+@Operations.implementation_for(CreateTriggerOp)
+def create_trigger(operations: Operations, operation: CreateTriggerOp):
+    operations.execute(
+        "CREATE FUNCTION %s() %s" % (operation.target.name, operation.target.function)
+    )
+    operations.execute(
+        "CREATE TRIGGER %s %s" % (operation.target.name, operation.target.trigger)
+    )
+
+
+@Operations.implementation_for(DropTriggerOp)
+def drop_trigger(operations: Operations, operation: DropTriggerOp):
+    operations.execute(
+        "DROP TRIGGER {} ON {};".format(operation.target.name, operation.target.table)
+    )
+    operations.execute("DROP FUNCTION %s();" % operation.target.name)

src/backend_api/app/alembic/versions/0f3f15f56b7d_added_unidentified_item_table.py

@@ -0,0 +1,123 @@
+"""Added Unidentified Item Table
+
+Revision ID: 0f3f15f56b7d
+Revises: fa3b02812f53
+Create Date: 2025-03-04 00:21:22.039605
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = "0f3f15f56b7d"
+down_revision: Union[str, None] = "fa3b02812f53"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "unidentified_item",
+        sa.Column("name", sa.Text(), nullable=False),
+        sa.Column("itemBaseTypeId", sa.SmallInteger(), nullable=False),
+        sa.Column("createdHoursSinceLaunch", sa.SmallInteger(), nullable=False),
+        sa.Column("league", sa.Text(), nullable=False),
+        sa.Column(
+            "itemId",
+            sa.BigInteger(),
+            sa.Identity(always=True, start=1, increment=1),
+            nullable=False,
+        ),
+        sa.Column("currencyId", sa.Integer(), nullable=False),
+        sa.Column("ilvl", sa.SmallInteger(), nullable=False),
+        sa.Column("currencyAmount", sa.Float(precision=4), nullable=False),
+        sa.Column("nItems", sa.SmallInteger(), nullable=False, default=1),
+        sa.Column("identified", sa.Boolean(), nullable=False),
+        sa.Column("rarity", sa.Text(), nullable=False),
+        sa.Column("aggregated", sa.Boolean(), nullable=False, default=False),
+        sa.CheckConstraint(
+            "\n            identified IS NOT TRUE\n            ",
+            name="identified_is_false",
+        ),
+        sa.ForeignKeyConstraint(
+            ["currencyId"], ["currency.currencyId"], ondelete="RESTRICT"
+        ),
+        sa.ForeignKeyConstraint(
+            ["itemBaseTypeId"],
+            ["item_base_type.itemBaseTypeId"],
+            onupdate="CASCADE",
+            ondelete="RESTRICT",
+        ),
+        sa.PrimaryKeyConstraint("itemId"),
+    )
+    op.create_index(
+        "ix_unid_item_name_itemBaseTypeId_createdHoursSinceLaunch_league",
+        "unidentified_item",
+        ["name", "itemBaseTypeId", "createdHoursSinceLaunch", "league"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_unidentified_item_currencyId"),
+        "unidentified_item",
+        ["currencyId"],
+        unique=False,
+    )
+    op.drop_index("item_createdHoursSinceLaunch_idx", table_name="item")
+    op.create_foreign_key(
+        None,
+        "item",
+        "item_base_type",
+        ["itemBaseTypeId"],
+        ["itemBaseTypeId"],
+        onupdate="CASCADE",
+        ondelete="RESTRICT",
+    )
+    op.create_foreign_key(
+        None, "item", "currency", ["currencyId"], ["currencyId"], ondelete="RESTRICT"
+    )
+    op.drop_index(
+        "item_modifier_createdHoursSinceLaunch_idx", table_name="item_modifier"
+    )
+    op.create_foreign_key(
+        None,
+        "item_modifier",
+        "modifier",
+        ["modifierId"],
+        ["modifierId"],
+        onupdate="CASCADE",
+        ondelete="CASCADE",
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_constraint(None, "item_modifier", type_="foreignkey")
+    op.create_index(
+        "item_modifier_createdHoursSinceLaunch_idx",
+        "item_modifier",
+        [sa.text('"createdHoursSinceLaunch" DESC')],
+        unique=False,
+    )
+    op.drop_constraint(None, "item", type_="foreignkey")
+    op.drop_constraint(None, "item", type_="foreignkey")
+    op.create_index(
+        "item_createdHoursSinceLaunch_idx",
+        "item",
+        [sa.text('"createdHoursSinceLaunch" DESC')],
+        unique=False,
+    )
+    op.drop_index(
+        op.f("ix_unidentified_item_currencyId"), table_name="unidentified_item"
+    )
+    op.drop_index(
+        "ix_unid_item_name_itemBaseTypeId_createdHoursSinceLaunch_league",
+        table_name="unidentified_item",
+    )
+    op.drop_table("unidentified_item")
+    # ### end Alembic commands ###

src/backend_api/app/alembic/versions/e38727349f3f_added_unidentified_aggregation_job.py

@@ -0,0 +1,83 @@
+"""Added unidentified aggregation job
+
+Revision ID: e38727349f3f
+Revises: 0f3f15f56b7d
+Create Date: 2025-03-31 14:15:02.926141
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+from app.alembic.replaceable_objects.main import ReplaceableTrigger
+
+
+# revision identifiers, used by Alembic.
+revision: str = "e38727349f3f"
+down_revision: Union[str, None] = "0f3f15f56b7d"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+unidentified_aggregation_trigger = ReplaceableTrigger(
+    "aggregate_unidentified",
+    "unidentified_item",
+    """
+    RETURNS TRIGGER AS $aggregate_unidentified$
+        DECLARE
+            current_hour INT;
+            divine_id INT;
+            divine_value FLOAT;
+        BEGIN
+			IF EXISTS (SELECT 1 FROM unidentified_item AS unid WHERE unid."createdHoursSinceLaunch" = NEW."createdHoursSinceLaunch") THEN
+				RETURN NEW;
+			END IF;
+			
+            current_hour := (SELECT MAX(unid."createdHoursSinceLaunch") FROM unidentified_item AS unid);
+            divine_id := (SELECT MAX(cur."currencyId") FROM currency AS cur WHERE cur."tradeName"='divine');
+            divine_value := (SELECT cur."valueInChaos" FROM currency AS cur WHERE cur."currencyId"=divine_id);
+			
+
+            WITH aggregates AS (
+                SELECT name, unid."itemBaseTypeId", unid."createdHoursSinceLaunch", league, ilvl, stddev(unid."currencyAmount" * cur."valueInChaos") AS std, AVG(unid."currencyAmount" * cur."valueInChaos") AS calc_avg, COUNT(unid."itemId") AS calc_count
+                FROM unidentified_item AS unid
+                NATURAL JOIN currency as cur
+                WHERE unid."createdHoursSinceLaunch" = current_hour
+				AND NOT aggregated
+                GROUP BY name, unid."itemBaseTypeId", unid."createdHoursSinceLaunch", league, ilvl
+            ), affected_item_ids AS (
+                SELECT unid."itemId" FROM unidentified_item AS unid WHERE NOT aggregated AND unid."createdHoursSinceLaunch"=current_hour
+            )
+
+            INSERT INTO unidentified_item (name, "itemBaseTypeId", "createdHoursSinceLaunch", league, "currencyId", ilvl, "currencyAmount", "nItems", identified, rarity, aggregated)
+                SELECT name, unid."itemBaseTypeId", unid."createdHoursSinceLaunch", league, divine_id, ilvl, AVG(unid."currencyAmount" * cur."valueInChaos") / divine_value, calc_count, identified, rarity, TRUE
+                    FROM unidentified_item AS unid
+                    NATURAL JOIN aggregates
+                    NATURAL JOIN currency AS cur
+                    WHERE unid."currencyAmount" * cur."valueInChaos" <= calc_avg + 1.97 * std
+                    	AND unid."currencyAmount" * cur."valueInChaos" >= calc_avg - 1.97 * std
+                    GROUP BY name, unid."itemBaseTypeId", unid."createdHoursSinceLaunch", league, ilvl, identified, rarity, calc_count;
+			
+			DELETE FROM unidentified_item as unid
+				WHERE NOT aggregated AND unid."createdHoursSinceLaunch"=current_hour;
+			
+			RETURN NEW;
+		END;
+    $aggregate_unidentified$ LANGUAGE plpgsql;
+    """,
+    """
+    BEFORE INSERT ON unidentified_item
+	FOR EACH ROW
+	EXECUTE FUNCTION aggregate_unidentified();
+    """,
+)
+
+
+def upgrade() -> None:
+    op.create_trigger(unidentified_aggregation_trigger)
+
+
+def downgrade() -> None:
+    op.drop_trigger(unidentified_aggregation_trigger)

src/backend_api/app/api/api.py

@@ -19,6 +19,8 @@
     test_prefix,
     turnstile,
     turnstile_prefix,
+    unidentified_item,
+    unidentified_item_prefix,
 )
 
 api_router = APIRouter()
@@ -40,6 +42,11 @@
 api_router.include_router(
     item.router, prefix=f"/{item_prefix}", tags=[f"{item_prefix}s"]
 )
+api_router.include_router(
+    unidentified_item.router,
+    prefix=f"/{unidentified_item_prefix}",
+    tags=[f"{unidentified_item_prefix}s"],
+)
 api_router.include_router(
     modifier.router, prefix=f"/{modifier_prefix}", tags=[f"{modifier_prefix}s"]
 )

src/backend_api/app/api/routes/__init__.py

@@ -2,6 +2,7 @@
 from app.api.routes.item_base_type import item_base_type_prefix
 from app.api.routes.item_modifier import item_modifier_prefix
 from app.api.routes.item import item_prefix
+from app.api.routes.unidentified_item import unidentified_item_prefix
 from app.api.routes.login import login_prefix
 from app.api.routes.modifier import modifier_prefix
 from app.api.routes.plot import plot_prefix