Merge pull request #150 from ddxv/main

Add Watchlists to Account Page and New Keywords ASO dash

Author: ddxv

backend/api_app/controllers/apps.py

@@ -104,7 +104,7 @@ def create_app_country_plot_dict(app_hist: pd.DataFrame) -> pd.DataFrame:
     Processes each country independently using groupby to maintain separate time series.
     """
     star_cols = ["one_star", "two_star", "three_star", "four_star", "five_star"]
-    metrics = ["rating", "review_count", "rating_count", *star_cols]
+    metrics = ["rating", "rating_count", *star_cols]
     weekly_metrics = [
         metric
         for metric in [
@@ -143,7 +143,7 @@ def process_country_group(group):
         group = group.resample("W").last()
 
         # Replace zeros with NaN for cumulative metrics (zeros are data holes, not valid values)
-        cumulative_metrics = ["rating_count", "review_count", *star_cols]
+        cumulative_metrics = ["rating_count", *star_cols]
 
         # Metrics to turn numeric / clean
         nmetrics = [
@@ -238,7 +238,6 @@ def create_app_plot_df(app_hist: pd.DataFrame) -> pd.DataFrame:
         "weekly_installs",
         "weekly_ratings",
         "weekly_active_users",
-        "monthly_active_users",
         "weekly_ad_revenue",
         "weekly_iap_revenue",
     ]
@@ -863,7 +862,14 @@ async def request_sdk_scan(
         return {"status": "success"}
 
     @get(path="/{store_id:str}/keywords", cache=3600)
-    async def get_app_keywords(self: Self, state: State, store_id: str) -> dict:
+    async def get_app_keywords(
+        self: Self,
+        state: State,
+        store_id: str,
+        keyword_text: list[str] | None = Parameter(
+            query="keyword_text", required=False
+        ),
+    ) -> dict:
         """Handle GET request for a list of apps.
 
         Returns
@@ -872,7 +878,12 @@ async def get_app_keywords(self: Self, state: State, store_id: str) -> dict:
 
         """
         start = time.perf_counter() * 1000
-        keywords_df = get_single_app_keywords(state, store_id)
+        if keyword_text is None:
+            keyword_text = []
+
+        keywords_df = get_single_app_keywords(
+            state, store_id, keyword_texts=keyword_text
+        )
         keyword_scores = keywords_df.to_dict(orient="records")
         keywords_list = keywords_df["keyword_text"].tolist()
         keywords_dict = {"keywords": keywords_list, "keyword_scores": keyword_scores}

backend/api_app/controllers/companies.py

@@ -20,6 +20,7 @@
 from litestar import Controller, get
 from litestar.config.response_cache import CACHE_FOREVER
 from litestar.datastructures import State
+from litestar.exceptions import NotFoundException
 from litestar.response import Stream
 
 from api_app.models import (
@@ -28,6 +29,7 @@
     CompaniesOverview,
     CompanyCategoryOverview,
     CompanyDetail,
+    CompanyFollowLookup,
     CompanyPatternsDict,
     CompanyPlatformOverview,
     CompanyPubIDOverview,
@@ -49,6 +51,7 @@
     get_company_adstxt_publisher_id_apps_raw,
     get_company_adstxt_publishers_overview,
     get_company_categories_topn,
+    get_company_follow_lookup,
     get_company_sdks,
     get_company_stats,
     get_company_tree,
@@ -823,6 +826,28 @@ async def company_overview(
         logger.info(f"GET /api/companies/{company_domain} took {duration}ms")
         return overview
 
+    @get(path="/companies/{company_domain:str}/lookup", cache=3600)
+    async def company_lookup(
+        self: Self,
+        state: State,
+        company_domain: str,
+    ) -> CompanyFollowLookup:
+        """Resolve a company domain to company_id metadata for follow actions."""
+        start = time.perf_counter() * 1000
+        df = get_company_follow_lookup(state=state, company_domain=company_domain)
+        if df.empty:
+            msg = f"Company domain not found: {company_domain!r}"
+            raise NotFoundException(msg, status_code=404)
+
+        row = df.to_dict(orient="records")[0]
+        duration = round((time.perf_counter() * 1000 - start), 2)
+        logger.info(f"GET /api/companies/{company_domain}/lookup took {duration}ms")
+        return CompanyFollowLookup(
+            company_id=int(row["company_id"]),
+            company_name=str(row["company_name"]),
+            company_domain=str(row["company_domain"]),
+        )
+
     @get(
         path="/companies/{company_domain:str}/topapps",
         cache=86400,

backend/api_app/controllers/keywords.py

@@ -5,12 +5,17 @@
 
 from typing import Self
 
+import pandas as pd
 from litestar import Controller, get
 from litestar.datastructures import State
 
 from api_app.utils import extend_app_icon_url
 from config import get_logger
-from dbcon.queries import get_keyword_apps, get_keyword_details
+from dbcon.queries import (
+    get_app_keywords_history,
+    get_keyword_apps,
+    get_keyword_details,
+)
 
 logger = get_logger(__name__)
 
@@ -49,3 +54,46 @@ async def get_keyword_apps(self: Self, state: State, keyword: str) -> dict:
             "apple": {"ranks": df_ios.to_dict(orient="records")},
             "google": {"ranks": df_android.to_dict(orient="records")},
         }
+
+    @get(path="/app/{store_app_id:int}", cache=3600)
+    async def get_app_keywords(
+        self: Self, state: State, store_app_id: int, keyword_ids: list[int]
+    ) -> list[dict]:
+        """Handle GET request for app keywords history.
+
+        Returns
+        -------
+            A list of dictionary representations of the history
+
+        """
+        if not keyword_ids:
+            return []
+        logger.info(
+            f"Getting keyword history for app {store_app_id} and keywords {keyword_ids}"
+        )
+        df = get_app_keywords_history(
+            state, store_app_id=store_app_id, keyword_ids=tuple(keyword_ids)
+        )
+        if df.empty:
+            return []
+
+        # Build one row per date with keyword_id columns, then carry rank forward.
+        df["crawled_date"] = pd.to_datetime(df["crawled_date"]).dt.strftime("%Y-%m-%d")
+        pivoted = (
+            df.pivot_table(
+                index="crawled_date",
+                columns="keyword_id",
+                values="app_rank",
+                aggfunc="first",
+            )
+            .sort_index()
+            .ffill()
+        )
+
+        # Use string keys for stable JSON object fields.
+        pivoted.columns = pivoted.columns.astype(str)
+        pivoted = pivoted.reset_index()
+        pivoted = pivoted.astype(object).where(pd.notna(pivoted), None)
+        pdicts = pivoted.to_dict(orient="records")
+
+        return pdicts

backend/api_app/models.py

@@ -211,6 +211,15 @@ class CompanyDetail:
     count: int
 
 
+@dataclass
+class CompanyFollowLookup:
+    """Canonical company metadata used by follow actions."""
+
+    company_id: int
+    company_name: str
+    company_domain: str
+
+
 @dataclass
 class CompanyPubIDTotals:
     """Totals for a publisher ID."""

backend/dbcon/queries.py

@@ -845,6 +845,16 @@ def search_companies(state: State, search_input: str, limit: int = 10) -> pd.Dat
     return df
 
 
+def get_company_follow_lookup(state: State, company_domain: str) -> pd.DataFrame:
+    """Resolve company domain to canonical company metadata."""
+    df = pd.read_sql(
+        sql.company_follow_lookup,
+        state.dbcon.engine,
+        params={"company_domain": company_domain},
+    )
+    return df
+
+
 def search_apps(state: State, search_input: str, limit: int = 100) -> pd.DataFrame:
     """Search apps by term in database."""
     logger.info(f"App search: {search_input=}")
@@ -967,10 +977,21 @@ def query_apps_crossfilter(
     return df
 
 
-def get_single_app_keywords(state: State, store_id: str) -> pd.DataFrame:
+def get_single_app_keywords(
+    state: State, store_id: str, keyword_texts: list[str] | None = None
+) -> pd.DataFrame:
     """Get single app keywords."""
+    if keyword_texts is None:
+        keyword_texts = []
+
+    keyword_texts = [
+        text.strip().lower() for text in keyword_texts if text and text.strip()
+    ]
+
     df = pd.read_sql(
-        sql.single_app_keywords, state.dbcon.engine, params={"store_id": store_id}
+        sql.single_app_keywords,
+        state.dbcon.engine,
+        params={"store_id": store_id, "keyword_texts": keyword_texts},
     )
     df = df.sort_values(by="d30_best_rank", ascending=True)
     return df
@@ -1056,3 +1077,25 @@ def insert_search_query(state: State, search_term: str, user_id: int | None) ->
 
     with state.dbconwrite.engine.connect() as connection, connection.begin():
         connection.execute(sql.insert_search_query, data)
+
+
+def get_app_keywords_history(
+    state: State, store_app_id: int, keyword_ids: tuple[int, ...], days: int = 120
+) -> pd.DataFrame:
+    """Get app keyword rank history."""
+    start_date = (
+        datetime.datetime.now(datetime.UTC) - datetime.timedelta(days=days)
+    ).strftime("%Y-%m-%d")
+    query = sql.app_keywords_history.bindparams(
+        bindparam("keyword_ids", expanding=True)
+    )
+    df = pd.read_sql_query(
+        query,
+        state.dbcon.engine,
+        params={
+            "store_app_id": store_app_id,
+            "keyword_ids": keyword_ids,
+            "start_date": start_date,
+        },
+    )
+    return df

backend/dbcon/sql/query_app_country_metrics_history.sql

@@ -3,7 +3,6 @@ SELECT
     c.alpha2 AS country,
     acmh.rating,
     acmh.rating_count,
-    acmh.review_count,
     acmh.one_star,
     acmh.two_star,
     acmh.three_star,

backend/dbcon/sql/query_app_global_metrics_history.sql

@@ -3,7 +3,6 @@ SELECT
     agmh.weekly_installs,
     agmh.weekly_ratings,
     agmh.weekly_active_users,
-    agmh.monthly_active_users,
     agmh.weekly_ad_revenue,
     agmh.weekly_iap_revenue,
     agmh.total_installs AS cumulative_installs,

backend/dbcon/sql/query_app_keywords_history.sql

@@ -0,0 +1,16 @@
+SELECT
+    crawled_date,
+    store,
+    country,
+    keyword_id,
+    store_app,
+    app_rank
+FROM
+    frontend.app_keyword_ranks_daily
+WHERE
+    store_app = :store_app_id
+    -- note this may need to change to = ANY() to support psycopg3
+    AND keyword_id IN :keyword_ids
+    AND crawled_date >= :start_date
+ORDER BY
+    crawled_date DESC;

backend/dbcon/sql/query_company_follow_lookup.sql

@@ -0,0 +1,16 @@
+SELECT
+    c.id AS company_id,
+    c.name AS company_name,
+    ad.domain_name AS company_domain
+FROM domains AS qd
+LEFT JOIN adtech.company_domain_mapping AS cdm
+    ON qd.id = cdm.domain_id
+LEFT JOIN adtech.companies AS c
+    ON cdm.company_id = c.id
+LEFT JOIN domains AS ad
+    ON c.domain_id = ad.id
+WHERE qd.domain_name = :company_domain
+ORDER BY
+    CASE WHEN ad.domain_name = :company_domain THEN 0 ELSE 1 END,
+    c.id
+LIMIT 1;