Merge pull request #139 from ddxv/main

Add App Ranks,  Revenue & MAU to App Explorer

Author: ddxv

.pre-commit-config.yaml

@@ -1,9 +1,9 @@
 repos:
   # Ruff
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.15.0
+    rev: v0.15.4
     hooks:
-      - id: ruff
+      - id: ruff-check
         args: [--fix]
       - id: ruff-format
   # Mypy
@@ -33,7 +33,7 @@ repos:
           - prettier
           - prettier-plugin-svelte
   - repo: https://github.com/pre-commit/mirrors-eslint
-    rev: v10.0.0
+    rev: v10.0.2
     hooks:
       - id: eslint
         name: eslint
@@ -61,12 +61,9 @@ repos:
         require_serial: true
   ## SQL ##
   - repo: https://github.com/sqlfluff/sqlfluff
-    rev: 4.0.0
+    rev: 4.0.4
     hooks:
       - id: sqlfluff-lint
         args: ["--config", "backend/pyproject.toml"]
       - id: sqlfluff-fix
         args: ["--config", "backend/pyproject.toml"]
-        # Arbitrary arguments to show an example
-        # args: [--rules, "LT02,CP02"]
-        # additional_dependencies: ['<dbt-adapter>', 'sqlfluff-templater-dbt']

backend/api_app/controllers/apps.py

@@ -72,7 +72,7 @@ def api_call_dfs(state: State, store_id: str) -> pd.DataFrame:
     return df
 
 
-def get_search_results(state: State, search_term: str) -> dict:
+def get_search_results(state: State, search_term: str) -> AppGroupByStore:
     """Parse search term and return resulting AppGroup."""
     decoded_input = urllib.parse.unquote(search_term)
     decoded_input = decoded_input.strip()
@@ -81,10 +81,12 @@ def get_search_results(state: State, search_term: str) -> dict:
     df = search_apps(state, search_input=decoded_input, limit=60)
     df = extend_app_icon_url(df)
     logger.info(f"{decoded_input=} returned rows: {df.shape[0]}")
-    apple_apps_dict = df[df["store"].str.startswith("Apple")].to_dict(orient="records")
-    google_apps_dict = df[df["store"].str.startswith("Google")].to_dict(
+    apple_apps_dict: list[AppDetail] = df[df["store"].str.startswith("Apple")].to_dict(
         orient="records"
     )
+    google_apps_dict: list[AppDetail] = df[
+        df["store"].str.startswith("Google")
+    ].to_dict(orient="records")
     app_group = AppGroupByStore(
         key=f"Apps matching '{search_term}'",
         apple=AppGroup(title="Apple", apps=apple_apps_dict),
@@ -337,7 +339,7 @@ async def get_new_apps(
     @get(path="/growth/{store:int}", cache=86400)
     async def get_growth_apps(
         self: Self, state: State, store: int, app_category: str | None = None
-    ) -> list[dict]:
+    ) -> dict:
         """Handle GET request for a list of fastest growing apps.
 
         Args:
@@ -555,7 +557,7 @@ async def get_sdk_details(self: Self, state: State, store_id: str) -> SDKsDetail
 
         cats = df.loc[df["category_slug"].notna(), "category_slug"].unique().tolist()
         company_sdk_dict = {}
-        found_sdk_tlds = []
+        found_sdk_tlds: list = []
         # example: {"ad-networks":
         # {"bytedance.com":
         # {"com.bytedance.sdk":
@@ -833,7 +835,7 @@ async def request_sdk_scan(
         insert_sdk_scan_request(state, store_id, user_id)
         return {"status": "success"}
 
-    @get(path="/{store_id:str}/keywords", cache=86400)
+    @get(path="/{store_id:str}/keywords", cache=3600)
     async def get_app_keywords(self: Self, state: State, store_id: str) -> dict:
         """Handle GET request for a list of apps.
 
@@ -851,7 +853,7 @@ async def get_app_keywords(self: Self, state: State, store_id: str) -> dict:
         logger.info(f"{self.path}/{store_id}/keywords took {duration}ms")
         return keywords_dict
 
-    @get(path="/{store_id:str}/apis", cache=86400)
+    @get(path="/{store_id:str}/apis", cache=3600)
     async def get_app_apis(self: Self, state: State, store_id: str) -> dict:
         """Handle GET request for a list of apps.
 
@@ -885,6 +887,7 @@ async def get_crossfilter_apps(self: Self, state: State, data: dict) -> dict:
         require_sdk_api = bool(data.get("require_sdk_api", False))
         require_iap = bool(data.get("require_iap", False))
         require_ads = bool(data.get("require_ads", False))
+        ranking_country = data.get("ranking_country")
         mydate = data.get("mydate", "2024-01-01")
         category = data.get("category")
         store = data.get("store")
@@ -894,8 +897,6 @@ async def get_crossfilter_apps(self: Self, state: State, data: dict) -> dict:
         max_rating_count = data.get("max_rating_count")
         min_installs_d30 = data.get("min_installs_d30")
         max_installs_d30 = data.get("max_installs_d30")
-        sort_col = data.get("sort_col", "installs")
-        sort_order = data.get("sort_order", "desc")
 
         # Ensure domains are lists of strings
         if isinstance(include_domains, str):
@@ -906,12 +907,14 @@ async def get_crossfilter_apps(self: Self, state: State, data: dict) -> dict:
         # Filter out empty strings
         include_domains = [d for d in include_domains if d and isinstance(d, str)]
         exclude_domains = [d for d in exclude_domains if d and isinstance(d, str)]
+        if ranking_country is not None:
+            ranking_country = str(ranking_country).strip() or None
 
         logger.info(
             f"Crossfilter query: include={len(include_domains)} domains, "
             f"exclude={len(exclude_domains)} domains, sdk_api={require_sdk_api}, "
-            f"iap={require_iap}, ads={require_ads}, date={mydate}, "
-            f"category={category}, store={store}, sort={sort_col} {sort_order}"
+            f"iap={require_iap}, ads={require_ads}, ranking_country={ranking_country}, date={mydate}, "
+            f"category={category}, store={store}"
         )
 
         try:
@@ -922,6 +925,7 @@ async def get_crossfilter_apps(self: Self, state: State, data: dict) -> dict:
                 require_sdk_api=require_sdk_api,
                 require_iap=require_iap,
                 require_ads=require_ads,
+                ranking_country=ranking_country,
                 mydate=mydate,
                 category=category,
                 store=store,
@@ -931,8 +935,6 @@ async def get_crossfilter_apps(self: Self, state: State, data: dict) -> dict:
                 max_rating_count=max_rating_count,
                 min_installs_d30=min_installs_d30,
                 max_installs_d30=max_installs_d30,
-                sort_col=sort_col,
-                sort_order=sort_order,
             )
             apps_df = extend_app_icon_url(apps_df)
             apps_list = apps_df.to_dict(orient="records")

backend/api_app/controllers/categories.py

@@ -29,7 +29,7 @@ def category_overview(state: State) -> CategoriesOverview:
     cats["name"] = cats["category"]
     cats["name"] = (
         cats["name"]
-        .str.replace("game_", "")
+        .str.replace("game_", "Games: ")
         .str.replace("_and_", " & ")
         .str.replace("_", " ")
         .str.title()
@@ -47,7 +47,20 @@ def category_overview(state: State) -> CategoriesOverview:
 
     cats = cats.sort_values("total_apps", ascending=False)
 
-    cats["type"] = np.where(cats.id.str.contains("_game|games"), "game", "app")
+    android_games_total = cats[cats.id.str.startswith("game")]["android"].sum()
+
+    cats.loc[cats["id"] == "games", "android"] = android_games_total
+
+    cats["type"] = np.where(
+        cats.id.str.contains("_game|games", regex=True), "game", "app"
+    )
+
+    cats["sort_priority"] = 2
+    cats.loc[cats["id"] == "overall", "sort_priority"] = 0
+    cats.loc[(cats["id"] == "games") | (cats["name"] == "Games"), "sort_priority"] = 1
+
+    cats = cats.sort_values(["sort_priority", "type", "name"])
+    cats = cats.drop(columns=["sort_priority"])
 
     category_dicts = cats.to_dict(orient="records")
 

backend/api_app/controllers/companies.py

@@ -344,8 +344,6 @@ def get_overviews(
     category_overview_stats = make_companies_stats(
         df=companies_df.copy(),
         tag_source_category_app_counts=tag_source_category_app_counts,
-        category=category,
-        type_slug=type_slug,
     )
 
     companies_df = prep_companies_overview_df(state, companies_df)
@@ -396,8 +394,6 @@ def append_overall_categories(df: pd.DataFrame) -> pd.DataFrame:
 def make_companies_stats(
     df: pd.DataFrame,
     tag_source_category_app_counts: pd.DataFrame,
-    category: str | None = None,
-    type_slug: str | None = None,
 ) -> CompaniesCategoryOverview:
     """Make category sums for overview."""
     overview = CompaniesCategoryOverview()

backend/api_app/controllers/creatives.py

@@ -136,7 +136,7 @@ async def company_creatives(
         logger.info(f"{self.path}/companies/{company_domain} took {duration}ms")
         return df.to_dict(orient="records")
 
-    @get(path="/apps/{store_id: str}/monetized", cache=86400)
+    @get(path="/apps/{store_id: str}/monetized", cache=3600)
     async def monetized_creatives(self: Self, state: State, store_id: str) -> dict:
         """Handle GET request for a list of creatives used for monetization by an app.
 
@@ -197,7 +197,7 @@ async def monetized_creatives(self: Self, state: State, store_id: str) -> dict:
             "by_creative": cdf.to_dict(orient="records"),
         }
 
-    @get(path="/apps/{store_id: str}/ads", cache=86400)
+    @get(path="/apps/{store_id: str}/ads", cache=3600)
     async def advertiser_creatives(self: Self, state: State, store_id: str) -> dict:
         """Handle GET request for a list of creatives for an app.
 
@@ -296,7 +296,7 @@ async def advertiser_creatives(self: Self, state: State, store_id: str) -> dict:
             "by_creative": cdf.to_dict(orient="records"),
         }
 
-    @get(path="/apps/{store_id: str}/ads/{vhash: str}", cache=86400)
+    @get(path="/apps/{store_id: str}/ads/{vhash: str}", cache=3600)
     async def get_advertiser_creative_records(
         self: Self, state: State, store_id: str, vhash: str
     ) -> dict:

backend/api_app/models.py

@@ -29,8 +29,8 @@ class AppHistory:
 class AdsTxtEntries:
     """App ads txt entries."""
 
-    direct_entries: dict
-    reseller_entries: dict
+    direct_entries: list[dict]
+    reseller_entries: list[dict]
 
 
 @dataclass
@@ -82,7 +82,7 @@ class PlatformDeveloper:
     """Developer details for a specific platform."""
 
     developer_id: str
-    developer_name: str
+    developer_name: str | None
     developer_url: str
     pub_domain_url: str | None
     apps: AppGroup
@@ -193,7 +193,7 @@ class ParentCompanyTree:
     parent_company_domain: str | None
     queried_company_domain: str
     queried_company_name: str
-    domains: list[str]
+    domains: list[dict]
     children_companies: list[ChildrenCompanyTree]
     queried_company_logo_url: str | None = None
     parent_company_logo_url: str | None = None
@@ -442,7 +442,7 @@ class AppRankOverview:
     """
 
     countries: list[str]
-    best_ranks: dict
+    best_ranks: list[dict]
 
 
 @dataclass

backend/dbcon/queries.py

@@ -637,7 +637,7 @@ def get_category_tag_type_stats(
 
 @cache_by_params
 def get_tag_source_category_totals(
-    state: State, app_category: str | None = None, type_slug: str | None = None
+    state: State, app_category: str | None = None
 ) -> pd.DataFrame:
     """Get category totals."""
     if app_category:
@@ -918,6 +918,7 @@ def query_apps_crossfilter(
     require_sdk_api: bool = False,
     require_iap: bool = False,
     require_ads: bool = False,
+    ranking_country: str | None = None,
     mydate: str = "2024-01-01",
     category: str | None = None,
     store: int | None = None,
@@ -927,38 +928,21 @@ def query_apps_crossfilter(
     max_rating_count: int | None = None,
     min_installs_d30: int | None = None,
     max_installs_d30: int | None = None,
-    sort_col: str = "installs",
-    sort_order: str = "desc",
 ) -> pd.DataFrame:
     """Query apps for analytics dashboard."""
     # Ensure domains are lists, not None
     include_domains = include_domains or []
     exclude_domains = exclude_domains or []
 
-    # Return empty dataframe if no include domains specified
-    if not include_domains:
-        return pd.DataFrame()
+    if category == "games":
+        category = "game%"
 
     # Parse date with fallback
     try:
         parsed_date = datetime.datetime.strptime(mydate, "%Y-%m-%d").date()
     except (ValueError, TypeError):
         parsed_date = datetime.date(2024, 1, 1)
 
-    # Validate sort_col to prevent injection (though utilizing param binding in SQL for logic selection)
-    allowed_sort_cols = [
-        "installs",
-        "rating_count",
-        "installs_d30",
-        "review_count",
-        "rating",
-    ]
-    if sort_col not in allowed_sort_cols:
-        sort_col = "installs"
-
-    if sort_order not in ["asc", "desc"]:
-        sort_order = "desc"
-
     df = pd.read_sql(
         sql.apps_crossfilter,
         state.dbcon.engine,
@@ -968,6 +952,7 @@ def query_apps_crossfilter(
             "require_sdk_api": bool(require_sdk_api),
             "require_iap": bool(require_iap),
             "require_ads": bool(require_ads),
+            "ranking_country": ranking_country,
             "mydate": parsed_date,
             "category": category,
             "store": store,
@@ -977,8 +962,6 @@ def query_apps_crossfilter(
             "max_rating_count": max_rating_count,
             "min_installs_d30": min_installs_d30,
             "max_installs_d30": max_installs_d30,
-            # "sort_col": sort_col,
-            # "sort_order": sort_order,
         },
     )
     return df

backend/dbcon/sql/query_adtech_categories.sql

@@ -1,3 +1,6 @@
-SELECT id, name, url_slug
+SELECT
+    id,
+    name,
+    url_slug
 FROM
     adtech.categories;