Merge pull request #51 from ddxv/main

Merge main

Author: ddxv

README.md

@@ -18,16 +18,13 @@ Note: This project is not a one click setup but feel free to reach out for help.
 - PostgreSQL: 17/18
 - Setup your database. The files here use the database name 'madrone'
 - Add a password to your default db user if you dont have one yet `ALTER USER postgres WITH PASSWORD 'xxx';`
-
 - Python environment: Python 3.13
 - Setup python environment `python3.12 -m venv .virtualenv` & `source .virtualenv/bin/activate`
 - `uv pip install -r pyproject.toml`
 - `cp example_config.toml ~/config/adscrawler/config.toml` and edit any needed values. For using all locally, the main thing that needs to be modified is the `xxx` for postgres pass and S3 host.
 - In your virtualenv, init db `python db_init.py` -> Initializes MVs, inserts 3m+ apps' store_ids from https://github.com/ddxv/appgoblin-data
-
 - Google Play App Ranks Require: NodeJS
 - `npm install --save google-play-scraper`
-
 - an S3 bucket used by app ranks, APK/IPA download, MITM
 
 ## Run

adscrawler/app_stores/apple.py

@@ -297,6 +297,7 @@ def get_developer_url(result: dict, urls: dict) -> str:
     """
     Decide if we should crawl the store html for the developer url.
     """
+    final_url: str
     should_crawl_html = False
     if "sellerUrl" not in result.keys():
         should_crawl_html = True
@@ -316,14 +317,14 @@ def get_developer_url(result: dict, urls: dict) -> str:
         if len(found_tlds) == 0:
             if "sellerUrl" not in result.keys():
                 raise Exception(f"No developer url found for {urls=}")
-            final_url: str = result["sellerUrl"]
+            final_url = result["sellerUrl"]
         elif len(found_tlds) == 1:
-            final_url: str = found_tlds[0]
+            final_url = found_tlds[0]
         else:
             logger.warning(f"Multiple developer sites found for {urls=} {found_tlds=}")
-            final_url: str = result["sellerUrl"]
+            final_url = result["sellerUrl"]
     else:
-        final_url: str = result["sellerUrl"]
+        final_url = result["sellerUrl"]
     return final_url
 
 
@@ -454,7 +455,9 @@ def clean_ios_app_df(df: pd.DataFrame) -> pd.DataFrame:
         logger.warning("Unable to parse histogram")
         df["histogram"] = None
     if "description" in df.columns:
-        df["description"] = df["description"].apply(truncate_utf8_bytes)
+        df.loc[df["description"].notna(), "description"] = df.loc[
+            df["description"].notna(), "description"
+        ].apply(truncate_utf8_bytes)
     if (
         "store_language_code" in df.columns
         and df["store_language_code"].str.len().all() == 2

adscrawler/app_stores/google.py

@@ -106,8 +106,7 @@ def clean_google_play_app_df(apps_df: pd.DataFrame) -> pd.DataFrame:
             apps_df["release_date"], format="%b %d, %Y"
         ).dt.date,
         store_last_updated=pd.to_datetime(
-            apps_df["store_last_updated"],
-            unit="s",
+            apps_df["store_last_updated"], unit="s", errors="coerce"
         ).fillna(apps_df["release_date"]),
     )
     if "developer_name" in apps_df.columns:

adscrawler/dbcon/sql/query_simplified_store_app_z_scores.sql

@@ -28,8 +28,8 @@ my_advs AS (
         ON
             ic.id = icc.company_id
     WHERE
-        vcasr.run_at > :target_week::date - interval '7 days'
-        AND vcasr.run_at <= :target_week::date
+        vcasr.run_at > :target_week ::date - interval '7 days'
+        AND vcasr.run_at <= :target_week ::date
         AND (
             hcc.category_id = 1
         )
@@ -50,10 +50,10 @@ baseline_period AS (
             s.store_app = m.advertiser_store_app_id
     WHERE
         s.week_start >= (
-            :target_week::date - interval '22 days'
+            :target_week ::date - interval '22 days'
         )
         AND s.week_start <= (
-            :target_week::date - interval '7 days'
+            :target_week ::date - interval '7 days'
         )
         AND s.country_id = 840
     GROUP BY
@@ -62,16 +62,16 @@ baseline_period AS (
 target_weeks_data AS (
     SELECT
         s.store_app,
-        :target_week::date AS target_week,
+        :target_week  ::date AS target_week,
         SUM(s.installs_diff) AS target_week_installs,
         SUM(s.rating_count_diff) AS target_week_rating_count
     FROM
         store_apps_history_weekly AS s
     WHERE
-        s.week_start >= :target_week::date
+        s.week_start >= :target_week ::date
         AND
         s.week_start < (
-            :target_week::date + interval '7 days'
+            :target_week ::date + interval '7 days'
         )
         AND s.country_id = 840
     GROUP BY
@@ -135,14 +135,13 @@ ranked_z_scores AS (
         sa.content_rating,
         sa.ad_supported,
         sa.in_app_purchases,
-        sa.editors_choice,
         sa.created_at,
         sa.updated_at,
         sa.crawl_result,
         sa.icon_url_100,
         sa.icon_url_512,
         sa.release_date,
-        sa.rating_count,
+        agm.rating_count,
         sa.featured_image_url,
         sa.phone_image_url_1,
         sa.phone_image_url_2,
@@ -157,16 +156,16 @@ ranked_z_scores AS (
                 sa.store,
                 (
                     CASE
-                        WHEN sa.store = 2 THEN 'rating'::text
-                        ELSE 'installs'::text
+                        WHEN sa.store = 2 THEN 'rating' ::text
+                        ELSE 'installs' ::text
                     END
                 )
             ORDER BY
                 (
                     CASE
                         WHEN sa.store = 2 THEN saz.ratings_z_score_1w
                         WHEN sa.store = 1 THEN saz.installs_z_score_1w
-                        ELSE NULL::numeric
+                        ELSE NULL ::numeric
                     END
                 ) DESC NULLS LAST
         ) AS rn
@@ -178,7 +177,7 @@ ranked_z_scores AS (
       LEFT JOIN app_global_metrics_latest AS agm
         ON sa.id = agm.store_app
     LEFT JOIN category_mapping AS cm ON
-        sa.category::text = cm.original_category::text
+        sa.category ::text = cm.original_category ::text
 )
 SELECT
     target_week,

adscrawler/tools/reports.py

@@ -7,8 +7,8 @@
 database_connection = get_db_connection(use_ssh_tunnel=use_tunnel)
 
 
-start_date = "2025-08-01"
-end_date = "2025-09-30"
+start_date = "2025-10-01"
+end_date = "2025-10-31"
 for week in pd.date_range(start=start_date, end=end_date, freq="W-Mon"):
     df = query_zscores(database_connection, target_week=week)
     df.to_sql(