Vastly speed up secondary apps query

Author: ddxv

adscrawler/app_stores/apple.py

@@ -229,7 +229,7 @@ def get_urls_from_html(soup: BeautifulSoup) -> dict:
     return urls
 
 
-def get_privacy_details(html: str, country: str, store_id: str) -> bool:
+def get_privacy_details(html: str, country: str, store_id: str) -> dict:
     """
     Get privacy details for an iOS app from the App Store.
 
@@ -265,7 +265,8 @@ def get_privacy_details(html: str, country: str, store_id: str) -> bool:
 
     if not data.get("data") or len(data["data"]) == 0:
         raise ValueError("App not found (404)")
-    return data["data"][0]["attributes"]["privacyDetails"]
+    privacy_details: dict = data["data"][0]["attributes"]["privacyDetails"]
+    return privacy_details
 
 
 def find_privacy_policy_id(soup: BeautifulSoup) -> str | None:
@@ -286,9 +287,10 @@ def find_privacy_policy_id(soup: BeautifulSoup) -> str | None:
                 # Find the position where "id" starts and extract everything after it
                 id_position = url.find("id")
                 if id_position != -1:
-                    id_value = url[id_position + 2 :]  # +2 to skip the "id" prefix
+                    id_value: str = url[id_position + 2 :]  # +2 to skip the "id" prefix
                     print(f"ID: {id_value}")  # This will print: 1538632801
                     return id_value
+    return None
 
 
 def get_developer_url(result: dict, urls: dict) -> str:
@@ -352,7 +354,7 @@ def scrape_app_ios(store_id: str, country: str, language: str) -> dict:
     result_dict: dict = scraper.get_app_details(
         store_id, country=country, add_ratings=True, timeout=10, lang=language
     )
-    logger.info(f"{store_id=}, {country=}, {language=} ios store scraped")
+    logger.info(f"store=2 {country=} {language=} {store_id=} ios store scraped")
     return result_dict
 
 

adscrawler/app_stores/google.py

@@ -43,7 +43,7 @@ def scrape_app_gp(store_id: str, country: str, language: str = "en") -> dict:
         country=country,
         timeout=10,
     )
-    logger.info(f"{store_id=}, {country=}, {language=} play store scraped")
+    logger.info(f"store=1 {country=} {language=} {store_id=} play store scraped")
     return result_dict
 
 

adscrawler/dbcon/queries.py

@@ -853,7 +853,7 @@ def query_store_apps_to_update(
     database_connection: PostgresCon,
     store: int,
     country_priority_group: int,
-    log_query=False,
+    log_query: bool = False,
     limit: int = 1000,
 ) -> pd.DataFrame:
     short_update_days = CONFIG["crawl-settings"].get("short_update_days", 1)
@@ -871,12 +871,6 @@ def query_store_apps_to_update(
         days=max_recrawl_days
     )
     year_ago_ts = datetime.datetime.now(tz=datetime.UTC) - datetime.timedelta(days=365)
-    rankings_mv_apps = ""
-    if check_mv_exists(database_connection, "store_apps_in_latest_rankings"):
-        # TODO: this is not included yet
-        rankings_mv_apps = (
-            " OR (sa.id in (select store_app from store_apps_in_latest_rankings))"
-        )
     params = {
         "store": store,
         "country_crawl_priority": country_priority_group,
@@ -1272,7 +1266,7 @@ def query_api_call_id_for_uuid(mitm_uuid: str, database_connection: PostgresCon)
     api_calls = query_api_calls_id_uuid_map(database_connection)
     filtered_df = api_calls[api_calls["mitm_uuid"] == mitm_uuid]
     assert filtered_df.shape[0] == 1, "Failed to find api_call_id for mitm_uuid"
-    api_call_id = filtered_df["api_call_id"].to_numpy()[0]
+    api_call_id: int = filtered_df["api_call_id"].to_numpy()[0]
     return api_call_id
 
 

adscrawler/dbcon/sql/query_apps_to_update_secondary.sql

@@ -1,4 +1,31 @@
-WITH countries_to_crawl AS (
+WITH target_apps AS (
+    SELECT
+        sa.store,
+        sa.id AS store_app,
+        sa.store_id,
+        sa.icon_url_100,
+        sa.additional_html_scraped_at,
+        sa.updated_at,
+        sa.store_last_updated,
+        agm.installs,
+        agm.rating_count
+    FROM
+        store_apps AS sa
+    LEFT JOIN app_global_metrics_latest AS agm
+        ON
+            sa.id = agm.store_app
+    WHERE
+        sa.store = :store
+        AND (
+            sa.crawl_result = 1
+            OR sa.id IN (
+                SELECT sailr.store_app
+                FROM
+                    store_apps_in_latest_rankings AS sailr
+            )
+        )
+),
+all_countries_to_crawl AS (
     SELECT
         cc.country_id,
         c.alpha2,
@@ -16,7 +43,30 @@ WITH countries_to_crawl AS (
         AND cc.enabled = TRUE
         AND cc.priority = :country_crawl_priority
 ),
-latest_crawls AS (
+oldest_country_crawls AS (
+    SELECT
+        country_id,
+        MIN(crawled_at) AS crawled_at
+    FROM
+        logging.app_country_crawls
+    GROUP BY
+        country_id
+),
+countries_to_crawl AS (
+    SELECT
+        actc.country_id,
+        actc.alpha2,
+        actc.priority
+    FROM
+        all_countries_to_crawl AS actc
+    LEFT JOIN oldest_country_crawls AS occ
+        ON
+            actc.country_id = occ.country_id
+    ORDER BY
+        occ.crawled_at NULLS FIRST
+    LIMIT 3
+),
+latest_app_crawls AS (
     SELECT DISTINCT ON
     (
         store_app,
@@ -28,14 +78,25 @@ latest_crawls AS (
         crawl_result
     FROM
         logging.app_country_crawls
+    WHERE
+        country_id IN (
+            SELECT cctc.country_id
+            FROM
+                countries_to_crawl AS cctc
+        )
+        AND store_app IN (
+            SELECT tta.store_app
+            FROM
+                target_apps AS tta
+        )
     ORDER BY
         store_app ASC,
         country_id ASC,
         crawled_at DESC
 )
 SELECT
     sa.store,
-    sa.id AS store_app,
+    sa.store_app,
     sa.store_id,
     ctc.country_id,
     ctc.alpha2 AS country_code,
@@ -45,46 +106,44 @@ SELECT
     sa.updated_at AS app_updated_at,
     lc.crawled_at AS country_crawled_at
 FROM
-    public.store_apps AS sa
+    target_apps AS sa
 CROSS JOIN countries_to_crawl AS ctc
-LEFT JOIN app_global_metrics_latest agm ON sa.id = agm.store_app
-LEFT JOIN latest_crawls AS lc
+LEFT JOIN latest_app_crawls AS lc
     ON
-        sa.id = lc.store_app
+        sa.store_app = lc.store_app
         AND ctc.country_id = lc.country_id
 WHERE
-    sa.store = :store
-    AND 
-        -- ensure it is at least a valid app to crawl many countries
-        (sa.crawl_result = 1 OR (sa.id in (select store_app from store_apps_in_latest_rankings)))
-    AND (
-            -- Long update conditions
-            (
-            lc.crawled_at <= :long_update_ts
-            AND 
-            sa.store_last_updated >= :year_ago_ts
-            )
-            -- Crawl at least once a year conditions
-            OR (
-                (lc.crawled_at <= :max_recrawl_ts
-                OR lc.crawl_result IS NULL)
-
-            )
+    -- Long update conditions
+    (
+        lc.crawled_at <= :long_update_ts
+        AND
+        sa.store_last_updated >= :year_ago_ts
+    )
+    -- Crawl at least once a year conditions
+    OR (
+        (
+            lc.crawled_at <= :max_recrawl_ts
+            OR lc.crawl_result IS NULL
+        )
     )
 ORDER BY
-    (CASE
-        WHEN lc.crawl_result IS NULL
-            THEN 0
-        ELSE 1
-    END),
-    (CASE
-        WHEN lc.crawled_at < :max_recrawl_ts
-            THEN 0
-        ELSE 1
-    END),
+    (
+        CASE
+            WHEN lc.crawl_result IS NULL
+                THEN 0
+            ELSE 1
+        END
+    ),
+    (
+        CASE
+            WHEN lc.crawled_at < :max_recrawl_ts
+                THEN 0
+            ELSE 1
+        END
+    ),
     GREATEST(
-        COALESCE(agm.installs, 0),
-        COALESCE(CAST(agm.rating_count AS bigint), 0)
+        COALESCE(sa.installs, 0),
+        COALESCE(CAST(sa.rating_count AS bigint), 0)
     )
     DESC NULLS LAST
 LIMIT :mylimit;