Prep for adding DAU / MAU / IAP Revenue estimates

Prep for adding DAU / MAU / Revenue

Author: ddxv

backend/api_app/controllers/apps.py

@@ -201,45 +201,34 @@ def process_country_group(group):
 def create_app_plot_dict(app_hist: pd.DataFrame) -> pd.DataFrame:
     """Create plot dicts for the app history with linear interpolation for missing weeks."""
     star_cols = ["one_star", "two_star", "three_star", "four_star", "five_star"]
-    metrics = ["installs", "rating", "review_count", "rating_count", *star_cols]
-    xaxis_col = "snapshot_date"
+    cumulative_metrics = ["rating", *star_cols]
+    weekly_metrics = [
+        "weekly_installs",
+        "weekly_ratings",
+        "weekly_reviews",
+        "weekly_active_users",
+        "monthly_active_users",
+        "weekly_ad_revenue",
+        "weekly_iap_revenue",
+    ]
+    xaxis_col = "week_start"
     # Convert to datetime and sort
     app_hist[xaxis_col] = pd.to_datetime(app_hist[xaxis_col])
     app_hist = app_hist.sort_values(xaxis_col)
-    app_hist = app_hist.set_index(xaxis_col)
-    # Resample to weekly frequency - this creates missing weeks with NaN
-    app_hist = app_hist.resample("W").last()
 
-    # Replace zeros with NaN for cumulative metrics (zeros are data holes, not valid values)
-    # Linear interpolation for cumulative metrics (installs, rating_count, review_count, star counts)
-    cumulative_metrics = ["installs", "rating_count", "review_count", *star_cols]
-    for metric in cumulative_metrics:
-        if metric in app_hist.columns:
-            app_hist[metric].head()
-            app_hist[metric].dtype
-            # Replace 0 with NaN (these are data holes, not valid cumulative values)
-            app_hist[metric] = app_hist[metric].replace(0, np.nan)
-            # Linear interpolation
-            app_hist[metric] = app_hist[metric].interpolate(
-                method="linear", limit_direction="forward"
-            )
-
-    # For rating (average), also replace zeros and interpolate
-    if "rating" in app_hist.columns:
-        # Replace 0 with NaN (invalid rating)
-        app_hist["rating"] = app_hist["rating"].replace(0, np.nan)
-        # Interpolate
-        app_hist["rating"] = app_hist["rating"].interpolate(
-            method="linear", limit_direction="forward"
-        )
-    app_hist = app_hist.reset_index()
-    # Calculate days between snapshots
-    app_hist["date_change"] = app_hist[xaxis_col] - app_hist[xaxis_col].shift(1)
-    app_hist["days_changed"] = app_hist["date_change"].apply(
-        lambda x: np.nan if pd.isna(x) else x.days,
-    )
     metrics_to_add = []
-    for metric in metrics:
+    for metric in weekly_metrics:
+        rate_of_change_metric = f"{metric}_rate_of_change"
+        avg_per_day_metric = f"{metric}_avg_per_day"
+        # Formula: ((new - old) / old) * 100
+        app_hist[rate_of_change_metric] = (
+            app_hist[metric] / app_hist[metric].shift(1)
+        ) * 100
+        app_hist[avg_per_day_metric] = app_hist[metric] / 7
+        metrics_to_add.append(rate_of_change_metric)
+        metrics_to_add.append(avg_per_day_metric)
+
+    for metric in cumulative_metrics:
         change_metric = f"new_{metric}"
         rate_of_change_metric = f"{metric}_rate_of_change"
         avg_per_day_metric = f"{metric}_avg_per_day"
@@ -250,14 +239,18 @@ def create_app_plot_dict(app_hist: pd.DataFrame) -> pd.DataFrame:
             (app_hist[metric] - app_hist[metric].shift(1)) / app_hist[metric].shift(1)
         ) * 100
         # Avg Per Day (daily average of the change)
-        app_hist[avg_per_day_metric] = (
-            app_hist[change_metric] / app_hist["days_changed"]
-        )
+        app_hist[avg_per_day_metric] = app_hist[change_metric] / 7
         metrics_to_add.append(change_metric)
         metrics_to_add.append(rate_of_change_metric)
         metrics_to_add.append(avg_per_day_metric)
+
+    # Include cumulative/base columns for charts (cumulative_installs, cumulative_ratings, rating, star_cols)
+    base_cols = ["cumulative_installs", "cumulative_ratings", "rating", *star_cols]
+    available_base = [c for c in base_cols if c in app_hist.columns]
     # Select final columns and drop the first row (no previous data to compare)
-    app_hist = app_hist[[xaxis_col, *metrics, *metrics_to_add]].drop(app_hist.index[0])
+    app_hist = app_hist[
+        [xaxis_col, *weekly_metrics, *available_base, *metrics_to_add]
+    ].drop(app_hist.index[0])
     # Replace infinite values with NaN
     app_hist = app_hist.replace([np.inf, -np.inf], np.nan)
     # Drop columns that are all NaN

backend/api_app/controllers/companies.py

@@ -95,15 +95,21 @@ def make_company_api_domains_dict(
         .reset_index()
     )
 
-    df["country"] = df["country"].apply(
-        lambda x: x.tolist() if isinstance(x, np.ndarray) else x
-    )
+    # Pandas 3.0 defaulting to ArrowStringArray
     df["org"] = df["org"].apply(
-        lambda x: x.tolist() if isinstance(x, np.ndarray) else x
+        lambda x: (
+            x.tolist()
+            if isinstance(x, np.ndarray) or isinstance(x, pd.arrays.ArrowStringArray)
+            else x
+        )
     )
 
     df["country"] = df["country"].apply(
-        lambda x: x.tolist() if isinstance(x, np.ndarray) else x
+        lambda x: (
+            x.tolist()
+            if isinstance(x, np.ndarray) or isinstance(x, pd.arrays.ArrowStringArray)
+            else x
+        )
     )
 
     missing_domains = [
@@ -558,16 +564,6 @@ def make_company_stats(df: pd.DataFrame) -> CompanyCategoryOverview:
         res_installs_d30["adstxt_reseller_android"],
     )
 
-    # (
-    # sdk_ios_rating_count_d30,
-    # adstxt_direct_ios_rating_count_d30,
-    # adstxt_reseller_ios_rating_count_d30,
-    # ) = (
-    #     res_rating_count_d30["sdk_ios"],
-    #     res_rating_count_d30["adstxt_direct_ios"],
-    #     res_rating_count_d30["adstxt_reseller_ios"],
-    # )
-
     sdk_total_apps = sdk_ios_total_apps + sdk_android_total_apps
     total_apps = (
         sdk_total_apps

backend/api_app/models.py

@@ -316,9 +316,6 @@ class CategoryCompanyStats:
     sdk_android_installs_d30: int = 0
     adstxt_direct_android_installs_d30: int = 0
     adstxt_reseller_android_installs_d30: int = 0
-    # sdk_ios_rating_count_d30: int = 0
-    # adstxt_direct_ios_rating_count_d30: int = 0
-    # adstxt_reseller_ios_rating_count_d30: int = 0
 
 
 @dataclass

backend/dbcon/queries.py

@@ -84,9 +84,7 @@ def get_growth_apps(
     )
     decimal_cols = [
         "installs_z_score_2w",
-        "ratings_z_score_2w",
         "installs_z_score_4w",
-        "ratings_z_score_4w",
     ]
     df[decimal_cols] = df[decimal_cols].round(2)
     return df
@@ -450,7 +448,6 @@ def get_company_stats(
                     "installs_total",
                     "installs_d30",
                     "rating_count_total",
-                    "rating_count_d30",
                 ]
             ]
             .sum()

backend/dbcon/sql/query_app_global_metrics_history.sql

@@ -1,18 +1,23 @@
 SELECT
-    agmh.snapshot_date,
-    agmh.installs,
+    agmh.week_start,
+    agmh.weekly_installs,
+    agmh.weekly_ratings,
+    agmh.weekly_reviews,
+    agmh.weekly_active_users,
+    agmh.monthly_active_users,
+    agmh.weekly_ad_revenue,
+    agmh.weekly_iap_revenue,
+    agmh.total_installs AS cumulative_installs,
+    agmh.total_ratings AS cumulative_ratings,
     agmh.rating,
-    agmh.rating_count,
-    agmh.review_count,
     agmh.one_star,
     agmh.two_star,
     agmh.three_star,
     agmh.four_star,
     agmh.five_star
-FROM app_global_metrics_history AS agmh
-inner join store_apps sa on agmh.store_app = sa.id
+FROM app_global_metrics_weekly AS agmh
+INNER JOIN store_apps AS sa ON agmh.store_app = sa.id
 WHERE
     sa.store_id = :store_id
-    AND agmh.snapshot_date >= CURRENT_DATE - INTERVAL '375 days'
-ORDER BY agmh.snapshot_date
-;
+    AND agmh.week_start >= CURRENT_DATE - INTERVAL '375 days'
+ORDER BY agmh.week_start;

backend/dbcon/sql/query_apps_crossfilter.sql

@@ -30,10 +30,9 @@ SELECT
     sao.id,
     sao.store_id,
     sao.name,
-    sao.installs_est AS installs,
+    sao.installs,
     sao.rating_count,
-    sao.installs_sum_4w_est AS installs_d30,
-    sao.ratings_sum_4w AS ratings_d30,
+    sao.installs_sum_4w AS installs_d30,
     sao.in_app_purchases,
     sao.ad_supported,
     sao.store,
@@ -55,12 +54,12 @@ WHERE
 
         :min_installs ::bigint IS NULL
         OR :min_installs = 0
-        OR sao.installs_est >= :min_installs
+        OR sao.installs >= :min_installs
     )
     AND (
 
         :max_installs ::bigint IS NULL
-        OR sao.installs_est <= :max_installs
+        OR sao.installs <= :max_installs
     )
     AND (
 
@@ -75,18 +74,18 @@ WHERE
     AND (
 
         :min_installs_d30 ::bigint IS NULL
-        OR sao.installs_sum_4w_est >= :min_installs_d30
+        OR sao.installs_sum_4w >= :min_installs_d30
     )
     AND (
 
         :max_installs_d30 ::bigint IS NULL
-        OR sao.installs_sum_4w_est <= :max_installs_d30
+        OR sao.installs_sum_4w <= :max_installs_d30
     )
     -- Exclusion check
     AND NOT EXISTS (
         SELECT 1 FROM exclude_apps AS ea
         WHERE ea.store_app = sao.id
     )
 ORDER BY
-    sao.installs_est DESC
+    sao.installs DESC
 LIMIT 100;

backend/dbcon/sql/query_company_secondary_top_apps.sql

@@ -13,7 +13,6 @@ SELECT
     ra.name,
     ra.store_id,
     ra.app_company_rank AS rank,
-    ra.rating_count_d30,
     ra.installs_d30,
     ra.sdk,
     ra.api_call,

backend/dbcon/sql/query_company_secondary_top_apps_category.sql

@@ -14,7 +14,6 @@ SELECT
     ra.name,
     ra.store_id,
     ra.app_company_category_rank AS rank,
-    ra.rating_count_d30,
     ra.installs_d30,
     ra.sdk,
     ra.api_call,

backend/dbcon/sql/query_company_top_apps.sql

@@ -15,7 +15,6 @@ SELECT
     ra.developer_name,
     ra.icon_url_100,
     ra.app_company_rank AS rank,
-    ra.rating_count_d30,
     ra.installs_d30,
     ra.sdk,
     ra.api_call,

backend/dbcon/sql/query_company_top_apps_category.sql

@@ -16,7 +16,6 @@ SELECT
     ra.developer_name,
     ra.icon_url_100,
     ra.app_company_category_rank AS rank,
-    ra.rating_count_d30,
     ra.installs_d30,
     ra.sdk,
     ra.api_call,