Squash the new stats tables more often

Author: norkans7

ureport/polls/tasks.py

@@ -333,17 +333,15 @@ def recheck_poll_flow_data(org_id=None):
 
 @app.task(name="polls.polls_stats_squash")
 def polls_stats_squash():
-    from ureport.stats.models import PollEngagementDailyCount, PollStats, PollStatsCounter
+    from ureport.stats.models import PollStats
 
     r = get_valkey_connection()
-    key = "squash_stats_lock"
+    key = "squash_polls_stats_lock"
 
     lock_timeout = 60 * 60 * 2
 
     if r.get(key):
-        logger.info("Skipping squashing stats as it is still running")
+        logger.info("Skipping polls app squashing stats as it is still running")
     else:
         with r.lock(key, timeout=lock_timeout):
             PollStats.squash()
-            PollStatsCounter.squash()
-            PollEngagementDailyCount.squash()

ureport/polls/tests.py

@@ -2315,7 +2315,10 @@ def verify_counts():
             )
 
             activities = (
-                ContactActivity.objects.filter(org=self.nigeria).values("date").annotate(count__sum=Count("id")).order_by("date")
+                ContactActivity.objects.filter(org=self.nigeria)
+                .values("date")
+                .annotate(count__sum=Count("id"))
+                .order_by("date")
             )
             self.assertEqual(12, activity_counts.count())
             self.assertEqual(12, activities.count())

ureport/settings_common.py

@@ -1101,6 +1101,12 @@
         "relative": True,
         "options": {"queue": "slow"},
     },
+    "stats_counts_squash": {
+        "task": "stats.stats_counts_squash",
+        "schedule": timedelta(minutes=5),
+        "relative": True,
+        "options": {"queue": "slow"},
+    },
     "stats_activities_squash": {
         "task": "stats.squash_contact_activities_counts",
         "schedule": timedelta(minutes=15),

ureport/stats/models.py

@@ -1053,7 +1053,7 @@ def _log_stats_comparison(cls, model_name, old_stats_dict, new_stats_dict, segme
 
 
 class PollStatsCounter(BaseScopedCount):
-    squash_max_distinct = 20000
+    squash_max_distinct = 100000
     squash_over = ("org_id", "flow_result_id", "flow_result_category_id", "scope")
 
     org = models.ForeignKey(Org, on_delete=models.PROTECT, related_name="poll_stats_segments")
@@ -1075,7 +1075,7 @@ class Meta:
 
 
 class PollEngagementDailyCount(BaseDailyCount):
-    squash_max_distinct = 10000
+    squash_max_distinct = 100000
     squash_over = ("org_id", "flow_result_id", "is_responded", "scope", "day")
 
     org = models.ForeignKey(Org, on_delete=models.PROTECT, related_name="poll_engagement_stats")

ureport/stats/tasks.py

@@ -112,3 +112,20 @@ def rebuild_contacts_activities_counts():
         logger.info(
             f"Task: rebuild_contacts_activities_counts finished recalculating contact activity and refreshing contacts activities engagement stats for org {org.id} in {time.time() - start_rebuild}s"
         )
+
+
+@app.task(name="stats.stats_counts_squash")
+def stats_counts_squash():
+    from ureport.stats.models import PollEngagementDailyCount, PollStatsCounter
+
+    r = get_valkey_connection()
+    key = "squash_stats_counts_lock"
+
+    lock_timeout = 60 * 60 * 2
+
+    if r.get(key):
+        logger.info("Skipping stats app squashing stats as it is still running")
+    else:
+        with r.lock(key, timeout=lock_timeout):
+            PollStatsCounter.squash()
+            PollEngagementDailyCount.squash()