Production south australia 2026 fork

.gitignore

@@ -19,4 +19,6 @@ db-a-logs-federal-election-2022/
 .dccache
 .iac-data/
 
-frontend/
+frontend/
+# Snyk Security Extension - AI Rules (auto-generated)
+.github/instructions/snyk_rules.instructions.md

.vscode/settings.json

@@ -6,4 +6,5 @@
         "editor.defaultFormatter": "biomejs.biome"
     },
     "biome.lspBin": "/Users/keithmoss/Projects/GitHub/demsausage/public-redesign/node_modules/@biomejs/biome/bin/biome",
+    "snyk.advanced.autoSelectOrganization": true,
 }

django/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9-slim-bullseye
+FROM python:3.10-slim-bookworm
 LABEL maintainer="<keithamoss@gmail.com>"
 
 # Python

django/demsausage/app/sausage/elections.py

@@ -36,6 +36,10 @@ def get_elections_cache_key():
     return "elections_list"
 
 
+def get_election_stats_cache_key(electionId):
+    return f"election_{electionId}_stats"
+
+
 def get_default_election():
     return Elections.objects.filter(is_primary=True).get()
 

django/demsausage/app/sausage/loader.py

@@ -1040,6 +1040,161 @@ def write_draft_polling_places(self):
             else:
                 self.logger.error("Polling place invalid: {}".format(serialiser.errors))
 
+    def migrate_unofficial_pending_stalls(self):
+        """
+        Handles pending stalls that were submitted against unofficial (user-provided)
+        polling places (i.e. location_info is set and polling_place is None).
+
+        This happens when stalls are submitted before the first official CSV data is
+        loaded. On first load, we match each such stall's location_info geom against
+        the incoming DRAFT polling places within a 100m threshold and repoint them.
+
+        Only runs on the first polling place load (polling_places_loaded=False).
+        """
+        if self.election.polling_places_loaded is True:
+            return
+
+        queryset = Stalls.objects.filter(
+            election=self.election,
+            status=StallStatus.PENDING,
+            polling_place__isnull=True,
+            location_info__isnull=False,
+        )
+
+        count = queryset.count()
+        if count == 0:
+            self.logger.info(
+                "Unofficial Pending Stall Migration: No unofficial pending stalls to migrate"
+            )
+            return
+
+        self.logger.info(
+            "Unofficial Pending Stall Migration: Found {} unofficial pending stall(s) to migrate".format(
+                count
+            )
+        )
+
+        matched_count = 0
+        for stall in queryset:
+            coordinates = stall.location_info["geom"]["coordinates"]
+            stall_geom = Point(coordinates[0], coordinates[1], srid=4326)
+
+            matching_polling_places = self.safe_find_by_distance(
+                "Unofficial Pending Stall Migration",
+                stall_geom,
+                distance_threshold_km=0.2,
+                limit=None,
+                qs=PollingPlaces.objects.filter(
+                    election=self.election, status=PollingPlaceStatus.DRAFT
+                ),
+            )
+
+            num_matches = len(matching_polling_places)
+
+            if num_matches == 0:
+                self.logger.error(
+                    "Unofficial Pending Stall Migration: No matching polling place found within 100m for stall {} "
+                    "(User-submitted location: '{}' at '{}', {}). "
+                    "The user-submitted location may not correspond to an official polling place in this CSV. "
+                    "Action needed: adjust the distance threshold or update the database manually.".format(
+                        stall.id,
+                        stall.location_info["name"],
+                        stall.location_info["address"],
+                        stall.location_info["state"],
+                    )
+                )
+
+                # Expand to 1km to give the human something to work with, but don't auto-match.
+                nearby = list(
+                    find_by_distance(
+                        stall_geom,
+                        distance_threshold_km=1.0,
+                        limit=None,
+                        qs=PollingPlaces.objects.filter(
+                            election=self.election, status=PollingPlaceStatus.DRAFT
+                        ),
+                    )
+                )
+                if len(nearby) == 0:
+                    self.logger.error(
+                        "Unofficial Pending Stall Migration: No polling places found within 1km of stall {} either.".format(
+                            stall.id
+                        )
+                    )
+                else:
+                    for pp in nearby:
+                        self.logger.error(
+                            "Unofficial Pending Stall Migration: Nearby polling place for stall {} ({:.0f}m away, not auto-matched): "
+                            "'{}' ({}) at '{}'".format(
+                                stall.id,
+                                pp.distance.m,
+                                pp.name,
+                                pp.premises,
+                                pp.address,
+                            )
+                        )
+
+            elif num_matches > 1:
+                self.logger.error(
+                    "Unofficial Pending Stall Migration: {} polling places found within 100m for stall {} "
+                    "(User-submitted location: '{}' at '{}', {}). "
+                    "Cannot determine the correct match unambiguously.".format(
+                        num_matches,
+                        stall.id,
+                        stall.location_info["name"],
+                        stall.location_info["address"],
+                        stall.location_info["state"],
+                    )
+                )
+                for pp in matching_polling_places:
+                    self.logger.error(
+                        "Unofficial Pending Stall Migration: Candidate polling place for stall {} ({:.0f}m away): "
+                        "'{}' ({}) at '{}'".format(
+                            stall.id, pp.distance.m, pp.name, pp.premises, pp.address
+                        )
+                    )
+            else:
+                official = matching_polling_places[0]
+                self.logger.info(
+                    "Unofficial Pending Stall Migration: Stall {} matched successfully ({:.0f}m away). "
+                    "User-submitted location: '{}' at '{}', {} "
+                    "| Official polling place: '{}' ({}) at '{}'. "
+                    "Please verify this match is correct.".format(
+                        stall.id,
+                        official.distance.m,
+                        stall.location_info["name"],
+                        stall.location_info["address"],
+                        stall.location_info["state"],
+                        official.name,
+                        official.premises,
+                        official.address,
+                    )
+                )
+
+                stall.polling_place_id = official.id
+                stall.save()
+                matched_count += 1
+
+        # Validate that all unofficial pending stalls have been successfully repointed
+        unmatched = Stalls.objects.filter(
+            election=self.election,
+            status=StallStatus.PENDING,
+            polling_place__isnull=True,
+            location_info__isnull=False,
+        ).count()
+        if unmatched > 0:
+            self.logger.error(
+                "Unofficial Pending Stall Migration: {} unofficial pending stall(s) still have no polling place after migration — this shouldn't happen.".format(
+                    unmatched
+                )
+            )
+
+        self.logger.info(
+            "Unofficial Pending Stall Migration: Matched and repointed {} of {} unofficial pending stall(s)".format(
+                matched_count, count
+            )
+        )
+
     def migrate_noms(self):
         def _getDistanceThreshold(polling_place):
             threshold = 0.1
@@ -1592,6 +1747,7 @@ def run(self):
 
             with transaction.atomic():
                 self.invoke_and_bail_if_errors("write_draft_polling_places")
+                self.invoke_and_bail_if_errors("migrate_unofficial_pending_stalls")
                 self.invoke_and_bail_if_errors("migrate_noms")
                 self.invoke_and_bail_if_errors("migrate_mpps")
                 self.invoke_and_bail_if_errors("migrate")

django/demsausage/app/serializers.py

@@ -1,3 +1,5 @@
+from datetime import timedelta
+
 from demsausage.app.enums import (
     MetaPollingPlaceStatus,
     MetaPollingPlaceTaskCategory,
@@ -22,7 +24,10 @@
     Profile,
     Stalls,
 )
-from demsausage.app.sausage.elections import getGamifiedElectionStats
+from demsausage.app.sausage.elections import (
+    get_election_stats_cache_key,
+    getGamifiedElectionStats,
+)
 from demsausage.app.sausage.polling_places import find_by_distance
 from demsausage.app.schemas import noms_schema, stall_location_info_schema
 from demsausage.util import daterange, get_or_none, get_url_safe_election_name
@@ -33,8 +38,10 @@
 
 from django.contrib.auth.models import User
 from django.contrib.gis.db.models.functions import Distance
+from django.core.cache import cache
 from django.db.models import Count, Q
 from django.db.models.functions import TruncDay
+from django.utils import timezone
 
 
 class HistoricalRecordField(serializers.ListField):
@@ -220,15 +227,18 @@ def _get_subs_by_category():
                 .count()
             )
 
-            pollingPlaceNomsAddedDirectlyAndAlsoWithApprovedSubs = (
-                PollingPlaceNoms.history.filter(
-                    id__in=PollingPlaces.objects.filter(
-                        election_id=obj.id,
-                        status=PollingPlaceStatus.ACTIVE,
-                        noms__isnull=False,
-                        noms__deleted=False,
-                    ).values_list("noms_id", flat=True)
-                )
+            # Build the shared subquery once and materialise the annotated history
+            # queryset into a list so we can derive both counts in Python without
+            # hitting the database twice.
+            active_noms_ids = PollingPlaces.objects.filter(
+                election_id=obj.id,
+                status=PollingPlaceStatus.ACTIVE,
+                noms__isnull=False,
+                noms__deleted=False,
+            ).values_list("noms_id", flat=True)
+
+            noms_history_annotated = list(
+                PollingPlaceNoms.history.filter(id__in=active_noms_ids)
                 .values("id")
                 .annotate(
                     has_added_directly=Count(
@@ -247,39 +257,18 @@ def _get_subs_by_category():
                         ),
                     ),
                 )
-                .filter(has_added_directly__gt=0, has_approval__gt=0)
-                .count()
             )
 
-            pollingPlacesNomsWithApprovedSubsAndNotDirectlySourced = (
-                PollingPlaceNoms.history.filter(
-                    id__in=PollingPlaces.objects.filter(
-                        election_id=obj.id,
-                        status=PollingPlaceStatus.ACTIVE,
-                        noms__isnull=False,
-                        noms__deleted=False,
-                    ).values_list("noms_id", flat=True)
-                )
-                .values("id")
-                .annotate(
-                    has_added_directly=Count(
-                        "id",
-                        filter=Q(
-                            history_change_reason=PollingPlaceNomsChangeReason.ADDED_DIRECTLY
-                        ),
-                    ),
-                    has_approval=Count(
-                        "id",
-                        filter=Q(
-                            history_change_reason__in=[
-                                PollingPlaceNomsChangeReason.APPROVED_AUTOMATIC,
-                                PollingPlaceNomsChangeReason.APPROVED_MANUAL,
-                            ]
-                        ),
-                    ),
-                )
-                .filter(has_added_directly=0, has_approval__gt=0)
-                .count()
+            pollingPlaceNomsAddedDirectlyAndAlsoWithApprovedSubs = sum(
+                1
+                for item in noms_history_annotated
+                if item["has_added_directly"] > 0 and item["has_approval"] > 0
+            )
+
+            pollingPlacesNomsWithApprovedSubsAndNotDirectlySourced = sum(
+                1
+                for item in noms_history_annotated
+                if item["has_added_directly"] == 0 and item["has_approval"] > 0
             )
 
             return {
@@ -341,7 +330,14 @@ def _get_top_submitters():
                 .exclude(count__lte=1),
             )[0]
 
-        return {
+        is_inactive = not obj.is_active()
+
+        if is_inactive:
+            cached = cache.get(get_election_stats_cache_key(obj.id))
+            if cached is not None:
+                return cached
+
+        result = {
             "with_data": PollingPlaces.objects.filter(
                 election=obj.id, status=PollingPlaceStatus.ACTIVE
             )
@@ -350,14 +346,21 @@ def _get_top_submitters():
             "total": PollingPlaces.objects.filter(
                 election=obj.id, status=PollingPlaceStatus.ACTIVE
             ).count(),
-            "by_source": _get_by_source(),
-            "subs_by_type_and_day": _get_subs_by_type_and_day(),
+            "by_source": list(_get_by_source()),
+            "subs_by_type_and_day": list(_get_subs_by_type_and_day()),
             "subs_by_category": _get_subs_by_category(),
-            "triage_actions_by_day": _get_triage_actions_by_day(),
-            "top_submitters": _get_top_submitters(),
+            "triage_actions_by_day": list(_get_triage_actions_by_day()),
+            "top_submitters": list(_get_top_submitters()),
             "noms_changes_by_user": getGamifiedElectionStats(obj.id),
         }
 
+        if is_inactive:
+            cache.set(
+                get_election_stats_cache_key(obj.id), result, timeout=60 * 60
+            )  # 1 hour
+
+        return result
+
 
 class NomsBooleanJSONField(serializers.JSONField):
     """Serializer for JSONField -- required to create the `other` boolean flag for the GeoJSON response"""

django/pyproject.toml

@@ -5,7 +5,7 @@ description = ""
 authors = ["Keith Moss <keithamoss@gmail.com>"]
 
 [tool.poetry.dependencies]
-python = "^3.9"
+python = "^3.10"
 Django = "^4.2.16"
 psycopg2-binary = "^2.9.9"
 djangorestframework = "^3.15.2"

scrapers/sa_2025/.gitignore

@@ -0,0 +1 @@
+venv/