Merge pull request #277 from cmu-delphi/development

Removed caching, made table stats async

Author: dmytrotsko

src/indicatorsets/urls.py

@@ -1,7 +1,7 @@
 from django.urls import path
 from django.urls.resolvers import URLPattern
 from indicatorsets.views import (IndicatorSetListView, epivis,
-                                 generate_export_data_url, preview_data, create_query_code, get_available_geos)
+                                 generate_export_data_url, preview_data, create_query_code, get_available_geos, get_related_indicators_json)
 
 urlpatterns: list[URLPattern] = [
     path("", IndicatorSetListView.as_view(), name="indicatorsets"),
@@ -10,4 +10,5 @@
     path("preview_data/", preview_data, name="preview_data"),
     path("create_query_code/", create_query_code, name="create_query_code"),
     path("get_available_geos/", get_available_geos, name="get_available_geos"),
+    path("get_related_indicators/", get_related_indicators_json, name="get_related_indicators"),
 ]

src/indicatorsets/utils.py

@@ -6,7 +6,6 @@
 
 import requests
 from django.conf import settings
-from django.core.cache import cache
 from django.http import JsonResponse
 from epiweeks import Week
 from delphi_utils import get_structured_logger
@@ -815,18 +814,14 @@ def get_num_locations_from_meta(indicators):
     indicators = set(
         (indicator["source"], indicator["name"]) for indicator in indicators
     )
-    metadata = cache.get("covidcast_meta")
-    if metadata is None:
-        try:
-            metadata = requests.get(
-                f"{settings.EPIDATA_URL}covidcast_meta/", timeout=5
-            ).json()["epidata"]
-            # Cache for 24 hours
-            cache.set("covidcast_meta", metadata, 60 * 60 * 24)
-        except Exception as e:
-            print(f"Error fetching covidcast metadata: {e}")
-            return 0
-    for r in metadata:
-        if (r["data_source"], r["signal"]) in indicators:
-            timeseries_count += r["num_locations"]
+    try:
+        metadata = requests.get(
+            f"{settings.EPIDATA_URL}covidcast_meta/", timeout=5
+        ).json()["epidata"]
+        for r in metadata:
+            if (r["data_source"], r["signal"]) in indicators:
+                timeseries_count += r["num_locations"]
+    except Exception as e:
+        print(f"Error fetching covidcast metadata: {e}")
+        return 0
     return timeseries_count

src/indicatorsets/views.py

@@ -50,6 +50,61 @@
 HEADER_DESCRIPTION = "Discover, display and download real-time infectious disease indicators (time series) that track a variety of pathogens, diseases and syndromes in a variety of locations (primarily within the USA). Browse the list, or filter it first by locations and pathogens of interest, by surveillance categories, and more. Expand any row to expose and select from a set of related indicators, then hit 'Show Selected Indicators' at bottom to plot or export your selected indicators, or to generate code snippets to retrieve them from the Delphi Epidata API. Most indicators are served from the Delphi Epidata real-time repository, but some may be available only from third parties or may require prior approval."
 
 
+def get_related_indicators(queryset, indicator_set_ids: list):
+    related_indicators = []
+    for indicator in queryset.filter(indicator_set__id__in=indicator_set_ids):
+        related_indicators.append(
+            {
+                "id": indicator.id,
+                "display_name": (
+                    indicator.get_display_name if indicator.get_display_name else ""
+                ),
+                "member_name": (
+                    indicator.member_name if indicator.member_name else ""
+                ),
+                "member_short_name": (
+                    indicator.member_short_name
+                    if indicator.member_short_name
+                    else ""
+                ),
+                "name": indicator.name if indicator.name else "",
+                "indicator_set": (
+                    indicator.indicator_set.id if indicator.indicator_set else ""
+                ),
+                "indicator_set_name": (
+                    indicator.indicator_set.name if indicator.indicator_set else ""
+                ),
+                "indicator_set_short_name": (
+                    indicator.indicator_set.short_name
+                    if indicator.indicator_set
+                    else ""
+                ),
+                "endpoint": (
+                    indicator.indicator_set.epidata_endpoint
+                    if indicator.indicator_set
+                    else ""
+                ),
+                "source": indicator.source.name if indicator.source else "",
+                "time_type": indicator.time_type if indicator.time_type else "",
+                "description": (
+                    indicator.description if indicator.description else ""
+                ),
+                "member_description": (
+                    indicator.member_description
+                    if indicator.member_description
+                    else indicator.description
+                ),
+                "restricted": (
+                    indicator.indicator_set.dua_required
+                    if indicator.indicator_set
+                    else ""
+                ),
+                "source_type": indicator.source_type,
+            }
+        )
+    return related_indicators
+
+
 class IndicatorSetListView(ListView):
     model = IndicatorSet
     template_name = "indicatorsets/indicatorSets.html"
@@ -67,60 +122,6 @@ def get_queryset(self):
             indicatorsets_logger.error(f"Error fetching indicator sets: {e}")
             return IndicatorSet.objects.none()
 
-    def get_related_indicators(self, queryset, indicator_set_ids: list):
-        related_indicators = []
-        for indicator in queryset.filter(indicator_set__id__in=indicator_set_ids):
-            related_indicators.append(
-                {
-                    "id": indicator.id,
-                    "display_name": (
-                        indicator.get_display_name if indicator.get_display_name else ""
-                    ),
-                    "member_name": (
-                        indicator.member_name if indicator.member_name else ""
-                    ),
-                    "member_short_name": (
-                        indicator.member_short_name
-                        if indicator.member_short_name
-                        else ""
-                    ),
-                    "name": indicator.name if indicator.name else "",
-                    "indicator_set": (
-                        indicator.indicator_set.id if indicator.indicator_set else ""
-                    ),
-                    "indicator_set_name": (
-                        indicator.indicator_set.name if indicator.indicator_set else ""
-                    ),
-                    "indicator_set_short_name": (
-                        indicator.indicator_set.short_name
-                        if indicator.indicator_set
-                        else ""
-                    ),
-                    "endpoint": (
-                        indicator.indicator_set.epidata_endpoint
-                        if indicator.indicator_set
-                        else ""
-                    ),
-                    "source": indicator.source.name if indicator.source else "",
-                    "time_type": indicator.time_type if indicator.time_type else "",
-                    "description": (
-                        indicator.description if indicator.description else ""
-                    ),
-                    "member_description": (
-                        indicator.member_description
-                        if indicator.member_description
-                        else indicator.description
-                    ),
-                    "restricted": (
-                        indicator.indicator_set.dua_required
-                        if indicator.indicator_set
-                        else ""
-                    ),
-                    "source_type": indicator.source_type,
-                }
-            )
-        return related_indicators
-
     def get_url_params(self):
         url_params_dict = {
             "pathogens": (
@@ -244,14 +245,6 @@ def get_context_data(self, **kwargs):
                 output_field=IntegerField(),
             ),
         ).order_by("beta_last", "-is_top_priority", "-delphi_hosted", "name")
-        related_indicators = self.get_related_indicators(
-            filter.indicators_qs.select_related("indicator_set", "source"),
-            filter.qs.values_list("id", flat=True),
-        )
-        context["related_indicators"] = json.dumps(related_indicators)
-        context["num_of_timeseries"] = get_num_locations_from_meta(
-            related_indicators
-        )
         context["filters_descriptions"] = (
             FilterDescription.get_all_descriptions_as_dict()
         )
@@ -540,3 +533,29 @@ def get_available_geos(request):
         return JsonResponse(
             {"geographic_granularities": geographic_granularities}, safe=False
         )
+
+
+def get_related_indicators_json(request):
+    try:
+        queryset = IndicatorSet.objects.all().prefetch_related(
+            "geographic_scope",
+            "pathogens",
+            "severity_pyramid_rungs",
+            "geographic_levels",
+        )
+    except Exception as e:
+        indicatorsets_logger.error(f"Error fetching indicator sets: {e}")
+        queryset = IndicatorSet.objects.none()
+
+    filter = IndicatorSetFilter(request.GET, queryset=queryset)
+    related_indicators = get_related_indicators(
+        filter.indicators_qs.select_related("indicator_set", "source"),
+        filter.qs.values_list("id", flat=True),
+    )
+    num_of_timeseries = get_num_locations_from_meta(related_indicators)
+    return JsonResponse(
+        {
+            "related_indicators": related_indicators,
+            "num_of_timeseries": num_of_timeseries,
+        }
+    )

src/templates/indicatorsets/indicatorSetsTable.html

@@ -355,32 +355,54 @@
 <script src="{% static 'js/indicatorSetsTable.js' %}"></script>
 <script src="{% static 'js/pluralize_word.js' %}"></script>
 <script>
-    var relatedIndicators = JSON.parse(JSON.stringify({{ related_indicators|safe }}));
-    var numOfTimeseries = {{ num_of_timeseries }};
+    var relatedIndicators = [];
+    var numOfTimeseries = 0;
 
     function numberWithCommas(x) {
         return x.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ",");
     }
 
-
     $(document).ready(function() {
         table.columns.adjust();
         const totalRowsSpan = document.createElement("span");
         totalRowsSpan.classList.add("table-stats-info");
-        if (numOfTimeseries > 0) {
-            totalRowsSpan.innerHTML = `Showing <b>${relatedIndicators.length}</b> distinct ${pluralize(relatedIndicators.length, 'indicator')} (arranged in <b>${table.page.info().recordsTotal}</b> ${pluralize(table.page.info().recordsTotal, 'set')}), including <b>${numberWithCommas(numOfTimeseries)}</b> Delphi-hosted time series across numerous locations.`;
-        } else {
-            totalRowsSpan.innerHTML = `Showing <b>${relatedIndicators.length}</b> distinct ${pluralize(relatedIndicators.length, 'indicator')} (arranged in <b>${table.page.info().recordsTotal}</b> ${pluralize(table.page.info().recordsTotal, 'set')}).`;
-        }
+        totalRowsSpan.innerHTML = "Loading indicators statistics...";
         const refElement = document.getElementById("indicatorSetsTable_wrapper");
         refElement.before(totalRowsSpan);
+
+        // Fetch related indicators asynchronously
+        $.ajax({
+            url: "{% url 'get_related_indicators' %}" + window.location.search,
+            method: "GET",
+            success: function(response) {
+                relatedIndicators = response.related_indicators;
+                numOfTimeseries = response.num_of_timeseries;
+
+                if (numOfTimeseries > 0) {
+                    totalRowsSpan.innerHTML = `Showing <b>${relatedIndicators.length}</b> distinct ${pluralize(relatedIndicators.length, 'indicator')} (arranged in <b>${table.page.info().recordsTotal}</b> ${pluralize(table.page.info().recordsTotal, 'set')}), including <b>${numberWithCommas(numOfTimeseries)}</b> Delphi-hosted time series across numerous locations.`;
+                } else {
+                    totalRowsSpan.innerHTML = `Showing <b>${relatedIndicators.length}</b> distinct ${pluralize(relatedIndicators.length, 'indicator')} (arranged in <b>${table.page.info().recordsTotal}</b> ${pluralize(table.page.info().recordsTotal, 'set')}).`;
+                }
+            },
+            error: function(xhr, status, error) {
+                console.error("Error fetching related indicators:", error);
+                totalRowsSpan.innerHTML = "Error loading indicators statistics.";
+            }
+        });
     });
 
     // Add event listener for openizng and closing details
     $('#indicatorSetsTable tbody').on('click', 'td.dt-control', function () {
         var tr = $(this).closest('tr');
         var row = table.row(tr);
-    
+
+        if (relatedIndicators.length === 0) {
+             // If data hasn't loaded yet, maybe wait or show alert?
+             // For now, let's assume it loads fast enough or just show empty/loading in child row if we wanted to be fancy.
+             // But existing behavior with empty array will just show "No available indicators yet." from format function.
+             // We can check if request is pending if needed, but let's stick to simple first.
+        }
+
         if (row.child.isShown()) {
             // This row is already open - close it
             row.child.hide();
@@ -389,9 +411,6 @@
             // Open this row
             row.child(format(tr.data('id'), relatedIndicators, tr.data("description"))).show();
         }
-        
-    });
-
-   
 
+    });
 </script>