histogram sql api working

Author: wardi

ckanext/datastore/backend/postgres.py

@@ -1649,14 +1649,15 @@ def search_data_buckets(context: Context, data_dict: dict[str, Any]):
         edges_{index} AS (
             SELECT DISTINCT
                 (data_stats_{index}.min_val + (
-                generate_series(0, {num_buckets}-1)
+                generate_series(0, {num_buckets})
                 * (data_stats_{index}.max_val - data_stats_{index}.min_val)
                 / {num_buckets}
-                ))::{ftype}
+                ))::{ftype} e
             FROM data_stats_{index}
+            ORDER BY e
         ),
         data_{index} AS (
-            SELECT val, freq
+            SELECT val, coalesce(freq, 0) freq
             FROM
                 unnest(array(select * from edges_{index})) with ordinality as val
             FULL JOIN
@@ -1705,7 +1706,7 @@ def search_data_buckets(context: Context, data_dict: dict[str, Any]):
                 resource=identifier(resource_id),
                 ts_query=ts_query,
                 where=where_clause,
-                num_buckets=MAX_BUCKETS,
+                num_buckets=data_dict['buckets'],
                 ftype=ftype,
             ))
 
@@ -1730,8 +1731,19 @@ def search_data_buckets(context: Context, data_dict: dict[str, Any]):
     ).mappings().one()
 
     for i, rf in enumerate(rfields, 1):
-        rf['buckets'] = result[f'freq_{i}']
-        rf['edges'] = result[f'edge_{i}']
+        buckets = result[f'freq_{i}']
+        edges = result[f'edge_{i}']
+        rf['nulls'] = 0
+        if result[f'edge_{i}'][-1:] == [None]:
+            edges.pop()
+            rf['nulls'] = buckets.pop()
+            if edges == [None]:  # all nulls returns two null edges
+                edges = []
+                buckets = []
+        # last value returned contains count exactly matching max value
+        # combine with bucket before
+        rf['buckets'] = buckets[:-2] + ([sum(buckets[-2:])] if buckets else [])
+        rf['edges'] = edges
 
     return {'fields': rfields}
 

ckanext/datastore/logic/action.py

@@ -777,11 +777,6 @@ def datastore_search_buckets(context: Context, data_dict: dict[str, Any]):
     :rtype: A dictionary with the following keys
     :param fields: fields/columns and their extra metadata
     :type fields: list of dictionaries
-    :param filters: query filters
-    :type filters: list of dictionaries
-    :param buckets: dict of matching results
-    :type buckets: dict of field ids and bucketed data
-
     '''
     backend = DatastoreBackend.get_active_backend()
     schema = context.get('schema', dsschema.datastore_search_buckets_schema())

ckanext/datastore/logic/schema.py

@@ -224,7 +224,11 @@ def datastore_search_buckets_schema() -> Schema:
         'resource_id': [not_missing, not_empty, unicode_safe],
         'id': [ignore_missing],
         'q': [ignore_missing, unicode_or_json_validator],
-        'buckets': [default(12), int_validator],
+        'buckets': [
+            default(12),
+            natural_number_validator,
+            limit_to_configured_maximum('ckan.datastore.search.buckets_max', 300),
+        ],
         'plain': [ignore_missing, boolean_validator],
         'filters': [ignore_missing, json_validator],
         'language': [ignore_missing, unicode_safe],

ckanext/datastore/tests/test_histogram.py

@@ -3,7 +3,7 @@
 import pytest
 import sqlalchemy as sa
 import sqlalchemy.orm as orm
-import decimal
+from datetime import date, datetime
 from unittest import mock
 
 import ckan.logic as logic
@@ -37,11 +37,56 @@ def test_histograms():
             {"one": 1, "short": 2, "nums": 10, "days": "2026-02-01", "ts": "2026-02-01"},
             {"one": 1, "short": 8, "nums": 15, "days": "2026-01-06", "ts": "2026-01-06"},
             {"one": 1, "short": 9, "nums": 15, "days": "2026-02-01", "ts": "2026-02-01"},
-            {"one": 2, "short": 3, "nums": -1, "days": "2026-01-06", "ts": "2026-01-06"},
+            {"one": 1, "short": 3, "nums": -1, "days": "2026-01-18", "ts": "2026-01-18"},
         ],
     )
     results = helpers.call_action(
         "datastore_search_buckets",
         resource_id=resource["id"],
+        buckets=4,
     )
-    assert results["fields"] == []
+    assert results["fields"] == [
+        {"id": "all_null", "buckets": [], "edges": [], "nulls": 6, "type": "int4"},
+        {"id": "one", "buckets": [6], "edges": [1], "nulls": 0, "type": "numeric"},
+        {
+            "id": "short",
+            "buckets": [3, 1, 0, 2],
+            "edges": [2, 3, 5, 7, 9],
+            "nulls": 0,
+            "type": "int4",
+        },
+        {
+            "id": "nums",
+            "buckets": [2, 0, 1, 3],
+            "edges": [-4, 2, 8, 14, 20],
+            "nulls": 0,
+            "type": "numeric",
+        },
+        {
+            "id": "days",
+            "buckets": [3, 0, 1, 2],
+            "edges": [
+                date(2026, 1, 1),
+                date(2026, 1, 8),
+                date(2026, 1, 16),
+                date(2026, 1, 24),
+                date(2026, 2, 1),
+            ],
+            "nulls": 0,
+            "type": "date",
+        },
+        {
+            "id": "ts",
+            "buckets": [3, 0, 1, 2],
+            "edges": [
+                datetime(2026, 1, 1, 0),
+                datetime(2026, 1, 8, 18),
+                datetime(2026, 1, 16, 12),
+                datetime(2026, 1, 24, 6),
+                datetime(2026, 2, 1, 0),
+
+            ],
+            "nulls": 0,
+            "type": "timestamp",
+        }
+    ]