Skip to content

Commit 1d7365f

Browse files
authored
Fix crawler endpoint (#301)
1 parent 07a3a28 commit 1d7365f

File tree

2 files changed

+7
-9
lines changed

2 files changed

+7
-9
lines changed

CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
- Better gz files extraction function name [#295](https://github.com/datagouv/hydra/pull/295)
1010
- Add more detailed statuses [#297](https://github.com/datagouv/hydra/pull/297)
1111
- Handle cases of too long columns labels for postgres [#298](https://github.com/datagouv/hydra/pull/298)
12+
- Fix rare issue in `/status/crawler/` endpoint [#299](https://github.com/datagouv/hydra/pull/299)
1213

1314
## 2.3.0 (2025-07-15)
1415

udata_hydra/routes/status.py

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,8 @@ async def get_crawler_status(request: web.Request) -> web.Response:
1111
# Count resources with no check and resources with a check
1212
q = f"""
1313
SELECT
14-
SUM(CASE WHEN last_check IS NULL THEN 1 ELSE 0 END) AS count_never_checked,
15-
SUM(CASE WHEN last_check IS NOT NULL THEN 1 ELSE 0 END) AS count_checked
14+
COALESCE(SUM(CASE WHEN last_check IS NULL THEN 1 ELSE 0 END), 0) AS count_never_checked,
15+
COALESCE(SUM(CASE WHEN last_check IS NOT NULL THEN 1 ELSE 0 END), 0) AS count_checked
1616
FROM catalog
1717
WHERE {Resource.get_excluded_clause()}
1818
AND catalog.deleted = False
@@ -22,22 +22,19 @@ async def get_crawler_status(request: web.Request) -> web.Response:
2222
now = datetime.now(timezone.utc)
2323
q = f"""
2424
SELECT
25-
SUM(CASE WHEN checks.next_check_at <= $1 THEN 1 ELSE 0 END) AS count_outdated
26-
--, SUM(CASE WHEN checks.next_check_at > $1 THEN 1 ELSE 0 END) AS count_fresh
25+
COALESCE(SUM(CASE WHEN checks.next_check_at <= $1 THEN 1 ELSE 0 END), 0) AS count_outdated
2726
FROM catalog, checks
2827
WHERE {Resource.get_excluded_clause()}
2928
AND catalog.last_check = checks.id
3029
AND catalog.deleted = False
3130
"""
3231
stats_checks: dict = await request.app["pool"].fetchrow(q, now)
3332

34-
count_pending_checks: int = stats_resources["count_never_checked"] + (
35-
stats_checks["count_outdated"] or 0
33+
count_pending_checks: int = (
34+
stats_resources["count_never_checked"] + stats_checks["count_outdated"]
3635
)
3736
# all w/ a check, minus those with an outdated checked
38-
count_fresh_checks: int = stats_resources["count_checked"] - (
39-
stats_checks["count_outdated"] or 0
40-
)
37+
count_fresh_checks: int = stats_resources["count_checked"] - stats_checks["count_outdated"]
4138
total: int = stats_resources["count_never_checked"] + stats_resources["count_checked"]
4239
rate_checked: float = round(stats_resources["count_checked"] / total * 100, 1)
4340
rate_checked_fresh: float = round(count_fresh_checks / total * 100, 1)

0 commit comments

Comments
 (0)