Intercept host logs Range header for Systemd v256+ compatibility (#5827)

Since Systemd v256 the Range header must not end with a trailing colon.
We relied on this undocumented feature when following logs, and the
frontend or CLI may still use it in requests. To fix the requests
failing with new Systemd version, intercept the header and fill in the
num_entries to maximum possible value, which avoids the journal-gatewayd
returning the response prematurely and also works on older Systemd
versions.

The journal-gatewayd would still return response if follow flag is used
along with num_entries, but this behavior is unchanged and would be
better fixed in the backend.

Link: systemd/systemd#37172

Author: sairon

supervisor/api/host.py

@@ -37,6 +37,7 @@
     LogFormat,
     LogFormatter,
 )
+from ..host.logs import SYSTEMD_JOURNAL_GATEWAYD_LINES_MAX
 from ..utils.systemd_journal import journal_logs_reader
 from .const import (
     ATTR_AGENT_VERSION,
@@ -238,13 +239,11 @@ async def advanced_logs_handler(
                 # return 2 lines at minimum.
                 lines = max(2, lines)
             # entries=cursor[[:num_skip]:num_entries]
-            range_header = f"entries=:-{lines - 1}:{'' if follow else lines}"
+            range_header = f"entries=:-{lines - 1}:{SYSTEMD_JOURNAL_GATEWAYD_LINES_MAX if follow else lines}"
         elif RANGE in request.headers:
             range_header = request.headers[RANGE]
         else:
-            range_header = (
-                f"entries=:-{DEFAULT_LINES - 1}:{'' if follow else DEFAULT_LINES}"
-            )
+            range_header = f"entries=:-{DEFAULT_LINES - 1}:{SYSTEMD_JOURNAL_GATEWAYD_LINES_MAX if follow else DEFAULT_LINES}"
 
         async with self.sys_host.logs.journald_logs(
             params=params, range_header=range_header, accept=LogFormat.JOURNAL

supervisor/host/logs.py

@@ -8,6 +8,7 @@
 import logging
 import os
 from pathlib import Path
+import re
 from typing import Self
 
 from aiohttp import ClientError, ClientSession, ClientTimeout
@@ -34,6 +35,8 @@
 )
 # pylint: enable=no-member
 
+SYSTEMD_JOURNAL_GATEWAYD_LINES_MAX = (1 << 64) - 1
+
 SYSTEMD_JOURNAL_GATEWAYD_SOCKET: Path = Path("/run/systemd-journal-gatewayd.sock")
 
 # From systemd catalog for message IDs (`journalctl --dump-catalog``)
@@ -42,6 +45,10 @@
 # Defined-By: systemd
 BOOT_IDS_QUERY = {"MESSAGE_ID": "b07a249cd024414a82dd00cd181378ff"}
 
+RE_ENTRIES_HEADER = re.compile(
+    r"^entries=(?P<cursor>[^:]*):(?P<num_skip>-?\d+):(?P<num_lines>\d*)$"
+)
+
 
 class LogsControl(CoreSysAttributes):
     """Handle systemd-journal logs."""
@@ -186,6 +193,16 @@ async def journald_logs(
             async with ClientSession(base_url=base_url, connector=connector) as session:
                 headers = {ACCEPT: accept}
                 if range_header:
+                    if range_header.endswith(":"):
+                        # Make sure that num_entries is always set - before Systemd v256 it was
+                        # possible to omit it, which made sense when the "follow" option was used,
+                        # but this syntax is now invalid and triggers HTTP 400.
+                        # See: https://github.com/systemd/systemd/issues/37172
+                        if not (matches := re.match(RE_ENTRIES_HEADER, range_header)):
+                            raise HostNotSupportedError(
+                                f"Invalid range header: {range_header}"
+                            )
+                        range_header = f"entries={matches.group('cursor')}:{matches.group('num_skip')}:{SYSTEMD_JOURNAL_GATEWAYD_LINES_MAX}"
                     headers[RANGE] = range_header
                 async with session.get(
                     f"{path}",

tests/api/__init__.py

@@ -7,7 +7,7 @@
 from supervisor.host.const import LogFormat
 
 DEFAULT_LOG_RANGE = "entries=:-99:100"
-DEFAULT_LOG_RANGE_FOLLOW = "entries=:-99:"
+DEFAULT_LOG_RANGE_FOLLOW = "entries=:-99:18446744073709551615"
 
 
 async def common_test_api_advanced_logs(

tests/api/test_host.py

@@ -16,7 +16,7 @@
 from tests.dbus_service_mocks.systemd import Systemd as SystemdService
 
 DEFAULT_RANGE = "entries=:-99:100"
-DEFAULT_RANGE_FOLLOW = "entries=:-99:"
+DEFAULT_RANGE_FOLLOW = "entries=:-99:18446744073709551615"
 # pylint: disable=protected-access
 
 

tests/host/test_logs.py

@@ -171,3 +171,24 @@ async def test_connection_refused_handled(
         with pytest.raises(HostServiceError):
             async with coresys.host.logs.journald_logs():
                 pass
+
+
+@pytest.mark.parametrize(
+    "range_header,range_reparse",
+    [
+        ("entries=:-99:", "entries=:-99:18446744073709551615"),
+        ("entries=:-99:100", "entries=:-99:100"),
+        ("entries=cursor:0:100", "entries=cursor:0:100"),
+    ],
+)
+async def test_range_header_reparse(
+    journald_gateway: MagicMock, coresys: CoreSys, range_header: str, range_reparse: str
+):
+    """Test that range header with trailing colon contains num_entries."""
+    async with coresys.host.logs.journald_logs(range_header=range_header):
+        journald_gateway.get.assert_called_with(
+            "/entries",
+            headers={"Accept": "text/plain", "Range": range_reparse},
+            params={},
+            timeout=None,
+        )