Refine uv lock maintenance loops

.github/workflows/clp-uv-checks.yaml

@@ -0,0 +1,50 @@
+name: "clp-uv-checks"
+
+on:
+  pull_request:
+    paths: &monitored_paths
+      - ".github/workflows/clp-uv-checks.yaml"
+      - "components/clp-mcp-server/pyproject.toml"
+      - "components/clp-mcp-server/uv.lock"
+      - "components/clp-package-utils/pyproject.toml"
+      - "components/clp-package-utils/uv.lock"
+      - "components/clp-py-utils/pyproject.toml"
+      - "components/clp-py-utils/uv.lock"
+      - "components/job-orchestration/pyproject.toml"
+      - "components/job-orchestration/uv.lock"
+      - "integration-tests/pyproject.toml"
+      - "integration-tests/uv.lock"
+  push:
+    paths: *monitored_paths
+  schedule:
+    # Run daily at 00:15 UTC (the 15 is to avoid periods of high load)
+    - cron: "15 0 * * *"
+  workflow_dispatch:
+
+concurrency:
+  group: "${{github.workflow}}-${{github.ref}}"
+  # Cancel in-progress jobs for efficiency
+  cancel-in-progress: true
+
+jobs:
+  rust-checks:
+    strategy:
+      matrix:
+        os:
+          - "macos-15"
+          - "ubuntu-22.04"
+          - "ubuntu-24.04"
+    runs-on: "${{matrix.os}}"
+    steps:
+      - # v5.0.0
+        uses: "actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8"
+        with:
+          submodules: "recursive"
+
+      - name: "Install task"
+        shell: "bash"
+        run: "npm install -g @go-task/[email protected]"
+
+      - name: "Validate lock files"
+        shell: "bash"
+        run: "task deps:lock:check-uv"

components/clp-mcp-server/clp_mcp_server/server/utils.py

@@ -83,8 +83,9 @@ def format_query_results(query_results: list[dict[str, Any]]) -> list[str]:
     kv-pairs:
     - "timestamp": An integer representing the epoch timestamp in milliseconds.
     - "message": A string representing the log message.
+    - "link": A string representing the link to open the log viewer displaying the message.
 
-    The message will be formatted as `timestamp: <date string>, message: <message>`:
+    The message will be formatted as `timestamp: <date string>, message: <message>, link: <link>`.
 
     :param query_results: A list of dictionaries representing kv-pair log events.
     :return: A list of strings representing formatted log events.
@@ -105,7 +106,9 @@ def format_query_results(query_results: list[dict[str, Any]]) -> list[str]:
             logger.warning("Empty message attached to a log event: %s.", obj)
             continue
 
-        formatted_log_events.append(f"timestamp: {timestamp_str}, message: {message}")
+        link = obj["link"]
+
+        formatted_log_events.append(f"timestamp: {timestamp_str}, message: {message}, link: {link}")
 
     return formatted_log_events
 

components/clp-mcp-server/tests/server/test_utils.py

@@ -13,6 +13,8 @@
 class TestUtils:
     """Test suite for utility functions."""
 
+    LINK = "http://localhost:4000/"
+
     # Error Messages:
     INVALID_DATE_STRING_ERROR = "Invalid date string"
     INVALID_DATE_STRING_FORMAT_ERROR = "Timestamp must end with 'Z' to indicate UTC."
@@ -24,54 +26,68 @@ class TestUtils:
         {
             "timestamp": None,
             "message": '{"message":"Log with None timestamp"}\n',
+            "link": LINK
         },
         {
             "timestamp": "1729267200000",  # str instead of int
             "message": '{"message":"Log with str timestamp"}\n',
+            "link": LINK
         },
         {
             "timestamp": 1729267200000.0,  # float instead of int
             "message": '{"message":"Log with float timestamp"}\n',
+            "link": LINK
         },
     ]
     EXPECTED_INVALID_TYPE = [
-        'timestamp: N/A, message: {"message":"Log with None timestamp"}\n',
-        'timestamp: N/A, message: {"message":"Log with str timestamp"}\n',
-        'timestamp: N/A, message: {"message":"Log with float timestamp"}\n',
+        f'timestamp: N/A, message: {{"message":"Log with None timestamp"}}\n, link: {LINK}',
+        f'timestamp: N/A, message: {{"message":"Log with str timestamp"}}\n, link: {LINK}',
+        f'timestamp: N/A, message: {{"message":"Log with float timestamp"}}\n, link: {LINK}'
     ]
 
     # Test case: invalid timestamp values.
     INVALID_VALUE_ENTRIES = [
         {
             "timestamp": 9999999999999999,
             "message": '{"message":"Log with overflow timestamp"}\n',
+            "link": LINK
         },
         {
             "timestamp": -9999999999999999,
             "message": '{"message":"Log with negative overflow timestamp"}\n',
+            "link": LINK
         },
     ]
     EXPECTED_INVALID_VALUE = [
-        'timestamp: N/A, message: {"message":"Log with overflow timestamp"}\n',
-        'timestamp: N/A, message: {"message":"Log with negative overflow timestamp"}\n',
+        (
+            f'timestamp: N/A, message: {{"message":"Log with overflow timestamp"}}\n,'
+            f' link: {LINK}'
+        ),
+        (
+            f'timestamp: N/A, message: {{"message":"Log with negative overflow timestamp"}}\n,'
+            f' link: {LINK}'
+        )
     ]
 
     # Test case: missing timestamp and message fields.
     MISSING_TIMESTAMP_AND_MESSAGE_ENTRY = [
         {
             "_id": "test001",
+            "link": LINK
         },
         {
             "_id": "test002",
             "message": '{"message":"Log with no timestamp"}\n',
+            "link": LINK
         },
         {
             "_id": "test003",
             "timestamp": 0,
+            "link": LINK
         }
     ]
     EXPECTED_MISSING_TIMESTAMP_AND_MESSAGE = [
-        'timestamp: N/A, message: {"message":"Log with no timestamp"}\n',
+        f'timestamp: N/A, message: {{"message":"Log with no timestamp"}}\n, link: {LINK}',
     ]
 
     # Testing basic functionality.
@@ -83,6 +99,7 @@ class TestUtils:
             "orig_file_path": "/var/log/app.log",
             "archive_id": "abc123",
             "log_event_ix": 99,
+            "link": LINK
         },
         {
             "_id": "test001",
@@ -91,6 +108,7 @@ class TestUtils:
             "orig_file_path": "/var/log/app.log",
             "archive_id": "abc123",
             "log_event_ix": 100,
+            "link": LINK
         },
         {
             "_id": "test002",
@@ -102,6 +120,7 @@ class TestUtils:
             "orig_file_path": "/var/log/app.log",
             "archive_id": "abc124",
             "log_event_ix": 101,
+            "link": LINK
         },
         {
             "_id": "test003",
@@ -113,29 +132,42 @@ class TestUtils:
             "orig_file_path": "/var/log/app.log",
             "archive_id": "abc125",
             "log_event_ix": 102,
+            "link": (
+                "http://localhost:4000/streamFile"
+                "?dataset=default"
+                '&type=json'
+                "&streamId=abc125"
+                "&logEventIdx=102"
+            ),
         },
     ]
 
     EXPECTED_RESULTS = [
         (
             'timestamp: 2024-10-18T16:00:00.123Z, message: '
             '{"ts":1729267200123,"pid":1234,"tid":5678,'
-            '"message":"Log with millisecond precision"}\n'
+            '"message":"Log with millisecond precision"}\n, '
+            "link: http://localhost:4000/streamFile"
+            "?dataset=default"
+            '&type=json'
+            "&streamId=abc125"
+            "&logEventIdx=102"
+
         ),
         (
-            'timestamp: 2024-10-18T16:00:00.000Z, message: '
-            '{"ts":1729267200000,"pid":1234,"tid":5678,'
-            '"message":"Log with zero milliseconds"}\n'
+            f'timestamp: 2024-10-18T16:00:00.000Z, message: '
+            f'{{"ts":1729267200000,"pid":1234,"tid":5678,'
+            f'"message":"Log with zero milliseconds"}}\n, link: {LINK}'
         ),
         (
-            'timestamp: 1970-01-01T00:00:00.000Z, message: '
-            '{"ts":0,"pid":null,"tid":null,'
-            '"message":"Log at epoch zero"}\n'
+            f'timestamp: 1970-01-01T00:00:00.000Z, message: '
+            f'{{"ts":0,"pid":null,"tid":null,'
+            f'"message":"Log at epoch zero"}}\n, link: {LINK}'
         ),
         (
-            'timestamp: N/A, message: '
-            '{"pid":null,"tid":null,'
-            '"message":"Log at epoch none"}\n'
+            f'timestamp: N/A, message: '
+            f'{{"pid":null,"tid":null,'
+            f'"message":"Log at epoch none"}}\n, link: {LINK}'
         ),
     ]
 

components/clp-package-utils/clp_package_utils/general.py

@@ -49,6 +49,9 @@
 
 DOCKER_MOUNT_TYPE_STRINGS = ["bind"]
 
+S3_KEY_PREFIX_COMPRESSION = "s3-key-prefix"
+S3_OBJECT_COMPRESSION = "s3-object"
+
 
 class DockerDependencyError(OSError):
     """Base class for errors related to Docker dependencies."""