Merge remote-tracking branch 'upstream/main' into remove-needs-flags

Author: gibber9809

.github/workflows/clp-core-build-macos.yaml

@@ -45,6 +45,7 @@ jobs:
     strategy:
       matrix:
         os:
+          - "macos-14"
           - "macos-15"
         use_shared_libs:
           - true
@@ -66,6 +67,12 @@ jobs:
           rm -f /usr/local/bin/pydoc3*
           rm -f /usr/local/bin/python3*
 
+      - name: "Remove preinstalled programs which will conflict with the pipx-installed packages"
+        run: |
+          if brew list --formula cmake >/dev/null 2>&1; then
+            brew uninstall --force cmake
+          fi
+
       - name: "Install dependencies"
         run: "./components/core/tools/scripts/lib_install/macos/install-all.sh"
 

.github/workflows/clp-s-generated-code-checks.yaml

@@ -40,7 +40,7 @@ jobs:
 
       - name: "Generate parsers"
         shell: "bash"
-        run: "task clp-s-generate-parsers"
+        run: "task codegen:clp-s-generate-parsers"
 
       - name: "Check if the generated parsers are the latest"
         shell: "bash"

.gitignore

@@ -1,3 +1,4 @@
+__pycache__/
 .clang-format
 .clang-tidy
 .lint-venv/

NOTICE

@@ -0,0 +1,2 @@
+`components/webui/client/sql-parser/SqlBase.g4` was originally authored by the contributors of
+https://github.com/prestodb/presto

components/clp-package-utils/clp_package_utils/scripts/compress.py

@@ -202,6 +202,12 @@ def main(argv):
         except Exception as e:
             logger.error(e)
             return -1
+
+        if parsed_args.timestamp_key is None:
+            logger.warning(
+                "`--timestamp-key` not specified. Events will not have assigned timestamps and can "
+                "only be searched from the command line without a timestamp filter."
+            )
     elif dataset is not None:
         logger.error(f"Dataset selection is not supported for storage engine: {storage_engine}.")
         return -1

components/clp-package-utils/clp_package_utils/scripts/native/search.py

@@ -9,6 +9,7 @@
 import sys
 
 import msgpack
+import psutil
 import pymongo
 from clp_py_utils.clp_config import (
     Database,
@@ -128,16 +129,11 @@ async def do_search_without_aggregation(
     path_filter: str | None,
     raw_output: bool,
 ):
-    ip_list = socket.gethostbyname_ex(socket.gethostname())[2]
-    if len(ip_list) == 0:
-        logger.error("Couldn't determine the current host's IP.")
+    host = _get_ipv4_address()
+    if host is None:
+        logger.error("Couldn't find an IPv4 address for receiving search results.")
         return
-
-    host = ip_list[0]
-    for ip in ip_list:
-        if ipaddress.ip_address(ip) not in ipaddress.IPv4Network("127.0.0.0/8"):
-            host = ip
-            break
+    logger.debug(f"Listening on {host} for search results.")
 
     server = await asyncio.start_server(
         client_connected_cb=get_worker_connection_handler(raw_output),
@@ -342,5 +338,30 @@ def main(argv):
     return 0
 
 
+def _get_ipv4_address() -> str | None:
+    """
+    Retrieves an IPv4 address of the host for network communication.
+
+    :returns: The first non-loopback IPv4 address it finds.
+    If no non-loopback address is available, returns the first loopback IPv4 address.
+    If no IPv4 address is found, returns None.
+    """
+    fallback_ip = None
+
+    for addresses in psutil.net_if_addrs().values():
+        for addr in addresses:
+            if addr.family != socket.AF_INET:
+                continue
+            ip = addr.address
+            if not ipaddress.ip_address(ip).is_loopback:
+                return ip
+            if fallback_ip is None:
+                fallback_ip = ip
+
+    if fallback_ip is not None:
+        logger.warning("Couldn't find a non-loopback IP address for receiving search results.")
+    return fallback_ip
+
+
 if "__main__" == __name__:
     sys.exit(main(sys.argv))

components/clp-package-utils/clp_package_utils/scripts/start_clp.py

@@ -26,6 +26,7 @@
     QUERY_JOBS_TABLE_NAME,
     QUERY_SCHEDULER_COMPONENT_NAME,
     QUERY_WORKER_COMPONENT_NAME,
+    QueryEngine,
     QUEUE_COMPONENT_NAME,
     REDIS_COMPONENT_NAME,
     REDUCER_COMPONENT_NAME,
@@ -887,6 +888,7 @@ def start_webui(
         "ClientDir": str(container_webui_dir / "client"),
         "LogViewerDir": str(container_webui_dir / "yscope-log-viewer"),
         "StreamTargetUncompressedSize": container_clp_config.stream_output.target_uncompressed_size,
+        "ClpQueryEngine": clp_config.package.query_engine,
     }
 
     container_cmd_extra_opts = []
@@ -912,6 +914,14 @@ def start_webui(
         server_settings_json_updates["StreamFilesS3PathPrefix"] = None
         server_settings_json_updates["StreamFilesS3Profile"] = None
 
+    query_engine = clp_config.package.query_engine
+    if QueryEngine.PRESTO == query_engine:
+        server_settings_json_updates["PrestoHost"] = clp_config.presto.host
+        server_settings_json_updates["PrestoPort"] = clp_config.presto.port
+    else:
+        server_settings_json_updates["PrestoHost"] = None
+        server_settings_json_updates["PrestoPort"] = None
+
     server_settings_json = read_and_update_settings_json(
         server_settings_json_path, server_settings_json_updates
     )

components/clp-package-utils/clp_package_utils/scripts/stop_clp.py

@@ -34,10 +34,12 @@
 logger = logging.getLogger(__file__)
 
 
-def stop_running_container(container_name: str, already_exited_containers: List[str], force: bool):
+def stop_running_container(
+    container_name: str, already_exited_containers: List[str], force: bool, timeout: int = 10
+):
     if is_container_running(container_name):
         logger.info(f"Stopping {container_name}...")
-        cmd = ["docker", "stop", container_name]
+        cmd = ["docker", "stop", "-t", str(timeout), container_name]
         subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True)
 
         logger.info(f"Removing {container_name}...")
@@ -148,9 +150,6 @@ def main(argv):
         if target in (ALL_TARGET_NAME, QUERY_WORKER_COMPONENT_NAME):
             container_name = f"clp-{QUERY_WORKER_COMPONENT_NAME}-{instance_id}"
             stop_running_container(container_name, already_exited_containers, force)
-        if target in (ALL_TARGET_NAME, COMPRESSION_WORKER_COMPONENT_NAME):
-            container_name = f"clp-{COMPRESSION_WORKER_COMPONENT_NAME}-{instance_id}"
-            stop_running_container(container_name, already_exited_containers, force)
         if target in (ALL_TARGET_NAME, CONTROLLER_TARGET_NAME, QUERY_SCHEDULER_COMPONENT_NAME):
             container_name = f"clp-{QUERY_SCHEDULER_COMPONENT_NAME}-{instance_id}"
             stop_running_container(container_name, already_exited_containers, force)
@@ -164,11 +163,14 @@ def main(argv):
             COMPRESSION_SCHEDULER_COMPONENT_NAME,
         ):
             container_name = f"clp-{COMPRESSION_SCHEDULER_COMPONENT_NAME}-{instance_id}"
-            stop_running_container(container_name, already_exited_containers, force)
+            stop_running_container(container_name, already_exited_containers, force, timeout=300)
 
             container_config_file_path = logs_dir / f"{container_name}.yml"
             if container_config_file_path.exists():
                 container_config_file_path.unlink()
+        if target in (ALL_TARGET_NAME, COMPRESSION_WORKER_COMPONENT_NAME):
+            container_name = f"clp-{COMPRESSION_WORKER_COMPONENT_NAME}-{instance_id}"
+            stop_running_container(container_name, already_exited_containers, force, timeout=60)
         if target in (ALL_TARGET_NAME, CONTROLLER_TARGET_NAME, REDIS_COMPONENT_NAME):
             container_name = f"clp-{REDIS_COMPONENT_NAME}-{instance_id}"
             stop_running_container(container_name, already_exited_containers, force)