feat: Replace existing webui codebase with new-webui. (#939)

Co-authored-by: kirkrodrigues <[email protected]>

Author: junhaoliao

components/clp-package-utils/clp_package_utils/general.py

@@ -15,7 +15,6 @@
     CLP_DEFAULT_CREDENTIALS_FILE_PATH,
     CLPConfig,
     DB_COMPONENT_NAME,
-    LOG_VIEWER_WEBUI_COMPONENT_NAME,
     QUEUE_COMPONENT_NAME,
     REDIS_COMPONENT_NAME,
     REDUCER_COMPONENT_NAME,
@@ -514,34 +513,17 @@ def validate_worker_config(clp_config: CLPConfig):
 
 
 def validate_webui_config(
-    clp_config: CLPConfig, logs_dir: pathlib.Path, settings_json_path: pathlib.Path
+    clp_config: CLPConfig,
+    client_settings_json_path: pathlib.Path,
+    server_settings_json_path: pathlib.Path,
 ):
-    if not settings_json_path.exists():
-        raise ValueError(
-            f"{WEBUI_COMPONENT_NAME} {settings_json_path} is not a valid path to Meteor settings.json"
-        )
-
-    try:
-        validate_path_could_be_dir(logs_dir)
-    except ValueError as ex:
-        raise ValueError(f"{WEBUI_COMPONENT_NAME} logs directory is invalid: {ex}")
+    for path in [client_settings_json_path, server_settings_json_path]:
+        if not path.exists():
+            raise ValueError(f"{WEBUI_COMPONENT_NAME} {path} is not a valid path to settings.json")
 
     validate_port(f"{WEBUI_COMPONENT_NAME}.port", clp_config.webui.host, clp_config.webui.port)
 
 
-def validate_log_viewer_webui_config(clp_config: CLPConfig, settings_json_path: pathlib.Path):
-    if not settings_json_path.exists():
-        raise ValueError(
-            f"{WEBUI_COMPONENT_NAME} {settings_json_path} is not a valid path to settings.json"
-        )
-
-    validate_port(
-        f"{LOG_VIEWER_WEBUI_COMPONENT_NAME}.port",
-        clp_config.log_viewer_webui.host,
-        clp_config.log_viewer_webui.port,
-    )
-
-
 def validate_path_for_container_mount(path: pathlib.Path) -> None:
     RESTRICTED_PREFIXES: List[pathlib.Path] = [
         CONTAINER_AWS_CONFIG_DIRECTORY,

components/clp-package-utils/clp_package_utils/scripts/start_clp.py

@@ -24,7 +24,6 @@
     CONTROLLER_TARGET_NAME,
     DB_COMPONENT_NAME,
     FILES_TABLE_SUFFIX,
-    LOG_VIEWER_WEBUI_COMPONENT_NAME,
     QUERY_JOBS_TABLE_NAME,
     QUERY_SCHEDULER_COMPONENT_NAME,
     QUERY_WORKER_COMPONENT_NAME,
@@ -57,7 +56,6 @@
     validate_and_load_queue_credentials_file,
     validate_and_load_redis_credentials_file,
     validate_db_config,
-    validate_log_viewer_webui_config,
     validate_queue_config,
     validate_redis_config,
     validate_reducer_config,
@@ -843,132 +841,62 @@ def read_and_update_settings_json(settings_file_path: pathlib.Path, updates: Dic
     return settings_object
 
 
-def start_webui(instance_id: str, clp_config: CLPConfig, mounts: CLPDockerMounts):
+def start_webui(
+    instance_id: str,
+    clp_config: CLPConfig,
+    container_clp_config: CLPConfig,
+    mounts: CLPDockerMounts,
+):
     component_name = WEBUI_COMPONENT_NAME
     logger.info(f"Starting {component_name}...")
 
     container_name = f"clp-{component_name}-{instance_id}"
     if container_exists(container_name):
         return
 
-    webui_logs_dir = clp_config.logs_directory / component_name
     container_webui_dir = CONTAINER_CLP_HOME / "var" / "www" / "webui"
-    node_path = str(container_webui_dir / "programs" / "server" / "npm" / "node_modules")
-    settings_json_path = get_clp_home() / "var" / "www" / "webui" / "settings.json"
-
-    validate_webui_config(clp_config, webui_logs_dir, settings_json_path)
-
-    # Create directories
-    webui_logs_dir.mkdir(exist_ok=True, parents=True)
+    node_path = str(container_webui_dir / "server" / "node_modules")
+    client_settings_json_path = (
+        get_clp_home() / "var" / "www" / "webui" / "client" / "settings.json"
+    )
+    server_settings_json_path = (
+        get_clp_home() / "var" / "www" / "webui" / "server" / "dist" / "server" / "settings.json"
+    )
 
-    container_webui_logs_dir = pathlib.Path("/") / "var" / "log" / component_name
+    validate_webui_config(clp_config, client_settings_json_path, server_settings_json_path)
 
-    # Read and update settings.json
+    # Read, update, and write back client's and server's settings.json
     clp_db_connection_params = clp_config.database.get_clp_connection_params_and_type(True)
     table_prefix = clp_db_connection_params["table_prefix"]
     if StorageEngine.CLP_S == clp_config.package.storage_engine:
         table_prefix = f"{table_prefix}{CLP_DEFAULT_DATASET_NAME}_"
-    meteor_settings_updates = {
-        "private": {
-            "SqlDbHost": clp_config.database.host,
-            "SqlDbPort": clp_config.database.port,
-            "SqlDbName": clp_config.database.name,
-            "SqlDbClpArchivesTableName": f"{table_prefix}{ARCHIVES_TABLE_SUFFIX}",
-            "SqlDbClpFilesTableName": f"{table_prefix}{FILES_TABLE_SUFFIX}",
-            "SqlDbCompressionJobsTableName": COMPRESSION_JOBS_TABLE_NAME,
-            "SqlDbQueryJobsTableName": QUERY_JOBS_TABLE_NAME,
-        },
-        "public": {
-            "ClpStorageEngine": clp_config.package.storage_engine,
-            "LogViewerWebuiUrl": (
-                f"http://{clp_config.log_viewer_webui.host}:{clp_config.log_viewer_webui.port}",
-            ),
-        },
+    client_settings_json_updates = {
+        "ClpStorageEngine": clp_config.package.storage_engine,
+        "MongoDbSearchResultsMetadataCollectionName": clp_config.webui.results_metadata_collection_name,
+        "SqlDbClpArchivesTableName": f"{table_prefix}{ARCHIVES_TABLE_SUFFIX}",
+        "SqlDbClpFilesTableName": f"{table_prefix}{FILES_TABLE_SUFFIX}",
+        "SqlDbCompressionJobsTableName": COMPRESSION_JOBS_TABLE_NAME,
     }
-    meteor_settings = read_and_update_settings_json(settings_json_path, meteor_settings_updates)
-
-    # Start container
-    # fmt: off
-    container_cmd = [
-        "docker", "run",
-        "-d",
-        "--network", "host",
-        "--name", container_name,
-        "--log-driver", "local",
-        "-u", f"{os.getuid()}:{os.getgid()}",
-    ]
-    # fmt: on
-    env_vars = [
-        f"NODE_PATH={node_path}",
-        f"MONGO_URL={clp_config.results_cache.get_uri()}",
-        f"PORT={clp_config.webui.port}",
-        f"ROOT_URL=http://{clp_config.webui.host}",
-        f"METEOR_SETTINGS={json.dumps(meteor_settings)}",
-        f"CLP_DB_USER={clp_config.database.username}",
-        f"CLP_DB_PASS={clp_config.database.password}",
-        f"WEBUI_LOGS_DIR={container_webui_logs_dir}",
-        f"WEBUI_LOGGING_LEVEL={clp_config.webui.logging_level}",
-    ]
-    necessary_mounts = [
-        mounts.clp_home,
-        DockerMount(DockerMountType.BIND, webui_logs_dir, container_webui_logs_dir),
-    ]
-    append_docker_options(container_cmd, necessary_mounts, env_vars)
-    container_cmd.append(clp_config.execution_container)
-
-    node_cmd = [
-        str(CONTAINER_CLP_HOME / "bin" / "node-14"),
-        str(container_webui_dir / "launcher.js"),
-        str(container_webui_dir / "main.js"),
-    ]
-    cmd = container_cmd + node_cmd
-    subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True)
-
-    logger.info(f"Started {component_name}.")
-
-
-def start_log_viewer_webui(
-    instance_id: str,
-    clp_config: CLPConfig,
-    container_clp_config: CLPConfig,
-    mounts: CLPDockerMounts,
-):
-    component_name = LOG_VIEWER_WEBUI_COMPONENT_NAME
-    logger.info(f"Starting {component_name}...")
-
-    container_name = f"clp-{component_name}-{instance_id}"
-    if container_exists(container_name):
-        return
-
-    container_log_viewer_webui_dir = CONTAINER_CLP_HOME / "var" / "www" / "log-viewer-webui"
-    node_path = str(container_log_viewer_webui_dir / "server" / "node_modules")
-    settings_json_path = (
-        get_clp_home()
-        / "var"
-        / "www"
-        / "log-viewer-webui"
-        / "server"
-        / "dist"
-        / "server"
-        / "settings.json"
+    client_settings_json = read_and_update_settings_json(
+        client_settings_json_path, client_settings_json_updates
     )
+    with open(client_settings_json_path, "w") as client_settings_json_file:
+        client_settings_json_file.write(json.dumps(client_settings_json))
 
-    validate_log_viewer_webui_config(clp_config, settings_json_path)
-
-    # Read, update, and write back settings.json
-    settings_json_updates = {
+    server_settings_json_updates = {
         "SqlDbHost": clp_config.database.host,
         "SqlDbPort": clp_config.database.port,
         "SqlDbName": clp_config.database.name,
         "SqlDbQueryJobsTableName": QUERY_JOBS_TABLE_NAME,
         "MongoDbHost": clp_config.results_cache.host,
         "MongoDbPort": clp_config.results_cache.port,
         "MongoDbName": clp_config.results_cache.db_name,
+        "MongoDbSearchResultsMetadataCollectionName": clp_config.webui.results_metadata_collection_name,
         "MongoDbStreamFilesCollectionName": clp_config.results_cache.stream_collection_name,
-        "ClientDir": str(container_log_viewer_webui_dir / "client"),
+        "ClientDir": str(container_webui_dir / "client"),
+        "LogViewerDir": str(container_webui_dir / "yscope-log-viewer"),
         "StreamFilesDir": str(container_clp_config.stream_output.get_directory()),
         "StreamTargetUncompressedSize": container_clp_config.stream_output.target_uncompressed_size,
-        "LogViewerDir": str(container_log_viewer_webui_dir / "yscope-log-viewer"),
     }
 
     container_cmd_extra_opts = []
@@ -977,23 +905,25 @@ def start_log_viewer_webui(
     if StorageType.S3 == stream_storage.type:
         s3_config = stream_storage.s3_config
 
-        settings_json_updates["StreamFilesS3Region"] = s3_config.region_code
-        settings_json_updates["StreamFilesS3PathPrefix"] = (
+        server_settings_json_updates["StreamFilesS3Region"] = s3_config.region_code
+        server_settings_json_updates["StreamFilesS3PathPrefix"] = (
             f"{s3_config.bucket}/{s3_config.key_prefix}"
         )
         auth = s3_config.aws_authentication
         if AwsAuthType.profile == auth.type:
-            settings_json_updates["StreamFilesS3Profile"] = auth.profile
+            server_settings_json_updates["StreamFilesS3Profile"] = auth.profile
         else:
-            settings_json_updates["StreamFilesS3Profile"] = None
+            server_settings_json_updates["StreamFilesS3Profile"] = None
     elif StorageType.FS == stream_storage.type:
-        settings_json_updates["StreamFilesS3Region"] = None
-        settings_json_updates["StreamFilesS3PathPrefix"] = None
-        settings_json_updates["StreamFilesS3Profile"] = None
+        server_settings_json_updates["StreamFilesS3Region"] = None
+        server_settings_json_updates["StreamFilesS3PathPrefix"] = None
+        server_settings_json_updates["StreamFilesS3Profile"] = None
 
-    settings_json = read_and_update_settings_json(settings_json_path, settings_json_updates)
-    with open(settings_json_path, "w") as settings_json_file:
-        settings_json_file.write(json.dumps(settings_json))
+    server_settings_json = read_and_update_settings_json(
+        server_settings_json_path, server_settings_json_updates
+    )
+    with open(server_settings_json_path, "w") as settings_json_file:
+        settings_json_file.write(json.dumps(server_settings_json))
 
     # fmt: off
     container_cmd = [
@@ -1009,8 +939,8 @@ def start_log_viewer_webui(
 
     necessary_env_vars = [
         f"NODE_PATH={node_path}",
-        f"HOST={clp_config.log_viewer_webui.host}",
-        f"PORT={clp_config.log_viewer_webui.port}",
+        f"HOST={clp_config.webui.host}",
+        f"PORT={clp_config.webui.port}",
         f"CLP_DB_USER={clp_config.database.username}",
         f"CLP_DB_PASS={clp_config.database.password}",
         f"NODE_ENV=production",
@@ -1027,7 +957,7 @@ def start_log_viewer_webui(
             necessary_env_vars.append(f"AWS_SECRET_ACCESS_KEY={credentials.secret_access_key}")
         else:
             aws_mount, aws_env_vars = generate_container_auth_options(
-                clp_config, LOG_VIEWER_WEBUI_COMPONENT_NAME
+                clp_config, WEBUI_COMPONENT_NAME
             )
             if aws_mount:
                 necessary_mounts.append(mounts.aws_config_dir)
@@ -1038,7 +968,7 @@ def start_log_viewer_webui(
 
     node_cmd = [
         str(CONTAINER_CLP_HOME / "bin" / "node-22"),
-        str(container_log_viewer_webui_dir / "server" / "dist" / "server" / "src" / "main.js"),
+        str(container_webui_dir / "server" / "dist" / "server" / "src" / "main.js"),
     ]
     cmd = container_cmd + node_cmd
     subprocess.run(cmd, stdout=subprocess.DEVNULL, check=True)
@@ -1147,7 +1077,6 @@ def main(argv):
     reducer_server_parser = component_args_parser.add_parser(REDUCER_COMPONENT_NAME)
     add_num_workers_argument(reducer_server_parser)
     component_args_parser.add_parser(WEBUI_COMPONENT_NAME)
-    component_args_parser.add_parser(LOG_VIEWER_WEBUI_COMPONENT_NAME)
 
     parsed_args = args_parser.parse_args(argv[1:])
 
@@ -1175,7 +1104,6 @@ def main(argv):
             COMPRESSION_SCHEDULER_COMPONENT_NAME,
             QUERY_SCHEDULER_COMPONENT_NAME,
             WEBUI_COMPONENT_NAME,
-            LOG_VIEWER_WEBUI_COMPONENT_NAME,
         ):
             validate_and_load_db_credentials_file(clp_config, clp_home, True)
         if target in (
@@ -1271,9 +1199,7 @@ def main(argv):
         if target in (ALL_TARGET_NAME, REDUCER_COMPONENT_NAME):
             start_reducer(instance_id, clp_config, container_clp_config, num_workers, mounts)
         if target in (ALL_TARGET_NAME, WEBUI_COMPONENT_NAME):
-            start_webui(instance_id, clp_config, mounts)
-        if target in (ALL_TARGET_NAME, LOG_VIEWER_WEBUI_COMPONENT_NAME):
-            start_log_viewer_webui(instance_id, clp_config, container_clp_config, mounts)
+            start_webui(instance_id, clp_config, container_clp_config, mounts)
 
     except Exception as ex:
         if type(ex) == ValueError:

components/clp-package-utils/clp_package_utils/scripts/stop_clp.py

@@ -11,7 +11,6 @@
     COMPRESSION_WORKER_COMPONENT_NAME,
     CONTROLLER_TARGET_NAME,
     DB_COMPONENT_NAME,
-    LOG_VIEWER_WEBUI_COMPONENT_NAME,
     QUERY_SCHEDULER_COMPONENT_NAME,
     QUERY_WORKER_COMPONENT_NAME,
     QUEUE_COMPONENT_NAME,
@@ -85,7 +84,6 @@ def main(argv):
     component_args_parser.add_parser(COMPRESSION_WORKER_COMPONENT_NAME)
     component_args_parser.add_parser(QUERY_WORKER_COMPONENT_NAME)
     component_args_parser.add_parser(WEBUI_COMPONENT_NAME)
-    component_args_parser.add_parser(LOG_VIEWER_WEBUI_COMPONENT_NAME)
 
     parsed_args = args_parser.parse_args(argv[1:])
 
@@ -104,7 +102,6 @@ def main(argv):
             ALL_TARGET_NAME,
             CONTROLLER_TARGET_NAME,
             DB_COMPONENT_NAME,
-            LOG_VIEWER_WEBUI_COMPONENT_NAME,
         ):
             validate_and_load_db_credentials_file(clp_config, clp_home, False)
         if target in (
@@ -133,9 +130,6 @@ def main(argv):
 
         already_exited_containers = []
         force = parsed_args.force
-        if target in (ALL_TARGET_NAME, LOG_VIEWER_WEBUI_COMPONENT_NAME):
-            container_name = f"clp-{LOG_VIEWER_WEBUI_COMPONENT_NAME}-{instance_id}"
-            stop_running_container(container_name, already_exited_containers, force)
         if target in (ALL_TARGET_NAME, WEBUI_COMPONENT_NAME):
             container_name = f"clp-{WEBUI_COMPONENT_NAME}-{instance_id}"
             stop_running_container(container_name, already_exited_containers, force)