fix(package): Update native/decompress.py to use CLI args and env vars for DB config (fixes #1199); Prevent IR extraction with clp-s storage engine and only use dataset argument for JSON extraction (fixes #1200). (#1202)

Author: junhaoliao

components/clp-package-utils/clp_package_utils/scripts/decompress.py

@@ -181,23 +181,13 @@ def handle_extract_stream_cmd(
         return -1
 
     job_command = parsed_args.command
+    if EXTRACT_IR_CMD == job_command and StorageEngine.CLP != storage_engine:
+        logger.error(f"IR extraction is not supported for storage engine `{storage_engine}`.")
+        return -1
     if EXTRACT_JSON_CMD == job_command and StorageEngine.CLP_S != storage_engine:
         logger.error(f"JSON extraction is not supported for storage engine `{storage_engine}`.")
         return -1
 
-    dataset = parsed_args.dataset
-    if StorageEngine.CLP_S == storage_engine:
-        dataset = CLP_DEFAULT_DATASET_NAME if dataset is None else dataset
-        try:
-            clp_db_connection_params = clp_config.database.get_clp_connection_params_and_type(True)
-            validate_dataset_name(clp_db_connection_params["table_prefix"], dataset)
-        except Exception as e:
-            logger.error(e)
-            return -1
-    elif dataset is not None:
-        logger.error(f"Dataset selection is not supported for storage engine: {storage_engine}.")
-        return -1
-
     container_name = generate_container_name(str(JobType.IR_EXTRACTION))
     container_clp_config, mounts = generate_container_config(clp_config, clp_home)
     generated_config_path_on_container, generated_config_path_on_host = dump_container_config(
@@ -229,6 +219,15 @@ def handle_extract_stream_cmd(
             extract_cmd.append("--target-uncompressed-size")
             extract_cmd.append(str(parsed_args.target_uncompressed_size))
     elif EXTRACT_JSON_CMD == job_command:
+        dataset = parsed_args.dataset
+        dataset = CLP_DEFAULT_DATASET_NAME if dataset is None else dataset
+        try:
+            clp_db_connection_params = clp_config.database.get_clp_connection_params_and_type(True)
+            validate_dataset_name(clp_db_connection_params["table_prefix"], dataset)
+        except Exception as e:
+            logger.error(e)
+            return -1
+
         extract_cmd.append(str(parsed_args.archive_id))
         if dataset is not None:
             extract_cmd.append("--dataset")

components/clp-package-utils/clp_package_utils/scripts/native/decompress.py

@@ -1,14 +1,14 @@
 import argparse
 import asyncio
 import logging
+import os
 import pathlib
 import subprocess
 import sys
 import uuid
 from contextlib import closing
 from typing import Optional
 
-import yaml
 from clp_py_utils.clp_config import (
     CLPConfig,
     Database,
@@ -229,18 +229,24 @@ def handle_extract_file_cmd(
     logs_dir = clp_config.logs_directory
     archives_dir = clp_config.archive_output.get_directory()
 
-    # Generate database config file for clp
-    db_config_file_path = logs_dir / f".decompress-db-config-{uuid.uuid4()}.yml"
-    with open(db_config_file_path, "w") as f:
-        yaml.safe_dump(clp_config.database.get_clp_connection_params_and_type(True), f)
-
+    # Configure CLP metadata DB connection params.
+    clp_db_connection_params = clp_config.database.get_clp_connection_params_and_type(True)
     # fmt: off
     extract_cmd = [
         str(clp_home / "bin" / "clp"),
         "x", str(archives_dir), str(extraction_dir),
-        "--db-config-file", str(db_config_file_path),
+        "--db-type", clp_db_connection_params["type"],
+        "--db-host", clp_db_connection_params["host"],
+        "--db-port", str(clp_db_connection_params["port"]),
+        "--db-name", clp_db_connection_params["name"],
+        "--db-table-prefix", clp_db_connection_params["table_prefix"],
     ]
     # fmt: on
+    extract_env = {
+        **os.environ,
+        "CLP_DB_USER": clp_db_connection_params["username"],
+        "CLP_DB_PASS": clp_db_connection_params["password"],
+    }
 
     files_to_extract_list_path = None
     if list_path is not None:
@@ -256,7 +262,7 @@ def handle_extract_file_cmd(
         extract_cmd.append("-f")
         extract_cmd.append(str(files_to_extract_list_path))
 
-    proc = subprocess.Popen(extract_cmd)
+    proc = subprocess.Popen(extract_cmd, env=extract_env)
     return_code = proc.wait()
     if 0 != return_code:
         logger.error(f"File extraction failed, return_code={return_code}")
@@ -265,7 +271,6 @@ def handle_extract_file_cmd(
     # Remove generated files
     if files_to_extract_list_path is not None:
         files_to_extract_list_path.unlink()
-    db_config_file_path.unlink()
 
     return 0