feat(package): Add verbose logging option to archive_manager and improve error handling. (#1173)

Co-authored-by: Junhao Liao <[email protected]>

Author: haiqi96

components/clp-package-utils/clp_package_utils/scripts/archive_manager.py

@@ -64,6 +64,12 @@ def main(argv: typing.List[str]) -> int:
         default=str(default_config_file_path),
         help="CLP package configuration file.",
     )
+    args_parser.add_argument(
+        "--verbose",
+        "-v",
+        action="store_true",
+        help="Enable debug logging.",
+    )
     args_parser.add_argument(
         "--dataset",
         type=str,
@@ -148,6 +154,10 @@ def main(argv: typing.List[str]) -> int:
     )
 
     parsed_args: argparse.Namespace = args_parser.parse_args(argv[1:])
+    if parsed_args.verbose:
+        logger.setLevel(logging.DEBUG)
+    else:
+        logger.setLevel(logging.INFO)
 
     begin_timestamp: typing.Optional[int]
     end_timestamp: typing.Optional[int]
@@ -169,7 +179,7 @@ def main(argv: typing.List[str]) -> int:
 
     storage_type: StorageType = clp_config.archive_output.storage.type
     if StorageType.FS != storage_type:
-        logger.error(f"Archive deletion is not supported for storage type: {storage_type}.")
+        logger.error(f"Archive manager is not supported for storage type: {storage_type}.")
         return -1
 
     storage_engine: StorageEngine = clp_config.package.storage_engine
@@ -224,6 +234,9 @@ def main(argv: typing.List[str]) -> int:
     if dataset is not None:
         archive_manager_cmd.append("--dataset")
         archive_manager_cmd.append(dataset)
+    if parsed_args.verbose:
+        archive_manager_cmd.append("--verbose")
+
     archive_manager_cmd.append(subcommand)
 
     # Add subcommand-specific arguments
@@ -249,15 +262,20 @@ def main(argv: typing.List[str]) -> int:
             archive_manager_cmd.extend([END_TS_ARG, str(end_timestamp)])
     else:
         logger.error(f"Unsupported subcommand: `{subcommand}`.")
+        return -1
 
     cmd: typing.List[str] = container_start_cmd + archive_manager_cmd
 
-    subprocess.run(cmd, check=True)
+    proc = subprocess.run(cmd)
+    ret_code = proc.returncode
+    if 0 != ret_code:
+        logger.error("Archive manager failed.")
+        logger.debug(f"Docker command failed: {' '.join(cmd)}")
 
     # Remove generated files
     generated_config_path_on_host.unlink()
 
-    return 0
+    return ret_code
 
 
 if "__main__" == __name__:

components/clp-package-utils/clp_package_utils/scripts/native/archive_manager.py

@@ -95,6 +95,12 @@ def main(argv: typing.List[str]) -> int:
         default=str(default_config_file_path),
         help="CLP configuration file.",
     )
+    args_parser.add_argument(
+        "--verbose",
+        "-v",
+        action="store_true",
+        help="Enable debug logging.",
+    )
     args_parser.add_argument(
         "--dataset",
         type=str,
@@ -176,6 +182,10 @@ def main(argv: typing.List[str]) -> int:
     )
 
     parsed_args: argparse.Namespace = args_parser.parse_args(argv[1:])
+    if parsed_args.verbose:
+        logger.setLevel(logging.DEBUG)
+    else:
+        logger.setLevel(logging.INFO)
 
     # Validate and load config file
     config_file_path: Path = Path(parsed_args.config)
@@ -258,7 +268,8 @@ def _find_archives(
     :return: 0 on success, 1 on failure.
     """
     archive_ids: typing.List[str]
-    logger.info("Starting to find archives from the database.")
+    dataset_specific_message = f" of dataset `{dataset}`" if dataset is not None else ""
+    logger.info(f"Starting to find archives{dataset_specific_message} from the database.")
     try:
         sql_adapter: SQL_Adapter = SQL_Adapter(database_config)
         clp_db_connection_params: dict[str, any] = (
@@ -323,7 +334,8 @@ def _delete_archives(
     """
 
     archive_ids: typing.List[str]
-    logger.info("Starting to delete archives from the database.")
+    dataset_specific_message = f" of dataset `{dataset}`" if dataset is not None else ""
+    logger.info(f"Starting to delete archives{dataset_specific_message} from the database.")
     sql_adapter: SQL_Adapter = SQL_Adapter(database_config)
     clp_db_connection_params: dict[str, any] = database_config.get_clp_connection_params_and_type(
         True