Skip to content

Commit 35760f7

Browse files
haiqi96junhaoliao
haiqi96
and
junhaoliao
authored
feat(package): Add --verbose option to package_utils scripts and improve error diagnostics (fixes #1081). (#1212)
Co-authored-by: Junhao Liao <[email protected]>
1 parent c7804f2 commit 35760f7

File tree

9 files changed

+123
-25
lines changed

9 files changed

+123
-25
lines changed

‎components/clp-package-utils/clp_package_utils/scripts/archive_manager.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import argparse
22
import logging
3+
import shlex
34
import subprocess
45
import sys
56
from pathlib import Path
@@ -238,11 +239,11 @@ def main(argv: List[str]) -> int:
238239
"--config", str(generated_config_path_on_container),
239240
]
240241
# fmt : on
242+
if parsed_args.verbose:
243+
archive_manager_cmd.append("--verbose")
241244
if dataset is not None:
242245
archive_manager_cmd.append("--dataset")
243246
archive_manager_cmd.append(dataset)
244-
if parsed_args.verbose:
245-
archive_manager_cmd.append("--verbose")
246247

247248
archive_manager_cmd.append(subcommand)
248249

@@ -277,7 +278,7 @@ def main(argv: List[str]) -> int:
277278
ret_code = proc.returncode
278279
if 0 != ret_code:
279280
logger.error("Archive manager failed.")
280-
logger.debug(f"Docker command failed: {' '.join(cmd)}")
281+
logger.debug(f"Docker command failed: {shlex.join(cmd)}")
281282

282283
# Remove generated files
283284
generated_config_path_on_host.unlink()

‎components/clp-package-utils/clp_package_utils/scripts/compress.py‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import argparse
22
import logging
33
import pathlib
4+
import shlex
45
import subprocess
56
import sys
67
import uuid
@@ -82,6 +83,8 @@ def _generate_compress_cmd(
8283
"--config", str(config_path),
8384
]
8485
# fmt: on
86+
if parsed_args.verbose:
87+
compress_cmd.append("--verbose")
8588
if dataset is not None:
8689
compress_cmd.append("--dataset")
8790
compress_cmd.append(dataset)
@@ -142,6 +145,12 @@ def main(argv):
142145
default=str(default_config_file_path),
143146
help="CLP package configuration file.",
144147
)
148+
args_parser.add_argument(
149+
"--verbose",
150+
"-v",
151+
action="store_true",
152+
help="Enable debug logging.",
153+
)
145154
args_parser.add_argument(
146155
"--dataset",
147156
type=str,
@@ -166,6 +175,10 @@ def main(argv):
166175
)
167176

168177
parsed_args = args_parser.parse_args(argv[1:])
178+
if parsed_args.verbose:
179+
logger.setLevel(logging.DEBUG)
180+
else:
181+
logger.setLevel(logging.INFO)
169182

170183
# Validate and load config file
171184
try:
@@ -235,13 +248,19 @@ def main(argv):
235248
compress_cmd = _generate_compress_cmd(
236249
parsed_args, dataset, generated_config_path_on_container, logs_list_path_on_container
237250
)
251+
238252
cmd = container_start_cmd + compress_cmd
239-
subprocess.run(cmd, check=True)
253+
254+
proc = subprocess.run(cmd)
255+
ret_code = proc.returncode
256+
if 0 != ret_code:
257+
logger.error("Compression failed.")
258+
logger.debug(f"Docker command failed: {shlex.join(cmd)}")
240259

241260
# Remove generated files
242261
generated_config_path_on_host.unlink()
243262

244-
return 0
263+
return ret_code
245264

246265

247266
if "__main__" == __name__:

‎components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import argparse
22
import logging
3+
import shlex
34
import subprocess
45
import sys
56
from pathlib import Path
@@ -162,6 +163,8 @@ def main(argv: List[str]) -> int:
162163
"--config", str(generated_config_path_on_container),
163164
]
164165
# fmt: on
166+
if parsed_args.verbose:
167+
dataset_manager_cmd.append("--verbose")
165168

166169
dataset_manager_cmd.append(subcommand)
167170

@@ -181,7 +184,7 @@ def main(argv: List[str]) -> int:
181184
ret_code = proc.returncode
182185
if 0 != ret_code:
183186
logger.error("Dataset manager failed.")
184-
logger.debug(f"Docker command failed: {' '.join(cmd)}")
187+
logger.debug(f"Docker command failed: {shlex.join(cmd)}")
185188

186189
# Remove generated files
187190
generated_config_path_on_host.unlink()

‎components/clp-package-utils/clp_package_utils/scripts/decompress.py‎

Lines changed: 30 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import argparse
22
import logging
33
import pathlib
4+
import shlex
45
import subprocess
56
import sys
67
from typing import Optional
@@ -69,7 +70,7 @@ def handle_extract_file_cmd(
6970
:param parsed_args:
7071
:param clp_home:
7172
:param default_config_file_path:
72-
:return: 0 on success, -1 otherwise.
73+
:return: exit code of extraction command, or -1 if an error is encountered.
7374
"""
7475
paths_to_extract_file_path = None
7576
if parsed_args.files_from:
@@ -143,23 +144,26 @@ def handle_extract_file_cmd(
143144
"-d", str(container_extraction_dir),
144145
]
145146
# fmt: on
147+
if parsed_args.verbose:
148+
extract_cmd.append("--verbose")
146149
for path in parsed_args.paths:
147150
extract_cmd.append(path)
148151
if container_paths_to_extract_file_path:
149152
extract_cmd.append("--input-list")
150153
extract_cmd.append(container_paths_to_extract_file_path)
151154

152155
cmd = container_start_cmd + extract_cmd
153-
try:
154-
subprocess.run(cmd, check=True)
155-
except subprocess.CalledProcessError:
156-
logger.exception("Docker or file extraction command failed.")
157-
return -1
156+
157+
proc = subprocess.run(cmd)
158+
ret_code = proc.returncode
159+
if 0 != ret_code:
160+
logger.error("file extraction failed.")
161+
logger.debug(f"Docker command failed: {shlex.join(cmd)}")
158162

159163
# Remove generated files
160164
generated_config_path_on_host.unlink()
161165

162-
return 0
166+
return ret_code
163167

164168

165169
def handle_extract_stream_cmd(
@@ -170,7 +174,7 @@ def handle_extract_stream_cmd(
170174
:param parsed_args:
171175
:param clp_home:
172176
:param default_config_file_path:
173-
:return: 0 on success, -1 otherwise.
177+
:return: exit code of extraction command, or -1 if an error is encountered.
174178
"""
175179
# Validate and load config file
176180
clp_config = validate_and_load_config(
@@ -218,6 +222,8 @@ def handle_extract_stream_cmd(
218222
job_command
219223
]
220224
# fmt: on
225+
if parsed_args.verbose:
226+
extract_cmd.append("--verbose")
221227

222228
if EXTRACT_IR_CMD == job_command:
223229
extract_cmd.append(str(parsed_args.msg_ix))
@@ -253,16 +259,16 @@ def handle_extract_stream_cmd(
253259

254260
cmd = container_start_cmd + extract_cmd
255261

256-
try:
257-
subprocess.run(cmd, check=True)
258-
except subprocess.CalledProcessError:
259-
logger.exception("Docker or stream extraction command failed.")
260-
return -1
262+
proc = subprocess.run(cmd)
263+
ret_code = proc.returncode
264+
if 0 != ret_code:
265+
logger.error("stream extraction failed.")
266+
logger.debug(f"Docker command failed: {shlex.join(cmd)}")
261267

262268
# Remove generated files
263269
generated_config_path_on_host.unlink()
264270

265-
return 0
271+
return ret_code
266272

267273

268274
def main(argv):
@@ -276,6 +282,12 @@ def main(argv):
276282
default=str(default_config_file_path),
277283
help="CLP configuration file.",
278284
)
285+
args_parser.add_argument(
286+
"--verbose",
287+
"-v",
288+
action="store_true",
289+
help="Enable debug logging.",
290+
)
279291
command_args_parser = args_parser.add_subparsers(dest="command", required=True)
280292

281293
# File extraction command parser
@@ -317,6 +329,10 @@ def main(argv):
317329
)
318330

319331
parsed_args = args_parser.parse_args(argv[1:])
332+
if parsed_args.verbose:
333+
logger.setLevel(logging.DEBUG)
334+
else:
335+
logger.setLevel(logging.INFO)
320336

321337
command = parsed_args.command
322338
if EXTRACT_FILE_CMD == command:

‎components/clp-package-utils/clp_package_utils/scripts/native/compress.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,12 @@ def main(argv):
199199
default=str(default_config_file_path),
200200
help="CLP package configuration file.",
201201
)
202+
args_parser.add_argument(
203+
"--verbose",
204+
"-v",
205+
action="store_true",
206+
help="Enable debug logging.",
207+
)
202208
args_parser.add_argument(
203209
"--dataset",
204210
type=str,
@@ -223,6 +229,10 @@ def main(argv):
223229
"-t", "--tags", help="A comma-separated list of tags to apply to the compressed archives."
224230
)
225231
parsed_args = args_parser.parse_args(argv[1:])
232+
if parsed_args.verbose:
233+
logger.setLevel(logging.DEBUG)
234+
else:
235+
logger.setLevel(logging.INFO)
226236

227237
# Validate and load config file
228238
try:

‎components/clp-package-utils/clp_package_utils/scripts/native/decompress.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -290,6 +290,12 @@ def main(argv):
290290
default=str(default_config_file_path),
291291
help="CLP configuration file.",
292292
)
293+
args_parser.add_argument(
294+
"--verbose",
295+
"-v",
296+
action="store_true",
297+
help="Enable debug logging.",
298+
)
293299
command_args_parser = args_parser.add_subparsers(dest="command", required=True)
294300

295301
# File extraction command parser
@@ -329,6 +335,10 @@ def main(argv):
329335
)
330336

331337
parsed_args = args_parser.parse_args(argv[1:])
338+
if parsed_args.verbose:
339+
logger.setLevel(logging.DEBUG)
340+
else:
341+
logger.setLevel(logging.INFO)
332342

333343
command = parsed_args.command
334344
if EXTRACT_FILE_CMD == command:

‎components/clp-package-utils/clp_package_utils/scripts/native/search.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -240,6 +240,12 @@ def main(argv):
240240

241241
args_parser = argparse.ArgumentParser(description="Searches the compressed logs.")
242242
args_parser.add_argument("--config", "-c", required=True, help="CLP configuration file.")
243+
args_parser.add_argument(
244+
"--verbose",
245+
"-v",
246+
action="store_true",
247+
help="Enable debug logging.",
248+
)
243249
args_parser.add_argument("wildcard_query", help="Wildcard query.")
244250
args_parser.add_argument(
245251
"--dataset",
@@ -281,6 +287,10 @@ def main(argv):
281287
"--raw", action="store_true", help="Output the search results as raw logs."
282288
)
283289
parsed_args = args_parser.parse_args(argv[1:])
290+
if parsed_args.verbose:
291+
logger.setLevel(logging.DEBUG)
292+
else:
293+
logger.setLevel(logging.INFO)
284294

285295
if (
286296
parsed_args.begin_time is not None

‎components/clp-package-utils/clp_package_utils/scripts/search.py‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import argparse
22
import logging
33
import pathlib
4+
import shlex
45
import subprocess
56
import sys
67

@@ -40,6 +41,12 @@ def main(argv):
4041
default=str(default_config_file_path),
4142
help="CLP package configuration file.",
4243
)
44+
args_parser.add_argument(
45+
"--verbose",
46+
"-v",
47+
action="store_true",
48+
help="Enable debug logging.",
49+
)
4350
args_parser.add_argument("wildcard_query", help="Wildcard query.")
4451
args_parser.add_argument(
4552
"--dataset",
@@ -76,6 +83,10 @@ def main(argv):
7683
"--raw", action="store_true", help="Output the search results as raw logs."
7784
)
7885
parsed_args = args_parser.parse_args(argv[1:])
86+
if parsed_args.verbose:
87+
logger.setLevel(logging.DEBUG)
88+
else:
89+
logger.setLevel(logging.INFO)
7990

8091
# Validate and load config file
8192
try:
@@ -134,6 +145,8 @@ def main(argv):
134145
parsed_args.wildcard_query,
135146
]
136147
# fmt: on
148+
if parsed_args.verbose:
149+
search_cmd.append("--verbose")
137150
if dataset is not None:
138151
search_cmd.append("--dataset")
139152
search_cmd.append(dataset)
@@ -159,12 +172,17 @@ def main(argv):
159172
if parsed_args.raw:
160173
search_cmd.append("--raw")
161174
cmd = container_start_cmd + search_cmd
162-
subprocess.run(cmd, check=True)
175+
176+
proc = subprocess.run(cmd)
177+
ret_code = proc.returncode
178+
if 0 != ret_code:
179+
logger.error("Search failed.")
180+
logger.debug(f"Docker command failed: {shlex.join(cmd)}")
163181

164182
# Remove generated files
165183
generated_config_path_on_host.unlink()
166184

167-
return 0
185+
return ret_code
168186

169187

170188
if "__main__" == __name__:

0 commit comments

Comments
 (0)