Skip to content

Commit a2b7ad5

Browse files
harshad16harshad16
authored
consolidate the logs of python script into container logs (#579)
Signed-off-by: Harshad Reddy Nalla <[email protected]>
1 parent 8461fce commit a2b7ad5

File tree

8 files changed

+304
-112
lines changed

8 files changed

+304
-112
lines changed

‎runtimes/datascience/ubi8-python-3.8/utils/bootstrapper.py

Lines changed: 38 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -472,32 +472,43 @@ def execute(self) -> None:
472472
"""Execute the Python script and upload results to object storage"""
473473
python_script = os.path.basename(self.filepath)
474474
python_script_name = python_script.replace(".py", "")
475-
python_script_output = f"{python_script_name}.log"
475+
# python_script_output = f"{python_script_name}.log"
476476

477477
try:
478478
OpUtil.log_operation_info(
479-
f"executing python script using 'python3 {python_script}' to '{python_script_output}'"
479+
f"executing python script using 'python3 {python_script}'"
480480
)
481481
t0 = time.time()
482482

483483
run_args = ["python3", python_script]
484484
if self.parameter_pass_method == "env":
485485
self.set_parameters_in_env()
486-
487-
with open(python_script_output, "w") as log_file:
488-
subprocess.run(run_args, stdout=log_file, stderr=subprocess.STDOUT, check=True)
489-
486+
487+
logger.info("----------------------Python logs start----------------------")
488+
# Removing support for the s3 storage of python script logs
489+
# with open(python_script_output, "w") as log_file:
490+
# process = subprocess.Popen(run_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
491+
process = subprocess.Popen(run_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
492+
493+
for line in iter(process.stdout.readline, b''):
494+
sys.stdout.write(line.decode())
495+
496+
process.stdout.close()
497+
return_code = process.wait()
498+
logger.info("----------------------Python logs ends----------------------")
499+
if return_code:
500+
raise subprocess.CalledProcessError(return_code, run_args)
490501
duration = time.time() - t0
491502
OpUtil.log_operation_info("python script execution completed", duration)
492503

493-
self.put_file_to_object_storage(python_script_output, python_script_output)
504+
# self.put_file_to_object_storage(python_script_output, python_script_output)
494505
self.process_outputs()
495506
except Exception as ex:
496507
# log in case of errors
497508
logger.error(f"Unexpected error: {sys.exc_info()[0]}")
498509
logger.error(f"Error details: {ex}")
499510

500-
self.put_file_to_object_storage(python_script_output, python_script_output)
511+
# self.put_file_to_object_storage(python_script_output, python_script_output)
501512
raise ex
502513

503514

@@ -508,30 +519,42 @@ def execute(self) -> None:
508519
"""Execute the R script and upload results to object storage"""
509520
r_script = os.path.basename(self.filepath)
510521
r_script_name = r_script.replace(".r", "")
511-
r_script_output = f"{r_script_name}.log"
522+
# r_script_output = f"{r_script_name}.log"
512523

513524
try:
514-
OpUtil.log_operation_info(f"executing R script using 'Rscript {r_script}' to '{r_script_output}'")
525+
OpUtil.log_operation_info(f"executing R script using 'Rscript {r_script}'")
515526
t0 = time.time()
516527

517528
run_args = ["Rscript", r_script]
518529
if self.parameter_pass_method == "env":
519530
self.set_parameters_in_env()
520531

521-
with open(r_script_output, "w") as log_file:
522-
subprocess.run(run_args, stdout=log_file, stderr=subprocess.STDOUT, check=True)
532+
logger.info("----------------------R script logs start----------------------")
533+
# Removing support for the s3 storage of R script logs
534+
# with open(r_script_output, "w") as log_file:
535+
# process = subprocess.Popen(run_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
536+
process = subprocess.Popen(run_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
537+
538+
for line in iter(process.stdout.readline, b''):
539+
sys.stdout.write(line.decode())
540+
541+
process.stdout.close()
542+
return_code = process.wait()
543+
logger.info("----------------------R script logs ends----------------------")
544+
if return_code:
545+
raise subprocess.CalledProcessError(return_code, run_args)
523546

524547
duration = time.time() - t0
525548
OpUtil.log_operation_info("R script execution completed", duration)
526549

527-
self.put_file_to_object_storage(r_script_output, r_script_output)
550+
# self.put_file_to_object_storage(r_script_output, r_script_output)
528551
self.process_outputs()
529552
except Exception as ex:
530553
# log in case of errors
531554
logger.error(f"Unexpected error: {sys.exc_info()[0]}")
532555
logger.error(f"Error details: {ex}")
533556

534-
self.put_file_to_object_storage(r_script_output, r_script_output)
557+
# self.put_file_to_object_storage(r_script_output, r_script_output)
535558
raise ex
536559

537560

@@ -727,6 +750,7 @@ def main():
727750
input_params = OpUtil.parse_arguments(sys.argv[1:])
728751
OpUtil.log_operation_info("starting operation")
729752
t0 = time.time()
753+
OpUtil.package_install(user_volume_path=input_params.get("user-volume-path"))
730754

731755
# Create the appropriate instance, process dependencies and execute the operation
732756
file_op = FileOpBase.get_instance(**input_params)

‎runtimes/datascience/ubi9-python-3.9/utils/bootstrapper.py

Lines changed: 38 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -472,32 +472,43 @@ def execute(self) -> None:
472472
"""Execute the Python script and upload results to object storage"""
473473
python_script = os.path.basename(self.filepath)
474474
python_script_name = python_script.replace(".py", "")
475-
python_script_output = f"{python_script_name}.log"
475+
# python_script_output = f"{python_script_name}.log"
476476

477477
try:
478478
OpUtil.log_operation_info(
479-
f"executing python script using 'python3 {python_script}' to '{python_script_output}'"
479+
f"executing python script using 'python3 {python_script}'"
480480
)
481481
t0 = time.time()
482482

483483
run_args = ["python3", python_script]
484484
if self.parameter_pass_method == "env":
485485
self.set_parameters_in_env()
486-
487-
with open(python_script_output, "w") as log_file:
488-
subprocess.run(run_args, stdout=log_file, stderr=subprocess.STDOUT, check=True)
489-
486+
487+
logger.info("----------------------Python logs start----------------------")
488+
# Removing support for the s3 storage of python script logs
489+
# with open(python_script_output, "w") as log_file:
490+
# process = subprocess.Popen(run_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
491+
process = subprocess.Popen(run_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
492+
493+
for line in iter(process.stdout.readline, b''):
494+
sys.stdout.write(line.decode())
495+
496+
process.stdout.close()
497+
return_code = process.wait()
498+
logger.info("----------------------Python logs ends----------------------")
499+
if return_code:
500+
raise subprocess.CalledProcessError(return_code, run_args)
490501
duration = time.time() - t0
491502
OpUtil.log_operation_info("python script execution completed", duration)
492503

493-
self.put_file_to_object_storage(python_script_output, python_script_output)
504+
# self.put_file_to_object_storage(python_script_output, python_script_output)
494505
self.process_outputs()
495506
except Exception as ex:
496507
# log in case of errors
497508
logger.error(f"Unexpected error: {sys.exc_info()[0]}")
498509
logger.error(f"Error details: {ex}")
499510

500-
self.put_file_to_object_storage(python_script_output, python_script_output)
511+
# self.put_file_to_object_storage(python_script_output, python_script_output)
501512
raise ex
502513

503514

@@ -508,30 +519,42 @@ def execute(self) -> None:
508519
"""Execute the R script and upload results to object storage"""
509520
r_script = os.path.basename(self.filepath)
510521
r_script_name = r_script.replace(".r", "")
511-
r_script_output = f"{r_script_name}.log"
522+
# r_script_output = f"{r_script_name}.log"
512523

513524
try:
514-
OpUtil.log_operation_info(f"executing R script using 'Rscript {r_script}' to '{r_script_output}'")
525+
OpUtil.log_operation_info(f"executing R script using 'Rscript {r_script}'")
515526
t0 = time.time()
516527

517528
run_args = ["Rscript", r_script]
518529
if self.parameter_pass_method == "env":
519530
self.set_parameters_in_env()
520531

521-
with open(r_script_output, "w") as log_file:
522-
subprocess.run(run_args, stdout=log_file, stderr=subprocess.STDOUT, check=True)
532+
logger.info("----------------------R script logs start----------------------")
533+
# Removing support for the s3 storage of R script logs
534+
# with open(r_script_output, "w") as log_file:
535+
# process = subprocess.Popen(run_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
536+
process = subprocess.Popen(run_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
537+
538+
for line in iter(process.stdout.readline, b''):
539+
sys.stdout.write(line.decode())
540+
541+
process.stdout.close()
542+
return_code = process.wait()
543+
logger.info("----------------------R script logs ends----------------------")
544+
if return_code:
545+
raise subprocess.CalledProcessError(return_code, run_args)
523546

524547
duration = time.time() - t0
525548
OpUtil.log_operation_info("R script execution completed", duration)
526549

527-
self.put_file_to_object_storage(r_script_output, r_script_output)
550+
# self.put_file_to_object_storage(r_script_output, r_script_output)
528551
self.process_outputs()
529552
except Exception as ex:
530553
# log in case of errors
531554
logger.error(f"Unexpected error: {sys.exc_info()[0]}")
532555
logger.error(f"Error details: {ex}")
533556

534-
self.put_file_to_object_storage(r_script_output, r_script_output)
557+
# self.put_file_to_object_storage(r_script_output, r_script_output)
535558
raise ex
536559

537560

@@ -727,6 +750,7 @@ def main():
727750
input_params = OpUtil.parse_arguments(sys.argv[1:])
728751
OpUtil.log_operation_info("starting operation")
729752
t0 = time.time()
753+
OpUtil.package_install(user_volume_path=input_params.get("user-volume-path"))
730754

731755
# Create the appropriate instance, process dependencies and execute the operation
732756
file_op = FileOpBase.get_instance(**input_params)

‎runtimes/minimal/ubi8-python-3.8/utils/bootstrapper.py

Lines changed: 38 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -472,32 +472,43 @@ def execute(self) -> None:
472472
"""Execute the Python script and upload results to object storage"""
473473
python_script = os.path.basename(self.filepath)
474474
python_script_name = python_script.replace(".py", "")
475-
python_script_output = f"{python_script_name}.log"
475+
# python_script_output = f"{python_script_name}.log"
476476

477477
try:
478478
OpUtil.log_operation_info(
479-
f"executing python script using 'python3 {python_script}' to '{python_script_output}'"
479+
f"executing python script using 'python3 {python_script}'"
480480
)
481481
t0 = time.time()
482482

483483
run_args = ["python3", python_script]
484484
if self.parameter_pass_method == "env":
485485
self.set_parameters_in_env()
486-
487-
with open(python_script_output, "w") as log_file:
488-
subprocess.run(run_args, stdout=log_file, stderr=subprocess.STDOUT, check=True)
489-
486+
487+
logger.info("----------------------Python logs start----------------------")
488+
# Removing support for the s3 storage of python script logs
489+
# with open(python_script_output, "w") as log_file:
490+
# process = subprocess.Popen(run_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
491+
process = subprocess.Popen(run_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
492+
493+
for line in iter(process.stdout.readline, b''):
494+
sys.stdout.write(line.decode())
495+
496+
process.stdout.close()
497+
return_code = process.wait()
498+
logger.info("----------------------Python logs ends----------------------")
499+
if return_code:
500+
raise subprocess.CalledProcessError(return_code, run_args)
490501
duration = time.time() - t0
491502
OpUtil.log_operation_info("python script execution completed", duration)
492503

493-
self.put_file_to_object_storage(python_script_output, python_script_output)
504+
# self.put_file_to_object_storage(python_script_output, python_script_output)
494505
self.process_outputs()
495506
except Exception as ex:
496507
# log in case of errors
497508
logger.error(f"Unexpected error: {sys.exc_info()[0]}")
498509
logger.error(f"Error details: {ex}")
499510

500-
self.put_file_to_object_storage(python_script_output, python_script_output)
511+
# self.put_file_to_object_storage(python_script_output, python_script_output)
501512
raise ex
502513

503514

@@ -508,30 +519,42 @@ def execute(self) -> None:
508519
"""Execute the R script and upload results to object storage"""
509520
r_script = os.path.basename(self.filepath)
510521
r_script_name = r_script.replace(".r", "")
511-
r_script_output = f"{r_script_name}.log"
522+
# r_script_output = f"{r_script_name}.log"
512523

513524
try:
514-
OpUtil.log_operation_info(f"executing R script using 'Rscript {r_script}' to '{r_script_output}'")
525+
OpUtil.log_operation_info(f"executing R script using 'Rscript {r_script}'")
515526
t0 = time.time()
516527

517528
run_args = ["Rscript", r_script]
518529
if self.parameter_pass_method == "env":
519530
self.set_parameters_in_env()
520531

521-
with open(r_script_output, "w") as log_file:
522-
subprocess.run(run_args, stdout=log_file, stderr=subprocess.STDOUT, check=True)
532+
logger.info("----------------------R script logs start----------------------")
533+
# Removing support for the s3 storage of R script logs
534+
# with open(r_script_output, "w") as log_file:
535+
# process = subprocess.Popen(run_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
536+
process = subprocess.Popen(run_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
537+
538+
for line in iter(process.stdout.readline, b''):
539+
sys.stdout.write(line.decode())
540+
541+
process.stdout.close()
542+
return_code = process.wait()
543+
logger.info("----------------------R script logs ends----------------------")
544+
if return_code:
545+
raise subprocess.CalledProcessError(return_code, run_args)
523546

524547
duration = time.time() - t0
525548
OpUtil.log_operation_info("R script execution completed", duration)
526549

527-
self.put_file_to_object_storage(r_script_output, r_script_output)
550+
# self.put_file_to_object_storage(r_script_output, r_script_output)
528551
self.process_outputs()
529552
except Exception as ex:
530553
# log in case of errors
531554
logger.error(f"Unexpected error: {sys.exc_info()[0]}")
532555
logger.error(f"Error details: {ex}")
533556

534-
self.put_file_to_object_storage(r_script_output, r_script_output)
557+
# self.put_file_to_object_storage(r_script_output, r_script_output)
535558
raise ex
536559

537560

@@ -727,6 +750,7 @@ def main():
727750
input_params = OpUtil.parse_arguments(sys.argv[1:])
728751
OpUtil.log_operation_info("starting operation")
729752
t0 = time.time()
753+
OpUtil.package_install(user_volume_path=input_params.get("user-volume-path"))
730754

731755
# Create the appropriate instance, process dependencies and execute the operation
732756
file_op = FileOpBase.get_instance(**input_params)

0 commit comments

Comments
 (0)