Add LogStore to retain build logs, and a S3LogStore implementation

repo2docker/app.py

@@ -11,6 +11,7 @@
 import sys
 import logging
 import os
+import boto3
 import getpass
 import shutil
 import tempfile
@@ -326,6 +327,12 @@ def _user_name_default(self):
         config=True,
     )
 
+    s3_logs_endpoint = Unicode("", help="S3 endpoint", config=True)
+    s3_logs_access_key = Unicode("", help="S3 access key ", config=True)
+    s3_logs_secret_key = Unicode("", help="S3 secret key", config=True)
+    s3_logs_bucket = Unicode("", help="S3 bucket", config=True)
+    s3_logs_region = Unicode("", help="S3 region", config=True)
+
     all_ports = Bool(
         False,
         help="""
@@ -625,7 +632,7 @@ def find_image(self):
                         return True
         return False
 
-    def build(self):
+    def build(self, logfile=None):
         """
         Build docker image
         """
@@ -714,6 +721,8 @@ def build(self):
                         bp.__class__.__name__,
                         extra=dict(phase="building"),
                     )
+                    if logfile:
+                        logfile.write("Using %s builder\n" % bp.__class__.__name__)
 
                     for l in picked_buildpack.build(
                         docker_client,
@@ -725,8 +734,12 @@ def build(self):
                     ):
                         if "stream" in l:
                             self.log.info(l["stream"], extra=dict(phase="building"))
+                            if logfile:
+                                logfile.write(l["stream"])
                         elif "error" in l:
                             self.log.info(l["error"], extra=dict(phase="failure"))
+                            if logfile:
+                                logfile.write(l["error"])
                             raise docker.errors.BuildError(l["error"], build_log="")
                         elif "status" in l:
                             self.log.info(
@@ -741,7 +754,35 @@ def build(self):
                 shutil.rmtree(checkout_path, ignore_errors=True)
 
     def start(self):
-        self.build()
+        logfile = None
+        if (
+            self.s3_logs_endpoint
+            and self.s3_logs_access_key
+            and self.s3_logs_secret_key
+            and self.s3_logs_bucket
+        ):
+            logfile = tempfile.NamedTemporaryFile("w", delete=False)
+        try:
+            self.build(logfile=logfile)
+        finally:
+            if logfile:
+                logfile.close()
+                if os.stat(logfile.name).st_size:
+                    dest = f"buildlogs/{self.output_image_spec}/repo2docker.log"
+                    self.log.info(f"Uploading log to {self.s3_logs_bucket}/{dest}")
+                    s3 = boto3.resource(
+                        "s3",
+                        endpoint_url=self.s3_logs_endpoint,
+                        aws_access_key_id=self.s3_logs_access_key,
+                        aws_secret_access_key=self.s3_logs_secret_key,
+                        config=boto3.session.Config(signature_version="s3v4"),
+                        region_name=self.s3_logs_region,
+                    )
+                    s3.Bucket(self.s3_logs_bucket).upload_file(
+                        logfile.name,
+                        dest,
+                        ExtraArgs={"ContentType": "text/plain"},
+                    )
 
         if self.push:
             self.push_image()

setup.py

@@ -47,6 +47,7 @@ def get_identifier(json):
     name="jupyter-repo2docker",
     version=versioneer.get_version(),
     install_requires=[
+        "boto3",
         "docker",
         "traitlets",
         "python-json-logger",