Merge pull request #1158 from betatim/image-cleaning-with-buster

Author: betatim

doc/doc-requirements.txt

@@ -17,18 +17,4 @@ sphinx-copybutton
 # automatically building documentation based on inspection of the binderhub
 # package, which means we need to install it on RTD so it is available for
 # inspection.
-#
-# The binderhub package dependencies include pycurl though, and pycurl cannot be
-# installed on RTD, so due to that we maintain a copy of binderhub's
-# requirements.txt where we comment out pyrcurl.
-docker
-escapism
-jinja2
-jsonschema
-jupyterhub
-kubernetes
-prometheus_client
-#pycurl
-python-json-logger
-tornado>=5.1
-traitlets
+-r ../requirements.txt

helm-chart/images/binderhub/Dockerfile

@@ -1,48 +1,59 @@
-# Using multi-stage builds
-ARG DIST=buster
-FROM buildpack-deps:$DIST as build-stage
-# ARG has to occur twice to be used in both contents and FROM
+# We use a build stage to package binderhub and pycurl into a wheel which we
+# then install by itself in the final image which is relatively slimmed.
 ARG DIST=buster
 
+
+# The build stage
+# ---------------
+FROM python:3.8-$DIST as build-stage
+# ARG DIST is defined again to be made available in this build stage's scope.
+# ref: https://docs.docker.com/engine/reference/builder/#understand-how-arg-and-from-interact
+ARG DIST
+
+# Install node as required to package binderhub to a wheel
 RUN echo "deb http://deb.nodesource.com/node_14.x $DIST main" > /etc/apt/sources.list.d/nodesource.list \
  && curl -s https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add -
-
-RUN apt-get update && \
-    apt-get install --yes \
+RUN apt-get update \
+ && apt-get install --yes \
         nodejs \
-        python3 \
-        python3-pip \
-        python3-wheel \
-        python3-setuptools
+ && rm -rf /var/lib/apt/lists/*
 
 # Copy the whole git repository to /tmp/binderhub
 COPY . /tmp/binderhub
 WORKDIR /tmp/binderhub
 
-# Build binderhub the python library
-RUN python3 setup.py bdist_wheel
+# Build the binderhub python library into a wheel and save it to the ./dist
+# folder. There are no pycurl wheels so we build our own in the build stage.
+RUN python setup.py bdist_wheel
+RUN pip wheel pycurl --wheel-dir ./dist
+
 
 # The final stage
 # ---------------
-FROM python:3.8-$DIST
+FROM python:3.8-slim-$DIST
 WORKDIR /
 
-# Copy the built binderhub python wheel from the build-stage
-# to the current directory within the container.
-COPY --from=build-stage /tmp/binderhub/dist/*.whl .
+# The slim version doesn't include git as required by binderhub
+RUN apt-get update \
+ && apt-get install --yes \
+        git \
+ && rm -rf /var/lib/apt/lists/*
 
-# Copy the additional Python requirements for our Docker container from the
-# build context. These can be certain pinned versions or peer dependencies.
+
+# Copy the built wheels from the build-stage. Also copy the image
+# requirements.txt built from the binderhub package requirements.txt and the
+# requirements.in file using the ./dependency script.
+COPY --from=build-stage /tmp/binderhub/dist/*.whl pre-built-wheels/
 COPY helm-chart/images/binderhub/requirements.txt .
+
+# Install pre-built wheels and the generated requirements.txt for the image.
 RUN pip install --no-cache-dir \
-        *.whl \
+        pre-built-wheels/*.whl \
         -r requirements.txt
 
-# when building the image used to compute the "frozen"
-# dependencies we add `pip-tools` which is for computing
-# the dependencies.
-# It is not added when chartpress builds the image used by
-# the helm chart when deploying BinderHub
+# When using the ./dependency script to output a frozen environment, we do it
+# from within this container. So below we conditionally install pip-tools for
+# use by the ./dependency script.
 ARG PIP_TOOLS=
 RUN test -z "$PIP_TOOLS" || pip install --no-cache pip-tools==$PIP_TOOLS
 

helm-chart/images/binderhub/requirements.in

@@ -11,6 +11,3 @@ google-cloud-logging
 kubernetes==9.*
 # jupyterhub is pinned to match the JupyterHub Helm chart's version of jupyterhub
 jupyterhub==1.1.*
-# For pycurl v7.43.0.6 there are no wheels available on pypi.org, so stick
-# to the previous version for now.
-pycurl==7.43.0.5

helm-chart/images/binderhub/requirements.txt

@@ -38,7 +38,6 @@ protobuf==3.14.0          # via google-api-core, googleapis-common-protos
 pyasn1-modules==0.2.8     # via google-auth
 pyasn1==0.4.8             # via pyasn1-modules, rsa
 pycparser==2.20           # via cffi
-pycurl==7.43.0.5          # via -r binderhub.in, -r requirements.in
 pyopenssl==19.1.0         # via certipy
 pyrsistent==0.17.3        # via jsonschema
 python-dateutil==2.8.1    # via alembic, jupyterhub, kubernetes

helm-chart/images/image-cleaner/Dockerfile

@@ -1,9 +1,9 @@
-FROM python:3.8-alpine3.11
+FROM python:3.8-slim-buster
 
-ADD requirements.txt /tmp/requirements.txt
-RUN pip3 install --no-cache-dir -r /tmp/requirements.txt
+COPY requirements.txt /tmp/requirements.txt
+RUN pip install --no-cache-dir -r /tmp/requirements.txt
 
-ADD image-cleaner.py /usr/local/bin/image-cleaner.py
+COPY image-cleaner.py /usr/local/bin/image-cleaner.py
 # set PYTHONUNBUFFERED to ensure output is produced
 ENV PYTHONUNBUFFERED=1
 CMD image-cleaner.py

requirements.txt

@@ -1,12 +1,19 @@
-# Keep this file in sync with `doc/doc-requirements.txt`!
+# About pycurl:
+# - pycurl is a dependency but not listed here, only in `setup.py`
+# - pycurl requires both `curl-config` and `gcc` to be available when installing
+#   it from source.
+# - the reason we do not list pycurl here is that it is not needed when our
+#   code is imported to generate the documentation and it is tricky to install
+#   in the ReadTheDocs build environment.
+# - instead we manually add pycurl to the list of dependencies in setup.py
+#   which is the list used when installing the binderhub package.
 docker
 escapism
 jinja2
 jsonschema
 jupyterhub
 kubernetes
 prometheus_client
-pycurl
 python-json-logger
 tornado>=5.1
 traitlets

setup.py

@@ -13,6 +13,8 @@
         l.strip() for l in f.readlines()
         if not l.strip().startswith('#')
     ]
+    # manually add pycurl here, see comment in requirements.txt
+    requirements.append("pycurl")
 
 with open(os.path.join(here, 'README.rst'), encoding="utf8") as f:
     readme = f.read()