diff --git a/.generator/Dockerfile b/.generator/Dockerfile index 0d98536ab42a..375aaabd9941 100644 --- a/.generator/Dockerfile +++ b/.generator/Dockerfile @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +# --- Builder Stage --- +# This stage installs all build dependencies and compiles all Python versions. FROM marketplace.gcr.io/google/ubuntu2404 AS builder # TODO(https://github.com/googleapis/librarian/issues/901): Install the necssary dependencies and build tools. @@ -38,54 +40,85 @@ RUN apt-get update && \ # Set up environment variables for tool versions to make updates easier. ENV BAZELISK_VERSION=v1.26.0 +# Install multiple Python versions from source. `make altinstall` is used to +# prevent replacing the system's default python binary. RUN for PYTHON_VERSION in 3.9.23 3.10.18 3.11.13 3.12.11 3.13.5; do \ - # Install Python from source wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz && \ tar -xvf Python-${PYTHON_VERSION}.tgz && \ cd Python-${PYTHON_VERSION} && \ - ./configure --enable-optimizations && \ + ./configure --enable-optimizations --prefix=/usr/local && \ + make -j$(nproc) && \ make altinstall && \ cd / && \ rm -rf Python-${PYTHON_VERSION}* \ ; done -# Get the pip installation script using the instructions below -# https://pip.pypa.io/en/stable/installation/#get-pip-py -# Attempts to use `ensurepip` instead of `get-pip.py` resulted in errors like `python3.13.5 not found` -RUN wget --no-check-certificate -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' - # Install pip for each python version -RUN for PYTHON_VERSION in 3.9 3.10 3.11 3.12 3.13; do \ - python${PYTHON_VERSION} /tmp/get-pip.py \ - ; done - -# Remove the pip installation script -RUN rm /tmp/get-pip.py - -# Test Pip -RUN for PYTHON_VERSION in 3.9 3.10 3.11 3.12 3.13; do \ - python${PYTHON_VERSION} -m pip \ - ; done - - -# TODO(https://github.com/googleapis/librarian/issues/904): Install protoc for gencode. +RUN wget --no-check-certificate -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' && \ + for PYTHON_VERSION in 3.9 3.10 3.11 3.12 3.13; do \ + python${PYTHON_VERSION} /tmp/get-pip.py; \ + done && \ + rm /tmp/get-pip.py # Install Bazelisk RUN wget https://github.com/bazelbuild/bazelisk/releases/download/${BAZELISK_VERSION}/bazelisk-linux-amd64 -O /usr/local/bin/bazelisk && \ chmod +x /usr/local/bin/bazelisk -# TODO(https://github.com/googleapis/librarian/issues/902): Create a dedicate non-root user and -# switch to the non-root user to run subsequent commands. - -# Set the working directory in the container. +# Set the working directory for build-related tasks. WORKDIR /app +# TODO(https://github.com/googleapis/librarian/issues/904): Install protoc for gencode. # TODO(https://github.com/googleapis/librarian/issues/907): Install Python dependencies from requirements.in. # TODO(https://github.com/googleapis/librarian/issues/905): Install Synthtool by cloning its repo. # TODO(https://github.com/googleapis/librarian/issues/906): Clone googleapis and run bazelisk build. -# Copy the CLI script into the container and set ownership. +# --- Final Stage --- +# This stage creates the lightweight final image, copying only the +# necessary artifacts from the builder stage. +FROM marketplace.gcr.io/google/ubuntu2404 + +# Install only the essential runtime libraries for Python. +# These are the non "-dev" versions of the libraries used in the builder. +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + libssl3 \ + zlib1g \ + libbz2-1.0 \ + libffi8 \ + libsqlite3-0 \ + libreadline8 \ + && apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# TODO(https://github.com/googleapis/librarian/issues/902): Create a dedicate non-root user and +# switch to the non-root user to run subsequent commands. +# Example: +# RUN groupadd --system --gid 1000 appgroup && \ +# useradd --system --uid 1000 --gid appgroup appuser +# USER appuser + +# Copy all Python interpreters, their pip executables, and their standard libraries from the builder. +COPY --from=builder /usr/local/bin/python3.9 /usr/local/bin/ +COPY --from=builder /usr/local/lib/python3.9 /usr/local/lib/python3.9 + +COPY --from=builder /usr/local/bin/python3.10 /usr/local/bin/ +COPY --from=builder /usr/local/lib/python3.10 /usr/local/lib/python3.10 + +COPY --from=builder /usr/local/bin/python3.11 /usr/local/bin/ +COPY --from=builder /usr/local/lib/python3.11 /usr/local/lib/python3.11 + +COPY --from=builder /usr/local/bin/python3.12 /usr/local/bin/ +COPY --from=builder /usr/local/lib/python3.12 /usr/local/lib/python3.12 + +COPY --from=builder /usr/local/bin/python3.13 /usr/local/bin/ +COPY --from=builder /usr/local/lib/python3.13 /usr/local/lib/python3.13 + +# Set the working directory in the container. +WORKDIR /app + +# Copy the CLI script into the container. COPY .generator/cli.py . # Set the entrypoint for the container to run the script. -ENTRYPOINT ["python3.11", "./cli.py"] +ENTRYPOINT ["python3.11", "./cli.py"] \ No newline at end of file diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 6c3dba47ec10..13903f79a9d4 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -14,18 +14,30 @@ # TODO(https://github.com/googleapis/google-cloud-python/issues/14142): # Reduce this timeout by moving the installation of Python runtimes to a separate base image -timeout: 7200s # 2 hours +timeout: 7200s # 2 hours for the first uncached run, can be lowered later. steps: - # This step builds the Docker image. + # STEP 1: Pull the latest version of the image to use as a cache. + # The '|| exit 0' part ensures the build doesn't fail if the image + # doesn't exist yet (e.g., on the very first run). + - name: 'gcr.io/cloud-builders/docker' + entrypoint: 'bash' + args: ['-c', 'docker pull gcr.io/$PROJECT_ID/python-librarian-generator:latest || exit 0'] + + # STEP 2: Build the new image, using the pulled image as a cache source. - name: 'gcr.io/cloud-builders/docker' args: - 'build' - - '-t' - - 'gcr.io/$PROJECT_ID/python-librarian-generator:latest' - - '-f' - - '.generator/Dockerfile' + - '--tag=gcr.io/$PROJECT_ID/python-librarian-generator:latest' + - '--cache-from=gcr.io/$PROJECT_ID/python-librarian-generator:latest' + - '--file=.generator/Dockerfile' - '.' -# This section automatically create a storage bucket for storing docker build logs. +# STEP 3: Push the newly built image to the registry so it can be used +# as the cache for the next run. Cloud Build does this automatically if +# the image is listed in the 'images' section. +images: + - 'gcr.io/$PROJECT_ID/python-librarian-generator:latest' + +# This section automatically creates a storage bucket for storing docker build logs. options: - default_logs_bucket_behavior: REGIONAL_USER_OWNED_BUCKET + default_logs_bucket_behavior: REGIONAL_USER_OWNED_BUCKET \ No newline at end of file