Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
5bdb746
chore(librarian): add ability to parse BUILD.bazel
parthea Sep 9, 2025
472c280
update docstring
parthea Sep 9, 2025
31dbb8e
add copyright header
parthea Sep 9, 2025
b59962a
address review feedback
parthea Sep 9, 2025
9e100a5
trim BUILD.bazel
parthea Sep 10, 2025
86c72b9
address feedback
parthea Sep 10, 2025
b8814ff
lint
parthea Sep 10, 2025
950350a
chore(librarian): migrate off of bazel for generate command
parthea Sep 3, 2025
aa6d60c
Merge branch 'main' into migrate-off-bazel
parthea Sep 10, 2025
5ce78cd
add requirements.in
parthea Sep 10, 2025
8b20163
copy requirements.in
parthea Sep 10, 2025
3c21b0d
Add missing file
parthea Sep 10, 2025
4578e14
refactor
parthea Sep 10, 2025
44817e0
lint
parthea Sep 10, 2025
62a5e1a
revert
parthea Sep 10, 2025
9718839
Move protoc download to builder step
parthea Sep 11, 2025
3938cef
move pandoc to builder step
parthea Sep 11, 2025
f499595
remove git from container
parthea Sep 11, 2025
2a8c9fd
remove apt-get install pandoc
parthea Sep 11, 2025
d34acca
add support for optional arguments
parthea Sep 16, 2025
65f3c28
migrate to shutil.copytree
parthea Sep 16, 2025
93c1a0b
Add comment
parthea Sep 16, 2025
3ed45fc
add comment
parthea Sep 16, 2025
234ae23
address feedback
parthea Sep 16, 2025
4da3e46
move env; remove obsolete code
parthea Sep 16, 2025
959d499
coverage
parthea Sep 16, 2025
d32ee67
add comment
parthea Sep 16, 2025
9bb0e3c
run black/isort
parthea Sep 16, 2025
c6a9b7f
remove erroneous import
parthea Sep 16, 2025
2ec1ce7
address review feedback
parthea Sep 16, 2025
8b182cf
address review feedback
parthea Sep 16, 2025
fcb444c
Merge branch 'main' into migrate-off-bazel
parthea Sep 16, 2025
097a54c
address review feedback
parthea Sep 16, 2025
fe18bf6
add tests
parthea Sep 16, 2025
faccd69
address review feedback
parthea Sep 16, 2025
2665fc6
add assert
parthea Sep 17, 2025
76c4061
add assert
parthea Sep 17, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
125 changes: 40 additions & 85 deletions .generator/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -16,22 +16,15 @@
# This stage installs all build dependencies and compiles all Python versions.
FROM marketplace.gcr.io/google/ubuntu2404 AS builder

# TODO(https://github.com/googleapis/librarian/issues/901): Install the necssary dependencies and build tools.
RUN apt-get update && \
apt-get install -y --no-install-recommends \
# Essential for compiling C code
build-essential \
# For downloading secure files
# For downloading and extracting secure files
git \
wget \
ca-certificates \
# For running bazelisk commands
openjdk-17-jdk \
zip \
unzip \
# To avoid bazel error
# "python interpreter `python3` not found in PATH"
python3-dev \
unzip \
# --- Critical libraries for a complete Python build ---
libssl-dev \
zlib1g-dev \
Expand All @@ -43,9 +36,6 @@ RUN apt-get update && \
&& apt-get clean && \
rm -rf /var/lib/apt/lists/*

# Set up environment variables for tool versions to make updates easier.
ENV BAZELISK_VERSION=v1.26.0

# Install multiple Python versions from source. `make altinstall` is used to
# prevent replacing the system's default python binary.
# TODO(http://github.com/googleapis/gapic-generator-python/issues/2435): Remove `3.10.18` when the linked issue is resolved.
Expand All @@ -68,77 +58,48 @@ RUN wget --no-check-certificate -O /tmp/get-pip.py 'https://bootstrap.pypa.io/ge
done && \
rm /tmp/get-pip.py

# Install Bazelisk
RUN wget https://github.com/bazelbuild/bazelisk/releases/download/${BAZELISK_VERSION}/bazelisk-linux-amd64 -O /usr/local/bin/bazelisk && \
chmod +x /usr/local/bin/bazelisk

# Set the working directory for build-related tasks.
WORKDIR /app

# Create the group and user, but only if they don't already exist.
ARG UID=1000
ARG GID=1000

RUN if ! getent group $GID > /dev/null; then \
groupadd -g $GID myuser; \
fi && \
if ! getent passwd $UID > /dev/null; then \
useradd -u $UID -g $GID -ms /bin/bash myuser; \
fi

# Set ownership of the app directory now, before we copy files into it.
RUN mkdir -p /app && chown $UID:$GID /app

# We'll point both to the /bazel_cache directory which will be mounted as a volume.
ENV BAZELISK_HOME="/bazel_cache/bazelisk"
ENV BAZEL_HOME="/bazel_cache/bazel"

# Ensure the cache directories within the non-root user's context exist and are writable.
# This is crucial as Bazel creates subdirectories under BAZEL_HOME.
RUN mkdir -p ${BAZEL_HOME}/_bazel_ubuntu/cache/repos \
${BAZEL_HOME}/_bazel_ubuntu/output_base \
${BAZELISK_HOME} && \
chown -R $UID:$GID ${BAZEL_HOME} ${BAZELISK_HOME}

RUN /usr/local/bin/python3.9 -m venv bazel_env
RUN . bazel_env/bin/activate

RUN git clone https://github.com/googleapis/googleapis.git \
&& cd googleapis \
&& bazelisk --output_base=/bazel_cache/_bazel_ubuntu/output_base build --disk_cache=/bazel_cache/_bazel_ubuntu/cache/repos --incompatible_strict_action_env //google/cloud/language/v1:language-v1-py
# Download/extract protoc
RUN wget https://github.com/protocolbuffers/protobuf/releases/download/v25.3/protoc-25.3-linux-x86_64.zip
RUN unzip protoc-25.3-linux-x86_64.zip -d protoc

# TODO(https://github.com/googleapis/librarian/issues/904): Install protoc for gencode.
# Download/extract pandoc
# Pandoc is required by gapic-generator-python for parsing documentation
RUN wget https://github.com/jgm/pandoc/releases/download/3.7.0.2/pandoc-3.7.0.2-linux-amd64.tar.gz
RUN tar -xvf pandoc-3.7.0.2-linux-amd64.tar.gz

# Download synthtool
RUN git clone --depth 1 https://github.com/googleapis/synthtool.git synthtool

# --- Final Stage ---
# This stage creates the lightweight final image, copying only the
# necessary artifacts from the builder stage.
FROM marketplace.gcr.io/google/ubuntu2404

# Tell synthtool to pull templates from this docker image instead of from
# the live repo.
ENV SYNTHTOOL_TEMPLATES="/synthtool/synthtool/gcp/templates"

# Install only the essential runtime libraries for Python.
# These are the non "-dev" versions of the libraries used in the builder.
RUN apt-get update && \
apt-get install -y --no-install-recommends \
ca-certificates \
git \
libssl3 \
zlib1g \
libbz2-1.0 \
libffi8 \
# This is needed to avoid the following error:
# `ImportError: libsqlite3.so.0: cannot open shared object file: No such file or directory`.
# `libsqlite3-0` is used by the `coverage` PyPI package which is used when testing libraries
libsqlite3-0 \
libreadline8 \
# For running bazelisk commands
openjdk-17-jdk \
# To avoid bazel error
# "python interpreter `python3` not found in PATH"
python3-dev \
# To avoid bazel error
# "Cannot find gcc or CC; either correct your path or set the CC environment variable"
build-essential \
# To avoid bazel error
# unzip command not found
unzip \
&& apt-get clean && \
rm -rf /var/lib/apt/lists/*
&& apt-get clean autoclean \
&& apt-get autoremove -y \
&& rm -rf /var/lib/apt/lists/* \
&& rm -f /var/cache/apt/archives/*.deb

COPY --from=builder protoc/bin /usr/local/bin
COPY --from=builder protoc/include /usr/local/include

COPY --from=builder pandoc-3.7.0.2/bin /usr/local/bin
COPY --from=builder synthtool /synthtool

# Copy all Python interpreters, their pip executables, and their standard libraries from the builder.
COPY --from=builder /usr/local/bin/python3.9 /usr/local/bin/
Expand All @@ -151,26 +112,16 @@ COPY --from=builder /usr/local/lib/python3.10 /usr/local/lib/python3.10
COPY --from=builder /usr/local/bin/python3.13 /usr/local/bin/
COPY --from=builder /usr/local/lib/python3.13 /usr/local/lib/python3.13

# Copy the bazelisk executable from the builder.
COPY --from=builder /usr/local/bin/bazelisk /usr/local/bin/

# Copy bazel cache from the builder.
COPY --from=builder /bazel_cache /bazel_cache
RUN chmod -R 777 /bazel_cache

# Set the working directory in the container.
WORKDIR /app

# Create a virtual env and set the Path to fix the missing nox error
# when running the post processor changes.
RUN /usr/local/bin/python3.9 -m venv bazel_env
RUN . bazel_env/bin/activate

ENV PATH=/app/bazel_env/bin:$PATH

RUN git clone --depth 1 https://github.com/googleapis/synthtool.git /tmp/synthtool && \
bazel_env/bin/python3.9 -m pip install /tmp/synthtool nox && \
rm -rf /tmp/synthtool
# Install dependencies of the CLI such as click.
# Install gapic-generator which is used to generate libraries.
# Install nox which is used for running client library tests.
# Install starlark-pyo3 which is used to parse BUILD.bazel files.
COPY .generator/requirements.in .
RUN python3.9 -m pip install -r requirements.in
RUN python3.9 -m pip install /synthtool

# Install build which is used to get the metadata of package config files.
COPY .generator/requirements.in .
Expand All @@ -180,4 +131,8 @@ RUN python3.9 -m pip install -r requirements.in
COPY .generator/cli.py .
RUN chmod a+rx ./cli.py

# Copy the script used to parse BUILD.bazel files.
COPY .generator/parse_googleapis_content.py .
RUN chmod a+rx ./parse_googleapis_content.py

ENTRYPOINT ["python3.9", "./cli.py"]
Loading
Loading