feat: implement build command to validate generated library (#14129)

Fixes googleapis/librarian#450

Author: parthea

.generator/Dockerfile

@@ -36,20 +36,37 @@ RUN apt-get update && \
     rm -rf /var/lib/apt/lists/*
 
 # Set up environment variables for tool versions to make updates easier.
-ENV PYTHON_VERSION=3.11.5
 ENV BAZELISK_VERSION=v1.26.0
 
-# Create a symbolic link for `python3` to point to our specific version.
-ENV PATH /usr/local/bin/python3.11:$PATH
-
-# Install Python from source
-RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz && \
+RUN for PYTHON_VERSION in 3.9.23 3.10.18 3.11.13 3.12.11 3.13.5; do \
+    # Install Python from source
+    wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz && \
     tar -xvf Python-${PYTHON_VERSION}.tgz && \
     cd Python-${PYTHON_VERSION} && \
     ./configure --enable-optimizations && \
     make altinstall && \
     cd / && \
-    rm -rf Python-${PYTHON_VERSION}*
+    rm -rf Python-${PYTHON_VERSION}* \
+    ; done
+
+# Get the pip installation script using the instructions below
+# https://pip.pypa.io/en/stable/installation/#get-pip-py
+# Attempts to use `ensurepip` instead of `get-pip.py` resulted in errors like `python3.13.5 not found`
+RUN wget --no-check-certificate -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py'
+
+# Install pip for each python version
+RUN for PYTHON_VERSION in 3.9 3.10 3.11 3.12 3.13; do \
+    python${PYTHON_VERSION} /tmp/get-pip.py \
+    ; done
+    
+# Remove the pip installation script
+RUN rm /tmp/get-pip.py
+
+# Test Pip
+RUN for PYTHON_VERSION in 3.9 3.10 3.11 3.12 3.13; do \
+    python${PYTHON_VERSION} -m pip \
+    ; done
+
 
 # TODO(https://github.com/googleapis/librarian/issues/904): Install protoc for gencode.
 

.generator/cli.py

@@ -16,18 +16,21 @@
 import json
 import logging
 import os
+import subprocess
 import sys
 import subprocess
+from typing import Dict, List
 
 logger = logging.getLogger()
 
 LIBRARIAN_DIR = "librarian"
 GENERATE_REQUEST_FILE = "generate-request.json"
 SOURCE_DIR = "source"
 OUTPUT_DIR = "output"
+REPO_DIR = "repo"
 
 
-def _read_json_file(path):
+def _read_json_file(path: str) -> Dict:
     """Helper function that reads a json file path and returns the loaded json content.
 
     Args:
@@ -50,7 +53,7 @@ def handle_configure():
     logger.info("'configure' command executed.")
 
 
-def _determine_bazel_rule(api_path):
+def _determine_bazel_rule(api_path: str) -> str:
     """Executes a `bazelisk query` to find a Bazel rule.
 
     Args:
@@ -83,7 +86,25 @@ def _determine_bazel_rule(api_path):
         raise ValueError(f"Bazelisk query `{query}` failed") from e
 
 
-def _build_bazel_target(bazel_rule):
+def _get_library_id(request_data: Dict) -> str:
+    """Retrieve the library id from the given request dictionary
+
+    Args:
+        request_data(Dict): The contents `generate-request.json`.
+
+    Raises:
+        ValueError: If the key `id` does not exist in `request_data`.
+
+    Returns:
+        str: The id of the library in `generate-request.json`
+    """
+    library_id = request_data.get("id")
+    if not library_id:
+        raise ValueError("Request file is missing required 'id' field.")
+    return library_id
+
+
+def _build_bazel_target(bazel_rule: str):
     """Executes `bazelisk build` on a given Bazel rule.
 
     Args:
@@ -167,9 +188,7 @@ def handle_generate():
     # Read a generate-request.json file
     try:
         request_data = _read_json_file(f"{LIBRARIAN_DIR}/{GENERATE_REQUEST_FILE}")
-        library_id = request_data.get("id")
-        if not library_id:
-            raise ValueError("Request file is missing required 'id' field.")
+        library_id = _get_library_id(request_data)
 
         for api in request_data.get("apis", []):
             api_path = api.get("path")
@@ -186,8 +205,59 @@ def handle_generate():
     logger.info("'generate' command executed.")
 
 
+def _run_nox_sessions(sessions: List[str]):
+    """Calls nox for all specified sessions.
+
+    Args:
+        path(List[str]): The list of nox sessions to run.
+    """
+    # Read a generate-request.json file
+    current_session = None
+    try:
+        request_data = _read_json_file(f"{LIBRARIAN_DIR}/{GENERATE_REQUEST_FILE}")
+        library_id = _get_library_id(request_data)
+        for nox_session in sessions:
+            _run_individual_session(nox_session, library_id)
+    except Exception as e:
+        raise ValueError(f"Failed to run the nox session: {current_session}") from e
+
+
+def _run_individual_session(nox_session: str, library_id: str):
+    """
+    Calls nox with the specified sessions.
+
+    Args:
+        nox_session(str): The nox session to run
+        library_id(str): The library id under test
+    """
+    command = [
+        "nox",
+        "-s",
+        nox_session,
+        "-f",
+        f"{REPO_DIR}/packages/{library_id}",
+    ]
+    result = subprocess.run(command, text=True, check=True)
+    logger.info(result)
+
+
 def handle_build():
-    # TODO(https://github.com/googleapis/librarian/issues/450): Implement build command and update docstring.
+    """The main coordinator for validating client library generation."""
+    sessions = [
+        "unit-3.9",
+        "unit-3.10",
+        "unit-3.11",
+        "unit-3.12",
+        "unit-3.13",
+        "docs",
+        "system",
+        "lint",
+        "lint_setup_py",
+        "mypy",
+        "check_lower_bounds",
+    ]
+    _run_nox_sessions(sessions)
+
     logger.info("'build' command executed.")
 
 

.generator/test_cli.py

@@ -12,24 +12,28 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
-import pytest
 import json
 import logging
+import os
 import subprocess
+from unittest.mock import MagicMock, mock_open
 
-from unittest.mock import mock_open, MagicMock
+import pytest
 
 from cli import (
-    _read_json_file,
-    _determine_bazel_rule,
+    GENERATE_REQUEST_FILE,
+    LIBRARIAN_DIR,
+    REPO_DIR,
     _build_bazel_target,
+    _determine_bazel_rule,
+    _get_library_id,
     _locate_and_extract_artifact,
-    handle_generate,
+    _read_json_file,
+    _run_individual_session,
+    _run_nox_sessions,
     handle_build,
     handle_configure,
-    LIBRARIAN_DIR,
-    GENERATE_REQUEST_FILE,
+    handle_generate,
 )
 
 
@@ -53,6 +57,42 @@ def mock_generate_request_file(tmp_path, monkeypatch):
     return request_file
 
 
+@pytest.fixture
+def mock_generate_request_data_for_nox():
+    """Returns mock data for generate-request.json for nox tests."""
+    return {
+        "id": "mock-library",
+        "apis": [
+            {"path": "google/mock/v1"},
+        ],
+    }
+
+
+def test_get_library_id_success():
+    """Tests that _get_library_id returns the correct ID when present."""
+    request_data = {"id": "test-library", "name": "Test Library"}
+    library_id = _get_library_id(request_data)
+    assert library_id == "test-library"
+
+
+def test_get_library_id_missing_id():
+    """Tests that _get_library_id raises ValueError when 'id' is missing."""
+    request_data = {"name": "Test Library"}
+    with pytest.raises(
+        ValueError, match="Request file is missing required 'id' field."
+    ):
+        _get_library_id(request_data)
+
+
+def test_get_library_id_empty_id():
+    """Tests that _get_library_id raises ValueError when 'id' is an empty string."""
+    request_data = {"id": "", "name": "Test Library"}
+    with pytest.raises(
+        ValueError, match="Request file is missing required 'id' field."
+    ):
+        _get_library_id(request_data)
+
+
 def test_handle_configure_success(caplog, mock_generate_request_file):
     """
     Tests the successful execution path of handle_configure.
@@ -191,12 +231,105 @@ def test_handle_generate_fail(caplog):
         handle_generate()
 
 
-def test_handle_build_success(caplog, mock_generate_request_file):
+def test_run_individual_session_success(mocker, caplog):
+    """Tests that _run_individual_session calls nox with correct arguments and logs success."""
+    caplog.set_level(logging.INFO)
+
+    mock_subprocess_run = mocker.patch(
+        "cli.subprocess.run", return_value=MagicMock(returncode=0)
+    )
+
+    test_session = "unit-3.9"
+    test_library_id = "test-library"
+    _run_individual_session(test_session, test_library_id)
+
+    expected_command = [
+        "nox",
+        "-s",
+        test_session,
+        "-f",
+        f"{REPO_DIR}/packages/{test_library_id}",
+    ]
+    mock_subprocess_run.assert_called_once_with(expected_command, text=True, check=True)
+
+
+def test_run_individual_session_failure(mocker):
+    """Tests that _run_individual_session raises CalledProcessError if nox command fails."""
+    mocker.patch(
+        "cli.subprocess.run",
+        side_effect=subprocess.CalledProcessError(
+            1, "nox", stderr="Nox session failed"
+        ),
+    )
+
+    with pytest.raises(subprocess.CalledProcessError):
+        _run_individual_session("lint", "another-library")
+
+
+def test_run_nox_sessions_success(mocker, mock_generate_request_data_for_nox):
+    """Tests that _run_nox_sessions successfully runs all specified sessions."""
+    mocker.patch("cli._read_json_file", return_value=mock_generate_request_data_for_nox)
+    mocker.patch("cli._get_library_id", return_value="mock-library")
+    mock_run_individual_session = mocker.patch("cli._run_individual_session")
+
+    sessions_to_run = ["unit-3.9", "lint"]
+    _run_nox_sessions(sessions_to_run)
+
+    assert mock_run_individual_session.call_count == len(sessions_to_run)
+    mock_run_individual_session.assert_has_calls(
+        [
+            mocker.call("unit-3.9", "mock-library"),
+            mocker.call("lint", "mock-library"),
+        ]
+    )
+
+
+def test_run_nox_sessions_read_file_failure(mocker):
+    """Tests that _run_nox_sessions raises ValueError if _read_json_file fails."""
+    mocker.patch("cli._read_json_file", side_effect=FileNotFoundError("file not found"))
+
+    with pytest.raises(ValueError, match="Failed to run the nox session"):
+        _run_nox_sessions(["unit-3.9"])
+
+
+def test_run_nox_sessions_get_library_id_failure(mocker):
+    """Tests that _run_nox_sessions raises ValueError if _get_library_id fails."""
+    mocker.patch("cli._read_json_file", return_value={"apis": []})  # Missing 'id'
+    mocker.patch(
+        "cli._get_library_id",
+        side_effect=ValueError("Request file is missing required 'id' field."),
+    )
+
+    with pytest.raises(ValueError, match="Failed to run the nox session"):
+        _run_nox_sessions(["unit-3.9"])
+
+
+def test_run_nox_sessions_individual_session_failure(
+    mocker, mock_generate_request_data_for_nox
+):
+    """Tests that _run_nox_sessions raises ValueError if _run_individual_session fails."""
+    mocker.patch("cli._read_json_file", return_value=mock_generate_request_data_for_nox)
+    mocker.patch("cli._get_library_id", return_value="mock-library")
+    mock_run_individual_session = mocker.patch(
+        "cli._run_individual_session",
+        side_effect=[None, subprocess.CalledProcessError(1, "nox", "session failed")],
+    )
+
+    sessions_to_run = ["unit-3.9", "lint"]
+    with pytest.raises(ValueError, match="Failed to run the nox session"):
+        _run_nox_sessions(sessions_to_run)
+
+    # Check that _run_individual_session was called at least once
+    assert mock_run_individual_session.call_count > 0
+
+
+def test_handle_build_success(caplog, mocker):
     """
     Tests the successful execution path of handle_build.
     """
     caplog.set_level(logging.INFO)
 
+    mocker.patch("cli._run_nox_sessions")
     handle_build()
 
     assert "'build' command executed." in caplog.text

cloudbuild.yaml

@@ -12,6 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# TODO(https://github.com/googleapis/google-cloud-python/issues/14142):
+# Reduce this timeout by moving the installation of Python runtimes to a separate base image
+timeout: 7200s  # 2 hours
 steps:
   # This step builds the Docker image.
   - name: 'gcr.io/cloud-builders/docker'