chore(librarian): add support for split repository

Author: parthea

.generator/Dockerfile

@@ -69,8 +69,10 @@ RUN wget https://github.com/jgm/pandoc/releases/download/${PANDOC_VERSION}/pando
 RUN tar -xvf pandoc-${PANDOC_VERSION}-linux-amd64.tar.gz -C pandoc-binary --strip-components=1
 
 # Pin synthtool for a more hermetic build
-RUN git clone https://github.com/googleapis/synthtool.git synthtool
-RUN cd synthtool && git checkout 35313ccd8cdd2d12d2447ccdc497a7919aae1e3e
+# This needs to be a single command so that the git clone command is not cached
+RUN git clone https://github.com/googleapis/synthtool.git synthtool && \
+    cd synthtool && \
+    git checkout 35313ccd8cdd2d12d2447ccdc497a7919aae1e3e
 
 # --- Final Stage ---
 # This stage creates the lightweight final image, copying only the

.generator/cli.py

@@ -33,7 +33,7 @@
 
 try:
     import synthtool
-    from synthtool.languages import python_mono_repo
+    from synthtool.languages import python, python_mono_repo
 
     SYNTHTOOL_INSTALLED = True
     SYNTHTOOL_IMPORT_ERROR = None
@@ -316,20 +316,28 @@ def _get_library_id(request_data: Dict) -> str:
     return library_id
 
 
-def _run_post_processor(output: str, library_id: str):
+def _run_post_processor(output: str, library_id: str, is_mono_repo: bool):
     """Runs the synthtool post-processor on the output directory.
 
     Args:
         output(str): Path to the directory in the container where code
             should be generated.
         library_id(str): The library id to be used for post processing.
-
+        is_mono_repo(bool): True if the current repository is a mono-repo.
     """
     os.chdir(output)
     path_to_library = f"packages/{library_id}"
     logger.info("Running Python post-processor...")
     if SYNTHTOOL_INSTALLED:
-        python_mono_repo.owlbot_main(path_to_library)
+        if is_mono_repo:
+            python_mono_repo.owlbot_main(path_to_library)
+        else:
+            # Some repositories have customizations in `owlbot.py`. If this file exists, 
+            # run those customizations instead of `owlbot_main`
+            if Path(f"{output}/owlbot.py").exists():
+                subprocess.run(["python3.14", f"{output}/owlbot.py"])
+            else:
+                python.owlbot_main(path_to_library)
     else:
         raise SYNTHTOOL_IMPORT_ERROR  # pragma: NO COVER
 
@@ -342,7 +350,9 @@ def _run_post_processor(output: str, library_id: str):
     logger.info("Python post-processor ran successfully.")
 
 
-def _copy_files_needed_for_post_processing(output: str, input: str, library_id: str):
+def _copy_files_needed_for_post_processing(
+    output: str, input: str, library_id: str, is_mono_repo: bool
+):
     """Copy files to the output directory whcih are needed during the post processing
     step, such as .repo-metadata.json and script/client-post-processing, using
     the input directory as the source.
@@ -353,9 +363,9 @@ def _copy_files_needed_for_post_processing(output: str, input: str, library_id:
         input(str): The path to the directory in the container
             which contains additional generator input.
         library_id(str): The library id to be used for post processing.
+        is_mono_repo(bool): True if the current repository is a mono-repo.
     """
-
-    path_to_library = f"packages/{library_id}"
+    path_to_library = f"packages/{library_id}" if is_mono_repo else "."
     repo_metadata_path = f"{input}/{path_to_library}/.repo-metadata.json"
 
     # We need to create these directories so that we can copy files necessary for post-processing.
@@ -373,6 +383,20 @@ def _copy_files_needed_for_post_processing(output: str, input: str, library_id:
             f"{output}/{path_to_library}/.repo-metadata.json",
         )
 
+    if not is_mono_repo:
+        setup_py_path = f"{input}/setup.py"
+        if os.path.exists(setup_py_path):
+            shutil.copy(
+                setup_py_path,
+                f"{output}/setup.py",
+            )
+        owlbot_py_path = f"{input}/owlbot.py"
+        if os.path.exists(owlbot_py_path):
+            shutil.copy(
+                owlbot_py_path,
+                f"{output}/owlbot.py",
+            )
+
     # copy post-procesing files
     for post_processing_file in glob.glob(
         f"{input}/client-post-processing/*.yaml"
@@ -385,7 +409,7 @@ def _copy_files_needed_for_post_processing(output: str, input: str, library_id:
                 )
 
 
-def _clean_up_files_after_post_processing(output: str, library_id: str):
+def _clean_up_files_after_post_processing(output: str, library_id: str, is_mono_repo: bool):
     """
     Clean up files which should not be included in the generated client.
     This function is idempotent and will not fail if files are already removed.
@@ -394,8 +418,9 @@ def _clean_up_files_after_post_processing(output: str, library_id: str):
         output(str): Path to the directory in the container where code
             should be generated.
         library_id(str): The library id to be used for post processing.
+        is_mono_repo(bool): True if the current repository is a mono-repo.
     """
-    path_to_library = f"packages/{library_id}"
+    path_to_library = f"packages/{library_id}" if is_mono_repo else "."
 
     # Safely remove directories, ignoring errors if they don't exist.
     shutil.rmtree(f"{output}/{path_to_library}/.nox", ignore_errors=True)
@@ -486,7 +511,7 @@ def _create_repo_metadata_from_service_config(
 
 
 def _generate_repo_metadata_file(
-    output: str, library_id: str, source: str, apis: List[Dict]
+    output: str, library_id: str, source: str, apis: List[Dict], is_mono_repo: bool
 ):
     """Generates the .repo-metadata.json file from the primary API service config.
 
@@ -495,8 +520,9 @@ def _generate_repo_metadata_file(
         library_id (str): The ID of the library.
         source (str): The path to the source directory.
         apis (List[Dict]): A list of APIs to generate.
+        is_mono_repo(bool): True if the current repository is a mono-repo.
     """
-    path_to_library = f"packages/{library_id}"
+    path_to_library = f"packages/{library_id}" if is_mono_repo else "."
     output_repo_metadata = f"{output}/{path_to_library}/.repo-metadata.json"
 
     # TODO(https://github.com/googleapis/librarian/issues/2334)): If `.repo-metadata.json`
@@ -523,7 +549,7 @@ def _generate_repo_metadata_file(
     _write_json_file(output_repo_metadata, metadata_content)
 
 
-def _copy_readme_to_docs(output: str, library_id: str):
+def _copy_readme_to_docs(output: str, library_id: str, is_mono_repo: bool):
     """Copies the README.rst file for a generated library to docs/README.rst.
 
     This function is robust against various symlink configurations that could
@@ -536,7 +562,7 @@ def _copy_readme_to_docs(output: str, library_id: str):
             should be generated.
         library_id(str): The library id.
     """
-    path_to_library = f"packages/{library_id}"
+    path_to_library = f"packages/{library_id}" if is_mono_repo else "."
     source_path = f"{output}/{path_to_library}/README.rst"
     docs_path = f"{output}/{path_to_library}/docs"
     destination_path = f"{docs_path}/README.rst"
@@ -593,6 +619,7 @@ def handle_generate(
     """
 
     try:
+        is_mono_repo = _is_mono_repo(input)
         # Read a generate-request.json file
         request_data = _read_json_file(f"{librarian}/{GENERATE_REQUEST_FILE}")
         library_id = _get_library_id(request_data)
@@ -601,12 +628,16 @@ def handle_generate(
         for api in apis_to_generate:
             api_path = api.get("path")
             if api_path:
-                _generate_api(api_path, library_id, source, output, version)
-        _copy_files_needed_for_post_processing(output, input, library_id)
-        _generate_repo_metadata_file(output, library_id, source, apis_to_generate)
-        _run_post_processor(output, library_id)
+                _generate_api(
+                    api_path, library_id, source, output, version, is_mono_repo
+                )
+        _copy_files_needed_for_post_processing(output, input, library_id, is_mono_repo)
+        _generate_repo_metadata_file(
+            output, library_id, source, apis_to_generate, is_mono_repo
+        )
+        _run_post_processor(output, library_id, is_mono_repo)
         _copy_readme_to_docs(output, library_id)
-        _clean_up_files_after_post_processing(output, library_id)
+        _clean_up_files_after_post_processing(output, library_id, is_mono_repo)
     except Exception as e:
         raise ValueError("Generation failed.") from e
     logger.info("'generate' command executed.")
@@ -821,7 +852,12 @@ def _stage_gapic_library(tmp_dir: str, staging_dir: str) -> None:
 
 
 def _generate_api(
-    api_path: str, library_id: str, source: str, output: str, gapic_version: str
+    api_path: str,
+    library_id: str,
+    source: str,
+    output: str,
+    gapic_version: str,
+    is_mono_repo: bool,
 ):
     """
     Handles the generation and staging process for a single API path.
@@ -833,6 +869,7 @@ def _generate_api(
         output (str): Path to the output directory where code should be staged.
         gapic_version(str): The desired version number for the GAPIC client library
             in a format which follows PEP-440.
+        is_mono_repo(bool): True if the current repository is a mono-repo.
     """
     py_gapic_config = _read_bazel_build_py_rule(api_path, source)
     is_proto_only_library = len(py_gapic_config) == 0
@@ -854,9 +891,10 @@ def _generate_api(
         staging_child_directory = _get_staging_child_directory(
             api_path, is_proto_only_library
         )
-        staging_dir = os.path.join(
-            output, "owl-bot-staging", library_id, staging_child_directory
-        )
+        staging_dir = os.path.join(output, "owl-bot-staging")
+        if is_mono_repo:
+            staging_dir = os.path.join(staging_dir, library_id)
+        staging_dir = os.path.join(staging_dir, staging_child_directory)
 
         # 4. Stage the generated code
         if is_proto_only_library:

.generator/test_cli.py

@@ -659,6 +659,8 @@ def test_handle_generate_success(
     """
     caplog.set_level(logging.INFO)
 
+    is_mono_repo = False
+
     mock_generate_api = mocker.patch("cli._generate_api")
     mock_run_post_processor = mocker.patch("cli._run_post_processor")
     mock_copy_files_needed_for_post_processing = mocker.patch(
@@ -671,12 +673,12 @@ def test_handle_generate_success(
 
     handle_generate()
 
-    mock_run_post_processor.assert_called_once_with("output", "google-cloud-language")
+    mock_run_post_processor.assert_called_once_with("output", "google-cloud-language", is_mono_repo)
     mock_copy_files_needed_for_post_processing.assert_called_once_with(
-        "output", "input", "google-cloud-language"
+        "output", "input", "google-cloud-language", is_mono_repo
     )
     mock_clean_up_files_after_post_processing.assert_called_once_with(
-        "output", "google-cloud-language"
+        "output", "google-cloud-language", is_mono_repo
     )
     mock_generate_api.assert_called_once()
 
@@ -836,8 +838,9 @@ def test_copy_files_needed_for_post_processing_copies_metadata_if_exists(mocker)
     mock_makedirs = mocker.patch("os.makedirs")
     mock_shutil_copy = mocker.patch("shutil.copy")
     mocker.patch("os.path.exists", return_value=True)
+    is_mono_repo = True
 
-    _copy_files_needed_for_post_processing("output", "input", "library_id")
+    _copy_files_needed_for_post_processing("output", "input", "library_id", is_mono_repo)
 
     mock_shutil_copy.assert_called_once()
     mock_makedirs.assert_called()
@@ -848,8 +851,9 @@ def test_copy_files_needed_for_post_processing_skips_metadata_if_not_exists(mock
     mock_makedirs = mocker.patch("os.makedirs")
     mock_shutil_copy = mocker.patch("shutil.copy")
     mocker.patch("os.path.exists", return_value=False)
+    is_mono_repo = True
 
-    _copy_files_needed_for_post_processing("output", "input", "library_id")
+    _copy_files_needed_for_post_processing("output", "input", "library_id", is_mono_repo)
 
     mock_shutil_copy.assert_not_called()
     mock_makedirs.assert_called()
@@ -1395,8 +1399,9 @@ def test_generate_repo_metadata_file(mocker):
             "path": "google/cloud/language/v1",
         }
     ]
+    is_mono_repo = True
 
-    _generate_repo_metadata_file(output, library_id, source, apis)
+    _generate_repo_metadata_file(output, library_id, source, apis, is_mono_repo)
 
     mock_create_metadata.assert_called_once_with(
         "service_config.yaml", "google/cloud/language/v1", source, library_id
@@ -1412,8 +1417,9 @@ def test_generate_repo_metadata_file_skips_if_exists(mocker):
     mock_write_json = mocker.patch("cli._write_json_file")
     mock_create_metadata = mocker.patch("cli._create_repo_metadata_from_service_config")
     mocker.patch("os.path.exists", return_value=True)
+    is_mono_repo = True
 
-    _generate_repo_metadata_file("output", "library_id", "source", [])
+    _generate_repo_metadata_file("output", "library_id", "source", [], is_mono_repo)
 
     mock_create_metadata.assert_not_called()
     mock_write_json.assert_not_called()