Flatten python packages (#743)

Author: groodt

python/pip_install/BUILD

@@ -6,7 +6,6 @@ filegroup(
         "BUILD",
         "pip_compile.py",
         "//python/pip_install/extract_wheels:distribution",
-        "//python/pip_install/parse_requirements_to_bzl:distribution",
         "//python/pip_install/private:distribution",
     ],
     visibility = ["//:__pkg__"],
@@ -24,7 +23,6 @@ filegroup(
     name = "py_srcs",
     srcs = [
         "//python/pip_install/extract_wheels:py_srcs",
-        "//python/pip_install/parse_requirements_to_bzl:py_srcs",
     ],
     visibility = ["//python/pip_install/private:__pkg__"],
 )

python/pip_install/extract_wheels/BUILD

@@ -1,20 +1,179 @@
-load("@rules_python//python:defs.bzl", "py_binary")
+load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test")
+load("//python/pip_install:repositories.bzl", "requirement")
+load(":annotations_test_helpers.bzl", "package_annotation", "package_annotations_file")
+
+py_library(
+    name = "lib",
+    srcs = [
+        "annotation.py",
+        "arguments.py",
+        "bazel.py",
+        "extract_single_wheel.py",
+        "extract_wheels.py",
+        "namespace_pkgs.py",
+        "parse_requirements_to_bzl.py",
+        "requirements.py",
+        "wheel.py",
+    ],
+    deps = [
+        requirement("installer"),
+        requirement("setuptools"),
+    ],
+)
 
 py_binary(
     name = "extract_wheels",
     srcs = [
-        "__init__.py",
-        "__main__.py",
+        "extract_wheels.py",
+    ],
+    deps = [":lib"],
+)
+
+py_binary(
+    name = "extract_single_wheel",
+    srcs = [
+        "extract_single_wheel.py",
+    ],
+    deps = [":lib"],
+)
+
+py_binary(
+    name = "parse_requirements_to_bzl",
+    srcs = [
+        "parse_requirements_to_bzl.py",
+    ],
+    deps = [":lib"],
+)
+
+package_annotations_file(
+    name = "mock_annotations",
+    annotations = {
+        "pkg_a": package_annotation(),
+        "pkg_b": package_annotation(
+            data_exclude_glob = [
+                "*.foo",
+                "*.bar",
+            ],
+        ),
+        "pkg_c": package_annotation(
+            # The `join` and `strip` here accounts for potential differences
+            # in new lines between unix and windows hosts.
+            additive_build_content = "\n".join([line.strip() for line in """\
+cc_library(
+    name = "my_target",
+    hdrs = glob(["**/*.h"]),
+    srcs = glob(["**/*.cc"]),
+)
+""".splitlines()]),
+            data = [":my_target"],
+        ),
+        "pkg_d": package_annotation(
+            srcs_exclude_glob = ["pkg_d/tests/**"],
+        ),
+    },
+    tags = ["manual"],
+)
+
+py_test(
+    name = "annotations_test",
+    size = "small",
+    srcs = ["annotations_test.py"],
+    data = [":mock_annotations"],
+    env = {"MOCK_ANNOTATIONS": "$(rootpath :mock_annotations)"},
+    tags = ["unit"],
+    deps = [
+        ":lib",
+        "//python/runfiles",
+    ],
+)
+
+py_test(
+    name = "bazel_test",
+    size = "small",
+    srcs = [
+        "bazel_test.py",
+    ],
+    tags = ["unit"],
+    deps = [
+        ":lib",
+    ],
+)
+
+py_test(
+    name = "namespace_pkgs_test",
+    size = "small",
+    srcs = [
+        "namespace_pkgs_test.py",
+    ],
+    tags = ["unit"],
+    deps = [
+        ":lib",
+    ],
+)
+
+py_test(
+    name = "requirements_test",
+    size = "small",
+    srcs = [
+        "requirements_test.py",
+    ],
+    tags = ["unit"],
+    deps = [
+        ":lib",
+    ],
+)
+
+py_test(
+    name = "arguments_test",
+    size = "small",
+    srcs = [
+        "arguments_test.py",
+    ],
+    tags = ["unit"],
+    deps = [
+        ":lib",
+    ],
+)
+
+py_test(
+    name = "whl_filegroup_test",
+    size = "small",
+    srcs = ["whl_filegroup_test.py"],
+    data = ["//examples/wheel:minimal_with_py_package"],
+    main = "whl_filegroup_test.py",
+    tags = ["unit"],
+    deps = [":lib"],
+)
+
+py_test(
+    name = "parse_requirements_to_bzl_test",
+    size = "small",
+    srcs = [
+        "parse_requirements_to_bzl_test.py",
+    ],
+    tags = ["unit"],
+    deps = [
+        ":lib",
+    ],
+)
+
+py_test(
+    name = "requirements_bzl_test",
+    size = "small",
+    srcs = [
+        "requirements_bzl_test.py",
+    ],
+    deps = [
+        ":lib",
     ],
-    main = "__main__.py",
-    deps = ["//python/pip_install/extract_wheels/lib"],
 )
 
 filegroup(
     name = "distribution",
-    srcs = glob(["*"]) + [
-        "//python/pip_install/extract_wheels/lib:distribution",
-    ],
+    srcs = glob(
+        ["*"],
+        exclude = ["*_test.py"],
+    ),
     visibility = ["//python/pip_install:__subpackages__"],
 )
 
@@ -23,8 +182,6 @@ filegroup(
     srcs = glob(
         include = ["**/*.py"],
         exclude = ["**/*_test.py"],
-    ) + [
-        "//python/pip_install/extract_wheels/lib:py_srcs",
-    ],
+    ),
     visibility = ["//python/pip_install:__subpackages__"],
 )

python/pip_install/extract_wheels/__init__.py

@@ -1,128 +0,0 @@
-"""extract_wheels
-
-extract_wheels resolves and fetches artifacts transitively from the Python Package Index (PyPI) based on a
-requirements.txt. It generates the required BUILD files to consume these packages as Python libraries.
-
-Under the hood, it depends on the `pip wheel` command to do resolution, download, and compilation into wheels.
-"""
-import argparse
-import glob
-import os
-import pathlib
-import subprocess
-import sys
-
-from python.pip_install.extract_wheels.lib import (
-    annotation,
-    arguments,
-    bazel,
-    requirements,
-    wheel,
-)
-
-
-def configure_reproducible_wheels() -> None:
-    """Modifies the environment to make wheel building reproducible.
-
-    Wheels created from sdists are not reproducible by default. We can however workaround this by
-    patching in some configuration with environment variables.
-    """
-
-    # wheel, by default, enables debug symbols in GCC. This incidentally captures the build path in the .so file
-    # We can override this behavior by disabling debug symbols entirely.
-    # https://github.com/pypa/pip/issues/6505
-    if "CFLAGS" in os.environ:
-        os.environ["CFLAGS"] += " -g0"
-    else:
-        os.environ["CFLAGS"] = "-g0"
-
-    # set SOURCE_DATE_EPOCH to 1980 so that we can use python wheels
-    # https://github.com/NixOS/nixpkgs/blob/master/doc/languages-frameworks/python.section.md#python-setuppy-bdist_wheel-cannot-create-whl
-    if "SOURCE_DATE_EPOCH" not in os.environ:
-        os.environ["SOURCE_DATE_EPOCH"] = "315532800"
-
-    # Python wheel metadata files can be unstable.
-    # See https://bitbucket.org/pypa/wheel/pull-requests/74/make-the-output-of-metadata-files/diff
-    if "PYTHONHASHSEED" not in os.environ:
-        os.environ["PYTHONHASHSEED"] = "0"
-
-
-def main() -> None:
-    """Main program.
-
-    Exits zero on successful program termination, non-zero otherwise.
-    """
-
-    configure_reproducible_wheels()
-
-    parser = argparse.ArgumentParser(
-        description="Resolve and fetch artifacts transitively from PyPI"
-    )
-    parser.add_argument(
-        "--requirements",
-        action="store",
-        required=True,
-        help="Path to requirements.txt from where to install dependencies",
-    )
-    parser.add_argument(
-        "--annotations",
-        type=annotation.annotations_map_from_str_path,
-        help="A json encoded file containing annotations for rendered packages.",
-    )
-    arguments.parse_common_args(parser)
-    args = parser.parse_args()
-    deserialized_args = dict(vars(args))
-    arguments.deserialize_structured_args(deserialized_args)
-
-    # Pip is run with the working directory changed to the folder containing the requirements.txt file, to allow for
-    # relative requirements to be correctly resolved. The --wheel-dir is therefore required to be repointed back to the
-    # current calling working directory (the repo root in .../external/name), where the wheel files should be written to
-    pip_args = (
-        [sys.executable, "-m", "pip"]
-        + (["--isolated"] if args.isolated else [])
-        + ["wheel", "-r", args.requirements]
-        + ["--wheel-dir", os.getcwd()]
-        + deserialized_args["extra_pip_args"]
-    )
-
-    env = os.environ.copy()
-    env.update(deserialized_args["environment"])
-
-    # Assumes any errors are logged by pip so do nothing. This command will fail if pip fails
-    subprocess.run(
-        pip_args,
-        check=True,
-        env=env,
-        cwd=str(pathlib.Path(args.requirements).parent.resolve()),
-    )
-
-    extras = requirements.parse_extras(args.requirements)
-
-    repo_label = "@%s" % args.repo
-
-    # Locate all wheels
-    wheels = [whl for whl in glob.glob("*.whl")]
-
-    # Collect all annotations
-    reqs = {whl: wheel.Wheel(whl).name for whl in wheels}
-    annotations = args.annotations.collect(reqs.values())
-
-    targets = [
-        '"{}{}"'.format(
-            repo_label,
-            bazel.extract_wheel(
-                wheel_file=whl,
-                extras=extras,
-                pip_data_exclude=deserialized_args["pip_data_exclude"],
-                enable_implicit_namespace_pkgs=args.enable_implicit_namespace_pkgs,
-                repo_prefix=args.repo_prefix,
-                annotation=annotations.get(name),
-            ),
-        )
-        for whl, name in reqs.items()
-    ]
-
-    with open("requirements.bzl", "w") as requirement_file:
-        requirement_file.write(
-            bazel.generate_requirements_file_contents(repo_label, targets)
-        )

python/pip_install/extract_wheels/__main__.py

@@ -5,7 +5,7 @@
 import unittest
 from pathlib import Path
 
-from python.pip_install.extract_wheels.lib.annotation import Annotation, AnnotationsMap
+from python.pip_install.extract_wheels.annotation import Annotation, AnnotationsMap
 from python.runfiles import runfiles
 
 

python/pip_install/extract_wheels/lib/annotation.py renamed to python/pip_install/extract_wheels/annotation.py

@@ -2,7 +2,7 @@
 import json
 import unittest
 
-from python.pip_install.extract_wheels.lib import arguments
+from python.pip_install.extract_wheels import arguments
 
 
 class ArgumentsTestCase(unittest.TestCase):

python/pip_install/extract_wheels/lib/annotations_test.py renamed to python/pip_install/extract_wheels/annotations_test.py

@@ -6,7 +6,7 @@
 from pathlib import Path
 from typing import Dict, Iterable, List, Optional, Set
 
-from python.pip_install.extract_wheels.lib import (
+from python.pip_install.extract_wheels import (
     annotation,
     namespace_pkgs,
     wheel,