Merge pull request #26 from dillon-giacoppo/dillon-format-and-lint

Author: dillon-giacoppo

.bazelrc

@@ -0,0 +1,2 @@
+build --aspects @mypy_integration//:mypy.bzl%mypy_aspect
+build --output_groups=+mypy

.gitignore

@@ -21,7 +21,6 @@ dist/
 downloads/
 eggs/
 .eggs/
-lib/
 lib64/
 parts/
 sdist/

README.md

@@ -1,38 +1,36 @@
 # rules_python_external
 
-Contains Bazel rules to fetch and install Python dependencies from a requirements.txt file.
+Bazel rules to transitively fetch and install Python dependencies from a requirements.txt file.
+
+## Features
+
+The rules address most of the top packaging issues in rules_python. This means the rules support common packages such
+as tensorflow and google.cloud natively.
+
+* Transitive dependency resolution:
+    [#35](https://github.com/bazelbuild/rules_python/issues/35),
+    [#102](https://github.com/bazelbuild/rules_python/issues/102)
+* Minimal runtime dependencies:
+    [#184](https://github.com/bazelbuild/rules_python/issues/184)
+* Support for [spreading purelibs](https://www.python.org/dev/peps/pep-0491/#installing-a-wheel-distribution-1-0-py32-none-any-whl):
+    [#71](https://github.com/bazelbuild/rules_python/issues/71)
+* Support for [namespace packages](https://packaging.python.org/guides/packaging-namespace-packages/):
+    [#14](https://github.com/bazelbuild/rules_python/issues/14),
+    [#55](https://github.com/bazelbuild/rules_python/issues/55),
+    [#65](https://github.com/bazelbuild/rules_python/issues/65),
+    [#93](https://github.com/bazelbuild/rules_python/issues/93),
+    [#189](https://github.com/bazelbuild/rules_python/issues/189)
+* Fetches pip packages only for building Python targets:
+    [#96](https://github.com/bazelbuild/rules_python/issues/96)
+* Reproducible builds:
+    [#154](https://github.com/bazelbuild/rules_python/issues/154),
+    [#176](https://github.com/bazelbuild/rules_python/issues/176)
 
 ## Usage
 
-#### Setup `requirements.txt` 
+#### Prerequisites
 
-While `rules_python_external` **does not** require a _transitively-closed_ `requirements.txt` file, it is recommended. But if you want to just have top-level packages listed, that works. 
-
-Transitively-closed requirements specs are very tedious to produce and maintain manually. To automate the process we recommend [`pip-compile` from `jazzband/pip-tools`](https://github.com/jazzband/pip-tools#example-usage-for-pip-compile).
-
-For example, `pip-compile` takes a `requirements.in` like this:
-
-```
-boto3~=1.9.227
-botocore~=1.12.247
-click~=7.0
-```
-
-These above are the third-party packages you can directly import.
-
-`pip-compile` 'compiles' it so you get a transitively-closed `requirements.txt` like this, which should be passed to `pip_install` below:
-
-```
-boto3==1.9.253
-botocore==1.12.253
-click==7.0
-docutils==0.15.2          # via botocore
-jmespath==0.9.4           # via boto3, botocore
-python-dateutil==2.8.1    # via botocore
-s3transfer==0.2.1         # via boto3
-six==1.14.0               # via python-dateutil
-urllib3==1.25.8           # via botocore
-```
+The rules support Python >= 3.5 (the oldest [maintained release](https://devguide.python.org/#status-of-python-branches)).
 
 #### Setup `WORKSPACE`
 
@@ -57,7 +55,7 @@ pip_install(
 )
 ```
 
-Example `BUILD` file.
+#### Example `BUILD` file.
 
 ```python
 load("@py_deps//:requirements.bzl", "requirement")
@@ -66,13 +64,48 @@ py_binary(
     name = "main",
     srcs = ["main.py"],
     deps = [
-        requirement("boto3"), # or @py_deps//pypi__boto3
+        requirement("boto3"),
     ],
 )
 ```
 
 Note that above you do not need to add transitively required packages to `deps = [ ... ]`
 
+#### Setup `requirements.txt`
+
+While `rules_python_external` **does not** require a _transitively-closed_ `requirements.txt` file, it is recommended.
+But if you want to just have top-level packages listed, that also will work.
+
+Transitively-closed requirements specs are very tedious to produce and maintain manually. To automate the process we
+recommend [`pip-compile` from `jazzband/pip-tools`](https://github.com/jazzband/pip-tools#example-usage-for-pip-compile).
+
+For example, `pip-compile` takes a `requirements.in` like this:
+
+```
+boto3~=1.9.227
+botocore~=1.12.247
+click~=7.0
+```
+
+`pip-compile` 'compiles' it so you get a transitively-closed `requirements.txt` like this, which should be passed to
+`pip_install` below:
+
+```
+boto3==1.9.253
+botocore==1.12.253
+click==7.0
+docutils==0.15.2          # via botocore
+jmespath==0.9.4           # via boto3, botocore
+python-dateutil==2.8.1    # via botocore
+s3transfer==0.2.1         # via boto3
+six==1.14.0               # via python-dateutil
+urllib3==1.25.8           # via botocore
+```
+
+### Demo
+
+You can find a demo in the [example/](./example) directory.
+
 ## Development
 
 ### Testing

WORKSPACE

@@ -4,12 +4,40 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
 http_archive(
     name = "rules_python",
-    url = "https://github.com/bazelbuild/rules_python/releases/download/0.0.1/rules_python-0.0.1.tar.gz",
-    sha256 = "aa96a691d3a8177f3215b14b0edc9641787abaaa30363a080165d06ab65e1161",
+    sha256 = "d2865e2ce23ee217aaa408ddaa024ca472114a6f250b46159d27de05530c75e3",
+    strip_prefix = "rules_python-7b222cfdb4e59b9fd2a609e1fbb233e94fdcde7c",
+    url = "https://github.com/bazelbuild/rules_python/archive/7b222cfdb4e59b9fd2a609e1fbb233e94fdcde7c.tar.gz",
 )
 
 load("@rules_python//python:repositories.bzl", "py_repositories")
 py_repositories()
 
 load("//:repositories.bzl", "rules_python_external_dependencies")
 rules_python_external_dependencies()
+
+mypy_integration_version = "0.0.7" # latest @ Feb 10th 2020
+
+http_archive(
+    name = "mypy_integration",
+    sha256 = "bf7ecd386740328f96c343dca095a63b93df7f86f8d3e1e2e6ff46e400880077", # for 0.0.7
+    strip_prefix = "bazel-mypy-integration-{version}".format(version = mypy_integration_version),
+    url = "https://github.com/thundergolfer/bazel-mypy-integration/archive/{version}.zip".format(
+        version = mypy_integration_version
+    ),
+)
+
+load(
+    "@mypy_integration//repositories:repositories.bzl",
+    mypy_integration_repositories = "repositories",
+)
+
+mypy_integration_repositories()
+
+load("@mypy_integration//:config.bzl", "mypy_configuration")
+mypy_configuration("//tools/typing:mypy.ini")
+
+load("@mypy_integration//repositories:deps.bzl", mypy_integration_deps = "deps")
+mypy_integration_deps("//tools/typing:mypy_version.txt")
+
+load("@mypy_integration//repositories:pip_repositories.bzl", "pip_deps")
+pip_deps()

defs.bzl

@@ -21,7 +21,8 @@ def _pip_repository_impl(rctx):
     result = rctx.execute(
         [
             rctx.which(rctx.attr.python_interpreter),
-            rctx.path(rctx.attr._script).dirname,
+            "-m",
+            "extract_wheels",
             "--requirements",
             rctx.path(rctx.attr.requirements),
             "--repo",
@@ -46,9 +47,6 @@ pip_repository = repository_rule(
         "python_interpreter": attr.string(default="python3"),
         # 600 is documented as default here: https://docs.bazel.build/versions/master/skylark/lib/repository_ctx.html#execute
         "timeout": attr.int(default = 600),
-        "_script": attr.label(
-            executable=True, default=Label("//src:__main__.py"), cfg="host",
-        ),
     },
     implementation=_pip_repository_impl,
 )

extract_wheels/BUILD

@@ -0,0 +1,8 @@
+load("//:repositories.bzl", "all_requirements")
+
+py_binary(
+    name = "extract_wheels",
+    srcs = ["__init__.py", "__main__.py"],
+    main = "__main__.py",
+    deps = ["//extract_wheels/lib"],
+)

extract_wheels/__init__.py

@@ -0,0 +1,81 @@
+"""extract_wheels
+
+extract_wheels resolves and fetches artifacts transitively from the Python Package Index (PyPI) based on a
+requirements.txt. It generates the required BUILD files to consume these packages as Python libraries.
+
+Under the hood, it depends on the `pip wheel` command to do resolution, download, and compilation into wheels.
+"""
+import argparse
+import glob
+import os
+import subprocess
+import sys
+
+from extract_wheels.lib import wheel, bazel
+
+
+def configure_reproducible_wheels() -> None:
+    """Modifies the environment to make wheel building reproducible.
+
+    Wheels created from sdists are not reproducible by default. We can however workaround this by
+    patching in some configuration with environment variables.
+    """
+
+    # wheel, by default, enables debug symbols in GCC. This incidentally captures the build path in the .so file
+    # We can override this behavior by disabling debug symbols entirely.
+    # https://github.com/pypa/pip/issues/6505
+    if "CFLAGS" in os.environ:
+        os.environ["CFLAGS"] += " -g0"
+    else:
+        os.environ["CFLAGS"] = "-g0"
+
+    # set SOURCE_DATE_EPOCH to 1980 so that we can use python wheels
+    # https://github.com/NixOS/nixpkgs/blob/master/doc/languages-frameworks/python.section.md#python-setuppy-bdist_wheel-cannot-create-whl
+    if "SOURCE_DATE_EPOCH" not in os.environ:
+        os.environ["SOURCE_DATE_EPOCH"] = "315532800"
+
+    # Python wheel metadata files can be unstable.
+    # See https://bitbucket.org/pypa/wheel/pull-requests/74/make-the-output-of-metadata-files/diff
+    if "PYTHONHASHSEED" not in os.environ:
+        os.environ["PYTHONHASHSEED"] = "0"
+
+
+def main() -> None:
+    """Main program.
+
+    Exits zero on successful program termination, non-zero otherwise.
+    """
+
+    configure_reproducible_wheels()
+
+    parser = argparse.ArgumentParser(
+        description="Resolve and fetch artifacts transitively from PyPI"
+    )
+    parser.add_argument(
+        "--requirements",
+        action="store",
+        required=True,
+        help="Path to requirements.txt from where to install dependencies",
+    )
+    parser.add_argument(
+        "--repo",
+        action="store",
+        required=True,
+        help="The external repo name to install dependencies. In the format '@{REPO_NAME}'",
+    )
+    args = parser.parse_args()
+
+    # Assumes any errors are logged by pip so do nothing. This command will fail if pip fails
+    subprocess.check_output(
+        [sys.executable, "-m", "pip", "wheel", "-r", args.requirements]
+    )
+
+    targets = [
+        '"%s%s"' % (args.repo, wheel.extract_wheel(whl, []))
+        for whl in glob.glob("*.whl")
+    ]
+
+    with open("requirements.bzl", "w") as requirement_file:
+        requirement_file.write(
+            bazel.generate_requirements_file_contents(args.repo, targets)
+        )

extract_wheels/__main__.py

@@ -0,0 +1,5 @@
+"""Main entry point."""
+import extract_wheels
+
+if __name__ == "__main__":
+    extract_wheels.main()

extract_wheels/lib/BUILD

@@ -0,0 +1,28 @@
+load("//:repositories.bzl", "requirement")
+
+py_library(
+    name = "lib",
+    visibility = ["//extract_wheels:__subpackages__"],
+    srcs = [
+        "bazel.py",
+        "namespace_pkgs.py",
+        "purelib.py",
+        "wheel.py",
+    ],
+    deps = [
+        requirement("pkginfo"),
+        requirement("setuptools"),
+    ],
+)
+
+py_test(
+    name = "namespace_pkgs_test",
+    size = "small",
+    srcs = [
+        "namespace_pkgs_test.py",
+    ],
+    tags = ["unit"],
+    deps = [
+        ":lib",
+    ],
+)