Update thirdparty fetching utilities

pombredanne · pombredanne · commit 6a3c5b0b9e35 · 2022-05-07T19:44:52.000+02:00
These were buggy in some corner cases.
They have been updated such that:
* --latest-version works.
* we can reliable fetch combinations of wheels and sdists for multiple
  OS combos at once
* we now support macOS universal wheels (for ARM CPUs)

Caching is now simpler: we have essentially a single file-based cache
under .cache. PyPI indexes are fetched and not cached, unless the new
--use-cached-index is used which can be useful when fetching many
thirdparty in a short timeframe.

The first PyPI repository in a list has precendence and we never fetch
from other repositories if we find wheels and sdsists there. This avoid
pounding too much on the self-hosted repo.

Signed-off-by: Philippe Ombredanne &lt;pombredanne@nexb.com&gt;
diff --git a/etc/scripts/check_thirdparty.py b/etc/scripts/check_thirdparty.py
@@ -16,7 +16,7 @@
 @click.command()
 @click.option(
     "-d",
-    "--dest_dir",
+    "--dest",
     type=click.Path(exists=True, readable=True, path_type=str, file_okay=False),
     required=True,
     help="Path to the thirdparty directory to check.",
@@ -35,7 +35,7 @@
 )
 @click.help_option("-h", "--help")
 def check_thirdparty_dir(
-    dest_dir,
+    dest,
     wheels,
     sdists,
 ):
@@ -45,7 +45,7 @@ def check_thirdparty_dir(
     # check for problems
     print(f"==> CHECK FOR PROBLEMS")
     utils_thirdparty.find_problems(
-        dest_dir=dest_dir,
+        dest_dir=dest,
         report_missing_sources=sdists,
         report_missing_wheels=wheels,
     )
diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py
@@ -100,11 +100,17 @@
     "index_urls",
     type=str,
     metavar="INDEX",
-    default=utils_thirdparty.PYPI_INDEXES,
+    default=utils_thirdparty.PYPI_INDEX_URLS,
     show_default=True,
     multiple=True,
     help="PyPI index URL(s) to use for wheels and sources, in order of preferences.",
 )
+@click.option(
+    "--use-cached-index",
+    is_flag=True,
+    help="Use on disk cached PyPI indexes list of packages and versions and do not refetch if present.",
+)
+
 @click.help_option("-h", "--help")
 def fetch_thirdparty(
     requirements_files,
@@ -116,26 +122,34 @@ def fetch_thirdparty(
     wheels,
     sdists,
     index_urls,
+    use_cached_index,
 ):
     """
-    Download to --dest-dir THIRDPARTY_DIR the PyPI wheels, source distributions,
+    Download to --dest THIRDPARTY_DIR the PyPI wheels, source distributions,
     and their ABOUT metadata, license and notices files.
 
     Download the PyPI packages listed in the combination of:
     - the pip requirements --requirements REQUIREMENT-FILE(s),
     - the pip name==version --specifier SPECIFIER(s)
     - any pre-existing wheels or sdsists found in --dest-dir THIRDPARTY_DIR.
 
-    Download wheels with the --wheels option for the ``--python-version`` PYVER(s)
-    and ``--operating_system`` OS(s) combinations defaulting to all supported combinations.
+    Download wheels with the --wheels option for the ``--python-version``
+    PYVER(s) and ``--operating_system`` OS(s) combinations defaulting to all
+    supported combinations.
 
     Download sdists tarballs with the --sdists option.
 
-    Generate or Download .ABOUT, .LICENSE and .NOTICE files for all the wheels and sources fetched.
+    Generate or Download .ABOUT, .LICENSE and .NOTICE files for all the wheels
+    and sources fetched.
 
-    Download wheels and sdists the provided PyPI simple --index-url INDEX(s) URLs.
+    Download from the provided PyPI simple --index-url INDEX(s) URLs.
     """
+    if not (wheels or sdists):
+        print("Error: one or both of --wheels  and --sdists is required.")
+        sys.exit(1)
+
     print(f"COLLECTING REQUIRED NAMES & VERSIONS FROM {dest_dir}")
+
     existing_packages_by_nv = {
         (package.name, package.version): package
         for package in utils_thirdparty.get_local_packages(directory=dest_dir)
@@ -151,134 +165,88 @@ def fetch_thirdparty(
         required_name_versions.update(nvs)
 
     for specifier in specifiers:
-        nv = utils_requirements.get_name_version(
+        nv = utils_requirements.get_required_name_version(
             requirement=specifier,
             with_unpinned=latest_version,
         )
         required_name_versions.add(nv)
 
+    if latest_version:
+        names = set(name for name, _version in sorted(required_name_versions))
+        required_name_versions = {(n, None) for n in names}
+
     if not required_name_versions:
         print("Error: no requirements requested.")
         sys.exit(1)
 
-    if not os.listdir(dest_dir) and not (wheels or sdists):
-        print("Error: one or both of --wheels  and --sdists is required.")
-        sys.exit(1)
-
-    if latest_version:
-        latest_name_versions = set()
-        names = set(name for name, _version in sorted(required_name_versions))
-        for name in sorted(names):
-            latests = utils_thirdparty.PypiPackage.sorted(
-                utils_thirdparty.get_package_versions(
-                    name=name, version=None, index_urls=index_urls
-                )
-            )
-            if not latests:
-                print(f"No distribution found for: {name}")
-            continue
-            latest = latests[-1]
-            latest_name_versions.add((latest.name, latest.version))
-        required_name_versions = latest_name_versions
-
-    if TRACE:
-        print("required_name_versions:", required_name_versions)
+    if TRACE_DEEP:
+        print("required_name_versions:")
+        for n, v in required_name_versions:
+            print(f"    {n} @ {v}")
 
+    # create the environments matrix we need for wheels
+    environments = None
     if wheels:
-        # create the environments matrix we need for wheels
         evts = itertools.product(python_versions, operating_systems)
         environments = [utils_thirdparty.Environment.from_pyver_and_os(pyv, os) for pyv, os in evts]
 
-    wheels_not_found = {}
-    sdists_not_found = {}
-    # iterate over requirements, one at a time
+    # Collect PyPI repos
+    repos = []
+    for index_url in index_urls:
+        index_url = index_url.strip("/")
+        existing = utils_thirdparty.DEFAULT_PYPI_REPOS_BY_URL.get(index_url)
+        if existing:
+            existing.use_cached_index = use_cached_index
+            repos.append(existing)
+        else:
+            repo = utils_thirdparty.PypiSimpleRepository(
+                index_url=index_url,
+                use_cached_index=use_cached_index,
+            )
+            repos.append(repo)
+
+    wheels_fetched = []
+    wheels_not_found = []
+
+    sdists_fetched = []
+    sdists_not_found = []
+
     for name, version in sorted(required_name_versions):
         nv = name, version
-        existing_package = existing_packages_by_nv.get(nv)
+        print(f"Processing: {name} @ {version}")
         if wheels:
             for environment in environments:
-                if existing_package:
-                    existing_wheels = list(
-                        existing_package.get_supported_wheels(environment=environment)
-                    )
-                else:
-                    existing_wheels = None
-
-                if existing_wheels:
-                    if TRACE_DEEP:
-                        print(
-                            f"====> Wheels already available: {name}=={version} on: {environment}: {existing_package.wheels!r}"
-                        )
-                    if all(w.is_pure() for w in existing_wheels):
-                        break
-                    else:
-                        continue
-
-                if TRACE_DEEP:
-                    print(f"Fetching wheel for: {name}=={version} on: {environment}")
-
-                try:
-                    (
-                        fetched_wheel_filenames,
-                        existing_wheel_filenames,
-                    ) = utils_thirdparty.download_wheel(
-                        name=name,
-                        version=version,
-                        environment=environment,
-                        dest_dir=dest_dir,
-                        index_urls=index_urls,
-                    )
-                    if TRACE:
-                        if existing_wheel_filenames:
-                            print(
-                                f"    ====> Wheels already available: {name}=={version} on: {environment}"
-                            )
-                            for whl in existing_wheel_filenames:
-                                print(f"        {whl}")
-                        if fetched_wheel_filenames:
-                            print(f"    ====> Wheels fetched: {name}=={version} on: {environment}")
-                            for whl in fetched_wheel_filenames:
-                                print(f"        {whl}")
-
-                    fwfns = fetched_wheel_filenames + existing_wheel_filenames
-
-                    if all(utils_thirdparty.Wheel.from_filename(f).is_pure() for f in fwfns):
-                        break
-
-                except utils_thirdparty.DistributionNotFound as e:
-                    wheels_not_found[f"{name}=={version}"] = str(e)
-
-        if sdists:
-            if existing_package and existing_package.sdist:
                 if TRACE:
-                    print(
-                        f"  ====> Sdist already available: {name}=={version}: {existing_package.sdist!r}"
-                    )
-                continue
-
-            if TRACE:
-                print(f"  Fetching sdist for: {name}=={version}")
-
-            try:
-                fetched = utils_thirdparty.download_sdist(
+                    print(f"  ==> Fetching wheel for envt: {environment}")
+                fwfns = utils_thirdparty.download_wheel(
                     name=name,
                     version=version,
+                    environment=environment,
                     dest_dir=dest_dir,
-                    index_urls=index_urls,
+                    repos=repos,
                 )
+                if fwfns:
+                    wheels_fetched.extend(fwfns)
+                else:
+                    wheels_not_found.append(f"{name}=={version} for: {environment}")
+                    if TRACE:
+                        print(f"      NOT FOUND")
 
+        if sdists:
+            if TRACE:
+                print(f"  ==> Fetching sdist: {name}=={version}")
+            fetched = utils_thirdparty.download_sdist(
+                name=name,
+                version=version,
+                dest_dir=dest_dir,
+                repos=repos,
+            )
+            if fetched:
+                sdists_fetched.append(fetched)
+            else:
+                sdists_not_found.append(f"{name}=={version}")
                 if TRACE:
-                    if not fetched:
-                        print(
-                            f"    ====> Sdist already available: {name}=={version}"
-                        )
-                    else:
-                        print(
-                            f"    ====> Sdist fetched: {fetched} for {name}=={version}"
-                        )
-
-            except utils_thirdparty.DistributionNotFound as e:
-                sdists_not_found[f"{name}=={version}"] = str(e)
+                    print(f"      NOT FOUND")
 
     if wheels and wheels_not_found:
         print(f"==> MISSING WHEELS")
@@ -291,7 +259,7 @@ def fetch_thirdparty(
             print(f"  {sd}")
 
     print(f"==> FETCHING OR CREATING ABOUT AND LICENSE FILES")
-    utils_thirdparty.fetch_abouts_and_licenses(dest_dir=dest_dir)
+    utils_thirdparty.fetch_abouts_and_licenses(dest_dir=dest_dir, use_cached_index=use_cached_index)
     utils_thirdparty.clean_about_files(dest_dir=dest_dir)
 
     # check for problems
diff --git a/etc/scripts/gen_pypi_simple.py b/etc/scripts/gen_pypi_simple.py
@@ -25,26 +25,26 @@ class InvalidDistributionFilename(Exception):
 
 def get_package_name_from_filename(filename):
     """
-    Return the package name extracted from a package ``filename``.
-    Optionally ``normalize`` the name according to distribution name rules.
+    Return the normalized package name extracted from a package ``filename``.
+    Normalization is done according to distribution name rules.
     Raise an ``InvalidDistributionFilename`` if the ``filename`` is invalid::
 
     >>> get_package_name_from_filename("foo-1.2.3_rc1.tar.gz")
     'foo'
-    >>> get_package_name_from_filename("foo-bar-1.2-py27-none-any.whl")
+    >>> get_package_name_from_filename("foo_bar-1.2-py27-none-any.whl")
     'foo-bar'
     >>> get_package_name_from_filename("Cython-0.17.2-cp26-none-linux_x86_64.whl")
     'cython'
     >>> get_package_name_from_filename("python_ldap-2.4.19-cp27-none-macosx_10_10_x86_64.whl")
     'python-ldap'
-    >>> get_package_name_from_filename("foo.whl")
-    Traceback (most recent call last):
-        ...
-    InvalidDistributionFilename: ...
-    >>> get_package_name_from_filename("foo.png")
-    Traceback (most recent call last):
-        ...
-    InvalidFilePackageName: ...
+    >>> try:
+    ...     get_package_name_from_filename("foo.whl")
+    ... except InvalidDistributionFilename:
+    ...     pass
+    >>> try:
+    ...     get_package_name_from_filename("foo.png")
+    ... except InvalidDistributionFilename:
+    ...     pass
     """
     if not filename or not filename.endswith(dist_exts):
         raise InvalidDistributionFilename(filename)
diff --git a/etc/scripts/requirements.txt b/etc/scripts/requirements.txt
@@ -1,12 +1,11 @@
 aboutcode_toolkit
-github-release-retry2
 attrs
 commoncode
 click
 requests
 saneyaml
-romp
 pip
 setuptools
 twine
-wheel
+wheel
+build
diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py
@@ -8,7 +8,6 @@
 # See https://github.com/nexB/skeleton for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
-import os
 import re
 import subprocess
 
@@ -42,23 +41,23 @@ def get_required_name_versions(requirement_lines, with_unpinned=False):
         if req_line.startswith("-") or (not with_unpinned and not "==" in req_line):
             print(f"Requirement line is not supported: ignored: {req_line}")
             continue
-        yield get_name_version(requirement=req_line, with_unpinned=with_unpinned)
+        yield get_required_name_version(requirement=req_line, with_unpinned=with_unpinned)
 
 
-def get_name_version(requirement, with_unpinned=False):
+def get_required_name_version(requirement, with_unpinned=False):
     """
     Return a (name, version) tuple given a`requirement` specifier string.
     Requirement version must be pinned. If ``with_unpinned`` is True, unpinned
     requirements are accepted and only the name portion is returned.
 
     For example:
-    >>> assert get_name_version("foo==1.2.3") == ("foo", "1.2.3")
-    >>> assert get_name_version("fooA==1.2.3.DEV1") == ("fooa", "1.2.3.dev1")
-    >>> assert get_name_version("foo==1.2.3", with_unpinned=False) == ("foo", "1.2.3")
-    >>> assert get_name_version("foo", with_unpinned=True) == ("foo", "")
-    >>> assert get_name_version("foo>=1.2", with_unpinned=True) == ("foo", ""), get_name_version("foo>=1.2")
+    >>> assert get_required_name_version("foo==1.2.3") == ("foo", "1.2.3")
+    >>> assert get_required_name_version("fooA==1.2.3.DEV1") == ("fooa", "1.2.3.dev1")
+    >>> assert get_required_name_version("foo==1.2.3", with_unpinned=False) == ("foo", "1.2.3")
+    >>> assert get_required_name_version("foo", with_unpinned=True) == ("foo", "")
+    >>> assert get_required_name_version("foo>=1.2", with_unpinned=True) == ("foo", ""), get_required_name_version("foo>=1.2")
     >>> try:
-    ...   assert not get_name_version("foo", with_unpinned=False)
+    ...   assert not get_required_name_version("foo", with_unpinned=False)
     ... except Exception as e:
     ...   assert "Requirement version must be pinned" in str(e)
     """
@@ -112,7 +111,7 @@ def get_installed_reqs(site_packages_dir):
     as a text.
     """
     if not os.path.exists(site_packages_dir):
-        raise Exception(f"site_packages directort: {site_packages_dir!r} does not exists")
+        raise Exception(f"site_packages directory: {site_packages_dir!r} does not exists")
     # Also include these packages in the output with --all: wheel, distribute,
     # setuptools, pip
     args = ["pip", "freeze", "--exclude-editable", "--all", "--path", site_packages_dir]
diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py
