Merge pull request #3159 from regro/pypi-api

feat: use pypi api directly to get urls

Author: beckermr

conda_forge_tick/update_recipe/version.py

@@ -13,6 +13,7 @@
 
 import jinja2
 import jinja2.sandbox
+import requests
 from conda_forge_feedstock_ops.container_utils import (
     get_default_log_level_args,
     run_container_operation,
@@ -140,6 +141,79 @@ def _render_jinja2(tmpl, context):
     )
 
 
+def _try_pypi_api(url_tmpl: str, context: MutableMapping, hash_type: str):
+    if "name" not in context:
+        return None, None
+
+    if "version" not in context:
+        return None, None
+
+    if not any(
+        pypi_slug in url_tmpl
+        for pypi_slug in ["/pypi.org", "/pypi.io", "/files.pythonhosted.org"]
+    ):
+        return None, None
+
+    r = requests.get(
+        f"https://pypi.org/simple/{context['name']}/",
+        headers={"Accept": "application/vnd.pypi.simple.v1+json"},
+    )
+    r.raise_for_status()
+
+    data = r.json()
+    logger.debug("PyPI API data:\n%s", pprint.pformat(data))
+
+    valid_src_exts = {".tar.gz", ".tar.bz2", ".tar.xz", ".zip", ".tgz"}
+    finfo = None
+    ext = None
+    for _finfo in data["files"]:
+        for valid_ext in valid_src_exts:
+            if _finfo["filename"].endswith(context["version"] + valid_ext):
+                ext = valid_ext
+                finfo = _finfo
+                break
+
+    if finfo is None or ext is None:
+        logger.debug(
+            "src dist for version %s not found in PyPI API", context["version"]
+        )
+        return None, None
+
+    bn, _ = os.path.split(url_tmpl)
+    pypi_name = finfo["filename"].split(context["version"] + ext)[0]
+    logger.debug("PyPI API file name: %s", pypi_name)
+    name_tmpl = None
+    for tmpl in [
+        "{{ name }}",
+        "{{ name.lower() }}",
+        "{{ name.replace('-', '_') }}",
+        "{{ name.replace('_', '-') }}",
+        "{{ name.replace('-', '_').lower() }}",
+        "{{ name.replace('_', '-').lower() }}",
+    ]:
+        if pypi_name == _render_jinja2(tmpl, context) + "-":
+            name_tmpl = tmpl
+            break
+
+    if name_tmpl is not None:
+        new_url_tmpl = os.path.join(bn, name_tmpl + "-" + "{{ version }}" + ext)
+    else:
+        new_url_tmpl = os.path.join(
+            bn, finfo["filename"].replace(context["version"], "{{ version }}")
+        )
+
+    logger.debug("new url template from PyPI API: %s", new_url_tmpl)
+    url = _render_jinja2(new_url_tmpl, context)
+    new_hash = _try_url_and_hash_it(url, hash_type)
+    if new_hash is not None:
+        return new_url_tmpl, new_hash
+
+    new_url_tmpl = finfo["url"]
+    new_hash = _try_url_and_hash_it(url, hash_type)
+    if new_hash is not None:
+        return new_url_tmpl, new_hash
+
+
 def _get_new_url_tmpl_and_hash(url_tmpl: str, context: MutableMapping, hash_type: str):
     logger.info(
         "hashing URL template: %s",
@@ -165,6 +239,13 @@ def _get_new_url_tmpl_and_hash(url_tmpl: str, context: MutableMapping, hash_type
     new_url_tmpl = None
     new_hash = None
 
+    try:
+        new_url_tmpl, new_hash = _try_pypi_api(url_tmpl, context, hash_type)
+        if new_hash is not None and new_url_tmpl is not None:
+            return new_url_tmpl, new_hash
+    except Exception as e:
+        logger.debug("PyPI API url+hash update failed: %s", repr(e), exc_info=e)
+
     for new_url_tmpl in gen_transformed_urls(url_tmpl):
         try:
             url = _render_jinja2(new_url_tmpl, context)

tests/test_version_migrator.py

@@ -61,6 +61,7 @@
         ("quart_trio", "0.11.1"),
         ("reproc", "14.2.5"),
         ("riskfolio_lib", "6.3.1"),
+        ("algotree", "0.7.3"),
         # these contain sources that depend on conda build config variants
         pytest.param(
             "polars_mixed_selectors",

tests/test_yaml/version_21cmfast_correct.yaml

@@ -6,7 +6,7 @@ package:
   version: {{ version }}
 
 source:
-  url: https://pypi.io/packages/source/{{ name[0] }}/{{ name|lower }}/{{ name|lower }}-{{ version }}.tar.gz
+  url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name.lower() }}-{{ version }}.tar.gz
   sha256: aff3b5a2adb30ad9a6c2274461901686606e9fdb5e3ff7040cbdf22755d7469f
 
 

tests/test_yaml/version_algotree.yaml

@@ -0,0 +1,45 @@
+{% set name = "algotree" %}
+{% set version = "0.7.2" %}
+
+package:
+  name: {{ name|lower }}
+  version: {{ version }}
+
+source:
+  url: https://pypi.org/packages/source/{{ name[0] }}/{{ name }}/algotree-{{ version }}.tar.gz
+  sha256: 13bb4dc6583a58e1784adee20f4d803bd4105359f1c6f52151e496d23f07152a
+
+build:
+  entry_points:
+    - jt=bin.jt:main
+  noarch: python
+  script: {{ PYTHON }} -m pip install . -vv --no-deps --no-build-isolation
+  number: 0
+
+requirements:
+  host:
+    - python >=3.6
+    - pip
+    - setuptools
+  run:
+    - python >=3.6
+
+test:
+  imports:
+    - AlgoTree
+  commands:
+    - pip check
+    - jt --help
+  requires:
+    - pip
+
+about:
+  home: https://github.com/queelius/AlgoTree
+  doc_url: https://queelius.github.io/AlgoTree/
+  summary: A algorithmic tookit for working with trees in Python
+  license: MIT
+  license_file: LICENSE
+
+extra:
+  recipe-maintainers:
+    - thewchan

tests/test_yaml/version_algotree_correct.yaml

@@ -0,0 +1,45 @@
+{% set name = "algotree" %}
+{% set version = "0.7.3" %}
+
+package:
+  name: {{ name|lower }}
+  version: {{ version }}
+
+source:
+  url: https://pypi.org/packages/source/{{ name[0] }}/{{ name }}/AlgoTree-{{ version }}.tar.gz
+  sha256: 349eee21a57c5f40f157be218788f944a723047b35b7945a56058219b383287d
+
+build:
+  entry_points:
+    - jt=bin.jt:main
+  noarch: python
+  script: {{ PYTHON }} -m pip install . -vv --no-deps --no-build-isolation
+  number: 0
+
+requirements:
+  host:
+    - python >=3.6
+    - pip
+    - setuptools
+  run:
+    - python >=3.6
+
+test:
+  imports:
+    - AlgoTree
+  commands:
+    - pip check
+    - jt --help
+  requires:
+    - pip
+
+about:
+  home: https://github.com/queelius/AlgoTree
+  doc_url: https://queelius.github.io/AlgoTree/
+  summary: A algorithmic tookit for working with trees in Python
+  license: MIT
+  license_file: LICENSE
+
+extra:
+  recipe-maintainers:
+    - thewchan

tests/test_yaml/version_dash_extensions_correct.yaml

@@ -7,7 +7,7 @@ package:
   version: {{ version }}
 
 source:
-  url: https://pypi.io/packages/source/{{ name[0] }}/{{ name.replace('-', '_').lower() }}/{{ name.replace('-', '_').lower() }}-{{ version }}.tar.gz
+  url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name.replace('-', '_') }}-{{ version }}.tar.gz
   sha256: b36fcf6fd74d87cafdbabc9568c3ae0097712ccee8f7d59be8e916b51d40b106
 
 build:

tests/test_yaml/version_pypiurl_correct.yaml

@@ -6,7 +6,7 @@ package:
   version: "{{ version }}"
 
 source:
-  url: "https://pypi.io/packages/source/{{ name[0] }}/{{ name.replace('_', '-').lower() }}/{{ name.replace('_', '-').lower() }}-{{ version }}.tar.gz"
+  url: "https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name.replace('_', '-') }}-{{ version }}.tar.gz"
   sha256: f3fe3f89011899b82451669cf1dbe4978523b8ac0f62c9c116429876fe8b6be8
 
 build:

tests/test_yaml/version_pyrsmq_correct.yaml

@@ -6,7 +6,7 @@ package:
   version: "{{ version }}"
 
 source:
-  url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name|lower }}-{{ version }}.tar.gz
+  url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name.lower() }}-{{ version }}.tar.gz
   sha256: dd1f8467e541935489be018dbb0ba1df8b903eb855bf1725947ceee41df92fa4
 
 build: