feat: use pypi api directly to get urls

Author: beckermr

conda_forge_tick/update_recipe/version.py

@@ -13,6 +13,7 @@
 
 import jinja2
 import jinja2.sandbox
+import requests
 from conda_forge_feedstock_ops.container_utils import (
     get_default_log_level_args,
     run_container_operation,
@@ -140,6 +141,72 @@ def _render_jinja2(tmpl, context):
     )
 
 
+def _try_pypi_api(url_tmpl: str, context: MutableMapping, hash_type: str):
+    if "name" not in context:
+        return None, None
+
+    if "version" not in context:
+        return None, None
+
+    if not any(pypi_slug in url_tmpl for pypi_slug in ["/pypi.org", "/pypi.io", "/files.pythonhosted.org"]):
+        return None, None
+
+    r = requests.get(
+        f"https://pypi.org/simple/{context['name']}/",
+        headers={"Accept": "application/vnd.pypi.simple.v1+json"}
+    )
+    r.raise_for_status()
+
+    data = r.json()
+    logger.debug("PyPI API data:\n%s", pprint.pformat(data))
+
+    valid_src_exts = {".tar.gz", ".tar.bz2", ".tar.xz", ".zip", ".tgz"}
+    finfo = None
+    ext = None
+    for _finfo in data["files"]:
+        for valid_ext in valid_src_exts:
+            if _finfo["filename"].endswith(context["version"] + valid_ext):
+                ext = valid_ext
+                finfo = _finfo
+                break
+
+    if finfo is None or ext is None:
+        logger.debug("src dist for version %s not found in PyPI API", context["version"])
+        return None, None
+
+    bn, _ = os.path.split(url_tmpl)
+    pypi_name = finfo["filename"].split(context["version"] + ext)[0]
+    logger.debug("PyPI API file name: %s", pypi_name)
+    name_tmpl = None
+    for tmpl in [
+        "{{ name }}",
+        "{{ name.lower() }}",
+        "{{ name.replace('-', '_') }}",
+        "{{ name.replace('_', '-') }}",
+        "{{ name.replace('-', '_').lower() }}",
+        "{{ name.replace('_', '-').lower() }}",
+    ]:
+        if pypi_name == _render_jinja2(tmpl, context) + "-":
+            name_tmpl = tmpl
+            break
+
+    if name_tmpl is not None:
+        new_url_tmpl = os.path.join(bn, name_tmpl + "-" + "{{ version }}" + ext)
+    else:
+        new_url_tmpl = os.path.join(bn, finfo["filename"].replace(context["version"], "{{ version }}"))
+
+    logger.debug("new url template from PyPI API: %s", new_url_tmpl)
+    url = _render_jinja2(new_url_tmpl, context)
+    new_hash = _try_url_and_hash_it(url, hash_type)
+    if new_hash is not None:
+        return new_url_tmpl, new_hash
+
+    new_url_tmpl = finfo["url"]
+    new_hash = _try_url_and_hash_it(url, hash_type)
+    if new_hash is not None:
+        return new_url_tmpl, new_hash
+
+
 def _get_new_url_tmpl_and_hash(url_tmpl: str, context: MutableMapping, hash_type: str):
     logger.info(
         "hashing URL template: %s",
@@ -165,6 +232,13 @@ def _get_new_url_tmpl_and_hash(url_tmpl: str, context: MutableMapping, hash_type
     new_url_tmpl = None
     new_hash = None
 
+    try:
+        new_url_tmpl, new_hash = _try_pypi_api(url_tmpl, context, hash_type)
+        if new_hash is not None and new_url_tmpl is not None:
+            return new_url_tmpl, new_hash
+    except Exception as e:
+        logger.debug("PyPI API url+hash update failed: %s", repr(e), exc_info=e)
+
     for new_url_tmpl in gen_transformed_urls(url_tmpl):
         try:
             url = _render_jinja2(new_url_tmpl, context)

tests/test_yaml/version_21cmfast_correct.yaml

@@ -6,7 +6,7 @@ package:
   version: {{ version }}
 
 source:
-  url: https://pypi.io/packages/source/{{ name[0] }}/{{ name|lower }}/{{ name|lower }}-{{ version }}.tar.gz
+  url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name.lower() }}-{{ version }}.tar.gz
   sha256: aff3b5a2adb30ad9a6c2274461901686606e9fdb5e3ff7040cbdf22755d7469f
 
 

tests/test_yaml/version_dash_extensions_correct.yaml

@@ -7,7 +7,7 @@ package:
   version: {{ version }}
 
 source:
-  url: https://pypi.io/packages/source/{{ name[0] }}/{{ name.replace('-', '_').lower() }}/{{ name.replace('-', '_').lower() }}-{{ version }}.tar.gz
+  url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name.replace('-', '_') }}-{{ version }}.tar.gz
   sha256: b36fcf6fd74d87cafdbabc9568c3ae0097712ccee8f7d59be8e916b51d40b106
 
 build:

tests/test_yaml/version_pypiurl_correct.yaml

@@ -6,7 +6,7 @@ package:
   version: "{{ version }}"
 
 source:
-  url: "https://pypi.io/packages/source/{{ name[0] }}/{{ name.replace('_', '-').lower() }}/{{ name.replace('_', '-').lower() }}-{{ version }}.tar.gz"
+  url: "https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name.replace('_', '-') }}-{{ version }}.tar.gz"
   sha256: f3fe3f89011899b82451669cf1dbe4978523b8ac0f62c9c116429876fe8b6be8
 
 build:

tests/test_yaml/version_pyrsmq_correct.yaml

@@ -6,7 +6,7 @@ package:
   version: "{{ version }}"
 
 source:
-  url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name|lower }}-{{ version }}.tar.gz
+  url: https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/{{ name.lower() }}-{{ version }}.tar.gz
   sha256: dd1f8467e541935489be018dbb0ba1df8b903eb855bf1725947ceee41df92fa4
 
 build: