Add ability to programatically determine available python versoins

relenv/common.py

@@ -404,6 +404,61 @@ def fetch_url(url, fp, backoff=3, timeout=30):
         # fp.close()
     log.info("Download complete %s", url)
 
+def fetch_url_content(url, backoff=3, timeout=30):
+    """
+    Fetch the contents of a url.
+
+    This method will store the contents in the given file like object.
+    """
+    # Late import so we do not import hashlib before runtime.bootstrap is called.
+    import urllib.error
+    import urllib.request
+    import io
+    import gzip
+
+    fp = io.BytesIO()
+
+    last = time.time()
+    if backoff < 1:
+        backoff = 1
+    n = 0
+    while n < backoff:
+        n += 1
+        try:
+            fin = urllib.request.urlopen(url, timeout=timeout)
+        except (
+            urllib.error.HTTPError,
+            urllib.error.URLError,
+            http.client.RemoteDisconnected,
+        ) as exc:
+            if n >= backoff:
+                raise RelenvException(f"Error fetching url {url} {exc}")
+            log.debug("Unable to connect %s", url)
+            time.sleep(n * 10)
+    log.info("url opened %s", url)
+    try:
+        total = 0
+        size = 1024 * 300
+        block = fin.read(size)
+        while block:
+            total += size
+            if time.time() - last > 10:
+                log.info("%s > %d", url, total)
+                last = time.time()
+            fp.write(block)
+            block = fin.read(10240)
+    finally:
+        fin.close()
+        # fp.close()
+    log.info("Download complete %s", url)
+    fp.seek(0)
+    info = fin.info()
+    if 'content-encoding' in info:
+        if info['content-encoding'] == 'gzip':
+            print("GZIPED")
+            fp = gzip.GzipFile(fileobj=fp)
+    return fp.read().decode()
+
 
 def download_url(url, dest, verbose=True, backoff=3, timeout=60):
     """

relenv/pyversions.py

@@ -0,0 +1,150 @@
+"""
+tools to determin python versions and download source code and signatures
+"""
+try:
+    import requests
+    from packaging.version import Version
+except ImportError:
+    raise RuntimeError("Required dependencies not found. Please pip install relenv[pyversions]")
+
+from relenv.common import fetch_url_content
+
+import subprocess
+import logging
+import re
+
+
+KEYSERVERS = [
+    "keyserver.ubuntu.com",
+    "keys.openpgp.org",
+    "pgp.mit.edu",
+]
+
+
+def ref_version(x):
+    _ = x.split("Python ", 1)[1].split("<", 1)[0]
+    return Version(_)
+
+def ref_path(x):
+    return x.split('href="')[1].split('"')[0]
+
+
+ARCHIVE = "https://www.python.org/ftp/python/{version}/Python-{version}.{ext}"
+
+def release_urls(version, gzip=False):
+    if gzip:
+        tarball = f"https://www.python.org/ftp/python/{version}/Python-{version}.tgz"
+    else:
+        tarball = f"https://www.python.org/ftp/python/{version}/Python-{version}.tar.xz"
+    # No signatures prior to 2.3
+    if version < Version("2.3"):
+        return tarball, None
+    return tarball, f"{tarball}.asc"
+
+
+print("Get downloads page")
+#reply = requests.get("https://www.python.org/downloads/")
+content = fetch_url_content("https://www.python.org/downloads/")
+print("Got downloads page")
+
+matched = re.findall(rf'<a href="/downloads/.*">Python.*</a>', content)
+
+versions = sorted([ref_version(_) for _ in matched], reverse=True)
+
+def download_file(url):
+    local_filename = url.split('/')[-1]
+    # NOTE the stream=True parameter below
+    with requests.get(url, stream=True) as r:
+        r.raise_for_status()
+        with open(local_filename, 'wb') as f:
+            for chunk in r.iter_content(chunk_size=65032):
+                # If you have chunk encoded response uncomment if
+                # and set chunk_size parameter to None.
+                #if chunk:
+                f.write(chunk)
+    return local_filename
+
+def check_status(url):
+    reply = requests.head(url)
+    if reply.status_code != 200:
+        print(f"Got {reply.status_code} for {url}")
+        return False
+    return True
+
+
+def receive_key(keyid, server):
+    proc = subprocess.run(["gpg", "--keyserver", server, "--recv-keys", keyid], capture_output=True)
+    if proc.returncode == 0:
+        return True
+    return False
+
+def get_keyid(proc):
+    try:
+        err = proc.stderr.decode()
+        return err.splitlines()[1].rsplit(" ", 1)[-1]
+    except (AttributeError, IndexError):
+        return False
+
+def verify_signature(path, signature):
+    proc = subprocess.run(["gpg", "--verify", signature, path], capture_output=True)
+    keyid = get_keyid(proc)
+    if proc.returncode == 0:
+        print(f"Valid signature {path} {keyid}")
+        return True
+    err = proc.stderr.decode()
+    if "No public key" in err:
+        for server in KEYSERVERS:
+            if receive_key(keyid, server):
+                print(f"found public key {keyid} on {server}")
+                break
+        else:
+            print("Unable to find key {keyid}  on any server")
+    else:
+        print(f"Signature verification failed {proc.stderr.decode()}")
+        return False
+    proc = subprocess.run(["gpg", "--verify", signature, path], capture_output=True)
+    if proc.returncode == 0:
+        print(f"Valid signature {path} {signature}")
+        return True
+    err = proc.stderr.decode()
+    print(f"Signature verification failed {proc.stderr.decode()}")
+    return False
+
+CHECK = True
+VERSION = None # '3.13.2'
+
+def main():
+    for version in versions:
+        if VERSION and Version(VERSION) != version:
+            continue
+        print(f"Check version {version}")
+
+        # Prior to 3.2.0 the url format only included major and minor.
+        if version <= Version('3.2') and version.micro == 0:
+            version = Version(f"{version.major}.{version.minor}")
+
+        # No xz archives prior to 3.1.4
+        if version >= Version('3.1.4'):
+            url = ARCHIVE.format(version=version, ext="tar.xz")
+            if CHECK:
+                check_status(url)
+                check_status(f"{url}.asc")
+            else:
+                path = download_file(url)
+                sig_path = download_file(f"{url}.asc")
+                verify_signature(path, sig_path)
+
+        url = ARCHIVE.format(version=version, ext="tgz")
+        if CHECK:
+            check_status(url)
+            # No signatures prior to 2.3
+            if version >= Version("2.3"):
+                check_status(f"{url}.asc")
+        else:
+            path = download_file(url)
+            if version >= Version("2.3"):
+                sig_path = download_file(f"{url}.asc")
+                verify_signature(path, sig_path)
+
+if __name__ == "__main__":
+    main()