File fetching from PyPI works

Author: chriskuehl

poetry.lock

@@ -3,9 +3,12 @@
 from starlette.applications import Starlette
 from starlette.staticfiles import StaticFiles
 from starlette.requests import Request
+from starlette.responses import PlainTextResponse
 from starlette.routing import Route
 from starlette.templating import Jinja2Templates
 
+from pypi_view import pypi
+
 
 install_root = os.path.dirname(__file__)
 
@@ -20,3 +23,35 @@
 @app.route('/')
 async def home(request: Request) -> templates.TemplateResponse:
     return templates.TemplateResponse("home.html", {"request": request})
+
+
+@app.route('/package/{package}')
+async def package(request: Request) -> templates.TemplateResponse:
+    package = request.path_params["package"]
+    version_to_files = await pypi.files_for_package(package)
+    return templates.TemplateResponse(
+        "package.html",
+        {
+            "request": request,
+            "package": package,
+            "version_to_files": version_to_files,
+        },
+    )
+
+
+@app.route('/package/{package}/{filename}')
+async def package_file(request: Request) -> templates.TemplateResponse:
+    package = request.path_params["package"]
+    filename = request.path_params["filename"]
+    archive = await pypi.downloaded_file_path(package, filename)
+    return PlainTextResponse(
+        f"Path: {archive} Size: {os.stat(archive)}",
+    )
+
+
+@app.route('/package/{package}/{filename}/{archive_path:path}')
+async def package_file_archive_path(request: Request) -> templates.TemplateResponse:
+    print(request.path_params["package"])
+    print(request.path_params["filename"])
+    print(request.path_params["archive_path"])
+    raise NotImplementedError()

pypi_view/app.py

@@ -0,0 +1,89 @@
+import base64
+import contextlib
+import os.path
+import itertools
+import typing
+
+import aiofiles
+import httpx
+
+
+# TODO: make configurable
+PYPI = 'https://pypi.org'
+STORAGE_DIR = '/home/ckuehl/tmp/pypi-view'
+
+
+async def package_metadata(client: httpx.AsyncClient, package: str) -> typing.Dict:
+    resp = await client.get(f'{PYPI}/pypi/{package}/json')
+    resp.raise_for_status()
+    return resp.json()
+
+
+async def files_for_package(package: str) -> typing.Dict[str, typing.Set[str]]:
+    async with httpx.AsyncClient() as client:
+        metadata = await package_metadata(client, package)
+
+    return {
+        version: {file_["filename"] for file_ in files}
+        for version, files in metadata["releases"].items()
+    }
+
+
+class CannotFindFileError(Exception):
+    pass
+
+
+def _storage_path(package: str, filename: str) -> str:
+    return os.path.join(
+        STORAGE_DIR,
+        # Base64-encoding the names to calculate the storage path just to be
+        # extra sure to avoid any path traversal vulnerabilities.
+        base64.urlsafe_b64encode(package.encode("utf8")).decode("ascii"),
+        base64.urlsafe_b64encode(filename.encode("utf8")).decode("ascii"),
+    )
+
+@contextlib.asynccontextmanager
+async def _atomic_file(path: str, mode: str = 'w') -> typing.Any:
+    async with aiofiles.tempfile.NamedTemporaryFile(mode, dir=os.path.dirname(path), delete=False) as f:
+        try:
+            yield f
+        except:
+            os.remove(f.name)
+            raise
+        else:
+            # This is atomic since the temporary file was created in the same directory.
+            os.rename(f.name, path)
+
+
+async def downloaded_file_path(package: str, filename: str) -> str:
+    """Return path on filesystem to downloaded PyPI file.
+
+    May be instant if the file is already cached; otherwise it will download
+    it and may take a while.
+    """
+    stored_path = _storage_path(package, filename)
+    if os.path.exists(stored_path):
+        return stored_path
+
+    async with httpx.AsyncClient() as client:
+        metadata = await package_metadata(client, package)
+
+        # Parsing versions from non-wheel Python packages isn't perfectly
+        # reliable, so just search through all releases until we find a
+        # matching file.
+        for file_ in itertools.chain.from_iterable(metadata["releases"].values()):
+            if file_["filename"] == filename:
+                url = file_["url"]
+                break
+        else:
+            raise CannotFindFileError(package, filename)
+
+        os.makedirs(os.path.dirname(stored_path), exist_ok=True)
+
+        async with _atomic_file(stored_path, 'wb') as f:
+            async with client.stream('GET', url) as resp:
+                resp.raise_for_status()
+                async for chunk in resp.aiter_bytes():
+                    await f.write(chunk)
+
+        return stored_path

pypi_view/pypi.py

@@ -4,8 +4,7 @@
         <link rel="stylesheet" href="{{url_for('static', path='/site.css')}}" />
     </head>
     <body class="page-{{page}}">
-        <h1>hello!</h1>
-
+        {% block content %}{% endblock %}
         <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.6.1/jquery.min.js"></script>
         {% block extra_js %}{% endblock %}
     </body>

pypi_view/templates/base.html

@@ -0,0 +1,19 @@
+{% set page = 'package' %}
+{% extends 'base.html' %}
+
+{% block content %}
+    <h1>{{package}}</h1>
+    {% for version in version_to_files|sort %}
+        <h2>{{version}}</h2>
+        <ul>
+            {% for file in version_to_files[version]|sort %}
+                <li>
+                    <a href="{{url_for('package_file', package=package, filename=file)}}">{{file}}</a>
+                </li>
+            {% endfor %}
+        </ul>
+    {% endfor %}
+{% endblock %}
+
+{# vim: ft=jinja
+#}

pypi_view/templates/package.html

@@ -12,6 +12,8 @@ python = "^3.8"
 starlette = "*"
 fluffy-code = "^0.0.0"
 Jinja2 = "^3.1.2"
+httpx = "^0.23.0"
+aiofiles = "^22.1.0"
 
 
 [tool.poetry.group.dev.dependencies]