Now renders files with fluffy-code

Author: chriskuehl

poetry.lock

@@ -1,6 +1,12 @@
+import io
 import mimetypes
 import os.path
 
+import pygments.lexers
+import pygments.lexers.special
+import fluffy_code.code
+import fluffy_code.prebuilt_styles
+from identify import identify
 from starlette.applications import Starlette
 from starlette.staticfiles import StaticFiles
 from starlette.requests import Request
@@ -16,6 +22,35 @@
 PACKAGE_TYPE_NOT_SUPPORTED_ERROR = (
     "Sorry, this package type is not yet supported (only .zip and .whl supported currently)."
 )
+TEXT_RENDER_FILESIZE_LIMIT = 20 * 1024  # 20 KiB
+
+# Mime types which are allowed to be presented as detected.
+# TODO: I think we actually only need to prevent text/html (and any HTML
+# variants like XHTML)?
+MIME_WHITELIST = (
+    'application/javascript',
+    'application/json',
+    'application/pdf',
+    'application/x-ruby',
+    'audio/',
+    'image/',
+    'text/css',
+    'text/plain',
+    'text/x-python',
+    'text/x-sh',
+    'video/',
+)
+
+# Mime types which should be displayed inline in the browser, as opposed to
+# being downloaded. This is used to populate the Content-Disposition header.
+# Only binary MIMEs need to be whitelisted here, since detected non-binary
+# files are always inline.
+INLINE_DISPLAY_MIME_WHITELIST = (
+    'application/pdf',
+    'audio/',
+    'image/',
+    'video/',
+)
 
 
 install_root = os.path.dirname(__file__)
@@ -124,17 +159,100 @@ async def package_file_archive_path(request: Request) -> Response:
             status_code=404,
         )
     entry = matching_entries[0]
+    mimetype, _ = mimetypes.guess_type(archive_path)
 
-    async def transfer_file():
-        async with package.open_from_archive(archive_path) as f:
-            data = None
-            while data is None or len(data) > 0:
-                data = await f.read(1024)
-                yield data
+    def _transfer_raw():
+        """Return the file verbatim."""
+        async def transfer_file():
+            async with package.open_from_archive(archive_path) as f:
+                data = None
+                while data is None or len(data) > 0:
+                    data = await f.read(1024)
+                    yield data
 
-    mimetype, _ = mimetypes.guess_type(archive_path)
-    return StreamingResponse(
-        transfer_file(),
-        media_type=mimetype or "text/plain",
-        headers={"Content-Length": str(entry.size)},
+        return StreamingResponse(
+            transfer_file(),
+            media_type=mimetype if mimetype.startswith(MIME_WHITELIST) else None,
+            headers={"Content-Length": str(entry.size)},
+        )
+
+    if "raw" in request.query_params:
+        return _transfer_raw()
+
+    # There are a few cases to handle here:
+    #   (1) Reasonable-length text: render syntax highlighted in HTML
+    #   (2) Extremely long text: don't render, just show error and offer
+    #       link to the raw file
+    #   (3) Binary file that browsers can display (e.g. image): render raw
+    #   (4) Binary file that browsers cannot display (e.g. tarball): don't
+    #       render, just show warning and link to the raw file
+    #
+    # Note that except for images, the file extension isn't too useful to
+    # determine the actual content since there are lots of files without
+    # extensions (and lots of extensions not recognized by `mimetypes`).
+    if mimetype is not None and mimetype.startswith(INLINE_DISPLAY_MIME_WHITELIST):
+        # Case 3: render binary
+        return _transfer_raw()
+
+    # Figure out if it looks like text or not.
+    async with package.open_from_archive(archive_path) as f:
+        first_chunk = await f.read(TEXT_RENDER_FILESIZE_LIMIT)
+
+    is_text = identify.is_text(io.BytesIO(first_chunk))
+
+    if is_text:
+        if entry.size <= TEXT_RENDER_FILESIZE_LIMIT:
+            # Case 1: render syntax-highlighted.
+            style_config = fluffy_code.prebuilt_styles.default_style()
+
+            try:
+                lexer = pygments.lexers.guess_lexer_for_filename(
+                    archive_path,
+                    first_chunk,
+                )
+            except pygments.lexers.ClassNotFound:
+                lexer = pygments.lexers.special.TextLexer()
+
+            return templates.TemplateResponse(
+                "package_file_archive_path.html",
+                {
+                    "request": request,
+                    "package": package_name,
+                    "filename": file_name,
+                    "archive_path": archive_path,
+                    "rendered_text": fluffy_code.code.render(
+                        first_chunk,
+                        style_config=style_config,
+                        highlight_config=fluffy_code.code.HighlightConfig(
+                            lexer=lexer,
+                            highlight_diff=False,
+                        ),
+                    ),
+                    "extra_css": fluffy_code.code.get_global_css() + "\n" + style_config.css,
+                    "extra_js": fluffy_code.code.get_global_javascript(),
+                },
+            )
+        else:
+            # Case 2: too long to syntax highlight.
+            return templates.TemplateResponse(
+                "package_file_archive_path_cannot_render.html",
+                {
+                    "request": request,
+                    "package": package_name,
+                    "filename": file_name,
+                    "archive_path": archive_path,
+                    "error": "This file is too long to syntax highlight.",
+                },
+            )
+
+    # Case 4: link to binary
+    return templates.TemplateResponse(
+        "package_file_archive_path_cannot_render.html",
+        {
+            "request": request,
+            "package": package_name,
+            "filename": file_name,
+            "archive_path": archive_path,
+            "error": "This file appears to be a binary.",
+        },
     )

pypi_view/app.py

@@ -2,6 +2,7 @@
 <html>
     <head>
         <link rel="stylesheet" href="{{url_for('static', path='/site.css')}}" />
+        {% block extra_head %}{% endblock %}
     </head>
     <body class="page-{{page}}">
         {% block content %}{% endblock %}

pypi_view/templates/base.html

@@ -0,0 +1,24 @@
+{% set page = 'package-file-archive-path' %}
+{% extends 'base.html' %}
+
+{% block extra_head %}
+    <style>
+        {{extra_css|safe}}
+    </style>
+{% endblock %}
+
+{% block extra_js %}
+    <script>
+        {{extra_js|safe}}
+    </script>
+{% endblock %}
+
+{% block content %}
+    <h1>{{package}}: {{filename}}</h1>
+    <h2>{{archive_path}}</h2>
+    {{rendered_text|safe}}
+    <p><a href="?raw">View Raw</a></p>
+{% endblock %}
+
+{# vim: ft=jinja
+#}

pypi_view/templates/package_file_archive_path.html

@@ -0,0 +1,12 @@
+{% set page = 'package-file-archive-path' %}
+{% extends 'base.html' %}
+
+{% block content %}
+    <h1>{{package}}: {{filename}}</h1>
+    <h2>{{archive_path}}</h2>
+    <p>{{error}}</p>
+    <p><a href="?raw">View Raw</a></p>
+{% endblock %}
+
+{# vim: ft=jinja
+#}

pypi_view/templates/package_file_archive_path_cannot_render.html

@@ -10,10 +10,12 @@ packages = [{include = "pypi_view"}]
 [tool.poetry.dependencies]
 python = "^3.9"
 starlette = "*"
-fluffy-code = "^0.0.0"
+fluffy-code = "^0.0.1"
 Jinja2 = "^3.1.2"
 httpx = "^0.23.0"
 aiofiles = "^22.1.0"
+identify = "^2.5.5"
+Pygments = "^2.13.0"
 
 
 [tool.poetry.group.dev.dependencies]