Add support for browsing tarballs

Author: chriskuehl

README.md

@@ -33,9 +33,6 @@ You can see all uploaded package archives for a given package.
 
 You can inspect a package archive's metadata and its contents.
 
-Note that this is currently only supported for `.zip` and `.whl` files, but
-support for tarballs is planned.
-
 
 ### Easily view files from package archives
 
@@ -80,12 +77,3 @@ $ make start-dev
 ```
 
 to run a copy of the application locally with hot reloading enabled.
-
-
-## Roadmap
-
-* Add support for non-ZIP files (e.g. `.tar.gz` source distributions).
-
-  This shouldn't be too hard, but it may be slow at runtime since unlike zip,
-  tarballs don't contain an index or support extracting individual files
-  without reading the entire tarball.

pypi_browser/app.py

@@ -26,6 +26,7 @@
 
 import pypi_browser.packaging
 from pypi_browser import pypi
+from pypi_browser.packaging import PackageFormat
 
 
 PACKAGE_TYPE_NOT_SUPPORTED_ERROR = (
@@ -207,6 +208,7 @@ async def package_file(request: Request) -> Response:
         {
             'request': request,
             'package': package_name,
+            'package_is_tarball': package.package_format is PackageFormat.TARBALL,
             'filename': file_name,
             'entries': entries,
             'metadata_path': metadata_path,
@@ -319,6 +321,7 @@ async def transfer_file():
                 {
                     'request': request,
                     'package': package_name,
+                    'package_is_tarball': package.package_format is PackageFormat.TARBALL,
                     'filename': file_name,
                     'archive_path': archive_path,
                     'rendered_text': fluffy_code.code.render(
@@ -341,6 +344,7 @@ async def transfer_file():
                 {
                     'request': request,
                     'package': package_name,
+                    'package_is_tarball': package.package_format is PackageFormat.TARBALL,
                     'filename': file_name,
                     'archive_path': archive_path,
                     'metadata': metadata,
@@ -354,6 +358,7 @@ async def transfer_file():
         {
             'request': request,
             'package': package_name,
+            'package_is_tarball': package.package_format is PackageFormat.TARBALL,
             'filename': file_name,
             'archive_path': archive_path,
             'metadata': metadata,

pypi_browser/packaging.py

@@ -5,6 +5,7 @@
 import os.path
 import re
 import stat
+import tarfile
 import typing
 import zipfile
 from dataclasses import dataclass
@@ -27,8 +28,6 @@ class PackageType(enum.Enum):
 class PackageFormat(enum.Enum):
     ZIPFILE = enum.auto()
     TARBALL = enum.auto()
-    TARBALL_GZ = enum.auto()
-    TARBALL_BZ2 = enum.auto()
 
 
 @dataclass(frozen=True)
@@ -51,6 +50,19 @@ def _package_entries_from_zipfile(path: str) -> typing.Set[PackageEntry]:
         }
 
 
+def _package_entries_from_tarball(path: str) -> typing.Set[PackageEntry]:
+    with tarfile.open(path) as tf:
+        return {
+            PackageEntry(
+                path=entry.name,
+                size=entry.size,
+                mode=stat.filemode(entry.mode),
+            )
+            for entry in tf.getmembers()
+            if not entry.isdir()
+        }
+
+
 ArchiveFile = typing.Union[zipfile.ZipExtFile]
 
 
@@ -90,8 +102,10 @@ def from_path(cls, path: str) -> 'Package':
         elif name.endswith('.egg'):
             package_type = PackageType.EGG
             package_format = PackageFormat.ZIPFILE
+        elif name.endswith(('.tar', '.tar.gz', '.tgz', '.tar.bz2')):
+            package_type = PackageType.SDIST
+            package_format = PackageFormat.TARBALL
         else:
-            # TODO: Add support for tarballs
             raise UnsupportedPackageType(name)
 
         return cls(
@@ -103,6 +117,8 @@ def from_path(cls, path: str) -> 'Package':
     async def entries(self) -> typing.Set[PackageEntry]:
         if self.package_format is PackageFormat.ZIPFILE:
             return await asyncio.to_thread(_package_entries_from_zipfile, self.path)
+        elif self.package_format is PackageFormat.TARBALL:
+            return await asyncio.to_thread(_package_entries_from_tarball, self.path)
         else:
             raise AssertionError(self.package_format)
 
@@ -116,5 +132,13 @@ async def open_from_archive(self, path: str) -> str:
                     yield zip_archive_file
             finally:
                 await asyncio.to_thread(zf.close)
+        elif self.package_format is PackageFormat.TARBALL:
+            tf = await asyncio.to_thread(tarfile.open, self.path)
+            archive_file = await asyncio.to_thread(tf.extractfile, path)
+            try:
+                async with AsyncArchiveFile(archive_file) as zip_archive_file:
+                    yield zip_archive_file
+            finally:
+                await asyncio.to_thread(tf.close)
         else:
             raise AssertionError(self.package_format)

pypi_browser/templates/_macros.html

@@ -8,5 +8,15 @@
     {% endif %}
 {% endmacro %}
 
+{% macro tarball_warning(is_tarball) %}
+    {% if is_tarball %}
+        <div class="alert alert-warning" role="alert">
+            You're browsing the contents of a tarball which may be slow as
+            tarballs do not support random access. Browse a <tt>.whl</tt> or
+            <tt>.zip</tt> file for best performance.
+        </div>
+    {% endif %}
+{% endmacro %}
+
 {# vim: ft=jinja
 #}

pypi_browser/templates/package_file.html

@@ -24,6 +24,8 @@ <h1 class="font-monospace">{{filename}}</h1>
 
     <h5>{{macros.package_type_pill(filename)}}</h5>
 
+    {{macros.tarball_warning(package_is_tarball)}}
+
     {% if metadata_path %}
         <h4>Package Metadata</h4>
         <table class="table table-bordered table-sm small metadata">

pypi_browser/templates/package_file_archive_path.html

@@ -1,6 +1,7 @@
 {% set page = 'package-file-archive-path' %}
 {% set selected_package_name = package %}
 {% extends '_base.html' %}
+{% import '_macros.html' as macros %}
 
 {% block title %}{{archive_path}} | {{filename}} | {{package}} | PyPI Browser{% endblock %}
 
@@ -40,6 +41,8 @@ <h1 class="font-monospace">{{archive_path}}</h1>
         </ol>
     </nav>
 
+    {{macros.tarball_warning(package_is_tarball)}}
+
     <div class="row mb-6">
         <div class="col col-sm-9">
             {% if cannot_render_error %}