DOP-1816: Permit tarballing of manpages

Author: i80and

snooty/builders/test_man.py

@@ -1,49 +1,17 @@
+import io
 import re
 import subprocess
+import tarfile
 from pathlib import Path
 from typing import Dict, Union, cast
 
 import pytest
 
-from ..diagnostics import CannotOpenFile
+from ..diagnostics import CannotOpenFile, UnsupportedFormat
 from ..types import FileId
 from ..util_test import make_test
 
-
-def normalize(text: str) -> str:
-    """Remove any non-word characters to make groff output comparable
-       across platforms."""
-    # Strip the strange escape characters that Groff inserts for TTYs
-    text = re.sub(r".\x08", "", text)
-
-    # Strip the header: this varies platform to platform.
-    text = text.split("\n", 1)[1]
-
-    # Remove non-word characters
-    return re.sub(r"[^\w]+", "", text)
-
-
-def test_manpage() -> None:
-    with make_test(
-        {
-            Path(
-                "snooty.toml"
-            ): """
-name = "test_manpage"
-
-[manpages.mongo]
-file = "index.txt"
-section = 1
-title = "The MongoDB Shell"
-
-[manpages.missing]
-file = "missing.txt"
-section = 1
-title = "This Manpage Doesn't Exist"
-""",
-            Path(
-                "source/index.txt"
-            ): """
+PAGE_TEXT = """
 .. _mongo:
 
 =========
@@ -147,35 +115,9 @@ def test_manpage() -> None:
    }
 
 Trailing paragraph.
-""",
-        }
-    ) as result:
-        # Ensure that we have an error about the missing manpage
-        assert [type(diag) for diag in result.diagnostics[FileId("snooty.toml")]] == [
-            CannotOpenFile
-        ]
-
-        static_files = cast(
-            Dict[str, Union[str, bytes]], result.metadata["static_files"]
-        )
-
-        troff = static_files["mongo.1"]
-
-        assert isinstance(troff, str)
-
-        # Empty lines are discouraged in troff source
-        assert "\n\n" not in troff
+"""
 
-        try:
-            # Use GNU Roff to turn our generated troff source into text we can compare.
-            text = subprocess.check_output(
-                ["groff", "-T", "utf8", "-t", "-man"], encoding="utf-8", input=troff,
-            )
-        except FileNotFoundError:
-            pytest.xfail("groff is not installed")
-
-        assert normalize(text).strip() == normalize(
-            """mongo(1)                    General Commands Manual                   mongo(1)
+MANPAGE_TEXT = """mongo(1)                    General Commands Manual                   mongo(1)
 
 
 
@@ -246,4 +188,98 @@ def test_manpage() -> None:
 
 
 mongo(1)"""
+
+
+def normalize(text: str) -> str:
+    """Remove any non-word characters to make groff output comparable
+       across platforms."""
+    # Strip the strange escape characters that Groff inserts for TTYs
+    text = re.sub(r".\x08", "", text)
+
+    # Strip the header: this varies platform to platform.
+    text = text.split("\n", 1)[1]
+
+    # Remove non-word characters
+    return re.sub(r"[^\w]+", "", text)
+
+
+def test_manpage() -> None:
+    with make_test(
+        {
+            Path(
+                "snooty.toml"
+            ): """
+name = "test_manpage"
+
+[bundle]
+manpages = "manpages.tar.gz"
+
+[manpages.mongo]
+file = "index.txt"
+section = 1
+title = "The MongoDB Shell"
+
+[manpages.missing]
+file = "missing.txt"
+section = 1
+title = "This Manpage Doesn't Exist"
+""",
+            Path("source/index.txt"): PAGE_TEXT,
+        }
+    ) as result:
+        # Ensure that we have an error about the missing manpage
+        assert [type(diag) for diag in result.diagnostics[FileId("snooty.toml")]] == [
+            CannotOpenFile
+        ]
+
+        static_files = cast(
+            Dict[str, Union[str, bytes]], result.metadata["static_files"]
         )
+
+        troff = static_files["mongo.1"]
+
+        assert isinstance(troff, str)
+
+        # Empty lines are discouraged in troff source
+        assert "\n\n" not in troff
+
+        try:
+            # Use GNU Roff to turn our generated troff source into text we can compare.
+            text = subprocess.check_output(
+                ["groff", "-T", "utf8", "-t", "-man"], encoding="utf-8", input=troff,
+            )
+        except FileNotFoundError:
+            pytest.xfail("groff is not installed")
+
+        assert normalize(text).strip() == normalize(MANPAGE_TEXT)
+
+        tarball_data = static_files["manpages.tar.gz"]
+        assert isinstance(tarball_data, bytes)
+        tarball = io.BytesIO(tarball_data)
+        with tarfile.open(None, "r:*", tarball) as tf:
+            names = tf.getnames()
+            assert sorted(names) == sorted(["mongo.1"])
+            assert tf.getmember("mongo.1").size == len(troff)
+
+
+def test_bundling_error() -> None:
+    with make_test(
+        {
+            Path(
+                "snooty.toml"
+            ): """
+name = "test_manpage"
+
+[bundle]
+manpages = "manpages.goofy"
+
+[manpages.mongo]
+file = "index.txt"
+section = 1
+title = "The MongoDB Shell"
+""",
+            Path("source/index.txt"): PAGE_TEXT,
+        }
+    ) as result:
+        diagnostics = result.diagnostics[FileId("snooty.toml")]
+        assert UnsupportedFormat in [type(diag) for diag in diagnostics]

snooty/diagnostics.py

@@ -1,6 +1,6 @@
 import enum
 from pathlib import Path
-from typing import Dict, List, Optional, Set, Tuple, Union
+from typing import Dict, List, Optional, Sequence, Set, Tuple, Union
 
 from . import n
 from .n import SerializableType
@@ -498,3 +498,20 @@ def __init__(
             start,
             end,
         )
+
+
+class UnsupportedFormat(Diagnostic):
+    severity = Diagnostic.Level.error
+
+    def __init__(
+        self,
+        actual: str,
+        expected: Sequence[str],
+        start: Union[int, Tuple[int, int]],
+        end: Union[None, int, Tuple[int, int]] = None,
+    ) -> None:
+        super().__init__(
+            f"Unsupported file format: {actual}. Must be one of {','.join(expected)}",
+            start,
+            end,
+        )

snooty/parser.py

@@ -1,12 +1,14 @@
 import collections
 import errno
 import getpass
+import io
 import logging
 import multiprocessing
 import os
 import re
 import subprocess
 import threading
+import time
 from copy import deepcopy
 from dataclasses import dataclass
 from functools import partial
@@ -52,6 +54,7 @@
     UnexpectedIndentation,
     UnknownTabID,
     UnknownTabset,
+    UnsupportedFormat,
 )
 from .gizaparser.nodes import GizaCategory
 from .gizaparser.published_branches import PublishedBranches, parse_published_branches
@@ -72,6 +75,35 @@
 logger = logging.getLogger(__name__)
 
 
+def bundle(
+    filename: PurePath, members: Iterable[Tuple[str, Union[str, bytes]]]
+) -> bytes:
+    if filename.suffixes[-2:] == [".tar", ".gz"] or filename.suffixes[-1] == ".tar":
+        import tarfile
+
+        compression_flag = "gz" if filename.suffix == ".gz" else "::"
+
+        current_time = time.time()
+        output_file = io.BytesIO()
+        with tarfile.open(
+            None, f"w:{compression_flag}", output_file, format=tarfile.PAX_FORMAT
+        ) as tf:
+            for member_name, member_data in members:
+                if isinstance(member_data, str):
+                    member_data = bytes(member_data, "utf-8")
+                member_file = io.BytesIO(member_data)
+                tar_info = tarfile.TarInfo(name=member_name)
+                tar_info.size = len(member_data)
+                tar_info.mtime = int(current_time)
+                tar_info.mode = 0x644
+                tf.addfile(tar_info, member_file)
+
+        return output_file.getvalue()
+
+    else:
+        raise ValueError(f"Unknown bundling format: {filename.as_posix()}")
+
+
 @dataclass
 class _DefinitionListTerm(n.InlineParent):
     """A vate node used for internal book-keeping that should not be exported to the AST."""
@@ -1318,6 +1350,7 @@ def create_page(filename: str) -> Tuple[Page, EmbeddedRstParser]:
                 self.backend.flush()
 
             # Build manpages
+            manpages: List[Tuple[str, str]] = []
             for name, definition in self.config.manpages.items():
                 fileid = FileId(definition.file)
                 manpage_page = self.pages.get(fileid)
@@ -1330,8 +1363,24 @@ def create_page(filename: str) -> Tuple[Page, EmbeddedRstParser]:
                 for filename, rendered in man.render(
                     manpage_page, name, definition.title, definition.section
                 ).items():
+                    manpages.append((filename.as_posix(), rendered))
                     static_files[filename.as_posix()] = rendered
 
+            if manpages and self.config.bundle.manpages:
+                try:
+                    static_files[self.config.bundle.manpages] = bundle(
+                        PurePath(self.config.bundle.manpages), manpages
+                    )
+                except ValueError:
+                    self.backend.on_diagnostics(
+                        FileId(self.config.config_path.relative_to(self.config.root)),
+                        [
+                            UnsupportedFormat(
+                                self.config.bundle.manpages, (".tar", ".tar.gz"), 0
+                            )
+                        ],
+                    )
+
             post_metadata["static_files"] = static_files
             for fileid, diagnostics in post_diagnostics.items():
                 self.backend.on_diagnostics(fileid, diagnostics)

snooty/types.py

@@ -104,6 +104,12 @@ class ManPageConfig:
     section: int
 
 
+@checked
+@dataclass
+class BundleConfig:
+    manpages: Optional[str] = field(default=None)
+
+
 @checked
 @dataclass
 class ProjectConfig:
@@ -122,6 +128,7 @@ class ProjectConfig:
     toc_landing_pages: List[str] = field(default_factory=list)
     page_groups: Dict[str, List[str]] = field(default_factory=dict)
     manpages: Dict[str, ManPageConfig] = field(default_factory=dict)
+    bundle: BundleConfig = field(default_factory=lambda: BundleConfig())
 
     @property
     def source_path(self) -> Path: