DOP-5816: Avoid parsing txt files in reserved directories (#669)

Author: rayangler

snooty/postprocess.py

@@ -73,7 +73,7 @@
 from .page import Page
 from .target_database import TargetDatabase
 from .types import Facet, ProjectConfig
-from .util import EXT_FOR_PAGE, SOURCE_FILE_EXTENSIONS, bundle
+from .util import EXT_FOR_PAGE, SOURCE_FILE_EXTENSIONS, bundle, is_txt_in_reserved_dir
 
 logger = logging.getLogger(__name__)
 _T = TypeVar("_T")
@@ -191,6 +191,9 @@ def propagate_facets(pages: Dict[FileId, Page], context: Context) -> None:
                     continue
 
                 file_path = Path(os.path.join(base, file))
+                if is_txt_in_reserved_dir(file_path):
+                    continue
+
                 fileid = config.get_fileid(file_path)
 
                 if ext == ".ast":

snooty/test_postprocess.py

@@ -4794,3 +4794,104 @@ def test_dismissible_skills_card() -> None:
             "skill": "WOW Lightsaber Skill",
             "url": "https://learn.mongodb.com/courses/crud-operations-in-mongodb",
         }
+
+
+def test_reserved_dirs() -> None:
+    with make_test(
+        {
+            Path(
+                "source/index.txt"
+            ): """
+========
+Homepage
+========
+
+txt files inside of code-examples directories should not be parsed as reStructuredText.
+
+.. literalinclude:: /code-examples/test.txt
+
+.. literalinclude:: /includes/code-examples/test.txt
+
+rst files inside of code-examples directories are okay to parse.
+
+.. include:: /includes/code-examples/foo.rst
+
+""",
+            Path(
+                "source/code-examples.txt"
+            ): """
+:orphan:
+
+==================
+Code Examples Page
+==================
+
+This page exists to make sure that pages titled code-examples are okay to have.
+
+""",
+            Path(
+                "source/code-examples/test.txt"
+            ): """
+This is a code example and should not be captured as a page.
+""",
+            Path(
+                "source/includes/code-examples/test.txt"
+            ): """
+This is another code example, but nested in a subdirectory, and should not be captured as a page.
+""",
+            Path(
+                "source/includes/code-examples/foo.rst"
+            ): """
+This file makes sure that rst files nested in a code-examples subdirectory is okay.
+
+.. warning::
+
+   This is a test.
+
+""",
+        }
+    ) as result:
+        # txt files nested under code-examples directories should not be parsed as reStructuredText
+        assert len(result.pages) == 3
+        assert result.pages.get(FileId("code-examples/test.txt")) == None
+        assert result.pages.get(FileId("includes/code-examples/test.txt")) == None
+
+        # Allow rst files under code-examples to be parsed
+        assert result.pages[FileId("includes/code-examples/foo.rst")]
+
+        # Allow code-examples.txt files to be parsed
+        assert result.pages[FileId("code-examples.txt")]
+
+        assert len(result.diagnostics[FileId("index.txt")]) == 0
+        check_ast_testing_string(
+            result.pages[FileId("index.txt")].ast,
+            """
+<root fileid="index.txt">
+    <section>
+        <heading id="homepage"><text>Homepage</text></heading>
+        <paragraph><text>txt files inside of code-examples directories should not be parsed as reStructuredText.</text></paragraph>
+        <directive name="literalinclude">
+            <text>/code-examples/test.txt</text>
+            <code copyable="True">
+                This is a code example and should not be captured as a page.
+            </code>
+        </directive>
+        <directive name="literalinclude">
+            <text>/includes/code-examples/test.txt</text>
+            <code copyable="True">
+                This is another code example, but nested in a subdirectory, and should not be captured as a page.
+            </code>
+        </directive>
+        <paragraph><text>rst files inside of code-examples directories are okay to parse.</text></paragraph>
+        <directive name="include">
+            <text>/includes/code-examples/foo.rst</text>
+            <root fileid="includes/code-examples/foo.rst">
+                <paragraph><text>This file makes sure that rst files nested in a code-examples subdirectory is okay.</text></paragraph>
+                <directive name="warning">
+                    <paragraph><text>This is a test.</text></paragraph>
+                </directive>
+            </root>
+        </directive>
+    </section>
+</root>""",
+        )

snooty/util.py

@@ -62,6 +62,7 @@
 PAT_URI = re.compile(r"^(?P<schema>[a-z]+)://")
 SOURCE_FILE_EXTENSIONS = {".txt", ".rst", ".yaml"}
 RST_EXTENSIONS = {".txt", ".rst"}
+RESERVED_DIRS = {"code-examples"}
 EXT_FOR_PAGE = ".txt"
 EMPTY_BLAKE2B = hashlib.blake2b(b"").hexdigest()
 SNOOTY_TOML = "snooty.toml"
@@ -175,7 +176,9 @@ def get_files(
 
             path = Path(os.path.join(base, name))
             # Detect and ignore symlinks outside of our jail
-            if is_relative_to(path.resolve(), must_be_relative_to):
+            if is_relative_to(
+                path.resolve(), must_be_relative_to
+            ) and not is_txt_in_reserved_dir(path):
                 yield path
 
 
@@ -781,3 +784,16 @@ def parse_toml_and_add_line_info(text: str) -> Dict[str, Any]:
             raise TOMLDecodeErrorWithSourceInfo(message, text.count("\n") + 1) from err
 
         raise err
+
+
+def is_txt_in_reserved_dir(path: Path) -> bool:
+    if path.suffix != ".txt":
+        return False
+
+    # Exclude checking files that have a reserved dir name AS the filename
+    path_parts = path.parts[:-1]
+    for part in path_parts:
+        if part in RESERVED_DIRS:
+            return True
+
+    return False