Skip to content

Commit e430549

Browse files
i80andi80and
authored
DOP-1018: Preserve source fileid through includes (#230)
* DOP-1018: Preserve source fileid through includes * Incorporate sophie's feedback
1 parent 5897876 commit e430549

31 files changed

+498
-328
lines changed

‎snooty/eventparser.py‎

Lines changed: 47 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,37 @@
1-
from typing import Any, Callable, Dict, Set, Tuple, Iterable, Union
1+
from typing import Any, Callable, Dict, Set, Tuple, Iterable, Union, List, Optional
22
from .types import FileId
33
from .page import Page
44
from . import n
55

66

7+
class FileIdStack:
8+
"""A stack which tracks file inclusion history, allowing a postprocessor
9+
pass to know at any point both the page where processing started, as well
10+
as what file is currently being processed."""
11+
12+
__slots__ = ("_stack",)
13+
14+
def __init__(self, initial_stack: Optional[List[FileId]] = None) -> None:
15+
self._stack: List[FileId] = initial_stack if initial_stack is not None else []
16+
17+
def pop(self) -> None:
18+
self._stack.pop()
19+
20+
def append(self, fileid: FileId) -> None:
21+
self._stack.append(fileid)
22+
23+
def clear(self) -> None:
24+
self._stack.clear()
25+
26+
@property
27+
def root(self) -> FileId:
28+
return self._stack[0]
29+
30+
@property
31+
def current(self) -> FileId:
32+
return self._stack[-1]
33+
34+
735
class EventListeners:
836
"""Manage the listener functions associated with an event-based parse operation"""
937

@@ -30,16 +58,16 @@ def get_event_listeners(self, event: str) -> Set[Callable[..., Any]]:
3058
def fire(
3159
self,
3260
event: str,
33-
filename: FileId,
61+
fileid: FileIdStack,
3462
*args: Union[n.Node, Page],
3563
**kwargs: Union[n.Node, Page],
3664
) -> None:
3765
"""Iterate through all universal listeners and all listeners of the specified type and call them"""
3866
for listener in self.get_event_listeners(event):
39-
listener(filename, *args, **kwargs)
67+
listener(fileid, *args, **kwargs)
4068

4169
for listener in self._universal_listeners:
42-
listener(filename, *args, **kwargs)
70+
listener(fileid, *args, **kwargs)
4371

4472

4573
class EventParser(EventListeners):
@@ -52,6 +80,7 @@ class EventParser(EventListeners):
5280

5381
def __init__(self) -> None:
5482
super(EventParser, self).__init__()
83+
self.fileid_stack = FileIdStack()
5584

5685
def consume(self, d: Iterable[Tuple[FileId, Page]]) -> None:
5786
"""Initializes a parse on the provided key-value map of pages"""
@@ -60,8 +89,14 @@ def consume(self, d: Iterable[Tuple[FileId, Page]]) -> None:
6089
self._iterate(page.ast, filename)
6190
self._on_page_exit_event(page, filename)
6291

92+
self.fileid_stack.clear()
93+
6394
def _iterate(self, d: n.Node, filename: FileId) -> None:
95+
if isinstance(d, n.Root):
96+
self.fileid_stack.append(d.fileid)
97+
6498
self._on_object_enter_event(d, filename)
99+
65100
if isinstance(d, n.Parent):
66101
if isinstance(d, n.DefinitionListItem):
67102
for child in d.term:
@@ -73,20 +108,24 @@ def _iterate(self, d: n.Node, filename: FileId) -> None:
73108

74109
for child in d.children:
75110
self._iterate(child, filename)
111+
76112
self._on_object_exit_event(d, filename)
77113

114+
if isinstance(d, n.Root):
115+
self.fileid_stack.pop()
116+
78117
def _on_page_enter_event(self, page: Page, filename: FileId) -> None:
79118
"""Called when an array is first encountered in tree"""
80-
self.fire(self.PAGE_START_EVENT, filename, page=page)
119+
self.fire(self.PAGE_START_EVENT, FileIdStack([filename]), page=page)
81120

82121
def _on_page_exit_event(self, page: Page, filename: FileId) -> None:
83122
"""Called when an array is first encountered in tree"""
84-
self.fire(self.PAGE_END_EVENT, filename, page=page)
123+
self.fire(self.PAGE_END_EVENT, FileIdStack([filename]), page=page)
85124

86125
def _on_object_enter_event(self, node: n.Node, filename: FileId) -> None:
87126
"""Called when an object is first encountered in tree"""
88-
self.fire(self.OBJECT_START_EVENT, filename, node=node)
127+
self.fire(self.OBJECT_START_EVENT, self.fileid_stack, node=node)
89128

90129
def _on_object_exit_event(self, node: n.Node, filename: FileId) -> None:
91130
"""Called when an object is first encountered in tree"""
92-
self.fire(self.OBJECT_END_EVENT, filename, node=node)
131+
self.fire(self.OBJECT_END_EVENT, self.fileid_stack, node=node)

‎snooty/gizaparser/extracts.py‎

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -28,15 +28,6 @@ def render(self, page: Page, rst_parser: EmbeddedRstParser) -> List[n.Node]:
2828
return children
2929

3030

31-
def extract_to_page(
32-
page: Page, extract: Extract, rst_parser: EmbeddedRstParser
33-
) -> n.Directive:
34-
rendered = extract.render(page, rst_parser)
35-
directive = n.Directive((extract.line,), [], "", "extract", [], {})
36-
directive.children = rendered
37-
return directive
38-
39-
4031
class GizaExtractsCategory(GizaCategory[Extract]):
4132
def parse(
4233
self, path: Path, text: Optional[str] = None
@@ -62,14 +53,20 @@ def to_pages(
6253
extracts: Sequence[Extract],
6354
) -> List[Page]:
6455
pages: List[Page] = []
56+
source_fileid = self.project_config.get_fileid(source_path)
57+
6558
for extract in extracts:
6659
assert extract.ref is not None
6760
if extract.ref.startswith("_"):
6861
continue
6962

7063
page, rst_parser = page_factory(f"{extract.ref}.rst")
7164
page.category = "extracts"
72-
page.ast = extract_to_page(page, extract, rst_parser)
65+
rendered = extract.render(page, rst_parser)
66+
extract_directive = n.Directive((extract.line,), [], "", "extract", [], {})
67+
extract_directive.children = rendered
68+
page.ast = n.Root((0,), [], source_fileid, {})
69+
page.ast.children.append(extract_directive)
7370
pages.append(page)
7471

7572
return pages

‎snooty/gizaparser/release.py‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -39,15 +39,6 @@ def render(self, page: Page, rst_parser: EmbeddedRstParser) -> List[n.Node]:
3939
return children
4040

4141

42-
def release_specification_to_page(
43-
page: Page, node: ReleaseSpecification, rst_parser: EmbeddedRstParser
44-
) -> n.Directive:
45-
rendered = node.render(page, rst_parser)
46-
directive = n.Directive((node.line,), [], "", "release_specification", [], {})
47-
directive.children = rendered
48-
return directive
49-
50-
5142
class GizaReleaseSpecificationCategory(GizaCategory[ReleaseSpecification]):
5243
def parse(
5344
self, path: Path, text: Optional[str] = None
@@ -73,14 +64,25 @@ def to_pages(
7364
nodes: Sequence[ReleaseSpecification],
7465
) -> List[Page]:
7566
pages: List[Page] = []
67+
source_fileid = self.project_config.get_fileid(source_path)
68+
7669
for node in nodes:
7770
assert node.ref is not None
7871
if node.ref.startswith("_"):
7972
continue
8073

8174
page, rst_parser = page_factory(f"{node.ref}.rst")
8275
page.category = "release"
83-
page.ast = release_specification_to_page(page, node, rst_parser)
76+
77+
rendered = node.render(page, rst_parser)
78+
release_directive = n.Directive(
79+
(node.line,), [], "", "release_specification", [], {}
80+
)
81+
release_directive.children = rendered
82+
83+
page.ast = n.Root((0,), [], source_fileid, {})
84+
page.ast.children.append(release_directive)
85+
8486
pages.append(page)
8587

8688
return pages

‎snooty/gizaparser/steps.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,11 @@ def to_pages(
127127
output_filename = output_filename[len("steps-") :]
128128
page, rst_parser = page_factory(output_filename)
129129
page.category = "steps"
130-
page.ast = n.Directive((0,), [], "", "steps", [], {})
131-
page.ast.children = [step_to_page(page, step, rst_parser) for step in steps]
130+
source_fileid = self.project_config.get_fileid(source_path)
131+
steps_directive = n.Directive((0,), [], "", "steps", [], {})
132+
steps_directive.children = [
133+
step_to_page(page, step, rst_parser) for step in steps
134+
]
135+
page.ast = n.Root((0,), [], source_fileid, {})
136+
page.ast.children.append(steps_directive)
132137
return [page]

‎snooty/gizaparser/test_extracts.py‎

Lines changed: 28 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,15 @@
55
from ..page import Page
66
from ..diagnostics import Diagnostic, FailedToInheritRef
77
from ..parser import EmbeddedRstParser
8-
from ..util_test import make_test, ast_to_testing_string
8+
from ..util_test import make_test, check_ast_testing_string
99

1010

1111
def test_extract() -> None:
12-
project_config = ProjectConfig(Path("test_data"), "")
12+
root_path = Path("test_data/test_gizaparser")
13+
project_config = ProjectConfig(root_path, "")
1314
category = GizaExtractsCategory(project_config)
14-
path = Path("test_data/extracts-test.yaml")
15-
parent_path = Path("test_data/extracts-test-parent.yaml")
15+
path = root_path.joinpath(Path("source/includes/extracts-test.yaml"))
16+
parent_path = root_path.joinpath(Path("source/includes/extracts-test-parent.yaml"))
1617

1718
def add_main_file() -> List[Diagnostic]:
1819
extracts, text, parse_diagnostics = category.parse(path)
@@ -43,24 +44,33 @@ def create_page(filename: Optional[str]) -> Tuple[Page, EmbeddedRstParser]:
4344

4445
pages = category.to_pages(path, create_page, giza_node.data)
4546
assert [str(page.fake_full_path()) for page in pages] == [
46-
"test_data/extracts/installation-directory-rhel.rst",
47-
"test_data/extracts/broken-inherit.rst",
48-
"test_data/extracts/another-file.rst",
49-
"test_data/extracts/missing-substitution.rst",
47+
"test_data/test_gizaparser/source/includes/extracts/installation-directory-rhel.rst",
48+
"test_data/test_gizaparser/source/includes/extracts/broken-inherit.rst",
49+
"test_data/test_gizaparser/source/includes/extracts/another-file.rst",
50+
"test_data/test_gizaparser/source/includes/extracts/missing-substitution.rst",
5051
]
5152

52-
assert ast_to_testing_string(pages[0].ast) == "".join(
53-
(
54-
'<directive name="extract"><paragraph><text>By default, MongoDB stores its data files in ',
55-
"</text><literal><text>/var/lib/mongo</text></literal><text> and its\nlog files in </text>",
56-
"<literal><text>/var/log/mongodb</text></literal><text>.</text></paragraph></directive>",
57-
)
53+
check_ast_testing_string(
54+
pages[0].ast,
55+
"""<root fileid="includes/extracts-test.yaml">
56+
<directive name="extract">
57+
<paragraph>
58+
<text>By default, MongoDB stores its data files in</text>
59+
<literal><text>/var/lib/mongo</text></literal>
60+
<text> and its\nlog files in </text>
61+
<literal><text>/var/log/mongodb</text></literal><text>.</text>
62+
</paragraph>
63+
</directive>
64+
</root>""",
5865
)
5966

60-
assert ast_to_testing_string(pages[3].ast) == "".join(
61-
(
62-
'<directive name="extract"><paragraph><text>Substitute</text></paragraph></directive>'
63-
)
67+
check_ast_testing_string(
68+
pages[3].ast,
69+
"""
70+
<root fileid="includes/extracts-test.yaml">
71+
<directive name="extract"><paragraph><text>Substitute</text></paragraph></directive>
72+
</root>
73+
""",
6474
)
6575

6676
# XXX: We need to track source file information for each property.

‎snooty/gizaparser/test_nodes.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,10 @@ def test_reify_all_files() -> None:
9393
project_config.constants["version"] = "3.4"
9494

9595
# Place good path and bad path here
96-
paths = ("test_data/release-base.yaml", "test_data/release-base-repeat.yaml")
96+
paths = (
97+
"test_data/test_gizaparser/source/includes/release-base.yaml",
98+
"test_data/test_gizaparser/source/includes/release-base-repeat.yaml",
99+
)
97100

98101
for i, path in enumerate(paths):
99102
test_path = Path(path)

‎snooty/gizaparser/test_release.py‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,12 @@
99

1010

1111
def test_release_specification() -> None:
12-
project_config = ProjectConfig(Path("test_data"), "")
12+
root_path = Path("test_data/test_gizaparser")
13+
project_config = ProjectConfig(root_path, "")
1314
project_config.constants["version"] = "3.4"
1415
category = GizaReleaseSpecificationCategory(project_config)
15-
path = Path("test_data/release-specifications.yaml")
16-
parent_path = Path("test_data/release-base.yaml")
16+
path = root_path.joinpath(Path("source/includes/release-specifications.yaml"))
17+
parent_path = root_path.joinpath(Path("source/includes/release-base.yaml"))
1718

1819
def add_main_file() -> List[Diagnostic]:
1920
extracts, text, parse_diagnostics = category.parse(path)
@@ -42,25 +43,25 @@ def create_page(filename: Optional[str]) -> Tuple[Page, EmbeddedRstParser]:
4243

4344
pages = category.to_pages(path, create_page, giza_node.data)
4445
assert [str(page.fake_full_path()) for page in pages] == [
45-
"test_data/release/untar-release-osx-x86_64.rst",
46-
"test_data/release/install-ent-windows-default.rst",
46+
"test_data/test_gizaparser/source/includes/release/untar-release-osx-x86_64.rst",
47+
"test_data/test_gizaparser/source/includes/release/install-ent-windows-default.rst",
4748
]
4849

4950
assert all((not diagnostics for diagnostics in all_diagnostics.values()))
5051

5152
check_ast_testing_string(
5253
pages[0].ast,
53-
"""<directive name="release_specification">
54+
"""<root fileid="includes/release-specifications.yaml"><directive name="release_specification">
5455
<code lang="sh" copyable="True">
5556
tar -zxvf mongodb-macos-x86_64-3.4.tgz\n</code>
56-
</directive>""",
57+
</directive></root>""",
5758
)
5859

5960
check_ast_testing_string(
6061
pages[1].ast,
61-
"""<directive name="release_specification">
62+
"""<root fileid="includes/release-specifications.yaml"><directive name="release_specification">
6263
<code lang="bat" copyable="True">
6364
msiexec.exe /l*v mdbinstall.log /qb /i mongodb-win32-x86_64-enterprise-windows-64-3.4-signed.msi\n
6465
</code>
65-
</directive>""",
66+
</directive></root>""",
6667
)

‎snooty/gizaparser/test_steps.py‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,16 @@
99

1010

1111
def test_step() -> None:
12-
project_config, project_diagnostics = ProjectConfig.open(Path("test_data"))
12+
root_path = Path("test_data/test_gizaparser")
13+
project_config, project_diagnostics = ProjectConfig.open(root_path)
1314
assert project_diagnostics == []
1415

1516
category = GizaStepsCategory(project_config)
16-
path = Path("test_data/steps-test.yaml")
17-
child_path = Path("test_data/steps-test-child.yaml")
18-
grandchild_path = Path("test_data/steps-test-grandchild.yaml")
17+
path = root_path.joinpath(Path("source/includes/steps-test.yaml"))
18+
child_path = root_path.joinpath(Path("source/includes/steps-test-child.yaml"))
19+
grandchild_path = root_path.joinpath(
20+
Path("source/includes/steps-test-grandchild.yaml")
21+
)
1922

2023
all_diagnostics: Dict[PurePath, List[Diagnostic]] = {}
2124
for current_path in [path, child_path, grandchild_path]:
@@ -33,7 +36,7 @@ def create_page(filename: Optional[str]) -> Tuple[Page, EmbeddedRstParser]:
3336

3437
pages = category.to_pages(path, create_page, giza_node.data)
3538
assert [str(page.fake_full_path()) for page in pages] == [
36-
"test_data/steps/test.rst"
39+
"test_data/test_gizaparser/source/includes/steps/test.rst"
3740
]
3841
# Ensure that no diagnostics were raised
3942
all_diagnostics = {k: v for k, v in all_diagnostics.items() if v}
@@ -42,6 +45,7 @@ def create_page(filename: Optional[str]) -> Tuple[Page, EmbeddedRstParser]:
4245
check_ast_testing_string(
4346
pages[0].ast,
4447
"""
48+
<root fileid="includes/steps-test.yaml">
4549
<directive name="steps">
4650
<directive name="step">
4751
<section>
@@ -94,5 +98,5 @@ def create_page(filename: Optional[str]) -> Tuple[Page, EmbeddedRstParser]:
9498
<code lang="sh" copyable="True">
9599
echo "mongodb-org hold" | sudo dpkg --set-selections
96100
</code><paragraph><text>bye</text></paragraph></section></directive>
97-
</directive>""",
101+
</directive></root>""",
98102
)

0 commit comments

Comments
 (0)