parser: Add source_constants

Author: i80and

parser/snooty/flutter.py

@@ -225,7 +225,7 @@ def check_type(ty: Type[C], data: object, ty_module: str = '') -> C:
 
         raise LoadError('Unsupported PEP-484 type', ty, data)
 
-    if ty is object or ty is Any:
+    if ty is object or ty is Any or isinstance(data, ty):
         return cast(C, data)
 
     raise LoadError('Unloadable type', ty, data)

parser/snooty/gizaparser/extracts.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from pathlib import PurePath
+from pathlib import Path
 from typing import Callable, List, Tuple, Sequence, Optional
 from ..flutter import checked
 from .nodes import Inheritable, GizaCategory, HeadingMixin
@@ -37,12 +37,11 @@ def extract_to_page(page: Page,
     }
 
 
-@dataclass
 class GizaExtractsCategory(GizaCategory[Extract]):
     def parse(self,
-              path: PurePath,
+              path: Path,
               text: Optional[str] = None) -> Tuple[Sequence[Extract], str, List[Diagnostic]]:
-        extracts, text, diagnostics = parse(Extract, path, text)
+        extracts, text, diagnostics = parse(Extract, path, self.project_config, text)
 
         def report_missing_ref(extract: Extract) -> bool:
             diagnostics.append(

parser/snooty/gizaparser/nodes.py

@@ -3,11 +3,11 @@
 import logging
 import re
 from dataclasses import dataclass, field
-from pathlib import PurePath
+from pathlib import Path, PurePath
 from typing import cast, Any, Callable, Dict, Set, Generic, Optional, \
                    TypeVar, Tuple, Iterator, Sequence, List, Union
 from ..flutter import checked
-from ..types import Diagnostic, Page, EmbeddedRstParser, SerializableType
+from ..types import Diagnostic, Page, EmbeddedRstParser, SerializableType, ProjectConfig
 
 _T = TypeVar('_T', str, object)
 PAT_SUBSTITUTION = re.compile(r'\{\{([\w-]+)\}\}')
@@ -116,7 +116,7 @@ class GizaFile(Generic[_I]):
     """A GizaFile represents a single Giza YAML file."""
     __slots__ = ('path', 'text', 'data')
 
-    path: PurePath
+    path: Path
     text: str
     data: Sequence[_I]
 
@@ -126,11 +126,12 @@ class GizaCategory(Generic[_I]):
     """A GizaCategory stores metadata about a "category" of Giza YAML files. For
        example, "steps", or "apiargs". Each GizaCategory contains all types necessary
        to transform a given path into Pages."""
+    project_config: ProjectConfig
     nodes: Dict[str, GizaFile[_I]] = field(default_factory=dict)
     dg: DependencyGraph = field(default_factory=DependencyGraph)
 
     def parse(self,
-              path: PurePath,
+              path: Path,
               text: Optional[str] = None) -> Tuple[Sequence[_I], str, List[Diagnostic]]:
         pass
 
@@ -139,7 +140,7 @@ def to_pages(self,
                  data: Sequence[_I]) -> List[Page]:
         pass
 
-    def add(self, path: PurePath, text: str, elements: Sequence[_I]) -> None:
+    def add(self, path: Path, text: str, elements: Sequence[_I]) -> None:
         file_id = path.name
         self.nodes[file_id] = GizaFile(path, text, elements)
         dependencies = set()

parser/snooty/gizaparser/parse.py

@@ -2,11 +2,11 @@
 import yaml
 import yaml.resolver
 import yaml.scanner
-from pathlib import PurePath
+from pathlib import Path
 from typing import List, Optional, Tuple, Type, TypeVar
 from yaml.composer import Composer
 from ..flutter import check_type, LoadError, mapping_dict
-from ..types import Diagnostic, SerializableType
+from ..types import Diagnostic, SerializableType, ProjectConfig
 
 _T = TypeVar('_T')
 logger = logging.getLogger(__name__)
@@ -48,23 +48,24 @@ def dict_constructor(loader: yaml.Loader, node: yaml.nodes.Node) -> mapping_dict
 
 
 def parse(ty: Type[_T],
-          path: PurePath,
+          path: Path,
+          project_config: ProjectConfig,
           text: Optional[str] = None) -> Tuple[List[_T], str, List[Diagnostic]]:
+    diagnostics: List[Diagnostic] = []
     if text is None:
-        with open(path, 'r') as f:
-            text = f.read()
+        text, diagnostics = project_config.read(path)
 
     try:
         parsed_yaml = load_yaml(text)
     except yaml.scanner.ScannerError as err:
         lineno = err.problem_mark.line
         col = err.problem_mark.column
-        return [], text, [Diagnostic.error(err.problem, (lineno, col))]
+        return [], text, diagnostics + [Diagnostic.error(err.problem, (lineno, col))]
 
     try:
         parsed = [check_type(ty, data) for data in parsed_yaml]
-        return parsed, text, []
+        return parsed, text, diagnostics
     except LoadError as err:
         mapping = err.bad_data if isinstance(err.bad_data, dict) else {}
         lineno = mapping.get('_start_line', 0)
-        return [], text, [Diagnostic.error(str(err), lineno)]
+        return [], text, diagnostics + [Diagnostic.error(str(err), lineno)]

parser/snooty/gizaparser/steps.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from pathlib import PurePath
+from pathlib import Path
 from typing import Callable, Optional, List, Sequence, Union, Tuple
 from ..flutter import checked
 from .parse import parse
@@ -110,12 +110,11 @@ def step_to_page(page: Page, step: Step, rst_parser: EmbeddedRstParser) -> Seria
     }
 
 
-@dataclass
 class GizaStepsCategory(GizaCategory[Step]):
     def parse(self,
-              path: PurePath,
+              path: Path,
               text: Optional[str] = None) -> Tuple[Sequence[Step], str, List[Diagnostic]]:
-        return parse(Step, path, text)
+        return parse(Step, path, self.project_config, text)
 
     def to_pages(self,
                  page_factory: Callable[[], Tuple[Page, EmbeddedRstParser]],

parser/snooty/gizaparser/test_extracts.py

@@ -2,12 +2,13 @@
 from typing import Dict, Tuple, List
 from .extracts import GizaExtractsCategory
 from .nodes import ast_to_testing_string
-from ..types import Diagnostic, Page, EmbeddedRstParser
+from ..types import Diagnostic, Page, EmbeddedRstParser, ProjectConfig
 from ..parser import make_embedded_rst_parser
 
 
 def test_extract() -> None:
-    category = GizaExtractsCategory()
+    project_config = ProjectConfig(Path('test_data'), '')
+    category = GizaExtractsCategory(project_config)
     path = Path('test_data/extracts-test.yaml')
     child_path = Path('test_data/extracts-test-child.yaml')
 
@@ -35,7 +36,7 @@ def add_child_file() -> List[Diagnostic]:
 
     def create_page() -> Tuple[Page, EmbeddedRstParser]:
         page = Page(path, '', {})
-        return page, make_embedded_rst_parser(path, page, all_diagnostics[path])
+        return page, make_embedded_rst_parser(project_config, page, all_diagnostics[path])
 
     pages = category.to_pages(create_page, giza_node.data)
     assert len(pages) == 4

parser/snooty/language_server.py

@@ -209,8 +209,8 @@ def set_diagnostics(self, page_path: PurePath, diagnostics: List[types.Diagnosti
             'diagnostics': workspace_item.create_lsp_diagnostics()
         })
 
-    def uri_to_path(self, uri: Uri) -> PurePath:
-        path = PurePath(uri.replace('file://', '', 1).replace(self.root_uri, ''))
+    def uri_to_path(self, uri: Uri) -> Path:
+        path = Path(uri.replace('file://', '', 1).replace(self.root_uri, ''))
         self.path_to_uri[path] = uri
         return path
 

parser/snooty/main.py

@@ -8,7 +8,7 @@
 from pathlib import Path, PurePath
 from typing import List
 
-from . import language_server
+# from . import language_server
 from .parser import Project, RST_EXTENSIONS
 from .types import Page, Diagnostic
 
@@ -106,9 +106,9 @@ def usage(exit_code: int) -> None:
 def main() -> None:
     logging.basicConfig(level=logging.INFO)
 
-    if len(sys.argv) == 2 and sys.argv[1] == 'language-server':
-        language_server.start()
-        return
+    # if len(sys.argv) == 2 and sys.argv[1] == 'language-server':
+    #     language_server.start()
+    #     return
 
     if len(sys.argv) not in (3, 4) or sys.argv[1] not in ('watch', 'build'):
         usage(1)

parser/snooty/parser.py

@@ -5,18 +5,16 @@
 import os
 import pwd
 import subprocess
-from dataclasses import dataclass
 from functools import partial
 from pathlib import Path, PurePath
-from typing import Any, Dict, Tuple, Optional, Set, List
+from typing import Any, Dict, Tuple, Optional, Set, List, Iterable
 from typing_extensions import Protocol
 import docutils.utils
-import toml
 
 from . import gizaparser, rstparser, util
-from .flutter import check_type, checked
 from .gizaparser.nodes import GizaCategory
-from .types import Diagnostic, SerializableType, EmbeddedRstParser, Page, StaticAsset
+from .types import Diagnostic, SerializableType, EmbeddedRstParser, Page, \
+    StaticAsset, ProjectConfigError, ProjectConfig
 
 RST_EXTENSIONS = {'.rst', '.txt'}
 logger = logging.getLogger(__name__)
@@ -171,6 +169,9 @@ def add_static_asset(self, path: PurePath) -> StaticAsset:
         self.static_assets.add(static_asset)
         return static_asset
 
+    def add_diagnostics(self, diagnostics: Iterable[Diagnostic]) -> None:
+        self.diagnostics.extend(diagnostics)
+
     def __make_child_visitor(self) -> 'JSONVisitor':
         visitor = type(self)(self.project_root, self.docpath, self.document)
         visitor.diagnostics = self.diagnostics
@@ -193,12 +194,9 @@ def dispatch_departure(self, node: docutils.nodes.Node) -> None:
 
 
 def parse_rst(parser: rstparser.Parser[JSONVisitor],
-              path: PurePath,
+              path: Path,
               text: Optional[str] = None) -> Tuple[Page, List[Diagnostic]]:
-    if text is None:
-        with open(path, 'r') as f:
-            text = f.read()
-    visitor = parser.parse(path, text)
+    visitor, text = parser.parse(path, text)
 
     return Page(
         path,
@@ -207,7 +205,7 @@ def parse_rst(parser: rstparser.Parser[JSONVisitor],
         visitor.static_assets), visitor.diagnostics
 
 
-def make_embedded_rst_parser(project_root: Path,
+def make_embedded_rst_parser(project_config: ProjectConfig,
                              page: Page,
                              diagnostics: List[Diagnostic]) -> EmbeddedRstParser:
     def parse_embedded_rst(rst: str,
@@ -216,8 +214,8 @@ def parse_embedded_rst(rst: str,
         # Crudely make docutils line numbers match
         text = '\n' * lineno + rst.strip()
         visitor_class = InlineJSONVisitor if inline else JSONVisitor
-        parser = rstparser.Parser(project_root, visitor_class)
-        visitor = parser.parse(page.source_path, text)
+        parser = rstparser.Parser(project_config, visitor_class)
+        visitor, _ = parser.parse(page.source_path, text)
         children: List[SerializableType] = visitor.state[-1]['children']
 
         diagnostics.extend(visitor.diagnostics)
@@ -242,55 +240,38 @@ def on_update(self, prefix: List[str], page_id: str, page: Page) -> None: ...
     def on_delete(self, page_id: str) -> None: ...
 
 
-@checked
-@dataclass
-class ProjectConfig:
-    name: str
-
-    @classmethod
-    def open(cls, root: Path) -> Tuple[Path, 'ProjectConfig']:
-        path = root
-        while path.parent != path:
-            try:
-                with open(path.joinpath('snooty.toml'), 'r') as f:
-                    return path, check_type(ProjectConfig, toml.load(f))
-            except FileNotFoundError:
-                pass
-            path = path.parent
-
-        return root, cls('untitled')
-
-
 class Project:
     def __init__(self,
                  root: Path,
                  backend: ProjectBackend) -> None:
         root = root.resolve(strict=True)
-        root, config = ProjectConfig.open(root)
+        root, self.config, config_diagnostics = ProjectConfig.open(root)
+
+        if config_diagnostics:
+            backend.on_diagnostics(root, config_diagnostics)
+            raise ProjectConfigError()
 
         self.root = root
-        self.parser = rstparser.Parser(self.root, JSONVisitor)
+        self.parser = rstparser.Parser(self.config, JSONVisitor)
         self.static_assets: Dict[PurePath, Set[StaticAsset]] = collections.defaultdict(set)
         self.backend = backend
 
-        self.steps_category = gizaparser.steps.GizaStepsCategory()
-        self.extracts_category = gizaparser.extracts.GizaExtractsCategory()
         self.yaml_mapping: Dict[str, GizaCategory[Any]] = {
-            'steps': self.steps_category,
-            'extracts': self.extracts_category
+            'steps': gizaparser.steps.GizaStepsCategory(self.config),
+            'extracts': gizaparser.extracts.GizaExtractsCategory(self.config)
         }
 
         username = pwd.getpwuid(os.getuid()).pw_name
         branch = subprocess.check_output(
             ['git', 'rev-parse', '--abbrev-ref', 'HEAD'],
             encoding='utf-8').strip()
-        self.prefix = [config.name, username, branch]
+        self.prefix = [self.config.name, username, branch]
 
     def get_page_id(self, path: PurePath) -> str:
         page_id = path.with_suffix('').relative_to(self.root).as_posix()
         return '/'.join(self.prefix + [page_id])
 
-    def update(self, path: PurePath, optional_text: Optional[str] = None) -> None:
+    def update(self, path: Path, optional_text: Optional[str] = None) -> None:
         diagnostics: Dict[PurePath, List[Diagnostic]] = {path: []}
         prefix = get_giza_category(path)
         _, ext = os.path.splitext(path)
@@ -302,9 +283,8 @@ def update(self, path: PurePath, optional_text: Optional[str] = None) -> None:
         elif ext == '.yaml' and prefix in self.yaml_mapping:
             file_id = os.path.basename(path)
             giza_category = self.yaml_mapping[prefix]
-            needs_rebuild = self.steps_category.dg.dependents[file_id].union(
-                self.extracts_category.dg.dependents[file_id]).union(
-                set([file_id]))
+            needs_rebuild = set((file_id)).union(*(
+                category.dg.dependents[file_id] for category in self.yaml_mapping.values()))
             logger.debug('needs_rebuild: %s', ','.join(needs_rebuild))
             for file_id in needs_rebuild:
                 file_diagnostics: List[Diagnostic] = []
@@ -319,7 +299,7 @@ def update(self, path: PurePath, optional_text: Optional[str] = None) -> None:
 
                 def create_page() -> Tuple[Page, EmbeddedRstParser]:
                     page = Page(giza_node.path, text, {})
-                    return page, make_embedded_rst_parser(self.root, page, file_diagnostics)
+                    return page, make_embedded_rst_parser(self.config, page, file_diagnostics)
 
                 giza_category.add(path, text, steps)
                 pages = giza_category.to_pages(create_page, giza_node.data)
@@ -352,7 +332,7 @@ def build(self) -> None:
 
         # Categorize our YAML files
         logger.debug('Categorizing YAML files')
-        categorized: Dict[str, List[PurePath]] = collections.defaultdict(list)
+        categorized: Dict[str, List[Path]] = collections.defaultdict(list)
         for path in util.get_files(self.root, ('.yaml',)):
             prefix = get_giza_category(path)
             if prefix in self.yaml_mapping:
@@ -373,7 +353,7 @@ def build(self) -> None:
                 def create_page() -> Tuple[Page, EmbeddedRstParser]:
                     page = Page(giza_node.path, giza_node.text, {})
                     return page, make_embedded_rst_parser(
-                        self.root, page, all_yaml_diagnostics.setdefault(giza_node.path, []))
+                        self.config, page, all_yaml_diagnostics.setdefault(giza_node.path, []))
 
                 for page in giza_category.to_pages(create_page, giza_node.data):
                     self.backend.on_update(self.prefix, self.get_page_id(page.get_id()), page)