bubble-up parsing problems via InheritedContext (#3218)

## Changes
Refactoring

### Linked issues
None

### Functionality
None

### Tests
- [X] ran unit tests

---------

Co-authored-by: Eric Vergnaud <[email protected]>

Author: ericvergnaud

src/databricks/labs/ucx/source_code/graph.py

@@ -527,11 +527,16 @@ def failed(self) -> bool:
         return len(self.problems) > 0
 
 
+@dataclass
 class InheritedContext:
 
+    tree: Tree | None
+    found: bool
+    problems: Iterable[DependencyProblem]
+
     @classmethod
     def from_route(cls, graph: DependencyGraph, path_lookup: PathLookup, route: list[Dependency]) -> InheritedContext:
-        context = InheritedContext(None, False)
+        context = InheritedContext(None, False, [])
         for i, dependency in enumerate(route):
             if i >= len(route) - 1:
                 break
@@ -545,41 +550,29 @@ def from_route(cls, graph: DependencyGraph, path_lookup: PathLookup, route: list
             context = context.append(local, i == len(route) - 2)
         return context.finalize()
 
-    def __init__(self, tree: Tree | None, found: bool):
-        self._tree = tree
-        self._found = found
-
-    @property
-    def tree(self) -> Tree | None:
-        return self._tree
-
-    @property
-    def found(self) -> bool:
-        return self._found
-
     def append(self, context: InheritedContext, copy_found: bool) -> InheritedContext:
         # we should never append to a found context
         if self.found:
             raise ValueError("Appending to an already resolved InheritedContext is illegal!")
         tree = context.tree
         found = copy_found and context.found
         if tree is None:
-            return InheritedContext(self._tree, found)
-        if self._tree is None:
-            self._tree = Tree.new_module()
-        self._tree.append_tree(tree)
-        return InheritedContext(self._tree, found)
+            return InheritedContext(self.tree, found, context.problems)
+        new_tree = self.tree or Tree.new_module()
+        new_tree.append_tree(tree)
+        new_problems = itertools.chain(self.problems, context.problems)
+        return InheritedContext(new_tree, found, new_problems)
 
     def finalize(self) -> InheritedContext:
         # hacky stuff for aligning with Astroid's inference engine behavior
         # the engine checks line numbers to skip variables that are not in scope of the current frame
         # see https://github.com/pylint-dev/astroid/blob/5b665e7e760a7181625a24b3635e9fec7b174d87/astroid/filter_statements.py#L113
         # this is problematic when linting code fragments that refer to inherited code with unrelated line numbers
         # here we fool the engine by pretending that all nodes from context have negative line numbers
-        if self._tree is None:
+        if self.tree is None:
             return self
-        tree = self._tree.renumber(-1)
-        return InheritedContext(tree, self.found)
+        tree = self.tree.renumber(-1)
+        return InheritedContext(tree, self.found, [])
 
 
 T = TypeVar("T")

src/databricks/labs/ucx/source_code/linters/files.py

@@ -66,8 +66,13 @@ def build_inherited_context(self, graph: DependencyGraph, child_path: Path) -> I
         if self._language is CellLanguage.PYTHON:
             context = graph.new_dependency_graph_context()
             analyzer = PythonCodeAnalyzer(context, self._original_code)
-            return analyzer.build_inherited_context(child_path)
-        return InheritedContext(None, False)
+            inherited = analyzer.build_inherited_context(child_path)
+            problems = list(inherited.problems)
+            for idx, problem in enumerate(problems):
+                if problem.is_path_missing():
+                    problems[idx] = dataclasses.replace(problem, source_path=self._path)
+            return dataclasses.replace(inherited, problems=problems)
+        return InheritedContext(None, False, [])
 
     def __repr__(self):
         return f"<LocalFile {self._path}>"

src/databricks/labs/ucx/source_code/notebooks/cells.py

@@ -84,7 +84,7 @@ def __repr__(self):
         return f"{self.language.name}: {self._original_code[:20]}"
 
     def build_inherited_context(self, _graph: DependencyGraph, _child_path: Path) -> InheritedContext:
-        return InheritedContext(None, False)
+        return InheritedContext(None, False, [])
 
 
 class PythonCell(Cell):
@@ -437,21 +437,21 @@ def notebook_path(self) -> Path | None:
     def build_inherited_context(self, context: DependencyGraphContext, child_path: Path) -> InheritedContext:
         path = self.notebook_path
         if path is None:
-            logger.warning("Missing notebook path in %run command")
-            return InheritedContext(None, False)
+            problem = DependencyProblem('missing-notebook-path', "Missing notebook path in %run command")
+            return InheritedContext(None, False, [problem])
         maybe = context.resolver.resolve_notebook(context.path_lookup, path, inherit_context=True)
         if not maybe.dependency:
-            logger.warning(f"Could not load notebook {path}")
-            return InheritedContext(None, False)
+            problem = DependencyProblem('missing-notebook', f"Could not locate notebook {path}")
+            return InheritedContext(None, False, [problem])
         child_path = maybe.dependency.path
         absolute_path = context.path_lookup.resolve(path)
         absolute_child = context.path_lookup.resolve(child_path)
         if absolute_path == absolute_child:
-            return InheritedContext(None, True)
+            return InheritedContext(None, True, [])
         container = maybe.dependency.load(context.path_lookup)
         if not container:
-            logger.warning(f"Could not load notebook {path}")
-            return InheritedContext(None, False)
+            problem = DependencyProblem('corrupt-notebook', f"Could not load notebook {path}")
+            return InheritedContext(None, False, [problem])
         return container.build_inherited_context(context.parent, child_path)
 
 

src/databricks/labs/ucx/source_code/notebooks/magic.py

@@ -73,7 +73,7 @@ def build_inherited_context(self, context: DependencyGraphContext, child_path: P
         magic = self.as_magic()
         if magic is not None:
             return magic.build_inherited_context(context, child_path)
-        return InheritedContext(None, False)
+        return InheritedContext(None, False, [])
 
 
 class MagicNode(NodeNG):
@@ -90,7 +90,7 @@ def __init__(self, node: NodeNG, code: str):
     def build_dependency_graph(self, parent: DependencyGraph) -> list[DependencyProblem]: ...
 
     def build_inherited_context(self, _context: DependencyGraphContext, _child_path: Path) -> InheritedContext:
-        return InheritedContext(None, False)
+        return InheritedContext(None, False, [])
 
 
 _FACTORIES: list[Callable[[str, NodeNG], MagicCommand | None]] = []

src/databricks/labs/ucx/source_code/notebooks/sources.py

@@ -110,10 +110,11 @@ def build_dependency_graph(self, parent: DependencyGraph) -> list[DependencyProb
         return problems
 
     def build_inherited_context(self, graph: DependencyGraph, child_path: Path) -> InheritedContext:
-        context = InheritedContext(None, False)
+        problems: list[DependencyProblem] = []
+        context = InheritedContext(None, False, problems)
         for cell in self._cells:
-            child = cell.build_inherited_context(graph, child_path)
-            context = context.append(child, True)
+            child_context = cell.build_inherited_context(graph, child_path)
+            context = context.append(child_context, True)
             if context.found:
                 return context
         return context

src/databricks/labs/ucx/source_code/python/python_analyzer.py

@@ -62,13 +62,11 @@ def build_graph(self) -> list[DependencyProblem]:
     def build_inherited_context(self, child_path: Path) -> InheritedContext:
         tree, nodes, problems = self._parse_and_extract_nodes()
         if problems:
-            # TODO: bubble up problems via InheritedContext
-            logger.warning(f"Failed to parse: {problems}")
-            return InheritedContext(None, False)
+            return InheritedContext(None, False, problems)
         assert tree is not None, "no problems should yield a tree"
         if len(nodes) == 0:
-            return InheritedContext(tree, False)
-        context = InheritedContext(Tree.new_module(), False)
+            return InheritedContext(tree, False, [])
+        context = InheritedContext(Tree.new_module(), False, [])
         assert context.tree is not None, "Tree should be initialized"
         last_line = -1
         for base_node in nodes:
@@ -123,22 +121,31 @@ def _build_graph_from_node(self, base_node: NodeBase) -> Iterable[DependencyProb
         elif isinstance(base_node, MagicLine):
             yield from base_node.build_dependency_graph(self._context.parent)
         else:
-            logger.warning(f"Can't build graph for node {NodeBase.__name__} of type {type(base_node).__name__}")
+            problem = DependencyProblem.from_node(
+                "unsupported-node-type",
+                f"Can't build graph for node {NodeBase.__name__} of type {type(base_node).__name__}",
+                base_node.node,
+            )
+            yield problem
 
     def _build_inherited_context_from_node(self, base_node: NodeBase, child_path: Path) -> InheritedContext:
         if isinstance(base_node, SysPathChange):
             self._mutate_path_lookup(base_node)
-            return InheritedContext(None, False)
+            return InheritedContext(None, False, [])
         if isinstance(base_node, ImportSource):
             # nothing to do, Astroid takes care of imports
-            return InheritedContext(None, False)
+            return InheritedContext(None, False, [])
         if isinstance(base_node, NotebookRunCall):
             # nothing to do, dbutils.notebook.run uses a dedicated context
-            return InheritedContext(None, False)
+            return InheritedContext(None, False, [])
         if isinstance(base_node, MagicLine):
             return base_node.build_inherited_context(self._context, child_path)
-        logger.warning(f"Can't build inherited context for node {NodeBase.__name__} of type {type(base_node).__name__}")
-        return InheritedContext(None, False)
+        problem = DependencyProblem.from_node(
+            "unsupported-node-type",
+            f"Can't build inherited context for node {NodeBase.__name__} of type {type(base_node).__name__}",
+            base_node.node,
+        )
+        return InheritedContext(None, False, [problem])
 
     def _register_import(self, base_node: ImportSource) -> Iterable[DependencyProblem]:
         prefix = ""