Added lint local code command (#1710)

Author: ericvergnaud

README.md

@@ -68,6 +68,7 @@ See [contributing instructions](CONTRIBUTING.md) to help improve this project.
   * [`move` command](#move-command)
   * [`alias` command](#alias-command)
 * [Code migration commands](#code-migration-commands)
+  * [`lint-local-code` command](#lint-local-code-command)
   * [`migrate-local-code` command](#migrate-local-code-command)
   * [`migrate-dbsql-dashboards` command](#migrate-dbsql-dashboards-command)
   * [`revert-dbsql-dashboards` command](#revert-dbsql-dashboards-command)
@@ -973,6 +974,25 @@ clusters to be UC compatible.
 
 [[back to top](#databricks-labs-ucx)]
 
+## `lint-local-code` command
+
+```text
+databricks labs ucx lint-local-code
+```
+
+At any time, you can run this command to assess all migrations required in a local directory or a file. It only takes seconds to run and it
+gives you an initial overview of what needs to be migrated without actually performing any migration. A great way to start a migration!
+
+This command detects all dependencies, and analyzes them. It is still experimental and at the moment only supports Python and SQL files.
+We expect this command to run within a minute on code bases up to 50.000 lines of code.
+Future versions of `ucx` will add support for more source types, and more migration details.
+
+When run from an IDE terminal, this command generates output as follows:
+![img.png](docs/lint-local-code-output.png)
+With modern IDEs, clicking on the file link opens the file at the problematic line
+
+[[back to top](#databricks-labs-ucx)]
+
 ## `migrate-local-code` command
 
 ```text

docs/lint-local-code-output.png

@@ -192,6 +192,9 @@ commands:
   - name: migrate-local-code
     description: (Experimental) Migrate files in the current directory to be more compatible with Unity Catalog.
 
+  - name:  lint-local-code
+    description: (Experimental) Lint files in the current directory to highlight incompatibilities with Unity Catalog.
+
   - name: show-all-metastores
     is_account_level: true
     description: Show all metastores available in the same region as the specified workspace

labs.yml

@@ -11,7 +11,7 @@
 
 from databricks.labs.ucx.config import WorkspaceConfig
 from databricks.labs.ucx.contexts.account_cli import AccountContext
-from databricks.labs.ucx.contexts.workspace_cli import WorkspaceContext
+from databricks.labs.ucx.contexts.workspace_cli import WorkspaceContext, LocalCheckoutContext
 from databricks.labs.ucx.hive_metastore.tables import What
 
 ucx = App(__file__)
@@ -398,7 +398,7 @@ def revert_cluster_remap(w: WorkspaceClient, prompts: Prompts):
 @ucx.command
 def migrate_local_code(w: WorkspaceClient, prompts: Prompts):
     """Fix the code files based on their language."""
-    ctx = WorkspaceContext(w)
+    ctx = LocalCheckoutContext(w)
     working_directory = Path.cwd()
     if not prompts.confirm("Do you want to apply UC migration to all files in the current directory?"):
         return
@@ -471,5 +471,15 @@ def revert_dbsql_dashboards(w: WorkspaceClient, dashboard_id: str | None = None)
     ctx.redash.revert_dashboards(dashboard_id)
 
 
+@ucx.command
+def lint_local_code(
+    w: WorkspaceClient, prompts: Prompts, path: str | None = None, ctx: LocalCheckoutContext | None = None
+):
+    """Lint local code files looking for problems."""
+    if ctx is None:
+        ctx = LocalCheckoutContext(w)
+    ctx.local_code_linter.lint(prompts, None if path is None else Path(path))
+
+
 if __name__ == "__main__":
     ucx()

src/databricks/labs/ucx/cli.py

@@ -17,6 +17,7 @@
 from databricks.labs.ucx.recon.schema_comparator import StandardSchemaComparator
 from databricks.labs.ucx.source_code.python_libraries import PipResolver
 from databricks.sdk import AccountClient, WorkspaceClient, core
+from databricks.sdk.errors import ResourceDoesNotExist
 from databricks.sdk.service import sql
 
 from databricks.labs.ucx.account.workspaces import WorkspaceInfo
@@ -46,7 +47,7 @@
     NotebookResolver,
     NotebookLoader,
 )
-from databricks.labs.ucx.source_code.files import FileLoader, ImportFileResolver
+from databricks.labs.ucx.source_code.files import FileLoader, FolderLoader, ImportFileResolver
 from databricks.labs.ucx.source_code.path_lookup import PathLookup
 from databricks.labs.ucx.source_code.graph import DependencyResolver
 from databricks.labs.ucx.source_code.whitelist import Whitelist
@@ -286,12 +287,15 @@ def aws_acl(self):
     @cached_property
     def principal_locations(self):
         eligible_locations = {}
-        if self.is_azure:
-            eligible_locations = self.azure_acl.get_eligible_locations_principals()
-        if self.is_aws:
-            eligible_locations = self.aws_acl.get_eligible_locations_principals()
-        if self.is_gcp:
-            raise NotImplementedError("Not implemented for GCP.")
+        try:
+            if self.is_azure:
+                eligible_locations = self.azure_acl.get_eligible_locations_principals()
+            if self.is_aws:
+                eligible_locations = self.aws_acl.get_eligible_locations_principals()
+            if self.is_gcp:
+                raise NotImplementedError("Not implemented for GCP.")
+        except ResourceDoesNotExist:
+            pass
         return eligible_locations
 
     @cached_property
@@ -382,6 +386,10 @@ def path_lookup(self):
     def file_loader(self):
         return FileLoader()
 
+    @cached_property
+    def folder_loader(self):
+        return FolderLoader(self.file_loader)
+
     @cached_property
     def whitelist(self):
         # TODO: fill in the whitelist

src/databricks/labs/ucx/contexts/application.py

@@ -8,13 +8,14 @@
 from databricks.labs.ucx.assessment.aws import run_command, AWSResources
 from databricks.labs.ucx.aws.access import AWSResourcePermissions
 from databricks.labs.ucx.aws.credentials import IamRoleMigration, IamRoleCreation
+from databricks.labs.ucx.aws.locations import AWSExternalLocationsMigration
 from databricks.labs.ucx.azure.access import AzureResourcePermissions
 from databricks.labs.ucx.azure.credentials import StorageCredentialManager, ServicePrincipalMigration
 from databricks.labs.ucx.azure.locations import ExternalLocationsMigration
-from databricks.labs.ucx.aws.locations import AWSExternalLocationsMigration
 from databricks.labs.ucx.azure.resources import AzureAPIClient, AzureResources
 from databricks.labs.ucx.contexts.application import CliContext
-from databricks.labs.ucx.source_code.files import LocalFileMigrator
+from databricks.labs.ucx.source_code.files import LocalFileMigrator, LocalCodeLinter
+from databricks.labs.ucx.source_code.notebooks.loaders import NotebookLoader
 from databricks.labs.ucx.workspace_access.clusters import ClusterAccess
 
 
@@ -31,10 +32,6 @@ def workspace_client(self) -> WorkspaceClient:
     def sql_backend(self) -> SqlBackend:
         return StatementExecutionBackend(self.workspace_client, self.config.warehouse_id)
 
-    @cached_property
-    def local_file_migrator(self):
-        return LocalFileMigrator(self.languages)
-
     @cached_property
     def cluster_access(self):
         return ClusterAccess(self.installation, self.workspace_client, self.prompts)
@@ -169,3 +166,22 @@ def iam_role_creation(self):
             self.workspace_client,
             self.aws_resource_permissions,
         )
+
+    @cached_property
+    def notebook_loader(self) -> NotebookLoader:
+        return NotebookLoader()
+
+
+class LocalCheckoutContext(WorkspaceContext):
+    """Local context extends Workspace context to provide extra properties
+    for running local operations."""
+
+    @cached_property
+    def local_file_migrator(self):
+        return LocalFileMigrator(self.languages)
+
+    @cached_property
+    def local_code_linter(self):
+        return LocalCodeLinter(
+            self.file_loader, self.folder_loader, self.path_lookup, self.dependency_resolver, lambda: self.languages
+        )

src/databricks/labs/ucx/contexts/workspace_cli.py

@@ -3,6 +3,8 @@
 from abc import abstractmethod
 from collections.abc import Iterable
 from dataclasses import dataclass
+from pathlib import Path
+
 
 # Code mapping between LSP, PyLint, and our own diagnostics:
 # | LSP                       | PyLint     | Our            |
@@ -54,6 +56,15 @@ def as_deprecation(self) -> 'Deprecation':
     def as_convention(self) -> 'Convention':
         return Convention(**self.__dict__)
 
+    def for_path(self, path: Path) -> LocatedAdvice:
+        return LocatedAdvice(self, path)
+
+
+@dataclass
+class LocatedAdvice:
+    advice: Advice
+    path: Path
+
 
 class Advisory(Advice):
     """A warning that does not prevent the code from running."""