Improve make known devex (#1820)

## Changes

Improve the devex around `known.json`:

- Improve logging so that it's clearer what it's doing, in particular
when there aren't any new distributions to scan.
 - Handle building from scratch when `known.json` has been removed.
 - Only rewrite the known.json file if it has changed.
- Use `importlib` instead of `pkgutil` when loading `known.json` (the
latter is deprecated from python 3.12).

### Functionality 

- [ ] added relevant user documentation
- [ ] added new CLI command
- [ ] modified existing command: `databricks labs ucx ...`
- [ ] added a new workflow
- [ ] modified existing workflow: `...`
- [ ] added a new table
- [ ] modified existing table: `...`

### Tests
<!-- How is this tested? Please see the checklist below and also
describe any other relevant tests -->

- [X] manually tested
- [X] added unit tests
- [ ] added integration tests
- [ ] verified on staging environment (screenshot attached)

Author: asnare

src/databricks/labs/ucx/source_code/known.py

@@ -103,13 +103,24 @@ def _cleanup_name(name):
     def rebuild(cls, root: Path):
         """rebuild the known.json file by analyzing the source code of installed libraries. Invoked by `make known`."""
         path_lookup = PathLookup.from_sys_path(root)
-        known_distributions = cls._get_known()
+        try:
+            known_distributions = cls._get_known()
+            logger.info("Scanning for newly installed distributions...")
+        except FileNotFoundError:
+            logger.info("No known distributions found; scanning all distributions...")
+            known_distributions = {}
+        updated_distributions = known_distributions.copy()
         for library_root in path_lookup.library_roots:
             for dist_info_folder in library_root.glob("*.dist-info"):
-                cls._analyze_dist_info(dist_info_folder, known_distributions, library_root)
-        known_json = Path(__file__).parent / "known.json"
-        with known_json.open('w') as f:
-            json.dump(dict(sorted(known_distributions.items())), f, indent=2)
+                cls._analyze_dist_info(dist_info_folder, updated_distributions, library_root)
+        updated_distributions = dict(sorted(updated_distributions.items()))
+        if known_distributions == updated_distributions:
+            logger.info("No new distributions found.")
+        else:
+            known_json = Path(__file__).parent / "known.json"
+            with known_json.open('w') as f:
+                json.dump(updated_distributions, f, indent=2)
+            logger.info(f"Updated known distributions: {known_json.relative_to(Path.cwd())}")
 
     @classmethod
     def _analyze_dist_info(cls, dist_info_folder, known_distributions, library_root):

tests/unit/source_code/test_known.py

@@ -1,4 +1,7 @@
 from pathlib import Path
+from unittest import mock
+
+import pytest
 
 from databricks.labs.ucx.source_code.known import Whitelist
 
@@ -32,5 +35,19 @@ def test_checks_library_compatibility():
 
 
 def test_loads_known_json():
+    known_json = Whitelist._get_known()  # pylint: disable=protected-access
+    assert known_json is not None and len(known_json) > 0
+
+
+def test_error_on_missing_known_json():
+    with (
+        mock.patch("pkgutil.get_data", side_effect=FileNotFoundError("simulate missing file")),
+        pytest.raises(FileNotFoundError),
+    ):
+        Whitelist._get_known()  # pylint: disable=protected-access
+
+
+def test_rebuild_trivial():
+    # No-op: the known.json file is already up-to-date
     cwd = Path.cwd()
     Whitelist.rebuild(cwd)