SNOW-3012459: Extend core telemetry (#2742)

Author: sfc-gh-turbaszek

src/snowflake/connector/_utils.py

@@ -7,6 +7,7 @@
 import platform
 import string
 import threading
+import time
 from enum import Enum
 from inspect import stack
 from pathlib import Path
@@ -141,6 +142,7 @@ def __init__(self):
         self._version: bytes | None = None
         self._error: Exception | None = None
         self._path: str | None = None
+        self._load_time: float | None = None
 
     @staticmethod
     def _detect_os() -> str:
@@ -229,7 +231,7 @@ def _get_lib_name() -> str:
             return "libsf_mini_core.so"
 
     @staticmethod
-    def _get_core_path() -> Path:
+    def _get_core_path():
         """Get the path to the minicore library for the current platform."""
         subdir = _CoreLoader._get_platform_subdir()
         lib_name = _CoreLoader._get_lib_name()
@@ -253,20 +255,49 @@ def _load_minicore(path: str) -> ctypes.CDLL:
             core = ctypes.CDLL(str(lib_path))
         return core
 
+    def get_present_binaries(self) -> str:
+        present_binaries = []
+        try:
+            minicore_files = importlib.resources.files("snowflake.connector.minicore")
+            # Iterate through all items in the minicore module
+            for item in minicore_files.iterdir():
+                # Skip non-platform directories like __pycache__
+                if item.is_dir() and not item.name.startswith("__"):
+                    # This is a platform subdirectory
+                    platform_name = item.name
+                    try:
+                        # List all files in this subdirectory
+                        for binary_file in item.iterdir():
+                            if binary_file.is_file():
+                                # Store as "platform/filename"
+                                present_binaries.append(
+                                    f"{platform_name}/{binary_file.name}"
+                                )
+                    except Exception as e:
+                        logger.debug(f"Error listing binaries in {platform_name}: {e}")
+        except Exception as e:
+            logger.debug(f"Error populating present binaries: {e}")
+
+        return ",".join(present_binaries)
+
     def _is_core_disabled(self) -> bool:
         value = str(os.getenv("SNOWFLAKE_DISABLE_MINICORE", None)).lower()
         return value in ["1", "true"]
 
     def _load(self) -> None:
+        start_time = time.perf_counter()
         try:
             path = self._get_core_path()
+            self._path = str(path)
             core = self._load_minicore(path)
             self._register_functions(core)
             self._version = core.sf_core_full_version()
             self._error = None
-            self._path = str(path)
         except Exception as err:
             self._error = err
+        end_time = time.perf_counter()
+        # Store load time in milliseconds (with sub-millisecond precision)
+        self._load_time = (end_time - start_time) * 1000
 
     def load(self):
         """Spawn a separate thread to load the minicore library (non-blocking)."""
@@ -291,6 +322,10 @@ def get_core_version(self) -> str | None:
     def get_file_name(self) -> str:
         return self._path
 
+    def get_load_time(self) -> float | None:
+        """Return the time it took to load the minicore binary in milliseconds."""
+        return self._load_time
+
 
 _core_loader = _CoreLoader()
 
@@ -308,5 +343,7 @@ def build_minicore_usage_for_telemetry() -> dict[str, str | None]:
         "OS": OPERATING_SYSTEM,
         "OS_VERSION": OS_VERSION,
         "CORE_LOAD_ERROR": _core_loader.get_load_error(),
+        "CORE_BINARIES_PRESENT": _core_loader.get_present_binaries(),
+        "CORE_LOAD_TIME": _core_loader.get_load_time(),
         **build_minicore_usage_for_session(),
     }

test/unit/test_util.py

@@ -38,13 +38,17 @@ def test_e2e(self):
         sleep(2)
         assert loader.get_load_error() == str(None)
         assert loader.get_core_version() == "0.0.1"
+        # Verify load time was measured
+        assert loader.get_load_time() is not None
+        assert loader.get_load_time() >= 0
 
     def test_core_loader_initialization(self):
         """Test that _CoreLoader initializes with None values."""
         loader = _CoreLoader()
         assert loader._version is None
         assert loader._error is None
         assert loader._path is None
+        assert loader._load_time is None
 
     @pytest.mark.parametrize(
         "system,expected",
@@ -194,7 +198,7 @@ def test_register_functions(self):
         assert mock_core.sf_core_full_version.restype == ctypes.c_char_p
 
     def test_load_minicore(self):
-        """Test that _load_minicore loads the library."""
+        """Test that _load_minicore loads the library correctly."""
         mock_path = mock.MagicMock()
         mock_lib_path = "/path/to/libsf_mini_core.so"
 
@@ -255,7 +259,7 @@ def test_load_skips_loading_when_core_disabled(self):
     def test_load_success(self):
         """Test successful load of the core library."""
         loader = _CoreLoader()
-        mock_path = mock.MagicMock()
+        mock_path = "/path/to/libsf_mini_core.so"
         mock_core = mock.MagicMock()
         mock_version = b"1.2.3"
         mock_core.sf_core_full_version = mock.MagicMock(return_value=mock_version)
@@ -270,15 +274,18 @@ def test_load_success(self):
                     with mock.patch.object(
                         loader, "_register_functions"
                     ) as mock_register:
-                        loader.load()
-                        sleep(2)
-
-                        mock_get_path.assert_called_once()
-                        mock_load.assert_called_once_with(mock_path)
-                        mock_register.assert_called_once_with(mock_core)
-                        assert loader._version == mock_version
-                        assert loader._error is None
-                        assert loader._path == str(mock_path)
+                        with mock.patch("time.perf_counter", side_effect=[0.0, 0.0155]):
+                            loader.load()
+                            sleep(2)
+
+                            mock_get_path.assert_called_once()
+                            mock_load.assert_called_once_with(mock_path)
+                            mock_register.assert_called_once_with(mock_core)
+                            assert loader._version == mock_version
+                            assert loader._error is None
+                            assert loader._path == mock_path
+                            # (0.0155 - 0.0) * 1000 = 15.5 ms
+                            assert loader._load_time == 15.5
 
     def test_load_failure(self):
         """Test that load captures exceptions."""
@@ -349,6 +356,102 @@ def test_get_file_name_no_path(self):
 
         assert result is None
 
+    def test_get_load_time_with_time(self):
+        """Test get_load_time returns the load time when it has been set."""
+        loader = _CoreLoader()
+        loader._load_time = 42.5
+
+        result = loader.get_load_time()
+
+        assert result == 42.5
+
+    def test_get_load_time_no_time(self):
+        """Test get_load_time returns None when no load time exists."""
+        loader = _CoreLoader()
+
+        result = loader.get_load_time()
+
+        assert result is None
+
+    def test_get_present_binaries_contains_expected_paths(self):
+        """Test get_present_binaries returns binaries for expected paths."""
+        loader = _CoreLoader()
+
+        result = loader.get_present_binaries()
+
+        assert isinstance(result, str)
+        assert result != ""
+
+    def test_get_present_binaries_with_mocked_structure(self, tmp_path):
+        """Test get_present_binaries with mocked directory structure."""
+        loader = _CoreLoader()
+
+        # Create a temporary directory structure mimicking minicore layout
+        # Create platform directories with binary files
+        linux_dir = tmp_path / "linux_x86_64_glibc"
+        linux_dir.mkdir()
+        (linux_dir / "libsf_mini_core.so").write_text("fake binary content")
+
+        macos_dir = tmp_path / "macos_aarch64"
+        macos_dir.mkdir()
+        (macos_dir / "libsf_mini_core.dylib").write_text("fake binary content")
+
+        windows_dir = tmp_path / "windows_x86_64"
+        windows_dir.mkdir()
+        (windows_dir / "sf_mini_core.dll").write_text("fake binary content")
+
+        # Create a __pycache__ directory that should be ignored
+        pycache_dir = tmp_path / "__pycache__"
+        pycache_dir.mkdir()
+        (pycache_dir / "some_file.pyc").write_text("cached file")
+
+        # Mock importlib.resources.files to return our temp directory
+        with mock.patch("importlib.resources.files") as mock_files:
+            mock_files.return_value = tmp_path
+
+            result = loader.get_present_binaries()
+
+            # Verify the function was called with correct module name
+            mock_files.assert_called_once_with("snowflake.connector.minicore")
+
+            # Parse the result
+            binaries = result.split(",")
+            assert len(binaries) == 3
+
+            # Verify all expected binaries are present
+            assert "linux_x86_64_glibc/libsf_mini_core.so" in binaries
+            assert "macos_aarch64/libsf_mini_core.dylib" in binaries
+            assert "windows_x86_64/sf_mini_core.dll" in binaries
+
+            # Verify __pycache__ files are not included
+            assert not any("__pycache__" in binary for binary in binaries)
+
+    def test_get_present_binaries_with_empty_directory(self, tmp_path):
+        """Test get_present_binaries returns empty string for empty directory."""
+        loader = _CoreLoader()
+
+        # Create an empty temp directory
+        # Mock importlib.resources.files to return our temp directory
+        with mock.patch("importlib.resources.files") as mock_files:
+            mock_files.return_value = tmp_path
+
+            result = loader.get_present_binaries()
+
+            assert result == ""
+
+    def test_get_present_binaries_handles_exceptions(self):
+        """Test get_present_binaries handles exceptions gracefully."""
+        loader = _CoreLoader()
+
+        # Mock importlib.resources.files to raise an exception
+        with mock.patch("importlib.resources.files") as mock_files:
+            mock_files.side_effect = Exception("Failed to access resources")
+
+            # Should not raise, but return empty string
+            result = loader.get_present_binaries()
+
+            assert result == ""
+
 
 def test_importing_snowflake_connector_triggers_core_loader_load():
     """Test that importing snowflake.connector triggers core_loader.load()."""