SCANPY-101: New bootstrapping: Fetch the Scanner Engine (#133)

Author: Seppli11

poetry.lock

@@ -32,6 +32,7 @@ python = '>=3.9'
 toml = '>=0.10.2'
 requests = "^2.32.3"
 responses = "^0.25.6"
+pyfakefs = "^5.7.4"
 
 [tool.poetry.group]
 [tool.poetry.group.dev]

pyproject.toml

@@ -17,11 +17,15 @@
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #
+import typing
 from dataclasses import dataclass
+
+import requests
+import requests.auth
+
 from pysonar_scanner.configuration import Configuration
 from pysonar_scanner.exceptions import SonarQubeApiException
 from pysonar_scanner.utils import remove_trailing_slash
-import requests
 
 
 @dataclass(frozen=True)
@@ -103,6 +107,12 @@ def __call__(self, r):
         return r
 
 
+@dataclass(frozen=True)
+class EngineInfo:
+    filename: str
+    sha256: str
+
+
 class SonarQubeApi:
     def __init__(self, base_urls: BaseUrls, token: str):
         self.base_urls = base_urls
@@ -121,3 +131,41 @@ def get_analysis_version(self) -> SQVersion:
             return SQVersion.from_str(res.text)
         except requests.RequestException as e:
             raise SonarQubeApiException("Error while fetching the analysis version") from e
+
+    def get_analysis_engine(self) -> EngineInfo:
+        try:
+            res = requests.get(
+                f"{self.base_urls.api_base_url}/analysis/engine", headers={"Accept": "application/json"}, auth=self.auth
+            )
+            res.raise_for_status()
+            json = res.json()
+            if "filename" not in json or "sha256" not in json:
+                raise SonarQubeApiException("Invalid response from the server")
+            return EngineInfo(filename=json["filename"], sha256=json["sha256"])
+        except requests.RequestException as e:
+            raise SonarQubeApiException("Error while fetching the analysis engine information") from e
+
+    def download_analysis_engine(self, handle: typing.BinaryIO) -> None:
+        """
+        This method can raise a SonarQubeApiException if the server doesn't respond successfully.
+        Alternative, if the file IO fails, an IOError or OSError can be raised.
+        """
+
+        def fetch_response():
+            res = requests.get(
+                f"{self.base_urls.api_base_url}/analysis/engine",
+                headers={"Accept": "application/octet-stream"},
+                auth=self.auth,
+            )
+            res.raise_for_status()
+            return res
+
+        def download_file(handle: typing.BinaryIO, requests: requests.Response):
+            for chunk in requests.iter_content(chunk_size=128):
+                handle.write(chunk)
+
+        try:
+            res = fetch_response()
+            download_file(handle, res)
+        except requests.RequestException as e:
+            raise SonarQubeApiException("Error while fetching the analysis engine information") from e

src/pysonar_scanner/api.py

@@ -0,0 +1,66 @@
+#
+# Sonar Scanner Python
+# Copyright (C) 2011-2024 SonarSource SA.
+# mailto:info AT sonarsource DOT com
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3 of the License, or (at your option) any later version.
+# This program is distributed in the hope that it will be useful,
+#
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+
+import pathlib
+import typing
+from dataclasses import dataclass
+
+from pysonar_scanner import utils
+
+OpenBinaryMode = typing.Literal["rb", "xb"]
+
+
+@dataclass(frozen=True)
+class CacheFile:
+    filepath: pathlib.Path
+    checksum: str
+
+    def is_valid(self) -> bool:
+        try:
+            with open(self.filepath, "rb") as f:
+                calculated_checksum = utils.calculate_checksum(f)
+
+            return calculated_checksum == self.checksum
+        except OSError:
+            return False
+
+    def open(self, mode: OpenBinaryMode = "rb") -> typing.BinaryIO:
+        return open(self.filepath, mode=mode)
+
+
+class Cache:
+    def __init__(self, cache_folder: pathlib.Path):
+        if not cache_folder.exists():
+            raise FileNotFoundError(f"Cache folder {cache_folder} does not exist")
+        self.cache_folder = cache_folder
+
+    def get_file(self, filename: str, checksum: str) -> CacheFile:
+        path = self.cache_folder / filename
+        return CacheFile(path, checksum)
+
+    @staticmethod
+    def create_cache(cache_folder: pathlib.Path):
+        if not cache_folder.exists():
+            cache_folder.mkdir(parents=True)
+        return Cache(cache_folder)
+
+
+def get_default() -> Cache:
+    return Cache.create_cache(pathlib.Path.home() / ".sonar-scanner")

src/pysonar_scanner/cache.py

@@ -18,7 +18,6 @@
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #
 import time
-
 from dataclasses import dataclass
 from enum import Enum
 from typing import Optional

src/pysonar_scanner/configuration.py

@@ -25,3 +25,7 @@ class SonarQubeApiException(Exception):
 
 class SQTooOldException(Exception):
     pass
+
+
+class ChecksumException(Exception):
+    pass

src/pysonar_scanner/exceptions.py

@@ -17,14 +17,42 @@
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #
-from pysonar_scanner.api import SonarQubeApi
 import pysonar_scanner.api as api
-from pysonar_scanner.exceptions import SQTooOldException
+from pysonar_scanner.api import SonarQubeApi
+from pysonar_scanner.cache import Cache, CacheFile
+from pysonar_scanner.exceptions import ChecksumException, SQTooOldException
+
+
+class ScannerEngineProvisioner:
+    def __init__(self, api: SonarQubeApi, cache: Cache):
+        self.api = api
+        self.cache = cache
+
+    def provision(self) -> None:
+        if self.__download_and_verify():
+            return
+        # Retry once in case the checksum failed due to the scanner engine being updated between getting the checksum and downloading the jar
+        if self.__download_and_verify():
+            return
+        else:
+            raise ChecksumException("Failed to download and verify scanner engine")
+
+    def __download_and_verify(self) -> bool:
+        engine_info = self.api.get_analysis_engine()
+        cache_file = self.cache.get_file(engine_info.filename, engine_info.sha256)
+        if not cache_file.is_valid():
+            self.__download_scanner_engine(cache_file)
+        return cache_file.is_valid()
+
+    def __download_scanner_engine(self, cache_file: CacheFile) -> None:
+        with cache_file.open(mode="wb") as f:
+            self.api.download_analysis_engine(f)
 
 
 class ScannerEngine:
-    def __init__(self, api: SonarQubeApi):
+    def __init__(self, api: SonarQubeApi, cache: Cache):
         self.api = api
+        self.cache = cache
 
     def __version_check(self):
         if self.api.is_sonar_qube_cloud():
@@ -34,3 +62,6 @@ def __version_check(self):
             raise SQTooOldException(
                 f"Only SonarQube versions >= {api.MIN_SUPPORTED_SQ_VERSION} are supported, but got {version}"
             )
+
+    def __fetch_scanner_engine(self):
+        ScannerEngineProvisioner(self.api, self.cache).provision()

src/pysonar_scanner/scannerengine.py

@@ -17,7 +17,16 @@
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #
+import hashlib
+import typing
 
 
 def remove_trailing_slash(url: str) -> str:
     return url.rstrip("/ ").lstrip()
+
+
+def calculate_checksum(filehandle: typing.BinaryIO) -> str:
+    sha256_hash = hashlib.sha256()
+    for byte_block in iter(lambda: filehandle.read(4096), b""):
+        sha256_hash.update(byte_block)
+    return sha256_hash.hexdigest()

src/pysonar_scanner/utils.py

@@ -19,9 +19,9 @@
 #
 import contextlib
 from typing import Optional
-from typing_extensions import Self
 from pysonar_scanner.api import BaseUrls, SonarQubeApi
 import responses
+from responses import matchers
 
 
 def get_sq_server() -> SonarQubeApi:
@@ -46,16 +46,40 @@ def __init__(self, base_url: str = "http://sq.home", rsps: Optional[responses.Re
         self.api_url = f"{base_url}/api/v2"
         self.rsps = rsps or responses
 
-    def mock_analysis_version(self, version: str = "", status: int = 200) -> Self:
-        self.rsps.get(url=f"{self.api_url}/analysis/version", body=version, status=status)
-        return self
+    def mock_analysis_version(self, version: str = "", status: int = 200) -> responses.BaseResponse:
+        return self.rsps.get(url=f"{self.api_url}/analysis/version", body=version, status=status)
 
-    def mock_server_version(self, version: str = "", status: int = 200) -> Self:
-        self.rsps.get(url=f"{self.base_url}/api/server/version", body=version, status=status)
-        return self
+    def mock_analysis_engine(
+        self, filename: Optional[str] = None, sha256: Optional[str] = None, status: int = 200
+    ) -> responses.BaseResponse:
+        def prepare_json_obj() -> dict:
+            json_response = {}
+            if filename:
+                json_response["filename"] = filename
+            if sha256:
+                json_response["sha256"] = sha256
+            return json_response
+
+        return self.rsps.get(
+            url=f"{self.api_url}/analysis/engine",
+            json=prepare_json_obj(),
+            status=status,
+            match=[matchers.header_matcher({"Accept": "application/json"})],
+        )
+
+    def mock_analysis_engine_download(self, body: bytes = b"", status: int = 200) -> responses.BaseResponse:
+        return self.rsps.get(
+            url=f"{self.api_url}/analysis/engine",
+            body=body,
+            status=status,
+            match=[matchers.header_matcher({"Accept": "application/octet-stream"})],
+        )
+
+    def mock_server_version(self, version: str = "", status: int = 200) -> responses.BaseResponse:
+        return self.rsps.get(url=f"{self.base_url}/api/server/version", body=version, status=status)
 
 
 @contextlib.contextmanager
-def sq_api_mocker(base_url: str = "http://sq.home"):
-    with responses.RequestsMock() as rsps:
+def sq_api_mocker(base_url: str = "http://sq.home", assert_all_requests_are_fired: bool = True):
+    with responses.RequestsMock(assert_all_requests_are_fired=assert_all_requests_are_fired) as rsps:
         yield SQApiMocker(base_url=base_url, rsps=rsps)