SCANPY-100: New bootstrapping: Resolve the JRE to use (#132)

Co-authored-by: Sebastian Zumbrunn <[email protected]>

Author: 2 people

src/pysonar_scanner/api.py

@@ -19,13 +19,14 @@
 #
 import typing
 from dataclasses import dataclass
+from typing import Optional
 
 import requests
 import requests.auth
 
 from pysonar_scanner.configuration import Configuration
-from pysonar_scanner.exceptions import SonarQubeApiException
-from pysonar_scanner.utils import remove_trailing_slash
+from pysonar_scanner.exceptions import MissingKeyException, SonarQubeApiException
+from pysonar_scanner.utils import Arch, Os, remove_trailing_slash
 
 
 @dataclass(frozen=True)
@@ -76,6 +77,32 @@ def __post_init__(self):
         object.__setattr__(self, "api_base_url", remove_trailing_slash(self.api_base_url))
 
 
+@dataclass(frozen=True)
+class JRE:
+    id: str
+    filename: str
+    sha256: str
+    java_path: str
+    os: str
+    arch: str
+    download_url: Optional[str]
+
+    @staticmethod
+    def from_dict(dict: dict) -> "JRE":
+        try:
+            return JRE(
+                id=dict["id"],
+                filename=dict["filename"],
+                sha256=dict["sha256"],
+                java_path=dict["javaPath"],
+                os=dict["os"],
+                arch=dict["arch"],
+                download_url=dict.get("downloadUrl", None),
+            )
+        except KeyError as e:
+            raise MissingKeyException(f"Missing key in dictionary {dict}") from e
+
+
 def get_base_urls(config: Configuration) -> BaseUrls:
     def is_sq_cloud_url(sonar_host_url: str) -> bool:
         sq_cloud_url = config.sonar.scanner.sonarcloud_url.strip() or "https://sonarcloud.io"
@@ -151,21 +178,51 @@ def download_analysis_engine(self, handle: typing.BinaryIO) -> None:
         Alternative, if the file IO fails, an IOError or OSError can be raised.
         """
 
-        def fetch_response():
+        try:
             res = requests.get(
                 f"{self.base_urls.api_base_url}/analysis/engine",
                 headers={"Accept": "application/octet-stream"},
                 auth=self.auth,
             )
+            self.__download_file(res, handle)
+        except requests.RequestException as e:
+            raise SonarQubeApiException("Error while fetching the analysis engine") from e
+
+    def get_analysis_jres(self, os: Optional[Os] = None, arch: Optional[Arch] = None) -> list[JRE]:
+        try:
+            params = {
+                "os": os.value if os else None,
+                "arch": arch.value if arch else None,
+            }
+            res = requests.get(
+                f"{self.base_urls.api_base_url}/analysis/jres",
+                auth=self.auth,
+                headers={"Accept": "application/json"},
+                params=params,
+            )
             res.raise_for_status()
-            return res
+            json_array = res.json()
+            return [JRE.from_dict(jre) for jre in json_array]
+        except (requests.RequestException, MissingKeyException) as e:
+            raise SonarQubeApiException("Error while fetching the analysis version") from e
 
-        def download_file(handle: typing.BinaryIO, requests: requests.Response):
-            for chunk in requests.iter_content(chunk_size=128):
-                handle.write(chunk)
+    def download_analysis_jre(self, id: str, handle: typing.BinaryIO) -> None:
+        """
+        This method can raise a SonarQubeApiException if the server doesn't respond successfully.
+        Alternative, if the file IO fails, an IOError or OSError can be raised.
+        """
 
         try:
-            res = fetch_response()
-            download_file(handle, res)
+            res = requests.get(
+                f"{self.base_urls.api_base_url}/analysis/jres/{id}",
+                headers={"Accept": "application/octet-stream"},
+                auth=self.auth,
+            )
+            self.__download_file(res, handle)
         except requests.RequestException as e:
-            raise SonarQubeApiException("Error while fetching the analysis engine information") from e
+            raise SonarQubeApiException("Error while fetching the JRE") from e
+
+    def __download_file(self, res: requests.Response, handle: typing.BinaryIO) -> None:
+        res.raise_for_status()
+        for chunk in res.iter_content(chunk_size=128):
+            handle.write(chunk)

src/pysonar_scanner/cache.py

@@ -24,14 +24,17 @@
 
 from pysonar_scanner import utils
 
-OpenBinaryMode = typing.Literal["rb", "xb"]
+OpenBinaryMode = typing.Literal["wb", "xb"]
 
 
 @dataclass(frozen=True)
 class CacheFile:
     filepath: pathlib.Path
     checksum: str
 
+    def exists(self) -> bool:
+        return self.filepath.exists()
+
     def is_valid(self) -> bool:
         try:
             with open(self.filepath, "rb") as f:
@@ -41,7 +44,7 @@ def is_valid(self) -> bool:
         except OSError:
             return False
 
-    def open(self, mode: OpenBinaryMode = "rb") -> typing.BinaryIO:
+    def open(self, mode: OpenBinaryMode) -> typing.BinaryIO:
         return open(self.filepath, mode=mode)
 
 
@@ -55,6 +58,9 @@ def get_file(self, filename: str, checksum: str) -> CacheFile:
         path = self.cache_folder / filename
         return CacheFile(path, checksum)
 
+    def get_file_path(self, filename: str) -> pathlib.Path:
+        return self.cache_folder / filename
+
     @staticmethod
     def create_cache(cache_folder: pathlib.Path):
         if not cache_folder.exists():
@@ -63,4 +69,4 @@ def create_cache(cache_folder: pathlib.Path):
 
 
 def get_default() -> Cache:
-    return Cache.create_cache(pathlib.Path.home() / ".sonar-scanner")
+    return Cache.create_cache(pathlib.Path.home() / ".sonar-scanner/cache")

src/pysonar_scanner/exceptions.py

@@ -19,6 +19,10 @@
 #
 
 
+class MissingKeyException(Exception):
+    pass
+
+
 class SonarQubeApiException(Exception):
     pass
 
@@ -29,3 +33,15 @@ class SQTooOldException(Exception):
 
 class ChecksumException(Exception):
     pass
+
+
+class JreProvisioningException(Exception):
+    pass
+
+
+class NoJreAvailableException(JreProvisioningException):
+    pass
+
+
+class UnsupportedArchiveFormat(JreProvisioningException):
+    pass

src/pysonar_scanner/jre.py

@@ -0,0 +1,142 @@
+#
+# Sonar Scanner Python
+# Copyright (C) 2011-2024 SonarSource SA.
+# mailto:info AT sonarsource DOT com
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 3 of the License, or (at your option) any later version.
+# This program is distributed in the hope that it will be useful,
+#
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+import pathlib
+import shutil
+import tarfile
+import zipfile
+from dataclasses import dataclass
+from typing import Optional
+
+from pysonar_scanner import utils
+from pysonar_scanner.api import JRE, SonarQubeApi
+from pysonar_scanner.cache import Cache
+from pysonar_scanner.configuration import Configuration
+from pysonar_scanner.exceptions import (
+    ChecksumException,
+    NoJreAvailableException,
+    UnsupportedArchiveFormat,
+)
+from pysonar_scanner.exceptions import JreProvisioningException
+
+
+@dataclass(frozen=True)
+class JREResolvedPath:
+    path: pathlib.Path
+
+    @staticmethod
+    def from_string(path: str) -> "JREResolvedPath":
+        if not path:
+            raise ValueError("JRE path cannot be empty")
+        return JREResolvedPath(pathlib.Path(path))
+
+
+class JREProvisioner:
+    def __init__(self, api: SonarQubeApi, cache: Cache):
+        self.api = api
+        self.cache = cache
+
+    def provision(self) -> JREResolvedPath:
+        jre, resolved_path = self.__attempt_provisioning_jre_with_retry()
+        return self.__unpack_jre(jre, resolved_path)
+
+    def __attempt_provisioning_jre_with_retry(self) -> tuple[JRE, pathlib.Path]:
+        jre_and_resolved_path = self.__attempt_provisioning_jre()
+        if jre_and_resolved_path is None:
+            jre_and_resolved_path = self.__attempt_provisioning_jre()
+        if jre_and_resolved_path is None:
+            raise ChecksumException(
+                f"Failed to download and verify JRE for {utils.get_os().value} and {utils.get_arch().value}"
+            )
+
+        return jre_and_resolved_path
+
+    def __attempt_provisioning_jre(self) -> Optional[tuple[JRE, pathlib.Path]]:
+        jre = self.__get_available_jre()
+
+        jre_path = self.__get_jre_from_cache(jre)
+        if jre_path is not None:
+            return (jre, jre_path)
+
+        jre_path = self.__download_jre(jre)
+        return (jre, jre_path) if jre_path is not None else None
+
+    def __get_available_jre(self) -> JRE:
+        jres = self.api.get_analysis_jres(os=utils.get_os(), arch=utils.get_arch())
+        if len(jres) == 0:
+            raise NoJreAvailableException(
+                f"No JREs are available for {utils.get_os().value} and {utils.get_arch().value}"
+            )
+        return jres[0]
+
+    def __get_jre_from_cache(self, jre: JRE) -> Optional[pathlib.Path]:
+        cache_file = self.cache.get_file(jre.filename, jre.sha256)
+        return cache_file.filepath if cache_file.is_valid() else None
+
+    def __download_jre(self, jre: JRE) -> Optional[pathlib.Path]:
+        cache_file = self.cache.get_file(jre.filename, jre.sha256)
+        cache_file.filepath.unlink(missing_ok=True)
+
+        with cache_file.open(mode="wb") as f:
+            self.api.download_analysis_jre(jre.id, f)
+
+        return cache_file.filepath if cache_file.is_valid() else None
+
+    def __unpack_jre(self, jre: JRE, file_path: pathlib.Path) -> JREResolvedPath:
+        unzip_dir = self.__prepare_unzip_dir(file_path)
+        self.__extract_jre(file_path, unzip_dir)
+        return JREResolvedPath(unzip_dir / jre.java_path)
+
+    def __prepare_unzip_dir(self, file_path: pathlib.Path) -> pathlib.Path:
+        unzip_dir = self.cache.get_file_path(f"{file_path}_unzip")
+        try:
+            if unzip_dir.exists():
+                shutil.rmtree(unzip_dir)
+            unzip_dir.mkdir(parents=True)
+            return unzip_dir
+        except OSError as e:
+            raise JreProvisioningException(f"Failed to prepare unzip directory: {unzip_dir}") from e
+
+    def __extract_jre(self, file_path: pathlib.Path, unzip_dir: pathlib.Path):
+        if file_path.suffix == ".zip":
+            with zipfile.ZipFile(file_path, "r") as zip_ref:
+                zip_ref.extractall(unzip_dir)
+        elif file_path.suffix in [".gz", ".tgz"]:
+            with tarfile.open(file_path, "r:gz") as tar_ref:
+                tar_ref.extractall(unzip_dir, filter="data")
+        else:
+            raise UnsupportedArchiveFormat(f"Unsupported archive format: {file_path.suffix}")
+
+
+class JREResolver:
+    def __init__(self, configuration: Configuration, jre_provisioner: JREProvisioner):
+        self.configuration = configuration
+        self.jre_provisioner = jre_provisioner
+
+    def resolve_jre(self) -> JREResolvedPath:
+        exe_suffix = ".exe" if self.configuration.sonar.scanner.os == "windows" else ""
+        if self.configuration.sonar.scanner.java_exe_path:
+            return JREResolvedPath(pathlib.Path(self.configuration.sonar.scanner.java_exe_path))
+        if not self.configuration.sonar.scanner.skip_jre_provisioning:
+            return self.__provision_jre()
+        java_path = pathlib.Path(f"java{exe_suffix}")
+        return JREResolvedPath(java_path)
+
+    def __provision_jre(self) -> JREResolvedPath:
+        return self.jre_provisioner.provision()

src/pysonar_scanner/utils.py

@@ -18,7 +18,11 @@
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #
 import hashlib
+import pathlib
+import platform
+import re
 import typing
+from enum import Enum
 
 
 def remove_trailing_slash(url: str) -> str:
@@ -30,3 +34,56 @@ def calculate_checksum(filehandle: typing.BinaryIO) -> str:
     for byte_block in iter(lambda: filehandle.read(4096), b""):
         sha256_hash.update(byte_block)
     return sha256_hash.hexdigest()
+
+
+class Os(Enum):
+    WINDOWS = "windows"
+    LINUX = "linux"
+    MACOS = "mac"
+    ALPINE = "alpine"
+    OTHER = "other"
+
+
+def get_os() -> Os:
+    def is_alpine() -> bool:
+        try:
+            os_release = pathlib.Path("/etc/os-release")
+            if not os_release.exists():
+                os_release = pathlib.Path("/usr/lib/os-release")
+            if not os_release.exists():
+                return False
+            os_release_str = os_release.read_text()
+            return "ID=alpine" in os_release_str or 'ID="alpine"' in os_release_str
+        except OSError:
+            return False
+
+    os_name = platform.system()
+    if os_name == "Windows":
+        return Os.WINDOWS
+    elif os_name == "Darwin":
+        return Os.MACOS
+    elif os_name == "Linux":
+        if is_alpine():
+            return Os.ALPINE
+        else:
+            return Os.LINUX
+    return Os.OTHER
+
+
+class Arch(Enum):
+    X64 = "x64"
+    AARCH64 = "aarch64"
+    OTHER = "other"
+
+
+def get_arch() -> Arch:
+    machine = platform.machine().lower()
+    if machine in ["amd64", "x86_64"]:
+        return Arch.X64
+    elif machine == "arm64":
+        return Arch.AARCH64
+    return Arch.OTHER
+
+
+def filter_none_values(dictionary: dict) -> dict:
+    return {k: v for k, v in dictionary.items() if v is not None}