feat(sca): move SCA models from ggshield to pygitguardian

Author: xblanchot-gg

pygitguardian/client.py

@@ -32,6 +32,12 @@
     SecretScanPreferences,
     ServerMetadata,
 )
+from .sca_models import (
+    ComputeSCAFilesResult,
+    SCAScanAllOutput,
+    SCAScanDiffOutput,
+    SCAScanParameters,
+)
 
 
 logger = logging.getLogger(__name__)
@@ -605,3 +611,89 @@ def create_jwt(
             obj = load_detail(resp)
         obj.status_code = resp.status_code
         return obj
+
+    def compute_sca_files(
+        self,
+        files: List[str],
+        extra_headers: Optional[Dict[str, str]] = None,
+    ) -> Union[Detail, ComputeSCAFilesResult]:
+        if len(files) == 0:
+            result = ComputeSCAFilesResult(sca_files=[], potential_siblings=[])
+            result.status_code = 200
+            return result
+
+        response = self.post(
+            endpoint="sca/compute_sca_files/",
+            data={"files": files},
+            extra_headers=extra_headers,
+        )
+        result: Union[Detail, ComputeSCAFilesResult]
+        if is_ok(response):
+            result = ComputeSCAFilesResult.from_dict(response.json())
+        else:
+            result = load_detail(response)
+
+        result.status_code = response.status_code
+        return result
+
+    def sca_scan_directory(
+        self,
+        tar_file: bytes,
+        scan_parameters: SCAScanParameters,
+        extra_headers: Optional[Dict[str, str]] = None,
+    ) -> Union[Detail, SCAScanAllOutput]:
+        """
+        Launches an SCA scan via SCA public API on a tar archive
+        """
+
+        result: Union[Detail, SCAScanAllOutput]
+
+        try:
+            # bypass self.post because data argument is needed in self.request and self.post use it as json
+            response = self.request(
+                "post",
+                endpoint="sca/sca_scan_all/",
+                files={"directory": tar_file},
+                data={
+                    "scan_parameters": SCAScanParameters.SCHEMA.dumps(scan_parameters)
+                },
+                extra_headers=extra_headers,
+            )
+        except requests.exceptions.ReadTimeout:
+            result = Detail("The request timed out.")
+            result.status_code = 504
+        else:
+            if is_ok(response):
+                result = SCAScanAllOutput.from_dict(response.json())
+            else:
+                result = load_detail(response)
+
+            result.status_code = response.status_code
+
+        return result
+
+    def scan_diff(
+        self,
+        reference: bytes,
+        current: bytes,
+        scan_parameters: SCAScanParameters,
+    ) -> Union[Detail, SCAScanDiffOutput]:
+        result: Union[Detail, SCAScanDiffOutput]
+        try:
+            response = self.post(
+                endpoint="sca/sca_scan_diff/",
+                files={"reference": reference, "current": current},
+                data={
+                    "scan_parameters": SCAScanParameters.SCHEMA.dumps(scan_parameters)
+                },
+            )
+        except requests.exceptions.ReadTimeout:
+            result = Detail("The request timed out.")
+            result.status_code = 504
+        else:
+            if is_ok(response):
+                result = SCAScanDiffOutput.from_dict(response.json())
+            else:
+                result = load_detail(response)
+            result.status_code = response.status_code
+        return result

pygitguardian/sca_models.py

@@ -0,0 +1,127 @@
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import List, Optional, cast
+
+import marshmallow_dataclass
+from typing_extensions import Literal
+
+from pygitguardian.models import Base, BaseSchema, FromDictMixin
+
+
+@dataclass
+class SCAIgnoredVulnerability(Base, FromDictMixin):
+    """
+    A model of an ignored vulnerability for SCA. This allows to ignore all occurrences
+    of a given vulnerability in a given dependency file.
+    - identifier: identifier (currently: GHSA id) of the vulnerability to ignore
+    - path: the path to the file in which ignore the vulnerability
+    """
+
+    identifier: str
+    path: str
+
+
+SCAIgnoredVulnerability.SCHEMA = cast(
+    BaseSchema,
+    marshmallow_dataclass.class_schema(
+        SCAIgnoredVulnerability, base_schema=BaseSchema
+    )(),
+)
+
+
+@dataclass
+class SCAScanParameters(Base, FromDictMixin):
+    minimum_severity: Optional[str] = None
+    ignored_vulnerabilities: List[SCAIgnoredVulnerability] = field(default_factory=list)
+
+
+SCAScanParameters.SCHEMA = cast(
+    BaseSchema,
+    marshmallow_dataclass.class_schema(SCAScanParameters, base_schema=BaseSchema)(),
+)
+
+
+@dataclass
+class ComputeSCAFilesResult(Base, FromDictMixin):
+    sca_files: List[str]
+    potential_siblings: List[str]
+
+
+ComputeSCAFilesResult.SCHEMA = cast(
+    BaseSchema,
+    marshmallow_dataclass.class_schema(ComputeSCAFilesResult, base_schema=BaseSchema)(),
+)
+
+
+@dataclass
+class SCAVulnerability(Base, FromDictMixin):
+    severity: str
+    summary: str
+    identifier: str
+    cve_ids: List[str] = field(default_factory=list)
+    created_at: Optional[datetime] = None
+    fixed_version: Optional[str] = None
+
+
+SCAVulnerability.SCHEMA = cast(
+    BaseSchema,
+    marshmallow_dataclass.class_schema(SCAVulnerability, base_schema=BaseSchema)(),
+)
+
+SCADependencyType = Literal["direct", "transitive"]
+
+
+@dataclass
+class SCAVulnerablePackageVersion(Base, FromDictMixin):
+    package_full_name: str
+    version: str
+    ecosystem: str
+    dependency_type: Optional[SCADependencyType] = None
+    vulns: List[SCAVulnerability] = field(default_factory=list)
+
+
+SCAVulnerablePackageVersion.SCHEMA = cast(
+    BaseSchema,
+    marshmallow_dataclass.class_schema(
+        SCAVulnerablePackageVersion, base_schema=BaseSchema
+    )(),
+)
+
+
+@dataclass
+class SCALocationVulnerability(Base, FromDictMixin):
+    location: str
+    package_vulns: List[SCAVulnerablePackageVersion] = field(default_factory=list)
+
+
+SCALocationVulnerability.SCHEMA = cast(
+    BaseSchema,
+    marshmallow_dataclass.class_schema(
+        SCALocationVulnerability, base_schema=BaseSchema
+    )(),
+)
+
+
+@dataclass
+class SCAScanAllOutput(Base, FromDictMixin):
+    scanned_files: List[str] = field(default_factory=list)
+    found_package_vulns: List[SCALocationVulnerability] = field(default_factory=list)
+
+
+SCAScanAllOutput.SCHEMA = cast(
+    BaseSchema,
+    marshmallow_dataclass.class_schema(SCAScanAllOutput, base_schema=BaseSchema)(),
+)
+
+
+@dataclass
+class SCAScanDiffOutput(Base, FromDictMixin):
+    scanned_files: List[str] = field(default_factory=list)
+    added_vulns: List[SCALocationVulnerability] = field(default_factory=list)
+    removed_vulns: List[SCALocationVulnerability] = field(default_factory=list)
+
+
+SCAScanDiffOutput.SCHEMA = cast(
+    BaseSchema,
+    marshmallow_dataclass.class_schema(SCAScanDiffOutput, base_schema=BaseSchema)(),
+)

tests/cassettes/test_sca_client_scan_diff.yaml

@@ -0,0 +1,62 @@
+interactions:
+  - request:
+      body: '{"files": ["Pipfile", "something_else"]}'
+      headers:
+        Accept:
+          - '*/*'
+        Accept-Encoding:
+          - gzip, deflate
+        Connection:
+          - keep-alive
+        Content-Length:
+          - '40'
+        Content-Type:
+          - application/json
+        User-Agent:
+          - pygitguardian/1.9.0 (Linux;py3.10.12)
+      method: POST
+      uri: https://api.gitguardian.com/v1/sca/compute_sca_files/
+    response:
+      body:
+        string: '{"sca_files":["Pipfile"],"potential_siblings":["Pipfile.lock"]}'
+      headers:
+        access-control-expose-headers:
+          - X-App-Version
+        allow:
+          - POST, OPTIONS
+        content-length:
+          - '63'
+        content-type:
+          - application/json
+        cross-origin-opener-policy:
+          - same-origin
+        date:
+          - Thu, 17 Aug 2023 08:43:29 GMT
+        referrer-policy:
+          - strict-origin-when-cross-origin
+        server:
+          - istio-envoy
+        strict-transport-security:
+          - max-age=31536000; includeSubDomains
+        vary:
+          - Cookie
+        x-app-version:
+          - v2.36.1
+        x-content-type-options:
+          - nosniff
+          - nosniff
+        x-envoy-upstream-service-time:
+          - '15'
+        x-frame-options:
+          - DENY
+          - SAMEORIGIN
+        x-sca-engine-version:
+          - 1.16.1
+        x-secrets-engine-version:
+          - 2.95.0
+        x-xss-protection:
+          - 1; mode=block
+      status:
+        code: 200
+        message: OK
+version: 1

tests/cassettes/test_sca_scan_compute_files.yaml

@@ -0,0 +1,64 @@
+interactions:
+  - request:
+      body:
+        "--770ddecb25300ec91a085b90915fe8f1\r\nContent-Disposition: form-data; name=\"scan_parameters\"\r\n\r\n{\"minimum_severity\":
+        null, \"ignored_vulnerabilities\": []}\r\n--770ddecb25300ec91a085b90915fe8f1\r\nContent-Disposition:
+        form-data; name=\"directory\"; filename=\"directory\"\r\n\r\n\r\n--770ddecb25300ec91a085b90915fe8f1--\r\n"
+      headers:
+        Accept:
+          - '*/*'
+        Accept-Encoding:
+          - gzip, deflate
+        Connection:
+          - keep-alive
+        Content-Length:
+          - '303'
+        Content-Type:
+          - multipart/form-data; boundary=770ddecb25300ec91a085b90915fe8f1
+        User-Agent:
+          - pygitguardian/1.9.0 (Linux;py3.10.12)
+      method: POST
+      uri: https://api.gitguardian.com/v1/sca/sca_scan_all/
+    response:
+      body:
+        string: '{"detail":"Directory is not a valid tarfile"}'
+      headers:
+        access-control-expose-headers:
+          - X-App-Version
+        allow:
+          - POST, OPTIONS
+        content-length:
+          - '45'
+        content-type:
+          - application/json
+        cross-origin-opener-policy:
+          - same-origin
+        date:
+          - Thu, 17 Aug 2023 08:43:30 GMT
+        referrer-policy:
+          - strict-origin-when-cross-origin
+        server:
+          - istio-envoy
+        strict-transport-security:
+          - max-age=31536000; includeSubDomains
+        vary:
+          - Cookie
+        x-app-version:
+          - v2.36.1
+        x-content-type-options:
+          - nosniff
+          - nosniff
+        x-envoy-upstream-service-time:
+          - '16'
+        x-frame-options:
+          - DENY
+        x-sca-engine-version:
+          - 1.16.1
+        x-secrets-engine-version:
+          - 2.95.0
+        x-xss-protection:
+          - 1; mode=block
+      status:
+        code: 400
+        message: Bad Request
+version: 1