Merge branch 'main' into christo/manifest-server-errors

Author: ChristoGrab

airbyte_cdk/cli/airbyte_cdk/_secrets.py

@@ -459,25 +459,60 @@ def _print_ci_secrets_masks(
         _print_ci_secrets_masks_for_config(config=config_dict)
 
 
+def _print_ci_secret_mask_for_string(secret: str) -> None:
+    """Print GitHub CI mask for a single secret string.
+
+    We expect single-line secrets, but we also handle the case where the secret contains newlines.
+    For multi-line secrets, we must print a secret mask for each line separately.
+    """
+    for line in secret.splitlines():
+        if line.strip():  # Skip empty lines
+            print(f"::add-mask::{line!s}")
+
+
+def _print_ci_secret_mask_for_value(value: Any) -> None:
+    """Print GitHub CI mask for a single secret value.
+
+    Call this function for any values identified as secrets, regardless of type.
+    """
+    if isinstance(value, dict):
+        # For nested dicts, we call recursively on each value
+        for v in value.values():
+            _print_ci_secret_mask_for_value(v)
+
+        return
+
+    if isinstance(value, list):
+        # For lists, we call recursively on each list item
+        for list_item in value:
+            _print_ci_secret_mask_for_value(list_item)
+
+        return
+
+    # For any other types besides dict and list, we convert to string and mask each line
+    # separately to handle multi-line secrets (e.g. private keys).
+    for line in str(value).splitlines():
+        if line.strip():  # Skip empty lines
+            _print_ci_secret_mask_for_string(line)
+
+
 def _print_ci_secrets_masks_for_config(
     config: dict[str, str] | list[Any] | Any,
 ) -> None:
     """Print GitHub CI mask for secrets config, navigating child nodes recursively."""
     if isinstance(config, list):
+        # Check each item in the list to look for nested dicts that may contain secrets:
         for item in config:
             _print_ci_secrets_masks_for_config(item)
 
-    if isinstance(config, dict):
+    elif isinstance(config, dict):
         for key, value in config.items():
             if _is_secret_property(key):
                 logger.debug(f"Masking secret for config key: {key}")
-                print(f"::add-mask::{value!s}")
-                if isinstance(value, dict):
-                    # For nested dicts, we also need to mask the json-stringified version
-                    print(f"::add-mask::{json.dumps(value)!s}")
-
-            if isinstance(value, (dict, list)):
-                _print_ci_secrets_masks_for_config(config=value)
+                _print_ci_secret_mask_for_value(value)
+            elif isinstance(value, (dict, list)):
+                # Recursively check nested dicts and lists
+                _print_ci_secrets_masks_for_config(value)
 
 
 def _is_secret_property(property_name: str) -> bool:

airbyte_cdk/sources/declarative/auth/jwt.py

@@ -6,14 +6,28 @@
 import json
 from dataclasses import InitVar, dataclass
 from datetime import datetime
-from typing import Any, Mapping, Optional, Union
+from typing import Any, Mapping, MutableMapping, Optional, Union, cast
 
 import jwt
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric.ec import EllipticCurvePrivateKey
+from cryptography.hazmat.primitives.asymmetric.ed448 import Ed448PrivateKey
+from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
+from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey
 
 from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
 from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
 from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
+from airbyte_cdk.sources.declarative.requesters.request_option import (
+    RequestOption,
+    RequestOptionType,
+)
+
+# Type alias for keys that JWT library accepts
+JwtKeyTypes = Union[
+    RSAPrivateKey, EllipticCurvePrivateKey, Ed25519PrivateKey, Ed448PrivateKey, str, bytes
+]
 
 
 class JwtAlgorithm(str):
@@ -74,6 +88,8 @@ class JwtAuthenticator(DeclarativeAuthenticator):
     aud: Optional[Union[InterpolatedString, str]] = None
     additional_jwt_headers: Optional[Mapping[str, Any]] = None
     additional_jwt_payload: Optional[Mapping[str, Any]] = None
+    passphrase: Optional[Union[InterpolatedString, str]] = None
+    request_option: Optional[RequestOption] = None
 
     def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         self._secret_key = InterpolatedString.create(self.secret_key, parameters=parameters)
@@ -103,6 +119,18 @@ def __post_init__(self, parameters: Mapping[str, Any]) -> None:
         self._additional_jwt_payload = InterpolatedMapping(
             self.additional_jwt_payload or {}, parameters=parameters
         )
+        self._passphrase = (
+            InterpolatedString.create(self.passphrase, parameters=parameters)
+            if self.passphrase
+            else None
+        )
+
+        # When we first implemented the JWT authenticator, we assumed that the signed token was always supposed
+        # to be loaded into the request headers under the `Authorization` key. This is not always the case, but
+        # this default option allows for backwards compatibility to be retained for existing connectors
+        self._request_option = self.request_option or RequestOption(
+            inject_into=RequestOptionType.header, field_name="Authorization", parameters=parameters
+        )
 
     def _get_jwt_headers(self) -> dict[str, Any]:
         """
@@ -149,11 +177,21 @@ def _get_jwt_payload(self) -> dict[str, Any]:
         payload["nbf"] = nbf
         return payload
 
-    def _get_secret_key(self) -> str:
+    def _get_secret_key(self) -> JwtKeyTypes:
         """
         Returns the secret key used to sign the JWT.
         """
         secret_key: str = self._secret_key.eval(self.config, json_loads=json.loads)
+
+        if self._passphrase:
+            passphrase_value = self._passphrase.eval(self.config, json_loads=json.loads)
+            if passphrase_value:
+                private_key = serialization.load_pem_private_key(
+                    secret_key.encode(),
+                    password=passphrase_value.encode(),
+                )
+                return cast(JwtKeyTypes, private_key)
+
         return (
             base64.b64encode(secret_key.encode()).decode()
             if self._base64_encode_secret_key
@@ -186,7 +224,8 @@ def _get_header_prefix(self) -> Union[str, None]:
 
     @property
     def auth_header(self) -> str:
-        return "Authorization"
+        options = self._get_request_options(RequestOptionType.header)
+        return next(iter(options.keys()), "")
 
     @property
     def token(self) -> str:
@@ -195,3 +234,18 @@ def token(self) -> str:
             if self._get_header_prefix()
             else self._get_signed_token()
         )
+
+    def get_request_params(self) -> Mapping[str, Any]:
+        return self._get_request_options(RequestOptionType.request_parameter)
+
+    def get_request_body_data(self) -> Union[Mapping[str, Any], str]:
+        return self._get_request_options(RequestOptionType.body_data)
+
+    def get_request_body_json(self) -> Mapping[str, Any]:
+        return self._get_request_options(RequestOptionType.body_json)
+
+    def _get_request_options(self, option_type: RequestOptionType) -> Mapping[str, Any]:
+        options: MutableMapping[str, Any] = {}
+        if self._request_option.inject_into == option_type:
+            self._request_option.inject_into_request(options, self.token, self.config)
+        return options

airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -1270,6 +1270,16 @@ definitions:
         title: Additional JWT Payload Properties
         description: Additional properties to be added to the JWT payload.
         additionalProperties: true
+      passphrase:
+        title: Passphrase
+        description: A passphrase/password used to encrypt the private key. Only provide a passphrase if required by the API for JWT authentication. The API will typically provide the passphrase when generating the public/private key pair.
+        type: string
+        examples:
+          - "{{ config['passphrase'] }}"
+      request_option:
+        title: Request Option
+        description: A request option describing where the signed JWT token that is generated should be injected into the outbound API request.
+        "$ref": "#/definitions/RequestOption"
       $parameters:
         type: object
         additionalProperties: true

airbyte_cdk/sources/declarative/interpolation/macros.py

@@ -6,6 +6,7 @@
 import datetime
 import re
 import typing
+import uuid
 from typing import Optional, Union
 from urllib.parse import quote_plus
 
@@ -207,6 +208,16 @@ def camel_case_to_snake_case(value: str) -> str:
     return re.sub(r"(?<!^)(?=[A-Z])", "_", value).lower()
 
 
+def generate_uuid() -> str:
+    """
+    Generates a UUID4
+
+    Usage:
+    `"{{ generate_uuid() }}"`
+    """
+    return str(uuid.uuid4())
+
+
 _macros_list = [
     now_utc,
     today_utc,
@@ -220,5 +231,6 @@ def camel_case_to_snake_case(value: str) -> str:
     str_to_datetime,
     sanitize_url,
     camel_case_to_snake_case,
+    generate_uuid,
 ]
 macros = {f.__name__: f for f in _macros_list}