Abu UI d2 4557 gcp validation support (#1286)

* Adding validations in GCP

---------

Co-authored-by: Release Workflow <[email protected]>

Author: abuabraham-ttd

.github/workflows/publish-gcp-oidc-enclave-docker.yaml

@@ -93,6 +93,7 @@ jobs:
           echo "jar_version=$(mvn help:evaluate -Dexpression=project.version | grep -e '^[1-9][^\[]')" >> $GITHUB_OUTPUT
           echo "git_commit=$(git show --format="%h" --no-patch)" >> $GITHUB_OUTPUT
           cp -r target ${{ env.DOCKER_CONTEXT_PATH }}/
+          cp scripts/confidential_compute.py ${{ env.DOCKER_CONTEXT_PATH }}/
 
       - name: Log in to the Docker container registry
         uses: docker/login-action@v3

pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.uid2</groupId>
     <artifactId>uid2-operator</artifactId>
-    <version>5.44.6</version>
+    <version>5.45.7-alpha-169-SNAPSHOT</version>
 
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

scripts/confidential_compute.py

@@ -19,14 +19,14 @@ def __init__(self, error_name, provider, extra_message=None):
         urls = {
             "EC2": "https://unifiedid.com/docs/guides/operator-guide-aws-marketplace#uid2-operator-error-codes",
             "Azure": "https://unifiedid.com/docs/guides/operator-guide-azure-enclave#uid2-operator-error-codes",
-            "GCP": "https://unifiedid.com/docs/guides/operator-private-gcp-confidential-space#uid2-operator-error-codes",
+            "GCPEntrypoint": "https://unifiedid.com/docs/guides/operator-private-gcp-confidential-space#uid2-operator-error-codes",
         }
         url = urls.get(provider)
         super().__init__(f"{error_name}\n" + (extra_message if extra_message else "") + f"\nVisit {url} for more details")
 
 class MissingInstanceProfile(ConfidentialComputeStartupException):
-    def __init__(self, cls):
-        super().__init__(error_name=f"E01: {self.__class__.__name__}", provider=cls)
+    def __init__(self, cls, message = None):
+        super().__init__(error_name=f"E01: {self.__class__.__name__}", provider=cls, extra_message=message)
 
 class ConfigNotFound(ConfidentialComputeStartupException):
     def __init__(self, cls, message = None):

scripts/gcp-oidc/Dockerfile

@@ -5,7 +5,11 @@ LABEL "tee.launch_policy.allow_env_override"="API_TOKEN_SECRET_NAME,DEPLOYMENT_E
 LABEL "tee.launch_policy.log_redirect"="always"
 
 # Install Packages
-RUN apk update && apk add jq
+RUN apk update && apk add --no-cache jq python3 py3-pip && \
+    python3 -m venv /venv && \
+    . /venv/bin/activate && \
+    pip install --no-cache-dir google-cloud-secret-manager google-auth google-api-core && \
+    rm -rf /var/cache/apk/*
 
 WORKDIR /app
 EXPOSE 8080
@@ -18,7 +22,6 @@ ENV JAR_NAME=${JAR_NAME}
 ENV JAR_VERSION=${JAR_VERSION}
 ENV IMAGE_VERSION=${IMAGE_VERSION}
 ENV REGION=default
-ENV LOKI_HOSTNAME=loki
 
 COPY ./target/${JAR_NAME}-${JAR_VERSION}-jar-with-dependencies.jar /app/${JAR_NAME}-${JAR_VERSION}.jar
 COPY ./target/${JAR_NAME}-${JAR_VERSION}-sources.jar /app
@@ -28,9 +31,10 @@ COPY ./conf/*.xml /app/conf/
 
 RUN tar xzvf /app/static.tar.gz --no-same-owner --no-same-permissions && rm -f /app/static.tar.gz
 
-COPY ./entrypoint.sh /app/
-RUN chmod a+x /app/entrypoint.sh
+COPY ./gcp.py /app/
+COPY ./confidential_compute.py /app
+RUN chmod a+x /app/gcp.py
 
 RUN mkdir -p /opt/uid2 && chmod 777 -R /opt/uid2 && mkdir -p /app && chmod 705 -R /app && mkdir -p /app/file-uploads && chmod 777 -R /app/file-uploads
 
-CMD ["/app/entrypoint.sh"]
+CMD ["/venv/bin/python", "/app/gcp.py"]

scripts/gcp-oidc/conf/default-config.json

@@ -1,42 +1,42 @@
-{
-  "service_verbose": true,
-  "service_instances": 12,
-  "core_s3_bucket": null,
-  "core_attest_url": null,
-  "core_api_token": null,
-  "storage_mock": false,
-  "optout_s3_bucket": null,
-  "optout_s3_folder": "optout/",
-  "optout_s3_path_compat": false,
-  "optout_data_dir": "/opt/uid2/operator-optout/",
-  "optout_api_token": null,
-  "optout_api_uri": null,
-  "optout_bloom_filter_size": 8192,
-  "optout_delta_rotate_interval": 300,
-  "optout_delta_backtrack_in_days": 1,
-  "optout_partition_interval": 86400,
-  "optout_max_partitions": 30,
-  "optout_heap_default_capacity": 8192,
-  "cloud_download_threads": 8,
-  "cloud_upload_threads": 2,
-  "cloud_refresh_interval": 60,
-  "sites_metadata_path": "sites/metadata.json",
-  "clients_metadata_path": "clients/metadata.json",
-  "client_side_keypairs_metadata_path": "client_side_keypairs/metadata.json",
-  "keysets_metadata_path": "keysets/metadata.json",
-  "keyset_keys_metadata_path": "keyset_keys/metadata.json",
-  "salts_metadata_path": "salts/metadata.json",
-  "services_metadata_path": "services/metadata.json",
-  "service_links_metadata_path": "service_links/metadata.json",
-  "optout_metadata_path": null,
-  "enclave_platform": "gcp-oidc",
-  "optout_inmem_cache": true,
-  "identity_token_expires_after_seconds": 86400,
-  "refresh_token_expires_after_seconds": 2592000,
-  "refresh_identity_token_after_seconds": 3600,
-  "allow_legacy_api": false,
-  "failure_shutdown_wait_hours": 120,
-  "sharing_token_expiry_seconds": 2592000,
-  "validate_service_links": false,
-  "operator_type": "private"
-}
+{
+    "service_verbose": true,
+    "service_instances": 12,
+    "core_s3_bucket": null,
+    "core_attest_url": null,
+    "core_api_token": null,
+    "storage_mock": false,
+    "optout_s3_bucket": null,
+    "optout_s3_folder": "optout/",
+    "optout_s3_path_compat": false,
+    "optout_data_dir": "/opt/uid2/operator-optout/",
+    "optout_api_token": null,
+    "optout_api_uri": null,
+    "optout_bloom_filter_size": 8192,
+    "optout_delta_rotate_interval": 300,
+    "optout_delta_backtrack_in_days": 1,
+    "optout_partition_interval": 86400,
+    "optout_max_partitions": 30,
+    "optout_heap_default_capacity": 8192,
+    "cloud_download_threads": 8,
+    "cloud_upload_threads": 2,
+    "cloud_refresh_interval": 60,
+    "sites_metadata_path": "sites/metadata.json",
+    "clients_metadata_path": "clients/metadata.json",
+    "client_side_keypairs_metadata_path": "client_side_keypairs/metadata.json",
+    "keysets_metadata_path": "keysets/metadata.json",
+    "keyset_keys_metadata_path": "keyset_keys/metadata.json",
+    "salts_metadata_path": "salts/metadata.json",
+    "services_metadata_path": "services/metadata.json",
+    "service_links_metadata_path": "service_links/metadata.json",
+    "optout_metadata_path": null,
+    "enclave_platform": "gcp-oidc",
+    "optout_inmem_cache": true,
+    "identity_token_expires_after_seconds": 86400,
+    "refresh_token_expires_after_seconds": 2592000,
+    "refresh_identity_token_after_seconds": 3600,
+    "allow_legacy_api": false,
+    "failure_shutdown_wait_hours": 120,
+    "sharing_token_expiry_seconds": 2592000,
+    "validate_service_links": false,
+    "operator_type": "private"
+}

scripts/gcp-oidc/conf/integ-config.json

@@ -0,0 +1,14 @@
+{
+    "sites_metadata_path": "https://core.uidapi.com/sites/refresh",
+    "clients_metadata_path": "https://core.uidapi.com/clients/refresh",
+    "keysets_metadata_path": "https://core.uidapi.com/key/keyset/refresh",
+    "keyset_keys_metadata_path": "https://core.uidapi.com/key/keyset-keys/refresh",
+    "client_side_keypairs_metadata_path": "https://core.uidapi.com/client_side_keypairs/refresh",
+    "salts_metadata_path": "https://core.uidapi.com/salt/refresh",
+    "services_metadata_path": "https://core.uidapi.com/services/refresh",
+    "service_links_metadata_path": "https://core.uidapi.com/service_links/refresh",
+    "optout_metadata_path": "https://optout.uidapi.com/optout/refresh",
+    "core_attest_url": "https://core.uidapi.com/attest",
+    "optout_api_uri": "https://optout.uidapi.com/optout/replicate",
+    "optout_s3_folder": "uid-optout-integ/"
+}

scripts/gcp-oidc/conf/integ-uid2-config.json

@@ -1,15 +1,15 @@
-{
-  "sites_metadata_path": "https://core.uidapi.com/sites/refresh",
-  "clients_metadata_path": "https://core.uidapi.com/clients/refresh",
-  "keysets_metadata_path": "https://core.uidapi.com/key/keyset/refresh",
-  "keyset_keys_metadata_path": "https://core.uidapi.com/key/keyset-keys/refresh",
-  "client_side_keypairs_metadata_path": "https://core.uidapi.com/client_side_keypairs/refresh",
-  "salts_metadata_path": "https://core.uidapi.com/salt/refresh",
-  "services_metadata_path": "https://core.uidapi.com/services/refresh",
-  "service_links_metadata_path": "https://core.uidapi.com/service_links/refresh",
-  "optout_metadata_path": "https://optout.uidapi.com/optout/refresh",
-  "core_attest_url": "https://core.uidapi.com/attest",
-  "optout_api_uri": "https://optout.uidapi.com/optout/replicate",
-  "optout_s3_folder": "optout-v2/",
-  "identity_token_expires_after_seconds": 259200
-}
+{
+  "sites_metadata_path": "https://core.uidapi.com/sites/refresh",
+  "clients_metadata_path": "https://core.uidapi.com/clients/refresh",
+  "keysets_metadata_path": "https://core.uidapi.com/key/keyset/refresh",
+  "keyset_keys_metadata_path": "https://core.uidapi.com/key/keyset-keys/refresh",
+  "client_side_keypairs_metadata_path": "https://core.uidapi.com/client_side_keypairs/refresh",
+  "salts_metadata_path": "https://core.uidapi.com/salt/refresh",
+  "services_metadata_path": "https://core.uidapi.com/services/refresh",
+  "service_links_metadata_path": "https://core.uidapi.com/service_links/refresh",
+  "optout_metadata_path": "https://optout.uidapi.com/optout/refresh",
+  "core_attest_url": "https://core.uidapi.com/attest",
+  "optout_api_uri": "https://optout.uidapi.com/optout/replicate",
+  "optout_s3_folder": "optout-v2/",
+  "identity_token_expires_after_seconds": 259200
+}

…ipts/gcp-oidc/conf/prod-uid2-config.json scripts/gcp-oidc/conf/prod-config.jsonscripts/gcp-oidc/conf/prod-uid2-config.json renamed to scripts/gcp-oidc/conf/prod-config.json scripts/gcp-oidc/conf/prod-uid2-config.json renamed to scripts/gcp-oidc/conf/prod-config.json

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+
+import os
+import shutil
+from typing import Dict
+import sys
+from google.cloud import secretmanager
+from google.auth import default
+from google.auth.exceptions import DefaultCredentialsError
+from google.api_core.exceptions import PermissionDenied, NotFound
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from confidential_compute import ConfidentialCompute, ConfidentialComputeConfig, MissingConfig, ConfigNotFound, MissingInstanceProfile, ConfidentialComputeStartupException
+
+class GCPEntrypoint(ConfidentialCompute):
+
+    def __init__(self):
+        super().__init__()
+
+    def _get_secret(self, secret_identifier=None) -> ConfidentialComputeConfig:
+        keys_mapping = {
+            "core_base_url": "CORE_BASE_URL",
+            "optout_base_url": "OPTOUT_BASE_URL",
+            "environment": "DEPLOYMENT_ENVIRONMENT",
+            "skip_validations": "SKIP_VALIDATIONS",
+            "debug_mode": "DEBUG_MODE",
+        }
+        config: ConfidentialComputeConfig = {
+            key: (os.environ[env_var].lower() == "true" if key in ["skip_validations", "debug_mode"] else os.environ[env_var])
+            for key, env_var in keys_mapping.items() if env_var in os.environ
+        }
+
+        if not os.getenv("API_TOKEN_SECRET_NAME"):
+            raise MissingConfig(self.__class__.__name__, ["API_TOKEN_SECRET_NAME"])
+        try:
+            client = secretmanager.SecretManagerServiceClient()
+            secret_version_name = f"{os.getenv("API_TOKEN_SECRET_NAME")}"
+            response = client.access_secret_version(name=secret_version_name)
+            secret_value = response.payload.data.decode("UTF-8")
+        except (PermissionDenied, DefaultCredentialsError) as e:
+            raise MissingInstanceProfile(self.__class__.__name__, str(e))
+        except NotFound:
+            raise ConfigNotFound(self.__class__.__name__, f"Secret Manager {os.getenv("API_TOKEN_SECRET_NAME")}")
+        config["api_token"] = secret_value
+        return config
+    
+    def __populate_operator_config(self, destination):
+        target_config = f"/app/conf/{self.configs["environment"].lower()}-config.json"
+        shutil.copy(target_config, destination)
+        with open(destination, 'r') as file:
+            config = file.read()
+        config = config.replace("https://core.uidapi.com", self.configs.get("core_base_url"))
+        config = config.replace("https://optout.uidapi.com", self.configs.get("optout_base_url"))
+        with open(destination, 'w') as file:
+            file.write(config)
+
+    def _setup_auxiliaries(self) -> None:
+        """ No Auxiliariy service required for GCP Confidential compute. """
+        pass
+
+    def _validate_auxiliaries(self) -> None:
+        """ No Auxiliariy service required for GCP Confidential compute. """
+        pass
+
+    def run_compute(self) -> None:
+        self.configs = self._get_secret('read_from_env_vars')
+        print(f"Fetched configs")
+        if not self.configs.get("skip_validations"):
+            self.validate_configuration()
+        config_locaton = "/tmp/final-config.json"
+        self.__populate_operator_config(config_locaton)
+        os.environ["gcp_secret_version_name"] = os.getenv("API_TOKEN_SECRET_NAME")
+        java_command = [
+            "java",
+            "-XX:MaxRAMPercentage=95", 
+            "-XX:-UseCompressedOops", 
+            "-XX:+PrintFlagsFinal",
+            "-Djava.security.egd=file:/dev/./urandom",
+            "-Dvertx.logger-delegate-factory-class-name=io.vertx.core.logging.SLF4JLogDelegateFactory",
+            "-Dlogback.configurationFile=/app/conf/logback.xml",
+            f"-Dvertx-config-path={config_locaton}",
+            "-jar",
+            f"{os.getenv("JAR_NAME")}-{os.getenv("JAR_VERSION")}.jar"
+        ]
+        self.run_command(java_command)
+
+if __name__ == "__main__":
+    try:
+        gcp = GCP()
+        gcp.run_compute()
+    except ConfidentialComputeStartupException as e:
+        print("Failed starting up Confidential Compute. Please checks the logs for errors and retry \n", e)
+    except Exception as e:
+         print("Unexpected failure while starting up Confidential Compute. Please contact UID support team with this log \n ", e)
+