Skip to content

Commit 3301aa4

Browse files
authored
UID2-4719 change azure cc starting process (#1260)
* UID2-4719 change azure cc starting process
1 parent 4c8e872 commit 3301aa4

File tree

7 files changed

+263
-61
lines changed

7 files changed

+263
-61
lines changed

.github/workflows/publish-azure-cc-enclave-docker.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ jobs:
9292
echo "jar_version=$(mvn help:evaluate -Dexpression=project.version | grep -e '^[1-9][^\[]')" >> $GITHUB_OUTPUT
9393
echo "git_commit=$(git show --format="%h" --no-patch)" >> $GITHUB_OUTPUT
9494
cp -r target ${{ env.DOCKER_CONTEXT_PATH }}/
95+
cp scripts/confidential_compute.py ${{ env.DOCKER_CONTEXT_PATH }}/
9596
9697
- name: Log in to the Docker container registry
9798
uses: docker/login-action@v3

pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
<groupId>com.uid2</groupId>
88
<artifactId>uid2-operator</artifactId>
9-
<version>5.45.7-alpha-169-SNAPSHOT</version>
9+
<version>5.45.8-alpha-172-SNAPSHOT</version>
1010

1111
<properties>
1212
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

scripts/aws/ec2.py

Lines changed: 19 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
import yaml
1717

1818
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
19-
from confidential_compute import ConfidentialCompute, ConfidentialComputeConfig, MissingInstanceProfile, ConfigNotFound, InvalidConfigValue, ConfidentialComputeStartupException
19+
from confidential_compute import ConfidentialCompute, ConfidentialComputeConfig, MissingInstanceProfile, ApiTokenNotFound, InvalidConfigValue, ConfidentialComputeStartupException
2020

2121
class AWSConfidentialComputeConfig(ConfidentialComputeConfig):
2222
enclave_memory_mb: int
@@ -48,10 +48,10 @@ def get_meta_url(cls) -> str:
4848
return f"http://{cls.AWS_METADATA}/latest/dynamic/instance-identity/document"
4949

5050

51-
class EC2(ConfidentialCompute):
51+
class EC2EntryPoint(ConfidentialCompute):
5252

5353
def __init__(self):
54-
super().__init__()
54+
self.configs: AWSConfidentialComputeConfig = {}
5555

5656
def __get_aws_token(self) -> str:
5757
"""Fetches a temporary AWS EC2 metadata token."""
@@ -74,38 +74,36 @@ def __get_current_region(self) -> str:
7474
except requests.RequestException as e:
7575
raise RuntimeError(f"Failed to fetch region: {e}")
7676

77-
def __validate_aws_specific_config(self, secret):
78-
if "enclave_memory_mb" in secret or "enclave_cpu_count" in secret:
77+
def __validate_aws_specific_config(self):
78+
if "enclave_memory_mb" in self.configs or "enclave_cpu_count" in self.configs:
7979
max_capacity = self.__get_max_capacity()
8080
min_capacity = {"enclave_memory_mb": 11000, "enclave_cpu_count" : 2 }
8181
for key in ["enclave_memory_mb", "enclave_cpu_count"]:
82-
if int(secret.get(key, 0)) > max_capacity.get(key):
83-
raise ValueError(f"{key} value ({secret.get(key, 0)}) exceeds the maximum allowed ({max_capacity.get(key)}).")
84-
if min_capacity.get(key) > int(secret.get(key, 10**9)):
85-
raise ValueError(f"{key} value ({secret.get(key, 0)}) needs to be higher than the minimum required ({min_capacity.get(key)}).")
82+
if int(self.configs.get(key, 0)) > max_capacity.get(key):
83+
raise ValueError(f"{key} value ({self.configs.get(key, 0)}) exceeds the maximum allowed ({max_capacity.get(key)}).")
84+
if min_capacity.get(key) > int(self.configs.get(key, 10**9)):
85+
raise ValueError(f"{key} value ({self.configs.get(key, 0)}) needs to be higher than the minimum required ({min_capacity.get(key)}).")
8686

87-
def _get_secret(self, secret_identifier: str) -> AWSConfidentialComputeConfig:
87+
def _set_confidential_config(self, secret_identifier: str) -> None:
8888
"""Fetches a secret value from AWS Secrets Manager and adds defaults"""
8989

90-
def add_defaults(configs: Dict[str, any]) -> AWSConfidentialComputeConfig:
90+
def add_defaults(configs: Dict[str, any]) -> None:
9191
"""Adds default values to configuration if missing."""
9292
default_capacity = self.__get_max_capacity()
9393
configs.setdefault("enclave_memory_mb", default_capacity["enclave_memory_mb"])
9494
configs.setdefault("enclave_cpu_count", default_capacity["enclave_cpu_count"])
9595
configs.setdefault("debug_mode", False)
96-
return configs
9796

9897
region = self.__get_current_region()
9998
print(f"Running in {region}")
10099
client = boto3.client("secretsmanager", region_name=region)
101100
try:
102-
secret = add_defaults(json.loads(client.get_secret_value(SecretId=secret_identifier)["SecretString"]))
103-
self.__validate_aws_specific_config(secret)
104-
return secret
101+
add_defaults(json.loads(client.get_secret_value(SecretId=secret_identifier)["SecretString"]))
102+
self.__validate_aws_specific_config()
105103
except NoCredentialsError as _:
106104
raise MissingInstanceProfile(self.__class__.__name__)
107105
except ClientError as _:
108-
raise ConfigNotFound(self.__class__.__name__, f"Secret Manager {secret_identifier} in {region}")
106+
raise ApiTokenNotFound(self.__class__.__name__, f"Secret Manager {secret_identifier} in {region}")
109107

110108
@staticmethod
111109
def __get_max_capacity():
@@ -137,7 +135,7 @@ def __run_config_server(self) -> None:
137135
json.dump(self.configs, config_file)
138136
os.chdir("/opt/uid2operator/config-server")
139137
command = ["./bin/flask", "run", "--host", AuxiliaryConfig.LOCALHOST, "--port", AuxiliaryConfig.FLASK_PORT]
140-
self.run_command(command, seperate_process=True)
138+
self.run_command(command, separate_process=True)
141139

142140
def __run_socks_proxy(self) -> None:
143141
"""
@@ -205,12 +203,12 @@ def __run_nitro_enclave(self):
205203
if self.configs.get('debug_mode', False):
206204
print("Running in debug_mode")
207205
command += ["--debug-mode", "--attach-console"]
208-
self.run_command(command, seperate_process=True)
206+
self.run_command(command, separate_process=True)
209207

210208
def run_compute(self) -> None:
211209
"""Main execution flow for confidential compute."""
212210
secret_manager_key = self.__get_secret_name_from_userdata()
213-
self.configs = self._get_secret(secret_manager_key)
211+
self._set_confidential_config(secret_manager_key)
214212
print(f"Fetched configs from {secret_manager_key}")
215213
if not self.configs.get("skip_validations"):
216214
self.validate_configuration()
@@ -246,13 +244,13 @@ def __kill_auxiliaries(self) -> None:
246244
parser.add_argument("-o", "--operation", choices=["stop", "start"], default="start", help="Operation to perform.")
247245
args = parser.parse_args()
248246
try:
249-
ec2 = EC2()
247+
ec2 = EC2EntryPoint()
250248
if args.operation == "stop":
251249
ec2.cleanup()
252250
else:
253251
ec2.run_compute()
254252
except ConfidentialComputeStartupException as e:
255253
print("Failed starting up Confidential Compute. Please checks the logs for errors and retry \n", e)
256254
except Exception as e:
257-
print("Unexpected failure while starting up Confidential Compute. Please contact UID support team with this log \n ", e)
255+
print("Unexpected failure while starting up Confidential Compute. Please contact UID support team with this log \n ", e)
258256

scripts/azure-cc/Dockerfile

Lines changed: 29 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,24 @@
1-
# sha from https://hub.docker.com/layers/amd64/eclipse-temurin/21.0.4_7-jre-alpine/images/sha256-8179ddc8a6c5ac9af935020628763b9a5a671e0914976715d2b61b21881cefca
1+
# Use Alpine-based JRE image
22
FROM eclipse-temurin@sha256:8179ddc8a6c5ac9af935020628763b9a5a671e0914976715d2b61b21881cefca
33

4-
# Install Packages
5-
RUN apk update && apk add jq
4+
# Install necessary packages and set up virtual environment
5+
RUN apk update && apk add --no-cache jq python3 py3-pip && \
6+
python3 -m venv /venv && \
7+
. /venv/bin/activate && \
8+
pip install --no-cache-dir requests azure-identity azure-keyvault-secrets && \
9+
rm -rf /var/cache/apk/*
610

11+
# Set virtual environment path
12+
ENV PATH="/venv/bin:$PATH"
13+
14+
# Working directory
715
WORKDIR /app
16+
17+
# Expose necessary ports
818
EXPOSE 8080
919
EXPOSE 9080
1020

21+
# ARG and ENV variables
1122
ARG JAR_NAME=uid2-operator
1223
ARG JAR_VERSION=1.0.0-SNAPSHOT
1324
ARG IMAGE_VERSION=1.0.0.unknownhash
@@ -17,18 +28,28 @@ ENV IMAGE_VERSION=${IMAGE_VERSION}
1728
ENV REGION=default
1829
ENV LOKI_HOSTNAME=loki
1930

31+
# Copy application files
2032
COPY ./target/${JAR_NAME}-${JAR_VERSION}-jar-with-dependencies.jar /app/${JAR_NAME}-${JAR_VERSION}.jar
2133
COPY ./target/${JAR_NAME}-${JAR_VERSION}-sources.jar /app
2234
COPY ./target/${JAR_NAME}-${JAR_VERSION}-static.tar.gz /app/static.tar.gz
2335
COPY ./conf/*.json /app/conf/
2436
COPY ./conf/*.xml /app/conf/
2537

26-
RUN tar xzvf /app/static.tar.gz --no-same-owner --no-same-permissions && rm -f /app/static.tar.gz
38+
# Extract and clean up tar.gz
39+
RUN tar xzvf /app/static.tar.gz --no-same-owner --no-same-permissions && \
40+
rm -f /app/static.tar.gz
41+
42+
COPY ./azureEntryPoint.py /app
43+
COPY ./confidential_compute.py /app
44+
RUN chmod a+x /app/*.py
2745

28-
COPY ./entrypoint.sh /app/
29-
RUN chmod a+x /app/entrypoint.sh
46+
# Create and configure non-root user
47+
RUN adduser -D uid2-operator && \
48+
mkdir -p /opt/uid2 && chmod 777 -R /opt/uid2 && \
49+
chmod 705 -R /app && mkdir -p /app/file-uploads && chmod 777 -R /app/file-uploads
3050

31-
RUN adduser -D uid2-operator && mkdir -p /opt/uid2 && chmod 777 -R /opt/uid2 && mkdir -p /app && chmod 705 -R /app && mkdir -p /app/file-uploads && chmod 777 -R /app/file-uploads
51+
# Switch to non-root user
3252
USER uid2-operator
3353

34-
CMD ["/app/entrypoint.sh"]
54+
# Run the Python entry point
55+
CMD python3 /app/azureEntryPoint.py
Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
#!/usr/bin/env python3
2+
3+
import json
4+
import os
5+
import time
6+
from typing import Dict
7+
import sys
8+
import shutil
9+
import requests
10+
import logging
11+
12+
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
13+
from confidential_compute import ConfidentialCompute, MissingConfig, MissingInstanceProfile, AuxiliariesException, SecretAccessDenied, ApiTokenNotFound, ConfidentialComputeStartupException
14+
from azure.keyvault.secrets import SecretClient
15+
from azure.identity import DefaultAzureCredential, CredentialUnavailableError
16+
from azure.core.exceptions import ResourceNotFoundError, ClientAuthenticationError
17+
18+
class AzureEntryPoint(ConfidentialCompute):
19+
20+
kv_name = os.getenv("VAULT_NAME")
21+
secret_name = os.getenv("OPERATOR_KEY_SECRET_NAME")
22+
env_name = os.getenv("DEPLOYMENT_ENVIRONMENT")
23+
jar_name = os.getenv("JAR_NAME", "default-jar-name")
24+
jar_version = os.getenv("JAR_VERSION", "default-jar-version")
25+
26+
FINAL_CONFIG = "/tmp/final-config.json"
27+
28+
def __init__(self):
29+
super().__init__()
30+
31+
def __check_env_variables(self):
32+
# Check essential env variables
33+
if AzureEntryPoint.kv_name is None:
34+
raise MissingConfig(self.__class__.__name__, ["VAULT_NAME"])
35+
if AzureEntryPoint.secret_name is None:
36+
raise MissingConfig(self.__class__.__name__, ["OPERATOR_KEY_SECRET_NAME"])
37+
if AzureEntryPoint.env_name is None:
38+
raise MissingConfig(self.__class__.__name__, ["DEPLOYMENT_ENVIRONMENT"])
39+
logging.info("Environment variables validation success")
40+
41+
def __create_final_config(self):
42+
TARGET_CONFIG = f"/app/conf/{AzureEntryPoint.env_name}-uid2-config.json"
43+
if not os.path.isfile(TARGET_CONFIG):
44+
logging.error(f"Unrecognized config {TARGET_CONFIG}")
45+
sys.exit(1)
46+
47+
logging.info(f"-- copying {TARGET_CONFIG} to {AzureEntryPoint.FINAL_CONFIG}")
48+
try:
49+
shutil.copy(TARGET_CONFIG, AzureEntryPoint.FINAL_CONFIG)
50+
except IOError as e:
51+
logging.error(f"Failed to create {AzureEntryPoint.FINAL_CONFIG} with error: {e}")
52+
sys.exit(1)
53+
54+
CORE_BASE_URL = os.getenv("CORE_BASE_URL")
55+
OPTOUT_BASE_URL = os.getenv("OPTOUT_BASE_URL")
56+
if CORE_BASE_URL and OPTOUT_BASE_URL and AzureEntryPoint.env_name != 'prod':
57+
logging.info(f"-- replacing URLs by {CORE_BASE_URL} and {OPTOUT_BASE_URL}")
58+
with open(AzureEntryPoint.FINAL_CONFIG, "r") as file:
59+
config = file.read()
60+
61+
config = config.replace("https://core-integ.uidapi.com", CORE_BASE_URL)
62+
config = config.replace("https://optout-integ.uidapi.com", OPTOUT_BASE_URL)
63+
64+
with open(AzureEntryPoint.FINAL_CONFIG, "w") as file:
65+
file.write(config)
66+
67+
with open(AzureEntryPoint.FINAL_CONFIG, "r") as file:
68+
logging.info(file.read())
69+
70+
def __set_base_urls(self):
71+
with open(AzureEntryPoint.FINAL_CONFIG, "r") as file:
72+
jdata = json.load(file)
73+
self.configs["core_base_url"] = jdata["core_attest_url"]
74+
self.configs["optout_base_url"] = jdata["optout_api_uri"]
75+
76+
def __set_api_token(self):
77+
try:
78+
credential = DefaultAzureCredential()
79+
kv_URL = f"https://{AzureEntryPoint.kv_name}.vault.azure.net"
80+
secret_client = SecretClient(vault_url=kv_URL, credential=credential)
81+
secret = secret_client.get_secret(AzureEntryPoint.secret_name)
82+
# print(f"Secret Value: {secret.value}")
83+
self.configs["api_token"] = secret.value
84+
85+
except (CredentialUnavailableError, ClientAuthenticationError) as auth_error:
86+
logging.error(f"Read operator key, authentication error: {auth_error}")
87+
raise SecretAccessDenied(self.__class__.__name__, str(auth_error))
88+
except ResourceNotFoundError as not_found_error:
89+
logging.error(f"Read operator key, secret not found: {AzureEntryPoint.secret_name}. Error: {not_found_error}")
90+
raise ApiTokenNotFound(self.__class__.__name__, str(not_found_error))
91+
92+
93+
def _set_confidential_config(self, secret_identifier: str = None):
94+
self.configs["skip_validations"] = os.getenv("SKIP_VALIDATIONS", "false").lower() == "true"
95+
self.configs["debug_mode"] = os.getenv("DEBUG_MODE", "false").lower() == "true"
96+
self.configs["environment"] = AzureEntryPoint.env_name
97+
98+
# set self.configs["api_token"]
99+
self.__set_api_token()
100+
# set base urls from final config file
101+
self.__set_base_urls()
102+
103+
def __run_operator(self):
104+
105+
# Start the operator
106+
os.environ["azure_vault_name"] = AzureEntryPoint.kv_name
107+
os.environ["azure_secret_name"] = AzureEntryPoint.secret_name
108+
109+
java_command = [
110+
"java",
111+
"-XX:MaxRAMPercentage=95", "-XX:-UseCompressedOops", "-XX:+PrintFlagsFinal",
112+
"-Djava.security.egd=file:/dev/./urandom",
113+
"-Dvertx.logger-delegate-factory-class-name=io.vertx.core.logging.SLF4JLogDelegateFactory",
114+
"-Dlogback.configurationFile=/app/conf/logback.xml",
115+
f"-Dvertx-config-path={AzureEntryPoint.FINAL_CONFIG}",
116+
"-jar",
117+
f"{AzureEntryPoint.jar_name}-{AzureEntryPoint.jar_version}.jar"
118+
]
119+
logging.info("-- starting java operator application")
120+
self.run_command(java_command, separate_process=False)
121+
122+
def _validate_auxiliaries(self):
123+
logging.info("Waiting for sidecar ...")
124+
125+
MAX_RETRIES = 15
126+
PING_URL = "http://169.254.169.254/ping"
127+
delay = 1
128+
129+
for attempt in range(1, MAX_RETRIES + 1):
130+
try:
131+
response = requests.get(PING_URL, timeout=5)
132+
if response.status_code in [200, 204]:
133+
logging.info("Sidecar started successfully.")
134+
return
135+
else:
136+
logging.warning(
137+
f"Attempt {attempt}: Unexpected status code {response.status_code}. Response: {response.text}"
138+
)
139+
except Exception as e:
140+
logging.info(f"Attempt {attempt}: Error during request - {e}")
141+
142+
if attempt == MAX_RETRIES:
143+
logging.error(
144+
f"Sidecar failed to start after {MAX_RETRIES} attempts. Exiting."
145+
)
146+
raise AuxiliariesException(self.__class__.__name__)
147+
148+
logging.info(f"Retrying in {delay} seconds... (Attempt {attempt}/{MAX_RETRIES})")
149+
time.sleep(delay)
150+
delay += 1
151+
152+
def run_compute(self) -> None:
153+
"""Main execution flow for confidential compute."""
154+
self.__check_env_variables()
155+
self.__create_final_config()
156+
self._set_confidential_config()
157+
if not self.configs.get("skip_validations"):
158+
self.validate_configuration()
159+
self._setup_auxiliaries()
160+
self.__run_operator()
161+
162+
def _setup_auxiliaries(self) -> None:
163+
""" setup auxiliary services are running."""
164+
pass
165+
166+
if __name__ == "__main__":
167+
168+
logging.basicConfig(level=logging.INFO)
169+
logging.info("Start AzureEntryPoint")
170+
try:
171+
operator = AzureEntryPoint()
172+
operator.run_compute()
173+
except ConfidentialComputeStartupException as e:
174+
logging.error(f"Failed starting up Azure Confidential Compute. Please checks the logs for errors and retry {e}", exc_info=True)
175+
except Exception as e:
176+
logging.error(f"Unexpected failure while starting up Azure Confidential Compute. Please contact UID support team with this log {e}", exc_info=True)

0 commit comments

Comments
 (0)