Make model-engine FIPS compliant by updating base chainguard image (#724)

* Update Dockerfile

* Update circleci config to login to chainguard

* fix typo in circleci config

* Add code to debug circleci errors

* Debug missing chainguard token

* Debug failing oidc token swap

* Update config

* Retry OIDC token swap with updated chainguard identity

* Update audience for token exchange request

* Simplify chainguard authentication with chainctl

* Specify audience cgr.dev in auth login

* Update system packages in Dockerfile

* Update Dockerfile packages for chainguard compatbility

* update Dockerfile

* Revert circleci python version to 3.10.14

* Update hardcoded model-engine image tag used in integration tests

* Fix CircleCI config trying to use hardcoded model-engine image tag for batch jobs pod

* Mount service_config_circleci.yaml in batch job pods

* Fix broken helm template

* Add missing infra config and service template config to batch job pods

* remove redundant config for batch job pods

* enable SHA256 checksums for Celery S3 backend to avoid MD5 decoding issues when creating endpoints

* Fix failing md5 monkey patch

* bump sqlalchemy to 2.0.21 to address md5 FIPS compliance

* Fix black linting errors

* wrap Dockerfile layers between root and nonroot user

* Remove the federal/ directory since Dockerfile is now FIPS compliant and doesn't require monkey patching

* set celery_enable_sha256 to true in all configs for FIPS compliance

* make changes backwards compatible by having separate Dockerfiles

* formatting

Author: ValentineDragan

.circleci/config.yml

@@ -105,6 +105,7 @@ jobs:
     executor: ubuntu-large
     steps:
       - checkout
+      - chainguard_login
       - run:
           name: Build Docker Image
           command: |
@@ -116,6 +117,7 @@ jobs:
       - aws-cli/setup:
           role-arn: ${CIRCLECI_ROLE_ARN}
           aws-region: AWS_REGION
+      - chainguard_login
       - run:
           name: Build Docker Image
           command: |
@@ -156,7 +158,11 @@ jobs:
       - run:
           name: Pre-load model-engine image to minikube
           command: |
+            # Load the base image for gateway/init containers
             minikube --logtostderr -v 1 image load model-engine:$CIRCLE_SHA1
+            # Tag and load with ECR prefix for batch job containers
+            docker tag model-engine:$CIRCLE_SHA1 $CIRCLECI_AWS_ACCOUNT_ID.dkr.ecr.us-west-2.amazonaws.com/model-engine:$CIRCLE_SHA1
+            minikube --logtostderr -v 1 image load $CIRCLECI_AWS_ACCOUNT_ID.dkr.ecr.us-west-2.amazonaws.com/model-engine:$CIRCLE_SHA1
       - run:
           name: Pre-load integration test images to minikube
           command: |
@@ -209,6 +215,20 @@ executors:
     resource_class: 2xlarge
 
 commands:
+  chainguard_login:
+    description: Authenticate to Chainguard Registry via OIDC
+    steps:
+      - run:
+          name: Install chainctl
+          command: |
+            curl -o chainctl "https://dl.enforce.dev/chainctl/latest/chainctl_$(uname -s | tr '[:upper:]' '[:lower:]')_$(uname -m | sed 's/aarch64/arm64/')"
+            sudo install -o $UID -g $(id -g) -m 0755 chainctl /usr/local/bin/
+      - run:
+          name: Login to Chainguard Registry
+          command: |
+            chainctl auth login --identity-token "${CIRCLE_OIDC_TOKEN}" --identity "${CHAINGUARD_IDENTITY_ID}" --audience cgr.dev
+            CHAINGUARD_TOKEN=$(chainctl auth token --audience cgr.dev)
+            echo "${CHAINGUARD_TOKEN}" | docker login cgr.dev -u "oauth2accesstoken" --password-stdin
   environment_setup:
     description: Basic Environment setup
     steps:

charts/model-engine/templates/service_template_config_map.yaml

@@ -1025,12 +1025,26 @@ data:
             {{- toYaml . | nindent 12 }}
           {{- end }}
           serviceAccountName: {{ $launch_name }}
-          {{- if $require_aws_config }}
           volumes:
+            {{- if $require_aws_config }}
             - name: config-volume
               configMap:
                 name: {{ $aws_config_map_name }}
-          {{- end }}
+            {{- end }}
+            {{- if $config_values }}
+            - name: service-config-volume
+              configMap:
+                name: {{ $launch_name }}-service-config
+                items:
+                  - key: launch_service_config
+                    path: service_config.yaml
+            - name: infra-service-config-volume
+              configMap:
+                name: {{ $launch_name }}-service-config
+                items:
+                  - key: infra_service_config
+                    path: config.yaml
+            {{- end }}
           containers:
             - name: main
               image: {{ $gateway_repository }}:${GIT_TAG}
@@ -1077,12 +1091,18 @@ data:
                   cpu: 4
                   memory: 32Gi
                   ephemeral-storage: 30Gi
-              {{- if $require_aws_config }}
               volumeMounts:
+                {{- if $require_aws_config }}
                 - name: config-volume
                   mountPath: /opt/.aws/config
                   subPath: config
-              {{- end }}
+                {{- end }}
+                {{- if $config_values }}
+                - name: service-config-volume
+                  mountPath: /workspace/model-engine/service_configs
+                - name: infra-service-config-volume
+                  mountPath: /workspace/model-engine/model_engine_server/core/configs
+                {{- end }}
   {{- range $device := tuple "cpu" "gpu" }}
   docker-image-batch-job-{{- $device }}.yaml: |-
     apiVersion: batch/v1
@@ -1134,6 +1154,14 @@ data:
               configMap:
                 name: {{ $aws_config_map_name }}
             {{- end }}
+            {{- if $config_values }}
+            - name: service-config-volume
+              configMap:
+                name: {{ $launch_name }}-service-config
+                items:
+                  - key: launch_service_config
+                    path: service_config.yaml
+            {{- end }}
             - name: workdir
               emptyDir: {}
             - name: dshm
@@ -1178,6 +1206,10 @@ data:
                   mountPath: /opt/.aws/config
                   subPath: config
                 {{- end }}
+                {{- if $config_values }}
+                - name: service-config-volume
+                  mountPath: /workspace/model-engine/service_configs
+                {{- end }}
                 - name: workdir
                   mountPath: ${MOUNT_PATH}
                 - mountPath: /dev/shm
@@ -1212,6 +1244,10 @@ data:
                   mountPath: /opt/.aws/config
                   subPath: config
                 {{- end }}
+                {{- if $config_values }}
+                - name: service-config-volume
+                  mountPath: /workspace/model-engine/service_configs
+                {{- end }}
                 - name: workdir
                   mountPath: ${MOUNT_PATH}
   {{- end }}

federal/sitecustomize.py

@@ -78,7 +78,7 @@ def my_model(**keyword_args):
     "flavor": {
         "flavor": "streaming_enhanced_runnable_image",
         "repository": "model-engine",
-        "tag": "830c81ecba2a147022e504917c6ce18b00c2af44",
+        "tag": os.environ.get("GIT_TAG"),
         "command": [
             "dumb-init",
             "--",
@@ -269,7 +269,7 @@ def my_model(**keyword_args):
 CREATE_DOCKER_IMAGE_BATCH_JOB_BUNDLE_REQUEST: Dict[str, Any] = {
     "name": format_name("di_batch_job_bundle_1"),
     "image_repository": "model-engine",
-    "image_tag": "830c81ecba2a147022e504917c6ce18b00c2af44",
+    "image_tag": os.environ.get("GIT_TAG"),
     "command": ["jq", ".", "/launch_mount_location/file"],
     "env": {"ENV1": "VAL1"},
     "mount_location": "/launch_mount_location/file",
@@ -289,7 +289,7 @@ def my_model(**keyword_args):
 CREATE_FINE_TUNE_DI_BATCH_JOB_BUNDLE_REQUEST: Dict[str, Any] = {
     "name": format_name("fine_tune_di_batch_job_bundle_1"),
     "image_repository": "model-engine",
-    "image_tag": "830c81ecba2a147022e504917c6ce18b00c2af44",
+    "image_tag": os.environ.get("GIT_TAG"),
     "command": ["cat", "/launch_mount_location/file"],
     "env": {"ENV1": "VAL1"},
     "mount_location": "/launch_mount_location/file",

integration_tests/rest_api_utils.py

@@ -1,9 +1,9 @@
-# federal/Dockerfile.chainguard
-FROM cgr.dev/scale.com/python-fips:3.10.15-dev
+FROM cgr.dev/scale.com/python-fips:3.10.19-dev
 WORKDIR /workspace
 USER root
 
-RUN apk update && apk add htop \
+RUN apk update && apk add \
+            htop \
             dumb-init \
             libssh \
             openssh-client \
@@ -13,7 +13,16 @@ RUN apk update && apk add htop \
             procps \
             libcurl-openssl4 \
             vim \
-            kubectl
+            kubectl \
+            jq \
+            gcc \
+            glibc-dev \
+            python-3.10-dev \
+            libffi-dev \
+            openssl-dev \
+            build-base \
+            postgresql-dev \
+            libpq-16
 
 RUN curl -Lo /bin/aws-iam-authenticator https://github.com/kubernetes-sigs/aws-iam-authenticator/releases/download/v0.5.9/aws-iam-authenticator_0.5.9_linux_amd64
 RUN chmod +x /bin/aws-iam-authenticator
@@ -23,8 +32,6 @@ RUN chmod -R 777 /workspace
 
 RUN pip install awscli==1.34.28 --no-cache-dir
 
-COPY federal/sitecustomize.py /usr/lib/python3.10/site-packages/sitecustomize.py
-
 WORKDIR /workspace/model-engine/
 COPY model-engine/requirements-test.txt /workspace/model-engine/requirements-test.txt
 COPY model-engine/requirements.txt /workspace/model-engine/requirements.txt
@@ -39,9 +46,9 @@ RUN pip install -e .
 COPY integration_tests /workspace/integration_tests
 
 WORKDIR /workspace
-ENV PYTHONPATH /workspace
-ENV WORKSPACE /workspace
+ENV PYTHONPATH=/workspace
+ENV WORKSPACE=/workspace
 
 USER nonroot
 
-EXPOSE 5000
+EXPOSE 5000

federal/Dockerfile.chainguard renamed to model-engine/Dockerfile.fips

@@ -61,7 +61,10 @@ def generate_destination(user_id: str, endpoint_name: str, endpoint_type: str) -
 
 
 def _generate_deployment_name_parts(user_id: str, endpoint_name: str) -> List[str]:
-    user_endpoint_hash = hashlib.md5((user_id + endpoint_name).encode("utf-8")).hexdigest()
+    # Use MD5 for deployment name hashing (non-security purpose) - FIPS compliant
+    user_endpoint_hash = hashlib.new(
+        "md5", (user_id + endpoint_name).encode("utf-8"), usedforsecurity=False
+    ).hexdigest()
     return [
         DEPLOYMENT_PREFIX,
         user_id[:24],

model-engine/model_engine_server/common/settings.py

@@ -69,7 +69,10 @@ class CeleryAutoscalerParams:
 
 
 def _hash_any_to_int(data: Any):
-    return int(hashlib.md5(str(data).encode()).hexdigest(), 16)  # nosemgrep
+    # Use MD5 for hashing (non-security purpose) - FIPS compliant with usedforsecurity=False
+    return int(
+        hashlib.new("md5", str(data).encode(), usedforsecurity=False).hexdigest(), 16
+    )  # nosemgrep
 
 
 async def list_deployments(core_api, apps_api) -> Dict[Tuple[str, str], CeleryAutoscalerParams]:

model-engine/model_engine_server/core/celery/celery_autoscaler.py

@@ -599,7 +599,10 @@ def _get_inject_bundle_image_params(
         bundle_id = model_bundle.id
         service_image_str = "-".join([base_image_params.image_tag, GIT_TAG, bundle_id])
         # nosemgrep
-        service_image_hash = hashlib.md5(str(service_image_str).encode("utf-8")).hexdigest()
+        # Use MD5 for image tag hashing (non-security purpose, required for Docker compatibility)
+        service_image_hash = hashlib.new(
+            "md5", str(service_image_str).encode("utf-8"), usedforsecurity=False
+        ).hexdigest()
         service_image_tag = f"inject-bundle-image-{service_image_hash}"
         ecr_repo = base_image_params.repo
 
@@ -812,7 +815,12 @@ def _get_restricted_env_vars(env_vars: Dict[str, str]) -> Set[str]:
     def _get_requirements_hash(requirements: List[str]) -> str:
         """Identifying hash for endpoint's Python requirements."""
         # nosemgrep
-        return hashlib.md5("\n".join(sorted(requirements)).encode("utf-8")).hexdigest()[:6]
+        # Use MD5 for requirements hashing (non-security purpose)
+        return hashlib.new(
+            "md5",
+            "\n".join(sorted(requirements)).encode("utf-8"),
+            usedforsecurity=False,
+        ).hexdigest()[:6]
 
     @staticmethod
     def _get_image_tag(base_image_tag: str, git_tag: str, requirements_hash: str) -> str:

model-engine/model_engine_server/infra/services/live_endpoint_builder_service.py

@@ -46,7 +46,7 @@ rich~=12.6
 sentencepiece==0.1.99
 sh~=1.13
 smart-open~=5.2
-sqlalchemy[asyncio]~=2.0.4
+sqlalchemy[asyncio]~=2.0.21
 sse-starlette==1.6.1
 sseclient-py==1.7.2
 starlette[full]>=0.36.2 # not used directly, but needs to be pinned for Microsoft security scan

model-engine/requirements.in

@@ -458,7 +458,7 @@ sniffio==1.3.0
     # via
     #   anyio
     #   httpx
-sqlalchemy[asyncio]==2.0.4
+sqlalchemy[asyncio]==2.0.21
     # via
     #   -r model-engine/requirements.in
     #   alembic