Make model-engine FIPS compliant by updating base chainguard image

.circleci/config.yml

@@ -105,6 +105,7 @@ jobs:
     executor: ubuntu-large
     steps:
       - checkout
+      - chainguard_login
       - run:
           name: Build Docker Image
           command: |
@@ -116,6 +117,7 @@ jobs:
       - aws-cli/setup:
           role-arn: ${CIRCLECI_ROLE_ARN}
           aws-region: AWS_REGION
+      - chainguard_login
       - run:
           name: Build Docker Image
           command: |
@@ -156,7 +158,11 @@ jobs:
       - run:
           name: Pre-load model-engine image to minikube
           command: |
+            # Load the base image for gateway/init containers
             minikube --logtostderr -v 1 image load model-engine:$CIRCLE_SHA1
+            # Tag and load with ECR prefix for batch job containers
+            docker tag model-engine:$CIRCLE_SHA1 $CIRCLECI_AWS_ACCOUNT_ID.dkr.ecr.us-west-2.amazonaws.com/model-engine:$CIRCLE_SHA1
+            minikube --logtostderr -v 1 image load $CIRCLECI_AWS_ACCOUNT_ID.dkr.ecr.us-west-2.amazonaws.com/model-engine:$CIRCLE_SHA1
       - run:
           name: Pre-load integration test images to minikube
           command: |
@@ -209,6 +215,20 @@ executors:
     resource_class: 2xlarge
 
 commands:
+  chainguard_login:
+    description: Authenticate to Chainguard Registry via OIDC
+    steps:
+      - run:
+          name: Install chainctl
+          command: |
+            curl -o chainctl "https://dl.enforce.dev/chainctl/latest/chainctl_$(uname -s | tr '[:upper:]' '[:lower:]')_$(uname -m | sed 's/aarch64/arm64/')"
+            sudo install -o $UID -g $(id -g) -m 0755 chainctl /usr/local/bin/
+      - run:
+          name: Login to Chainguard Registry
+          command: |
+            chainctl auth login --identity-token "${CIRCLE_OIDC_TOKEN}" --identity "${CHAINGUARD_IDENTITY_ID}" --audience cgr.dev
+            CHAINGUARD_TOKEN=$(chainctl auth token --audience cgr.dev)
+            echo "${CHAINGUARD_TOKEN}" | docker login cgr.dev -u "oauth2accesstoken" --password-stdin
   environment_setup:
     description: Basic Environment setup
     steps:

charts/model-engine/templates/service_template_config_map.yaml

@@ -1025,12 +1025,26 @@ data:
             {{- toYaml . | nindent 12 }}
           {{- end }}
           serviceAccountName: {{ $launch_name }}
-          {{- if $require_aws_config }}
           volumes:
+            {{- if $require_aws_config }}
             - name: config-volume
               configMap:
                 name: {{ $aws_config_map_name }}
-          {{- end }}
+            {{- end }}
+            {{- if $config_values }}
+            - name: service-config-volume
+              configMap:
+                name: {{ $launch_name }}-service-config
+                items:
+                  - key: launch_service_config
+                    path: service_config.yaml
+            - name: infra-service-config-volume
+              configMap:
+                name: {{ $launch_name }}-service-config
+                items:
+                  - key: infra_service_config
+                    path: config.yaml
+            {{- end }}
           containers:
             - name: main
               image: {{ $gateway_repository }}:${GIT_TAG}
@@ -1077,12 +1091,18 @@ data:
                   cpu: 4
                   memory: 32Gi
                   ephemeral-storage: 30Gi
-              {{- if $require_aws_config }}
               volumeMounts:
+                {{- if $require_aws_config }}
                 - name: config-volume
                   mountPath: /opt/.aws/config
                   subPath: config
-              {{- end }}
+                {{- end }}
+                {{- if $config_values }}
+                - name: service-config-volume
+                  mountPath: /workspace/model-engine/service_configs
+                - name: infra-service-config-volume
+                  mountPath: /workspace/model-engine/model_engine_server/core/configs
+                {{- end }}
   {{- range $device := tuple "cpu" "gpu" }}
   docker-image-batch-job-{{- $device }}.yaml: |-
     apiVersion: batch/v1
@@ -1134,6 +1154,14 @@ data:
               configMap:
                 name: {{ $aws_config_map_name }}
             {{- end }}
+            {{- if $config_values }}
+            - name: service-config-volume
+              configMap:
+                name: {{ $launch_name }}-service-config
+                items:
+                  - key: launch_service_config
+                    path: service_config.yaml
+            {{- end }}
             - name: workdir
               emptyDir: {}
             - name: dshm
@@ -1178,6 +1206,10 @@ data:
                   mountPath: /opt/.aws/config
                   subPath: config
                 {{- end }}
+                {{- if $config_values }}
+                - name: service-config-volume
+                  mountPath: /workspace/model-engine/service_configs
+                {{- end }}
                 - name: workdir
                   mountPath: ${MOUNT_PATH}
                 - mountPath: /dev/shm
@@ -1212,6 +1244,10 @@ data:
                   mountPath: /opt/.aws/config
                   subPath: config
                 {{- end }}
+                {{- if $config_values }}
+                - name: service-config-volume
+                  mountPath: /workspace/model-engine/service_configs
+                {{- end }}
                 - name: workdir
                   mountPath: ${MOUNT_PATH}
   {{- end }}

charts/model-engine/values.yaml

@@ -3,7 +3,7 @@ spellbook:
   enabled: false
 
 # celery_enable_sha256 [optional] uses SHA256 hashes for federal compliance mode (FIPS, enhanced security)
-celery_enable_sha256: null
+celery_enable_sha256: true
 
 # debug_mode [optional] enables detailed debug logging for infrastructure components
 debug_mode: null

charts/model-engine/values_circleci.yaml

@@ -1,6 +1,6 @@
 # This is a YAML-formatted file.
 
-celery_enable_sha256: null
+celery_enable_sha256: true
 celery_broker_type_redis: null
 debug_mode: null
 
@@ -39,6 +39,7 @@ balloons:
 # commit from which the image was built.
 # tag:
 context: circleci
+
 image:
   gatewayRepository: model-engine
   builderRepository: model-engine

charts/model-engine/values_sample.yaml

@@ -1,7 +1,7 @@
 # This is a YAML-formatted file.
 
 # celery_enable_sha256 [optional] uses SHA256 hashes for federal compliance mode (FIPS, enhanced security)
-celery_enable_sha256: null
+celery_enable_sha256: true
 
 # debug_mode [optional] enables detailed debug logging for infrastructure components  
 debug_mode: null

integration_tests/rest_api_utils.py

@@ -78,7 +78,7 @@ def my_model(**keyword_args):
     "flavor": {
         "flavor": "streaming_enhanced_runnable_image",
         "repository": "model-engine",
-        "tag": "830c81ecba2a147022e504917c6ce18b00c2af44",
+        "tag": os.environ.get("GIT_TAG"),
         "command": [
             "dumb-init",
             "--",
@@ -269,7 +269,7 @@ def my_model(**keyword_args):
 CREATE_DOCKER_IMAGE_BATCH_JOB_BUNDLE_REQUEST: Dict[str, Any] = {
     "name": format_name("di_batch_job_bundle_1"),
     "image_repository": "model-engine",
-    "image_tag": "830c81ecba2a147022e504917c6ce18b00c2af44",
+    "image_tag": os.environ.get("GIT_TAG"),
     "command": ["jq", ".", "/launch_mount_location/file"],
     "env": {"ENV1": "VAL1"},
     "mount_location": "/launch_mount_location/file",
@@ -289,7 +289,7 @@ def my_model(**keyword_args):
 CREATE_FINE_TUNE_DI_BATCH_JOB_BUNDLE_REQUEST: Dict[str, Any] = {
     "name": format_name("fine_tune_di_batch_job_bundle_1"),
     "image_repository": "model-engine",
-    "image_tag": "830c81ecba2a147022e504917c6ce18b00c2af44",
+    "image_tag": os.environ.get("GIT_TAG"),
     "command": ["cat", "/launch_mount_location/file"],
     "env": {"ENV1": "VAL1"},
     "mount_location": "/launch_mount_location/file",

model-engine/Dockerfile

@@ -1,42 +1,37 @@
-# syntax = docker/dockerfile:experimental
-
-FROM python:3.10.15-slim as model-engine
+FROM cgr.dev/scale.com/python-fips:3.10.19-dev
 WORKDIR /workspace
-
-RUN apt-get update && apt-get install -y \
-  apt-utils \
-  dumb-init \
-  git \
-  ssh \
-  htop \
-  iftop \
-  vim \
-  curl \
-  procps \
-  libcurl4-openssl-dev \
-  libssl-dev \
-  python3-dev \
-  gcc \
-  build-essential \
-  telnet \
-  && rm -rf /var/lib/apt/lists/*
+USER root
+
+RUN apk update && apk add \
+            htop \
+            dumb-init \
+            libssh \
+            openssh-client \
+            iftop \
+            curl \
+            curl-dev \
+            procps \
+            libcurl-openssl4 \
+            vim \
+            kubectl \
+            jq \
+            gcc \
+            glibc-dev \
+            python-3.10-dev \
+            libffi-dev \
+            openssl-dev \
+            build-base \
+            postgresql-dev \
+            libpq-16
 
 RUN curl -Lo /bin/aws-iam-authenticator https://github.com/kubernetes-sigs/aws-iam-authenticator/releases/download/v0.5.9/aws-iam-authenticator_0.5.9_linux_amd64
 RUN chmod +x /bin/aws-iam-authenticator
 
-# Install kubectl
-RUN curl -LO "https://dl.k8s.io/release/v1.23.13/bin/linux/amd64/kubectl" \
-  && chmod +x kubectl \
-  && mv kubectl /usr/local/bin/kubectl
-
-# Pin pip version
 RUN pip install pip==24.2
 RUN chmod -R 777 /workspace
 
-# Install AWS CLI
 RUN pip install awscli==1.34.28 --no-cache-dir
 
-## grab model_engine_server project (w/ requirements install layer caching)
 WORKDIR /workspace/model-engine/
 COPY model-engine/requirements-test.txt /workspace/model-engine/requirements-test.txt
 COPY model-engine/requirements.txt /workspace/model-engine/requirements.txt
@@ -51,7 +46,9 @@ RUN pip install -e .
 COPY integration_tests /workspace/integration_tests
 
 WORKDIR /workspace
-ENV PYTHONPATH /workspace
-ENV WORKSPACE /workspace
+ENV PYTHONPATH=/workspace
+ENV WORKSPACE=/workspace
+
+USER nonroot
 
 EXPOSE 5000

model-engine/model_engine_server/common/settings.py

@@ -61,7 +61,10 @@ def generate_destination(user_id: str, endpoint_name: str, endpoint_type: str) -
 
 
 def _generate_deployment_name_parts(user_id: str, endpoint_name: str) -> List[str]:
-    user_endpoint_hash = hashlib.md5((user_id + endpoint_name).encode("utf-8")).hexdigest()
+    # Use MD5 for deployment name hashing (non-security purpose) - FIPS compliant
+    user_endpoint_hash = hashlib.new(
+        "md5", (user_id + endpoint_name).encode("utf-8"), usedforsecurity=False
+    ).hexdigest()
     return [
         DEPLOYMENT_PREFIX,
         user_id[:24],

model-engine/model_engine_server/core/celery/celery_autoscaler.py

@@ -69,7 +69,10 @@ class CeleryAutoscalerParams:
 
 
 def _hash_any_to_int(data: Any):
-    return int(hashlib.md5(str(data).encode()).hexdigest(), 16)  # nosemgrep
+    # Use MD5 for hashing (non-security purpose) - FIPS compliant with usedforsecurity=False
+    return int(
+        hashlib.new("md5", str(data).encode(), usedforsecurity=False).hexdigest(), 16
+    )  # nosemgrep
 
 
 async def list_deployments(core_api, apps_api) -> Dict[Tuple[str, str], CeleryAutoscalerParams]:

model-engine/model_engine_server/infra/services/live_endpoint_builder_service.py

@@ -599,7 +599,10 @@ def _get_inject_bundle_image_params(
         bundle_id = model_bundle.id
         service_image_str = "-".join([base_image_params.image_tag, GIT_TAG, bundle_id])
         # nosemgrep
-        service_image_hash = hashlib.md5(str(service_image_str).encode("utf-8")).hexdigest()
+        # Use MD5 for image tag hashing (non-security purpose, required for Docker compatibility)
+        service_image_hash = hashlib.new(
+            "md5", str(service_image_str).encode("utf-8"), usedforsecurity=False
+        ).hexdigest()
         service_image_tag = f"inject-bundle-image-{service_image_hash}"
         ecr_repo = base_image_params.repo
 
@@ -812,7 +815,12 @@ def _get_restricted_env_vars(env_vars: Dict[str, str]) -> Set[str]:
     def _get_requirements_hash(requirements: List[str]) -> str:
         """Identifying hash for endpoint's Python requirements."""
         # nosemgrep
-        return hashlib.md5("\n".join(sorted(requirements)).encode("utf-8")).hexdigest()[:6]
+        # Use MD5 for requirements hashing (non-security purpose)
+        return hashlib.new(
+            "md5",
+            "\n".join(sorted(requirements)).encode("utf-8"),
+            usedforsecurity=False,
+        ).hexdigest()[:6]
 
     @staticmethod
     def _get_image_tag(base_image_tag: str, git_tag: str, requirements_hash: str) -> str: