Add actual pytests to integration tests (#227)

Author: phil-scale

.circleci/config.yml

@@ -138,12 +138,48 @@ jobs:
           name: Pre-load model-engine image to minikube
           command: |
             minikube --logtostderr -v 1 image load model-engine:$CIRCLE_SHA1
+      - run:
+          name: Pre-load integration test images to minikube
+          command: |
+            docker build -f model-engine/model_engine_server/inference/pytorch_or_tf.base.Dockerfile \
+            --build-arg BASE_IMAGE=pytorch/pytorch:1.7.1-cuda11.0-cudnn8-runtime \
+            --build-arg REQUIREMENTS_FILE="$CIRCLE_SHA1-base-requirements.txt" \
+            -t temp:1.7.1-cuda11.0-cudnn8-runtime-$CIRCLE_SHA1 .
+
+            touch $CIRCLE_SHA1-requirements.txt
+            echo -e "cloudpickle==2.1.0\npyyaml==6.0" > $CIRCLE_SHA1-requirements.txt
+
+            DOCKER_BUILDKIT=1 docker build -f model-engine/model_engine_server/inference/pytorch_or_tf.user.Dockerfile \
+            --build-arg BASE_IMAGE=temp:1.7.1-cuda11.0-cudnn8-runtime-$CIRCLE_SHA1 \
+            --build-arg REQUIREMENTS_FILE="$CIRCLE_SHA1-requirements.txt" \
+            -t $CIRCLECI_AWS_ACCOUNT_ID.dkr.ecr.us-west-2.amazonaws.com/hosted-model-inference/async-pytorch:1.7.1-cuda11.0-cudnn8-runtime-$CIRCLE_SHA1-021694 .
+            rm $CIRCLE_SHA1-requirements.txt
+
+            minikube --logtostderr -v 1 image load $CIRCLECI_AWS_ACCOUNT_ID.dkr.ecr.us-west-2.amazonaws.com/hosted-model-inference/async-pytorch:1.7.1-cuda11.0-cudnn8-runtime-$CIRCLE_SHA1-021694
       - run:
           name: Install helm chart
           command: |
             pushd $HOME/project/charts
             cat model-engine/values_circleci.yaml | envsubst > model-engine/values_circleci_subst.yaml
             helm install model-engine model-engine --values model-engine/values_circleci_subst.yaml --set tag=$CIRCLE_SHA1 --atomic --debug
+      - run:
+          name: Change python version to 3.8.12
+          command: |
+            pyenv install 3.8.12
+            pyenv global 3.8.12
+      - run:
+          name: Install integration test dependencies
+          command: |
+            sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev libssl-dev python3-dev
+            pip install -r model-engine/requirements.txt
+      - install_client
+      - install_server
+      - run:
+          name: Run integration tests
+          command: |
+            pushd $HOME/project
+            kubectl port-forward svc/model-engine 5001:80 &
+            GIT_TAG=$CIRCLE_SHA1 pytest integration_tests
 
 executors:
   ubuntu-large:
@@ -188,6 +224,7 @@ commands:
       - run:
           name: Install LLM Engine client
           command: |
+            pip install --upgrade pip
             pip install -e $HOME/project/clients/python
   run_unit_tests_python_client:
     description: Unit tests of the python client

charts/model-engine/templates/_helpers.tpl

@@ -281,6 +281,10 @@ env:
   - name: REDIS_AUTH_TOKEN
     value: {{ .Values.redis.auth }}
   {{- end }}
+  {{- if eq .Values.context "circleci" }}
+  - name: CIRCLECI
+    value: "true"
+  {{- end }}
 {{- end }}
 
 {{- define "modelEngine.serviceEnvGitTagFromHelmVar" }}

charts/model-engine/templates/service_config_map.yaml

@@ -23,4 +23,32 @@ data:
     {{ $key }}: {{ $value | quote }}
     {{- end }}
     {{- end }}
+
+---
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "modelEngine.fullname" . }}-service-config
+  namespace: {{ .Values.config.values.launch.endpoint_namespace }}
+  labels:
+    {{- include "modelEngine.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": pre-install,pre-upgrade
+    "helm.sh/hook-weight": "-2"
+data:
+  launch_service_config: |-
+    datadog_trace_enabled: {{ .Values.datadog_trace_enabled | default false | quote }}
+    {{- with .Values.config.values.launch }}
+    {{- range $key, $value := . }}
+    {{ $key }}: {{ $value | quote }}
+    {{- end }}
+    {{- end }}
+  infra_service_config: |-
+    env: {{ .Values.context | quote }}
+    {{- with .Values.config.values.infra }}
+    {{- range $key, $value := . }}
+    {{ $key }}: {{ $value | quote }}
+    {{- end }}
+    {{- end }}
 {{- end }}

charts/model-engine/templates/service_template_config_map.yaml

@@ -143,7 +143,7 @@ data:
                   name: http
             {{- else if eq $mode "streaming" }}
             - name: http-forwarder
-              image: {{ $forwarder_repository }}:{{ $tag }}
+              image: {{ $forwarder_repository }}:${GIT_TAG}
               imagePullPolicy: IfNotPresent
               command:
                 - /usr/bin/dumb-init

charts/model-engine/values_circleci.yaml

@@ -82,7 +82,7 @@ config:
       dns_host_domain: localhost
       default_region: us-west-2
       ml_account_id: "$CIRCLECI_AWS_ACCOUNT_ID"
-      docker_repo_prefix: "CIRCLECI_AWS_ACCOUNT_ID.dkr.ecr.us-west-2.amazonaws.com"
+      docker_repo_prefix: "$CIRCLECI_AWS_ACCOUNT_ID.dkr.ecr.us-west-2.amazonaws.com"
       redis_host: redis-message-broker-master.default
       s3_bucket: "$CIRCLECI_AWS_S3_BUCKET"
       profile_ml_worker: "default"