CircleCI + Performance Regression

.circleci/README.md

@@ -16,6 +16,7 @@ Following _jobs_ are executed under each workflow -
 3. **python-tests** : Executes pytests from _mms/tests/unit_tests/_
 4. **benchmark** : Executes latency benchmark using resnet-18 model
 5. (NEW!) **api-tests** : Executes newman test suite for API testing
+6. (NEW!) **performance-regression** : Executes performance regression suite
 
 Following _executors_ are available for job execution -
 1. py27

.circleci/config.yml

@@ -40,12 +40,7 @@ commands:
       - run:
           name: Start MMS, Execute << parameters.collection >> API Tests, Stop MMS
           command: .circleci/scripts/linux_test_api.sh << parameters.collection >>
-      - store_artifacts:
-          name: Store server logs from << parameters.collection >> API tests
-          path: mms_<< parameters.collection >>.log
-      - store_artifacts:
-          name: Store << parameters.collection >> API test results
-          path: test/<< parameters.collection >>-api-report.html
+          when: always
 
 
 jobs:
@@ -57,7 +52,7 @@ jobs:
       steps:
         - checkout
         - run:
-            name: Build frontend
+            name: Build MMS
             command: .circleci/scripts/linux_build.sh
         - store_artifacts:
             name: Store gradle testng results
@@ -75,11 +70,12 @@ jobs:
       steps:
         - attach-mms-workspace
         - run:
-            name: Execute python unit tests
+            name: Execute python lint and unit tests
             command: .circleci/scripts/linux_test_python.sh
         - store_artifacts:
             name: Store python Test results
-            path: htmlcov
+            path: result_units
+            destination: units
 
     api-tests:
       parameters:
@@ -95,6 +91,9 @@ jobs:
             collection: inference
         - exeucute-api-tests:
             collection: https
+        - store_artifacts:
+            name: Store server logs and test results
+            path: tests/api/artifacts/
 
     benchmark:
       parameters:
@@ -109,11 +108,31 @@ jobs:
             command: .circleci/scripts/linux_test_benchmark.sh
         - store_artifacts:
             name: Store server logs from benchmark tests
-            path: mms.log
+            path: logs/
         - store_artifacts:
             name: Store Benchmark Latency resnet-18 results
-            path: /tmp/MMSBenchmark/out/latency/resnet-18/report/
-            destination: benchmark-latency-resnet-18
+            path: /tmp/MMSBenchmark
+
+    performance-regression:
+      parameters:
+        executor:
+          type: executor
+      executor: << parameters.executor >>
+      steps:
+        - attach-mms-workspace
+        - install-mms-server
+        - run:
+            name: Start MMS, Execute performance regression testcases, Stop MMS
+            command: .circleci/scripts/linux_test_perf_regression.sh
+        - store_artifacts:
+            name: Store server logs
+            path: tests/performance/logs/
+        - store_artifacts:
+            name: Store artifacts from performance regression run
+            path: tests/performance/run_artifacts/
+        - store_test_results:
+            name: Store performance regression test results for CircleCI
+            path: tests/performance/run_artifacts/report/
 
     modelarchiver:
       parameters:
@@ -123,11 +142,11 @@ jobs:
       steps:
         - checkout
         - run:
-            name: Execute lint, unit and integration tests
+            name: Execute python lint, unit and integration tests
             command: .circleci/scripts/linux_test_modelarchiver.sh
         - store_artifacts:
             name: Store unit tests results from model archiver tests
-            path: model-archiver/results_units
+            path: model-archiver/result_units
             destination: units
 
 
@@ -187,3 +206,8 @@ workflows:
           requires:
             - build-<< matrix.executor >>
           matrix: *matrix
+      - performance-regression:
+          name: performance-regression-<< matrix.executor >>
+          requires:
+            - build-<< matrix.executor >>
+          matrix: *matrix

.circleci/scripts/linux_test_api.sh

@@ -1,21 +1,23 @@
 #!/bin/bash
 
-MODEL_STORE_DIR='test/model_store'
+BASE_DIR="tests/api"
+MODEL_STORE_DIR="model_store"
 
-MMS_LOG_FILE_MANAGEMENT='mms_management.log'
-MMS_LOG_FILE_INFERENCE='mms_inference.log'
-MMS_LOG_FILE_HTTPS='mms_https.log'
-MMS_CONFIG_FILE_HTTPS='test/resources/config.properties'
+ARTIFACTS_BASE_DIR="artifacts"
+ARTIFACTS_MANAGEMENT_DIR="$ARTIFACTS_BASE_DIR/management"
+ARTIFACTS_INFERENCE_DIR="$ARTIFACTS_BASE_DIR/inference"
+ARTIFACTS_HTTPS_DIR="$ARTIFACTS_BASE_DIR/https"
 
-POSTMAN_ENV_FILE='test/postman/environment.json'
-POSTMAN_COLLECTION_MANAGEMENT='test/postman/management_api_test_collection.json'
-POSTMAN_COLLECTION_INFERENCE='test/postman/inference_api_test_collection.json'
-POSTMAN_COLLECTION_HTTPS='test/postman/https_test_collection.json'
-POSTMAN_DATA_FILE_INFERENCE='test/postman/inference_data.json'
+MMS_CONSOLE_LOG_FILE="mms_console.log"
+MMS_CONFIG_FILE_HTTPS="resources/config.properties"
 
-REPORT_FILE_MANAGEMENT='test/management-api-report.html'
-REPORT_FILE_INFERENCE='test/inference-api-report.html'
-REPORT_FILE_HTTPS='test/https-api-report.html'
+POSTMAN_ENV_FILE="postman/environment.json"
+POSTMAN_COLLECTION_MANAGEMENT="postman/management_api_test_collection.json"
+POSTMAN_COLLECTION_INFERENCE="postman/inference_api_test_collection.json"
+POSTMAN_COLLECTION_HTTPS="postman/https_test_collection.json"
+POSTMAN_DATA_FILE_INFERENCE="postman/inference_data.json"
+
+REPORT_FILE="report.html"
 
 start_mms_server() {
   multi-model-server --start --model-store $1 >> $2 2>&1
@@ -29,45 +31,78 @@ start_mms_secure_server() {
 
 stop_mms_server() {
   multi-model-server --stop
+  sleep 10
+}
+
+cleanup_model_store(){
+  rm -rf $MODEL_STORE_DIR/*
+}
+
+move_logs(){
+  mv $1 logs/
+  mv logs/ $2
 }
 
 trigger_management_tests(){
-  start_mms_server $MODEL_STORE_DIR $MMS_LOG_FILE_MANAGEMENT
+  start_mms_server $MODEL_STORE_DIR $MMS_CONSOLE_LOG_FILE
   newman run -e $POSTMAN_ENV_FILE $POSTMAN_COLLECTION_MANAGEMENT \
-             -r cli,html --reporter-html-export $REPORT_FILE_MANAGEMENT --verbose
+             -r cli,html --reporter-html-export $ARTIFACTS_MANAGEMENT_DIR/$REPORT_FILE --verbose
+  local EXIT_CODE=$?
   stop_mms_server
+  move_logs $MMS_CONSOLE_LOG_FILE $ARTIFACTS_MANAGEMENT_DIR
+  cleanup_model_store
+  return $EXIT_CODE
 }
 
 trigger_inference_tests(){
-  start_mms_server $MODEL_STORE_DIR $MMS_LOG_FILE_INFERENCE
+  start_mms_server $MODEL_STORE_DIR $MMS_CONSOLE_LOG_FILE
   newman run -e $POSTMAN_ENV_FILE $POSTMAN_COLLECTION_INFERENCE -d $POSTMAN_DATA_FILE_INFERENCE \
-             -r cli,html --reporter-html-export $REPORT_FILE_INFERENCE --verbose
+             -r cli,html --reporter-html-export $ARTIFACTS_INFERENCE_DIR/$REPORT_FILE --verbose
+  local EXIT_CODE=$?
   stop_mms_server
+  move_logs $MMS_CONSOLE_LOG_FILE $ARTIFACTS_INFERENCE_DIR
+  cleanup_model_store
+  return $EXIT_CODE
 }
 
 trigger_https_tests(){
-  start_mms_secure_server $MODEL_STORE_DIR $MMS_LOG_FILE_HTTPS
+  start_mms_secure_server $MODEL_STORE_DIR $MMS_CONSOLE_LOG_FILE
   newman run --insecure -e $POSTMAN_ENV_FILE $POSTMAN_COLLECTION_HTTPS \
-             -r cli,html --reporter-html-export $REPORT_FILE_HTTPS --verbose
+             -r cli,html --reporter-html-export $ARTIFACTS_HTTPS_DIR/$REPORT_FILE --verbose
+  local EXIT_CODE=$?
   stop_mms_server
+  move_logs $MMS_CONSOLE_LOG_FILE $ARTIFACTS_HTTPS_DIR
+  cleanup_model_store
+  return $EXIT_CODE
 }
 
-mkdir -p $MODEL_STORE_DIR
+cd $BASE_DIR
+mkdir -p $MODEL_STORE_DIR $ARTIFACTS_MANAGEMENT_DIR $ARTIFACTS_INFERENCE_DIR $ARTIFACTS_HTTPS_DIR
 
 case $1 in
    'management')
       trigger_management_tests
+      exit $?
       ;;
    'inference')
       trigger_inference_tests
+      exit $?
       ;;
    'https')
       trigger_https_tests
+      exit $?
       ;;
    'ALL')
       trigger_management_tests
+      MGMT_EXIT_CODE=$?
       trigger_inference_tests
+      INFR_EXIT_CODE=$?
       trigger_https_tests
+      HTTPS_EXIT_CODE=$?
+      # If any one of the tests fail, exit with error
+      if [ "$MGMT_EXIT_CODE" -ne 0 ] || [ "$INFR_EXIT_CODE" -ne 0 ] || [ "$HTTPS_EXIT_CODE" -ne 0 ]
+      then exit 1
+      fi
       ;;
    *)
      echo $1 'Invalid'

.circleci/scripts/linux_test_benchmark.sh

@@ -2,13 +2,24 @@
 
 # Hack needed to make it work with existing benchmark.py
 # benchmark.py expects jmeter to be present at a very specific location
-mkdir -p /home/ubuntu/.linuxbrew/Cellar/jmeter/5.3/libexec/bin/
-ln -s /opt/apache-jmeter-5.3/bin/jmeter /home/ubuntu/.linuxbrew/Cellar/jmeter/5.3/libexec/bin/jmeter
+mkdir -p /home/ubuntu/.linuxbrew/Cellar/jmeter/5.3/libexec/
+ln -s /opt/apache-jmeter-5.3/lib/ /home/ubuntu/.linuxbrew/Cellar/jmeter/5.3/libexec/lib
+ln -s /opt/apache-jmeter-5.3/bin/ /home/ubuntu/.linuxbrew/Cellar/jmeter/5.3/libexec/bin
 
-multi-model-server --start >> mms.log 2>&1
+# Start MMS and redirect console ouptut and errors to a log file
+multi-model-server --start >> mms_console.log 2>&1
 sleep 30
 
 cd benchmarks
 python benchmark.py latency
+EXIT_CODE=$?
 
-multi-model-server --stop
+multi-model-server --stop
+
+# Moving MMS console log file to logs directory
+# Just a convenience for CircleCI to pick up logs from one directory
+cd ../
+mv mms_console.log logs/
+
+# Exit with the same error code as that of benchmark script
+exit $EXIT_CODE

.circleci/scripts/linux_test_modelarchiver.sh

@@ -4,14 +4,23 @@ cd model-archiver/
 
 # Lint test
 pylint -rn --rcfile=./model_archiver/tests/pylintrc model_archiver/.
+PY_LINT_EXIT_CODE=$?
 
 # Execute python unit tests
-python -m pytest --cov-report html:results_units --cov=./ model_archiver/tests/unit_tests
+python -m pytest --cov-report html:result_units --cov=./ model_archiver/tests/unit_tests
+PY_UNITS_EXIT_CODE=$?
 
 
 # Install model archiver module
-pip install .
+python setup.py bdist_wheel --universal && \
+pip install dist/*.whl
+MODL_ARCHVR_EXIT_CODE=$?
 
 # Execute integration tests
-python -m pytest model_archiver/tests/integ_tests
-# ToDo - Report for Integration tests ?
+python -m pytest model_archiver/tests/integ_tests # ToDo - Report for Integration tests ?
+PY_INTEG_EXIT_CODE=$?
+
+# If any one of the steps fail, exit with error
+if [ "$PY_LINT_EXIT_CODE" -ne 0 ] || [ "$PY_UNITS_EXIT_CODE" -ne 0 ] || [ "$MODL_ARCHVR_EXIT_CODE" -ne 0 ] || [ "$PY_INTEG_EXIT_CODE" -ne 0 ]
+then exit 1
+fi

.circleci/scripts/linux_test_perf_regression.sh

@@ -1,28 +1,31 @@
 #!/bin/bash
 
-multi-model-server --start \
-                   --models squeezenet=https://s3.amazonaws.com/model-server/model_archive_1.0/squeezenet_v1.1.mar \
-                   >> mms.log 2>&1
-sleep 90
+ARTIFACTS_DIR='run_artifacts'
+RESULT_DIR=$ARTIFACTS_DIR'/report/performance/'
+JMETER_PATH='/opt/apache-jmeter-5.3/bin/jmeter'
 
-cd performance_regression
+cd tests/performance
 
 # Only on a python 2 environment -
 PY_MAJOR_VER=$(python -c 'import sys; major = sys.version_info.major; print(major);')
 if [ $PY_MAJOR_VER -eq 2 ]; then
   # Hack to use python 3.6.5 for bzt installation and execution
+  # While MMS continues to use system python which is at 2.7.x
   export PATH="/root/.pyenv/bin:/root/.pyenv/shims:$PATH"
-  pyenv local 3.6.5
+  pyenv local 3.6.5 system
 fi
 
 # Install dependencies
+pip install -r requirements.txt
 pip install bzt
 
-curl -O https://s3.amazonaws.com/model-server/inputs/kitten.jpg
-bzt -o modules.jmeter.path=/opt/apache-jmeter-5.3/bin/jmeter \
-    -o settings.artifacts-dir=/tmp/mms-performance-regression/ \
-    -o modules.console.disable=true \
-    imageInputModelPlan.jmx.yaml \
-    -report
+# Execute performance test suite and store exit code
+./run_performance_suite.py -j $JMETER_PATH -e ci_linux_medium -x example* --no-compare-local --no-monit
+EXIT_CODE=$?
 
-multi-model-server --stop
+# Collect and store test results in result directory to be picked up by CircleCI
+mkdir -p $RESULT_DIR
+cp $ARTIFACTS_DIR/**/performance_results.xml $RESULT_DIR
+
+# Exit with the same error code as that of test execution
+exit $EXIT_CODE

.circleci/scripts/linux_test_python.sh

@@ -2,6 +2,13 @@
 
 # Lint Test
 pylint -rn --rcfile=./mms/tests/pylintrc mms/.
+PY_LINT_EXIT_CODE=$?
 
 # Execute python tests
-python -m pytest --cov-report html:htmlcov --cov=mms/ mms/tests/unit_tests/
+python -m pytest --cov-report html:result_units --cov=mms/ mms/tests/unit_tests/
+PYTEST_EXIT_CODE=$?
+
+# If any one of the tests fail, exit with error
+if [ "$PY_LINT_EXIT_CODE" -ne 0 ] || [ "$PYTEST_EXIT_CODE" -ne 0 ]
+then exit 1
+fi