Add nightly performance tests into Gitlab NV pipeline

Add small fixes to shell script.
Add heaptrack and heaptrack report converting to svg, then saving it as GitLab Pages for human checks.

Relates-To: OLPEDGE-746

Signed-off-by: Yaroslav Stefinko <[email protected]>

Author: ystefinko

.gitlab-ci.yml

@@ -6,6 +6,7 @@ variables:
 
 stages:
   - build
+  - test
   - translate_report
   - deploy
 
@@ -52,13 +53,13 @@ build_test_nv:
       - $CI_PROJECT_DIR
     expire_in: 1 year
 
-upload_sonar_nv:
-  stage: deploy
+test_performance_nv:
+  stage: test
   tags:
   - docker-prod
-  image: ${DOCKER_REGISTRY}/${DOCKER_IMAGE_COVERAGE}:${DOCKER_IMAGE_COVERAGE_VERSION}
+  image: ${DOCKER_REGISTRY}/${DOCKER_IMAGE_PERFORMANCE}:${DOCKER_IMAGE_PERFORMANCE_VERSION}
   script:
-  - $CI_PROJECT_DIR/scripts/linux/nv/gitlab_cppcheck_and_upload_sonar.sh
+  - $CI_PROJECT_DIR/scripts/linux/nv/gitlab_test_performance.sh
   only:
     refs:
       - master
@@ -68,9 +69,27 @@ upload_sonar_nv:
   artifacts:
     when: always
     paths:
+      - ./*.gz
+      - ./*.html
+      - cache
+      - heaptrack
       - reports
     expire_in: 1 year
 
+upload_sonar_nv:
+  stage: deploy
+  tags:
+  - docker-prod
+  image: ${DOCKER_REGISTRY}/${DOCKER_IMAGE_COVERAGE}:${DOCKER_IMAGE_COVERAGE_VERSION}
+  script:
+  - $CI_PROJECT_DIR/scripts/linux/nv/gitlab_cppcheck_and_upload_sonar.sh
+  only:
+    refs:
+      - master
+      - schedules
+    variables:
+      - $NIGHTLY
+
 translate_report:
   stage: translate_report
   tags:
@@ -83,6 +102,7 @@ translate_report:
     # - python -m junit2htmlreport --merge olp-merged-report.xml reports/*.xml
     # - python -m junit2htmlreport olp-merged-report.xml
     - python -m junit2htmlreport --report-matrix reports/index.html reports/*.xml
+    - cat heaptrack_report.html >> reports/index.html
     - mkdir -p .public
     - cp reports/*.html .public/
   artifacts:
@@ -94,6 +114,7 @@ translate_report:
       - schedules
     variables:
       - $FULLY
+      - $NIGHTLY
 
 pages:
   stage: deploy
@@ -110,4 +131,5 @@ pages:
       - master
       - schedules
     variables:
-      - $FULLY
+      - $FULLY
+      - $NIGHTLY

scripts/linux/fv/gitlab-olp-cpp-sdk-functional-test.sh

@@ -62,4 +62,4 @@ done
 # Kill local server
 kill -15 ${SERVER_PID}
 # Waiter for server process to be exited correctly
-wait ${SERVER_PID}
+wait ${SERVER_PID}

scripts/linux/nv/gitlab_test_performance.sh

@@ -17,7 +17,8 @@
 # SPDX-License-Identifier: Apache-2.0
 # License-Filename: LICENSE
 
-ulimit -c unlimited # for core dump backtrace
+# For core dump backtrace
+ulimit -c unlimited
 
 # Start local server
 node tests/utils/olp_server/server.js & export SERVER_PID=$!
@@ -38,11 +39,46 @@ echo ">>> Local Server started for further performance test ... >>>"
 
 export cache_location="cache"
 
-heaptrack ./build/tests/performance/olp-cpp-sdk-performance-tests --gtest_filter="*short_test_null_cache"
-heaptrack ./build/tests/performance/olp-cpp-sdk-performance-tests --gtest_filter="*short_test_memory_cache"
-heaptrack ./build/tests/performance/olp-cpp-sdk-performance-tests --gtest_filter="*short_test_disk_cache"
+echo ">>> Start performance tests ... >>>"
+heaptrack ./build/tests/performance/olp-cpp-sdk-performance-tests --gtest_filter="*short_test_null_cache" 2>> errors.txt || TEST_FAILURE=1
+mv heaptrack.olp-cpp-sdk-performance-tests.*.gz short_test_null_cache.gz
+heaptrack ./build/tests/performance/olp-cpp-sdk-performance-tests --gtest_filter="*short_test_memory_cache" 2>> errors.txt || TEST_FAILURE=1
+mv heaptrack.olp-cpp-sdk-performance-tests.*.gz short_test_memory_cache.gz
+heaptrack ./build/tests/performance/olp-cpp-sdk-performance-tests --gtest_filter="*short_test_disk_cache" 2>> errors.txt || TEST_FAILURE=1
+mv heaptrack.olp-cpp-sdk-performance-tests.*.gz short_test_disk_cache.gz
+echo ">>> Finished performance tests . >>>"
 
-du -h $cache_location
+if [[ ${TEST_FAILURE} == 1 ]]; then
+        echo "Printing error.txt ###########################################"
+        cat errors.txt
+        echo "End of error.txt #############################################"
+        echo "CRASH ERROR. One of test groups contains crash. Report was not generated for that group ! "
+else
+    echo "OK. Full list of tests passed. "
+fi
+
+mkdir reports/heaptrack
+mkdir heaptrack
+
+# Third party dependency needed for pretty graph generation below
+git clone --depth=1 https://github.com/brendangregg/FlameGraph
+
+for archive_name in short_test_null_cache short_test_memory_cache short_test_disk_cache
+do
+    heaptrack_print --print-leaks \
+      --print-flamegraph heaptrack/flamegraph_${archive_name}.data \
+      --file ${archive_name}.gz > reports/heaptrack/report_${archive_name}.txt
+    # Pretty graph generation
+    ./FlameGraph/flamegraph.pl --title="Flame Graph: ${archive_name}" heaptrack/flamegraph_${archive_name}.data > reports/heaptrack/flamegraph_${archive_name}.svg
+    cat reports/heaptrack/flamegraph_${archive_name}.svg >> heaptrack_report.html
+done
+cp heaptrack_report.html reports
+ls -la heaptrack
+ls -la
+
+du ${cache_location}
+du ${cache_location}/*
+ls -la ${cache_location}
 
 # TODO:
 #  1. print the total allocations done
@@ -51,6 +87,7 @@ du -h $cache_location
 #  4. track the disk IO made by SDK
 #  5. track the CPU load
 
-#Kill local server
-kill -15 $SERVER_PID
-wait $SERVER_PID # Waiter for server process to be exited correctly
+# Gracefully stop local server
+kill -15 ${SERVER_PID}
+# Waiter for server process to be exited correctly
+wait ${SERVER_PID}