Merge pull request #244 from azure-rtos/tizho/test

Release ThreadX regression system

Author: TiejunZhou-CN

.devcontainer/devcontainer.json

@@ -0,0 +1,4 @@
+{
+    "image": "tizho/azurertos-regression",
+    "runArgs": [ "--cap-add=NET_ADMIN"]
+}

.pipelines/smp.yml

@@ -0,0 +1,59 @@
+trigger:
+- master
+
+pr:
+- master
+
+pool:
+  vmImage: "ubuntu-22.04"
+
+steps:
+  - bash: sudo $(Build.SourcesDirectory)/scripts/install.sh
+    displayName: 'Install softwares'
+    env:
+      INDEX_URL: $(INDEX_URL)
+
+  - task: Bash@3
+    displayName: 'SDL check'
+    inputs:
+      filePath: '$(Build.SourcesDirectory)/scripts/sdl_check.sh'
+
+  - task: Bash@3
+    displayName: 'Build SMP'
+    inputs:
+      filePath: '$(Build.SourcesDirectory)/scripts/build_smp.sh'
+
+  - task: Bash@3
+    displayName: 'Test SMP'
+    inputs:
+      filePath: '$(Build.SourcesDirectory)/scripts/test_smp.sh'
+
+  - task: PublishTestResults@2
+    condition: succeededOrFailed()
+    displayName: 'Test SMP (PublishTestResults)'
+    inputs:
+      testResultsFormat: 'cTest'
+      testResultsFiles: '*/Testing/**/*.xml'
+      searchFolder: '$(Build.SourcesDirectory)/test/smp/cmake/build'
+      testRunTitle: 'SMP-Tests'
+      buildConfiguration: 'Release'
+
+  - task: CopyFiles@2
+    condition: succeededOrFailed()
+    displayName: 'Test SMP (PublishTestReports)'
+    inputs:
+      SourceFolder: '$(Build.SourcesDirectory)/test/smp/cmake'
+      Contents: |
+        build/*.txt
+        build/*/Testing/**/*.xml
+        coverage_report/**/*
+      TargetFolder: '$(ob_outputDirectory)/test_reports_SMP'
+
+  - task: PublishCodeCoverageResults@1
+    condition: succeededOrFailed()
+    displayName: 'Test SMP (PublishCodeCoverageResults)'
+    inputs:
+      codeCoverageTool: 'Cobertura'
+      summaryFileLocation: '$(Build.SourcesDirectory)/test/smp/cmake/coverage_report/default_build_coverage.xml'
+      pathToSources: '$(Build.SourcesDirectory)/test/smp/cmake'
+      reportDirectory: '$(Build.SourcesDirectory)/test/smp/cmake/coverage_report/default_build_coverage'

.pipelines/tx.yml

@@ -0,0 +1,59 @@
+trigger:
+- master
+
+pr:
+- master
+
+pool:
+  vmImage: "ubuntu-22.04"
+
+steps:
+  - bash: sudo $(Build.SourcesDirectory)/scripts/install.sh
+    displayName: 'Install softwares'
+    env:
+      INDEX_URL: $(INDEX_URL)
+
+  - task: Bash@3
+    displayName: 'SDL check'
+    inputs:
+      filePath: '$(Build.SourcesDirectory)/scripts/sdl_check.sh'
+
+  - task: Bash@3
+    displayName: 'Build TX'
+    inputs:
+      filePath: '$(Build.SourcesDirectory)/scripts/build_tx.sh'
+
+  - task: Bash@3
+    displayName: 'Test TX'
+    inputs:
+      filePath: '$(Build.SourcesDirectory)/scripts/test_tx.sh'
+
+  - task: PublishTestResults@2
+    condition: succeededOrFailed()
+    displayName: 'Test TX (PublishTestResults)'
+    inputs:
+      testResultsFormat: 'cTest'
+      testResultsFiles: '*/Testing/**/*.xml'
+      searchFolder: '$(Build.SourcesDirectory)/test/tx/cmake/build'
+      testRunTitle: 'TX-Tests'
+      buildConfiguration: 'Release'
+
+  - task: CopyFiles@2
+    condition: succeededOrFailed()
+    displayName: 'Test TX (PublishTestReports)'
+    inputs:
+      SourceFolder: '$(Build.SourcesDirectory)/test/tx/cmake'
+      Contents: |
+        build/*.txt
+        build/*/Testing/**/*.xml
+        coverage_report/**/*
+      TargetFolder: '$(ob_outputDirectory)/test_reports_TX'
+
+  - task: PublishCodeCoverageResults@1
+    condition: succeededOrFailed()
+    displayName: 'Test TX (PublishCodeCoverageResults)'
+    inputs:
+      codeCoverageTool: 'Cobertura'
+      summaryFileLocation: '$(Build.SourcesDirectory)/test/tx/cmake/coverage_report/default_build_coverage.xml'
+      pathToSources: '$(Build.SourcesDirectory)/test/tx/cmake'
+      reportDirectory: '$(Build.SourcesDirectory)/test/tx/cmake/coverage_report/default_build_coverage'

scripts/build_smp.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+$(dirname `realpath $0`)/../test/smp/cmake/run.sh build all

scripts/build_tx.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+$(dirname `realpath $0`)/../test/tx/cmake/run.sh build all

scripts/cmake_bootstrap.sh

@@ -0,0 +1,120 @@
+#!/bin/bash
+
+set -e
+
+function help() {
+    echo "Usage: $0 [build|test] [all|<build_configuration> <build_configuration>...]"
+    echo "Available build_configuration:"
+    for build in ${build_configurations[*]}; do
+        echo "  $build"
+    done
+    exit 1
+}
+
+function validate() {
+    for build in ${build_configurations[*]}; do
+        if [ "$1" == "$build" ]; then
+            return
+        fi
+    done
+    help
+}
+
+function generate() {
+    build=$1
+    cmake -Bbuild/$build -GNinja -DBUILD_SHARED_LIBS=ON -DCMAKE_TOOLCHAIN_FILE=$(dirname $(realpath $0))/../cmake/linux.cmake -DCMAKE_BUILD_TYPE=$build .
+}
+
+function build() {
+    cmake --build build/$1
+}
+
+function build_libs() {
+    cmake -Bbuild/libs -GNinja -DBUILD_SHARED_LIBS=ON -DCMAKE_TOOLCHAIN_FILE=$(dirname $(realpath $0))/../cmake/linux.cmake libs
+    cmake --build build/libs
+}
+
+function test() {
+    pushd build/$1
+    [ -z "${CTEST_PARALLEL_LEVEL}" ] && parallel="-j$2"
+    if [ -z "${CTEST_REPEAT_FAIL}" ]; 
+    then
+        repeat_fail=2
+    else
+        repeat_fail=${CTEST_REPEAT_FAIL}
+    fi
+    ctest $parallel --timeout 1000 -O $1.txt -T test --no-compress-output --test-output-size-passed 4194304 --test-output-size-failed 4194304 --output-on-failure --repeat until-pass:${repeat_fail}
+    popd
+    grep -E "^(\s*[0-9]+|Total)" build/$1/$1.txt >build/$1.txt
+    sed -i "s/\x1B\[[0-9;]*[JKmsu]//g" build/$1.txt
+    if [[ $1 = *"_coverage" ]]; then
+        ./coverage.sh $1
+    fi
+}
+
+cd $(dirname $0)
+
+result=$(sed -n "/(BUILD_CONFIGURATIONS/,/)/p" CMakeLists.txt|sed ':label;N;s/\n/ /;b label'|grep -Pzo "[a-zA-Z0-9_]*build[a-zA-Z0-9_]*\s*"| tr -d '\0')
+IFS=' '
+read -ra build_configurations <<< "$result"
+
+if [ $# -lt 1 ]; then
+    help
+fi
+
+command=$1
+shift
+
+if [ "$#" == "0" ]; then
+    builds=${build_configurations[0]}
+elif [ "$*" == "all" ]; then
+    builds=${build_configurations[@]}
+else
+    for item in $*; do
+        validate $item
+    done
+    builds=$*
+fi
+
+if [ "$command" == "build" ]; then
+    for item in $builds; do
+        generate $item
+        echo ""
+    done
+
+    for item in $builds; do
+        echo "Building $item"
+        build $item
+        echo ""
+    done
+elif [ "$command" == "test" ]; then
+    cores=$(nproc)
+    if [ -z "${CTEST_PARALLEL_LEVEL}" ];
+    then
+        # Run builds in parallel
+        build_counts=$(echo $builds | wc -w)
+        parallel_jobs=$(($cores / $build_counts))
+        parallel_jobs=$(($parallel_jobs + 2))
+        pids=""
+        for item in $builds; do
+            echo "Testing $item"
+            test $item $parallel_jobs &
+            pids+=" $!"
+        done
+        exit_code=0
+        for p in $pids; do
+            wait $p || exit_code=$?
+        done
+        exit $exit_code
+    else
+        # Run builds in serial
+        for item in $builds; do
+            echo "Testing $item"
+            test $item $parallel_jobs
+        done
+    fi
+elif [ "$command" == "build_libs" ]; then
+    build_libs
+else
+    help
+fi

scripts/install.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+#
+# Install necessary softwares for Ubuntu.
+
+apt-get update
+apt-get install -y \
+    gcc-multilib \
+    git \
+    g++ \
+    python3-pip \
+    ninja-build \
+    unifdef \
+    p7zip-full \
+    tofrodos \
+    gawk \
+    cmake \
+    software-properties-common
+
+python3 -m pip install --upgrade pip
+pip3 install artifacts-keyring
+pip3 install gcovr==4.1 $INDEX_URL
+

scripts/sdl_check.sh

@@ -0,0 +1,19 @@
+# !/bin/bash
+dir_list="common common_smp common_modules ports ports_module ports_smp samples"
+exclude_list="-path TX"
+file_list=$(find $dir_list \( $exclude_list \) -prune -o -type f -name '*.[ch]' -print)
+cd $(dirname `realpath $0`)/..
+echo "Checking for unexpected usage of memcpy..."
+echo ""
+echo "Excluding:"
+echo $exclude_list | grep -P "[^\s]*/[^\s]*" -o
+echo ""
+echo "Result:"
+grep -i "memcpy(" -i $file_list -n | grep -i "use case of .* is verified" -v
+if [ "$?" -eq "1" ];
+then
+    echo "CLEAN"
+    exit 0
+else
+    exit 1
+fi

scripts/test_smp.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+CTEST_PARALLEL_LEVEL=4 $(dirname `realpath $0`)/../test/smp/cmake/run.sh test all

scripts/test_tx.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+CTEST_PARALLEL_LEVEL=4 $(dirname `realpath $0`)/../test/tx/cmake/run.sh test all