Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions .github/workflows/Nightly_Perf_Env_CI.yml
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ jobs:
if [[ "${{ matrix.workload }}" == "bootstorm_vm_scale" ]]
then
# Warm-up: Pull the Fedora image from quay.io for each node
ssh -t provision "podman run --rm -t -e WORKLOAD='$WORKLOAD' -e KUBEADMIN_PASSWORD='$KUBEADMIN_PASSWORD' -e KUBECONFIG_PATH='$CONTAINER_KUBECONFIG_PATH' -e SCALE='$SCALE' -e SCALE_NODES=$SCALE_NODES -e REDIS='$REDIS' -e RUN_ARTIFACTS_URL='$RUN_ARTIFACTS_URL' -e BUILD_VERSION='$build_version' -e RUN_TYPE='$RUN_TYPE' -e KATA_CPUOFFLINE_WORKAROUND='True' -e SAVE_ARTIFACTS_LOCAL='False' -e ENABLE_PROMETHEUS_SNAPSHOT='$ENABLE_PROMETHEUS_SNAPSHOT' -e THREADS_LIMIT='$THREADS_LIMIT' -e WINDOWS_URL='$WINDOWS_URL' -e TIMEOUT='$TIMEOUT' -e log_level='INFO' -v '$CONTAINER_KUBECONFIG_PATH':'$CONTAINER_KUBECONFIG_PATH' --privileged '${{ secrets.QUAY_BENCHMARK_RUNNER_REPOSITORY }}'"
ssh -t provision "podman run --rm -t -e WORKLOAD='$WORKLOAD' -e KUBEADMIN_PASSWORD='$KUBEADMIN_PASSWORD' -e SCALE='$SCALE' -e SCALE_NODES=$SCALE_NODES -e REDIS='$REDIS' -e RUN_ARTIFACTS_URL='$RUN_ARTIFACTS_URL' -e BUILD_VERSION='$build_version' -e RUN_TYPE='$RUN_TYPE' -e KATA_CPUOFFLINE_WORKAROUND='True' -e SAVE_ARTIFACTS_LOCAL='False' -e ENABLE_PROMETHEUS_SNAPSHOT='$ENABLE_PROMETHEUS_SNAPSHOT' -e THREADS_LIMIT='$THREADS_LIMIT' -e WINDOWS_URL='$WINDOWS_URL' -e TIMEOUT='$TIMEOUT' -e log_level='INFO' -v '$CONTAINER_KUBECONFIG_PATH':'$CONTAINER_KUBECONFIG_PATH' --privileged '${{ secrets.QUAY_BENCHMARK_RUNNER_REPOSITORY }}'"
SCALE=$BOOTSTORM_SCALE
elif [[ "$WORKLOAD" == "windows_vm" ]]
then
Expand All @@ -212,11 +212,11 @@ jobs:
*) echo "Unknown Windows scale workload ${{ matrix.workload }}"; exit 1 ;;
esac
# Warm-up: Load DV for Windows
ssh -t provision "podman run --rm -t -e WORKLOAD='$WORKLOAD' -e KUBEADMIN_PASSWORD='$KUBEADMIN_PASSWORD' -e KUBECONFIG_PATH='$CONTAINER_KUBECONFIG_PATH' -e SCALE='$SCALE' -e SCALE_NODES=$SCALE_NODES -e REDIS='$REDIS' -e RUN_ARTIFACTS_URL='$RUN_ARTIFACTS_URL' -e BUILD_VERSION='$build_version' -e RUN_TYPE='$RUN_TYPE' -e KATA_CPUOFFLINE_WORKAROUND='True' -e SAVE_ARTIFACTS_LOCAL='False' -e ENABLE_PROMETHEUS_SNAPSHOT='$ENABLE_PROMETHEUS_SNAPSHOT' -e THREADS_LIMIT='$THREADS_LIMIT' -e WINDOWS_URL='$WINDOWS_URL' -e TIMEOUT='$TIMEOUT' -e log_level='INFO' -v '$CONTAINER_KUBECONFIG_PATH':'$CONTAINER_KUBECONFIG_PATH' --privileged '${{ secrets.QUAY_BENCHMARK_RUNNER_REPOSITORY }}'"
ssh -t provision "podman run --rm -t -e WORKLOAD='$WORKLOAD' -e KUBEADMIN_PASSWORD='$KUBEADMIN_PASSWORD' -e SCALE='$SCALE' -e SCALE_NODES=$SCALE_NODES -e REDIS='$REDIS' -e RUN_ARTIFACTS_URL='$RUN_ARTIFACTS_URL' -e BUILD_VERSION='$build_version' -e RUN_TYPE='$RUN_TYPE' -e KATA_CPUOFFLINE_WORKAROUND='True' -e SAVE_ARTIFACTS_LOCAL='False' -e ENABLE_PROMETHEUS_SNAPSHOT='$ENABLE_PROMETHEUS_SNAPSHOT' -e THREADS_LIMIT='$THREADS_LIMIT' -e WINDOWS_URL='$WINDOWS_URL' -e TIMEOUT='$TIMEOUT' -e log_level='INFO' -v '$CONTAINER_KUBECONFIG_PATH':'$CONTAINER_KUBECONFIG_PATH' --privileged '${{ secrets.QUAY_BENCHMARK_RUNNER_REPOSITORY }}'"
SCALE=$WINDOWS_SCALE
fi
# SCALE_NODES is a list, not add ''
ssh -t provision "podman run --rm -t -e WORKLOAD='$WORKLOAD' -e KUBEADMIN_PASSWORD='$KUBEADMIN_PASSWORD' -e KUBECONFIG_PATH='$CONTAINER_KUBECONFIG_PATH' -e SCALE='$SCALE' -e SCALE_NODES=$SCALE_NODES -e REDIS='$REDIS' -e PIN_NODE_BENCHMARK_OPERATOR='$PIN_NODE_BENCHMARK_OPERATOR' -e PIN_NODE1='$PIN_NODE1' -e PIN_NODE2='$PIN_NODE2' -e ELASTICSEARCH='$ELASTICSEARCH' -e ELASTICSEARCH_PORT='$ELASTICSEARCH_PORT' -e ELASTICSEARCH_USER='$ELASTICSEARCH_USER' -e ELASTICSEARCH_PASSWORD='$ELASTICSEARCH_PASSWORD' -e IBM_REGION_NAME='$IBM_REGION_NAME' -e IBM_ENDPOINT_URL='$IBM_ENDPOINT_URL' -e IBM_ACCESS_KEY_ID='$IBM_ACCESS_KEY_ID' -e IBM_SECRET_ACCESS_KEY='$IBM_SECRET_ACCESS_KEY' -e IBM_BUCKET='$IBM_BUCKET' -e IBM_KEY='$IBM_KEY' -e RUN_ARTIFACTS_URL='$RUN_ARTIFACTS_URL' -e BUILD_VERSION='$build_version' -e RUN_TYPE='$RUN_TYPE' -e KATA_CPUOFFLINE_WORKAROUND='True' -e SAVE_ARTIFACTS_LOCAL='False' -e ENABLE_PROMETHEUS_SNAPSHOT='$ENABLE_PROMETHEUS_SNAPSHOT' -e THREADS_LIMIT='$THREADS_LIMIT' -e WINDOWS_URL='$WINDOWS_URL' -e TIMEOUT='$TIMEOUT' -e log_level='INFO' -v '$CONTAINER_KUBECONFIG_PATH':'$CONTAINER_KUBECONFIG_PATH' --privileged '${{ secrets.QUAY_BENCHMARK_RUNNER_REPOSITORY }}'"
ssh -t provision "podman run --rm -t -e WORKLOAD='$WORKLOAD' -e KUBEADMIN_PASSWORD='$KUBEADMIN_PASSWORD' -e SCALE='$SCALE' -e SCALE_NODES=$SCALE_NODES -e REDIS='$REDIS' -e PIN_NODE_BENCHMARK_OPERATOR='$PIN_NODE_BENCHMARK_OPERATOR' -e PIN_NODE1='$PIN_NODE1' -e PIN_NODE2='$PIN_NODE2' -e ELASTICSEARCH='$ELASTICSEARCH' -e ELASTICSEARCH_PORT='$ELASTICSEARCH_PORT' -e ELASTICSEARCH_USER='$ELASTICSEARCH_USER' -e ELASTICSEARCH_PASSWORD='$ELASTICSEARCH_PASSWORD' -e IBM_REGION_NAME='$IBM_REGION_NAME' -e IBM_ENDPOINT_URL='$IBM_ENDPOINT_URL' -e IBM_ACCESS_KEY_ID='$IBM_ACCESS_KEY_ID' -e IBM_SECRET_ACCESS_KEY='$IBM_SECRET_ACCESS_KEY' -e IBM_BUCKET='$IBM_BUCKET' -e IBM_KEY='$IBM_KEY' -e RUN_ARTIFACTS_URL='$RUN_ARTIFACTS_URL' -e BUILD_VERSION='$build_version' -e RUN_TYPE='$RUN_TYPE' -e KATA_CPUOFFLINE_WORKAROUND='True' -e SAVE_ARTIFACTS_LOCAL='False' -e ENABLE_PROMETHEUS_SNAPSHOT='$ENABLE_PROMETHEUS_SNAPSHOT' -e THREADS_LIMIT='$THREADS_LIMIT' -e WINDOWS_URL='$WINDOWS_URL' -e TIMEOUT='$TIMEOUT' -e log_level='INFO' -v '$CONTAINER_KUBECONFIG_PATH':'$CONTAINER_KUBECONFIG_PATH' --privileged '${{ secrets.QUAY_BENCHMARK_RUNNER_REPOSITORY }}'"
# NOT SCALE RUN
else
if [[ "${{ matrix.workload }}" == "clusterbuster" ]]
Expand All @@ -227,9 +227,9 @@ jobs:
if [[ "${{ matrix.workload }}" == "vdbench_vm" ]]
then
# Warm-up: download vdbench_vm container disk image per node (centos stream8)
ssh -t provision "podman run --rm -t -e WORKLOAD='${{ matrix.workload }}' -e KUBEADMIN_PASSWORD='$KUBEADMIN_PASSWORD' -e KUBECONFIG_PATH='$CONTAINER_KUBECONFIG_PATH' -e SCALE='$SCALE' -e SCALE_NODES=$SCALE_NODES -e REDIS='$REDIS' -e RUN_TYPE='test_ci' -e SAVE_ARTIFACTS_LOCAL='False' -e THREADS_LIMIT='$THREADS_LIMIT' -e TIMEOUT='$TIMEOUT' -e log_level='INFO' -v '$CONTAINER_KUBECONFIG_PATH':'$CONTAINER_KUBECONFIG_PATH' --privileged '${{ secrets.QUAY_BENCHMARK_RUNNER_REPOSITORY }}'"
ssh -t provision "podman run --rm -t -e WORKLOAD='${{ matrix.workload }}' -e KUBEADMIN_PASSWORD='$KUBEADMIN_PASSWORD' -e SCALE='$SCALE' -e SCALE_NODES=$SCALE_NODES -e REDIS='$REDIS' -e RUN_TYPE='test_ci' -e SAVE_ARTIFACTS_LOCAL='False' -e THREADS_LIMIT='$THREADS_LIMIT' -e TIMEOUT='$TIMEOUT' -e log_level='INFO' -v '$CONTAINER_KUBECONFIG_PATH':'$CONTAINER_KUBECONFIG_PATH' --privileged '${{ secrets.QUAY_BENCHMARK_RUNNER_REPOSITORY }}'"
fi
ssh -t provision "podman run --rm -t -e WORKLOAD='${{ matrix.workload }}' -e KUBEADMIN_PASSWORD='$KUBEADMIN_PASSWORD' -e KUBECONFIG_PATH='$CONTAINER_KUBECONFIG_PATH' -e PIN_NODE_BENCHMARK_OPERATOR='$PIN_NODE_BENCHMARK_OPERATOR' -e PIN_NODE1='$PIN_NODE1' -e PIN_NODE2='$PIN_NODE2' -e ELASTICSEARCH='$ELASTICSEARCH' -e ELASTICSEARCH_PORT='$ELASTICSEARCH_PORT' -e ELASTICSEARCH_USER='$ELASTICSEARCH_USER' -e ELASTICSEARCH_PASSWORD='$ELASTICSEARCH_PASSWORD' -e IBM_REGION_NAME='$IBM_REGION_NAME' -e IBM_ENDPOINT_URL='$IBM_ENDPOINT_URL' -e IBM_ACCESS_KEY_ID='$IBM_ACCESS_KEY_ID' -e IBM_SECRET_ACCESS_KEY='$IBM_SECRET_ACCESS_KEY' -e IBM_BUCKET='$IBM_BUCKET' -e IBM_KEY='$IBM_KEY' -e RUN_ARTIFACTS_URL='$RUN_ARTIFACTS_URL' -e BUILD_VERSION='$build_version' -e RUN_TYPE='$RUN_TYPE' -e KATA_CPUOFFLINE_WORKAROUND='True' -e SAVE_ARTIFACTS_LOCAL='False' -e ENABLE_PROMETHEUS_SNAPSHOT='$ENABLE_PROMETHEUS_SNAPSHOT' -e WORKER_DISK_IDS=$WORKER_DISK_IDS -e WORKER_DISK_PREFIX='$WORKER_DISK_PREFIX' -e TIMEOUT='$TIMEOUT' -e log_level='INFO' -v '$CONTAINER_KUBECONFIG_PATH':'$CONTAINER_KUBECONFIG_PATH' --privileged '${{ secrets.QUAY_BENCHMARK_RUNNER_REPOSITORY }}'"
ssh -t provision "podman run --rm -t -e WORKLOAD='${{ matrix.workload }}' -e KUBEADMIN_PASSWORD='$KUBEADMIN_PASSWORD' -e PIN_NODE_BENCHMARK_OPERATOR='$PIN_NODE_BENCHMARK_OPERATOR' -e PIN_NODE1='$PIN_NODE1' -e PIN_NODE2='$PIN_NODE2' -e ELASTICSEARCH='$ELASTICSEARCH' -e ELASTICSEARCH_PORT='$ELASTICSEARCH_PORT' -e ELASTICSEARCH_USER='$ELASTICSEARCH_USER' -e ELASTICSEARCH_PASSWORD='$ELASTICSEARCH_PASSWORD' -e IBM_REGION_NAME='$IBM_REGION_NAME' -e IBM_ENDPOINT_URL='$IBM_ENDPOINT_URL' -e IBM_ACCESS_KEY_ID='$IBM_ACCESS_KEY_ID' -e IBM_SECRET_ACCESS_KEY='$IBM_SECRET_ACCESS_KEY' -e IBM_BUCKET='$IBM_BUCKET' -e IBM_KEY='$IBM_KEY' -e RUN_ARTIFACTS_URL='$RUN_ARTIFACTS_URL' -e BUILD_VERSION='$build_version' -e RUN_TYPE='$RUN_TYPE' -e KATA_CPUOFFLINE_WORKAROUND='True' -e SAVE_ARTIFACTS_LOCAL='False' -e ENABLE_PROMETHEUS_SNAPSHOT='$ENABLE_PROMETHEUS_SNAPSHOT' -e WORKER_DISK_IDS=$WORKER_DISK_IDS -e WORKER_DISK_PREFIX='$WORKER_DISK_PREFIX' -e TIMEOUT='$TIMEOUT' -e log_level='INFO' -v '$CONTAINER_KUBECONFIG_PATH':'$CONTAINER_KUBECONFIG_PATH' --privileged '${{ secrets.QUAY_BENCHMARK_RUNNER_REPOSITORY }}'"
fi
ssh -t provision "podman rmi -f '${{ secrets.QUAY_BENCHMARK_RUNNER_REPOSITORY }}'"
echo '>>>>>>>>>>>>>>>>>>>>>>>>>> End E2E workload: ${{ matrix.workload }} >>>>>>>>>>>>>>>>>>>>>>>>>>>>'
Expand Down Expand Up @@ -330,7 +330,7 @@ jobs:
# Check for workload failure or success => return pass/failed
if [[ "${{needs.workload.outputs.job_status}}" == "failure" || "${{needs.workload.outputs.job_status}}" == "cancelled" ]]; then status="failed"; else status="pass"; fi
echo '>>>>>>>>>>>>>>>>>>>>>>>>>> Update CI status $status >>>>>>>>>>>>>>>>>>>>>>>>>>'
ssh -t provision "podman run --rm -e KUBEADMIN_PASSWORD='$KUBEADMIN_PASSWORD' -e KUBECONFIG_PATH='$CONTAINER_KUBECONFIG_PATH' -e PIN_NODE_BENCHMARK_OPERATOR='$PIN_NODE_BENCHMARK_OPERATOR' -e PIN_NODE1='$PIN_NODE1' -e PIN_NODE2='$PIN_NODE2' -e ELASTICSEARCH='$ELASTICSEARCH' -e ELASTICSEARCH_PORT='$ELASTICSEARCH_PORT' -e ELASTICSEARCH_USER='$ELASTICSEARCH_USER' -e ELASTICSEARCH_PASSWORD='$ELASTICSEARCH_PASSWORD' -e BUILD_VERSION='$build_version' -e CI_STATUS='$status' -e CI_MINUTES_TIME='$ci_minutes_time' -e BENCHMARK_RUNNER_ID='$BENCHMARK_RUNNER_ID' -e BENCHMARK_OPERATOR_ID='$BENCHMARK_OPERATOR_ID' -e BENCHMARK_WRAPPER_ID='$BENCHMARK_WRAPPER_ID' -e RUN_TYPE='$RUN_TYPE' -e TIMEOUT='3600' -e log_level='INFO' -v '$CONTAINER_KUBECONFIG_PATH':'$CONTAINER_KUBECONFIG_PATH' --privileged '${{ secrets.QUAY_BENCHMARK_RUNNER_REPOSITORY }}'"
ssh -t provision "podman run --rm -e KUBEADMIN_PASSWORD='$KUBEADMIN_PASSWORD' -e PIN_NODE_BENCHMARK_OPERATOR='$PIN_NODE_BENCHMARK_OPERATOR' -e PIN_NODE1='$PIN_NODE1' -e PIN_NODE2='$PIN_NODE2' -e ELASTICSEARCH='$ELASTICSEARCH' -e ELASTICSEARCH_PORT='$ELASTICSEARCH_PORT' -e ELASTICSEARCH_USER='$ELASTICSEARCH_USER' -e ELASTICSEARCH_PASSWORD='$ELASTICSEARCH_PASSWORD' -e BUILD_VERSION='$build_version' -e CI_STATUS='$status' -e CI_MINUTES_TIME='$ci_minutes_time' -e BENCHMARK_RUNNER_ID='$BENCHMARK_RUNNER_ID' -e BENCHMARK_OPERATOR_ID='$BENCHMARK_OPERATOR_ID' -e BENCHMARK_WRAPPER_ID='$BENCHMARK_WRAPPER_ID' -e RUN_TYPE='$RUN_TYPE' -e TIMEOUT='3600' -e log_level='INFO' -v '$CONTAINER_KUBECONFIG_PATH':'$CONTAINER_KUBECONFIG_PATH' --privileged '${{ secrets.QUAY_BENCHMARK_RUNNER_REPOSITORY }}'"
echo '>>>>>>>>>>>>>>>>>>>>>>>>>> Remove image on provision >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>'
echo "if [[ \"\$(sudo podman images -q ${{ secrets.QUAY_BENCHMARK_RUNNER_REPOSITORY }} 2> /dev/null)\" != \"\" ]]; then sudo podman rmi -f \$(sudo podman images -q ${{ secrets.QUAY_BENCHMARK_RUNNER_REPOSITORY }} 2> /dev/null); fi" > "$RUNNER_PATH/remove_image.sh"
scp -r "$RUNNER_PATH/remove_image.sh" provision:"/tmp/remove_image.sh"
Expand Down
Loading