Skip to content

Commit fbf8714

Browse files
wind57wind57
authored
1339 3.0.x (#1376)
1 parent 5be0951 commit fbf8714

File tree

9 files changed

+623
-74
lines changed

9 files changed

+623
-74
lines changed

‎.github/workflows/composites/download-docker-images/action.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ description: download docker images
66
runs:
77
using: "composite"
88
steps:
9+
910
- uses: actions/cache@v3
1011
with:
1112
path: /tmp/docker/images

‎.github/workflows/composites/matrix-bounds-on-test-times-cache-hit/action.yaml‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
########################################### find test times #######################################
2+
# when cache of test times is present, compute matrix steps
3+
# we do this because the number of steps required in a matrix is not stable
4+
# it is computed as sum_of_times_of_all_tests / max_time_of_a_single_test
5+
# For example if there is no test times cache, we will be using 32 steps in the matrix
6+
# on the other hand if cache is present, we might need less steps:
7+
8+
# sum_of_times_of_all_tests = 8000 seconds
9+
# max_time_of_a_single_test = 400
10+
# we would require 20 steps in the matrix, as having 32 would make no sense
11+
# because one step will run around 400 seconds and others will wait for this one to finish
12+
# as they finish earlier
13+
14+
name: matrix bounds on test times cache hit
15+
description: matrix bounds on test times cache hit
16+
runs:
17+
using: "composite"
18+
19+
steps:
20+
21+
- name: compute matrix steps
22+
shell: bash
23+
run: |
24+
25+
sum_of_all_tests=$(awk -F' ' '{sum+=$2;} END{print sum;}' /tmp/sorted.txt)
26+
sum_of_all_tests_as_int=$(printf "%.0f\n" "$sum_of_all_tests")
27+
echo "sum of all tests : $sum_of_all_tests_as_int"
28+
29+
max_test_time=$(tail -1 /tmp/sorted.txt | awk '{print $2}')
30+
max_test_time_as_int=$(printf "%.0f\n" "$max_test_time")
31+
echo "max test time : $max_test_time_as_int"
32+
33+
number_of_instances=$(( $sum_of_all_tests_as_int / $max_test_time_as_int ))
34+
35+
number_of_instances_as_array=()
36+
number_of_instances_as_array+='['
37+
number_of_instances_as_array+=$number_of_instances
38+
number_of_instances_as_array+=']'
39+
40+
average_time_per_instance=$(( sum_of_all_tests_as_int / $number_of_instances ))
41+
echo "average time per instance $average_time_per_instance"
42+
43+
matrix_array=()
44+
matrix_array+='['
45+
for ((i=0; i<=${number_of_instances}; i++)); do
46+
if (( $i == $(( $number_of_instances )) )); then
47+
matrix_array+=$i
48+
else
49+
matrix_array+=$i,
50+
fi
51+
done
52+
53+
matrix_array+=']'
54+
55+
echo "number of instances : $number_of_instances_as_array"
56+
echo "average time per instance : $average_time_per_instance"
57+
echo "matrix_array : $matrix_array"
58+
echo "********************************************************************************************************"
59+
60+
number_of_instances_json=$(jq -r -c . <<< $number_of_instances_as_array)
61+
echo "number_of_instances_json : $number_of_instances_json"
62+
63+
matrix_array_json=$(jq -r -c . <<< $matrix_array)
64+
echo "matrix_array_json : $matrix_array_json"
65+
66+
echo "TEST_TIMES_CACHE_PRESENT=true" >> $GITHUB_ENV
67+
echo "NUMBER_OF_MATRIX_INSTANCES=$(echo $number_of_instances_json)" >> $GITHUB_ENV
68+
echo "MATRIX_ARRAY=$(echo $matrix_array_json)" >> $GITHUB_ENV
69+
echo "AVERAGE_TIME_PER_INSTANCE=$(echo $average_time_per_instance)" >> $GITHUB_ENV
70+
71+

‎.github/workflows/composites/pre-test-actions/action.yaml‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
# common steps before running actula tests for both
2+
# when a cache of test times is presen and when it is not present
3+
4+
name: pre-test-actions
5+
description: pre-test-actions
6+
runs:
7+
using: "composite"
8+
9+
steps:
10+
11+
- name: set env variables
12+
uses: ./.github/workflows/composites/env-variables
13+
14+
- name: setup project jdk-17
15+
uses: ./.github/workflows/composites/setup-jdk17
16+
if: env.BASE_BRANCH == 'main'
17+
18+
- name: setup project jdk-8
19+
uses: ./.github/workflows/composites/setup-jdk1.8
20+
if: env.BASE_BRANCH == '2.1.x'
21+
22+
- name: cache local maven repository
23+
uses: ./.github/workflows/composites/cache
24+
25+
- name: download docker images
26+
uses: ./.github/workflows/composites/download-docker-images
27+
if: env.BASE_BRANCH != '2.1.x'
28+
29+
- name: load docker images into local repo
30+
uses: ./.github/workflows/composites/load-docker-images
31+
if: env.BASE_BRANCH != '2.1.x'
32+
33+
- name: download tests
34+
uses: actions/download-artifact@v3
35+
with:
36+
name: tests.txt
37+
path: /tmp
38+
39+
40+
41+

‎.github/workflows/composites/run-and-save-test-times-when-cache-missing/action.yaml‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
########################################### find test times #######################################
2+
# 1. find files that look like this: 'org.springframework.xxx.txt', these files are generated
3+
# by surefire-plugin when tests run.
4+
# 2. extract the time it took to run a certain test (from the above .txt file)
5+
# 3. create a file that contains test times from this current matrix
6+
7+
name: run and save test times when cache missing
8+
description: run and save test times when cache missing
9+
runs:
10+
using: "composite"
11+
12+
steps:
13+
14+
- name: run and save test times
15+
shell: bash
16+
run: |
17+
18+
# take only likes that look like :
19+
# 'spring.cloud.k8s.test.to.run -> org.springframework.cloud.kubernetes.fabric8.discovery.ServicePortSecureResolverTest'
20+
PLAIN_TEST_CLASSNAMES=($(cat /tmp/tests.txt | grep -o 'spring.cloud.k8s.test.to.run -> org.*' | awk '{print $3}'))
21+
IFS=$'\n'
22+
SORTED_TEST_CLASSNAMES=( $(sort <<< "${PLAIN_TEST_CLASSNAMES[*]}") )
23+
unset IFS
24+
25+
start_from_name=start_from_${CURRENT_INDEX}
26+
how_many_name=how_many_${CURRENT_INDEX}
27+
28+
start_from=${!start_from_name}
29+
how_many=${!how_many_name}
30+
31+
echo "${SORTED_TEST_CLASSNAMES[@]}"
32+
echo "$start_from_name : ${start_from}"
33+
echo "$how_many_name : ${how_many}"
34+
35+
sliced_array=(${SORTED_TEST_CLASSNAMES[@]:$start_from:$how_many})
36+
37+
TEST_ARG=$(echo ${sliced_array[@]} | sed 's/ /,/g')
38+
echo "$TEST_ARG"
39+
40+
if [[ $baseBranch == "2.1.x" ]]; then
41+
42+
./mvnw -s .settings.xml -pl '-:kubernetes-leader-election-example' -pl '-:kubernetes-hello-world-example' \
43+
-pl '-:kubernetes-reload-example' -pl '-:kubernetes-loadbalancer-example' \
44+
-pl '-:spring-cloud-kubernetes-configserver' -pl '-:spring-cloud-kubernetes-configuration-watcher' \
45+
-pl '-:spring-cloud-kubernetes-discoveryserver' \
46+
-DtestsToRun=${TEST_ARG[@]} \
47+
-e clean install \
48+
-U -P sonar -nsu --batch-mode \
49+
-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \
50+
-Dhttp.keepAlive=false \
51+
-Dmaven.wagon.http.pool=false \
52+
-Dmaven.wagon.http.retryHandler.class=standard \
53+
-Dmaven.wagon.http.retryHandler.count=3 \
54+
-Dskip.build.image=true
55+
56+
else
57+
58+
version=$(java -version)
59+
echo "version of java: $version"
60+
61+
./mvnw -s .settings.xml \
62+
-DtestsToRun=${TEST_ARG[@]} \
63+
-e clean install \
64+
-U -P sonar -nsu --batch-mode \
65+
-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \
66+
-Dhttp.keepAlive=false \
67+
-Dmaven.wagon.http.pool=false \
68+
-Dmaven.wagon.http.retryHandler.class=standard \
69+
-Dmaven.wagon.http.retryHandler.count=3 \
70+
-Dskip.build.image=true
71+
fi
72+
73+
touch /tmp/test_times_${{ env.CURRENT_INDEX }}.txt
74+
75+
for i in "${sliced_array[@]}"; do
76+
filename="${i}.txt"
77+
echo "searching for filename: ${filename}"
78+
file=$(find . -name "${filename}")
79+
echo "found file: ${file}"
80+
result=$(cat "${file}" | grep 'elapsed' | awk '{print $12, $13}')
81+
82+
echo "run test: ${i} in : ${result}" >> /tmp/test_times_${{ env.CURRENT_INDEX }}.txt
83+
done
84+
85+
- name: show individual test times
86+
shell: bash
87+
if: env.BASE_BRANCH != '2.1.x'
88+
run: cat /tmp/test_times_${{ env.CURRENT_INDEX }}.txt
89+
90+
- name: upload individual tests
91+
if: env.BASE_BRANCH != '2.1.x'
92+
uses: actions/upload-artifact@v3
93+
with:
94+
name: test_times_${{ env.CURRENT_INDEX }}.txt
95+
path: /tmp/test_times_${{ env.CURRENT_INDEX }}.txt
96+
97+

0 commit comments

Comments
 (0)