Skip to content

Commit 7c587ee

Browse files
davewichersdavewichers
authored
Merge pull request #196 from zoobinn/runSonarQubesh-9.9LTS
make runSonarqube.sh to work on LTS9.9
2 parents f5e90e4 + 340f4da commit 7c587ee

File tree

1 file changed

+61
-28
lines changed

1 file changed

+61
-28
lines changed

‎scripts/runSonarQube.sh

Lines changed: 61 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,25 @@
11
#!/usr/bin/env bash
2+
## This script Launches the benchmark, populates results on Sonarqube Dashboard and then fetch same results back from the SonarQube Server using SONAR Host,Project and Token
3+
## This Script is totaly experimental. Tested Against SonarQube Enterprise Server version 9.9 LTS
4+
## To run SonarQube benchmark you need to be on the /BenchmarkJava path and run ./scripts/runSonarQube.sh
25

36
source scripts/requireCommand.sh
47

58
requireCommand curl
69
requireCommand jq
710

811
# Check for install/updates at https://github.com/SonarSource/sonarqube
12+
# This is Page size, If facing JQ Errors due to Long Arguments, Decrease this Number. Tested with SonarQube 9.9 LTS, 50 and 100 where producing lots of errors,
13+
elements_per_request=20
914

10-
if [ ! -f scripts/SonarQubeCredentials.sh ]; then
11-
cat > scripts/SonarQubeCredentials.sh << EOF
15+
if [ ! -f scripts/SonarQubeCredentials.sh ]; then cat > scripts/SonarQubeCredentials.sh << EOF
1216
#!/usr/bin/env bash
13-
1417
sonar_host="" # e. g. http://localhost:9000
1518
sonar_project=""
1619
sonar_token=""
1720
EOF
18-
chmod +x scripts/SonarQubeCredentials.sh
19-
fi
21+
chmod +x scripts/SonarQubeCredentials.sh
22+
fi
2023

2124
source scripts/SonarQubeCredentials.sh
2225

@@ -27,7 +30,7 @@ fi
2730

2831
mvn sonar:sonar -Dsonar.projectKey="$sonar_project" -Dsonar.host.url="$sonar_host" -Dsonar.login="$sonar_token"
2932

30-
sleep 300s # might be replaced with polling of $sonar_host/api/ce/component?component=$sonar_project
33+
sleep 300
3134

3235
benchmark_version=$(scripts/getBenchmarkVersion.sh)
3336
sonarqube_version=$(curl --silent -u "$sonar_token:" "$sonar_host/api/server/version")
@@ -38,35 +41,65 @@ result_file="results/Benchmark_$benchmark_version-sonarqube-v$sonarqube_version.
3841
result='{"issues":[], "hotspots": []}'
3942
rules='[]'
4043

41-
# sonarqube does not allow us to grab more than 10k issues, but most of them are information exposure which is not even
42-
# considered by benchmark so let's just get all relevant rules and receive results for only those rules
4344

44-
rules_count=$(curl --silent -u "$sonar_token:" "$sonar_host/api/rules/search?p=1&ps=1" | jq -r '.total')
45-
page=1
46-
47-
while (((page - 1) * 500 < rules_count)); do
48-
rules=$(echo "$rules" | jq ". += $(curl --silent -u "$sonar_token:" "$sonar_host/api/rules/search?p=$page&ps=500" | jq '.rules | map( .key ) | map( select(. | contains("java:") ) )')")
49-
page=$((page+1))
50-
done
45+
## WE ARE GOING TO DISCARD RULE CHERRY PICKING. SO ALL RESULTS ARE REPORTED REGARDLESS SO THAT BENCHMARK CAN POPULATE RESULTS & SCORE ACCORDINGLY.
46+
## The content/data structure returned is controled by SONARQUEBE end server, Benchmark Script picks them accordingly and match them back to test cases and create the score.
47+
## If returned data are not structured in a way expected by Benchmark/Score calculator. Example: CWE/DataPoint missed then results will not be counted/scored. This can end up in in-correct/Lower Score calculation.
48+
## rules_count=$(curl --silent -u "$sonar_token:" "$sonar_host/api/rules/search?p=1&ps=1" | jq -r '.total')
49+
##page=1
50+
##echo "rule count is: $rules_count"
5151

52-
rules=$(echo "$rules" | jq '. | join(",")' | sed 's/java:S1989,//')
52+
## while (((page - 1) * elements_per_request < rules_count)); do
53+
## rules=$(echo "$rules" | jq ". += $(curl --silent -u "$sonar_token:" "$sonar_host/api/rules/search?p=$page&ps=$elements_per_request" | jq '.rules | map( .key ) | map( select(. | contains("java:") ) )')")
54+
## page=$((page+1))
55+
## echo "rule page: $page"
56+
## sleep 1;
57+
## done
58+
## rules=$(echo "$rules" | jq '. | join(",")' | sed 's/java:S1989,//')
5359

54-
issues_count=$(curl --silent -u "$sonar_token:" "$sonar_host/api/issues/search?p=1&ps=1&types=VULNERABILITY&componentKeys=$sonar_project&rules=$rules" | jq -r '.paging.total')
60+
issues_count=$(curl --silent -u "$sonar_token:" "$sonar_host/api/issues/search?p=1&ps=1&types=VULNERABILITY&componentKeys=$sonar_project" | jq -r '.paging.total')
5561
page=1
5662

57-
while (((page - 1) * 500 < issues_count)); do
58-
issues_page=$(curl --silent -u "$sonar_token:" "$sonar_host/api/issues/search?types=VULNERABILITY&p=$page&ps=500&componentKeys=$sonar_project&rules=$rules" | jq '.issues')
59-
60-
result=$(echo "$result" | jq ".issues += $issues_page")
61-
page=$((page+1))
63+
echo "Vulnerability Issue count is: $issues_count"
64+
65+
## We are using two files to write results to. One as buffer the other as final to incrementally add results and swap in-between.
66+
## This helps to have some sort of fault tolerance. If jq hits long argument or sonarqube sends back impaired data/empty for a single page, previous progress of result collection will not be erased/lost retroactively.
67+
echo '{"issues":[], "hotspots": []}' > buffdump.json;
68+
echo '{"issues":[], "hotspots": []}' > resdump.json;
69+
70+
while (((page - 1) * elements_per_request < issues_count)); do
71+
cat resdump.json > buffdump.json;
72+
itemcount=$(($page * $elements_per_request))
73+
echo "processing Vulnerablity issues, page: $page up to $itemcount items out of total $issues_count"
74+
issues_page=$(curl --silent -u "$sonar_token:" "$sonar_host/api/issues/search?types=VULNERABILITY&p=$page&ps=$elements_per_request&componentKeys=$sonar_project" | jq '.issues')
75+
if [ "$issues_page" ]; then
76+
cat buffdump.json | jq ".issues += ${issues_page}" > resdump.json;
77+
else
78+
echo "Empty. Error reading Vulnerability issues at Page:$page !"
79+
fi
80+
page=$((page+1))
6281
done
63-
64-
hotspot_count=$(curl --silent -u "$sonar_token:" "$sonar_host/api/hotspots/search?projectKey=benchmark&p=1&ps=1" | jq -r '.paging.total')
82+
83+
hotspot_count=$(curl --silent -u "$sonar_token:" "$sonar_host/api/hotspots/search?projectKey=$sonar_project&p=1&ps=1" | jq -r '.paging.total')
6584
page=1
66-
67-
while (((page - 1) * 500 < hotspot_count)); do
68-
result=$(echo "$result" | jq ".hotspots += $(curl --silent -u "$sonar_token:" "$sonar_host/api/hotspots/search?projectKey=$sonar_project&p=$page&ps=500" | jq '.hotspots')")
85+
echo "Hotspot Count is: $hotspot_count"
86+
87+
cat resdump.json > buffdump.json
88+
while (((page - 1) * elements_per_request < hotspot_count)); do
89+
cat resdump.json > buffdump.json
90+
itemcount=$(($page * $elements_per_request))
91+
echo "processing Hotspots, page: $page up to $itemcount items out of total $hotspot_count"
92+
hotspot_page=$(curl --silent -u "$sonar_token:" "$sonar_host/api/hotspots/search?projectKey=$sonar_project&p=$page&ps=$elements_per_request" | jq '.hotspots')
93+
if [ "$hotspot_page" ]; then
94+
cat buffdump.json | jq ".hotspots += ${hotspot_page}" > resdump.json;
95+
else
96+
echo "Empty. Error reading Hotspot at Page:$page !"
97+
fi
6998
page=$((page+1))
7099
done
100+
echo "Writing end results json content";
101+
cp resdump.json "${result_file}";
102+
echo "Done, please go ahead an generate the scorecard";
103+
## cleanup the two files generated to record results, if want them for debug, you can comment the following line
104+
rm resdump.json buffdump.json;
71105

72-
echo "$result" > "$result_file"

0 commit comments

Comments
 (0)