Skip to content

Commit 4591081

Browse files
davewichersdavewichers
committed
Change SonarQube script to the one run with Docker.
1 parent 2d37d5a commit 4591081

File tree

1 file changed

+81
-72
lines changed

1 file changed

+81
-72
lines changed

‎scripts/runSonarQube.sh

Lines changed: 81 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,71 @@
11
#!/usr/bin/env bash
2-
## This script Launches the benchmark, populates results on Sonarqube Dashboard and then fetch same results back from the SonarQube Server using SONAR Host,Project and Token
3-
## This Script is totaly experimental. Tested Against SonarQube Enterprise Server version 9.9 LTS
4-
## To run SonarQube benchmark you need to be on the /BenchmarkJava path and run ./scripts/runSonarQube.sh
2+
3+
# this script will
4+
# - create a sonarqube server using the default configuration
5+
# - setup basic things (account, project, token)
6+
# - start a scan (takes >= 1 hour on mac)
7+
# - create a report file
8+
# - shutdown sonarqube server
59

610
source scripts/requireCommand.sh
711

812
requireCommand curl
13+
requireCommand docker
914
requireCommand jq
1015

1116
# Check for install/updates at https://github.com/SonarSource/sonarqube
12-
# This is Page size, If facing JQ Errors due to Long Arguments, Decrease this Number. Tested with SonarQube 9.9 LTS, 50 and 100 where producing lots of errors,
13-
elements_per_request=20
1417

15-
if [ ! -f scripts/SonarQubeCredentials.sh ]; then cat > scripts/SonarQubeCredentials.sh << EOF
16-
#!/usr/bin/env bash
17-
sonar_host="" # e. g. http://localhost:9000
18-
sonar_project=""
19-
sonar_token=""
20-
EOF
21-
chmod +x scripts/SonarQubeCredentials.sh
22-
fi
18+
sonar_port="9876"
19+
sonar_host="http://localhost:$sonar_port"
20+
sonar_project="benchmark"
21+
sonar_user="admin"
22+
sonar_default_password="admin"
23+
sonar_password="password"
24+
25+
echo "Creating temporary SonarQube instance"
26+
27+
docker pull sonarqube
28+
29+
# start local sonarqube
30+
container_id=$(docker run --rm -d -e SONAR_ES_BOOTSTRAP_CHECKS_DISABLE=true -p "$sonar_port:9000" sonarqube)
31+
32+
echo "Waiting for instance to come up"
33+
34+
# wait for container to come up
35+
while [[ "$(curl --connect-timeout 5 --max-time 5 --retry 60 --retry-delay 0 --retry-max-time 120 -s -o /dev/null -w '%{http_code}' "$sonar_host")" != "200" ]]; do
36+
sleep 3;
37+
done
38+
39+
# wait for sonarqube to be started
40+
while [[ "$(curl --silent "$sonar_host/api/system/status" | jq -r '.status')" != "UP" ]]; do
41+
sleep 3;
42+
done
43+
44+
echo "Setting up instance"
45+
46+
# change default password
47+
curl "$sonar_host/api/users/change_password" --silent -u "$sonar_user:$sonar_default_password" -X POST --data-raw "login=$sonar_user&password=$sonar_password&previousPassword=$sonar_default_password" -o /dev/null
48+
49+
# create project
50+
curl "$sonar_host/api/projects/create" --silent -u "$sonar_user:$sonar_password" -X POST --data-raw "project=$sonar_project&name=$sonar_project" -o /dev/null
2351

24-
source scripts/SonarQubeCredentials.sh
52+
# create token
53+
sonar_token=$(curl "$sonar_host/api/user_tokens/generate" --silent -u "$sonar_user:$sonar_password" -X POST --data-raw "name=$(date)" | jq -r '.token')
2554

26-
if [ -z "$sonar_host" ] || [ -z "$sonar_project" ] || [ -z "$sonar_token" ]; then
27-
echo "Please provide credentials in SonarQubeCredentials.sh"
28-
exit 1
29-
fi
55+
echo "Starting scan (might take some time!)"
3056

31-
mvn sonar:sonar -Dsonar.projectKey="$sonar_project" -Dsonar.host.url="$sonar_host" -Dsonar.login="$sonar_token"
57+
# run scan (using net=host to be able to connect to localhost sonarqube)
58+
docker run --env SONAR_SCANNER_OPTS=-Xmx4g --net=host --rm -v ~/.m2:/root/.m2 -v "$(pwd)":"$(pwd)" -w "$(pwd)" sonarsource/sonar-scanner-cli \
59+
-Dsonar.java.binaries="target" -Dsonar.projectKey="$sonar_project" -Dsonar.host.url="$sonar_host" -Dsonar.login="$sonar_token" \
60+
-Dsonar.sources="src" -Dsonar.exclusions="results/**,scorecard/**,scripts/**,tools/**,VMs/**"
3261

33-
sleep 300
62+
echo "Waiting for SonarQube CE to finish task"
63+
64+
while [[ "$(curl --silent -u "$sonar_token:" "$sonar_host/api/ce/component?component=$sonar_project" | jq -r '.current.status')" != "SUCCESS" ]]; do
65+
sleep 3;
66+
done
67+
68+
echo "Generating report..."
3469

3570
benchmark_version=$(scripts/getBenchmarkVersion.sh)
3671
sonarqube_version=$(curl --silent -u "$sonar_token:" "$sonar_host/api/server/version")
@@ -41,65 +76,39 @@ result_file="results/Benchmark_$benchmark_version-sonarqube-v$sonarqube_version.
4176
result='{"issues":[], "hotspots": []}'
4277
rules='[]'
4378

79+
# sonarqube does not allow us to grab more than 10k issues, but most of them are information exposure which is not even
80+
# considered by benchmark so let's just get all relevant rules and receive results for only those rules
4481

45-
## WE ARE GOING TO DISCARD RULE CHERRY PICKING. SO ALL RESULTS ARE REPORTED REGARDLESS SO THAT BENCHMARK CAN POPULATE RESULTS & SCORE ACCORDINGLY.
46-
## The content/data structure returned is controled by SONARQUEBE end server, Benchmark Script picks them accordingly and match them back to test cases and create the score.
47-
## If returned data are not structured in a way expected by Benchmark/Score calculator. Example: CWE/DataPoint missed then results will not be counted/scored. This can end up in in-correct/Lower Score calculation.
48-
## rules_count=$(curl --silent -u "$sonar_token:" "$sonar_host/api/rules/search?p=1&ps=1" | jq -r '.total')
49-
##page=1
50-
##echo "rule count is: $rules_count"
82+
rules_count=$(curl --silent -u "$sonar_token:" "$sonar_host/api/rules/search?p=1&ps=1" | jq -r '.total')
83+
page=1
84+
85+
while (((page - 1) * 500 < rules_count)); do
86+
rules=$(echo "$rules" | jq ". += $(curl --silent -u "$sonar_token:" "$sonar_host/api/rules/search?p=$page&ps=500" | jq '.rules | map( .key ) | map( select(. | contains("java:") ) )')")
87+
page=$((page+1))
88+
done
5189

52-
## while (((page - 1) * elements_per_request < rules_count)); do
53-
## rules=$(echo "$rules" | jq ". += $(curl --silent -u "$sonar_token:" "$sonar_host/api/rules/search?p=$page&ps=$elements_per_request" | jq '.rules | map( .key ) | map( select(. | contains("java:") ) )')")
54-
## page=$((page+1))
55-
## echo "rule page: $page"
56-
## sleep 1;
57-
## done
58-
## rules=$(echo "$rules" | jq '. | join(",")' | sed 's/java:S1989,//')
90+
rules=$(echo "$rules" | jq '. | join(",")' | sed 's/java:S1989,//')
5991

60-
issues_count=$(curl --silent -u "$sonar_token:" "$sonar_host/api/issues/search?p=1&ps=1&types=VULNERABILITY&componentKeys=$sonar_project" | jq -r '.paging.total')
92+
issues_count=$(curl --silent -u "$sonar_token:" "$sonar_host/api/issues/search?p=1&ps=1&types=VULNERABILITY&componentKeys=$sonar_project&rules=$rules" | jq -r '.paging.total')
6193
page=1
6294

63-
echo "Vulnerability Issue count is: $issues_count"
64-
65-
## We are using two files to write results to. One as buffer the other as final to incrementally add results and swap in-between.
66-
## This helps to have some sort of fault tolerance. If jq hits long argument or sonarqube sends back impaired data/empty for a single page, previous progress of result collection will not be erased/lost retroactively.
67-
echo '{"issues":[], "hotspots": []}' > buffdump.json;
68-
echo '{"issues":[], "hotspots": []}' > resdump.json;
69-
70-
while (((page - 1) * elements_per_request < issues_count)); do
71-
cat resdump.json > buffdump.json;
72-
itemcount=$(($page * $elements_per_request))
73-
echo "processing Vulnerablity issues, page: $page up to $itemcount items out of total $issues_count"
74-
issues_page=$(curl --silent -u "$sonar_token:" "$sonar_host/api/issues/search?types=VULNERABILITY&p=$page&ps=$elements_per_request&componentKeys=$sonar_project" | jq '.issues')
75-
if [ "$issues_page" ]; then
76-
cat buffdump.json | jq ".issues += ${issues_page}" > resdump.json;
77-
else
78-
echo "Empty. Error reading Vulnerability issues at Page:$page !"
79-
fi
80-
page=$((page+1))
95+
while (((page - 1) * 500 < issues_count)); do
96+
issues_page=$(curl --silent -u "$sonar_token:" "$sonar_host/api/issues/search?types=VULNERABILITY&p=$page&ps=500&componentKeys=$sonar_project&rules=$rules" | jq '.issues')
97+
98+
result=$(echo "$result" | jq ".issues += $issues_page")
99+
page=$((page+1))
81100
done
82-
83-
hotspot_count=$(curl --silent -u "$sonar_token:" "$sonar_host/api/hotspots/search?projectKey=$sonar_project&p=1&ps=1" | jq -r '.paging.total')
101+
102+
hotspot_count=$(curl --silent -u "$sonar_token:" "$sonar_host/api/hotspots/search?projectKey=benchmark&p=1&ps=1" | jq -r '.paging.total')
84103
page=1
85-
echo "Hotspot Count is: $hotspot_count"
86-
87-
cat resdump.json > buffdump.json
88-
while (((page - 1) * elements_per_request < hotspot_count)); do
89-
cat resdump.json > buffdump.json
90-
itemcount=$(($page * $elements_per_request))
91-
echo "processing Hotspots, page: $page up to $itemcount items out of total $hotspot_count"
92-
hotspot_page=$(curl --silent -u "$sonar_token:" "$sonar_host/api/hotspots/search?projectKey=$sonar_project&p=$page&ps=$elements_per_request" | jq '.hotspots')
93-
if [ "$hotspot_page" ]; then
94-
cat buffdump.json | jq ".hotspots += ${hotspot_page}" > resdump.json;
95-
else
96-
echo "Empty. Error reading Hotspot at Page:$page !"
97-
fi
104+
105+
while (((page - 1) * 500 < hotspot_count)); do
106+
result=$(echo "$result" | jq ".hotspots += $(curl --silent -u "$sonar_token:" "$sonar_host/api/hotspots/search?projectKey=$sonar_project&p=$page&ps=500" | jq '.hotspots')")
98107
page=$((page+1))
99108
done
100-
echo "Writing end results json content";
101-
cp resdump.json "${result_file}";
102-
echo "Done, please go ahead and generate the scorecard";
103-
## cleanup the two files generated to record results, if you want them for debug purposes, comment out the following line
104-
rm resdump.json buffdump.json;
105109

110+
echo "$result" > "$result_file"
111+
112+
echo "Shutting down SonarQube"
113+
114+
docker stop "$container_id"

0 commit comments

Comments
 (0)