Skip to content

Commit 1dc6376

Browse files
guillaume-dequenneguillaume-dequenne
committed
SONARPY-974 Add extended ruling test for bug rules
1 parent 72d7959 commit 1dc6376

File tree

6 files changed

+340
-10
lines changed

6 files changed

+340
-10
lines changed

‎.cirrus.yml

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -139,9 +139,33 @@ ruling_task:
139139
- source cirrus-env QA
140140
- source set_maven_build_version $BUILD_NUMBER
141141
- cd its/ruling
142-
- mvn verify -Dsonar.runtimeVersion=LATEST_RELEASE -Dmaven.test.redirectTestOutputToFile=false -B -e -V
142+
- mvn verify -Dsonar.runtimeVersion=LATEST_RELEASE -Dmaven.test.redirectTestOutputToFile=false -B -e -V -Dtest=PythonRulingTest
143143
cleanup_before_cache_script: cleanup_maven_repository
144-
144+
145+
146+
extended_ruling_task:
147+
depends_on:
148+
- build
149+
only_if: $CIRRUS_PR != "" || $CIRRUS_BRANCH == "master" || $CIRRUS_BRANCH =~ "branch-.*"
150+
gke_container:
151+
<<: *CONTAINER_DEFINITION
152+
cpu: 4
153+
memory: 8G
154+
env:
155+
CIRRUS_CLONE_DEPTH: 10
156+
SONARSOURCE_QA: true
157+
maven_cache:
158+
folder: ${CIRRUS_WORKING_DIR}/.m2/repository
159+
submodules_script:
160+
- git submodule update --init
161+
ruling_script:
162+
- source cirrus-env QA
163+
- source set_maven_build_version $BUILD_NUMBER
164+
- cd its/ruling
165+
- mvn verify -Dsonar.runtimeVersion=LATEST_RELEASE -Dmaven.test.redirectTestOutputToFile=false -B -e -V -Dtest=PythonExtendedRulingTest
166+
cleanup_before_cache_script: cleanup_maven_repository
167+
168+
145169
promote_task:
146170
depends_on:
147171
- ruling

‎.gitmodules

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
[submodule "its/sources"]
22
path = its/sources
33
url = https://github.com/SonarCommunity/python-test-sources.git
4+
[submodule "its/sources_extended"]
5+
path = its/sources_extended
6+
url = https://github.com/SonarCommunity/python-test-sources.git
7+
branch = extended
48
[submodule "tools/typeshed_serializer/resources/typeshed"]
59
path = python-frontend/typeshed_serializer/resources/typeshed
610
url = https://github.com/python/typeshed.git

‎its/ruling/src/test/java/org/sonar/python/it/PythonExtendedRulingTest.java

Lines changed: 211 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,211 @@
1+
/*
2+
* SonarQube Python Plugin
3+
* Copyright (C) 2012-2022 SonarSource SA
4+
* mailto:info AT sonarsource DOT com
5+
*
6+
* This program is free software; you can redistribute it and/or
7+
* modify it under the terms of the GNU Lesser General Public
8+
* License as published by the Free Software Foundation; either
9+
* version 3 of the License, or (at your option) any later version.
10+
*
11+
* This program is distributed in the hope that it will be useful,
12+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14+
* Lesser General Public License for more details.
15+
*
16+
* You should have received a copy of the GNU Lesser General Public License
17+
* along with this program; if not, write to the Free Software Foundation,
18+
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19+
*/
20+
package org.sonar.python.it;
21+
22+
import com.sonar.orchestrator.Orchestrator;
23+
import com.sonar.orchestrator.build.SonarScanner;
24+
import com.sonar.orchestrator.locator.FileLocation;
25+
import java.io.File;
26+
import java.io.IOException;
27+
import java.nio.file.Files;
28+
import java.util.List;
29+
import org.junit.BeforeClass;
30+
import org.junit.ClassRule;
31+
import org.junit.Test;
32+
33+
import static java.nio.charset.StandardCharsets.UTF_8;
34+
import static org.assertj.core.api.Assertions.assertThat;
35+
import static org.sonar.python.it.RulingHelper.bugRuleKeys;
36+
import static org.sonar.python.it.RulingHelper.getOrchestrator;
37+
38+
// Ruling test for bug rules, to ensure they are properly tested without slowing down the CI
39+
public class PythonExtendedRulingTest {
40+
41+
@ClassRule
42+
public static final Orchestrator ORCHESTRATOR = getOrchestrator();
43+
44+
@BeforeClass
45+
public static void prepare_quality_profile() throws IOException {
46+
List<String> ruleKeys = bugRuleKeys();
47+
String pythonProfile = profile("py", "python", ruleKeys);
48+
loadProfile(ORCHESTRATOR, pythonProfile);
49+
}
50+
51+
@Test
52+
public void test_airflow() throws IOException {
53+
SonarScanner build = buildWithCommonProperties("airflow");
54+
build.setProperty("sonar.sources", "airflow");
55+
build.setProperty("sonar.tests", "tests");
56+
executeBuild(build);
57+
}
58+
59+
@Test
60+
public void test_archery() throws IOException {
61+
executeBuild(buildWithCommonProperties("Archery"));
62+
}
63+
64+
@Test
65+
public void test_autokeras() throws IOException {
66+
executeBuild(buildWithCommonProperties("autokeras"));
67+
}
68+
69+
@Test
70+
public void test_black() throws IOException {
71+
SonarScanner build = buildWithCommonProperties("black");
72+
build.setProperty("sonar.sources", "src");
73+
build.setProperty("sonar.tests", "tests");
74+
build.setProperty("sonar.test.exclusions", "tests/data/async_as_identifier.py");
75+
executeBuild(build);
76+
}
77+
78+
@Test
79+
public void test_calibre() throws IOException {
80+
SonarScanner build = buildWithCommonProperties("calibre");
81+
build.setProperty("sonar.sources", "src");
82+
executeBuild(build);
83+
}
84+
85+
@Test
86+
public void test_chalice() throws IOException {
87+
SonarScanner build = buildWithCommonProperties("chalice");
88+
build.setProperty("sonar.sources", "chalice");
89+
build.setProperty("sonar.tests", "tests");
90+
executeBuild(build);
91+
}
92+
93+
@Test
94+
public void test_django_shop() throws IOException {
95+
SonarScanner build = buildWithCommonProperties("django-shop");
96+
build.setProperty("sonar.sources", "shop");
97+
build.setProperty("sonar.tests", "tests");
98+
executeBuild(build);
99+
}
100+
101+
@Test
102+
public void test_indico() throws IOException {
103+
SonarScanner build = buildWithCommonProperties("indico");
104+
build.setProperty("sonar.sources", "indico");
105+
executeBuild(build);
106+
}
107+
108+
@Test
109+
public void test_LibCST() throws IOException {
110+
SonarScanner build = buildWithCommonProperties("LibCST");
111+
build.setProperty("sonar.sources", "libcst");
112+
build.setProperty("sonar.tests", "libcst/tests");
113+
build.setProperty("sonar.test.inclusions", "**/");
114+
executeBuild(build);
115+
}
116+
117+
@Test
118+
public void test_nltk() throws IOException {
119+
SonarScanner build = buildWithCommonProperties("nltk");
120+
build.setProperty("sonar.sources", ".");
121+
build.setProperty("sonar.exclusions", "**/test/**/*");
122+
executeBuild(build);
123+
}
124+
125+
@Test
126+
public void test_saleor() throws IOException {
127+
SonarScanner build = buildWithCommonProperties("saleor");
128+
build.setProperty("sonar.sources", "saleor");
129+
executeBuild(build);
130+
}
131+
132+
@Test
133+
public void test_salt() throws IOException {
134+
SonarScanner build = buildWithCommonProperties("salt");
135+
build.setProperty("sonar.sources", "salt");
136+
build.setProperty("sonar.tests", "tests");
137+
executeBuild(build);
138+
}
139+
140+
@Test
141+
public void test_scikit_learn() throws IOException {
142+
SonarScanner build = buildWithCommonProperties("scikit-learn");
143+
build.setProperty("sonar.sources", "sklearn");
144+
executeBuild(build);
145+
}
146+
147+
@Test
148+
public void test_timesketch() throws IOException {
149+
SonarScanner build = buildWithCommonProperties("timesketch");
150+
build.setProperty("sonar.sources", "timesketch");
151+
build.setProperty("sonar.test.inclusions", "**/*_test.py");
152+
executeBuild(build);
153+
}
154+
155+
public SonarScanner buildWithCommonProperties(String projectKey) {
156+
return buildWithCommonProperties(projectKey, projectKey);
157+
}
158+
159+
public SonarScanner buildWithCommonProperties(String projectKey, String projectName) {
160+
ORCHESTRATOR.getServer().provisionProject(projectKey, projectKey);
161+
ORCHESTRATOR.getServer().associateProjectToQualityProfile(projectKey, "py", "customProfile");
162+
return SonarScanner.create(FileLocation.of(String.format("../sources_extended/%s", projectName)).getFile())
163+
.setProjectKey(projectKey)
164+
.setProjectName(projectKey)
165+
.setProjectVersion("1")
166+
.setLanguage("py")
167+
.setSourceEncoding("UTF-8")
168+
.setSourceDirs(".")
169+
.setProperty("sonar.lits.dump.old", FileLocation.of(String.format("src/test/resources/expected_extended/%s", projectKey)).getFile().getAbsolutePath())
170+
.setProperty("sonar.lits.dump.new", FileLocation.of(String.format("target/actual_extended/%s", projectKey)).getFile().getAbsolutePath())
171+
.setProperty("sonar.cpd.exclusions", "**/*")
172+
.setProperty("sonar.internal.analysis.failFast", "true")
173+
.setEnvironmentVariable("SONAR_RUNNER_OPTS", "-Xmx2000m");
174+
}
175+
176+
void executeBuild(SonarScanner build) throws IOException {
177+
File litsDifferencesFile = FileLocation.of("target/differences").getFile();
178+
build.setProperty("sonar.lits.differences", litsDifferencesFile.getAbsolutePath());
179+
ORCHESTRATOR.executeBuild(build);
180+
String litsDifferences = new String(Files.readAllBytes(litsDifferencesFile.toPath()), UTF_8);
181+
assertThat(litsDifferences).isEmpty();
182+
}
183+
184+
private static String profile(String language, String repositoryKey, List<String> ruleKeys) {
185+
StringBuilder sb = new StringBuilder()
186+
.append("<profile>")
187+
.append("<name>").append("customProfile").append("</name>")
188+
.append("<language>").append(language).append("</language>")
189+
.append("<rules>");
190+
ruleKeys.forEach(ruleKey -> {
191+
sb.append("<rule>")
192+
.append("<repositoryKey>").append(repositoryKey).append("</repositoryKey>")
193+
.append("<key>").append(ruleKey).append("</key>")
194+
.append("<priority>INFO</priority>")
195+
.append("</rule>");
196+
});
197+
198+
return sb
199+
.append("</rules>")
200+
.append("</profile>")
201+
.toString();
202+
}
203+
204+
private static void loadProfile(Orchestrator orchestrator, String profile) throws IOException {
205+
File file = File.createTempFile("profile", ".xml");
206+
Files.write(file.toPath(), profile.getBytes());
207+
orchestrator.getServer().restoreProfile(FileLocation.of(file));
208+
file.delete();
209+
}
210+
211+
}

‎its/ruling/src/test/java/org/sonar/python/it/PythonRulingTest.java

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
import com.sonar.orchestrator.Orchestrator;
2323
import com.sonar.orchestrator.build.SonarScanner;
2424
import com.sonar.orchestrator.locator.FileLocation;
25-
import com.sonar.orchestrator.locator.MavenLocation;
2625
import java.io.File;
2726
import java.nio.file.Files;
2827
import java.util.Collections;
@@ -33,19 +32,14 @@
3332

3433
import static java.nio.charset.StandardCharsets.UTF_8;
3534
import static org.assertj.core.api.Assertions.assertThat;
35+
import static org.sonar.python.it.RulingHelper.getOrchestrator;
3636

3737
public class PythonRulingTest {
3838

39-
private static final String SQ_VERSION_PROPERTY = "sonar.runtimeVersion";
40-
private static final String DEFAULT_SQ_VERSION = "LATEST_RELEASE";
4139
public static final String PROJECT_KEY = "project";
4240

4341
@ClassRule
44-
public static final Orchestrator ORCHESTRATOR = Orchestrator.builderEnv()
45-
.setSonarVersion(System.getProperty(SQ_VERSION_PROPERTY, DEFAULT_SQ_VERSION))
46-
.addPlugin(FileLocation.byWildcardMavenFilename(new File("../../sonar-python-plugin/target"), "sonar-python-plugin-*.jar"))
47-
.addPlugin(MavenLocation.of("org.sonarsource.sonar-lits-plugin", "sonar-lits-plugin", "0.10.0.2181"))
48-
.build();
42+
public static final Orchestrator ORCHESTRATOR = getOrchestrator();
4943

5044
@BeforeClass
5145
public static void prepare_quality_profile() {

‎its/ruling/src/test/java/org/sonar/python/it/RulingHelper.java

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
/*
2+
* SonarQube Python Plugin
3+
* Copyright (C) 2012-2022 SonarSource SA
4+
* mailto:info AT sonarsource DOT com
5+
*
6+
* This program is free software; you can redistribute it and/or
7+
* modify it under the terms of the GNU Lesser General Public
8+
* License as published by the Free Software Foundation; either
9+
* version 3 of the License, or (at your option) any later version.
10+
*
11+
* This program is distributed in the hope that it will be useful,
12+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14+
* Lesser General Public License for more details.
15+
*
16+
* You should have received a copy of the GNU Lesser General Public License
17+
* along with this program; if not, write to the Free Software Foundation,
18+
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19+
*/
20+
package org.sonar.python.it;
21+
22+
import com.sonar.orchestrator.Orchestrator;
23+
import com.sonar.orchestrator.locator.FileLocation;
24+
import com.sonar.orchestrator.locator.MavenLocation;
25+
import java.io.File;
26+
import java.util.Arrays;
27+
import java.util.List;
28+
29+
public class RulingHelper {
30+
31+
private static final String SQ_VERSION_PROPERTY = "sonar.runtimeVersion";
32+
private static final String DEFAULT_SQ_VERSION = "LATEST_RELEASE";
33+
34+
static Orchestrator getOrchestrator() {
35+
return Orchestrator.builderEnv()
36+
.setSonarVersion(System.getProperty(SQ_VERSION_PROPERTY, DEFAULT_SQ_VERSION))
37+
.addPlugin(FileLocation.byWildcardMavenFilename(new File("../../sonar-python-plugin/target"), "sonar-python-plugin-*.jar"))
38+
.addPlugin(MavenLocation.of("org.sonarsource.sonar-lits-plugin", "sonar-lits-plugin", "0.10.0.2181"))
39+
.build();
40+
}
41+
42+
// TODO: SONARPY-984, read rules metadata instead of hardcoding this list
43+
static List<String> bugRuleKeys() {
44+
return Arrays.asList(
45+
"PreIncrementDecrement",
46+
"S1045",
47+
"S1143",
48+
"S1226",
49+
"S1656",
50+
"S1716",
51+
"S1751",
52+
"S1763",
53+
"S1764",
54+
"S1862",
55+
"S2159",
56+
"S2190",
57+
"S2201",
58+
"S2275",
59+
"S2711",
60+
"S2712",
61+
"S2734",
62+
"S2757",
63+
"S2823",
64+
"S3403",
65+
"S3827",
66+
"S3862",
67+
"S3923",
68+
"S3981",
69+
"S3984",
70+
"S4143",
71+
"S5549",
72+
"S5607",
73+
"S5632",
74+
"S5644",
75+
"S5707",
76+
"S5708",
77+
"S5714",
78+
"S5719",
79+
"S5722",
80+
"S5724",
81+
"S5756",
82+
"S5796",
83+
"S5807",
84+
"S5828",
85+
"S5842",
86+
"S5850",
87+
"S5855",
88+
"S5856",
89+
"S5868",
90+
"S5996",
91+
"S6002",
92+
"S905",
93+
"S930"
94+
);
95+
}
96+
}

‎its/sources_extended

Submodule sources_extended added at 66c3a5d

0 commit comments

Comments
 (0)