Skip to content

Commit 0b44f90

Browse files
joke1196ghislainpiot
joke1196
authored and
ghislainpiot
committed
SONARPY-1986: The IPythonSensor should register to .ipynb files
1 parent c238536 commit 0b44f90

File tree

14 files changed

+359
-18
lines changed

14 files changed

+359
-18
lines changed

‎its/plugin/it-python-plugin-test/projects/ipynb_json_project/json_notebook.ipynb

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": 1,
6+
"metadata": {},
7+
"outputs": [
8+
{
9+
"name": "stdout",
10+
"output_type": "stream",
11+
"text": [
12+
"hello world\n"
13+
]
14+
}
15+
],
16+
"source": [
17+
"x = None\n",
18+
"if x is not None:\n",
19+
" print \"not none\"\n",
20+
"\n",
21+
"\n",
22+
"def foo():\n",
23+
" x = 42\n",
24+
" x = 17\n",
25+
" a = f\"test\"\n",
26+
" print(x)"
27+
]
28+
},
29+
{
30+
"cell_type": "markdown",
31+
"metadata": {},
32+
"source": [
33+
"# Hello\n",
34+
"This is some markdown"
35+
]
36+
},
37+
{
38+
"cell_type": "markdown",
39+
"metadata": {},
40+
"source": [
41+
"This is another markdown cell"
42+
]
43+
},
44+
{
45+
"cell_type": "code",
46+
"execution_count": null,
47+
"metadata": {},
48+
"outputs": [],
49+
"source": [
50+
"if x is not None:\n",
51+
" print(\"hello\")"
52+
]
53+
},
54+
{
55+
"cell_type": "code",
56+
"execution_count": null,
57+
"metadata": {},
58+
"outputs": [],
59+
"source": [
60+
"x = 42"
61+
]
62+
}
63+
],
64+
"metadata": {
65+
"kernelspec": {
66+
"display_name": "jupyter-experiment_venv",
67+
"language": "python",
68+
"name": "python3"
69+
},
70+
"language_info": {
71+
"codemirror_mode": {
72+
"name": "ipython",
73+
"version": 3
74+
},
75+
"file_extension": ".py",
76+
"mimetype": "text/x-python",
77+
"name": "python",
78+
"nbconvert_exporter": "python",
79+
"pygments_lexer": "ipython3",
80+
"version": "3.12.2"
81+
}
82+
},
83+
"nbformat": 4,
84+
"nbformat_minor": 2
85+
}

‎its/plugin/it-python-plugin-test/src/test/java/com/sonar/python/it/plugin/NotebookPluginTest.java

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
/*
2+
* SonarQube Python Plugin
3+
* Copyright (C) 2012-2024 SonarSource SA
4+
* mailto:info AT sonarsource DOT com
5+
*
6+
* This program is free software; you can redistribute it and/or
7+
* modify it under the terms of the GNU Lesser General Public
8+
* License as published by the Free Software Foundation; either
9+
* version 3 of the License, or (at your option) any later version.
10+
*
11+
* This program is distributed in the hope that it will be useful,
12+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14+
* Lesser General Public License for more details.
15+
*
16+
* You should have received a copy of the GNU Lesser General Public License
17+
* along with this program; if not, write to the Free Software Foundation,
18+
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19+
*/
20+
package com.sonar.python.it.plugin;
21+
22+
import com.sonar.orchestrator.build.SonarScanner;
23+
import com.sonar.orchestrator.junit5.OrchestratorExtension;
24+
import java.io.File;
25+
import java.util.List;
26+
import org.junit.jupiter.api.BeforeAll;
27+
import org.junit.jupiter.api.Test;
28+
import org.junit.jupiter.api.extension.RegisterExtension;
29+
import org.sonarqube.ws.Issues;
30+
31+
import static com.sonar.python.it.plugin.TestsUtils.issues;
32+
import static org.assertj.core.api.Assertions.assertThat;
33+
34+
class NotebookPluginTest {
35+
36+
private static final String PROJECT_KEY = "ipynb_json_project";
37+
38+
@RegisterExtension
39+
public static final OrchestratorExtension ORCHESTRATOR = TestsUtils.ORCHESTRATOR;
40+
41+
@BeforeAll
42+
static void startServer() {
43+
ORCHESTRATOR.getServer().provisionProject(PROJECT_KEY, PROJECT_KEY);
44+
SonarScanner build = SonarScanner.create()
45+
.setProjectDir(new File("projects", PROJECT_KEY))
46+
.setProjectKey(PROJECT_KEY)
47+
.setProjectName(PROJECT_KEY)
48+
.setProjectVersion("1.0-SNAPSHOT")
49+
.setSourceDirs(".");
50+
ORCHESTRATOR.executeBuild(build);
51+
}
52+
53+
@Test
54+
void test() {
55+
List<Issues.Issue> issues = issues(PROJECT_KEY);
56+
assertThat(issues).isEmpty();
57+
}
58+
}
59+

‎its/ruling/src/test/java/org/sonar/python/it/PythonExtendedRulingTest.java

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,15 +38,18 @@
3838
// Ruling test for bug rules, to ensure they are properly tested without slowing down the CI
3939
class PythonExtendedRulingTest {
4040

41-
4241
@RegisterExtension
4342
public static final OrchestratorExtension ORCHESTRATOR = getOrchestrator();
4443

44+
private static final String PROFILE_NAME = "customProfile";
45+
4546
@BeforeAll
4647
static void prepare_quality_profile() throws IOException {
4748
List<String> ruleKeys = bugRuleKeys();
48-
String pythonProfile = RulingHelper.profile("customProfile", "py", "python", ruleKeys);
49+
String pythonProfile = RulingHelper.profile(PROFILE_NAME, "py", "python", ruleKeys);
4950
RulingHelper.loadProfile(ORCHESTRATOR, pythonProfile);
51+
String iPythonProfile = RulingHelper.profile(PROFILE_NAME, "ipynb", "python", ruleKeys);
52+
RulingHelper.loadProfile(ORCHESTRATOR, iPythonProfile);
5053
}
5154

5255
@Test
@@ -169,7 +172,8 @@ public SonarScanner buildWithCommonProperties(String projectKey) {
169172

170173
public SonarScanner buildWithCommonProperties(String projectKey, String projectName) {
171174
ORCHESTRATOR.getServer().provisionProject(projectKey, projectKey);
172-
ORCHESTRATOR.getServer().associateProjectToQualityProfile(projectKey, "py", "customProfile");
175+
ORCHESTRATOR.getServer().associateProjectToQualityProfile(projectKey, "py", PROFILE_NAME);
176+
ORCHESTRATOR.getServer().associateProjectToQualityProfile(projectKey, "ipynb", PROFILE_NAME);
173177
return SonarScanner.create(FileLocation.of(String.format("../sources_extended/%s", projectName)).getFile())
174178
.setProjectKey(projectKey)
175179
.setProjectName(projectKey)

‎its/ruling/src/test/java/org/sonar/python/it/PythonPrAnalysisTest.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,9 @@ static void prepare_quality_profile() throws IOException {
9292
String profile = RulingHelper.profile(INCREMENTAL_ANALYSIS_PROFILE, "py", "python", List.of("S5713"));
9393
RulingHelper.loadProfile(ORCHESTRATOR, profile);
9494
ORCHESTRATOR.getServer().associateProjectToQualityProfile(PR_ANALYSIS_PROJECT_KEY, "py", INCREMENTAL_ANALYSIS_PROFILE);
95+
String iPythonProfile = RulingHelper.profile(INCREMENTAL_ANALYSIS_PROFILE, "ipynb", "ipython", List.of("S5713"));
96+
RulingHelper.loadProfile(ORCHESTRATOR, iPythonProfile);
97+
ORCHESTRATOR.getServer().associateProjectToQualityProfile(PR_ANALYSIS_PROJECT_KEY, "ipynb", INCREMENTAL_ANALYSIS_PROFILE);
9598
}
9699

97100
// @Parameters(name = "{index}: {0}")

‎its/ruling/src/test/java/org/sonar/python/it/PythonRulingTest.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,12 +57,15 @@ static void prepare_quality_profile() {
5757
String serverUrl = ORCHESTRATOR.getServer().getUrl();
5858
File profileFile = ProfileGenerator.generateProfile(serverUrl, "py", "python", parameters, Collections.emptySet());
5959
ORCHESTRATOR.getServer().restoreProfile(FileLocation.of(profileFile));
60+
File iPythonProfileFile = ProfileGenerator.generateProfile(serverUrl, "ipynb", "python", parameters, Collections.emptySet());
61+
ORCHESTRATOR.getServer().restoreProfile(FileLocation.of(iPythonProfileFile));
6062
}
6163

6264
@Test
6365
void test() throws Exception {
6466
ORCHESTRATOR.getServer().provisionProject(PROJECT_KEY, PROJECT_KEY);
6567
ORCHESTRATOR.getServer().associateProjectToQualityProfile(PROJECT_KEY, "py", "rules");
68+
ORCHESTRATOR.getServer().associateProjectToQualityProfile(PROJECT_KEY, "ipynb", "rules");
6669
File litsDifferencesFile = FileLocation.of("target/differences").getFile();
6770
SonarScanner build = SonarScanner.create(FileLocation.of("../sources").getFile())
6871
.setProjectKey(PROJECT_KEY)

‎sonar-python-plugin/src/main/java/org/sonar/plugins/python/IPynb.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ public class IPynb extends AbstractLanguage {
2525

2626
public static final String KEY = "ipynb";
2727

28-
private static final String[] DEFAULT_FILE_SUFFIXES = { ".ipynb" };
28+
private static final String[] DEFAULT_FILE_SUFFIXES = { KEY };
2929

3030
public IPynb() {
3131
super(KEY, "IPython Notebooks");

‎sonar-python-plugin/src/main/java/org/sonar/plugins/python/IPynbSensor.java

Lines changed: 40 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@
2222
import java.util.ArrayList;
2323
import java.util.Collections;
2424
import java.util.List;
25+
import javax.annotation.Nullable;
26+
import org.sonar.api.SonarProduct;
2527
import org.sonar.api.batch.fs.FilePredicates;
2628
import org.sonar.api.batch.fs.InputFile;
2729
import org.sonar.api.batch.rule.CheckFactory;
@@ -30,9 +32,13 @@
3032
import org.sonar.api.batch.sensor.SensorDescriptor;
3133
import org.sonar.api.issue.NoSonarFilter;
3234
import org.sonar.api.measures.FileLinesContextFactory;
35+
import org.sonar.plugins.python.IpynbNotebookParser.ParseResult;
3336
import org.sonar.plugins.python.api.ProjectPythonVersion;
3437
import org.sonar.plugins.python.api.PythonVersionUtils;
38+
import org.sonar.plugins.python.api.caching.CacheContext;
3539
import org.sonar.plugins.python.indexer.PythonIndexer;
40+
import org.sonar.plugins.python.indexer.SonarQubePythonIndexer;
41+
import org.sonar.python.caching.CacheContextImpl;
3642
import org.sonar.python.checks.CheckList;
3743
import org.sonar.python.parser.PythonParser;
3844

@@ -45,7 +51,11 @@ public final class IPynbSensor implements Sensor {
4551
private final NoSonarFilter noSonarFilter;
4652
private final PythonIndexer indexer;
4753

48-
public IPynbSensor(FileLinesContextFactory fileLinesContextFactory, CheckFactory checkFactory, NoSonarFilter noSonarFilter, PythonIndexer indexer) {
54+
public IPynbSensor(FileLinesContextFactory fileLinesContextFactory, CheckFactory checkFactory, NoSonarFilter noSonarFilter) {
55+
this(fileLinesContextFactory, checkFactory, noSonarFilter, null);
56+
}
57+
58+
public IPynbSensor(FileLinesContextFactory fileLinesContextFactory, CheckFactory checkFactory, NoSonarFilter noSonarFilter, @Nullable PythonIndexer indexer) {
4959
this.checks = new PythonChecks(checkFactory)
5060
.addChecks(CheckList.IPYTHON_REPOSITORY_KEY, CheckList.getChecks());
5161
this.fileLinesContextFactory = fileLinesContextFactory;
@@ -67,10 +77,38 @@ public void execute(SensorContext context) {
6777
if (pythonVersions.length != 0) {
6878
ProjectPythonVersion.setCurrentVersions(PythonVersionUtils.fromStringArray(pythonVersions));
6979
}
70-
PythonScanner scanner = new PythonScanner(context, checks, fileLinesContextFactory, noSonarFilter, PythonParser.createIPythonParser(), indexer);
80+
if (isInSonarLintRuntime(context)) {
81+
PythonScanner scanner = new PythonScanner(context, checks, fileLinesContextFactory, noSonarFilter, PythonParser.createIPythonParser(), indexer);
82+
scanner.execute(pythonFiles, context);
83+
} else {
84+
processNotebooksFiles(pythonFiles, context);
85+
}
86+
}
87+
88+
private void processNotebooksFiles(List<PythonInputFile> pythonFiles, SensorContext context) {
89+
pythonFiles = parseNotebooks(pythonFiles);
90+
// Disable caching for IPynb files for now
91+
CacheContext cacheContext = CacheContextImpl.dummyCache();
92+
PythonIndexer pythonIndexer = new SonarQubePythonIndexer(pythonFiles, cacheContext, context);
93+
PythonScanner scanner = new PythonScanner(context, checks, fileLinesContextFactory, noSonarFilter, PythonParser.createIPythonParser(), pythonIndexer);
7194
scanner.execute(pythonFiles, context);
7295
}
7396

97+
private static List<PythonInputFile> parseNotebooks(List<PythonInputFile> pythonFiles) {
98+
List<PythonInputFile> generatedIPythonFiles = new ArrayList<>();
99+
for (PythonInputFile inputFile : pythonFiles) {
100+
ParseResult result = IpynbNotebookParser.parseNotebook(inputFile);
101+
generatedIPythonFiles.add(result.inputFile());
102+
}
103+
return generatedIPythonFiles;
104+
}
105+
106+
private static boolean isInSonarLintRuntime(SensorContext context) {
107+
// SL preprocesses notebooks and send us Python files
108+
// SQ/SC sends us the actual JSON files
109+
return context.runtime().getProduct().equals(SonarProduct.SONARLINT);
110+
}
111+
74112
private static List<PythonInputFile> getInputFiles(SensorContext context) {
75113
FilePredicates p = context.fileSystem().predicates();
76114
Iterable<InputFile> it = context.fileSystem().inputFiles(p.and(p.hasLanguage(IPynb.KEY)));

‎sonar-python-plugin/src/main/java/org/sonar/plugins/python/Python.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ public Python(Configuration configuration) {
4040

4141
@Override
4242
public String[] getFileSuffixes() {
43-
String[] suffixes = filterEmptyStrings(configuration.getStringArray(PythonPlugin.FILE_SUFFIXES_KEY));
43+
String[] suffixes = filterEmptyStrings(configuration.getStringArray(PythonPlugin.PYTHON_FILE_SUFFIXES_KEY));
4444
return suffixes.length == 0 ? Python.DEFAULT_FILE_SUFFIXES : suffixes;
4545
}
4646

‎sonar-python-plugin/src/main/java/org/sonar/plugins/python/PythonPlugin.java

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -56,13 +56,14 @@ public class PythonPlugin implements Plugin {
5656
private static final String EXTERNAL_ANALYZERS_CATEGORY = "External Analyzers";
5757
private static final String DEPRECATED_PREFIX = "DEPRECATED : Use " + PythonCoverageSensor.REPORT_PATHS_KEY + " instead. ";
5858

59-
public static final String FILE_SUFFIXES_KEY = "sonar.python.file.suffixes";
59+
public static final String PYTHON_FILE_SUFFIXES_KEY = "sonar.python.file.suffixes";
60+
public static final String IPYNB_FILE_SUFFIXES_KEY = "sonar.ipynb.file.suffixes";
6061

6162
@Override
6263
public void define(Context context) {
6364

6465
context.addExtensions(
65-
PropertyDefinition.builder(FILE_SUFFIXES_KEY)
66+
PropertyDefinition.builder(PYTHON_FILE_SUFFIXES_KEY)
6667
.index(10)
6768
.name("File Suffixes")
6869
.description("List of suffixes of Python files to analyze.")
@@ -73,6 +74,7 @@ public void define(Context context) {
7374
.defaultValue("py")
7475
.build(),
7576

77+
7678
PropertyDefinition.builder(PYTHON_VERSION_KEY)
7779
.index(11)
7880
.name("Python versions")
@@ -91,6 +93,22 @@ public void define(Context context) {
9193
PythonRuleRepository.class,
9294
AnalysisWarningsWrapper.class);
9395

96+
context.addExtensions(
97+
PropertyDefinition.builder(IPYNB_FILE_SUFFIXES_KEY)
98+
.index(12)
99+
.name("IPython File Suffixes")
100+
.description("List of suffixes of IPython Notebooks files to analyze.")
101+
.multiValues(true)
102+
.category(PYTHON_CATEGORY)
103+
.subCategory(GENERAL)
104+
.onQualifiers(Qualifiers.PROJECT)
105+
.defaultValue("ipynb")
106+
.build(),
107+
IPynb.class,
108+
IPynbProfile.class,
109+
IPynbSensor.class,
110+
IPynbRuleRepository.class);
111+
94112
SonarRuntime sonarRuntime = context.getRuntime();
95113
if (sonarRuntime.getProduct() != SonarProduct.SONARLINT) {
96114
addCoberturaExtensions(context);
@@ -101,14 +119,11 @@ public void define(Context context) {
101119
addMypyExtensions(context);
102120
addRuffExtensions(context);
103121
}
122+
123+
104124
if (sonarRuntime.getProduct() == SonarProduct.SONARLINT) {
105125
SonarLintPluginAPIManager sonarLintPluginAPIManager = new SonarLintPluginAPIManager();
106126
sonarLintPluginAPIManager.addSonarlintPythonIndexer(context, new SonarLintPluginAPIVersion());
107-
context.addExtensions(
108-
IPynb.class,
109-
IPynbProfile.class,
110-
IPynbSensor.class,
111-
IPynbRuleRepository.class);
112127
}
113128
}
114129

0 commit comments

Comments
 (0)