SONARPY-1231 Cache CPD tokens (#1325)

Author: gyula-sallai-sonarsource

its/ruling/src/test/java/org/sonar/python/it/PythonPrAnalysisTest.java

@@ -43,6 +43,7 @@
 
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.sonar.python.it.RulingHelper.getMeasure;
 import static org.sonar.python.it.RulingHelper.getOrchestrator;
 
 @RunWith(Parameterized.class)
@@ -53,6 +54,8 @@ public class PythonPrAnalysisTest {
 
   private static final String PR_ANALYSIS_PROJECT_KEY = "prAnalysis";
   private static final String INCREMENTAL_ANALYSIS_PROFILE = "incrementalPrAnalysis";
+  private static final String PR_KEY = "1";
+  private static final String PR_BRANCH_NAME = "incremental";
 
   @Rule
   public TemporaryFolder temporaryFolder = new TemporaryFolder();
@@ -62,13 +65,15 @@ public class PythonPrAnalysisTest {
   private final int expectedRecomputed;
   private final int expectedSkipped;
   private final List<String> deletedFiles;
+  private final Integer expectedDuplicatedLines;
 
-  public PythonPrAnalysisTest(String scenario, int expectedTotalFiles, int expectedRecomputed, int expectedSkipped, List<String> deletedFiles) {
+  public PythonPrAnalysisTest(String scenario, int expectedTotalFiles, int expectedRecomputed, int expectedSkipped, List<String> deletedFiles, Integer expectedDuplication) {
     this.scenario = scenario;
     this.expectedTotalFiles = expectedTotalFiles;
     this.expectedRecomputed = expectedRecomputed;
     this.expectedSkipped = expectedSkipped;
     this.deletedFiles = deletedFiles;
+    this.expectedDuplicatedLines = expectedDuplication;
   }
 
   @BeforeClass
@@ -83,15 +88,16 @@ public static void prepare_quality_profile() throws IOException {
 
   @Parameters(name = "{index}: {0}")
   public static Collection<Object[]> data() {
-    return List.of(new Object[][] {
-      // {<scenario>, <total files>, <recomputed>, <skipped>, <deleted>}
-      {"newFile", 10, 1, 9, Collections.emptyList()},
-      {"changeInImportedModule", 9, 1, 7, Collections.emptyList()},
-      {"changeInParent", 9, 1, 6, Collections.emptyList()},
-      {"changeInPackageInit", 9, 1, 7, Collections.emptyList()},
-      {"changeInRelativeImport", 9, 2, 4, Collections.emptyList()},
-      {"deletedFile", 8, 0, 7, List.of("submodule.py")}}
-    );
+    return List.of(new Object[][]{
+      // {<scenario>, <total files>, <recomputed>, <skipped>, <deleted>, <duplication on index.py>}
+      {"newFile", 10, 1, 9, Collections.emptyList(), null},
+      {"changeInImportedModule", 9, 1, 7, Collections.emptyList(), null},
+      {"changeInParent", 9, 1, 6, Collections.emptyList(), null},
+      {"changeInPackageInit", 9, 1, 7, Collections.emptyList(), null},
+      {"changeInRelativeImport", 9, 2, 4, Collections.emptyList(), null},
+      {"deletedFile", 8, 0, 7, List.of("submodule.py"), null},
+      {"duplication", 10, 1, 9, Collections.emptyList(), 55}
+    });
   }
 
   @Test
@@ -105,11 +111,12 @@ public void pr_analysis_logs() throws IOException {
     // Analyze the changed branch
     setUpChanges(tempDirectory, scenario);
     SonarScanner build = prepareScanner(tempDirectory, PR_ANALYSIS_PROJECT_KEY, scenario, litsDifferencesFile)
-      .setProperty("sonar.pullrequest.key", "1")
-      .setProperty("sonar.pullrequest.branch", "incremental");
+      .setProperty("sonar.pullrequest.key", PR_KEY)
+      .setProperty("sonar.pullrequest.branch", PR_BRANCH_NAME);
 
     BuildResult result = ORCHESTRATOR.executeBuild(build);
     assertPrAnalysisLogs(result);
+    assertMeasures();
   }
 
   @Test
@@ -153,6 +160,16 @@ private void assertPrAnalysisLogs(BuildResult result) {
       .contains(expectedFinalLog);
   }
 
+  private void assertMeasures() {
+    if (expectedDuplicatedLines != null) {
+      var duplicatedLines = getMeasure(ORCHESTRATOR, PR_KEY, PR_ANALYSIS_PROJECT_KEY + ":index_duplicate.py", "duplicated_lines");
+      assertThat(duplicatedLines)
+        .isNotNull();
+      assertThat(Integer.parseInt(duplicatedLines.getValue()))
+        .isEqualTo(expectedDuplicatedLines);
+    }
+  }
+
   private void analyzeAndAssertBaseCommit(File tempFile, File litsDifferencesFile) throws IOException {
     FileUtils.copyDirectory(new File("../sources_pr_analysis", "baseCommit"), tempFile);
 
@@ -178,7 +195,6 @@ private SonarScanner prepareScanner(File path, String projectKey, String scenari
       .setSourceDirs(".")
       .setProperty("sonar.lits.dump.old", FileLocation.of("src/test/resources/expected_pr_analysis/" + scenario).getFile().getAbsolutePath())
       .setProperty("sonar.lits.dump.new", FileLocation.of("target/actual").getFile().getAbsolutePath())
-      .setProperty("sonar.cpd.exclusions", "**/*")
       .setProperty("sonar.lits.differences", litsDifferencesFile.getAbsolutePath())
       .setProperty("sonar.internal.analysis.failFast", "true")
       .setEnvironmentVariable("SONAR_RUNNER_OPTS", "-Xmx2000m");

its/ruling/src/test/java/org/sonar/python/it/RulingHelper.java

@@ -29,6 +29,14 @@
 import java.nio.file.Files;
 import java.util.Arrays;
 import java.util.List;
+import org.sonarqube.ws.Measures;
+import org.sonarqube.ws.client.HttpConnector;
+import org.sonarqube.ws.client.WsClient;
+import org.sonarqube.ws.client.WsClientFactories;
+import org.sonarqube.ws.client.measures.ComponentRequest;
+
+import static java.lang.Double.parseDouble;
+import static java.util.Collections.singletonList;
 
 public class RulingHelper {
 
@@ -139,4 +147,24 @@ static List<String> bugRuleKeys() {
       "S930"
     );
   }
+
+  static Measures.Measure getMeasure(Orchestrator orchestrator, String branch, String componentKey, String metricKey) {
+    List<Measures.Measure> measures = getMeasures(orchestrator, branch, componentKey, singletonList(metricKey));
+    return measures != null && measures.size() == 1 ? measures.get(0) : null;
+  }
+
+  private static List<Measures.Measure> getMeasures(Orchestrator orchestrator, String prKey, String componentKey, List<String> metricKeys) {
+    Measures.ComponentWsResponse response = newWsClient(orchestrator).measures().component(new ComponentRequest()
+      .setComponent(componentKey)
+      .setPullRequest(prKey)
+      .setMetricKeys(metricKeys));
+    return response.getComponent().getMeasuresList();
+  }
+
+  static WsClient newWsClient(Orchestrator orchestrator) {
+    return WsClientFactories.getDefault().newClient(HttpConnector.newBuilder()
+      .url(orchestrator.getServer().getUrl())
+      .credentials(null, null)
+      .build());
+  }
 }

its/ruling/src/test/resources/expected_pr_analysis/duplication/python-S5713.json

@@ -0,0 +1,9 @@
+{
+'prAnalysis:index_duplicate.py':[
+9,
+19,
+29,
+39,
+49
+],
+}

its/sources_pr_analysis/duplication/index_duplicate.py

@@ -0,0 +1,55 @@
+from submodule import SubmoduleException
+from subpackage.parent import ParentException
+from subpackage import SubpackageException
+from relative_imports.module import RelativeImportException
+from relative_imports.sub.inner import RelativeImportSubpackageException
+
+try:
+    raise SubmoduleException()
+except (SubmoduleException, NotImplementedError): # Noncompliant
+    print("Foo")
+
+try:
+    raise SubmoduleException()
+except (SubmoduleException, ValueError): # OK
+    print("Foo")
+
+try:
+    raise ParentException()
+except (ParentException, NotImplementedError): # Noncompliant
+    print("Foo")
+
+try:
+    raise ParentException()
+except (ParentException, ValueError): # OK
+    print("Foo")
+
+try:
+    raise SubpackageException()
+except (SubpackageException, NotImplementedError): # Noncompliant
+    print("Foo")
+
+try:
+    raise SubpackageException()
+except (SubpackageException, ValueError): # OK
+    print("Foo")
+
+try:
+    raise RelativeImportException()
+except (RelativeImportException, NotImplementedError): # Noncompliant
+    print("Foo")
+
+try:
+    raise RelativeImportException()
+except (RelativeImportException, ValueError): # OK
+    print("Foo")
+
+try:
+    raise RelativeImportSubpackageException()
+except (RelativeImportSubpackageException, NotImplementedError): # Noncompliant
+    print("Foo")
+
+try:
+    raise RelativeImportSubpackageException()
+except (RelativeImportSubpackageException, ValueError): # OK
+    print("Foo")

python-frontend/src/main/java/org/sonar/python/caching/CpdSerializer.java

@@ -0,0 +1,76 @@
+/*
+ * SonarQube Python Plugin
+ * Copyright (C) 2011-2022 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.python.caching;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+import java.util.List;
+import java.util.stream.Collectors;
+import org.sonar.plugins.python.api.tree.Token;
+import org.sonar.python.TokenLocation;
+
+public class CpdSerializer {
+
+  private CpdSerializer() {
+    // Prevent instantiation
+  }
+
+  public static final class TokenInfo implements Serializable {
+    public final int startLine;
+    public final int startLineOffset;
+    public final int endLine;
+    public final int endLineOffset;
+    public final String value;
+
+    public static TokenInfo from(Token token) {
+      TokenLocation location = new TokenLocation(token);
+      return new TokenInfo(location.startLine(), location.startLineOffset(), location.endLine(), location.endLineOffset(), token.value());
+    }
+
+    public TokenInfo(int startLine, int startLineOffset, int endLine, int endLineOffset, String value) {
+      this.startLine = startLine;
+      this.startLineOffset = startLineOffset;
+      this.endLine = endLine;
+      this.endLineOffset = endLineOffset;
+      this.value = value;
+    }
+  }
+
+  public static byte[] toBytes(List<Token> tokens) throws IOException {
+    List<TokenInfo> tokenInfos = tokens.stream()
+      .map(TokenInfo::from)
+      .collect(Collectors.toList());
+
+    ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+    ObjectOutputStream objectOutputStream = new ObjectOutputStream(byteArrayOutputStream);
+    objectOutputStream.writeObject(tokenInfos);
+
+    return byteArrayOutputStream.toByteArray();
+  }
+
+  public static List<TokenInfo> fromBytes(byte[] bytes) throws IOException, ClassNotFoundException {
+    ObjectInputStream objectInputStream = new ObjectInputStream(new ByteArrayInputStream(bytes));
+    return (List<TokenInfo>) objectInputStream.readObject();
+  }
+}

python-frontend/src/test/java/org/sonar/python/caching/CpdSerializerTest.java

@@ -0,0 +1,54 @@
+/*
+ * SonarQube Python Plugin
+ * Copyright (C) 2011-2022 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.python.caching;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.List;
+import org.junit.Test;
+import org.sonar.plugins.python.api.tree.Token;
+import org.sonar.python.api.PythonKeyword;
+import org.sonar.python.tree.TokenImpl;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class CpdSerializerTest {
+
+  @Test
+  public void to_bytes_from_bytes() throws IOException, ClassNotFoundException {
+    var sslrToken = com.sonar.sslr.api.Token.builder()
+      .setLine(1)
+      .setColumn(0)
+      .setValueAndOriginalValue("pass")
+      .setURI(URI.create(""))
+      .setType(PythonKeyword.PASS)
+      .build();
+
+    List<Token> tokens = List.of(new TokenImpl(sslrToken));
+    byte[] bytes = CpdSerializer.toBytes(tokens);
+
+    List<CpdSerializer.TokenInfo> tokenInfos = CpdSerializer.fromBytes(bytes);
+
+    assertThat(tokenInfos)
+      .hasSize(1);
+    assertThat(tokenInfos.get(0))
+      .usingRecursiveComparison().isEqualTo(new CpdSerializer.TokenInfo(1, 0, 1, 4, "pass"));
+  }
+}

sonar-python-plugin/src/main/java/org/sonar/plugins/python/PythonScanner.java

@@ -152,7 +152,8 @@ public boolean scanFileWithoutParsing(InputFile inputFile) {
         return false;
       }
     }
-    return true;
+
+    return restoreAndPushMeasuresIfApplicable(inputFile);
   }
 
   @Override
@@ -296,6 +297,14 @@ private void saveMeasures(InputFile inputFile, PythonVisitorContext visitorConte
     }
   }
 
+  private boolean restoreAndPushMeasuresIfApplicable(InputFile inputFile) {
+    if (inputFile.type() == InputFile.Type.TEST) {
+      return true;
+    }
+
+    return cpdAnalyzer.pushCachedCpdTokens(inputFile, indexer.cacheContext());
+  }
+
   private void saveMetricOnFile(InputFile inputFile, Metric<Integer> metric, Integer value) {
     context.<Integer>newMeasure()
       .withValue(value)

sonar-python-plugin/src/main/java/org/sonar/plugins/python/caching/Caching.java

@@ -45,6 +45,7 @@ public class Caching {
   public static final String PROJECT_FILES_KEY = "python:files";
   public static final String TYPESHED_MODULES_KEY = "python:typeshed_modules";
   public static final String CACHE_VERSION_KEY = "python:cache_version";
+  public static final String CPD_TOKENS_CACHE_KEY_PREFIX = "python:cpd:data:";
 
   private static final Logger LOG = Loggers.get(Caching.class);