SONARTEXT-6 Do not analyze every non binary file in SonarQube or SonarCloud context (#109)

Author: nils-werner-sonarsource

its/ruling/src/test/java/org/sonarsource/text/TextRulingTest.java

@@ -72,6 +72,7 @@ public void test() throws Exception {
       .setSourceEncoding("UTF-8")
       .setProperty("dump.old", FileLocation.of("src/test/resources/expected").getFile().getAbsolutePath())
       .setProperty("dump.new", FileLocation.of("target/actual").getFile().getAbsolutePath())
+      .setProperty("sonar.text.analyzeAllFiles", "true")
       .setProperty("sonar.cpd.exclusions", "**/*")
       .setProperty("lits.differences", LITS_DIFFERENCES_FILE.getAbsolutePath());
     build.setEnvironmentVariable("SONAR_RUNNER_OPTS", "-Xmx1000m");

sonar-text-plugin/src/main/java/org/sonar/plugins/common/TextAndSecretsSensor.java

@@ -24,6 +24,8 @@
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
+import org.sonar.api.SonarProduct;
+import org.sonar.api.batch.fs.FilePredicate;
 import org.sonar.api.batch.fs.FileSystem;
 import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.rule.CheckFactory;
@@ -42,8 +44,12 @@ public class TextAndSecretsSensor implements Sensor {
 
   public static final String EXCLUDED_FILE_SUFFIXES_KEY = "sonar.text.excluded.file.suffixes";
 
+  private static final String ANALYZE_ALL_FILES_KEY = "sonar.text.analyzeAllFiles";
+
   public static final String TEXT_CATEGORY = "Secrets";
 
+  private static final FilePredicate LANGUAGE_FILE_PREDICATE = inputFile -> inputFile.language() != null;
+
   private final CheckFactory checkFactory;
 
   private boolean displayHelpAboutExcludingBinaryFile = true;
@@ -68,17 +74,19 @@ public void execute(SensorContext sensorContext) {
     }
 
     NotBinaryFilePredicate notBinaryFilePredicate = binaryFilePredicate(sensorContext);
-    List<InputFile> allInputFiles = getAllInputFiles(sensorContext, notBinaryFilePredicate);
-    if (allInputFiles.isEmpty()) {
+    FilePredicate filePredicate = isSonarLintContext(sensorContext) || analyzeAllFiles(sensorContext)
+      ? notBinaryFilePredicate : LANGUAGE_FILE_PREDICATE;
+    List<InputFile> inputFiles = getInputFiles(sensorContext, filePredicate);
+    if (inputFiles.isEmpty()) {
       return;
     }
 
-    List<String> filenames = allInputFiles.stream().map(InputFile::toString).collect(Collectors.toList());
+    List<String> filenames = inputFiles.stream().map(InputFile::toString).collect(Collectors.toList());
     ProgressReport progressReport = new ProgressReport("Progress of the text and secrets analysis", TimeUnit.SECONDS.toMillis(10));
     progressReport.start(filenames);
     boolean cancelled = false;
     try {
-      for (InputFile inputFile : allInputFiles) {
+      for (InputFile inputFile : inputFiles) {
         if (sensorContext.isCancelled()) {
           cancelled = true;
           break;
@@ -100,13 +108,25 @@ private static NotBinaryFilePredicate binaryFilePredicate(SensorContext sensorCo
     return new NotBinaryFilePredicate(sensorContext.config().getStringArray(TextAndSecretsSensor.EXCLUDED_FILE_SUFFIXES_KEY));
   }
 
-  private static List<InputFile> getAllInputFiles(SensorContext sensorContext, NotBinaryFilePredicate notBinaryFilePredicate) {
-    List<InputFile> allInputFiles = new ArrayList<>();
+  private static boolean isSonarLintContext(SensorContext sensorContext) {
+    return sensorContext.runtime().getProduct().equals(SonarProduct.SONARLINT);
+  }
+
+  private static boolean analyzeAllFiles(SensorContext sensorContext) {
+    return "true".equals(sensorContext.config().get(ANALYZE_ALL_FILES_KEY).orElse("false"));
+  }
+
+  /**
+   * In SonarLint context we want to analyze all non-binary input files, even when they are not analyzed or assigned to a language.
+   * To avoid analyzing all non-binary files to reduce time and memory consumption in a non SonarLint context only files assigned to a language are analyzed.
+   */
+  private static List<InputFile> getInputFiles(SensorContext sensorContext, FilePredicate filePredicate) {
+    List<InputFile> inputFiles = new ArrayList<>();
     FileSystem fileSystem = sensorContext.fileSystem();
-    for (InputFile inputFile : fileSystem.inputFiles(notBinaryFilePredicate)) {
-      allInputFiles.add(inputFile);
+    for (InputFile inputFile : fileSystem.inputFiles(filePredicate)) {
+      inputFiles.add(inputFile);
     }
-    return allInputFiles;
+    return inputFiles;
   }
 
   private void analyze(SensorContext sensorContext, List<Check> activeChecks, InputFile inputFile, NotBinaryFilePredicate notBinaryFilePredicate) {

sonar-text-plugin/src/test/java/org/sonar/plugins/common/TestUtils.java

@@ -28,6 +28,10 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import javax.annotation.Nullable;
+import org.sonar.api.SonarEdition;
+import org.sonar.api.SonarProduct;
+import org.sonar.api.SonarQubeSide;
+import org.sonar.api.SonarRuntime;
 import org.sonar.api.batch.fs.InputFile;
 import org.sonar.api.batch.fs.TextRange;
 import org.sonar.api.batch.fs.internal.TestInputFileBuilder;
@@ -37,14 +41,20 @@
 import org.sonar.api.batch.sensor.internal.SensorContextTester;
 import org.sonar.api.batch.sensor.issue.Issue;
 import org.sonar.api.batch.sensor.issue.IssueLocation;
+import org.sonar.api.internal.SonarRuntimeImpl;
 import org.sonar.api.rule.RuleKey;
+import org.sonar.api.utils.Version;
 import org.sonar.plugins.secrets.SecretsRulesDefinition;
 import org.sonar.plugins.text.TextRuleDefinition;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 public class TestUtils {
 
+  private static final Version VERSION = Version.create(9, 9);
+  public static final SonarRuntime SONARLINT_RUNTIME = SonarRuntimeImpl.forSonarLint(VERSION);
+  public static final SonarRuntime SONARQUBE_RUNTIME = SonarRuntimeImpl.forSonarQube(VERSION, SonarQubeSide.SERVER, SonarEdition.COMMUNITY);
+
   public static List<String> analyze(Check check, String fileContent) throws IOException {
     return analyze(check, inputFile(fileContent));
   }
@@ -133,8 +143,12 @@ public static SensorContextTester sensorContext(String... activeRules) {
     return sensorContext(new File(".").getAbsoluteFile(), activeRules);
   }
 
+  /**
+   * By default, the SonarLint runtime is chosen because it allows to analyze all files and not only files assigned to a language
+   */
   public static SensorContextTester sensorContext(File baseDir, String... activeRules) {
     return SensorContextTester.create(baseDir)
+      .setRuntime(SONARLINT_RUNTIME)
       .setActiveRules(activeRules(activeRules));
   }
 

sonar-text-plugin/src/test/java/org/sonar/plugins/common/TextAndSecretsSensorTest.java

@@ -35,6 +35,7 @@
 import org.sonar.api.batch.sensor.error.AnalysisError;
 import org.sonar.api.batch.sensor.internal.DefaultSensorDescriptor;
 import org.sonar.api.batch.sensor.internal.SensorContextTester;
+import org.sonar.api.config.internal.MapSettings;
 import org.sonar.api.utils.log.LogTesterJUnit5;
 import org.sonar.check.Rule;
 import org.sonar.plugins.text.api.TextCheck;
@@ -51,6 +52,8 @@
 
 class TextAndSecretsSensorTest {
 
+  private static final String SENSITIVE_BIDI_CHARS = "\u0002\u0004";
+
   @RegisterExtension
   LogTesterJUnit5 logTester = new LogTesterJUnit5();
 
@@ -201,6 +204,36 @@ public void analyze(InputFileContext ctx) {
     }
   }
 
+  @Test
+  void shouldNotAnalyzeNonLanguageAssignedFilesInSonarQubeContext() {
+    Check check = new ReportIssueAtLineOneCheck();
+    SensorContextTester context = sensorContext(check);
+    context.setRuntime(TestUtils.SONARQUBE_RUNTIME);
+    analyse(sensor(check), context, inputFile(Path.of("Foo.java"), SENSITIVE_BIDI_CHARS, null));
+    assertThat(logTester.logs()).isEmpty();
+  }
+
+  @Test
+  void shouldAnalyzeLanguageAssignedFilesInSonarQubeContext() {
+    Check check = new ReportIssueAtLineOneCheck();
+    SensorContextTester context = sensorContext(check);
+    context.setRuntime(TestUtils.SONARQUBE_RUNTIME);
+    analyse(sensor(check), context, inputFile(Path.of("Foo.java"), SENSITIVE_BIDI_CHARS, "java"));
+    assertThat(logTester.logs()).containsExactly(
+      "1 source file to be analyzed",
+      "1/1 source file has been analyzed");
+  }
+
+  @Test
+  void shouldNotAnalyzeNonLanguageAssignedFilesInSonarQubeContextWhenPropertyIsSet() {
+    Check check = new ReportIssueAtLineOneCheck();
+    SensorContextTester context = sensorContext(check);
+    context.setRuntime(TestUtils.SONARQUBE_RUNTIME);
+    context.setSettings(new MapSettings().setProperty("sonar.text.analyzeAllFiles", true));
+    analyse(sensor(check), context, inputFile(Path.of("Foo.java"), SENSITIVE_BIDI_CHARS, null));
+    assertThat(logTester.logs()).contains("1 source file to be analyzed");
+  }
+
   @Test
   void should_not_execute_checks_on_binary_file_names() {
     Check check = new BoomCheck();
@@ -215,7 +248,7 @@ void should_not_execute_checks_on_binary_file_names() {
   void should_not_exclude_binary_file_content_if_language_is_not_null() throws IOException {
     Check check = new ReportIssueAtLineOneCheck();
     SensorContextTester context = defaultSensorContext();
-    analyse(sensor(check), context, inputFile(Path.of("Foo.java"), "\u0002\u0004", "java"));
+    analyse(sensor(check), context, inputFile(Path.of("Foo.java"), SENSITIVE_BIDI_CHARS, "java"));
 
     assertThat(asString(context.allIssues())).containsExactly(
       "text:IssueAtLineOne [1:0-1:2] testIssue");
@@ -229,8 +262,8 @@ void should_exclude_binary_file_content_if_language_is_null_and_exclude_the_exte
     Check check = new ReportIssueAtLineOneCheck();
     SensorContextTester context = defaultSensorContext();
     analyse(sensor(check), context,
-      inputFile(Path.of("Foo.txt"), "\u0002\u0004", null),
-      inputFile(Path.of("FileWithoutExtension"), "\u0002\u0004", null));
+      inputFile(Path.of("Foo.txt"), SENSITIVE_BIDI_CHARS, null),
+      inputFile(Path.of("FileWithoutExtension"), SENSITIVE_BIDI_CHARS, null));
 
     assertThat(asString(context.allIssues())).isEmpty();
     assertThat(logTester.logs()).containsExactlyInAnyOrder(