SONARPY-1233 Allow import of mypy reports (#1431)

Author: gyula-sallai-sonarsource

its/plugin/it-python-plugin-test/projects/mypy_project/mypy_output.txt

@@ -0,0 +1,7 @@
+src/type_hints_noncompliant.py:11:11: error: Argument 1 to "greet_all" has incompatible type "List[int]"; expected "List[str]"  [arg-type]
+src/type_hints_noncompliant.py:13:1: error: Function is missing a type annotation  [no-untyped-def]
+src/type_hints_noncompliant.py:16:1: error: Cannot find implementation or library stub for module named "unknown"  [import]
+src/type_hints_noncompliant.py:16:1: note: See https://mypy.readthedocs.io/en/stable/running_mypy.html#missing-imports
+src/type_hints_noncompliant.py:19:11: error: Call to untyped function "no_type_hints" in typed context  [no-untyped-call]
+src/type_hints_noncompliant.py:24: error: Unused "type: ignore" comment
+Found 5 errors in 1 file (checked 1 source file)

its/plugin/it-python-plugin-test/projects/mypy_project/sonar-project.properties

@@ -0,0 +1,9 @@
+# Required metadata
+sonar.projectKey=mypy_project
+sonar.projectName=mypy_project
+sonar.projectVersion=1
+sonar.sourceEncoding=UTF8
+
+# Comma-separated paths to directories with sources (required)
+sonar.sources=src
+sonar.python.mypy.reportPaths=mypy_output.txt

its/plugin/it-python-plugin-test/projects/mypy_project/src/type_hints_noncompliant.py

@@ -0,0 +1,25 @@
+# Example from the mypy documentation
+
+def greet_all(names: list[str]) -> None:
+    for name in names:
+        print('Hello ' + name)
+
+names = ["Alice", "Bob", "Charlie"]
+ages = [10, 20, 30]
+
+greet_all(names)
+greet_all(ages)
+
+def no_type_hints(x):
+    return [x]
+
+from unknown import unknown
+
+if __name__ == "__name__":
+    print(no_type_hints(0))
+    greet_all(unknown())
+
+from typing import List
+
+def ignore_comment() -> List[str]: # type: ignore
+    return []

its/plugin/it-python-plugin-test/src/test/java/com/sonar/python/it/plugin/MypyReportTest.java

@@ -0,0 +1,66 @@
+/*
+ * SonarQube Python Plugin
+ * Copyright (C) 2012-2023 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package com.sonar.python.it.plugin;
+
+import com.sonar.orchestrator.Orchestrator;
+import com.sonar.orchestrator.build.SonarScanner;
+import java.io.File;
+import java.util.List;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.sonarqube.ws.Common;
+import org.sonarqube.ws.Issues;
+
+import static com.sonar.python.it.plugin.Tests.issues;
+import static org.assertj.core.api.Assertions.assertThat;
+
+public class MypyReportTest {
+
+  private static final String PROJECT = "mypy_project";
+
+  @ClassRule
+  public static final Orchestrator ORCHESTRATOR = Tests.ORCHESTRATOR;
+
+  @Test
+  public void import_report() {
+    ORCHESTRATOR.getServer().provisionProject(PROJECT, PROJECT);
+    ORCHESTRATOR.getServer().associateProjectToQualityProfile(PROJECT, "py", "no_rule");
+    ORCHESTRATOR.executeBuild(
+      SonarScanner.create()
+        .setProjectDir(new File("projects/mypy_project")));
+
+    List<Issues.Issue> issues = issues(PROJECT);
+    assertThat(issues).hasSize(5);
+    assertIssue(issues.get(0), "external_mypy:arg-type", "Argument 1 to \"greet_all\" has incompatible type \"List[int]\"; expected \"List[str]\"");
+    assertIssue(issues.get(1), "external_mypy:no-untyped-def", "Function is missing a type annotation");
+    assertIssue(issues.get(2), "external_mypy:import", "Cannot find implementation or library stub for module named \"unknown\"");
+    assertIssue(issues.get(3), "external_mypy:no-untyped-call", "Call to untyped function \"no_type_hints\" in typed context");
+    assertIssue(issues.get(4), "external_mypy:????", "Unused \"type: ignore\" comment");
+  }
+
+  private static void assertIssue(Issues.Issue issue, String rule, String message) {
+    assertThat(issue.getComponent()).isEqualTo("mypy_project:src/type_hints_noncompliant.py");
+    assertThat(issue.getRule()).isEqualTo(rule);
+    assertThat(issue.getMessage()).isEqualTo(message);
+    assertThat(issue.getType()).isEqualTo(Common.RuleType.CODE_SMELL);
+    assertThat(issue.getSeverity()).isEqualTo(Common.Severity.MAJOR);
+    assertThat(issue.getEffort()).isEqualTo("5min");
+  }
+}

sonar-python-plugin/src/main/java/org/sonar/plugins/python/PythonPlugin.java

@@ -33,6 +33,8 @@
 import org.sonar.plugins.python.flake8.Flake8RulesDefinition;
 import org.sonar.plugins.python.flake8.Flake8Sensor;
 import org.sonar.plugins.python.indexer.SonarLintPythonIndexer;
+import org.sonar.plugins.python.mypy.MypyRulesDefinition;
+import org.sonar.plugins.python.mypy.MypySensor;
 import org.sonar.plugins.python.pylint.PylintRulesDefinition;
 import org.sonar.plugins.python.pylint.PylintSensor;
 import org.sonar.plugins.python.warnings.AnalysisWarningsWrapper;
@@ -82,6 +84,7 @@ public void define(Context context) {
       addPylintExtensions(context);
       addBanditExtensions(context);
       addFlake8Extensions(context);
+      addMypyExtensions(context);
     }
     if (sonarRuntime.getProduct() == SonarProduct.SONARLINT) {
       SonarLintPluginAPIManager sonarLintPluginAPIManager = new SonarLintPluginAPIManager();
@@ -184,6 +187,19 @@ private static void addFlake8Extensions(Context context) {
       Flake8RulesDefinition.class);
   }
 
+  private static void addMypyExtensions(Context context) {
+    context.addExtensions(MypySensor.class,
+      PropertyDefinition.builder(MypySensor.REPORT_PATH_KEY)
+        .name("Mypy Report Files")
+        .description("Paths (absolute or relative) to report files with Mypy issues.")
+        .category(EXTERNAL_ANALYZERS_CATEGORY)
+        .subCategory(PYTHON_CATEGORY)
+        .onQualifiers(Qualifiers.PROJECT)
+        .multiValues(true)
+        .build(),
+      MypyRulesDefinition.class);
+  }
+
   static class SonarLintPluginAPIManager {
 
     public void addSonarlintPythonIndexer(Context context, SonarLintPluginAPIVersion sonarLintPluginAPIVersion) {

sonar-python-plugin/src/main/java/org/sonar/plugins/python/mypy/MypyRulesDefinition.java

@@ -0,0 +1,37 @@
+/*
+ * SonarQube Python Plugin
+ * Copyright (C) 2011-2023 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.plugins.python.mypy;
+
+import org.sonar.api.server.rule.RulesDefinition;
+import org.sonar.plugins.python.Python;
+import org.sonarsource.analyzer.commons.ExternalRuleLoader;
+
+public class MypyRulesDefinition implements RulesDefinition {
+
+  private static final String RULES_JSON = "org/sonar/plugins/python/mypy/rules.json";
+
+  private static final ExternalRuleLoader RULE_LOADER = new ExternalRuleLoader(MypySensor.LINTER_KEY, MypySensor.LINTER_NAME, RULES_JSON, Python.KEY);
+
+  @Override
+  public void define(RulesDefinition.Context context) {
+    RULE_LOADER.createExternalRuleRepository(context);
+  }
+
+}

sonar-python-plugin/src/main/java/org/sonar/plugins/python/mypy/MypySensor.java

@@ -0,0 +1,127 @@
+/*
+ * SonarQube Python Plugin
+ * Copyright (C) 2011-2023 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.plugins.python.mypy;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.Scanner;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.sonar.api.batch.fs.FileSystem;
+import org.sonar.api.batch.sensor.SensorContext;
+import org.sonar.api.config.Configuration;
+import org.sonar.api.utils.log.Logger;
+import org.sonar.api.utils.log.Loggers;
+import org.sonar.plugins.python.ExternalIssuesSensor;
+import org.sonar.plugins.python.TextReportReader;
+
+public class MypySensor extends ExternalIssuesSensor {
+
+  private static final Logger LOG = Loggers.get(MypySensor.class);
+
+  public static final String LINTER_NAME = "Mypy";
+  public static final String LINTER_KEY = "mypy";
+  public static final String REPORT_PATH_KEY = "sonar.python.mypy.reportPaths";
+
+  private static final String FALLBACK_RULE_KEY = "unknown_mypy_rule";
+
+  // Pattern -> Location ': ' Severity ':' Message '['Code']'
+  // Location -> File ':' StartLine
+  private static final Pattern PATTERN =
+    Pattern.compile("^(?<file>[^:]+):(?<startLine>\\d+)(?::(?<startCol>\\d+))?(?::\\d+:\\d+)?: (?<severity>\\S+[^:]): (?<message>.*?)(?: \\[(?<code>.*)])?\\s*$");
+
+  @Override
+  protected void importReport(File reportPath, SensorContext context, Set<String> unresolvedInputFiles) throws IOException {
+    List<TextReportReader.Issue> issues = parse(reportPath, context.fileSystem());
+    issues.forEach(i -> saveIssue(context, i, unresolvedInputFiles, LINTER_KEY));
+  }
+
+  private static List<TextReportReader.Issue> parse(File report, FileSystem fileSystem) throws IOException {
+    List<TextReportReader.Issue> issues = new ArrayList<>();
+    try (Scanner scanner = new Scanner(report.toPath(), fileSystem.encoding().name())) {
+      while (scanner.hasNextLine()) {
+        TextReportReader.Issue issue = parseLine(scanner.nextLine());
+        if (issue != null) {
+          issues.add(issue);
+        }
+      }
+    }
+    return issues;
+  }
+
+  private static TextReportReader.Issue parseLine(String line) {
+    if (line.length() > 0) {
+      Matcher m = PATTERN.matcher(line);
+      if (m.matches()) {
+        return extractIssue(m);
+      }
+      LOG.debug("Cannot parse the line: {}", line);
+    }
+
+    return null;
+  }
+
+  private static TextReportReader.Issue extractIssue(Matcher m) {
+    String severity = m.group("severity");
+    if (!"error".equals(severity)) {
+      return null;
+    }
+
+    String filePath = m.group("file");
+    int lineNumber = Integer.parseInt(m.group("startLine"));
+    String message = m.group("message");
+    String errorCode = m.group("code");
+    if (errorCode == null) {
+      // Sometimes mypy does not report an error code, however the API expects a non-null error code.
+      errorCode = FALLBACK_RULE_KEY;
+    }
+
+    Integer columnNumber = Optional.ofNullable(m.group("startCol"))
+      .map(Integer::parseInt)
+      .map(i -> i - 1)
+      .orElse(null);
+
+    return new TextReportReader.Issue(filePath, errorCode, message, lineNumber, columnNumber);
+  }
+
+  @Override
+  protected boolean shouldExecute(Configuration conf) {
+    return conf.hasKey(REPORT_PATH_KEY);
+  }
+
+  @Override
+  protected String linterName() {
+    return LINTER_NAME;
+  }
+
+  @Override
+  protected String reportPathKey() {
+    return REPORT_PATH_KEY;
+  }
+
+  @Override
+  protected Logger logger() {
+    return LOG;
+  }
+}