SONARPY-2032 Adapt (executable) lines of code metric for Jupyter Notebooks (#1883)

Author: ghislainpiot

python-frontend/src/main/java/org/sonar/python/metrics/FileLinesVisitor.java

@@ -61,9 +61,18 @@ public class FileLinesVisitor extends PythonSubscriptionCheck {
   private final Set<Integer> linesOfComments = new HashSet<>();
   private final Set<Integer> linesOfDocstring = new HashSet<>();
   private final Set<Integer> executableLines = new HashSet<>();
+  private final boolean isNotebook;
   private int statements = 0;
   private int classDefs = 0;
 
+  public FileLinesVisitor(boolean isNotebook) {
+    this.isNotebook = isNotebook;
+  }
+
+  public FileLinesVisitor() {
+    this(false);
+  }
+
   @Override
   public void scanFile(PythonVisitorContext visitorContext) {
     SubscriptionVisitor.analyze(Collections.singleton(this), visitorContext);
@@ -158,7 +167,7 @@ private void visitComment(Trivia trivia, Token parentToken) {
     }
   }
 
-  public static boolean containsNoSonarComment(Trivia trivia){
+  public static boolean containsNoSonarComment(Trivia trivia) {
     String commentLine = getContents(trivia.token().value());
     return commentLine.contains("NOSONAR");
   }
@@ -186,7 +195,7 @@ public int getCommentLineCount() {
   }
 
   public Set<Integer> getExecutableLines() {
-    return Collections.unmodifiableSet(executableLines);
+    return isNotebook ? Set.of() : Collections.unmodifiableSet(executableLines);
   }
 
   private static boolean isBlank(String line) {

python-frontend/src/main/java/org/sonar/python/metrics/FileMetrics.java

@@ -35,9 +35,9 @@ public class FileMetrics {
   private final FileLinesVisitor fileLinesVisitor;
   private List<Integer> functionComplexities = new ArrayList<>();
 
-  public FileMetrics(PythonVisitorContext context) {
+  public FileMetrics(PythonVisitorContext context, boolean isNotebook) {
     FileInput fileInput = context.rootTree();
-    fileLinesVisitor = new FileLinesVisitor();
+    fileLinesVisitor = new FileLinesVisitor(isNotebook);
     fileLinesVisitor.scanFile(context);
     numberOfStatements = fileLinesVisitor.getStatements();
     numberOfClasses = fileLinesVisitor.getClassDefs();
@@ -46,6 +46,10 @@ public FileMetrics(PythonVisitorContext context) {
     fileInput.accept(new FunctionVisitor());
   }
 
+  public FileMetrics(PythonVisitorContext context) {
+    this(context, false);
+  }
+
   private class FunctionVisitor extends BaseTreeVisitor {
     @Override
     public void visitFunctionDef(FunctionDef pyFunctionDefTree) {

python-frontend/src/test/java/org/sonar/python/FileLinesVisitorTest.java

@@ -66,4 +66,12 @@ void empty_file() {
     TestPythonVisitorRunner.scanFile(new File(BASE_DIR, "empty.py"), visitor);
     assertThat(visitor.getExecutableLines()).isEmpty();
   }
+
+  @Test
+  void notebook_locs() {
+    FileLinesVisitor visitor = new FileLinesVisitor(true);
+    TestPythonVisitorRunner.scanFile(new File(BASE_DIR, "notebook_loc.ipynb"), visitor);
+    assertThat(visitor.getExecutableLines()).isEmpty();
+    assertThat(visitor.getLinesOfCode()).hasSize(17);
+  }
 }

python-frontend/src/test/resources/metrics/notebook_loc.ipynb

@@ -0,0 +1,27 @@
+x = None
+if x is not None:
+    print "not none"
+
+
+def foo():
+    x = 42
+    x = 17
+    print(x)
+#SONAR_PYTHON_NOTEBOOK_CELL_DELIMITER
+if x is not None:
+    print("hello")
+#SONAR_PYTHON_NOTEBOOK_CELL_DELIMITER
+x = 42
+#SONAR_PYTHON_NOTEBOOK_CELL_DELIMITER
+#Some code
+print("hello world\n")
+#SONAR_PYTHON_NOTEBOOK_CELL_DELIMITER
+print("My\ntext")
+print("Something else\n")
+#SONAR_PYTHON_NOTEBOOK_CELL_DELIMITER
+print("My\ntext")
+print("Something else\n")
+#SONAR_PYTHON_NOTEBOOK_CELL_DELIMITER
+a = "A bunch of characters \n \t \f \r / // \ "
+b = None
+#SONAR_PYTHON_NOTEBOOK_CELL_DELIMITER

sonar-python-plugin/src/main/java/org/sonar/plugins/python/PythonScanner.java

@@ -302,7 +302,7 @@ private static NewIssueLocation newLocation(PythonInputFile inputFile, NewIssue
   }
 
   private void saveMeasures(PythonInputFile inputFile, PythonVisitorContext visitorContext) {
-    FileMetrics fileMetrics = new FileMetrics(visitorContext);
+    FileMetrics fileMetrics = new FileMetrics(visitorContext, isNotebook(inputFile));
     FileLinesVisitor fileLinesVisitor = fileMetrics.fileLinesVisitor();
 
     noSonarFilter.noSonarInFile(inputFile.wrappedFile(), fileLinesVisitor.getLinesWithNoSonar());
@@ -330,6 +330,10 @@ private void saveMeasures(PythonInputFile inputFile, PythonVisitorContext visito
     }
   }
 
+  static boolean isNotebook(PythonInputFile inputFile) {
+    return inputFile.kind() == PythonInputFile.Kind.IPYTHON;
+  }
+
   private boolean restoreAndPushMeasuresIfApplicable(PythonInputFile inputFile) {
     if (inputFile.wrappedFile().type() == InputFile.Type.TEST) {
       return true;

sonar-python-plugin/src/test/java/org/sonar/plugins/python/PythonSensorTest.java

@@ -1290,6 +1290,17 @@ void cpd_tokens_failure_does_not_execute_checks_multiple_times() throws IOExcept
       .containsEntry(CPD_TOKENS_STRING_TABLE_KEY_PREFIX + inputFile.wrappedFile().key(), cpdTokens.stringTable);
   }
 
+  @Test
+  void test_scanner_isNotebook(){
+    var regularPythonFile = mock(PythonInputFile.class);
+    when(regularPythonFile.kind()).thenReturn(PythonInputFile.Kind.PYTHON);
+    assertThat(PythonScanner.isNotebook(regularPythonFile)).isFalse();
+
+    var notebookPythonFile = mock(PythonInputFile.class);
+    when(notebookPythonFile.kind()).thenReturn(PythonInputFile.Kind.IPYTHON);
+    assertThat(PythonScanner.isNotebook(notebookPythonFile)).isTrue();
+  }
+
   private com.sonar.sslr.api.Token passToken(URI uri) {
     return com.sonar.sslr.api.Token.builder()
       .setType(PythonKeyword.PASS)