SONARPY-1236 Use protobuf to cache CPD tokens (#1331)

Author: gyula-sallai-sonarsource

python-frontend/src/main/java/org/sonar/python/caching/CpdSerializer.java

@@ -19,24 +19,20 @@
  */
 package org.sonar.python.caching;
 
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
 import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import java.io.Serializable;
 import java.util.List;
 import java.util.stream.Collectors;
 import org.sonar.plugins.python.api.tree.Token;
 import org.sonar.python.TokenLocation;
+import org.sonar.python.types.protobuf.CpdTokenProtos;
 
 public class CpdSerializer {
 
   private CpdSerializer() {
     // Prevent instantiation
   }
 
-  public static final class TokenInfo implements Serializable {
+  public static final class TokenInfo {
     public final int startLine;
     public final int startLineOffset;
     public final int endLine;
@@ -58,19 +54,29 @@ public TokenInfo(int startLine, int startLineOffset, int endLine, int endLineOff
   }
 
   public static byte[] toBytes(List<Token> tokens) throws IOException {
-    List<TokenInfo> tokenInfos = tokens.stream()
-      .map(TokenInfo::from)
-      .collect(Collectors.toList());
+    CpdTokenProtos.FileCpdTokens.Builder builder = CpdTokenProtos.FileCpdTokens.newBuilder();
 
-    ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
-    ObjectOutputStream objectOutputStream = new ObjectOutputStream(byteArrayOutputStream);
-    objectOutputStream.writeObject(tokenInfos);
+    for (Token token : tokens) {
+      TokenLocation location = new TokenLocation(token);
+      CpdTokenProtos.Token protoToken = CpdTokenProtos.Token.newBuilder()
+        .setValue(token.value())
+        .setStartLine(location.startLine())
+        .setStartLineOffset(location.startLineOffset())
+        .setEndLine(location.endLine())
+        .setEndLineOffset(location.endLineOffset())
+        .build();
 
-    return byteArrayOutputStream.toByteArray();
+      builder.addTokens(protoToken);
+    }
+
+    return builder.build().toByteArray();
   }
 
-  public static List<TokenInfo> fromBytes(byte[] bytes) throws IOException, ClassNotFoundException {
-    ObjectInputStream objectInputStream = new ObjectInputStream(new ByteArrayInputStream(bytes));
-    return (List<TokenInfo>) objectInputStream.readObject();
+  public static List<TokenInfo> fromBytes(byte[] bytes) throws IOException {
+    return CpdTokenProtos.FileCpdTokens.parseFrom(bytes)
+      .getTokensList()
+      .stream()
+      .map(proto -> new TokenInfo(proto.getStartLine(), proto.getStartLineOffset(), proto.getEndLine(), proto.getEndLineOffset(), proto.getValue()))
+      .collect(Collectors.toList());
   }
 }

python-frontend/src/main/protobuf/tokens.proto

@@ -0,0 +1,19 @@
+syntax = "proto3";
+
+// Not necessary for Python but should still be declared to avoid name collisions
+// in the Protocol Buffers namespace and non-Python languages
+package protoblog;
+option java_outer_classname = "CpdTokenProtos";
+option java_package = "org.sonar.python.types.protobuf";
+
+message Token {
+    string value = 1;
+    int32 startLine = 2;
+    int32 startLineOffset = 3;
+    int32 endLine = 4;
+    int32 endLineOffset = 5;
+}
+
+message FileCpdTokens {
+    repeated Token tokens = 1;
+}

sonar-python-plugin/src/main/java/org/sonar/plugins/python/cpd/PythonCpdAnalyzer.java

@@ -90,7 +90,7 @@ public boolean pushCachedCpdTokens(InputFile inputFile, CacheContext cacheContex
       cpdTokens.save();
       cacheContext.getWriteCache().copyFromPrevious(key);
       return true;
-    } catch (IOException | ClassNotFoundException | ClassCastException e) {
+    } catch (IOException e) {
       LOG.warn("Failed to deserialize CPD tokens ({}: {})", e.getClass().getSimpleName(), e.getMessage());
     }
 

sonar-python-plugin/src/test/java/org/sonar/plugins/python/PythonSensorTest.java

@@ -1033,7 +1033,7 @@ public void read_cpd_tokens_from_cache_corrupted_format() throws IOException {
     sensor().execute(context);
 
     assertThat(logTester.logs(LoggerLevel.WARN))
-      .anyMatch(line -> line.startsWith("Failed to deserialize CPD tokens (ClassCastException: class java.lang.String cannot be cast to class java.util.List"));
+      .anyMatch(line -> line.startsWith("Failed to deserialize CPD tokens"));
 
     // Verify the written CPD tokens
     List<TokensLine> tokensLines = context.cpdTokens("moduleKey:pass.py");