Merge pull request #31 from snyk/feat/add_sharding_to_get_analysis

feat: ✨ support sharding in snyk code

Author: ArtsiomCh

CHANGELOG.md

@@ -1,4 +1,7 @@
-## [2.2.1] - 2020-12-10
+## [2.2.2] - 2022-03
+- feat: provide unique (per project) `shard` to getAnalysis call
+
+## [2.2.1] - 2021-12-10
 - fix: don't upload empty files
 
 ## [2.2.0] - 2020-11-29

build.gradle

@@ -6,7 +6,7 @@ plugins {
 
 group = "io.snyk.code.sdk"
 archivesBaseName = "snyk-code-client"
-version = "2.2.1"
+version = "2.2.2"
 
 repositories {
     mavenLocal()

src/integTest/java/ai/deepcode/javaclient/DeepCodeRestApiTest.java

@@ -370,7 +370,7 @@ private GetAnalysisResponse doAnalysisAndWait(List<String> analysedFiles, Intege
       throws InterruptedException {
     GetAnalysisResponse response = null;
     for (int i = 0; i < 120; i++) {
-      response = DeepCodeRestApi.getAnalysis(loggedToken, bundleId, severity, analysedFiles);
+      response = DeepCodeRestApi.getAnalysis(loggedToken, bundleId, severity, analysedFiles, bundleId);
       if (response.getStatus().equals("COMPLETE")) break;
       Thread.sleep(1000);
     }

src/main/java/ai/deepcode/javaclient/DeepCodeRestApi.java

@@ -338,12 +338,12 @@ Call<GetAnalysisResponse> doGetAnalysis(
    */
   @NotNull
   public static GetAnalysisResponse getAnalysis(
-      String token, String bundleId, Integer severity, List<String> filesToAnalyse) {
+    String token, String bundleId, Integer severity, List<String> filesToAnalyse, String shard) {
     GetAnalysisCall getAnalysisCall = retrofit.create(GetAnalysisCall.class);
     try {
       Response<GetAnalysisResponse> retrofitResponse =
           getAnalysisCall
-              .doGetAnalysis(token, new GetAnalysisRequest(bundleId, filesToAnalyse, severity))
+              .doGetAnalysis(token, new GetAnalysisRequest(bundleId, filesToAnalyse, severity, shard))
               .execute();
       GetAnalysisResponse result = retrofitResponse.body();
       if (result == null) result = new GetAnalysisResponse();

src/main/java/ai/deepcode/javaclient/core/AnalysisDataBase.java

@@ -577,7 +577,9 @@ private GetAnalysisResponse doGetAnalysis(
               deepCodeParams.getSessionToken(),
               bundleId,
               deepCodeParams.getMinSeverity(),
-              filesToAnalyse);
+              filesToAnalyse,
+              HashContentUtilsBase.calculateHash(pdUtils.getProjectName(project))
+          );
 
       pdUtils.progressCheckCanceled(progress);
       dcLogger.logInfo(response.toString());

src/main/java/ai/deepcode/javaclient/core/HashContentUtilsBase.java

@@ -46,7 +46,7 @@ private static String bytesToHex(byte[] hash) {
   public boolean isHashChanged(@NotNull Object file) {
     // fixme debug only
     // DCLogger.getInstance().info("hash check started");
-    String newHash = doGetHash(doGetFileContent(file));
+    String newHash = calculateHash(doGetFileContent(file));
     String oldHash = mapFile2Hash.put(file, newHash);
     // fixme debug only
     /*
@@ -67,17 +67,17 @@ public String getHash(@NotNull Object file) {
   }
 
   private String doGetHash(@NotNull Object file) {
-    return doGetHash(getFileContent(file));
+    return calculateHash(getFileContent(file));
   }
 
-  private static String doGetHash(@NotNull String fileText) {
+  protected static String calculateHash(@NotNull String plain) {
     MessageDigest messageDigest;
     try {
       messageDigest = MessageDigest.getInstance("SHA-256");
     } catch (NoSuchAlgorithmException e) {
       throw new RuntimeException(e);
     }
-    byte[] encodedHash = messageDigest.digest(fileText.getBytes(StandardCharsets.UTF_8));
+    byte[] encodedHash = messageDigest.digest(plain.getBytes(StandardCharsets.UTF_8));
     return bytesToHex(encodedHash);
   }
 

src/main/java/ai/deepcode/javaclient/requests/GetAnalysisRequest.java

@@ -6,41 +6,45 @@
 public class GetAnalysisRequest {
   private GetAnalysisKey key;
   private Integer severity;
-  private boolean prioritized = false;
-  private boolean legacy = true;
+  private boolean prioritized;
+  private boolean legacy;
 
   /**
    * @param bundleHash
    * @param limitToFiles list of filePath
    * @param severity
+   * @param shard uniq String (hash) per Project to optimize jobs on backend (run on the same worker to reuse caches)
    * @param prioritized
    * @param legacy
    */
   public GetAnalysisRequest(
-      String bundleHash,
-      List<String> limitToFiles,
-      Integer severity,
-      boolean prioritized,
-      boolean legacy) {
-    this.key = new GetAnalysisKey(bundleHash, limitToFiles);
+    String bundleHash,
+    List<String> limitToFiles,
+    Integer severity,
+    String shard,
+    boolean prioritized,
+    boolean legacy
+  ) {
+    this.key = new GetAnalysisKey(bundleHash, limitToFiles, shard);
     this.severity = severity;
     this.prioritized = prioritized;
     this.legacy = legacy;
   }
 
-  public GetAnalysisRequest(String bundleHash, List<String> limitToFiles, Integer severity) {
-    this(bundleHash, limitToFiles, severity, false, true);
+  public GetAnalysisRequest(String bundleHash, List<String> limitToFiles, Integer severity, String shard) {
+    this(bundleHash, limitToFiles, severity, shard, false, true);
   }
 
-  private class GetAnalysisKey {
-    private String type = "file";
-    private String hash;
-    private List<String> limitToFiles;
+  private static class GetAnalysisKey {
+    private final String type = "file";
+    private final String hash;
+    private final List<String> limitToFiles;
+    private final String shard;
 
-    public GetAnalysisKey(String hash, List<String> limitToFiles) {
-      this.type = type;
+    public GetAnalysisKey(String hash, List<String> limitToFiles, String shard) {
       this.hash = hash;
       this.limitToFiles = limitToFiles;
+      this.shard = shard;
     }
 
     public String getHash() {
@@ -51,6 +55,10 @@ public List<String> getLimitToFiles() {
       return limitToFiles;
     }
 
+    public String getShard() {
+      return shard;
+    }
+
     @Override
     public boolean equals(Object o) {
       if (this == o) return true;