Merge pull request #1071 from launchableinc/AIENG-183

[AIENG-183] optional mode to collect file content, off by default

Author: kohsuke

.editorconfig

@@ -8,4 +8,4 @@ charset=utf-8
 indent_style=space
 
 [*.java]
-indent_size = 4
+indent_size = 2

WORKSPACE

@@ -9,9 +9,11 @@ http_archive(
 )
 
 load("@rules_jvm_external//:repositories.bzl", "rules_jvm_external_deps")
+
 rules_jvm_external_deps()
 
 load("@rules_jvm_external//:setup.bzl", "rules_jvm_external_setup")
+
 rules_jvm_external_setup()
 
 load("@rules_jvm_external//:defs.bzl", "maven_install")
@@ -25,6 +27,7 @@ maven_install(
         "com.fasterxml.jackson.core:jackson-core:2.18.2",
         "com.fasterxml.jackson.core:jackson-databind:2.18.2",
         "com.google.guava:guava:33.3.1-jre",
+        "org.apache.commons:commons-compress:1.27.1",
         "org.apache.httpcomponents:httpclient:4.5.14",
         # This is the last release that produce Java 8 class files.
         "org.eclipse.jgit:org.eclipse.jgit:5.13.3.202401111512-r",
@@ -45,10 +48,10 @@ maven_install(
             version = "1.4.4",
         ),
     ],
+    fetch_sources = True,
     maven_install_json = "//src:maven_install.json",
     repositories = ["https://repo1.maven.org/maven2"],
     version_conflict_policy = "pinned",
-    fetch_sources = True,
 )
 
 load("@maven//:defs.bzl", "pinned_maven_install")

launchable/commands/record/commit.py

@@ -65,10 +65,12 @@ def commit(ctx, source: str, executable: bool, max_days: int, scrub_pii: bool, i
 
     # Commit messages are not collected in the default.
     is_collect_message = False
+    is_collect_files = False
     try:
         res = client.request("get", "commits/collect/options")
         res.raise_for_status()
         is_collect_message = res.json().get("commitMessage", False)
+        is_collect_files = res.json().get("files", False)
     except Exception as e:
         tracking_client.send_error_event(
             event_name=Tracking.ErrorEvent.INTERNAL_CLI_ERROR,
@@ -79,7 +81,7 @@ def commit(ctx, source: str, executable: bool, max_days: int, scrub_pii: bool, i
 
     cwd = os.path.abspath(source)
     try:
-        exec_jar(cwd, max_days, ctx.obj, is_collect_message)
+        exec_jar(cwd, max_days, ctx.obj, is_collect_message, is_collect_files)
     except Exception as e:
         if os.getenv(REPORT_ERROR_KEY):
             raise e
@@ -89,7 +91,7 @@ def commit(ctx, source: str, executable: bool, max_days: int, scrub_pii: bool, i
                 "If not, please set a directory use by --source option.\nerror: {}".format(cwd, e))
 
 
-def exec_jar(source: str, max_days: int, app: Application, is_collect_message: bool):
+def exec_jar(source: str, max_days: int, app: Application, is_collect_message: bool, is_collect_files: bool):
     java = get_java_command()
 
     if not java:
@@ -120,6 +122,8 @@ def exec_jar(source: str, max_days: int, app: Application, is_collect_message: b
         command.append("-skip-cert-verification")
     if is_collect_message:
         command.append("-commit-message")
+    if is_collect_files:
+        command.append("-files")
     if os.getenv(COMMIT_TIMEOUT):
         command.append("-enable-timeout")
     command.append(cygpath(source))

launchable/jar/exe_deploy.jar

@@ -1,5 +1,5 @@
 package(
-    default_visibility = ["//visibility:public"]
+    default_visibility = ["//visibility:public"],
 )
 
 java_library(
@@ -11,6 +11,7 @@ java_library(
         "@maven//:com_fasterxml_jackson_core_jackson_core",
         "@maven//:com_fasterxml_jackson_core_jackson_databind",
         "@maven//:com_google_guava_guava",
+        "@maven//:org_apache_commons_commons_compress",
         "@maven//:org_apache_httpcomponents_httpclient",
         "@maven//:org_apache_httpcomponents_httpcore",
         "@maven//:org_eclipse_jgit_org_eclipse_jgit",

src/main/java/com/launchableinc/ingest/commits/BUILD

@@ -0,0 +1,59 @@
+package com.launchableinc.ingest.commits;
+
+import org.apache.http.entity.ContentProducer;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.UncheckedIOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Consumer;
+
+/**
+ * Accepts T, buffers them, and writes them out as a batch.
+ */
+abstract class ChunkStreamer<T> implements Consumer<T>, Closeable {
+  /**
+   * Encapsulation of how batches are sent.
+   */
+  private final IOConsumer<ContentProducer> sender;
+  private final int chunkSize;
+  private final List<T> spool = new ArrayList<>();
+
+  ChunkStreamer(IOConsumer<ContentProducer> sender, int chunkSize) {
+    this.sender = sender;
+    this.chunkSize = chunkSize;
+  }
+
+  @Override
+  public void accept(T f) {
+    spool.add(f);
+    if (spool.size() >= chunkSize) {
+      try {
+        flush();
+      } catch (IOException e) {
+        throw new UncheckedIOException(e);
+      }
+    }
+  }
+
+  @Override
+  public void close() throws IOException {
+    flush();
+  }
+
+  private void flush() throws IOException {
+    if (spool.isEmpty()) {
+      return;
+    }
+
+    try {
+      sender.accept(os -> writeTo(spool,os));
+    } finally {
+      spool.clear();
+    }
+  }
+
+  protected abstract void writeTo(List<T> spool, OutputStream os) throws IOException;
+}

src/main/java/com/launchableinc/ingest/commits/ChunkStreamer.java

@@ -0,0 +1,37 @@
+package com.launchableinc.ingest.commits;
+
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonGenerator;
+import org.apache.http.entity.ContentProducer;
+import org.eclipse.jgit.revwalk.RevCommit;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.List;
+import java.util.function.Consumer;
+
+/**
+ * {@link Consumer} that groups commits into chunks and write them as JSON, using streams supplied
+ * by the factory.
+ */
+final class CommitChunkStreamer extends ChunkStreamer<JSCommit> {
+  CommitChunkStreamer(IOConsumer<ContentProducer> sender, int chunkSize) {
+    super(sender, chunkSize);
+  }
+
+  @Override
+  protected void writeTo(List<JSCommit> spool, OutputStream os) throws IOException {
+    JsonGenerator w = new JsonFactory().createGenerator(os).useDefaultPrettyPrinter();
+    w.setCodec(CommitGraphCollector.objectMapper);
+    w.writeStartObject();
+    w.writeArrayFieldStart("commits");
+
+    for (JSCommit commit : spool) {
+      w.writeObject(commit);
+    }
+
+    w.writeEndArray();
+    w.writeEndObject();
+    w.close();
+  }
+}