Add benchmark metrics to iOS CI pipeline (#4286)

Measured and reported VM translation time and native compilation time in the PR comment.
Updated `scripts/build-ios-app.sh` to capture VM translation time.
Updated `scripts/run-ios-ui-tests.sh` to capture compilation time and pass metrics.
Updated `RenderScreenshotReport.java` to display benchmark results.

Co-authored-by: google-labs-jules[bot] <161369871+google-labs-jules[bot]@users.noreply.github.com>

Author: google-labs-jules[bot]

scripts/build-ios-app.sh

@@ -62,14 +62,22 @@ xcodebuild -version
 
 bia_log "Building iOS Xcode project using Codename One port"
 cd $APP_DIR
+VM_START=$(date +%s)
 ./mvnw package \
   -DskipTests \
   -Dcodename1.platform=ios \
   -Dcodename1.buildTarget=ios-source \
   -Dopen=false \
   -U -e
+VM_END=$(date +%s)
+VM_TIME=$((VM_END - VM_START))
 cd ../..
 
+ARTIFACTS_DIR="${ARTIFACTS_DIR:-$REPO_ROOT/artifacts}"
+mkdir -p "$ARTIFACTS_DIR"
+echo "$VM_TIME" > "$ARTIFACTS_DIR/vm_time.txt"
+bia_log "VM translation time: ${VM_TIME}s (saved to $ARTIFACTS_DIR/vm_time.txt)"
+
 IOS_TARGET_DIR="$APP_DIR/ios/target"
 if [ ! -d "$IOS_TARGET_DIR" ]; then
   bia_log "iOS target directory not found at $IOS_TARGET_DIR" >&2

scripts/common/java/RenderScreenshotReport.java

@@ -32,7 +32,7 @@ public static void main(String[] args) throws Exception {
 
         CoverageSummary coverage = loadCoverage(arguments.coverageSummary, arguments.coverageHtmlUrl);
 
-        SummaryAndComment output = buildSummaryAndComment(data, title, marker, successMessage, coverage);
+        SummaryAndComment output = buildSummaryAndComment(data, title, marker, successMessage, coverage, arguments.vmTime, arguments.compilationTime);
         writeLines(arguments.summaryOut, output.summaryLines);
         writeLines(arguments.commentOut, output.commentLines);
     }
@@ -51,7 +51,7 @@ private static void writeLines(Path path, List<String> lines) throws IOException
         Files.writeString(path, sb.toString(), StandardCharsets.UTF_8);
     }
 
-    private static SummaryAndComment buildSummaryAndComment(Map<String, Object> data, String title, String marker, String successMessage, CoverageSummary coverage) {
+    private static SummaryAndComment buildSummaryAndComment(Map<String, Object> data, String title, String marker, String successMessage, CoverageSummary coverage, Long vmTime, Long compilationTime) {
         List<String> summaryLines = new ArrayList<>();
         List<String> commentLines = new ArrayList<>();
         Object resultsObj = data.get("results");
@@ -148,22 +148,34 @@ private static SummaryAndComment buildSummaryAndComment(Map<String, Object> data
 
         appendCoverageComment(commentLines, coverage);
 
-        if (commentLines.isEmpty()) {
+        // If no visual regressions or errors were found (commentEntries is empty),
+        // we should display the success message.
+        // However, we also need to include benchmark results.
+        if (commentEntries.isEmpty()) {
             commentLines.add(successMessage != null ? successMessage : DEFAULT_SUCCESS_MESSAGE);
             commentLines.add("");
-            comparisonOverviewAdded = appendComparisonOverview(commentLines, comparisonSummary);
-            appendCoverageComment(commentLines, coverage);
-        } else if (commentLines.size() == 1 && commentLines.get(0).isEmpty()) {
-            commentLines.add(successMessage != null ? successMessage : DEFAULT_SUCCESS_MESSAGE);
-            commentLines.add("");
-            comparisonOverviewAdded = appendComparisonOverview(commentLines, comparisonSummary);
+            if (!comparisonOverviewAdded) {
+                comparisonOverviewAdded = appendComparisonOverview(commentLines, comparisonSummary);
+            }
             appendCoverageComment(commentLines, coverage);
         }
 
-        if (commentLines.isEmpty()) {
-            commentLines.add(successMessage != null ? successMessage : DEFAULT_SUCCESS_MESSAGE);
-            commentLines.add("");
-            appendComparisonOverview(commentLines, comparisonSummary);
+        // Add benchmark results at the end
+        appendBenchmarkResults(commentLines, vmTime, compilationTime);
+
+        if (commentLines.isEmpty() || (commentLines.size() == 1 && commentLines.get(0).isEmpty())) {
+            // This fallback block might be redundant now, but kept for safety.
+            // If for some reason we still have empty lines (e.g. no benchmarks and no visual changes and logic skipped above),
+            // ensure we output something.
+            if (commentEntries.isEmpty()) {
+                 if (commentLines.isEmpty() || !commentLines.contains(successMessage != null ? successMessage : DEFAULT_SUCCESS_MESSAGE)) {
+                     commentLines.add(0, successMessage != null ? successMessage : DEFAULT_SUCCESS_MESSAGE);
+                     commentLines.add(1, "");
+                 }
+            }
+            if (!comparisonOverviewAdded) {
+                comparisonOverviewAdded = appendComparisonOverview(commentLines, comparisonSummary);
+            }
         }
 
         if (marker != null && !marker.isEmpty()) {
@@ -219,6 +231,24 @@ private static void appendCoverageSummary(List<String> summaryLines, CoverageSum
         }
     }
 
+    private static void appendBenchmarkResults(List<String> commentLines, Long vmTime, Long compilationTime) {
+        if (vmTime == null && compilationTime == null) {
+            return;
+        }
+        if (!commentLines.isEmpty() && !commentLines.get(commentLines.size() - 1).isEmpty()) {
+            commentLines.add("");
+        }
+        commentLines.add("### Benchmark Results");
+        commentLines.add("");
+        if (vmTime != null) {
+            commentLines.add(String.format("- **VM Translation Time:** %d seconds", vmTime));
+        }
+        if (compilationTime != null) {
+            commentLines.add(String.format("- **Compilation Time:** %d seconds", compilationTime));
+        }
+        commentLines.add("");
+    }
+
     private static void appendCoverageComment(List<String> commentLines, CoverageSummary coverage) {
         if (coverage == null) {
             return;
@@ -418,8 +448,10 @@ private static class Arguments {
         final String successMessage;
         final Path coverageSummary;
         final String coverageHtmlUrl;
+        final Long vmTime;
+        final Long compilationTime;
 
-        private Arguments(Path compareJson, Path commentOut, Path summaryOut, String marker, String title, String successMessage, Path coverageSummary, String coverageHtmlUrl) {
+        private Arguments(Path compareJson, Path commentOut, Path summaryOut, String marker, String title, String successMessage, Path coverageSummary, String coverageHtmlUrl, Long vmTime, Long compilationTime) {
             this.compareJson = compareJson;
             this.commentOut = commentOut;
             this.summaryOut = summaryOut;
@@ -428,6 +460,8 @@ private Arguments(Path compareJson, Path commentOut, Path summaryOut, String mar
             this.successMessage = successMessage;
             this.coverageSummary = coverageSummary;
             this.coverageHtmlUrl = coverageHtmlUrl;
+            this.vmTime = vmTime;
+            this.compilationTime = compilationTime;
         }
 
         static Arguments parse(String[] args) {
@@ -439,6 +473,8 @@ static Arguments parse(String[] args) {
             String successMessage = null;
             Path coverageSummary = null;
             String coverageHtmlUrl = null;
+            Long vmTime = null;
+            Long compilationTime = null;
             for (int i = 0; i < args.length; i++) {
                 String arg = args[i];
                 switch (arg) {
@@ -498,6 +534,30 @@ static Arguments parse(String[] args) {
                         }
                         coverageHtmlUrl = args[i];
                     }
+                    case "--vm-time" -> {
+                        if (++i >= args.length) {
+                            System.err.println("Missing value for --vm-time");
+                            return null;
+                        }
+                        try {
+                            vmTime = Long.parseLong(args[i]);
+                        } catch (NumberFormatException e) {
+                            System.err.println("Invalid value for --vm-time: " + args[i]);
+                            return null;
+                        }
+                    }
+                    case "--compilation-time" -> {
+                        if (++i >= args.length) {
+                            System.err.println("Missing value for --compilation-time");
+                            return null;
+                        }
+                        try {
+                            compilationTime = Long.parseLong(args[i]);
+                        } catch (NumberFormatException e) {
+                            System.err.println("Invalid value for --compilation-time: " + args[i]);
+                            return null;
+                        }
+                    }
                     default -> {
                         System.err.println("Unknown argument: " + arg);
                         return null;
@@ -508,7 +568,7 @@ static Arguments parse(String[] args) {
                 System.err.println("--compare-json, --comment-out, and --summary-out are required");
                 return null;
             }
-            return new Arguments(compare, comment, summary, marker, title, successMessage, coverageSummary, coverageHtmlUrl);
+            return new Arguments(compare, comment, summary, marker, title, successMessage, coverageSummary, coverageHtmlUrl, vmTime, compilationTime);
         }
     }
 

scripts/lib/cn1ss.sh

@@ -354,6 +354,12 @@ cn1ss_process_and_report() {
   if [ -n "${CN1SS_COVERAGE_HTML_URL:-}" ]; then
     render_args+=(--coverage-html-url "$CN1SS_COVERAGE_HTML_URL")
   fi
+  if [ -n "${CN1SS_VM_TIME:-}" ]; then
+    render_args+=(--vm-time "$CN1SS_VM_TIME")
+  fi
+  if [ -n "${CN1SS_COMPILATION_TIME:-}" ]; then
+    render_args+=(--compilation-time "$CN1SS_COMPILATION_TIME")
+  fi
 
   if ! cn1ss_java_run "$CN1SS_RENDER_CLASS" "${render_args[@]}"; then
     cn1ss_log "FATAL: Failed to render screenshot summary/comment"

scripts/run-ios-ui-tests.sh

@@ -346,6 +346,7 @@ rm -rf "$DERIVED_DATA_DIR"
 BUILD_LOG="$ARTIFACTS_DIR/xcodebuild-build.log"
 
 ri_log "Building simulator app with xcodebuild"
+COMPILE_START=$(date +%s)
 if ! xcodebuild \
   -workspace "$WORKSPACE_PATH" \
   -scheme "$SCHEME" \
@@ -358,6 +359,9 @@ if ! xcodebuild \
   ri_log "STAGE:XCODE_BUILD_FAILED -> See $BUILD_LOG"
   exit 10
 fi
+COMPILE_END=$(date +%s)
+COMPILATION_TIME=$((COMPILE_END - COMPILE_START))
+ri_log "Compilation time: ${COMPILATION_TIME}s"
 
 BUILD_SETTINGS="$(xcodebuild -workspace "$WORKSPACE_PATH" -scheme "$SCHEME" -sdk iphonesimulator -configuration Debug -showBuildSettings 2>/dev/null || true)"
 TARGET_BUILD_DIR="$(printf '%s\n' "$BUILD_SETTINGS" | awk -F' = ' '/ TARGET_BUILD_DIR /{print $2; exit}')"
@@ -612,6 +616,15 @@ export CN1SS_COMMENT_MARKER="<!-- CN1SS_IOS_COMMENT -->"
 export CN1SS_COMMENT_LOG_PREFIX="[run-ios-device-tests]"
 export CN1SS_PREVIEW_SUBDIR="ios"
 
+# Load VM translation time if available
+CN1SS_VM_TIME=0
+if [ -f "$ARTIFACTS_DIR/vm_time.txt" ]; then
+  CN1SS_VM_TIME=$(cat "$ARTIFACTS_DIR/vm_time.txt")
+  ri_log "Loaded VM translation time: ${CN1SS_VM_TIME}s"
+fi
+export CN1SS_VM_TIME
+export CN1SS_COMPILATION_TIME="$COMPILATION_TIME"
+
 cn1ss_process_and_report \
   "iOS screenshot updates" \
   "$COMPARE_JSON" \