Add iOS UI screenshot regression tests

.github/workflows/scripts-ios.yml

@@ -7,6 +7,9 @@ on:
       - 'scripts/setup-workspace.sh'
       - 'scripts/build-ios-port.sh'
       - 'scripts/build-ios-app.sh'
+      - 'scripts/run-ios-ui-tests.sh'
+      - 'scripts/ios/tests/**'
+      - 'scripts/ios/screenshots/**'
       - 'scripts/templates/**'
       - '!scripts/templates/**/*.md'
       - 'CodenameOne/src/**'
@@ -24,6 +27,9 @@ on:
       - 'scripts/setup-workspace.sh'
       - 'scripts/build-ios-port.sh'
       - 'scripts/build-ios-app.sh'
+      - 'scripts/run-ios-ui-tests.sh'
+      - 'scripts/ios/tests/**'
+      - 'scripts/ios/screenshots/**'
       - 'scripts/templates/**'
       - '!scripts/templates/**/*.md'
       - 'CodenameOne/src/**'
@@ -37,12 +43,20 @@ on:
 
 jobs:
   build-ios:
+    permissions:
+      contents: read
+      pull-requests: write
+      issues: write
     runs-on: macos-15           # pinning macos-15 avoids surprises during the cutover window
     timeout-minutes: 60         # allow enough time for dependency installs and full build
     concurrency:                # ensure only one mac build runs at once
       group: mac-ci
       cancel-in-progress: false # queue new ones instead of canceling in-flight
 
+    env:
+      GITHUB_TOKEN: ${{ secrets.CN1SS_GH_TOKEN }}
+      GH_TOKEN: ${{ secrets.CN1SS_GH_TOKEN }}
+
     steps:
       - uses: actions/checkout@v4
 
@@ -75,6 +89,24 @@ jobs:
         timeout-minutes: 25
 
       - name: Build sample iOS app and compile workspace
+        id: build-ios-app
         run: ./scripts/build-ios-app.sh -q -DskipTests
         timeout-minutes: 30
 
+      - name: Run iOS UI screenshot tests
+        env:
+          ARTIFACTS_DIR: ${{ github.workspace }}/artifacts
+        run: |
+          mkdir -p "${ARTIFACTS_DIR}"
+          ./scripts/run-ios-ui-tests.sh "${{ steps.build-ios-app.outputs.workspace }}" "${{ steps.build-ios-app.outputs.app_bundle }}"
+        timeout-minutes: 25
+
+      - name: Upload iOS artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: ios-ui-tests
+          path: artifacts
+          if-no-files-found: warn
+          retention-days: 14
+

scripts/android/tests/PostPrComment.java

@@ -22,8 +22,10 @@
 import java.util.regex.Pattern;
 
 public class PostPrComment {
-    private static final String MARKER = "<!-- CN1SS_SCREENSHOT_COMMENT -->";
-    private static final String LOG_PREFIX = "[run-android-instrumentation-tests]";
+    private static final String DEFAULT_MARKER = "<!-- CN1SS_SCREENSHOT_COMMENT -->";
+    private static final String DEFAULT_LOG_PREFIX = "[run-android-instrumentation-tests]";
+    private static String marker = DEFAULT_MARKER;
+    private static String logPrefix = DEFAULT_LOG_PREFIX;
 
     public static void main(String[] args) throws Exception {
         int exitCode = execute(args);
@@ -35,6 +37,9 @@ private static int execute(String[] args) throws Exception {
         if (arguments == null) {
             return 2;
         }
+        marker = arguments.marker != null ? arguments.marker : DEFAULT_MARKER;
+        logPrefix = arguments.logPrefix != null ? arguments.logPrefix : DEFAULT_LOG_PREFIX;
+
         Path bodyPath = arguments.body;
         if (!Files.isRegularFile(bodyPath)) {
             return 0;
@@ -44,10 +49,10 @@ private static int execute(String[] args) throws Exception {
         if (body.isEmpty()) {
             return 0;
         }
-        if (!body.contains(MARKER)) {
-            body = body.stripTrailing() + "\n\n" + MARKER;
+        if (!body.contains(marker)) {
+            body = body.stripTrailing() + "\n\n" + marker;
         }
-        String bodyWithoutMarker = body.replace(MARKER, "").trim();
+        String bodyWithoutMarker = body.replace(marker, "").trim();
         if (bodyWithoutMarker.isEmpty()) {
             return 0;
         }
@@ -154,7 +159,7 @@ private static CommentContext locateExistingComment(HttpClient client, Map<Strin
             for (Object comment : comments) {
                 Map<String, Object> commentMap = JsonUtil.asObject(comment);
                 String bodyText = stringValue(commentMap.get("body"), "");
-                if (bodyText.contains(MARKER)) {
+                if (bodyText.contains(marker)) {
                     existingComment = commentMap;
                     Map<String, Object> user = JsonUtil.asObject(commentMap.get("user"));
                     String login = stringValue(user.get("login"), null);
@@ -181,7 +186,7 @@ private static CommentContext locateExistingComment(HttpClient client, Map<Strin
                     .uri(URI.create("https://api.github.com/repos/" + repo + "/issues/" + prNumber + "/comments"))
                     .timeout(Duration.ofSeconds(20))
                     .headers(headers.entrySet().stream().flatMap(e -> java.util.stream.Stream.of(e.getKey(), e.getValue())).toArray(String[]::new))
-                    .POST(HttpRequest.BodyPublishers.ofString(JsonUtil.stringify(Map.of("body", MARKER))))
+                    .POST(HttpRequest.BodyPublishers.ofString(JsonUtil.stringify(Map.of("body", marker))))
                     .build();
             HttpResponse<String> createResponse = client.send(createRequest, HttpResponse.BodyHandlers.ofString(StandardCharsets.UTF_8));
             if (createResponse.statusCode() >= 200 && createResponse.statusCode() < 300) {
@@ -406,11 +411,11 @@ private static String stringValue(Object value, String fallback) {
     }
 
     private static void log(String message) {
-        System.out.println(LOG_PREFIX + " " + message);
+        System.out.println(logPrefix + " " + message);
     }
 
     private static void err(String message) {
-        System.err.println(LOG_PREFIX + " " + message);
+        System.err.println(logPrefix + " " + message);
     }
 
     private record CommentContext(long commentId, boolean createdPlaceholder) {
@@ -422,15 +427,21 @@ private record AttachmentReplacement(String body, List<String> missing) {
     private static class Arguments {
         final Path body;
         final Path previewDir;
+        final String marker;
+        final String logPrefix;
 
-        private Arguments(Path body, Path previewDir) {
+        private Arguments(Path body, Path previewDir, String marker, String logPrefix) {
             this.body = body;
             this.previewDir = previewDir;
+            this.marker = marker;
+            this.logPrefix = logPrefix;
         }
 
         static Arguments parse(String[] args) {
             Path body = null;
             Path previewDir = null;
+            String marker = null;
+            String logPrefix = null;
             for (int i = 0; i < args.length; i++) {
                 String arg = args[i];
                 switch (arg) {
@@ -448,6 +459,20 @@ static Arguments parse(String[] args) {
                         }
                         previewDir = Path.of(args[i]);
                     }
+                    case "--marker" -> {
+                        if (++i >= args.length) {
+                            System.err.println("Missing value for --marker");
+                            return null;
+                        }
+                        marker = args[i];
+                    }
+                    case "--log-prefix" -> {
+                        if (++i >= args.length) {
+                            System.err.println("Missing value for --log-prefix");
+                            return null;
+                        }
+                        logPrefix = args[i];
+                    }
                     default -> {
                         System.err.println("Unknown argument: " + arg);
                         return null;
@@ -458,7 +483,7 @@ static Arguments parse(String[] args) {
                 System.err.println("--body is required");
                 return null;
             }
-            return new Arguments(body, previewDir);
+            return new Arguments(body, previewDir, marker, logPrefix);
         }
     }
 

scripts/android/tests/RenderScreenshotReport.java

@@ -8,7 +8,9 @@
 import java.util.Map;
 
 public class RenderScreenshotReport {
-    private static final String MARKER = "<!-- CN1SS_SCREENSHOT_COMMENT -->";
+    private static final String DEFAULT_MARKER = "<!-- CN1SS_SCREENSHOT_COMMENT -->";
+    private static final String DEFAULT_TITLE = "Android screenshot updates";
+    private static final String DEFAULT_SUCCESS_MESSAGE = "✅ Native Android screenshot tests passed.";
 
     public static void main(String[] args) throws Exception {
         Arguments arguments = Arguments.parse(args);
@@ -24,7 +26,11 @@ public static void main(String[] args) throws Exception {
         String text = Files.readString(comparePath, StandardCharsets.UTF_8);
         Object parsed = JsonUtil.parse(text);
         Map<String, Object> data = JsonUtil.asObject(parsed);
-        SummaryAndComment output = buildSummaryAndComment(data);
+        String marker = arguments.marker != null ? arguments.marker : DEFAULT_MARKER;
+        String title = arguments.title != null ? arguments.title : DEFAULT_TITLE;
+        String successMessage = arguments.successMessage != null ? arguments.successMessage : DEFAULT_SUCCESS_MESSAGE;
+
+        SummaryAndComment output = buildSummaryAndComment(data, title, marker, successMessage);
         writeLines(arguments.summaryOut, output.summaryLines);
         writeLines(arguments.commentOut, output.commentLines);
     }
@@ -43,7 +49,7 @@ private static void writeLines(Path path, List<String> lines) throws IOException
         Files.writeString(path, sb.toString(), StandardCharsets.UTF_8);
     }
 
-    private static SummaryAndComment buildSummaryAndComment(Map<String, Object> data) {
+    private static SummaryAndComment buildSummaryAndComment(Map<String, Object> data, String title, String marker, String successMessage) {
         List<String> summaryLines = new ArrayList<>();
         List<String> commentLines = new ArrayList<>();
         Object resultsObj = data.get("results");
@@ -114,8 +120,10 @@ private static SummaryAndComment buildSummaryAndComment(Map<String, Object> data
         }
 
         if (!commentEntries.isEmpty()) {
-            commentLines.add("### Android screenshot updates");
-            commentLines.add("");
+            if (title != null && !title.isEmpty()) {
+                commentLines.add("### " + title);
+                commentLines.add("");
+            }
             for (Map<String, Object> entry : commentEntries) {
                 String test = stringValue(entry.get("test"), "");
                 String status = stringValue(entry.get("status"), "");
@@ -127,11 +135,11 @@ private static SummaryAndComment buildSummaryAndComment(Map<String, Object> data
             if (!commentLines.isEmpty() && !commentLines.get(commentLines.size() - 1).isEmpty()) {
                 commentLines.add("");
             }
-            commentLines.add(MARKER);
+            commentLines.add(marker);
         } else {
-            commentLines.add("✅ Native Android screenshot tests passed.");
+            commentLines.add(successMessage != null ? successMessage : DEFAULT_SUCCESS_MESSAGE);
             commentLines.add("");
-            commentLines.add(MARKER);
+            commentLines.add(marker);
         }
         return new SummaryAndComment(summaryLines, commentLines);
     }
@@ -258,17 +266,26 @@ private static class Arguments {
         final Path compareJson;
         final Path commentOut;
         final Path summaryOut;
+        final String marker;
+        final String title;
+        final String successMessage;
 
-        private Arguments(Path compareJson, Path commentOut, Path summaryOut) {
+        private Arguments(Path compareJson, Path commentOut, Path summaryOut, String marker, String title, String successMessage) {
             this.compareJson = compareJson;
             this.commentOut = commentOut;
             this.summaryOut = summaryOut;
+            this.marker = marker;
+            this.title = title;
+            this.successMessage = successMessage;
         }
 
         static Arguments parse(String[] args) {
             Path compare = null;
             Path comment = null;
             Path summary = null;
+            String marker = null;
+            String title = null;
+            String successMessage = null;
             for (int i = 0; i < args.length; i++) {
                 String arg = args[i];
                 switch (arg) {
@@ -293,6 +310,27 @@ static Arguments parse(String[] args) {
                         }
                         summary = Path.of(args[i]);
                     }
+                    case "--marker" -> {
+                        if (++i >= args.length) {
+                            System.err.println("Missing value for --marker");
+                            return null;
+                        }
+                        marker = args[i];
+                    }
+                    case "--title" -> {
+                        if (++i >= args.length) {
+                            System.err.println("Missing value for --title");
+                            return null;
+                        }
+                        title = args[i];
+                    }
+                    case "--success-message" -> {
+                        if (++i >= args.length) {
+                            System.err.println("Missing value for --success-message");
+                            return null;
+                        }
+                        successMessage = args[i];
+                    }
                     default -> {
                         System.err.println("Unknown argument: " + arg);
                         return null;
@@ -303,7 +341,7 @@ static Arguments parse(String[] args) {
                 System.err.println("--compare-json, --comment-out, and --summary-out are required");
                 return null;
             }
-            return new Arguments(compare, comment, summary);
+            return new Arguments(compare, comment, summary, marker, title, successMessage);
         }
     }
 }

scripts/build-ios-app.sh

@@ -174,6 +174,18 @@ fi
 
 bia_log "Found generated iOS project at $PROJECT_DIR"
 
+UITEST_TEMPLATE="$SCRIPT_DIR/ios/tests/HelloCodenameOneUITests.swift.tmpl"
+if [ -f "$UITEST_TEMPLATE" ]; then
+  IOS_UITEST_DIR="$(find "$PROJECT_DIR" -maxdepth 1 -type d -name '*UITests' -print -quit 2>/dev/null || true)"
+  if [ -n "$IOS_UITEST_DIR" ]; then
+    UI_TEST_DEST="$IOS_UITEST_DIR/templateUITests.swift"
+    bia_log "Installing UI test template at $UI_TEST_DEST"
+    cp "$UITEST_TEMPLATE" "$UI_TEST_DEST"
+  else
+    bia_log "Warning: Could not locate a *UITests target directory under $PROJECT_DIR; UI tests will be skipped"
+  fi
+fi
+
 if [ -f "$PROJECT_DIR/Podfile" ]; then
   bia_log "Installing CocoaPods dependencies"
   (

scripts/ios/screenshots/README.md

@@ -0,0 +1,3 @@
+# iOS screenshot baselines
+
+This directory stores the reference images that the CI iOS UI tests compare against. Add PNG files named after the test identifiers (e.g. `MainActivity.png`) once the first successful baseline has been captured.

scripts/ios/tests/HelloCodenameOneUITests.swift.tmpl

@@ -0,0 +1,55 @@
+import XCTest
+
+final class HelloCodenameOneUITests: XCTestCase {
+    private var app: XCUIApplication!
+    private var outputDirectory: URL!
+
+    override func setUpWithError() throws {
+        continueAfterFailure = false
+
+        app = XCUIApplication()
+        let environment = ProcessInfo.processInfo.environment
+        if let outputPath = environment["CN1SS_OUTPUT_DIR"], !outputPath.isEmpty {
+            outputDirectory = URL(fileURLWithPath: outputPath)
+        } else {
+            outputDirectory = URL(fileURLWithPath: NSTemporaryDirectory())
+        }
+        try FileManager.default.createDirectory(at: outputDirectory, withIntermediateDirectories: true)
+
+        app.launch()
+    }
+
+    override func tearDownWithError() throws {
+        app?.terminate()
+        app = nil
+    }
+
+    private func captureScreenshot(named name: String) throws {
+        let screenshot = XCUIScreen.main.screenshot()
+        let pngURL = outputDirectory.appendingPathComponent("\(name).png")
+        try screenshot.pngRepresentation.write(to: pngURL)
+
+        let attachment = XCTAttachment(screenshot: screenshot)
+        attachment.name = name
+        attachment.lifetime = .keepAlways
+        add(attachment)
+    }
+
+    func testMainScreenScreenshot() throws {
+        XCTAssertTrue(app.staticTexts["Hello Codename One"].waitForExistence(timeout: 10))
+        sleep(1)
+        try captureScreenshot(named: "MainActivity")
+    }
+
+    func testBrowserComponentScreenshot() throws {
+        let button = app.buttons["Open Browser Screen"]
+        XCTAssertTrue(button.waitForExistence(timeout: 10))
+        button.tap()
+
+        let webView = app.webViews.firstMatch
+        XCTAssertTrue(webView.waitForExistence(timeout: 15))
+        sleep(2)
+        try captureScreenshot(named: "BrowserComponent")
+    }
+}
+