Continue tests after a crash when using isolated = false (#1)

* Add ability to continue test execution after failure with isolated=true
* Fix bug where isolated=false would execute all the tests.
* Write tests to a file instead of passing them as arguments avoiding the
argument limitation.

Author: runningcode

build_and_install.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# Script to build AndroidX Test m2repository and install orchestrator APKs
+# Based on CONTRIBUTING.md instructions
+
+set -e  # Exit on any error
+
+echo "🔨 Building AndroidX Test m2repository..."
+bazelisk build :axt_m2repository
+
+echo "📦 Unpacking m2repository to ~/.m2/"
+unzip -o bazel-bin/axt_m2repository.zip -d ~/.m2/
+
+echo "📱 Installing orchestrator and services APKs..."
+
+# Find and install test services APK
+SERVICES_APK=~/.m2/repository/androidx/test/services/test-services/1.7.0-alpha01/test-services-1.7.0-alpha01.apk
+if [ -n "$SERVICES_APK" ]; then
+    echo "Installing test services APK: $SERVICES_APK"
+    adb install --force-queryable -r "$SERVICES_APK"
+else
+    echo "❌ Test services APK not found in ~/.m2/repository"
+    exit 1
+fi
+
+# Find and install orchestrator APK  
+ORCHESTRATOR_APK=~/.m2/repository/androidx/test/orchestrator/1.7.0-alpha01/orchestrator-1.7.0-alpha01.apk
+if [ -n "$ORCHESTRATOR_APK" ]; then
+    echo "Installing orchestrator APK: $ORCHESTRATOR_APK"
+    adb install --force-queryable -r "$ORCHESTRATOR_APK"
+else
+    echo "❌ Orchestrator APK not found in ~/.m2/repository"
+    exit 1
+fi
+
+echo "✅ Build and installation complete!"

runner/android_test_orchestrator/java/androidx/test/orchestrator/AndroidTestOrchestrator.java

@@ -40,26 +40,37 @@
 import android.os.Build;
 import android.os.Bundle;
 import android.os.Debug;
+import android.os.Environment;
 import android.os.IBinder;
 import android.os.RemoteException;
 import android.text.TextUtils;
 import android.util.Log;
+
 import androidx.annotation.VisibleForTesting;
 import androidx.core.content.ContextCompat;
 import androidx.test.orchestrator.TestRunnable.RunFinishedListener;
 import androidx.test.orchestrator.junit.ParcelableDescription;
+import androidx.test.orchestrator.junit.ParcelableFailure;
 import androidx.test.orchestrator.listeners.OrchestrationListenerManager;
 import androidx.test.orchestrator.listeners.OrchestrationResult;
 import androidx.test.orchestrator.listeners.OrchestrationResultPrinter;
+import androidx.test.orchestrator.listeners.OrchestrationRunListener;
 import androidx.test.services.shellexecutor.ClientNotConnected;
 import androidx.test.services.shellexecutor.ShellExecSharedConstants;
 import androidx.test.services.shellexecutor.ShellExecutor;
 import androidx.test.services.shellexecutor.ShellExecutorFactory;
+
+import java.io.BufferedWriter;
 import java.io.ByteArrayOutputStream;
+import java.io.File;
 import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
+import java.io.OutputStreamWriter;
 import java.io.PrintStream;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -127,6 +138,27 @@
 public final class AndroidTestOrchestrator extends android.app.Instrumentation
     implements RunFinishedListener {
 
+  private class OrchestratorListener extends OrchestrationRunListener {
+    @Override
+    public void testStarted(ParcelableDescription description) {
+      if (!runsInIsolatedMode(arguments)) {
+        test = description.getClassName() + "#" + description.getMethodName();
+        testsToExecuteNoIsolation.remove(test);
+      }
+    }
+
+    @Override
+    public void testFailure(ParcelableFailure failure) {
+        isRecoveringFromCrash = true;
+    }
+
+    @Override
+    public void testProcessFinished(String message) {
+      super.testProcessFinished(message);
+      Log.i(TAG, "Test process finished: " + message);
+    }
+  }
+
   private static final String TAG = "AndroidTestOrchestrator";
   // As defined in the AndroidManifest of the Orchestrator app.
   private static final String ORCHESTRATOR_SERVICE_LOCATION = "OrchestratorService";
@@ -145,6 +177,7 @@ public final class AndroidTestOrchestrator extends android.app.Instrumentation
   private final OrchestrationResultPrinter resultPrinter = new OrchestrationResultPrinter();
   private final OrchestrationListenerManager listenerManager =
       new OrchestrationListenerManager(this);
+  private final OrchestratorListener orchestratorListener = new OrchestratorListener();
 
   private final ExecutorService executorService;
 
@@ -158,6 +191,9 @@ public final class AndroidTestOrchestrator extends android.app.Instrumentation
   private String test;
   private Iterator<String> testIterator;
 
+  private List<String> testsToExecuteNoIsolation;
+  private boolean isRecoveringFromCrash = false;
+
   public AndroidTestOrchestrator() {
     super();
     // We never want to execute multiple tests in parallel.
@@ -315,6 +351,7 @@ public void runFinished() {
     if (null == test) {
       List<String> allTests = callbackLogic.provideCollectedTests();
       testIterator = allTests.iterator();
+      testsToExecuteNoIsolation = new ArrayList<>(allTests);
       addListeners(allTests.size());
 
       if (allTests.isEmpty()) {
@@ -333,17 +370,76 @@ public void runFinished() {
   }
 
   private void executeEntireTestSuite() {
+    Log.i(TAG, "Executing entire test suite...");
+    // If we're recovering from a crash, continue with remaining tests
+    if (isRecoveringFromCrash && testsToExecuteNoIsolation != null && !testsToExecuteNoIsolation.isEmpty()) {
+      isRecoveringFromCrash = false;
+      executeRemainingTests();
+      return;
+    }
+    
     if (null != test) {
       finish(Activity.RESULT_OK, createResultBundle());
       return;
     }
 
-    // We don't actually need test to have any particular value,
-    // just to indicate we've started execution.
+    executeRemainingTests();
+  }
+
+  private void executeRemainingTests() {
+    Log.i(TAG, "Executing remaining tests...");
+    // Create a list of remaining tests from the current iterator position
+    List<String> remainingTests = new ArrayList<>(testsToExecuteNoIsolation);
+    
+    if (remainingTests.isEmpty()) {
+      Log.i(TAG, "No remaining tests to execute.");
+      finish(Activity.RESULT_OK, createResultBundle());
+      return;
+    }
+    
+    // Set test to indicate execution has started
     test = "";
+    // Execute remaining tests using a subset TestRunnable, we need to prevent the argument list from being too long
+    // Run 500 tests at a time
+    Log.i(TAG, "Executing subset of remaining tests: " + remainingTests.size() + " tests.");
+    
+    // Write tests to file to avoid argument length limits
+    String testFilePath = writeTestsToFile(remainingTests);
+    Log.i(TAG, "Test file path: " + testFilePath);
+    
     executorService.execute(
-        TestRunnable.legacyTestRunnable(
-            getContext(), getSecret(arguments), arguments, getOutputStream(), this));
+            TestRunnable.testSubsetRunnable(
+                    getContext(), getSecret(arguments), arguments, getOutputStream(), this, testFilePath));
+  }
+
+  private String writeTestsToFile(List<String> tests) {
+    String fileName = "test_subset.txt";
+    String filePath = "/data/local/tmp/" + fileName;
+    Context context = getContext();
+    String secret = getSecret(arguments);
+    
+    try {
+      // Create the file using shell command to ensure proper permissions
+      execShellCommandSync(context, secret, "touch", Arrays.asList(filePath));
+      
+      // Make the file world readable/writable so both processes can access it
+      execShellCommandSync(context, secret, "chmod", Arrays.asList("666", filePath));
+      
+      // Write the test content to the file using shell commands
+      StringBuilder content = new StringBuilder();
+      for (String test : tests) {
+        content.append(test).append("\n");
+      }
+      
+      // Use echo to write content (escape any special characters)
+      String escapedContent = content.toString().replace("\"", "\\\"").replace("$", "\\$");
+      execShellCommandSync(context, secret, "sh", Arrays.asList("-c", "echo \"" + escapedContent + "\" > " + filePath));
+      
+      return filePath;
+    } catch (Exception e) {
+      Log.e(TAG, "Failed to write tests to file using shell commands", e);
+      throw new RuntimeException("Could not write tests to file", e);
+    }
   }
 
   private void executeNextTest() {
@@ -434,6 +530,7 @@ private String getOutputFile(String testName) {
   private void addListeners(int testSize) {
     listenerManager.addListener(resultBuilder);
     listenerManager.addListener(resultPrinter);
+    listenerManager.addListener(orchestratorListener);
     listenerManager.orchestrationRunStarted(testSize);
   }
 

runner/android_test_orchestrator/java/androidx/test/orchestrator/TestRunnable.java

@@ -45,65 +45,68 @@ public class TestRunnable implements Runnable {
   private final RunFinishedListener listener;
   private final OutputStream outputStream;
   private final String test;
+  private final String testFilePath;
   private final boolean collectTests;
   private final Context context;
   private final String secret;
 
   /**
-   * Constructs a TestRunnable executes all tests in arguments.
+   * Constructs a TestRunnable which will run a single test.
    *
    * @param context A context
    * @param secret A string representing the speakeasy binder key
    * @param arguments contains arguments to be passed to the target instrumentation
    * @param outputStream the stream to write the results of the test process
    * @param listener a callback listener to know when the run has completed
+   * @param test contains a specific test#method to run. Will override whatever is specified in the
+   *     bundle.
    */
-  public static TestRunnable legacyTestRunnable(
+  public static TestRunnable singleTestRunnable(
       Context context,
       String secret,
       Bundle arguments,
       OutputStream outputStream,
-      RunFinishedListener listener) {
-    return new TestRunnable(context, secret, arguments, outputStream, listener, null, false);
+      RunFinishedListener listener,
+      String test) {
+    return new TestRunnable(context, secret, arguments, outputStream, listener, test, null, false);
   }
 
   /**
-   * Constructs a TestRunnable which will run a single test.
+   * Constructs a TestRunnable which will ask the instrumentation to list out its tests.
    *
    * @param context A context
    * @param secret A string representing the speakeasy binder key
    * @param arguments contains arguments to be passed to the target instrumentation
    * @param outputStream the stream to write the results of the test process
    * @param listener a callback listener to know when the run has completed
-   * @param test contains a specific test#method to run. Will override whatever is specified in the
-   *     bundle.
    */
-  public static TestRunnable singleTestRunnable(
+  public static TestRunnable testCollectionRunnable(
       Context context,
       String secret,
       Bundle arguments,
       OutputStream outputStream,
-      RunFinishedListener listener,
-      String test) {
-    return new TestRunnable(context, secret, arguments, outputStream, listener, test, false);
+      RunFinishedListener listener) {
+    return new TestRunnable(context, secret, arguments, outputStream, listener, null, null, true);
   }
 
   /**
-   * Constructs a TestRunnable which will ask the instrumentation to list out its tests.
+   * Constructs a TestRunnable which will run a specific subset of tests from a file.
    *
    * @param context A context
    * @param secret A string representing the speakeasy binder key
    * @param arguments contains arguments to be passed to the target instrumentation
    * @param outputStream the stream to write the results of the test process
    * @param listener a callback listener to know when the run has completed
+   * @param testFilePath the path to a file containing the tests to run
    */
-  public static TestRunnable testCollectionRunnable(
+  public static TestRunnable testSubsetRunnable(
       Context context,
       String secret,
       Bundle arguments,
       OutputStream outputStream,
-      RunFinishedListener listener) {
-    return new TestRunnable(context, secret, arguments, outputStream, listener, null, true);
+      RunFinishedListener listener,
+      String testFilePath) {
+    return new TestRunnable(context, secret, arguments, outputStream, listener, null, testFilePath, false);
   }
 
   @VisibleForTesting
@@ -114,13 +117,15 @@ public static TestRunnable testCollectionRunnable(
       OutputStream outputStream,
       RunFinishedListener listener,
       String test,
+      String testFilePath,
       boolean collectTests) {
     this.context = context;
     this.secret = secret;
     this.arguments = new Bundle(arguments);
     this.outputStream = outputStream;
     this.listener = listener;
     this.test = test;
+    this.testFilePath = testFilePath;
     this.collectTests = collectTests;
   }
 
@@ -178,9 +183,12 @@ private Bundle getTargetInstrumentationArguments() {
       targetArgs.remove("testFile");
     }
 
-    // Override the class parameter with the current test target.
+    // Override the class parameter with the current test target or use testFile for test subset.
     if (test != null) {
       targetArgs.putString(AJUR_CLASS_ARGUMENT, test);
+    } else if (testFilePath != null && !testFilePath.isEmpty()) {
+      // For test subset, use testFile parameter to read tests from file
+      targetArgs.putString("testFile", testFilePath);
     }
 
     return targetArgs;

runner/android_test_orchestrator/javatests/androidx/test/orchestrator/TestRunnableTest.java

@@ -59,7 +59,7 @@ private static class FakeTestRunnable extends TestRunnable {
         RunFinishedListener listener,
         String test,
         boolean collectTests) {
-      super(context, secret, arguments, outputStream, listener, test, collectTests);
+      super(context, secret, arguments, outputStream, listener, test, null, collectTests);
     }
 
     @Override