Add ability to perform UI Automation via connected console test runner

Author: Morten Nielsen

Automation/MSTestX.Console/Adb/AdbClient.UIAutomation.cs

@@ -0,0 +1,53 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace MSTestX.Console.Adb
+{
+    internal partial class AdbClient
+    {
+        // See https://commandmasters.com/commands/input-android/ for reference to ADB input commands.
+
+        public async Task PerformUIAutomationAction(UIAutomationMessage message)
+        {
+            if (message is TapMessage tap)
+            {
+                // send a tap command to the device over ADB.
+                await SendShellCommandAsync($"adb shell input tap {tap.X} {tap.Y}", "deviceId");
+            }
+            if (message is SwipeMessage swipe)
+            {
+                // send a swipe command to the device over ADB.
+                await SendShellCommandAsync($"adb shell input swipe {swipe.FromX} {swipe.FromY} {swipe.ToX} {swipe.ToY} {swipe.DurationMs}", "deviceId");
+            }
+            else if (message is KeyboardMessage keyboard)
+            {
+                // TODO: Send modifiers
+                if(keyboard.Key.Length == 1)
+                    await SendShellCommandAsync($"adb shell input keyevent {KeyToKeyEventCode(keyboard.Key[0])}", "deviceId");
+                else
+                    await SendShellCommandAsync($"adb shell input text \"{keyboard.Key.Replace(" ", "%20")}\"", "deviceId");
+            }
+        }
+
+        private static int KeyToKeyEventCode(char key)
+        {
+            // This is a very simplified mapping. A real implementation would need to handle more keys and modifiers.
+            if (char.IsLetterOrDigit(key))
+            {
+                return char.ToUpper(key) - 'A' + 29; // KeyEvent.KEYCODE_A starts at 29
+            }
+            switch (key)
+            {
+                case ' ':
+                    return 62; // KeyEvent.KEYCODE_SPACE
+                case '\n':
+                    return 66; // KeyEvent.KEYCODE_ENTER
+                default:
+                    throw new NotSupportedException($"Key '{key}' is not supported.");
+            }
+        }
+    }
+}

Automation/MSTestX.Console/Adb/AdbClient.cs

@@ -14,7 +14,7 @@ namespace MSTestX.Console.Adb
 {
     // Good command-reference overview here: https://android.googlesource.com/platform/packages/modules/adb/+/HEAD/SERVICES.TXT
     // ADB Source and overview: https://android.googlesource.com/platform/packages/modules/adb/
-    internal class AdbClient
+    internal partial class AdbClient
     {
         private int port;
 
@@ -173,5 +173,5 @@ private static async Task WriteCommandToSocket(Socket s, string command)
             }
             System.Diagnostics.Debug.WriteLine($"OKAY");
         }
-    }
+    }   
 }

Automation/MSTestX.Console/Program.cs

@@ -397,7 +397,7 @@ private static async Task OnApplicationLaunched(System.Net.IPEndPoint? endpoint
                 return;
             appLaunchDetected = true;
             int result = 0;
-            var runner = new TestRunner(endpoint);
+            var runner = new TestRunner(endpoint, client);
             try
             {
                 await Task.Delay(5000); //Give app some time to start up

Automation/MSTestX.Console/TestRunner.cs

@@ -14,14 +14,16 @@
 
 namespace MSTestX.Console
 {
-    public class TestRunner : IDisposable
+    internal class TestRunner : IDisposable
     {
         private SocketCommunicationManager socket;
         private System.Net.IPEndPoint _endpoint;
+        private Adb.AdbClient? _adbClient;
 
-        public TestRunner(System.Net.IPEndPoint endpoint = null)
+        public TestRunner(System.Net.IPEndPoint endpoint = null, Adb.AdbClient adbClient = null)
         {
             _endpoint = endpoint;
+            _adbClient = adbClient;
         }
 
         public async Task RunTests(string outputFilename, string settingsXml, CancellationToken cancellationToken)
@@ -106,8 +108,22 @@ private async Task RunTestsInternal(string outputFilename, string settingsXml, T
                 {
                     continue;
                 }
-
-                if (msg.MessageType == MessageType.TestHostLaunched)
+                if (msg.MessageType.StartsWith(UIAutomationMessage.AutomationMessageType))
+                {
+                    var uia = UIAutomationMessage.FromMessageType(msg.MessageType, msg.Payload);
+                    if (uia is not null)
+                    {
+                        if (_adbClient is not null)
+                        {
+                            _ = _adbClient.PerformUIAutomationAction(uia);
+                        }
+                        else
+                        {
+                            System.Console.WriteLine("Cannot send UI Automation message to device.");
+                        }
+                    }
+                }
+                else if (msg.MessageType == MessageType.TestHostLaunched)
                 {
                     var thl = JsonDataSerializer.Instance.DeserializePayload<TestHostLaunchedPayload>(msg);
                     pid = thl.ProcessId;

src/TestAppRunner/TestAppRunner/TestAdapterConnection.cs

@@ -17,6 +17,8 @@ internal class TestAdapterConnection
         private int port;
         private System.Threading.Thread? messageLoopThread;
 
+        public bool IsConnected => isConnected;
+
         public TestAdapterConnection(int port)
         {
             this.port = port;

src/TestAppRunner/TestAppRunner/UIAutomation/AutomationMessages.cs

@@ -0,0 +1,56 @@
+#nullable enable
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Microsoft.VisualStudio.TestPlatform.CommunicationUtilities;
+using Newtonsoft.Json;
+
+namespace MSTestX
+{
+    internal class UIAutomationMessage
+    {      
+        internal static string AutomationMessageType => "UIAutomation";
+
+        public string MessageType => $"{AutomationMessageType}.{GetType().Name}";
+
+        public static UIAutomationMessage? FromMessageType(string messageType, Newtonsoft.Json.Linq.JToken payload)
+        {
+            switch (messageType)
+            {
+                case var s when s == $"{AutomationMessageType}.{nameof(TapMessage)}":
+                    return payload.ToObject<TapMessage>();
+                case var s when s == $"{AutomationMessageType}.{nameof(KeyboardMessage)}":
+                    return payload.ToObject<KeyboardMessage>();
+                case var s when s == $"{AutomationMessageType}.{nameof(SwipeMessage)}":
+                    return payload.ToObject<SwipeMessage>();
+                default:
+                    throw new InvalidOperationException($"Unknown message type: {messageType}");
+            }
+        }
+    }
+
+    internal class TapMessage : UIAutomationMessage
+    {
+        public int X { get; init; }
+        public int Y { get; init; }
+    }
+
+    internal class SwipeMessage : UIAutomationMessage
+    {
+        public int FromX { get; init; }
+        public int FromY { get; init; }
+        public int ToX { get; init; }
+        public int ToY { get; init; }
+        public int DurationMs { get; init; }
+    }
+
+    internal class KeyboardMessage : UIAutomationMessage
+    {
+        public string? Key { get; set; }
+
+        public string[]? Modifiers { get; set; }
+    }
+}

src/TestAppRunner/TestAppRunner/UIAutomation/UIAutomation.cs

@@ -0,0 +1,45 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace MSTestX
+{
+    /// <summary>
+    /// Allows the unit test to reach out to the test runner and send UI automation messages to be performed by the host machine.
+    /// This is useful for testing scenarios where the test needs to interact with the UI of the host machine, such as clicking buttons or entering text.
+    /// </summary>
+    public static class UIAutomation
+    {
+        /// <summary>
+        /// Tests if the test runner is currently connected to a host runner. If this is false, any attempt to send a message will throw an exception.
+        /// </summary>
+        public static bool IsConnected => TestAppRunner.ViewModels.TestRunnerVM.Instance.Connection?.IsConnected ?? false;
+
+        public static void Tap(int x, int y)
+        {
+            SendMessage(new TapMessage { X = x, Y = y });
+        }
+
+        public static void SendSwipe(int fromX, int fromY, int toX, int toY, int durationMs = 500)
+        {
+            SendMessage(new SwipeMessage { FromX = fromX, FromY = fromY, ToX = toX, ToY = toY, DurationMs = durationMs });
+        }
+
+        public static void SendLongPress(int x, int y, int durationMs = 1000)
+        {
+            SendMessage(new SwipeMessage { FromX = x, FromY = y, ToX = x, ToY = y, DurationMs = durationMs });
+        }
+
+        public void SendTextInput(string text) => SendMessage(new KeyboardMessage { Key = text });
+
+        public static void SendKeyboardInput(char key) => SendMessage(new KeyboardMessage { Key = key.ToString() });
+
+        private static void SendMessage(UIAutomationMessage message)
+        {
+            if (!IsConnected) throw new InvalidOperationException("Test Runner is not connected to host runner");
+            TestAppRunner.ViewModels.TestRunnerVM.Instance.Connection.SendMessage(message.MessageType, message);
+        }
+    }
+}

src/TestAppRunner/TestAppRunner/ViewModels/TestRunnerVM.cs

@@ -32,6 +32,7 @@ internal class TestRunnerVM : VMBase, ITestExecutionRecorder
         private System.IO.StreamWriter logOutput;
         private TrxWriter trxWriter;
         private TestAdapterConnection connection;
+        internal TestAdapterConnection Connection => connection;
 
         private static TestRunnerVM _Instance;