Merge pull request #81 from NakaokaRei/feat/keyboard-input-source-context

feat: add keyboard input source context to AI agent

Author: NakaokaRei

Sources/SwiftAutoGUI/OpenAIVisionBackend.swift

@@ -249,13 +249,18 @@ extension OpenAIVisionBackend {
 
             You will also receive structured screen context alongside the screenshot. This includes:
             - The frontmost application name and bundle identifier
+            - The current keyboard input source / IME mode (e.g., "U.S.", "日本語ローマ字")
             - A list of visible windows with their titles, owning apps, and screen bounds
             - An accessibility tree of the focused window showing UI elements with their roles, labels, values, and positions
 
             Use this context to precisely locate UI elements. The bounding boxes in the accessibility tree \
             give exact coordinates you can use with move/click actions. Prefer using accessibility tree \
             coordinates over guessing positions from the screenshot when available. \
             The accessibility tree may be truncated ([...]) for deeply nested elements.
+
+            When the keyboard input source indicates a non-ASCII input mode (e.g., Japanese), \
+            consider switching to an ASCII-capable source before using the 'write' action for English text. \
+            Common shortcuts to toggle input source include Control+Space or Caps Lock, depending on user settings.
             """
         }
 

Sources/SwiftAutoGUI/ScreenContext.swift

@@ -5,6 +5,7 @@
 
 import AppKit
 import ApplicationServices
+import Carbon
 import Foundation
 
 // MARK: - Data Types
@@ -24,10 +25,14 @@ public struct ScreenContext: Sendable, Codable {
     /// The accessibility tree of the focused window (nil if unavailable).
     public let focusedWindowAXTree: AXNode?
 
-    public init(frontmostApp: AppInfo?, visibleWindows: [WindowInfo], focusedWindowAXTree: AXNode?) {
+    /// The current keyboard input source / IME mode (nil if unavailable).
+    public let keyboardInputSource: InputSourceInfo?
+
+    public init(frontmostApp: AppInfo?, visibleWindows: [WindowInfo], focusedWindowAXTree: AXNode?, keyboardInputSource: InputSourceInfo? = nil) {
         self.frontmostApp = frontmostApp
         self.visibleWindows = visibleWindows
         self.focusedWindowAXTree = focusedWindowAXTree
+        self.keyboardInputSource = keyboardInputSource
     }
 }
 
@@ -44,6 +49,20 @@ public struct AppInfo: Sendable, Codable {
     }
 }
 
+/// Information about the current keyboard input source (IME state).
+public struct InputSourceInfo: Sendable, Codable {
+    /// The input source identifier (e.g., "com.apple.inputmethod.Japanese.RomajiTyping").
+    public let id: String
+
+    /// The localized display name (e.g., "日本語ローマ字", "U.S.").
+    public let localizedName: String
+
+    public init(id: String, localizedName: String) {
+        self.id = id
+        self.localizedName = localizedName
+    }
+}
+
 /// Information about a visible window on screen.
 public struct WindowInfo: Sendable, Codable {
     public let title: String?
@@ -157,6 +176,7 @@ public struct ScreenContextProvider: Sendable {
     public static func gather(options: Options = Options()) -> ScreenContext {
         let frontmostApp = gatherFrontmostApp()
         let visibleWindows = gatherVisibleWindows()
+        let inputSource = gatherKeyboardInputSource()
 
         var axTree: AXNode?
         if options.includeAXTree, let app = frontmostApp {
@@ -167,7 +187,8 @@ public struct ScreenContextProvider: Sendable {
         return ScreenContext(
             frontmostApp: frontmostApp,
             visibleWindows: visibleWindows,
-            focusedWindowAXTree: axTree
+            focusedWindowAXTree: axTree,
+            keyboardInputSource: inputSource
         )
     }
 }
@@ -185,6 +206,26 @@ extension ScreenContextProvider {
     }
 }
 
+// MARK: - Gathering: Keyboard Input Source
+
+extension ScreenContextProvider {
+    private static func gatherKeyboardInputSource() -> InputSourceInfo? {
+        guard let source = TISCopyCurrentKeyboardInputSource()?.takeRetainedValue() else {
+            return nil
+        }
+
+        let idPtr = TISGetInputSourceProperty(source, kTISPropertyInputSourceID)
+        let namePtr = TISGetInputSourceProperty(source, kTISPropertyLocalizedName)
+
+        guard let idPtr, let namePtr else { return nil }
+
+        let id = Unmanaged<CFString>.fromOpaque(idPtr).takeUnretainedValue() as String
+        let localizedName = Unmanaged<CFString>.fromOpaque(namePtr).takeUnretainedValue() as String
+
+        return InputSourceInfo(id: id, localizedName: localizedName)
+    }
+}
+
 // MARK: - Gathering: Visible Windows
 
 extension ScreenContextProvider {
@@ -378,6 +419,11 @@ extension ScreenContext {
             lines.append("Frontmost app: \(app.name)\(bundle)")
         }
 
+        // Keyboard input source
+        if let inputSource = keyboardInputSource {
+            lines.append("Keyboard input source: \(inputSource.localizedName) (\(inputSource.id))")
+        }
+
         // Visible windows
         if !visibleWindows.isEmpty {
             lines.append("Visible windows:")

Tests/SwiftAutoGUITests/ScreenContextTests.swift

@@ -10,6 +10,21 @@ import Testing
 @Suite("ScreenContext Tests")
 struct ScreenContextTests {
 
+    // MARK: - InputSourceInfo Tests
+
+    @Suite("InputSourceInfo")
+    struct InputSourceInfoTests {
+
+        @Test("round-trip encoding/decoding")
+        func roundTrip() throws {
+            let info = InputSourceInfo(id: "com.apple.keylayout.US", localizedName: "U.S.")
+            let data = try JSONEncoder().encode(info)
+            let decoded = try JSONDecoder().decode(InputSourceInfo.self, from: data)
+            #expect(decoded.id == info.id)
+            #expect(decoded.localizedName == info.localizedName)
+        }
+    }
+
     // MARK: - CodableRect Tests
 
     @Suite("CodableRect")
@@ -196,6 +211,33 @@ struct ScreenContextTests {
             #expect(output.contains("[...]"))
         }
 
+        @Test("formats keyboard input source")
+        func keyboardInputSource() {
+            let context = ScreenContext(
+                frontmostApp: nil,
+                visibleWindows: [],
+                focusedWindowAXTree: nil,
+                keyboardInputSource: InputSourceInfo(
+                    id: "com.apple.keylayout.US",
+                    localizedName: "U.S."
+                )
+            )
+            let output = context.formatted()
+            #expect(output.contains("Keyboard input source: U.S. (com.apple.keylayout.US)"))
+        }
+
+        @Test("omits keyboard input source when nil")
+        func noKeyboardInputSource() {
+            let context = ScreenContext(
+                frontmostApp: nil,
+                visibleWindows: [],
+                focusedWindowAXTree: nil,
+                keyboardInputSource: nil
+            )
+            let output = context.formatted()
+            #expect(!output.contains("Keyboard input source"))
+        }
+
         @Test("full context output combines all sections")
         func fullContext() {
             let context = ScreenContext(