feat: add get_visual_tree tool (#293)

Author: DearVa

src/Everywhere.Core/Chat/Plugins/VisualContextPlugin.cs

@@ -79,6 +79,13 @@ public interface IVisualElementContext : IObservable<TextSelectionData>
     /// <returns></returns>
     IVisualElement? ElementFromPointer(ScreenSelectionMode mode = ScreenSelectionMode.Element);
 
+    /// <summary>
+    /// Get the element from a native window handle.
+    /// </summary>
+    /// <param name="windowHandle"></param>
+    /// <returns></returns>
+    IVisualElement? ElementFromWindowHandle(nint windowHandle);
+
     /// <summary>
     /// Let the user pick an element from the screen.
     /// </summary>

src/Everywhere.Core/Interop/IVisualElementContext.cs

@@ -85,11 +85,11 @@
 
       <DockPanel Grid.Row="2">
         <Button
-          DockPanel.Dock="Right" Click="HandleBuildXmlButtonClicked"
-          Content="Build XML"/>
+          DockPanel.Dock="Right" Click="HandleBuildButtonClicked"
+          Content="Build"/>
         <TextBox
           x:Name="TokenLimitTextBox" Text="8000"
-          Watermark="Token limit"/>
+          Watermark="Limit"/>
       </DockPanel>
 
       <Button

src/Everywhere.Core/Views/Controls/VisualTreeDebugger.axaml

@@ -138,46 +138,60 @@ private async void HandleCaptureButtonClicked(object? sender, RoutedEventArgs e)
         }
     }
 
-    private async void HandleBuildXmlButtonClicked(object? sender, RoutedEventArgs e)
+    private async void HandleBuildButtonClicked(object? sender, RoutedEventArgs e)
     {
         try
         {
+            const VisualTreeDetailLevel level = VisualTreeDetailLevel.Compact;
             var tokenLimit = int.Parse(TokenLimitTextBox.Text ?? "8000");
             var builder = new VisualTreeBuilder(
                 VisualTreeView.SelectedItems.AsValueEnumerable().OfType<IVisualElement>().ToList(),
                 tokenLimit,
                 0,
-                VisualTreeDetailLevel.Compact);
+                level);
 #if DEBUG
-            // use profiler to measure xml building time in debug mode
-            var xml = await Task.Run(() =>
+            // use profiler to measure building time in debug mode
+            var visualTree = await Task.Run(() =>
             {
                 var originalThreadName = Thread.CurrentThread.Name;
-                Thread.CurrentThread.Name = "XML Builder Thread";
-                MeasureProfiler.StartCollectingData("BuildXml");
+                Thread.CurrentThread.Name = "Visual Tree Builder Thread";
+                MeasureProfiler.StartCollectingData("BuildVisualTree");
 
                 try
                 {
                     return builder.Build(CancellationToken.None);
                 }
                 finally
                 {
-                    MeasureProfiler.SaveData("BuildXml");
+                    MeasureProfiler.SaveData("BuildVisualTree");
                     Thread.CurrentThread.Name = originalThreadName;
                 }
             });
 #else
-            var xml = await Task.Run(() => builder.Build(CancellationToken.None));
+            var visualTree = await Task.Run(() => builder.Build(CancellationToken.None));
 #endif
             var timestamp = DateTime.Now.ToString("yyyyMMdd_HHmmss");
-            var filename = $"visual_tree_{timestamp}.xml";
-            var xmlPath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, filename);
-            await File.WriteAllTextAsync(xmlPath, xml);
-            await ServiceLocator.Resolve<ILauncher>().LaunchFileInfoAsync(new FileInfo(xmlPath));
+            var extension = level switch
+            {
+                VisualTreeDetailLevel.Compact => "json",
+                VisualTreeDetailLevel.Detailed => "xml",
+                _ => "toon"
+            };
+            var filename = $"visual_tree_{timestamp}.{extension}";
+            var filePath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, filename);
+            await File.WriteAllTextAsync(filePath, visualTree);
+            await ServiceLocator.Resolve<ILauncher>().LaunchFileInfoAsync(new FileInfo(filePath));
         }
+#if DEBUG
+        catch (Exception ex)
+        {
+            _ = ex;
+            Debugger.Break();
+#else
         catch
         {
             // ignored
+#endif
         }
     }
 }

src/Everywhere.Core/Views/Controls/VisualTreeDebugger.axaml.cs

@@ -2132,4 +2132,10 @@ Use scroll wheel or number keys to switch selection mode</value>
     <data name="ChatPluginConsentRequest_CommonNone_Header" xml:space="preserve">
         <value>Do you allow '{0}' to execute?</value>
     </data>
+    <data name="BuiltInChatPlugin_VisualContext_GetVisualTree_Header" xml:space="preserve">
+        <value>Read Visual Tree</value>
+    </data>
+    <data name="BuiltInChatPlugin_VisualContext_GetVisualTree_Description" xml:space="preserve">
+        <value>Read the visual tree of a window or element on screen.</value>
+    </data>
 </root>

src/Everywhere.I18N/Strings.resx

@@ -2133,4 +2133,10 @@
     <data name="ChatPluginConsentRequest_CommonNone_Header" xml:space="preserve">
         <value>是否允许“{0}”执行？</value>
     </data>
+    <data name="BuiltInChatPlugin_VisualContext_GetVisualTree_Header" xml:space="preserve">
+        <value>读取视觉树</value>
+    </data>
+    <data name="BuiltInChatPlugin_VisualContext_GetVisualTree_Description" xml:space="preserve">
+        <value>读取屏幕上窗口或元素的视觉树。</value>
+    </data>
 </root>

src/Everywhere.I18N/Strings.zh-hans.resx

@@ -78,6 +78,11 @@ public IVisualElement? FocusedElement
         return ElementFromPoint(point, mode);
     }
 
+    public IVisualElement? ElementFromWindowHandle(IntPtr windowHandle)
+    {
+        throw new NotImplementedException();
+    }
+
     public async Task<IVisualElement?> PickElementAsync(ScreenSelectionMode? initialMode)
     {
         if (Application.Current is not { ApplicationLifetime: ClassicDesktopStyleApplicationLifetime desktopLifetime })
@@ -92,18 +97,18 @@ public IVisualElement? FocusedElement
         return result;
     }
 
-    public async Task<Avalonia.Media.Imaging.Bitmap?> ScreenshotAsync(ScreenSelectionMode? initialMode)
+    public async Task<Bitmap?> ScreenshotAsync(ScreenSelectionMode? initialMode)
     {
-         if (Application.Current is not { ApplicationLifetime: ClassicDesktopStyleApplicationLifetime desktopLifetime })
+        if (Application.Current is not { ApplicationLifetime: ClassicDesktopStyleApplicationLifetime desktopLifetime })
         {
             return null;
         }
 
         var windows = desktopLifetime.Windows.AsValueEnumerable().Where(w => w.IsVisible).ToList();
         foreach (var window in windows) window.Hide();
-        
+
         var result = await ScreenshotPicker.ScreenshotAsync(this, backend, initialMode);
-        
+
         foreach (var window in windows) window.IsVisible = true;
         return result;
     }
@@ -112,4 +117,4 @@ public IDisposable Subscribe(IObserver<TextSelectionData> observer)
     {
         return backend.Subscribe(observer);
     }
-}
+}

src/Everywhere.Linux/Interop/VisualElementContext.cs

@@ -277,9 +277,6 @@ public Task<Bitmap> CaptureAsync(CancellationToken cancellationToken)
             var screen = NSScreen.Screens.FirstOrDefault(s => s.Frame.IntersectsWith(rect));
             var scale = screen?.BackingScaleFactor ?? 1.0;
 
-            var targetWidth = rect.Width * scale;
-            var targetHeight = rect.Height * scale;
-
             // cgImage captures the window content starting at (0,0) in Window Local Coordinates.
             // rect contains Screen Coordinates (including Dock/Menu bar offsets).
             // To crop correctly, we must transform rect to Window-Relative coordinates.
@@ -303,7 +300,8 @@ public Task<Bitmap> CaptureAsync(CancellationToken cancellationToken)
 
             // Check if captured image approximately matches target size (allowing for rounding/shadows).
             // If it matches, we assume full window capture and start at 0,0.
-            bool isFullWindow = cgImage.Width >= targetWidth - 2 && cgImage.Width <= targetWidth + 100;
+            var targetWidth = rect.Width * scale;
+            var isFullWindow = cgImage.Width >= targetWidth - 2 && cgImage.Width <= targetWidth + 100;
 
             // If full window, offset is 0. 
             // If partial (element inside window), offset is (ElementScreenPos - WindowScreenPos).
@@ -406,6 +404,70 @@ private void PerformAction(NSString actionName)
         return handle != 0 ? new AXUIElement(handle) : null;
     }
 
+    /// <summary>
+    /// Gets the AXUIElement corresponding to the specified CGWindowID.
+    /// This is a reverse lookup using _AXUIElementGetWindow under the hood.
+    /// </summary>
+    /// <param name="cgWindowId">The target CGWindowID.</param>
+    /// <returns>The matching AXUIElement, or null if not found.</returns>
+    public static AXUIElement? ElementFromWindowId(uint cgWindowId)
+    {
+        if (cgWindowId == 0) return null;
+
+        // 1. Get the owner PID from the CGWindowID using CoreGraphics
+        var ownerPid = 0;
+        var windowInfoArrayPtr = CGInterop.CGWindowListCopyWindowInfo(CGWindowListOption.IncludingWindow, cgWindowId);
+
+        if (windowInfoArrayPtr != 0)
+        {
+            // Take ownership of the CFArray returned by Create/Copy rule
+            using var windowInfoArray = Runtime.GetNSObject<NSArray>(windowInfoArrayPtr, owns: true);
+            if (windowInfoArray is { Count: > 0 })
+            {
+                using var windowInfo = windowInfoArray.GetItem<NSDictionary>(0);
+                using var pidKey = new NSString("kCGWindowOwnerPID");
+
+                if (windowInfo?.ObjectForKey(pidKey) is NSNumber pidNumber)
+                {
+                    ownerPid = pidNumber.Int32Value;
+                }
+            }
+        }
+
+        if (ownerPid == 0) return null;
+
+        // 2. Create the AXApplication element from the PID
+        using var appElement = ElementFromPid(ownerPid);
+        if (appElement is null) return null;
+
+        // 3. Get all windows of the application
+        // Note: Replace with AXAttributeConstants.Windows if you have it defined.
+        using var windowsKey = new NSString("AXWindows");
+        using var windows = appElement.GetAttribute<NSArray>(windowsKey);
+
+        if (windows is null) return null;
+
+        // 4. Iterate through the windows and find the matching CGWindowID
+        for (nuint i = 0; i < windows.Count; i++)
+        {
+            var windowElement = FromCopyArray(windows, i);
+            if (windowElement is null) continue;
+
+            // Use your existing property which correctly handles the _AXUIElementGetWindow P/Invoke
+            if (windowElement.NativeWindowHandle == (nint)cgWindowId)
+            {
+                // Found it! Return the retained element.
+                return windowElement;
+            }
+
+            // Not a match: explicitly dispose to release the CFRetain applied in FromCopyArray,
+            // avoiding memory leaks during traversal.
+            windowElement.Dispose();
+        }
+
+        return null;
+    }
+
     [LibraryImport(AppServices, EntryPoint = "AXUIElementCreateSystemWide")]
     private static partial nint CreateSystemWide();
 

src/Everywhere.Mac/Interop/AXUIElement.cs

@@ -66,6 +66,11 @@ public partial class VisualElementContext(IWindowHelper windowHelper) : IVisualE
         return point is null ? null : ElementFromPoint(point.Value, mode);
     }
 
+    public IVisualElement? ElementFromWindowHandle(IntPtr windowHandle)
+    {
+        return AXUIElement.ElementFromWindowId((uint)windowHandle);
+    }
+
     public Task<IVisualElement?> PickElementAsync(ScreenSelectionMode? initialMode)
     {
         return PickerSession.PickAsync(windowHelper, initialMode);

src/Everywhere.Mac/Interop/VisualElementContext.cs

@@ -60,6 +60,11 @@ public partial class VisualElementContext(IWindowHelper windowHelper) : IVisualE
         return !PInvoke.GetCursorPos(out var point) ? null : ElementFromPoint(new PixelPoint(point.X, point.Y), mode);
     }
 
+    public IVisualElement? ElementFromWindowHandle(IntPtr windowHandle)
+    {
+        return TryCreateVisualElement(() => Automation.FromHandle(windowHandle));
+    }
+
     public Task<IVisualElement?> PickElementAsync(ScreenSelectionMode? initialMode) => PickerSession.PickAsync(windowHelper, initialMode);
 
     public Task<Bitmap?> ScreenshotAsync(ScreenSelectionMode? initialMode) => ScreenshotSession.ScreenshotAsync(windowHelper, initialMode);