Refactor UIA (#337)

* Refactor UIA

* Update UIA when visual ordering changes

* Fix keyboard input and support UIA removal of all children

---------

Co-authored-by: adpa-ms

Author: adpa-ms

Samples/Islands/DrawingIsland/DrawingCppTestApp/WinMain.cpp

@@ -40,6 +40,11 @@ int __stdcall wWinMain(HINSTANCE, HINSTANCE, PWSTR, int)
     siteBridge.Show();
     siteBridge.Connect(drawing.Island());
 
+    // Move initial focus to the island.
+    auto focusNavigationHost = winrt::InputFocusNavigationHost::GetForSiteBridge(siteBridge);
+    focusNavigationHost.NavigateFocus(winrt::FocusNavigationRequest::Create(
+        winrt::FocusNavigationReason::Programmatic));
+
     queue.RunEventLoop();
 
     controller.ShutdownQueue();

Samples/Islands/DrawingIsland/DrawingIslandComponents/DrawingIsland.cpp

@@ -56,8 +56,6 @@ namespace winrt::DrawingIslandComponents::implementation
 
     DrawingIsland::~DrawingIsland()
     {
-        m_uia.FragmentRoot = nullptr;
-        m_uia.FragmentFactory = nullptr;
         m_output.Compositor = nullptr;
     }
 
@@ -104,9 +102,18 @@ namespace winrt::DrawingIslandComponents::implementation
         }
 #endif
 
-        m_uia.FragmentFactory = nullptr;
-        m_uia.FragmentRoot = nullptr;
-        m_uia.VisualToFragmentMap.clear();
+        // Make sure automation is destroyed.
+        for (auto& automationPeer : m_uia.AutomationPeers)
+        {
+            automationPeer.GetAutomationProvider()->SetCallbackHandler(nullptr);
+        }
+        m_uia.AutomationPeers.clear();
+
+        if (nullptr != m_uia.FragmentRoot)
+        {
+            m_uia.FragmentRoot->SetCallbackHandler(nullptr);
+            m_uia.FragmentRoot = nullptr;
+        }
 
 
         // Destroy Content:
@@ -173,9 +180,105 @@ namespace winrt::DrawingIslandComponents::implementation
     }
 
 
+    winrt::Windows::Graphics::RectInt32
+    DrawingIsland::GetScreenBoundsForAutomationFragment(
+        _In_::IUnknown const* const sender) const
+    {
+        // Check if the automation provider that has sent this callback is the fragment root.
+        if (m_uia.FragmentRoot.as<::IUnknown>().get() == sender)
+        {
+            winrt::Rect logicalRect;
+            logicalRect.X = 0;
+            logicalRect.Y = 0;
+            logicalRect.Width = m_island.ActualSize().x;
+            logicalRect.Height = m_island.ActualSize().y;
+
+            // This will convert from the logical coordinate space of the ContentIsland to
+            // Win32 screen coordinates that are needed by Accessibility.
+            return m_island.CoordinateConverter().ConvertLocalToScreen(logicalRect);
+        }
+
+        // Else find the matching automation peer entry for the sending fragment.
+        auto iterator = std::find_if(
+            m_uia.AutomationPeers.begin(), m_uia.AutomationPeers.end(), [&sender](auto const& automationPeer)
+            {
+                return automationPeer.Match(sender);
+            });
+
+        if (m_uia.AutomationPeers.end() == iterator)
+        {
+            // Did not find any match for this automation provider.
+            return { 0, 0, 0, 0 };
+        }
+
+        auto const& visualPeer = iterator->GetVisual();
+        winrt::Rect logicalRect;
+        logicalRect.X = visualPeer.Offset().x;
+        logicalRect.Y = visualPeer.Offset().y;
+        logicalRect.Width = visualPeer.Size().x;
+        logicalRect.Height = visualPeer.Size().y;
+
+        // This will convert from the logical coordinate space of the ContentIsland to
+        // Win32 screen coordinates that are needed by Accessibility.
+        return m_island.CoordinateConverter().ConvertLocalToScreen(logicalRect);
+    }
+
+
+    winrt::com_ptr<IRawElementProviderFragment>
+    DrawingIsland::GetFragmentFromPoint(
+        _In_ double x,
+        _In_ double y) const
+    {
+        // UIA provides hit test points in screen space.
+        // Convert the point into a dummy empty rectangle to use with the coordinate converter.
+        winrt::Windows::Graphics::RectInt32 screenRect{ static_cast<int>(x + 0.5), static_cast<int>(y + 0.5), 0, 0 };
+        auto logicalRect = m_island.CoordinateConverter().ConvertScreenToLocal(screenRect);
+        float2 localPoint{ logicalRect.X, logicalRect.Y };
+        auto hitTestVisual = HitTestVisual(localPoint);
+
+        // Find the automation peer for the hit test visual if any.
+        if (nullptr != hitTestVisual)
+        {
+            auto iterator = std::find_if(
+                m_uia.AutomationPeers.begin(), m_uia.AutomationPeers.end(), [&hitTestVisual](auto const& automationPeer)
+                {
+                    return automationPeer.Match(hitTestVisual);
+                });
+
+            if (m_uia.AutomationPeers.end() != iterator)
+            {
+                // Return the automation provider if we found an automation peer for the hit test visual.
+                return iterator->GetAutomationProvider().as<IRawElementProviderFragment>();
+            }
+        }
+
+        return nullptr;
+    }
+
+
+    winrt::com_ptr<IRawElementProviderFragment>
+    DrawingIsland::GetFragmentInFocus() const
+    {
+        // Find the currently selected visual's automation peer.
+        auto iterator = std::find_if(
+            m_uia.AutomationPeers.begin(), m_uia.AutomationPeers.end(), [visual = m_items.SelectedVisual](auto const& automationPeer)
+            {
+                return automationPeer.Match(visual);
+            });
+
+        if (m_uia.AutomationPeers.end() != iterator)
+        {
+            // Return the automation provider if we found an automation peer for the selected visual.
+            return iterator->GetAutomationProvider().as<IRawElementProviderFragment>();
+        }
+
+        return nullptr;
+    }
+
+
     winrt::Visual
     DrawingIsland::HitTestVisual(
-        float2 const point)
+        float2 const& point) const
     {
         winrt::Visual selectedVisual{ nullptr };
         for (winrt::Visual visual : m_items.Visuals)
@@ -199,53 +302,51 @@ namespace winrt::DrawingIslandComponents::implementation
     void
     DrawingIsland::Accessibility_Initialize()
     {
-        m_uia.FragmentRoot = winrt::make_self<IslandFragmentRoot>(m_island);
+        // Create an UI automation fragment root for our island's content.
+        m_uia.FragmentRoot = winrt::make_self<IslandFragmentRoot>();
         m_uia.FragmentRoot->SetName(L"Drawing Squares");
-
-        m_uia.FragmentFactory = winrt::make_self<NodeSimpleFragmentFactory>();
+        m_uia.FragmentRoot->SetProviderOptions(
+            ProviderOptions_ServerSideProvider | ProviderOptions_UseComThreading | ProviderOptions_RefuseNonClientSupport);
+        m_uia.FragmentRoot->SetUiaControlTypeId(UIA_WindowControlTypeId);
+        m_uia.FragmentRoot->SetCallbackHandler(this);
 
         (void)m_island.AutomationProviderRequested(
             { this, &DrawingIsland::Accessibility_OnAutomationProviderRequested });
     }
 
 
     void
-    DrawingIsland::Accessibility_CreateItemFragment()
+    DrawingIsland::Accessibility_CreateItemFragment(
+        const winrt::Visual& itemVisual)
     {
-        // Create a new fragment for the new item.
-        winrt::com_ptr<NodeSimpleFragment> fragment = m_uia.FragmentFactory->Create(
-            s_colorNames[m_output.CurrentColorIndex].c_str(), m_uia.FragmentRoot);
+        // Create a new automation fragment.
+        auto fragment = winrt::make_self<NodeSimpleFragment>();
+        fragment->SetName(s_colorNames[m_output.CurrentColorIndex].c_str());
+        fragment->SetCallbackHandler(this);
 
-        // Store the mapping between the Visual and the Fragment:
+        // Add an entry to our automation peers which is a mapping between the Visual and the Fragment:
         // - This is keeping the UIA objects alive.
         // - Although not used yet, the lookup would be used to get back to the item Fragment for
         //   specific operations, such as hit-testing or tree walking.  This avoids adding to more
         //   expensive data storage, such as the Visual.CustomProperties map.
+        m_uia.AutomationPeers.push_back(AutomationPeer{ itemVisual, fragment });
 
-        m_uia.VisualToFragmentMap[m_items.SelectedVisual] = fragment;
-
-        // Give the fragment a backpointer to the Visual to get real-time information.
-        fragment->SetVisual(m_items.SelectedVisual);
-
-        // Add the new fragment into the UIA tree.
-        m_uia.FragmentRoot->AddChild(fragment);
-
-        // Finally set up parent chain
-        fragment->SetParent(m_uia.FragmentRoot);
+        // Connect the automation fragment to our fragment root.
+        m_uia.FragmentRoot->AddChildToEnd(fragment);
     }
 
 
     void
     DrawingIsland::Accessibility_OnAutomationProviderRequested(
-        const winrt::ContentIsland& /*island*/,
+        const winrt::ContentIsland& island,
         const winrt::ContentIslandAutomationProviderRequestedEventArgs& args)
     {
-        IInspectable providerAsIInspectable;
-        m_uia.FragmentRoot->QueryInterface(
-            winrt::guid_of<IInspectable>(),
-            winrt::put_abi(providerAsIInspectable));
+        // Make sure the host provider is up to date.
+        m_uia.FragmentRoot->SetHostProvider(
+            island.GetAutomationHostProvider().as<IRawElementProviderSimple>());
 
-        args.AutomationProvider(std::move(providerAsIInspectable));
+        // Return the fragment root as an IInspectable.
+        args.AutomationProvider(m_uia.FragmentRoot.as<IInspectable>());
 
         args.Handled(true);
     }
@@ -286,8 +387,7 @@ namespace winrt::DrawingIslandComponents::implementation
                 Input_OnPointerReleased();
             });
 
-        // Set up the keyboard source. We store this in a member variable so we can easily call 
-        // TrySetFocus() in response to left clicks in the content later on.
+        // Set up the keyboard source.
         m_input.KeyboardSource = winrt::InputKeyboardSource::GetForIsland(m_island);
 
         m_input.KeyboardSource.KeyDown(
@@ -407,6 +507,11 @@ namespace winrt::DrawingIslandComponents::implementation
             case winrt::Windows::System::VirtualKey::Escape:
             {
                 m_items.Visuals.RemoveAll();
+
+                // Update accessibility.
+                m_uia.FragmentRoot->RemoveAllChildren();
+                m_uia.AutomationPeers.clear();
+
                 handled = true;
                 break;
             }
@@ -515,7 +620,19 @@ namespace winrt::DrawingIslandComponents::implementation
             m_items.Visuals.Remove(m_items.SelectedVisual);
             m_items.Visuals.InsertAtTop(m_items.SelectedVisual);
 
-            // TODO: The m_uia.FragmentRoots child should be removed and added to the end as well.
+            // Update automation.
+            // First find the existing automation peer.
+            auto iterator = std::find_if(
+                m_uia.AutomationPeers.begin(), m_uia.AutomationPeers.end(), [visual = m_items.SelectedVisual](auto const& automationPeer)
+                {
+                    return automationPeer.Match(visual);
+                });
+            if (m_uia.AutomationPeers.end() != iterator)
+            {
+                // Mirror the change to the visuals above.
+                m_uia.FragmentRoot->RemoveChild(iterator->GetAutomationProvider());
+                m_uia.FragmentRoot->AddChildToEnd(iterator->GetAutomationProvider());
+            }
         }
         else
         {
@@ -644,7 +761,7 @@ namespace winrt::DrawingIslandComponents::implementation
         m_items.Offset.x = -BlockSize / 2.0f;
         m_items.Offset.y = -BlockSize / 2.0f;
 
-        Accessibility_CreateItemFragment();
+        Accessibility_CreateItemFragment(visual);
     }
 
 

Samples/Islands/DrawingIsland/DrawingIslandComponents/DrawingIsland.h

@@ -4,14 +4,15 @@
 #pragma once
 
 #include "DrawingIsland.g.h"
-#include "NodeSimpleFragment.h"
+#include "IslandFragmentRoot.h"
 
 namespace winrt::DrawingIslandComponents::implementation
 {
     struct DrawingIsland :
         public DrawingIslandT<
         DrawingIsland,
-        Microsoft::UI::Input::IInputPreTranslateKeyboardSourceHandler>
+        Microsoft::UI::Input::IInputPreTranslateKeyboardSourceHandler>,
+        IAutomationCallbackHandler
     {
     public:
         DrawingIsland(
@@ -78,13 +79,24 @@ namespace winrt::DrawingIslandComponents::implementation
             UINT keyboardModifiers,
             _Inout_ bool* handled);
 
+        // IAutomationCallbackHandler overrides.
+        winrt::Windows::Graphics::RectInt32 GetScreenBoundsForAutomationFragment(
+            _In_::IUnknown const* const sender) const override;
+
+        winrt::com_ptr<IRawElementProviderFragment> GetFragmentFromPoint(
+            _In_ double x,
+            _In_ double y) const override;
+
+        winrt::com_ptr<IRawElementProviderFragment> GetFragmentInFocus() const override;
+
     private:
         winrt::Visual HitTestVisual(
-            float2 const point);
+            float2 const& point) const;
 
         void Accessibility_Initialize();
 
-        void Accessibility_CreateItemFragment();
+        void Accessibility_CreateItemFragment(
+            const winrt::Visual& itemVisual);
 
         void Accessibility_OnAutomationProviderRequested(
             const winrt::ContentIsland& island,
@@ -244,17 +256,50 @@ namespace winrt::DrawingIslandComponents::implementation
             winrt::ContentLayoutDirection LayoutDirection =
                 winrt::ContentLayoutDirection::LeftToRight;
         } m_prevState;
+            
+        struct AutomationPeer
+        {
+            explicit AutomationPeer(
+                winrt::Visual const& visual,
+                winrt::com_ptr<NodeSimpleFragment> const& automationProvider) :
+                _visual{ visual },
+                _automationProvider{ automationProvider }
+            {
+
+            }
+
+            winrt::Visual const& GetVisual() const
+            {
+                return _visual;
+            }
+
+            winrt::com_ptr<NodeSimpleFragment> const& GetAutomationProvider() const
+            {
+                return _automationProvider;
+            }
+
+            bool Match(winrt::Visual const& visual) const noexcept
+            {
+                return visual == _visual;
+            }
+
+            bool Match(::IUnknown const* const automationProviderAsIUnknown) const noexcept
+            {
+                return automationProviderAsIUnknown == _automationProvider.try_as<::IUnknown>().get();
+            }
+
+        private:
+            winrt::Visual _visual{ nullptr };
+            winrt::com_ptr<NodeSimpleFragment> _automationProvider{ nullptr };
+        };
 
         struct
         {
-            // Our UIA object to create fragments
-            winrt::com_ptr<NodeSimpleFragmentFactory> FragmentFactory{ nullptr };
-
             // The UIA parent Root for this Island that contains the fragment children.
             winrt::com_ptr<IslandFragmentRoot> FragmentRoot{ nullptr };
 
-            // Map a given square (Visual) to its UIA fragment object
-            std::map<winrt::Visual, winrt::com_ptr<NodeSimpleFragment>> VisualToFragmentMap;
+            // Map a given square (Visual) to its UIA fragment object.
+            std::vector<AutomationPeer> AutomationPeers{};
         } m_uia;
     };
 }