Skip to content

Updated Sort Order #244

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
RoyalPineapple merged 3 commits into from
Jun 24, 2025
Merged

Updated Sort Order #244

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c1d1d4a
updating sort order to take the userinterfaceidiom into account
RoyalPineapple May 27, 2025
cda21b4
updating tests
RoyalPineapple May 27, 2025
89ec76b
Update value to 13.0
RoyalPineapple May 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
102 changes: 92 additions & 10 deletions Example/UnitTests/AccessibilityHierarchyParserTests.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,49 +19,125 @@ import UIKit
import XCTest

final class AccessibilityHierarchyParserTests: XCTestCase {

func testUserInterfaceLayoutDirection() {
let gridView = UIView(frame: .init(x: 0, y: 0, width: 20, height: 20))

let elementA = UIView(frame: .init(x: 0, y: 0, width: 10, height: 10))
elementA.isAccessibilityElement = true
elementA.accessibilityLabel = "A"
elementA.accessibilityFrame = elementA.frame
gridView.addSubview(elementA)

let elementB = UIView(frame: .init(x: 10, y: 0, width: 10, height: 10))
elementB.isAccessibilityElement = true
elementB.accessibilityLabel = "B"
elementB.accessibilityFrame = elementB.frame
gridView.addSubview(elementB)

let elementC = UIView(frame: .init(x: 0, y: 10, width: 10, height: 10))
elementC.isAccessibilityElement = true
elementC.accessibilityLabel = "C"
elementC.accessibilityFrame = elementC.frame
gridView.addSubview(elementC)

let elementD = UIView(frame: .init(x: 10, y: 10, width: 10, height: 10))
elementD.isAccessibilityElement = true
elementD.accessibilityLabel = "D"
elementD.accessibilityFrame = elementD.frame
gridView.addSubview(elementD)

let parser = AccessibilityHierarchyParser()

let ltrElements = parser.parseAccessibilityElements(
in: gridView,
userInterfaceLayoutDirectionProvider: TestUserInterfaceLayoutDirectionProvider(userInterfaceLayoutDirection: .leftToRight)
userInterfaceLayoutDirectionProvider: TestUserInterfaceLayoutDirectionProvider(userInterfaceLayoutDirection: .leftToRight),
userInterfaceIdiomProvider: TestUserInterfaceIdiomProvider(userInterfaceIdiom: .phone)
).map { $0.description }
XCTAssertEqual(ltrElements, ["A", "B", "C", "D"])

let rtlElements = parser.parseAccessibilityElements(
in: gridView,
userInterfaceLayoutDirectionProvider: TestUserInterfaceLayoutDirectionProvider(userInterfaceLayoutDirection: .rightToLeft)
userInterfaceLayoutDirectionProvider: TestUserInterfaceLayoutDirectionProvider(userInterfaceLayoutDirection: .rightToLeft),
userInterfaceIdiomProvider: TestUserInterfaceIdiomProvider(userInterfaceIdiom: .phone)
).map { $0.description }
XCTAssertEqual(rtlElements, ["B", "A", "D", "C"])
}




func testVerticalSeperation() {
let magicNumber = 8.0 // This is enough to trigger vertical separation for phone but not for pad

let gridView = UIView(frame: .init(x: 0, y: 0, width: 100, height: 20))

let elementA = UIView(frame: .init(x: 0, y: magicNumber, width: 10, height: 10))
elementA.isAccessibilityElement = true
elementA.accessibilityLabel = "A"
elementA.accessibilityFrame = elementA.frame
gridView.addSubview(elementA)

let elementB = UIView(frame: .init(x: 10, y: 0, width: 0, height: 10))
elementB.isAccessibilityElement = true
elementB.accessibilityLabel = "B"
elementB.accessibilityFrame = elementB.frame
gridView.addSubview(elementB)

let elementC = UIView(frame: .init(x: 20, y: -(magicNumber), width: 10, height: 10))
elementC.isAccessibilityElement = true
elementC.accessibilityLabel = "C"
elementC.accessibilityFrame = elementC.frame
gridView.addSubview(elementC)

let elementD = UIView(frame: .init(x: 30, y: -(magicNumber), width: 10, height: 10))
elementD.isAccessibilityElement = true
elementD.accessibilityLabel = "D"
elementD.accessibilityFrame = elementD.frame
gridView.addSubview(elementD)

let parser = AccessibilityHierarchyParser()

let padElements = parser.parseAccessibilityElements(
in: gridView,
userInterfaceLayoutDirectionProvider:
TestUserInterfaceLayoutDirectionProvider(userInterfaceLayoutDirection: .leftToRight),
userInterfaceIdiomProvider: TestUserInterfaceIdiomProvider(userInterfaceIdiom: .pad)
).map { $0.description }
// on pad elements are sorted horizontally
XCTAssertEqual(padElements, ["A", "B", "C", "D"])

let phoneElements = parser.parseAccessibilityElements(
in: gridView,
userInterfaceLayoutDirectionProvider:
TestUserInterfaceLayoutDirectionProvider(userInterfaceLayoutDirection: .leftToRight),
userInterfaceIdiomProvider: TestUserInterfaceIdiomProvider(userInterfaceIdiom: .phone)
).map { $0.description }
// on phone elements are sorted vertically and then left to right
XCTAssertEqual(phoneElements, ["C", "D", "B", "A"])


let padMagicNumber = 25

elementA.accessibilityFrame = .init(x: 0, y: padMagicNumber, width: 10, height: 10)
elementB.accessibilityFrame = .init(x: 10, y: 0, width: 0, height: 10)
elementC.accessibilityFrame = .init(x: 20, y: -(padMagicNumber), width: 10, height: 10)
elementD.accessibilityFrame = .init(x: 30, y: -(padMagicNumber), width: 10, height: 10)



let padAgain = parser.parseAccessibilityElements(
in: gridView,
userInterfaceLayoutDirectionProvider:
TestUserInterfaceLayoutDirectionProvider(userInterfaceLayoutDirection: .leftToRight),
userInterfaceIdiomProvider: TestUserInterfaceIdiomProvider(userInterfaceIdiom: .pad)
).map { $0.description }


// Now pad elements are sorted vertically and then left to right
XCTAssertEqual(padAgain, ["C", "D", "B", "A"])

}
}

// MARK: -
Expand All @@ -71,3 +147,9 @@ private struct TestUserInterfaceLayoutDirectionProvider: UserInterfaceLayoutDire
var userInterfaceLayoutDirection: UIUserInterfaceLayoutDirection

}

private struct TestUserInterfaceIdiomProvider: UserInterfaceIdiomProviding {

var userInterfaceIdiom: UIUserInterfaceIdiom

}
42 changes: 30 additions & 12 deletions Sources/AccessibilitySnapshot/Core/Swift/Classes/AccessibilityHierarchyParser.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,9 +79,18 @@ public protocol UserInterfaceLayoutDirectionProviding {
var userInterfaceLayoutDirection: UIUserInterfaceLayoutDirection { get }

}

extension UIApplication: UserInterfaceLayoutDirectionProviding {}


public protocol UserInterfaceIdiomProviding {

var userInterfaceIdiom: UIUserInterfaceIdiom { get }

}

extension UIDevice: UserInterfaceIdiomProviding {}


// MARK: -

public final class AccessibilityHierarchyParser {
Expand Down Expand Up @@ -171,17 +180,20 @@ public final class AccessibilityHierarchyParser {
/// In most cases, this should use the default value, `UIApplication.shared`.
public func parseAccessibilityElements(
in root: UIView,
userInterfaceLayoutDirectionProvider: UserInterfaceLayoutDirectionProviding = UIApplication.shared
userInterfaceLayoutDirectionProvider: UserInterfaceLayoutDirectionProviding = UIApplication.shared,
userInterfaceIdiomProvider: UserInterfaceIdiomProviding = UIDevice.current
Copy link
Copy Markdown

@kyleve kyleve May 28, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Noting that in the Market snapshot tests we will want to inject a different UserInterfaceIdiomProviding, since they're kinda magic: They can run on any 3x iOS device and do swizzling and such to simulate all different device sizes, safe areas, etc, that we need.

) -> [AccessibilityMarker] {
let userInterfaceLayoutDirection = userInterfaceLayoutDirectionProvider.userInterfaceLayoutDirection

let userInterfaceIdiom = userInterfaceIdiomProvider.userInterfaceIdiom

let accessibilityNodes = root.recursiveAccessibilityHierarchy()

let uncontextualizedElements = sortedElements(
for: accessibilityNodes,
explicitlyOrdered: false,
in: root,
userInterfaceLayoutDirection: userInterfaceLayoutDirection
userInterfaceLayoutDirection: userInterfaceLayoutDirection,
userInterfaceIdiom: userInterfaceIdiom
)

let accessibilityElements = uncontextualizedElements.map { element in
Expand All @@ -190,7 +202,8 @@ public final class AccessibilityHierarchyParser {
context: context(
for: element.object,
from: element.contextProvider,
userInterfaceLayoutDirection: userInterfaceLayoutDirection
userInterfaceLayoutDirection: userInterfaceLayoutDirection,
userInterfaceIdiom: userInterfaceIdiom
)
)
}
Expand Down Expand Up @@ -271,11 +284,13 @@ public final class AccessibilityHierarchyParser {
/// this should typically be `false`.
/// - parameter root: The root view to which the nodes' shapes are relative.
/// - parameter userInterfaceLayoutDirection: The device's current user interface layout direction.
/// - parameter userInterfaceIdiom: the device's interface idiom, used to calculate the sort order
private func sortedElements(
for nodes: [AccessibilityNode],
explicitlyOrdered: Bool,
in root: UIView,
userInterfaceLayoutDirection: UIUserInterfaceLayoutDirection
userInterfaceLayoutDirection: UIUserInterfaceLayoutDirection,
userInterfaceIdiom: UIUserInterfaceIdiom = UIDevice.current.userInterfaceIdiom
) -> [Element] {
// VoiceOver flick navigation iterates through elements in a horizontal, then vertical order. The horizontal
// ordering matches the application's user interface layout direction. The vertical ordering is always
Expand All @@ -300,9 +315,9 @@ public final class AccessibilityHierarchyParser {
fatalError("Unknown user interface layout direction: \(userInterfaceLayoutDirection)")
}

// 8 seems to be the magic number for VoiceOver to consider it
// to be vertically "above" other views.
let minimumVerticalSeparation = 8.0
// Derived via experimentation, these magic numbers are the cutoff for VoiceOver to consider
// an element to be vertically "above" other views.
let minimumVerticalSeparation = userInterfaceIdiom == .phone ? 8.0 : 13.0
Copy link
Copy Markdown

@kyleve kyleve May 28, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not for this PR, but I am wondering if Voiceover is also a bit more clever when seeing a hierarchy like this:

┌──────────────────────────────────┐
│                                  │
│             ┌───────┐            │
│  ┌───────┐  │2      │ ┌───────┐  │
│  │1      │  │       │ │3      │  │
│  └───────┘  │       │ │       │  │
│             └───────┘ │       │  │
│                       └───────┘  │
│                                  │
│                                  │
│  ┌───────┐                       │
│  │4      │  ┌───────┐            │
│  └───────┘  │5      │ ┌───────┐  │
│             │       │ │6      │  │
│             │       │ └───────┘  │
│             └───────┘            │
│                                  │
└──────────────────────────────────┘

Eg does it try to also group content into "rows" (eg, 1, 2, 3 would be in one group, 4, 5, 6) in another, and then orders the items in the groups LTR/RTL based on layout direction, rather than purely by vertical separation? That would make a bit more sense to me.

Copy link
Copy Markdown
Collaborator

@meherkasam meherkasam Jul 8, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I thought I responded to this comment, but maybe not? I set up a test project and tested out some scenarios similar to the one you described. In all cases, it seems like VoiceOver is simply doing it based on the leading-to-trailing, top to bottom (within a tolerance) regardless of the layout of the elements.


let sortedNodes = explicitlyOrdered ? nodes : nodes
.map { ($0, accessibilitySortFrame(for: $0, in: root)) }
Expand Down Expand Up @@ -331,7 +346,8 @@ public final class AccessibilityHierarchyParser {
for: elements,
explicitlyOrdered: explicitlyOrdered,
in: root,
userInterfaceLayoutDirection: userInterfaceLayoutDirection
userInterfaceLayoutDirection: userInterfaceLayoutDirection,
userInterfaceIdiom: userInterfaceIdiom
)
)
}
Expand Down Expand Up @@ -391,7 +407,8 @@ public final class AccessibilityHierarchyParser {
private func context(
for element: NSObject,
from contextProvider: ContextProvider?,
userInterfaceLayoutDirection: UIUserInterfaceLayoutDirection
userInterfaceLayoutDirection: UIUserInterfaceLayoutDirection,
userInterfaceIdiom: UIUserInterfaceIdiom
) -> Context? {
guard let contextProvider = contextProvider else {
return nil
Expand Down Expand Up @@ -436,7 +453,8 @@ public final class AccessibilityHierarchyParser {
for: hierarchy,
explicitlyOrdered: false,
in: view,
userInterfaceLayoutDirection: userInterfaceLayoutDirection
userInterfaceLayoutDirection: userInterfaceLayoutDirection,
userInterfaceIdiom: userInterfaceIdiom
).map { $0.object }
viewToElementsMap[view] = accessibleElements
}
Expand Down