Reduce flakiness by improving some checks to ensure the app is ready. (#61)

Changes:
1. Ensures the runloop is fully flushed before continuing
2. Force enables the accessibility engine and waits for it to load if it
is the first time loading on a device
3. Improve detection for view did appear to hopefully detect when the
window's root view controller has completed appearance better and reduce
issues from slow sim boot
4. Check for unfinished animations with the assumption that we can use
those for transitions.
5. Adds a new settings object to be able to configure some of these
options

Author: gabriellanata

HammerTests.podspec

@@ -1,6 +1,6 @@
 Pod::Spec.new do |spec|
   spec.name          = "HammerTests"
-  spec.version       = "0.16.0"
+  spec.version       = "0.17.0"
   spec.summary       = "iOS touch and keyboard synthesis library for unit tests."
   spec.description   = "Hammer is a touch and keyboard synthesis library for emulating user interaction events. It enables new ways of triggering UI actions in unit tests, replicating a real world environment as much as possible."
   spec.homepage      = "https://github.com/lyft/Hammer"

Sources/Hammer/EventGenerator/EventGenerator+Settings.swift

@@ -0,0 +1,22 @@
+import Foundation
+
+extension EventGenerator {
+    /// Shared setting values for all event generators
+    public static var settings = Settings()
+
+    /// Shared setting values for all event generators
+    public struct Settings {
+        /// The delay to wait after activating the accessibility engine.
+        public var accessibilityActivateDelay: TimeInterval = 0.02
+
+        /// The delay to wait after activating the accessibility engine for the first time in a simulator.
+        public var accessibilityActivateFirstTimeDelay: TimeInterval = 5.0
+
+        /// The accessibility engine is required for finding accessibility labels. We proactively enable it
+        /// to avoid issues with the first test case that uses it.
+        public var forceActivateAccessibilityEngine: Bool = true
+
+        /// If we should wait for animations to complete when an event generator is created.
+        public var waitForAnimations: Bool = false
+    }
+}

Sources/Hammer/EventGenerator/EventGenerator.swift

@@ -114,11 +114,25 @@ public final class EventGenerator {
     public func waitUntilWindowIsReady(timeout: TimeInterval = 3) throws {
         do {
             try self.waitUntil(self.isWindowReady, timeout: timeout)
+            try self.waitUntilAccessibilityActivate()
+
+            if EventGenerator.settings.waitForAnimations {
+                try self.waitUntilAnimationsAreFinished(timeout: timeout)
+            }
+
+            try self.waitUntilRunloopIsFlushed(timeout: timeout)
         } catch {
             throw HammerError.windowIsNotReadyForInteraction
         }
     }
 
+    /// Waits until animations are finished.
+    ///
+    /// - parameter timeout: The maximum time to wait for the window to be ready.
+    public func waitUntilAnimationsAreFinished(timeout: TimeInterval) throws {
+        try self.waitUntil(!self.hasRunningAnimations, timeout: timeout)
+    }
+
     /// Returns if the window is ready to receive user interaction events
     public var isWindowReady: Bool {
         guard !(UIApplication.shared as UIApplicationDeprecated).isIgnoringInteractionEvents
@@ -139,9 +153,39 @@ public final class EventGenerator {
             }
         }
 
+        if let hammerWindow = self.window as? HammerWindow, !hammerWindow.viewControllerHasAppeared {
+            return false
+        }
+
         return true
     }
 
+    // Returns if the view or any of its subviews has running animations.
+    public var hasRunningAnimations: Bool {
+        // Recursive
+        func hasRunningAnimations(currentView: UIView) -> Bool {
+            // If the view is not visible, we do not need to consider it as running animation
+            guard self.viewIsVisible(currentView) else {
+                return false
+            }
+
+            // If there are animations running on the layer, return true
+            if currentView.layer.animationKeys()?.isEmpty == false {
+                return true
+            }
+
+            // Special case for parallax dimming view which happens during some animations
+            if String(describing: type(of: currentView)) == "_UIParallaxDimmingView" {
+                return true
+            }
+
+            // Traverse subviews
+            return currentView.subviews.contains { hasRunningAnimations(currentView: $0) }
+        }
+
+        return hasRunningAnimations(currentView: self.window)
+    }
+
     /// Gets the next event ID to use. Event IDs are global and sequential.
     ///
     /// - returns: The next event ID.
@@ -180,6 +224,42 @@ public final class EventGenerator {
         try self.sendMarkerEvent { try? waiter.complete() }
         try waiter.start()
     }
+
+    // MARK: - Accessibility initialization
+
+    private var isAccessibilityActivated = false
+
+    private func waitUntilAccessibilityActivate() throws {
+        guard EventGenerator.settings.forceActivateAccessibilityEngine else {
+            return
+        }
+
+        UIApplication.shared.accessibilityActivate()
+        if self.isAccessibilityActivated {
+            return
+        }
+
+        // The first time the accessibility engine is activated in a simulator it needs more time to warm up
+        // and start producing consistent results, after that only a short delay per test case is enough
+        let simAccessibilityActivatedKey = "accessibility_activated"
+        let simAccessibilityActivated = UserDefaults.standard.bool(forKey: simAccessibilityActivatedKey)
+        if !simAccessibilityActivated {
+            print("Activating accessibility engine for the first time in this simulator and waiting 5s")
+        } else {
+            print("Activating accessibility engine and waiting 0.1s")
+        }
+
+        try self.wait(
+            simAccessibilityActivated
+            ? EventGenerator.settings.accessibilityActivateDelay // Default: 0.02s
+            : EventGenerator.settings.accessibilityActivateFirstTimeDelay // Default: 5.0s
+        )
+
+        self.isAccessibilityActivated = true
+        if !simAccessibilityActivated {
+            UserDefaults.standard.set(true, forKey: simAccessibilityActivatedKey)
+        }
+    }
 }
 
 // Bypasses deprecation warning for `isIgnoringInteractionEvents`

Sources/Hammer/Utilties/HammerWindow.swift

@@ -8,6 +8,10 @@ final class HammerWindow: UIWindow {
         return .zero
     }
 
+    var viewControllerHasAppeared: Bool {
+        return self.hammerViewController.hasAppeared
+    }
+
     init() {
         super.init(frame: UIScreen.main.bounds)
         self.rootViewController = self.hammerViewController
@@ -41,6 +45,8 @@ private final class HammerViewController: UIViewController {
     override var shouldAutomaticallyForwardAppearanceMethods: Bool { false }
     override var prefersStatusBarHidden: Bool { true }
 
+    var hasAppeared = false
+
     override func viewDidLoad() {
         super.viewDidLoad()
         self.view.backgroundColor = .clear
@@ -56,6 +62,16 @@ private final class HammerViewController: UIViewController {
         ])
     }
 
+    override func viewDidAppear(_ animated: Bool) {
+        super.viewDidAppear(animated)
+        self.hasAppeared = true
+    }
+
+    override func viewWillDisappear(_ animated: Bool) {
+        super.viewWillDisappear(animated)
+        self.hasAppeared = false
+    }
+
     func presentContained(_ viewController: UIViewController) {
         viewController.beginAppearanceTransition(true, animated: false)
         self.addChild(viewController)

Sources/Hammer/Utilties/Subviews.swift

@@ -154,31 +154,7 @@ extension EventGenerator {
     ///
     /// - returns: If the view is visible
     public func viewIsVisible(_ view: UIView, visibility: Visibility = .partial) -> Bool {
-        guard view.isDescendant(of: self.window) else {
-            return false
-        }
-
-        // Recursive
-        func viewIsVisible(currentView: UIView) -> Bool {
-            guard !currentView.isHidden && currentView.alpha >= 0.01 else {
-                return false
-            }
-
-            guard let superview = currentView.superview else {
-                return currentView == self.window
-            }
-
-            if superview.clipsToBounds {
-                let adjustedBounds = view.convert(view.bounds, to: superview)
-                guard superview.bounds.isVisible(adjustedBounds, visibility: visibility) else {
-                    return false
-                }
-            }
-
-            return viewIsVisible(currentView: superview)
-        }
-
-        return viewIsVisible(currentView: view)
+        return view.isVisible(inWindow: self.window, visibility: visibility)
     }
 
     /// Returns if the specified rect is visible.

Sources/Hammer/Utilties/UIKit+Extensions.swift

@@ -75,4 +75,38 @@ extension UIView {
     var topLevelView: UIView {
         return self.superview?.topLevelView ?? self
     }
+
+    /// Returns if the view is visible.
+    ///
+    /// - parameter window:     The window to check if the view is part of.
+    /// - parameter visibility: How determine if the view is visible.
+    ///
+    /// - returns: If the view is visible
+    func isVisible(inWindow window: UIWindow, visibility: EventGenerator.Visibility = .partial) -> Bool {
+        guard self.isDescendant(of: window) else {
+            return false
+        }
+
+        // Recursive
+        func isVisible(currentView: UIView) -> Bool {
+            guard !currentView.isHidden && currentView.alpha >= 0.01 else {
+                return false
+            }
+
+            guard let superview = currentView.superview else {
+                return currentView == window
+            }
+
+            if superview.clipsToBounds {
+                let adjustedBounds = self.convert(self.bounds, to: superview)
+                guard superview.bounds.isVisible(adjustedBounds, visibility: visibility) else {
+                    return false
+                }
+            }
+
+            return isVisible(currentView: superview)
+        }
+
+        return isVisible(currentView: self)
+    }
 }

Sources/Hammer/Utilties/Waiting.swift

@@ -239,4 +239,29 @@ extension EventGenerator {
         try self.waitUntil(self.viewIsHittable(self.mainView),
                            timeout: timeout, checkInterval: checkInterval)
     }
+
+    // MARK: - System waiting
+
+    /// Waits for the main runloop is flushed and all scheduled tasks have executed.
+    ///
+    /// - parameter timeout: The maximum time to wait.
+    ///
+    /// - throws: An error if the runloop is not flushed within the specified time.
+    public func waitUntilRunloopIsFlushed(timeout: TimeInterval) throws {
+        var errorCompleting: Error?
+
+        let waiter = Waiter(timeout: timeout)
+        DispatchQueue.main.async {
+            do {
+                try waiter.complete()
+            } catch {
+                errorCompleting = error
+            }
+        }
+
+        try waiter.start()
+        if let errorCompleting {
+            throw errorCompleting
+        }
+    }
 }