Merge pull request #7551 from woocommerce/try/prologue-cta-order-ab-test

Swap the order of prologue CTAs as an A/B experiment

Author: jaclync

Experiments/Experiments/ABTest.swift

@@ -20,6 +20,10 @@ public enum ABTest: String, CaseIterable {
     /// Experiment ref: pbxNRc-1Pp-p2
     case linkedProductsPromo = "woocommerceios_product_details_linked_products_banner"
 
+    /// A/B test for the login button order on the prologues screen.
+    /// Experiment ref: pbxNRc-1VA-p2
+    case loginPrologueButtonOrder = "woocommerceios_login_prologue_button_order"
+
     /// Returns a variation for the given experiment
     public var variation: Variation {
         ExPlat.shared?.experiment(rawValue) ?? .control
@@ -29,14 +33,30 @@ public enum ABTest: String, CaseIterable {
 public extension ABTest {
     /// Start the AB Testing platform if any experiment exists
     ///
-    static func start() {
-        guard ABTest.allCases.count > 1 else {
-            return
-        }
-
-        let experimentNames = ABTest.allCases.filter { $0 != .null }.map { $0.rawValue }
-        ExPlat.shared?.register(experiments: experimentNames)
+    static func start() async {
+        await withCheckedContinuation { continuation in
+            guard ABTest.allCases.count > 1 else {
+                return continuation.resume(returning: ())
+            }
+
+            let experimentNames = ABTest.allCases.filter { $0 != .null }.map { $0.rawValue }
+            ExPlat.shared?.register(experiments: experimentNames)
+
+            ExPlat.shared?.refresh {
+                continuation.resume(returning: ())
+            }
+        } as Void
+    }
+}
 
-        ExPlat.shared?.refresh()
+public extension Variation {
+    /// Used in an analytics event property value.
+    var analyticsValue: String {
+        switch self {
+        case .control:
+            return "control"
+        case .treatment(let string):
+            return string.map { "treatment: \($0)" } ?? "treatment"
+        }
     }
 }

WooCommerce/Classes/AppDelegate.swift

@@ -45,26 +45,20 @@ class AppDelegate: UIResponder, UIApplicationDelegate {
         appCoordinator?.tabBarController
     }
 
-    /// Checks on whether the Apple ID credential is valid when the app is logged in and becomes active.
-    ///
-    private lazy var appleIDCredentialChecker = AppleIDCredentialChecker()
-
     private let universalLinkRouter = UniversalLinkRouter.defaultUniversalLinkRouter()
 
     // MARK: - AppDelegate Methods
 
     func application(_ application: UIApplication, willFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]? = nil) -> Bool {
         // Setup Components
         setupAnalytics()
-        setupAuthenticationManager()
         setupCocoaLumberjack()
         setupLogLevel(.verbose)
         setupPushNotificationsManagerIfPossible()
         setupAppRatingManager()
         setupWormholy()
         setupKeyboardStateProvider()
         handleLaunchArguments()
-        appleIDCredentialChecker.observeLoggedInStateForAppleIDObservations()
         setupUserNotificationCenter()
 
         // Components that require prior Auth
@@ -78,19 +72,20 @@ class AppDelegate: UIResponder, UIApplicationDelegate {
         // ever new source code is injected into our application.
         Inject.animation = .interactiveSpring()
 
+        Task { @MainActor in
+            await startABTesting()
+
+            // Upgrade check...
+            // This has to be called after A/B testing setup in `startABTesting` if any of the Tracks events
+            // in `checkForUpgrades` is used as an exposure event for an experiment.
+            // For example, `application_installed` could be the exposure event for logged-out experiments.
+            checkForUpgrades()
+        }
+
         return true
     }
 
     func application(_ application: UIApplication, didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]?) -> Bool {
-
-        startABTesting()
-
-        // Upgrade check...
-        // This has to be called after A/B testing setup in `startABTesting` if any of the Tracks events
-        // in `checkForUpgrades` is used as an exposure event for an experiment.
-        // For example, `application_installed` could be the exposure event for logged-out experiments.
-        checkForUpgrades()
-
         // Setup the Interface!
         setupMainWindow()
         setupComponentsAppearance()
@@ -267,12 +262,6 @@ private extension AppDelegate {
         ServiceLocator.analytics.initialize()
     }
 
-    /// Sets up the WordPress Authenticator.
-    ///
-    func setupAuthenticationManager() {
-        ServiceLocator.authenticationManager.initialize()
-    }
-
     /// Sets up CocoaLumberjack logging.
     ///
     func setupCocoaLumberjack() {
@@ -382,8 +371,8 @@ private extension AppDelegate {
 
     /// Starts the AB testing platform
     ///
-    func startABTesting() {
-        ABTest.start()
+    func startABTesting() async {
+        await ABTest.start()
     }
 }
 
@@ -397,7 +386,9 @@ private extension AppDelegate {
         let versionOfLastRun = UserDefaults.standard[.versionOfLastRun] as? String
         if versionOfLastRun == nil {
             // First run after a fresh install
-            ServiceLocator.analytics.track(.applicationInstalled)
+            ServiceLocator.analytics.track(.applicationInstalled,
+                                           withProperties: ["after_abtest_setup": true,
+                                                            "prologue_experiment_variant": ABTest.loginPrologueButtonOrder.variation.analyticsValue])
         } else if versionOfLastRun != currentVersion {
             // App was upgraded
             ServiceLocator.analytics.track(.applicationUpgraded, withProperties: ["previous_version": versionOfLastRun ?? String()])

WooCommerce/Classes/Authentication/AuthenticationManager.swift

@@ -4,6 +4,7 @@ import WordPressAuthenticator
 import WordPressKit
 import Yosemite
 import class Networking.UserAgent
+import enum Experiments.ABTest
 import struct Networking.Settings
 import protocol Storage.StorageManagerType
 
@@ -45,6 +46,7 @@ class AuthenticationManager: Authentication {
     func initialize() {
         let isWPComMagicLinkPreferredToPassword = ServiceLocator.featureFlagService.isFeatureFlagEnabled(.loginMagicLinkEmphasis)
         let isWPComMagicLinkShownAsSecondaryActionOnPasswordScreen = ServiceLocator.featureFlagService.isFeatureFlagEnabled(.loginMagicLinkEmphasisM2)
+        let continueWithSiteAddressFirst = ABTest.loginPrologueButtonOrder.variation == .control
         let configuration = WordPressAuthenticatorConfiguration(wpcomClientId: ApiCredentials.dotcomAppId,
                                                                 wpcomSecret: ApiCredentials.dotcomSecret,
                                                                 wpcomScheme: ApiCredentials.dotcomAuthScheme,
@@ -60,7 +62,7 @@ class AuthenticationManager: Authentication {
                                                                 enableSignInWithApple: true,
                                                                 enableSignupWithGoogle: false,
                                                                 enableUnifiedAuth: true,
-                                                                continueWithSiteAddressFirst: true,
+                                                                continueWithSiteAddressFirst: continueWithSiteAddressFirst,
                                                                 enableSiteCredentialsLoginForSelfHostedSites: true,
                                                                 isWPComLoginRequiredForSiteCredentialsLogin: true,
                                                                 isWPComMagicLinkPreferredToPassword: isWPComMagicLinkPreferredToPassword,

WooCommerce/Classes/Authentication/Epilogue/StorePickerViewController.swift

@@ -548,7 +548,9 @@ private extension StorePickerViewController {
         guard ServiceLocator.stores.isAuthenticated else {
             return
         }
-        ABTest.start()
+        Task { @MainActor in
+            await ABTest.start()
+        }
     }
 }
 

WooCommerce/Classes/ViewRelated/AppCoordinator.swift

@@ -27,6 +27,10 @@ final class AppCoordinator {
     private var localNotificationResponsesSubscription: AnyCancellable?
     private var isLoggedIn: Bool = false
 
+    /// Checks on whether the Apple ID credential is valid when the app is logged in and becomes active.
+    ///
+    private lazy var appleIDCredentialChecker = AppleIDCredentialChecker()
+
     init(window: UIWindow,
          stores: StoresManager = ServiceLocator.stores,
          storageManager: StorageManagerType = ServiceLocator.storageManager,
@@ -64,11 +68,12 @@ final class AppCoordinator {
                 // More details about the UI states: https://github.com/woocommerce/woocommerce-ios/pull/3498
                 switch (isLoggedIn, needsDefaultStore) {
                 case (false, true), (false, false):
-                    self.displayAuthenticator()
+                    self.displayAuthenticatorWithOnboardingIfNeeded()
                 case (true, true):
                     self.displayLoggedInStateWithoutDefaultStore()
                 case (true, false):
                     self.validateRoleEligibility {
+                        self.configureAuthenticator()
                         self.displayLoggedInUI()
                         self.synchronizeAndShowWhatsNew()
                     }
@@ -121,34 +126,63 @@ private extension AppCoordinator {
 
     /// Displays the WordPress.com Authentication UI.
     ///
-    func displayAuthenticator() {
+    func displayAuthenticatorWithOnboardingIfNeeded() {
+        if canPresentLoginOnboarding() {
+            // Sets a placeholder view controller as the window's root view as it is required
+            // at the end of app launch.
+            setWindowRootViewControllerAndAnimateIfNeeded(.init())
+            presentLoginOnboarding { [weak self] in
+                guard let self = self else { return }
+                // Only displays the authenticator when dismissing onboarding to allow time for A/B test setup.
+                self.configureAndDisplayAuthenticator()
+            }
+        } else {
+            configureAndDisplayAuthenticator()
+        }
+    }
+
+    /// Configures the WPAuthenticator and sets the authenticator UI as the window's root view.
+    func configureAndDisplayAuthenticator() {
+        configureAuthenticator()
+
         let authenticationUI = authenticationManager.authenticationUI()
         setWindowRootViewControllerAndAnimateIfNeeded(authenticationUI) { [weak self] _ in
             guard let self = self else { return }
             self.tabBarController.removeViewControllers()
-
-            self.presentLoginOnboarding()
         }
-        ServiceLocator.analytics.track(.openedLogin)
+        ServiceLocator.analytics.track(.openedLogin, withProperties: ["prologue_experiment_variant": ABTest.loginPrologueButtonOrder.variation.analyticsValue])
     }
 
-    /// Presents onboarding on top of the authentication UI under certain criteria.
-    func presentLoginOnboarding() {
+    /// Configures the WPAuthenticator for usage in both logged-in and logged-out states.
+    func configureAuthenticator() {
+        authenticationManager.initialize()
+        appleIDCredentialChecker.observeLoggedInStateForAppleIDObservations()
+    }
+
+    /// Determines whether the login onboarding should be shown.
+    func canPresentLoginOnboarding() -> Bool {
         // Since we cannot control the user defaults in the simulator where UI tests are run on,
         // login onboarding is not shown in UI tests for now.
         // If we want to add UI tests for the login onboarding, we can add another launch argument
         // so that we can show/hide the onboarding screen consistently.
         let isUITesting: Bool = CommandLine.arguments.contains("-ui_testing")
         guard isUITesting == false else {
-            return
+            return false
         }
 
         guard featureFlagService.isFeatureFlagEnabled(.loginPrologueOnboarding),
-        loggedOutAppSettings.hasFinishedOnboarding == false else {
-            return
+              loggedOutAppSettings.hasFinishedOnboarding == false else {
+            return false
         }
+        return true
+    }
+
+    /// Presents onboarding on top of the authentication UI under certain criteria.
+    /// - Parameter onDismiss: invoked when the onboarding is dismissed.
+    func presentLoginOnboarding(onDismiss: @escaping () -> Void) {
         let onboardingViewController = LoginOnboardingViewController { [weak self] action in
             guard let self = self else { return }
+            onDismiss()
             self.loggedOutAppSettings.setHasFinishedOnboarding(true)
             self.window.rootViewController?.dismiss(animated: true)
 
@@ -182,6 +216,8 @@ private extension AppCoordinator {
             return
         }
 
+        configureAuthenticator()
+
         let matcher = ULAccountMatcher(storageManager: storageManager)
         matcher.refreshStoredSites()
 
@@ -209,7 +245,7 @@ private extension AppCoordinator {
         storePickerCoordinator?.onDismiss = { [weak self] in
             guard let self = self else { return }
             if self.isLoggedIn == false {
-                self.displayAuthenticator()
+                self.displayAuthenticatorWithOnboardingIfNeeded()
             }
         }
     }

WooCommerce/WooCommerceTests/AppCoordinatorTests.swift

@@ -222,7 +222,7 @@ final class AppCoordinatorTests: XCTestCase {
         appCoordinator.start()
 
         // Then
-        assertThat(window.rootViewController, isAnInstanceOf: LoginNavigationController.self)
+        assertThat(window.rootViewController, isAnInstanceOf: UIViewController.self)
         assertThat(window.rootViewController?.presentedViewController, isAnInstanceOf: LoginOnboardingViewController.self)
     }