chore(nimbus): Update External Configs

.circleci/config.yml

@@ -601,23 +601,21 @@ jobs:
       - run:
           name: Check for External Config Update
           command: |
-            git checkout main
-            git pull origin main
             cp .env.sample .env
             touch ./experimenter/fetch-summary.txt
             env GITHUB_BEARER_TOKEN="${GH_EXTERNAL_CONFIG_TOKEN}" make fetch_external_resources FETCH_ARGS="--summary fetch-summary.txt"
             mv ./experimenter/fetch-summary.txt /tmp/pr-body.txt
             echo -e "\nCircle CI Task: ${CIRCLE_BUILD_URL}" >> /tmp/pr-body.txt
             if python3 ./experimenter/bin/should-pr.py
               then
-                git checkout -B external-config
+                git checkout -B mibrahim-external-config-test
                 git add .
                 git commit -m 'chore(nimbus): Update External Configs'
-                if (($((git diff external-config origin/external-config || git diff HEAD~1) | wc -c) > 0))
+                if (($((git diff mibrahim-external-config-test origin/mibrahim-external-config-test || git diff HEAD~1) | wc -c) > 0))
                   then
-                    git push origin external-config -f
-                    gh pr create -t "chore(nimbus): Update External Configs" -F /tmp/pr-body.txt --base main --head external-config --repo mozilla/experimenter || \
-                      gh pr edit external-config -F /tmp/pr-body.txt
+                    git push origin mibrahim-external-config-test -f
+                    gh pr create -t "chore(nimbus): Update External Configs" -F /tmp/pr-body.txt --base main --head mibrahim-external-config-test --repo mozilla/experimenter || \
+                      gh pr edit mibrahim-external-config-test -F /tmp/pr-body.txt
                   else
                     echo "Changes already committed, skipping"
                 fi
@@ -886,3 +884,9 @@ workflows:
           filters:
             branches:
               only: main
+      - update_external_configs:
+          name: Update External Configs
+          filters:
+            branches:
+              ignore: 
+                - main

experimenter/experimenter/features/manifests/apps.yaml

@@ -18,6 +18,8 @@ fenix:
           - "main"
           - "beta"
           - "release"
+  targeting_files:
+    - "mobile/android/fenix/app/src/main/java/org/mozilla/fenix/experiments/RecordedNimbusContext.kt"
 
 firefox_ios:
   slug: "ios"
@@ -42,6 +44,8 @@ firefox_ios:
         branch_re: 'release/v(?P<major>\d+)(?:\.(?P<minor>\d+))?'
         tag_re: 'firefox-v(?P<major>\d+)\.(?P<minor>\d+)'
       - type: "branched"
+  targeting_files:
+    - "firefox-ios/Client/Experiments/RecordedNimbusContext.swift"
 
 monitor_cirrus:
   slug: "monitor-web"
@@ -85,6 +89,8 @@ firefox_desktop:
           - "esr115"
           - "esr128"
           - "esr140"
+  targeting_files:
+    - "toolkit/components/nimbus/lib/TargetingContextRecorder.sys.mjs"
 
 experimenter_cirrus:
   slug: "experimenter"

experimenter/experimenter/features/manifests/fenix/.ref-cache.yaml

@@ -1,3 +1,3 @@
 beta: 511b1cd099f9120d0753fcc110df4a355f7a88c3
-main: 403431c6882c8de419338a19469f862f22f8095b
+main: 6f66908ea0c0f0b51c134ff1e6646a0c7fb8e82b
 release: c7fa3c91990bac266ee99a9e31863c202469f369

experimenter/experimenter/features/manifests/fenix/RecordedNimbusContext.kt

@@ -0,0 +1,283 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package org.mozilla.fenix.experiments
+
+import android.content.Context
+import android.os.Build
+import androidx.annotation.VisibleForTesting
+import mozilla.components.support.locale.LocaleManager
+import mozilla.components.support.locale.LocaleManager.getSystemDefault
+import mozilla.components.support.utils.ext.packageManagerCompatHelper
+import org.json.JSONArray
+import org.json.JSONObject
+import org.mozilla.experiments.nimbus.NIMBUS_DATA_DIR
+import org.mozilla.experiments.nimbus.NimbusDeviceInfo
+import org.mozilla.experiments.nimbus.internal.JsonObject
+import org.mozilla.experiments.nimbus.internal.RecordedContext
+import org.mozilla.experiments.nimbus.internal.getCalculatedAttributes
+import org.mozilla.fenix.GleanMetrics.NimbusSystem
+import org.mozilla.fenix.GleanMetrics.Pings
+import org.mozilla.fenix.ext.settings
+import org.mozilla.fenix.home.pocket.ContentRecommendationsFeatureHelper
+import org.mozilla.fenix.termsofuse.experimentation.TermsOfUseAdvancedTargetingHelper
+import org.mozilla.fenix.termsofuse.experimentation.utils.DefaultTermsOfUseDataProvider
+import org.mozilla.fenix.utils.Settings
+import java.io.File
+
+/**
+ * The following constants are string constants of the keys that appear in the [EVENT_QUERIES] map.
+ */
+const val DAYS_OPENED_IN_LAST_28 = "days_opened_in_last_28"
+
+/**
+ * [EVENT_QUERIES] is a map of keys to Nimbus SDK EventStore queries.
+ */
+private val EVENT_QUERIES = mapOf(
+    DAYS_OPENED_IN_LAST_28 to "'events.app_opened'|eventCountNonZero('Days', 28, 0)",
+)
+
+/**
+ * The RecordedNimbusContext class inherits from an internal Nimbus interface that provides methods
+ * for obtaining a JSON value for the object and recording the object's value to Glean. Its JSON
+ * value is loaded into the Nimbus targeting context.
+ *
+ * The value recorded to Glean is used to automate population sizing. Any additions to this object
+ * require a new data review for the `nimbus_system.recorded_nimbus_context` metric.
+ */
+@Suppress("complexity:LongParameterList")
+class RecordedNimbusContext(
+    val isFirstRun: Boolean,
+    private val eventQueries: Map<String, String> = mapOf(),
+    private var eventQueryValues: Map<String, Double> = mapOf(),
+    val utmSource: String,
+    val utmMedium: String,
+    val utmCampaign: String,
+    val utmTerm: String,
+    val utmContent: String,
+    val androidSdkVersion: String = Build.VERSION.SDK_INT.toString(),
+    val appVersion: String?,
+    val locale: String,
+    val daysSinceInstall: Int?,
+    val daysSinceUpdate: Int?,
+    val language: String?,
+    val region: String?,
+    val deviceManufacturer: String = Build.MANUFACTURER,
+    val deviceModel: String = Build.MODEL,
+    val userAcceptedTou: Boolean,
+    val noShortcutsOrStoriesOptOuts: Boolean,
+    val addonIds: List<String>,
+    val touPoints: Int?,
+) : RecordedContext {
+    /**
+     * [getEventQueries] is called by the Nimbus SDK Rust code to retrieve the map of event
+     * queries. The are then executed against the Nimbus SDK's EventStore to retrieve their values.
+     *
+     * @return Map<String, String>
+     */
+    override fun getEventQueries(): Map<String, String> {
+        return eventQueries
+    }
+
+    /**
+     * [record] is called when experiment enrollments are evolved. It should apply the
+     * [RecordedNimbusContext]'s values to a [NimbusSystem.RecordedNimbusContextObject] instance,
+     * and use that instance to record the values to Glean.
+     */
+    override fun record() {
+        val eventQueryValuesObject = NimbusSystem.RecordedNimbusContextObjectItemEventQueryValuesObject(
+            daysOpenedInLast28 = eventQueryValues[DAYS_OPENED_IN_LAST_28]?.toInt(),
+        )
+        NimbusSystem.recordedNimbusContext.set(
+            NimbusSystem.RecordedNimbusContextObject(
+                isFirstRun = isFirstRun,
+                eventQueryValues = eventQueryValuesObject,
+                installReferrerResponseUtmSource = utmSource,
+                installReferrerResponseUtmMedium = utmMedium,
+                installReferrerResponseUtmCampaign = utmCampaign,
+                installReferrerResponseUtmTerm = utmTerm,
+                installReferrerResponseUtmContent = utmContent,
+                androidSdkVersion = androidSdkVersion,
+                appVersion = appVersion,
+                locale = locale,
+                daysSinceInstall = daysSinceInstall,
+                daysSinceUpdate = daysSinceUpdate,
+                language = language,
+                region = region,
+                deviceManufacturer = deviceManufacturer,
+                deviceModel = deviceModel,
+                userAcceptedTou = userAcceptedTou,
+                noShortcutsOrStoriesOptOuts = noShortcutsOrStoriesOptOuts,
+                addonIds = NimbusSystem.RecordedNimbusContextObjectAddonIds(addonIds.toMutableList()),
+                touPoints = touPoints,
+            ),
+        )
+        Pings.nimbus.submit()
+    }
+
+    /**
+     * [setEventQueryValues] is called by the Nimbus SDK Rust code after the event queries have been
+     * executed. The [eventQueryValues] should be written back to the Kotlin object.
+     *
+     * @param [eventQueryValues] The values for each query after they have been executed in the
+     * Nimbus SDK Rust environment.
+     */
+    override fun setEventQueryValues(eventQueryValues: Map<String, Double>) {
+        this.eventQueryValues = eventQueryValues
+    }
+
+    /**
+     * [toJson] is called by the Nimbus SDK Rust code after the event queries have been executed,
+     * and before experiment enrollments have been evolved. The value returned from this method
+     * will be applied directly to the Nimbus targeting context, and its keys/values take
+     * precedence over those in the main Nimbus targeting context.
+     *
+     * @return JSONObject
+     */
+    override fun toJson(): JsonObject {
+        val obj = JSONObject(
+            mapOf(
+                "is_first_run" to isFirstRun,
+                "events" to JSONObject(eventQueryValues),
+                "install_referrer_response_utm_source" to utmSource,
+                "install_referrer_response_utm_medium" to utmMedium,
+                "install_referrer_response_utm_campaign" to utmCampaign,
+                "install_referrer_response_utm_term" to utmTerm,
+                "install_referrer_response_utm_content" to utmContent,
+                "android_sdk_version" to androidSdkVersion,
+                "app_version" to appVersion,
+                "locale" to locale,
+                "days_since_install" to daysSinceInstall,
+                "days_since_update" to daysSinceUpdate,
+                "language" to language,
+                "region" to region,
+                "device_manufacturer" to deviceManufacturer,
+                "device_model" to deviceModel,
+                "user_accepted_tou" to userAcceptedTou,
+                "no_shortcuts_or_stories_opt_outs" to noShortcutsOrStoriesOptOuts,
+                "addon_ids" to JSONArray(addonIds),
+                "tou_points" to touPoints,
+            ),
+        )
+        return obj
+    }
+
+    /**
+     * Companion object for RecordedNimbusContext
+     */
+    companion object {
+
+        /**
+         * Creates a RecordedNimbusContext instance, populated with the application-defined
+         * eventQueries
+         *
+         * @return RecordedNimbusContext
+         */
+        fun create(
+            context: Context,
+            isFirstRun: Boolean,
+        ): RecordedNimbusContext {
+            val settings = context.settings()
+            val langTag = LocaleManager.getCurrentLocale(context)
+                ?.toLanguageTag() ?: getSystemDefault().toLanguageTag()
+            val termsOfUseAdvancedTargetingHelper = TermsOfUseAdvancedTargetingHelper(
+                DefaultTermsOfUseDataProvider(settings),
+                langTag,
+            )
+
+            val packageInfo =
+                context.packageManagerCompatHelper.getPackageInfoCompat(context.packageName, 0)
+            val deviceInfo = NimbusDeviceInfo.default()
+            val db = File(context.applicationInfo.dataDir, NIMBUS_DATA_DIR)
+            val calculatedAttributes = getCalculatedAttributes(
+                packageInfo.firstInstallTime,
+                db.path,
+                deviceInfo.localeTag,
+            )
+
+            return RecordedNimbusContext(
+                isFirstRun = isFirstRun,
+                eventQueries = EVENT_QUERIES,
+                utmSource = settings.utmSource,
+                utmMedium = settings.utmMedium,
+                utmCampaign = settings.utmCampaign,
+                utmTerm = settings.utmTerm,
+                utmContent = settings.utmContent,
+                appVersion = packageInfo.versionName,
+                locale = deviceInfo.localeTag,
+                daysSinceInstall = calculatedAttributes.daysSinceInstall,
+                daysSinceUpdate = calculatedAttributes.daysSinceUpdate,
+                language = calculatedAttributes.language,
+                region = calculatedAttributes.region,
+                userAcceptedTou = settings.hasAcceptedTermsOfService,
+                noShortcutsOrStoriesOptOuts = settings.noShortcutsOrStoriesOptOuts(context),
+                addonIds = getFormattedAddons(settings),
+                touPoints = termsOfUseAdvancedTargetingHelper.getTouPoints(),
+            )
+        }
+
+        @VisibleForTesting
+        internal fun getFormattedAddons(settings: Settings): List<String> =
+            settings.installedAddonsList.split(",").map { it.trim() }
+
+        /**
+         * Checks whether an eligible user has opted out of any sponsored top sites or stories.
+         *
+         *  @return `true` if the user has opted out of any sponsored top sites or stories,
+         * `false` otherwise.
+         */
+        private fun Settings.noShortcutsOrStoriesOptOuts(context: Context) =
+            !optedOutOfSponsoredTopSites() && !optedOutOfSponsoredStories(context)
+
+        /**
+         * Checks whether an eligible user has opted out of the sponsored top sites feature.
+         *
+         * This is not entirely self evident from the API descriptions, please note:
+         * [Settings.showContileFeature] indicates whether the sponsored shortcuts are shown.
+         * [Settings.showTopSitesFeature] indicates whether the feature should be shown at all.
+         */
+        private fun Settings.optedOutOfSponsoredTopSites() =
+            !showContileFeature || !showTopSitesFeature
+
+        private fun Settings.optedOutOfSponsoredStories(context: Context) =
+            isEligibleForStories(context) && (!showPocketSponsoredStories || !showPocketRecommendationsFeature)
+
+        private fun isEligibleForStories(context: Context): Boolean =
+            ContentRecommendationsFeatureHelper.isContentRecommendationsFeatureEnabled(context)
+
+        /**
+         * Creates a RecordedNimbusContext instance for test purposes
+         *
+         * @return RecordedNimbusContext
+         */
+        @VisibleForTesting
+        internal fun createForTest(
+            isFirstRun: Boolean = false,
+            eventQueries: Map<String, String> = EVENT_QUERIES,
+            eventQueryValues: Map<String, Double> = mapOf(),
+            addonIds: List<String> = emptyList(),
+        ): RecordedNimbusContext {
+            return RecordedNimbusContext(
+                isFirstRun = isFirstRun,
+                eventQueries = eventQueries,
+                eventQueryValues = eventQueryValues,
+                utmSource = "",
+                utmMedium = "",
+                utmCampaign = "",
+                utmTerm = "",
+                utmContent = "",
+                appVersion = "",
+                locale = "",
+                daysSinceInstall = 5,
+                daysSinceUpdate = 0,
+                language = "en",
+                region = "US",
+                userAcceptedTou = true,
+                noShortcutsOrStoriesOptOuts = true,
+                addonIds = addonIds,
+                touPoints = 3,
+            )
+        }
+    }
+}

experimenter/experimenter/features/manifests/fenix/beta.fml.yaml

@@ -281,7 +281,6 @@ features:
         description: This property provides a lookup table of whether or not the given section should be enabled. If the section is enabled, it should be toggleable in the settings screen, and on by default.
         type: Map<HomeScreenSection, Boolean>
         default:
-          header: true
           top-sites: true
           jump-back-in: true
           bookmarks: true
@@ -799,6 +798,13 @@ features:
           Whether or not to enable private browsing mode lock.
         type: Boolean
         default: true
+  private-mode-and-stories-entry-point:
+    description: Feature to enable the Private Mode and Stories Entry Point Experiment.
+    variables:
+      enabled:
+        description: If true, Private Mode and Stories Entry Point Experiment is enabled.
+        type: Boolean
+        default: false
   remote-search-configuration:
     description: Feature to use search configurations from remote servers.
     variables:
@@ -1129,8 +1135,6 @@ enums:
         description: The sites the user has bookmarked.
       collections:
         description: The collections section of the homepage.
-      header:
-        description: The header of the homescreen.
       jump-back-in:
         description: The tabs the user was looking immediately before being interrupted.
       pocket: