Skip to content

Commit 696a0ba

Browse files
dbrantdbrant
authored
Followup (final?) to machine suggestion analytics. (#3924)
1 parent 58b1ca1 commit 696a0ba

11 files changed

+90
-65
lines changed

‎app/src/main/java/org/wikipedia/analytics/eventplatform/ABTest.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import kotlin.random.Random
55

66
open class ABTest(private val abTestName: String, private val abTestGroupCount: Int) {
77

8-
val aBTestGroup: Int
8+
val group: Int
99
get() {
1010
testGroup = PrefsIoUtil.getInt(AB_TEST_KEY_PREFIX + abTestName, -1)
1111
if (testGroup == -1) {

‎app/src/main/java/org/wikipedia/analytics/eventplatform/MachineGeneratedArticleDescriptionABCTest.kt‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,24 +3,25 @@ package org.wikipedia.analytics.eventplatform
33
import kotlinx.coroutines.runBlocking
44
import org.wikipedia.WikipediaApp
55
import org.wikipedia.auth.AccountUtil
6+
import org.wikipedia.settings.Prefs
67
import org.wikipedia.util.log.L
78

89
class MachineGeneratedArticleDescriptionABCTest : ABTest("mBART25", GROUP_SIZE_2) {
910

1011
override fun assignGroup() {
1112
super.assignGroup()
12-
if (testGroup == GROUP_2 && AccountUtil.isLoggedIn) {
13+
if (AccountUtil.isLoggedIn) {
1314
runBlocking {
1415
try {
1516
MachineGeneratedArticleDescriptionsAnalyticsHelper.setUserExperienced()
16-
if (MachineGeneratedArticleDescriptionsAnalyticsHelper.isUserExperienced) {
17+
if (testGroup == GROUP_2 && Prefs.suggestedEditsMachineGeneratedDescriptionsIsExperienced) {
1718
testGroup = GROUP_3
1819
}
1920
} catch (e: Exception) {
2021
L.e(e)
2122
}
2223
}
2324
}
24-
MachineGeneratedArticleDescriptionsAnalyticsHelper.logGroupAssigned(WikipediaApp.instance, testGroup)
25+
MachineGeneratedArticleDescriptionsAnalyticsHelper().logGroupAssigned(WikipediaApp.instance, testGroup)
2526
}
2627
}

‎app/src/main/java/org/wikipedia/analytics/eventplatform/MachineGeneratedArticleDescriptionsAnalyticsHelper.kt‎

Lines changed: 41 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -6,60 +6,55 @@ import kotlinx.coroutines.withContext
66
import org.wikipedia.WikipediaApp
77
import org.wikipedia.auth.AccountUtil
88
import org.wikipedia.dataclient.ServiceFactory
9-
import org.wikipedia.dataclient.okhttp.HttpStatusException
109
import org.wikipedia.page.PageTitle
10+
import org.wikipedia.settings.Prefs
1111

12-
object MachineGeneratedArticleDescriptionsAnalyticsHelper {
12+
class MachineGeneratedArticleDescriptionsAnalyticsHelper {
1313

14-
private const val MACHINE_GEN_DESC_SUGGESTIONS = "machineSuggestions"
1514
private var apiFailed = false
16-
val machineGeneratedDescriptionsABTest = MachineGeneratedArticleDescriptionABCTest()
1715
var apiOrderList = emptyList<String>()
1816
var displayOrderList = emptyList<String>()
19-
var chosenSuggestion = ""
20-
var isUserExperienced = false
21-
var isUserInExperiment = false
17+
private var chosenSuggestion = ""
2218
private var startTime = 0L
2319

2420
fun articleDescriptionEditingStart(context: Context) {
25-
log(context, "ArticleDescriptionEditing.start")
26-
startTime = System.currentTimeMillis()
21+
log(context, composeGroupString() + ".start")
2722
}
2823

2924
fun resetTimer() {
3025
startTime = System.currentTimeMillis()
3126
}
3227

3328
fun articleDescriptionEditingEnd(context: Context) {
34-
log(context, "ArticleDescriptionEditing.end.timeSpentMs.${System.currentTimeMillis() - startTime}")
29+
log(context, composeGroupString() + ".end.timeSpentMs.${System.currentTimeMillis() - startTime}")
3530
}
3631

3732
fun logAttempt(context: Context, finalDescription: String, wasChosen: Boolean, wasModified: Boolean, title: PageTitle) {
38-
log(
39-
context, composeLogString(title) + ".attempt:$finalDescription.suggestionChosen:${if (!wasChosen) -1 else displayOrderList.indexOf(chosenSuggestion) + 1}" +
40-
".api.${apiOrderList.indexOf(chosenSuggestion) + 1}.modified:$wasModified"
41-
)
33+
log(context, composeLogString(title) + ".attempt:$finalDescription" +
34+
".suggestion1:${encode(apiOrderList.first())}" + (if (apiOrderList.size > 1) ".suggestion2.${encode(apiOrderList.last())}" else "") +
35+
getOrderString(wasChosen, chosenSuggestion) + ".modified:$wasModified")
4236
}
4337

4438
fun logSuccess(context: Context, finalDescription: String, wasChosen: Boolean, wasModified: Boolean, title: PageTitle, revId: Long) {
45-
log(context, composeLogString(title) + ".success:$finalDescription.suggestionChosen:${if (!wasChosen) -1 else displayOrderList.indexOf(
46-
chosenSuggestion) + 1}.api.${apiOrderList.indexOf(chosenSuggestion) + 1}.modified:$wasModified.revId:$revId")
39+
log(context, composeLogString(title) + ".success:$finalDescription" +
40+
".suggestion1:${encode(apiOrderList.first())}" + (if (apiOrderList.size > 1) ".suggestion2.${encode(apiOrderList.last())}" else "") +
41+
getOrderString(wasChosen, chosenSuggestion) + ".modified:$wasModified.revId:$revId")
4742
}
4843

4944
fun logSuggestionsReceived(context: Context, isBlp: Boolean, title: PageTitle) {
5045
apiFailed = false
51-
log(context, composeLogString(title) + ".blp:$isBlp.count:${apiOrderList.size}.api1:${apiOrderList.first()}" +
52-
if (apiOrderList.size > 1) ".api2.${apiOrderList.last()}" else "")
46+
log(context, composeLogString(title) + ".blp:$isBlp.count:${apiOrderList.size}.suggestion1:${encode(apiOrderList.first())}" +
47+
if (apiOrderList.size > 1) ".suggestion2.${encode(apiOrderList.last())}" else "")
5348
}
5449

5550
fun logSuggestionsShown(context: Context, title: PageTitle) {
56-
log(context, composeLogString(title) + ".count:${displayOrderList.size}.display1:${displayOrderList.first()} " +
57-
if (displayOrderList.size > 1) ".display2.${displayOrderList.last()}" else "")
51+
log(context, composeLogString(title) + ".count:${displayOrderList.size}.display1:${encode(displayOrderList.first())}" +
52+
if (displayOrderList.size > 1) ".display2.${encode(displayOrderList.last())}" else "")
5853
}
5954

6055
fun logSuggestionChosen(context: Context, suggestion: String, title: PageTitle) {
6156
chosenSuggestion = suggestion
62-
log(context, composeLogString(title) + ".selected:$suggestion.${getOrderString()}")
57+
log(context, composeLogString(title) + ".selected:${encode(suggestion)}${getOrderString(true, suggestion)}")
6358
}
6459

6560
fun logSuggestionsDismissed(context: Context, title: PageTitle) {
@@ -68,25 +63,23 @@ object MachineGeneratedArticleDescriptionsAnalyticsHelper {
6863

6964
fun logSuggestionReported(context: Context, suggestion: String, reportReasonsList: List<String>, title: PageTitle) {
7065
val reportReasons = reportReasonsList.joinToString("|")
71-
log(context, composeLogString(title) + ".reportDialog.$suggestion.${getOrderString()}.reasons:$reportReasons.reported")
66+
log(context, composeLogString(title) + ".reportDialog.${encode(suggestion)}${getOrderString(true, suggestion)}.reasons:$reportReasons.reported")
7267
}
7368

7469
fun logReportDialogDismissed(context: Context) {
7570
log(context, composeGroupString() + ".reportDialog.dismissed")
7671
}
7772

7873
fun logOnboardingShown(context: Context) {
79-
log(context, "$MACHINE_GEN_DESC_SUGGESTIONS.onboardingShown")
74+
log(context, composeGroupString() + ".onboardingShown")
8075
}
8176

8277
fun logGroupAssigned(context: Context, testGroup: Int) {
8378
log(context, "$MACHINE_GEN_DESC_SUGGESTIONS.groupAssigned:$testGroup")
8479
}
8580

8681
fun logApiFailed(context: Context, throwable: Throwable, title: PageTitle) {
87-
if (throwable is HttpStatusException) {
88-
log(context, "Api failed with response code ${throwable.code} for : ${composeLogString(title)} ")
89-
}
82+
log(context, composeLogString(title) + ".apiError.${throwable.message}")
9083
apiFailed = true
9184
}
9285

@@ -97,22 +90,35 @@ object MachineGeneratedArticleDescriptionsAnalyticsHelper {
9790
EventPlatformClient.submit(BreadCrumbLogEvent(BreadCrumbViewUtil.getReadableScreenName(context), logString))
9891
}
9992

100-
private fun getOrderString(): String {
101-
return "api.${apiOrderList.indexOf(chosenSuggestion) + 1}.display.${displayOrderList.indexOf(chosenSuggestion) + 1}"
93+
private fun getOrderString(wasChosen: Boolean, suggestion: String): String {
94+
return ".chosenApiIndex.${if (!wasChosen) -1 else apiOrderList.indexOf(suggestion) + 1}" +
95+
".chosenDisplayIndex:${if (!wasChosen) -1 else displayOrderList.indexOf(suggestion) + 1}"
10296
}
10397

10498
private fun composeLogString(title: PageTitle): String {
105-
return "${composeGroupString()}.lang:${title.wikiSite.languageCode}.title:${title.prefixedText}"
99+
return "${composeGroupString()}.lang:${title.wikiSite.languageCode}.title:${encode(title.prefixedText)}"
106100
}
107101

108102
private fun composeGroupString(): String {
109-
return "$MACHINE_GEN_DESC_SUGGESTIONS.group:${machineGeneratedDescriptionsABTest.aBTestGroup}.experienced:$isUserExperienced"
103+
return "$MACHINE_GEN_DESC_SUGGESTIONS.group:${abcTest.group}.experienced:${Prefs.suggestedEditsMachineGeneratedDescriptionsIsExperienced}"
110104
}
111105

112-
suspend fun setUserExperienced() =
113-
withContext(Dispatchers.Default) {
114-
val totalContributions = ServiceFactory.get(WikipediaApp.instance.wikiSite)
115-
.globalUserInfo(AccountUtil.userName!!).query?.globalUserInfo?.editCount ?: 0
116-
isUserExperienced = totalContributions > 50
106+
companion object {
107+
private const val MACHINE_GEN_DESC_SUGGESTIONS = "machineSuggestions"
108+
val abcTest = MachineGeneratedArticleDescriptionABCTest()
109+
var isUserInExperiment = false
110+
111+
// HACK: We're using periods and colons as delimiting characters in these events, so let's
112+
// urlencode just those characters, if they appear in our strings.
113+
private fun encode(str: String): String {
114+
return str.replace(".", "%2E").replace(":", "%3A")
117115
}
116+
117+
suspend fun setUserExperienced() =
118+
withContext(Dispatchers.Default) {
119+
val totalContributions = ServiceFactory.get(WikipediaApp.instance.wikiSite)
120+
.globalUserInfo(AccountUtil.userName!!).query?.globalUserInfo?.editCount ?: 0
121+
Prefs.suggestedEditsMachineGeneratedDescriptionsIsExperienced = totalContributions > 50
122+
}
123+
}
118124
}

‎app/src/main/java/org/wikipedia/descriptions/DescriptionEditActivity.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ class DescriptionEditActivity : SingleFragmentActivity<DescriptionEditFragment>(
4040
SuggestedArticleDescriptionsDialog.availableLanguages.contains(pageTitle.wikiSite.languageCode))
4141

4242
val shouldShowAIOnBoarding = MachineGeneratedArticleDescriptionsAnalyticsHelper.isUserInExperiment &&
43-
MachineGeneratedArticleDescriptionsAnalyticsHelper.machineGeneratedDescriptionsABTest.aBTestGroup != GROUP_1
43+
MachineGeneratedArticleDescriptionsAnalyticsHelper.abcTest.group != GROUP_1
4444

4545
if (action == Action.ADD_DESCRIPTION && Prefs.isDescriptionEditTutorialEnabled) {
4646
Prefs.isDescriptionEditTutorialEnabled = false

‎app/src/main/java/org/wikipedia/descriptions/DescriptionEditFragment.kt‎

Lines changed: 19 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,8 @@ class DescriptionEditFragment : Fragment() {
6666
private var highlightText: String? = null
6767
private var editingAllowed = true
6868

69+
private val analyticsHelper = MachineGeneratedArticleDescriptionsAnalyticsHelper()
70+
6971
private val disposables = CompositeDisposable()
7072

7173
private val loginLauncher = registerForActivityResult(ActivityResultContracts.StartActivityForResult()) {
@@ -193,7 +195,8 @@ class DescriptionEditFragment : Fragment() {
193195

194196
private fun setUpEditView(savedInstanceState: Bundle?) {
195197
if (action == DescriptionEditActivity.Action.ADD_DESCRIPTION) {
196-
MachineGeneratedArticleDescriptionsAnalyticsHelper.articleDescriptionEditingStart(requireContext())
198+
analyticsHelper.resetTimer()
199+
analyticsHelper.articleDescriptionEditingStart(requireContext())
197200
}
198201
binding.fragmentDescriptionEditView.setAction(action)
199202
binding.fragmentDescriptionEditView.setPageTitle(pageTitle)
@@ -211,7 +214,7 @@ class DescriptionEditFragment : Fragment() {
211214
binding.fragmentDescriptionEditView.isSuggestionButtonEnabled =
212215
SuggestedArticleDescriptionsDialog.availableLanguages.contains(pageTitle.wikiSite.languageCode) &&
213216
action == DescriptionEditActivity.Action.ADD_DESCRIPTION && pageTitle.description.isNullOrEmpty() &&
214-
MachineGeneratedArticleDescriptionsAnalyticsHelper.machineGeneratedDescriptionsABTest.aBTestGroup != GROUP_1
217+
MachineGeneratedArticleDescriptionsAnalyticsHelper.abcTest.group != GROUP_1
215218

216219
if (binding.fragmentDescriptionEditView.isSuggestionButtonEnabled) {
217220
binding.fragmentDescriptionEditView.showSuggestedDescriptionsLoadingProgress()
@@ -223,7 +226,7 @@ class DescriptionEditFragment : Fragment() {
223226
lifecycleScope.launch(CoroutineExceptionHandler { _, throwable ->
224227
binding.fragmentDescriptionEditView.isSuggestionButtonEnabled = false
225228
L.e(throwable)
226-
MachineGeneratedArticleDescriptionsAnalyticsHelper.logApiFailed(requireContext(), throwable, pageTitle)
229+
analyticsHelper.logApiFailed(requireContext(), throwable, pageTitle)
227230
}) {
228231
withContext(Dispatchers.IO) {
229232
val response = ServiceFactory[pageTitle.wikiSite, DescriptionSuggestionService.API_URL, DescriptionSuggestionService::class.java]
@@ -235,19 +238,19 @@ class DescriptionEditFragment : Fragment() {
235238
val list = (if (pageTitle.wikiSite.languageCode == "en") {
236239
response.prediction.map { StringUtil.capitalize(it)!! }
237240
} else response.prediction).distinct()
238-
MachineGeneratedArticleDescriptionsAnalyticsHelper.apiOrderList = list
239-
MachineGeneratedArticleDescriptionsAnalyticsHelper.logSuggestionsReceived(requireContext(), response.blp, pageTitle)
241+
analyticsHelper.apiOrderList = list
242+
analyticsHelper.logSuggestionsReceived(requireContext(), response.blp, pageTitle)
240243
L.d("Received suggestion: " + list.first())
241244
L.d("And is it a BLP? " + response.blp)
242245

243246
// Randomize the display order
244-
if (!response.blp || MachineGeneratedArticleDescriptionsAnalyticsHelper.machineGeneratedDescriptionsABTest.aBTestGroup == GROUP_3) {
247+
if (!response.blp || MachineGeneratedArticleDescriptionsAnalyticsHelper.abcTest.group == GROUP_3) {
245248
val randomizedListIndex = (0 until 2).random()
246249
val firstSuggestion = if (list.size == 2) list[randomizedListIndex] else list.first()
247250
val secondSuggestion = if (list.size == 2) { if (randomizedListIndex == 0) list.last() else list.first() } else null
248-
MachineGeneratedArticleDescriptionsAnalyticsHelper.displayOrderList = listOfNotNull(firstSuggestion, secondSuggestion)
251+
analyticsHelper.displayOrderList = listOfNotNull(firstSuggestion, secondSuggestion)
249252
binding.fragmentDescriptionEditView.showSuggestedDescriptionsButton(firstSuggestion, secondSuggestion)
250-
MachineGeneratedArticleDescriptionsAnalyticsHelper.logSuggestionsShown(requireContext(), pageTitle)
253+
analyticsHelper.logSuggestionsShown(requireContext(), pageTitle)
251254
}
252255
}
253256
}
@@ -267,15 +270,15 @@ class DescriptionEditFragment : Fragment() {
267270
override fun onSaveClick() {
268271
if (!binding.fragmentDescriptionEditView.showingReviewContent()) {
269272
if (action == DescriptionEditActivity.Action.ADD_DESCRIPTION) {
270-
MachineGeneratedArticleDescriptionsAnalyticsHelper.articleDescriptionEditingEnd(requireContext())
273+
analyticsHelper.articleDescriptionEditingEnd(requireContext())
271274
}
272275
binding.fragmentDescriptionEditView.loadReviewContent(true)
273276
} else {
274277
binding.fragmentDescriptionEditView.setError(null)
275278
binding.fragmentDescriptionEditView.setSaveState(true)
276279
cancelCalls()
277280
if (action == DescriptionEditActivity.Action.ADD_DESCRIPTION) {
278-
MachineGeneratedArticleDescriptionsAnalyticsHelper.logAttempt(requireContext(),
281+
analyticsHelper.logAttempt(requireContext(),
279282
binding.fragmentDescriptionEditView.description.orEmpty(), binding.fragmentDescriptionEditView.wasSuggestionChosen,
280283
binding.fragmentDescriptionEditView.wasSuggestionModified, pageTitle
281284
)
@@ -333,7 +336,7 @@ class DescriptionEditFragment : Fragment() {
333336
AnonymousNotificationHelper.onEditSubmitted()
334337
waitForUpdatedRevision(newRevId)
335338
EditAttemptStepEvent.logSaveSuccess(pageTitle, EditAttemptStepEvent.INTERFACE_OTHER)
336-
MachineGeneratedArticleDescriptionsAnalyticsHelper.logSuccess(requireContext(),
339+
analyticsHelper.logSuccess(requireContext(),
337340
binding.fragmentDescriptionEditView.description.orEmpty(),
338341
binding.fragmentDescriptionEditView.wasSuggestionChosen,
339342
binding.fragmentDescriptionEditView.wasSuggestionModified,
@@ -382,7 +385,7 @@ class DescriptionEditFragment : Fragment() {
382385
AnonymousNotificationHelper.onEditSubmitted()
383386
if (response.success > 0) {
384387
requireView().postDelayed(successRunnable, TimeUnit.SECONDS.toMillis(4))
385-
MachineGeneratedArticleDescriptionsAnalyticsHelper.logSuccess(requireContext(),
388+
analyticsHelper.logSuccess(requireContext(),
386389
binding.fragmentDescriptionEditView.description.orEmpty(),
387390
binding.fragmentDescriptionEditView.wasSuggestionChosen,
388391
binding.fragmentDescriptionEditView.wasSuggestionModified,
@@ -467,7 +470,6 @@ class DescriptionEditFragment : Fragment() {
467470
override fun onCancelClick() {
468471
if (binding.fragmentDescriptionEditView.showingReviewContent()) {
469472
binding.fragmentDescriptionEditView.loadReviewContent(false)
470-
MachineGeneratedArticleDescriptionsAnalyticsHelper.resetTimer()
471473
} else {
472474
DeviceUtil.hideSoftKeyboard(requireActivity())
473475
requireActivity().onBackPressed()
@@ -486,6 +488,10 @@ class DescriptionEditFragment : Fragment() {
486488
FeedbackUtil.showMessage(requireActivity(), R.string.error_voice_search_not_available)
487489
}
488490
}
491+
492+
override fun getAnalyticsHelper(): MachineGeneratedArticleDescriptionsAnalyticsHelper {
493+
return analyticsHelper
494+
}
489495
}
490496

491497
private fun updateDescriptionInArticle(articleText: String, newDescription: String): String {

‎app/src/main/java/org/wikipedia/descriptions/DescriptionEditTutorialFragment.kt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ class DescriptionEditTutorialFragment : OnboardingFragment() {
1919

2020
override fun onCreate(savedInstanceState: Bundle?) {
2121
super.onCreate(savedInstanceState)
22-
showAIOnBoarding = requireArguments().getBoolean(DescriptionEditTutorialActivity.SHOULD_SHOW_AI_ON_BOARDING)!!
22+
showAIOnBoarding = requireArguments().getBoolean(DescriptionEditTutorialActivity.SHOULD_SHOW_AI_ON_BOARDING)
2323
}
2424

2525
override fun getAdapter(): FragmentStateAdapter {
@@ -42,7 +42,7 @@ class DescriptionEditTutorialFragment : OnboardingFragment() {
4242
val position = requireArguments().getInt(ARG_POSITION, 0)
4343
val view = inflater.inflate(pages[position], container, false) as OnboardingPageView
4444
if (position == 2) {
45-
MachineGeneratedArticleDescriptionsAnalyticsHelper.logOnboardingShown(requireContext())
45+
MachineGeneratedArticleDescriptionsAnalyticsHelper().logOnboardingShown(requireContext())
4646
}
4747
view.callback = OnboardingPageView.DefaultCallback()
4848
return view

0 commit comments

Comments
 (0)