diff --git a/analytics/metricsplatform/.gitignore b/analytics/metricsplatform/.gitignore new file mode 100644 index 00000000000..796b96d1c40 --- /dev/null +++ b/analytics/metricsplatform/.gitignore @@ -0,0 +1 @@ +/build diff --git a/analytics/metricsplatform/build.gradle b/analytics/metricsplatform/build.gradle new file mode 100644 index 00000000000..e74625f1781 --- /dev/null +++ b/analytics/metricsplatform/build.gradle @@ -0,0 +1,77 @@ +plugins { + id 'com.android.library' + id 'com.google.devtools.ksp' + id 'kotlin-android' + id 'kotlinx-serialization' +} + +final JavaVersion JAVA_VERSION = JavaVersion.VERSION_17 + +android { + namespace 'org.wikimedia.metricsplatform' + + compileOptions { + coreLibraryDesugaringEnabled true + + sourceCompatibility = JAVA_VERSION + targetCompatibility = JAVA_VERSION + } + + kotlinOptions { + jvmTarget = JAVA_VERSION + } + compileSdk 35 + + defaultConfig { + targetSdk 35 + + buildConfigField "String", "EVENTGATE_ANALYTICS_EXTERNAL_BASE_URI", '"https://intake-analytics.wikimedia.org"' + buildConfigField "String", "EVENTGATE_LOGGING_EXTERNAL_BASE_URI", '"https://intake-logging.wikimedia.org"' + } + + buildFeatures { + buildConfig true + } +} + +dependencies { + coreLibraryDesugaring libs.desugar.jdk.libs + + implementation libs.kotlin.stdlib.jdk8 + implementation libs.kotlinx.coroutines.core + implementation libs.kotlinx.coroutines.android + implementation libs.kotlinx.serialization.json + + implementation libs.material + implementation libs.appcompat + implementation libs.core.ktx + implementation libs.browser + implementation libs.constraintlayout + implementation libs.fragment.ktx + implementation libs.paging.runtime.ktx + implementation libs.palette.ktx + implementation libs.preference.ktx + implementation libs.recyclerview + implementation libs.viewpager2 + implementation libs.flexbox + implementation libs.drawerlayout + implementation libs.swiperefreshlayout + implementation libs.work.runtime.ktx + + implementation libs.okhttp.tls + implementation libs.okhttp3.logging.interceptor + implementation libs.retrofit + implementation libs.commons.lang3 + implementation libs.jsoup + implementation libs.photoview + implementation libs.balloon + implementation libs.retrofit2.kotlinx.serialization.converter + + implementation libs.android.sdk + implementation libs.android.plugin.annotation.v9 + + implementation libs.androidx.room.runtime + annotationProcessor libs.androidx.room.compiler + ksp libs.androidx.room.compiler + implementation libs.androidx.room.ktx +} diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/ContextController.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/ContextController.kt new file mode 100644 index 00000000000..2b8c90a7bbf --- /dev/null +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/ContextController.kt @@ -0,0 +1,94 @@ +package org.wikimedia.metricsplatform + +import org.wikimedia.metricsplatform.config.StreamConfig +import org.wikimedia.metricsplatform.context.AgentData +import org.wikimedia.metricsplatform.context.ClientData +import org.wikimedia.metricsplatform.context.ContextValue +import org.wikimedia.metricsplatform.context.MediawikiData +import org.wikimedia.metricsplatform.context.PageData +import org.wikimedia.metricsplatform.context.PerformerData +import org.wikimedia.metricsplatform.event.EventProcessed + +class ContextController { + fun enrichEvent(event: EventProcessed, streamConfig: StreamConfig) { + if (!streamConfig.hasRequestedContextValuesConfig()) { + return + } + // Check stream config for which contextual values should be added to the event. + val requestedValuesFromConfig = streamConfig.producerConfig?.metricsPlatformClientConfig?.requestedValues.orEmpty() + // Add required properties. + val requestedValues= mutableSetOf() + requestedValues.addAll(requestedValuesFromConfig) + requestedValues.addAll(REQUIRED_PROPERTIES) + val filteredData = filterClientData(event.clientData, requestedValues) + event.applyClientData(filteredData) + } + + private fun filterClientData(clientData: ClientData, requestedValues: Collection): ClientData { + val newAgentData = AgentData() + val newPageData = PageData() + val newMediawikiData = MediawikiData() + val newPerformerData = PerformerData() + + for (requestedValue in requestedValues) { + when (requestedValue) { + ContextValue.AGENT_APP_INSTALL_ID -> newAgentData.appInstallId = clientData.agentData?.appInstallId + ContextValue.AGENT_CLIENT_PLATFORM -> newAgentData.clientPlatform = clientData.agentData?.clientPlatform + ContextValue.AGENT_CLIENT_PLATFORM_FAMILY -> newAgentData.clientPlatformFamily = clientData.agentData?.clientPlatformFamily + ContextValue.AGENT_APP_FLAVOR -> newAgentData.appFlavor = clientData.agentData?.appFlavor + ContextValue.AGENT_APP_THEME -> newAgentData.appTheme = clientData.agentData?.appTheme + ContextValue.AGENT_APP_VERSION -> newAgentData.appVersion = clientData.agentData?.appVersion + ContextValue.AGENT_APP_VERSION_NAME -> newAgentData.appVersionName = clientData.agentData?.appVersionName + ContextValue.AGENT_DEVICE_FAMILY -> newAgentData.deviceFamily = clientData.agentData?.deviceFamily + ContextValue.AGENT_DEVICE_LANGUAGE -> newAgentData.deviceLanguage = clientData.agentData?.deviceLanguage + ContextValue.AGENT_RELEASE_STATUS -> newAgentData.releaseStatus = clientData.agentData?.releaseStatus + ContextValue.PAGE_ID -> newPageData.id = clientData.pageData?.id + ContextValue.PAGE_TITLE -> newPageData.title = clientData.pageData?.title + ContextValue.PAGE_NAMESPACE_ID -> newPageData.namespaceId = clientData.pageData?.namespaceId + ContextValue.PAGE_NAMESPACE_NAME -> newPageData.namespaceName = clientData.pageData?.namespaceName + ContextValue.PAGE_REVISION_ID -> newPageData.revisionId = clientData.pageData?.revisionId + ContextValue.PAGE_WIKIDATA_QID -> newPageData.wikidataItemQid = clientData.pageData?.wikidataItemQid + ContextValue.PAGE_CONTENT_LANGUAGE -> newPageData.contentLanguage = clientData.pageData?.contentLanguage + ContextValue.MEDIAWIKI_DATABASE -> newMediawikiData.database = clientData.mediawikiData?.database + ContextValue.PERFORMER_ID -> newPerformerData.id = clientData.performerData?.id + ContextValue.PERFORMER_NAME -> newPerformerData.name = clientData.performerData?.name + ContextValue.PERFORMER_IS_LOGGED_IN -> newPerformerData.isLoggedIn = clientData.performerData?.isLoggedIn + ContextValue.PERFORMER_IS_TEMP -> newPerformerData.isTemp = clientData.performerData?.isTemp + ContextValue.PERFORMER_SESSION_ID -> newPerformerData.sessionId = clientData.performerData?.sessionId + ContextValue.PERFORMER_PAGEVIEW_ID -> newPerformerData.pageviewId = clientData.performerData?.pageviewId + ContextValue.PERFORMER_GROUPS -> newPerformerData.groups = clientData.performerData?.groups + ContextValue.PERFORMER_LANGUAGE_GROUPS -> { + var languageGroups = clientData.performerData?.languageGroups + if (languageGroups != null && languageGroups.length > 255) { + languageGroups = languageGroups.substring(0, 255) + } + newPerformerData.languageGroups = languageGroups + } + + ContextValue.PERFORMER_LANGUAGE_PRIMARY -> newPerformerData.languagePrimary = clientData.performerData?.languagePrimary + ContextValue.PERFORMER_REGISTRATION_DT -> newPerformerData.registrationDt = clientData.performerData?.registrationDt + else -> throw IllegalArgumentException("Unknown property: $requestedValue") + } + } + + return ClientData(newAgentData, newPageData, newMediawikiData, newPerformerData) + } + + companion object { + /** + * @see [Metrics Platform/Contextual attributes](https://wikitech.wikimedia.org/wiki/Metrics_Platform/Contextual_attributes) + */ + private val REQUIRED_PROPERTIES = listOf( + "agent_app_flavor", + "agent_app_install_id", + "agent_app_theme", + "agent_app_version", + "agent_app_version_name", + "agent_client_platform", + "agent_client_platform_family", + "agent_device_family", + "agent_device_language", + "agent_release_status" + ) + } +} diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/CurationController.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/CurationController.kt new file mode 100644 index 00000000000..adecb3908ad --- /dev/null +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/CurationController.kt @@ -0,0 +1,11 @@ +package org.wikimedia.metricsplatform + +import org.wikimedia.metricsplatform.config.StreamConfig +import org.wikimedia.metricsplatform.event.EventProcessed + +class CurationController { + fun shouldProduceEvent(event: EventProcessed, streamConfig: StreamConfig): Boolean { + if (!streamConfig.hasCurationFilter()) return true + return streamConfig.curationFilter.test(event) + } +} diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventProcessor.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventProcessor.kt new file mode 100644 index 00000000000..f9f9cb645e2 --- /dev/null +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventProcessor.kt @@ -0,0 +1,99 @@ +package org.wikimedia.metricsplatform + +import org.wikimedia.metricsplatform.config.DestinationEventService +import org.wikimedia.metricsplatform.config.SourceConfig +import org.wikimedia.metricsplatform.config.StreamConfig +import org.wikimedia.metricsplatform.event.EventProcessed +import java.net.SocketTimeoutException +import java.net.UnknownHostException +import java.util.concurrent.BlockingQueue +import java.util.concurrent.atomic.AtomicReference + +class EventProcessor( + private val contextController: ContextController, + private val curationController: CurationController, + private val sourceConfig: AtomicReference, + private val samplingController: SamplingController, + private val eventSender: EventSender, + private val eventQueue: BlockingQueue +) { + + /** + * Send all events currently in the output buffer. + * + * + * A shallow clone of the output buffer is created and passed to the integration layer for + * submission by the client. If the event submission succeeds, the events are removed from the + * output buffer. (Note that the shallow copy created by clone() retains pointers to the original + * Event objects.) If the event submission fails, a client error is produced, and the events remain + * in buffer to be retried on the next submission attempt. + */ + fun sendEnqueuedEvents() { + val config: SourceConfig? = sourceConfig.get() + if (config == null) { + //log.log(Level.FINE, "Configuration is missing, enqueued events are not sent.") + return + } + + val pending = mutableListOf() + synchronized(eventQueue) { + eventQueue.drainTo(pending) + } + + val streamConfigsMap = config.streamConfigs + + pending.filter { event -> streamConfigsMap.containsKey(event.stream) } + .filter { event -> + val cfg = streamConfigsMap[event.stream] + if (cfg == null) { + false + } else { + cfg.sampleConfig?.let { event.sample = it } + samplingController.isInSample(cfg) + } + } + .filter { event -> eventPassesCurationRules(event, streamConfigsMap) } + .groupBy { event -> destinationEventService(event, streamConfigsMap) } + .forEach { (destinationEventService, pendingValidEvents) -> + sendEventsToDestination(destinationEventService, pendingValidEvents) + } + } + + fun eventPassesCurationRules( + event: EventProcessed, + streamConfigMap: Map + ): Boolean { + val streamConfig = streamConfigMap[event.stream] + if (streamConfig == null) { + return false + } + contextController.enrichEvent(event, streamConfig) + return curationController.shouldProduceEvent(event, streamConfig) + } + + private fun destinationEventService( + event: EventProcessed, + streamConfigMap: Map + ): DestinationEventService { + val streamConfig = streamConfigMap[event.stream] + return streamConfig?.destinationEventService ?: DestinationEventService.ANALYTICS + } + + private fun sendEventsToDestination( + destinationEventService: DestinationEventService, + pendingValidEvents: List + ) { + try { + //TODO! + //eventSender.sendEvents(destinationEventService.getBaseUri(isDebug), pendingValidEvents) + } catch (e: UnknownHostException) { + //log.log(Level.WARNING, "Network error while sending " + pendingValidEvents.size + " events. Adding back to queue.", e) + eventQueue.addAll(pendingValidEvents) + } catch (e: SocketTimeoutException) { + //log.log(Level.WARNING, "Network error while sending " + pendingValidEvents.size + " events. Adding back to queue.", e) + eventQueue.addAll(pendingValidEvents) + } catch (e: Exception) { + //log.log(Level.WARNING, "Failed to send " + pendingValidEvents.size + " events.", e) + } + } +} diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventSender.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventSender.kt new file mode 100644 index 00000000000..caeb22948b7 --- /dev/null +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventSender.kt @@ -0,0 +1,14 @@ +package org.wikimedia.metricsplatform + +import android.net.Uri +import org.wikimedia.metricsplatform.event.EventProcessed + +fun interface EventSender { + /** + * Transmit an event to a destination intake service. + * + * @param baseUri base uri of destination intake service + * @param events events to be sent + */ + fun sendEvents(baseUri: Uri, events: List) +} diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventSenderDefault.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventSenderDefault.kt new file mode 100644 index 00000000000..83caec284d4 --- /dev/null +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventSenderDefault.kt @@ -0,0 +1,39 @@ +package org.wikimedia.metricsplatform + +import android.net.Uri +import kotlinx.serialization.json.Json +import okhttp3.MediaType.Companion.toMediaTypeOrNull +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.RequestBody.Companion.toRequestBody +import org.wikimedia.metricsplatform.event.EventProcessed +import java.io.IOException + +class EventSenderDefault( + private val json: Json, + private val httpClient: OkHttpClient +) : EventSender { + override fun sendEvents(baseUri: Uri, events: List) { + val request = Request.Builder() + .url(baseUri.toString()) + .header("Accept", "application/json") + .header( + "User-Agent", + "Metrics Platform Client/Java " + MetricsClient.METRICS_PLATFORM_LIBRARY_VERSION + ) + .post(json.encodeToString(events).toRequestBody("application/json".toMediaTypeOrNull())) + .build() + + httpClient.newCall(request).execute().use { response -> + val status = response.code + val body = response.body + if (!response.isSuccessful || status == 207) { + // In the case of a multi-status response (207), it likely means that one or more + // events were rejected. In such a case, the error is actually contained in + // the normal response body. + throw IOException(body?.string().orEmpty()) + } + //log.log(java.util.logging.Level.INFO, "Sent " + events.size + " events successfully.") + } + } +} diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/MetricsClient.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/MetricsClient.kt new file mode 100644 index 00000000000..67b74d000a1 --- /dev/null +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/MetricsClient.kt @@ -0,0 +1,325 @@ +package org.wikimedia.metricsplatform + +import org.wikimedia.metricsplatform.config.SourceConfig +import org.wikimedia.metricsplatform.config.StreamConfig +import org.wikimedia.metricsplatform.context.ClientData +import org.wikimedia.metricsplatform.context.InteractionData +import org.wikimedia.metricsplatform.event.Event +import org.wikimedia.metricsplatform.event.EventProcessed +import java.time.ZoneId +import java.time.ZonedDateTime +import java.time.format.DateTimeFormatter +import java.util.concurrent.BlockingQueue +import java.util.concurrent.LinkedBlockingQueue +import java.util.concurrent.atomic.AtomicReference +import kotlin.math.max + +class MetricsClient( + clientData: ClientData, + eventSender: EventSender, + sourceConfigInit: SourceConfig? = null, + val queueCapacity: Int = 100 +) { + + private val sourceConfig = AtomicReference(sourceConfigInit) + + /** + * Handles logging session management. A new session begins (and a new session ID is created) + * if the app has been inactive for 15 minutes or more. + */ + private val sessionController = SessionController() + + /** + * Evaluates whether events for a given stream are in-sample based on the stream configuration. + */ + private val samplingController = SamplingController(clientData, sessionController) + + private val eventQueue: BlockingQueue = LinkedBlockingQueue(queueCapacity) + + private val eventProcessor: EventProcessor = EventProcessor( + ContextController(), + CurationController(), + sourceConfig, + samplingController, + eventSender, + eventQueue + ) + + /** + * Submit an event to be enqueued and sent to the Event Platform. + * + * + * If stream configs are not yet fetched, the event will be held temporarily in the input + * buffer (provided there is space to do so). + * + * + * If stream configs are available, the event will be validated and enqueued for submission + * to the configured event platform intake service. + * + * + * Supplemental metadata is added immediately on intake, regardless of the presence or absence + * of stream configs, so that the event timestamp is recorded accurately. + * + * @param event event data + */ + fun submit(event: Event) { + val eventProcessed = EventProcessed.fromEvent(event) + addRequiredMetadata(eventProcessed) + addToEventQueue(eventProcessed) + } + + /** + * Construct and submits a Metrics Platform Event from the schema id, event name, page metadata, and custom data for + * the stream that is interested in those events. + * + * @param streamName stream name + * @param schemaId schema id + * @param eventName event name + * @param clientData client context data + * @param customData custom data + */ + fun submitMetricsEvent( + streamName: String, + schemaId: String, + eventName: String, + clientData: ClientData? = null, + customData: Map? = null, + interactionData: InteractionData? = null + ) { + // If we already have stream configs, then we can pre-validate certain conditions and exclude the event from the queue entirely. + var streamConfig: StreamConfig? = null + if (sourceConfig.get() != null) { + streamConfig = sourceConfig.get().getStreamConfigByName(streamName) + if (streamConfig == null) { + //log.log(Level.FINE, "No stream config exists for this stream, the submitMetricsEvent event is ignored and dropped.") + return + } + if (!samplingController.isInSample(streamConfig)) { + //log.log(Level.FINE, "Not in sample, the submitMetricsEvent event is ignored and dropped.") + return + } + } + + val event = Event(streamName) + event.schema = schemaId + event.name = eventName + if (clientData != null) { + event.clientData = clientData + } + if (customData != null) { + event.customData = customData.mapValues { it.value.toString() } + } + if (interactionData != null) { + event.interactionData = interactionData + } + if (streamConfig?.sampleConfig != null) { + event.sample = streamConfig.sampleConfig + } + submit(event) + } + + /** + * Submit an interaction event to a stream. + * + * + * See above - takes additional parameters (custom data + custom schema id) to submit an interaction event. + * + * @param streamName stream name + * @param schemaId schema id + * @param eventName event name + * @param clientData client context data + * @param interactionData common data for the base interaction schema + * @param customData custom data for the interaction + * + * @see [Metrics Platform/Java API](https://wikitech.wikimedia.org/wiki/Metrics_Platform/Java_API) + */ + fun submitInteraction( + streamName: String, + eventName: String, + schemaId: String = METRICS_PLATFORM_SCHEMA_BASE, + clientData: ClientData? = null, + interactionData: InteractionData? = null, + customData: Map? = null + ) { + submitMetricsEvent(streamName, schemaId, eventName, clientData, customData, interactionData) + } + + /** + * Submit a click event to a stream. + * + * @param streamName stream name + * @param clientData client context data + * @param interactionData common data for the base interaction schema + * + * @see [Metrics Platform/Java API](https://wikitech.wikimedia.org/wiki/Metrics_Platform/Java_API) + */ + fun submitClick( + streamName: String, + clientData: ClientData, + interactionData: InteractionData + ) { + submitMetricsEvent( + streamName, + METRICS_PLATFORM_SCHEMA_BASE, + "click", + clientData, + null, + interactionData + ) + } + + /** + * Submit a click event to a stream with custom data. + * + * @param streamName stream name + * @param schemaId schema id + * @param eventName event name + * @param clientData client context data + * @param customData custom data for the interaction + * @param interactionData common data for the base interaction schema + * + * @see [Metrics Platform/Java API](https://wikitech.wikimedia.org/wiki/Metrics_Platform/Java_API) + */ + fun submitClick( + streamName: String, + schemaId: String, + eventName: String, + clientData: ClientData, + customData: Map?, + interactionData: InteractionData + ) { + submitMetricsEvent(streamName, schemaId, eventName, clientData, customData, interactionData) + } + + /** + * Submit a view event to a stream. + * + * @param streamName stream name + * @param clientData client context data + * @param interactionData common data for the base interaction schema + */ + fun submitView( + streamName: String, + clientData: ClientData, + interactionData: InteractionData + ) { + submitMetricsEvent( + streamName, + METRICS_PLATFORM_SCHEMA_BASE, + "view", + clientData, + null, + interactionData + ) + } + + /** + * Submit a view event to a stream with custom data. + * + * @param streamName stream name + * @param schemaId schema id + * @param eventName event name + * @param clientData client context data + * @param customData custom data for the interaction + * @param interactionData common data for the base interaction schema + */ + fun submitView( + streamName: String, + schemaId: String, + eventName: String, + clientData: ClientData, + customData: Map?, + interactionData: InteractionData + ) { + submitMetricsEvent(streamName, schemaId, eventName, clientData, customData, interactionData) + } + + /** + * Convenience method to be called when + * [ + * the onPause() activity lifecycle callback](https://developer.android.com/guide/components/activities/activity-lifecycle#onpause) is called. + * + * + * Touches the session so that we can determine whether it's session has expired if and when the + * application is resumed. + */ + fun onAppPause() { + sessionController.touchSession() + eventProcessor.sendEnqueuedEvents() + } + + /** + * Convenience method to be called when + * [ + * the onResume() activity lifecycle callback](https://developer.android.com/guide/components/activities/activity-lifecycle#onresume) is called. + * + * + * Touches the session so that we can determine whether it has expired. + */ + fun onAppResume() { + sessionController.touchSession() + } + + /** + * Closes the session. + */ + fun onAppClose() { + sessionController.closeSession() + eventProcessor.sendEnqueuedEvents() + } + + /** + * Begins a new session and touches the session. + */ + fun resetSession() { + sessionController.beginSession() + } + + /** + * Supplement the outgoing event with additional metadata. + * These include: + * - app_session_id: the current session ID + * - dt: ISO 8601 timestamp + * - domain: hostname + * + * @param event event + */ + private fun addRequiredMetadata(event: EventProcessed) { + event.performerData?.let { it.sessionId = sessionController.sessionId } + event.timestamp = DATE_FORMAT.format(ZonedDateTime.now(ZONE_Z)) + event.setDomain(event.clientData.domain) + } + + /** + * Append an enriched event to the queue. + * If the queue is full, we remove the oldest events from the queue to add the current event. + * Number of attempts to add to the queue is 1/50 of the number queue capacity but at least 10 + * + * @param event a processed event + */ + private fun addToEventQueue(event: EventProcessed?) { + var eventQueueAppendAttempts = max(eventQueue.size / 50, 10) + + if (eventQueue.size > queueCapacity / 2) { + eventProcessor.sendEnqueuedEvents() + } + + while (!eventQueue.offer(event)) { + val removedEvent = eventQueue.remove() + if (removedEvent != null) { + //log.log(Level.FINE, removedEvent.name + " was dropped so that a newer event could be added to the queue.") + } + if (eventQueueAppendAttempts-- <= 0) break + } + } + + companion object { + val DATE_FORMAT: DateTimeFormatter = DateTimeFormatter.ISO_DATE_TIME + val ZONE_Z: ZoneId? = ZoneId.of("Z") + + const val METRICS_PLATFORM_LIBRARY_VERSION: String = "2.8" + const val METRICS_PLATFORM_BASE_VERSION: String = "1.2.2" + + const val METRICS_PLATFORM_SCHEMA_BASE: String = "/analytics/product_metrics/app/base/$METRICS_PLATFORM_BASE_VERSION" + } +} diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/SamplingController.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/SamplingController.kt new file mode 100644 index 00000000000..d7cf470da45 --- /dev/null +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/SamplingController.kt @@ -0,0 +1,57 @@ +package org.wikimedia.metricsplatform + +import org.wikimedia.metricsplatform.config.StreamConfig +import org.wikimedia.metricsplatform.config.sampling.SampleConfig +import org.wikimedia.metricsplatform.context.ClientData +import java.util.UUID + +/** + * SamplingController: computes various sampling functions on the client + * + * Sampling is based on associative identifiers, each of which have a + * well-defined scope, and sampling config, which each stream provides as + * part of its configuration. + */ +class SamplingController internal constructor( + private val clientData: ClientData, + private val sessionController: SessionController +) { + /** + * @param streamConfig stream config + * @return true if in sample or false otherwise + */ + fun isInSample(streamConfig: StreamConfig): Boolean { + if (streamConfig.sampleConfig == null) { + return true + } + val sampleConfig = streamConfig.sampleConfig!! + if (sampleConfig.rate == 1.0) { + return true + } + if (sampleConfig.rate == 0.0) { + return false + } + return getSamplingValue(sampleConfig.unit) < sampleConfig.rate + } + + fun getSamplingValue(unit: String): Double { + val token = getSamplingId(unit).substring(0, 8) + return token.toLong(16).toDouble() / 0xFFFFFFFFL.toDouble() + } + + /** + * Returns the ID string to be used when evaluating presence in sample. + * The ID used is configured in stream config. + * + * @param unit Identifier enum value + * @return the requested ID string + */ + fun getSamplingId(unit: String): String { + return when (unit) { + SampleConfig.UNIT_SESSION -> return sessionController.sessionId + SampleConfig.UNIT_DEVICE -> return clientData.agentData?.appInstallId.orEmpty() + SampleConfig.UNIT_PAGEVIEW -> return clientData.performerData?.pageviewId.orEmpty() + else -> UUID.randomUUID().toString() + } + } +} diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/SessionController.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/SessionController.kt new file mode 100644 index 00000000000..a10cf366c3f --- /dev/null +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/SessionController.kt @@ -0,0 +1,58 @@ +package org.wikimedia.metricsplatform + +import java.security.SecureRandom +import java.time.Duration +import java.time.Instant +import java.util.Locale +import java.util.Random + +/** + * Manages sessions and session IDs for the Metrics Platform Client. + * + * A session begins when the application is launched and expires when the app is in the background + * for 30 minutes or more. + */ +class SessionController internal constructor( + private var sessionTouched: Instant? = Instant.now() +) { + @get:Synchronized + var sessionId = generateSessionId() + private set + + @Synchronized + fun touchSession() { + if (sessionExpired()) { + sessionId = generateSessionId() + } + sessionTouched = Instant.now() + } + + @Synchronized + fun beginSession() { + sessionId = generateSessionId() + sessionTouched = Instant.now() + } + + @Synchronized + fun closeSession() { + // @ToDo Determine how to close the session. + sessionTouched = Instant.now() + } + + @Synchronized + fun sessionExpired(): Boolean { + return Duration.between(sessionTouched, Instant.now()) >= SESSION_LENGTH + } + + companion object { + private val SESSION_LENGTH = Duration.ofMinutes(30) + private val RANDOM = SecureRandom() + + private fun generateSessionId(): String { + val random: Random = RANDOM + return String.format(Locale.US, "%08x", random.nextInt()) + + String.format(Locale.US, "%08x", random.nextInt()) + + String.format(Locale.US, "%04x", random.nextInt() and 0xFFFF) + } + } +} diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/CurationFilter.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/CurationFilter.kt new file mode 100644 index 00000000000..587f5c31cd5 --- /dev/null +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/CurationFilter.kt @@ -0,0 +1,95 @@ +@file:UseSerializers(InstantSerializer::class) + +package org.wikimedia.metricsplatform.config + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import kotlinx.serialization.UseSerializers +import org.wikimedia.metricsplatform.config.curation.CollectionCurationRules +import org.wikimedia.metricsplatform.config.curation.ComparableCurationRules +import org.wikimedia.metricsplatform.config.curation.CurationRules +import org.wikimedia.metricsplatform.context.AgentData +import org.wikimedia.metricsplatform.context.InstantSerializer +import org.wikimedia.metricsplatform.context.MediawikiData +import org.wikimedia.metricsplatform.context.PageData +import org.wikimedia.metricsplatform.context.PerformerData +import org.wikimedia.metricsplatform.event.EventProcessed +import java.time.Instant +import java.util.function.Predicate + +@Serializable +class CurationFilter : Predicate { + @SerialName("agent_app_install_id") var agentAppInstallIdRules: CurationRules? = null + @SerialName("agent_client_platform") var agentClientPlatformRules: CurationRules? = null + @SerialName("agent_client_platform_family") var agentClientPlatformFamilyRules: CurationRules? = null + @SerialName("mediawiki_database") var mediawikiDatabase: CurationRules? = null + @SerialName("page_id") var pageIdRules: ComparableCurationRules? = null + @SerialName("page_namespace_id") var pageNamespaceIdRules: ComparableCurationRules? = null + @SerialName("page_namespace_name") var pageNamespaceNameRules: CurationRules? = null + @SerialName("page_title") var pageTitleRules: CurationRules? = null + @SerialName("page_revision_id") var pageRevisionIdRules: ComparableCurationRules? = null + @SerialName("page_wikidata_qid") var pageWikidataQidRules: CurationRules? = null + @SerialName("page_content_language") var pageContentLanguageRules: CurationRules? = null + @SerialName("performer_id") var performerIdRules: ComparableCurationRules? = null + @SerialName("performer_name") var performerNameRules: CurationRules? = null + @SerialName("performer_session_id") var performerSessionIdRules: CurationRules? = null + @SerialName("performer_pageview_id") var performerPageviewIdRules: CurationRules? = null + @SerialName("performer_groups") var performerGroupsRules: CollectionCurationRules? = null + @SerialName("performer_is_logged_in") var performerIsLoggedInRules: CurationRules? = null + @SerialName("performer_is_temp") var performerIsTempRules: CurationRules? = null + @SerialName("performer_registration_dt") var performerRegistrationDtRules: ComparableCurationRules? = null + + @SerialName("performer_language_groups") + var performerLanguageGroupsRules: CurationRules? = null + + @SerialName("performer_language_primary") + var performerLanguagePrimaryRules: CurationRules? = null + + override fun test(event: EventProcessed): Boolean { + return applyAgentRules(event.agentData) + && applyMediaWikiRules(event.mediawikiData) + && applyPageRules(event.pageData) + && applyPerformerRules(event.performerData) + } + + private fun applyAgentRules(data: AgentData?): Boolean { + return applyPredicate(this.agentAppInstallIdRules, data?.appInstallId) + && applyPredicate(this.agentClientPlatformRules, data?.clientPlatform) + && applyPredicate(this.agentClientPlatformFamilyRules, data?.clientPlatformFamily) + } + + private fun applyMediaWikiRules(data: MediawikiData?): Boolean { + return applyPredicate(this.mediawikiDatabase, data?.database) + } + + private fun applyPageRules(data: PageData?): Boolean { + return applyPredicate(this.pageIdRules, data?.id) + && applyPredicate(this.pageNamespaceIdRules, data?.namespaceId) + && applyPredicate(this.pageNamespaceNameRules, data?.namespaceName) + && applyPredicate(this.pageTitleRules, data?.title) + && applyPredicate(this.pageRevisionIdRules, data?.revisionId) + && applyPredicate(this.pageWikidataQidRules, data?.wikidataItemQid) + && applyPredicate(this.pageContentLanguageRules, data?.contentLanguage) + } + + private fun applyPerformerRules(data: PerformerData?): Boolean { + return applyPredicate(this.performerIdRules, data?.id) + && applyPredicate(this.performerNameRules, data?.name) + && applyPredicate(this.performerSessionIdRules, data?.sessionId) + && applyPredicate(this.performerPageviewIdRules, data?.pageviewId) + && applyPredicate(this.performerGroupsRules, data?.groups) + && applyPredicate(this.performerIsLoggedInRules, data?.isLoggedIn) + && applyPredicate(this.performerIsTempRules, data?.isTemp) + && applyPredicate(this.performerRegistrationDtRules, data?.registrationDt) + && applyPredicate(this.performerLanguageGroupsRules, data?.languageGroups) + && applyPredicate(this.performerLanguagePrimaryRules, data?.languagePrimary) + } + + companion object { + private fun applyPredicate(rules: Predicate?, value: T?): Boolean { + if (rules == null) return true + if (value == null) return false + return rules.test(value) + } + } +} diff --git a/app/src/main/java/org/wikipedia/analytics/eventplatform/DestinationEventService.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/DestinationEventService.kt similarity index 52% rename from app/src/main/java/org/wikipedia/analytics/eventplatform/DestinationEventService.kt rename to analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/DestinationEventService.kt index 02f7c9eb962..6fd93bacd73 100644 --- a/app/src/main/java/org/wikipedia/analytics/eventplatform/DestinationEventService.kt +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/DestinationEventService.kt @@ -1,17 +1,15 @@ -package org.wikipedia.analytics.eventplatform +package org.wikimedia.metricsplatform.config -import org.wikipedia.BuildConfig +import org.wikimedia.metricsplatform.BuildConfig /** * Possible event destination endpoints which can be specified in stream configurations. - * https://wikitech.wikimedia.org/wiki/Event_Platform/EventGate#EventGate_clusters + * For now, we'll assume that we always want to send to ANALYTICS. * - * N.B. Currently our streamconfigs API request is filtering for streams with their destination - * event service configured as eventgate-analytics-external. However, that will likely change in - * the future, so flexible destination event service support is added optimistically now. + * https://wikitech.wikimedia.org/wiki/Event_Platform/EventGate#EventGate_clusters */ enum class DestinationEventService(val id: String, val baseUri: String) { - ANALYTICS("eventgate-analytics-external", BuildConfig.EVENTGATE_ANALYTICS_EXTERNAL_BASE_URI), - LOGGING("eventgate-logging-external", BuildConfig.EVENTGATE_LOGGING_EXTERNAL_BASE_URI) + LOGGING("eventgate-logging-external", BuildConfig.EVENTGATE_LOGGING_EXTERNAL_BASE_URI), + LOCAL("eventgate-logging-local", "http://localhost:8192"); } diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/SourceConfig.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/SourceConfig.kt new file mode 100644 index 00000000000..78be1055211 --- /dev/null +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/SourceConfig.kt @@ -0,0 +1,7 @@ +package org.wikimedia.metricsplatform.config + +class SourceConfig(val streamConfigs: Map) { + fun getStreamConfigByName(streamName: String): StreamConfig? { + return streamConfigs[streamName] + } +} diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfig.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfig.kt new file mode 100644 index 00000000000..89227b1c4e7 --- /dev/null +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfig.kt @@ -0,0 +1,102 @@ +package org.wikimedia.metricsplatform.config + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import kotlinx.serialization.Transient +import org.wikimedia.metricsplatform.config.sampling.SampleConfig + +@Serializable +class StreamConfig { + @SerialName("stream") + var streamName: String = "" + + @SerialName("canary_events_enabled") + var canaryEventsEnabled = false + + @SerialName("destination_event_service") + val destinationEventServiceKey: String = "eventgate-analytics-external" + + @Transient + var destinationEventService: DestinationEventService = DestinationEventService.ANALYTICS + + @SerialName("schema_title") var schemaTitle: String? = null + + @SerialName("producers") var producerConfig: ProducerConfig? = null + + @SerialName("sample") var sampleConfig: SampleConfig? = null + + @SerialName("topic_prefixes") + val topicPrefixes: List = emptyList() + + val topics: List = emptyList() + + fun hasRequestedContextValuesConfig(): Boolean { + return producerConfig?.metricsPlatformClientConfig?.requestedValues != null + } + + val events + get() = producerConfig?.metricsPlatformClientConfig?.events.orEmpty() + + fun hasCurationFilter(): Boolean { + return producerConfig?.metricsPlatformClientConfig?.curationFilter != null + } + + val curationFilter get() = producerConfig?.metricsPlatformClientConfig?.curationFilter ?: CurationFilter() + + init { + try { + destinationEventService = DestinationEventService.valueOf(destinationEventServiceKey) + } catch (_: Exception) { } + } + + @Serializable + class ProducerConfig { + @SerialName("metrics_platform_client") var metricsPlatformClientConfig: MetricsPlatformClientConfig? = null + } + + @Serializable + class MetricsPlatformClientConfig { + @SerialName("events") var events: List? = null + @SerialName("provide_values") var requestedValues: List? = null + @SerialName("curation") var curationFilter: CurationFilter? = null + } + + fun isInterestedInEvent(eventName: String): Boolean { + for (streamEventName in this.events) { + // Match string prefixes for event names of interested streams. + if (eventName.startsWith(streamEventName)) { + return true + } + } + return false + } + + companion object { + /** + * The context attributes that the Metrics Platform Client can add to an event. + */ + val CONTEXTUAL_ATTRIBUTES = arrayOf( + // Agent + "agent_app_install_id", + "agent_client_platform", + "agent_client_platform_family", // Page + "page_id", + "page_title", + "page_namespace_id", + "page_namespace_name", + "page_revision_id", + "page_wikidata_qid", + "page_content_language", // MediaWiki + "mediawiki_database", // Performer + "performer_is_logged_in", + "performer_id", + "performer_name", + "performer_session_id", + "performer_pageview_id", + "performer_groups", + "performer_language_primary", + "performer_language_groups", + "performer_registration_dt", + ) + } +} diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfigCollection.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfigCollection.kt new file mode 100644 index 00000000000..fdcfa025239 --- /dev/null +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfigCollection.kt @@ -0,0 +1,10 @@ +package org.wikimedia.metricsplatform.config + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable + +// TODO: make this inherit from MwResponse, and properly handle MW errors +@Serializable +class StreamConfigCollection { + @SerialName("streams") var streamConfigs: Map = emptyMap() +} diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/curation/CollectionCurationRules.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/curation/CollectionCurationRules.kt new file mode 100644 index 00000000000..e0c31917ca2 --- /dev/null +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/curation/CollectionCurationRules.kt @@ -0,0 +1,24 @@ +package org.wikimedia.metricsplatform.config.curation + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import java.util.function.Predicate + +@Serializable +class CollectionCurationRules : Predicate?> { + var contains: T? = null + + @SerialName("does_not_contain") var doesNotContain: T? = null + + @SerialName("contains_all") var containsAll: Collection? = null + + @SerialName("contains_any") var containsAny: Collection? = null + + override fun test(value: Collection?): Boolean { + if (value == null) return false + return (contains == null || value.contains(contains)) + && (doesNotContain == null || !value.contains(doesNotContain)) + && (containsAll == null || value.containsAll(containsAll!!)) + && (containsAny == null || value.count { v: T? -> containsAny!!.contains(v) } > 0) + } +} diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/curation/ComparableCurationRules.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/curation/ComparableCurationRules.kt new file mode 100644 index 00000000000..632b9808f70 --- /dev/null +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/curation/ComparableCurationRules.kt @@ -0,0 +1,36 @@ +package org.wikimedia.metricsplatform.config.curation + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import java.util.function.Predicate + +@Serializable +class ComparableCurationRules?> : Predicate { + @SerialName("is_equals") var isEquals: T? = null + + @SerialName("not_equals") var isNotEquals: T? = null + + @SerialName("greater_than") var greaterThan: T? = null + + @SerialName("less_than") var lessThan: T? = null + + @SerialName("greater_than_or_equals") var greaterThanOrEquals: T? = null + + @SerialName("less_than_or_equals") var lessThanOrEquals: T? = null + + @SerialName("in") var isIn: MutableCollection? = null + + @SerialName("not_in") var isNotIn: MutableCollection? = null + + override fun test(value: T): Boolean { + if (value == null) return false + return (isEquals == null || isEquals == value) + && (isNotEquals == null || isNotEquals != value) + && (greaterThan == null || greaterThan!!.compareTo(value) < 0) + && (lessThan == null || lessThan!!.compareTo(value) > 0) + && (greaterThanOrEquals == null || greaterThanOrEquals!!.compareTo(value) <= 0) + && (lessThanOrEquals == null || lessThanOrEquals!!.compareTo(value) >= 0) + && (isIn == null || isIn!!.contains(value)) + && (isNotIn == null || !isNotIn!!.contains(value)) + } +} diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/curation/CurationRules.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/curation/CurationRules.kt new file mode 100644 index 00000000000..7a015958ea6 --- /dev/null +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/curation/CurationRules.kt @@ -0,0 +1,24 @@ +package org.wikimedia.metricsplatform.config.curation + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import java.util.function.Predicate + +@Serializable +class CurationRules : Predicate { + @SerialName("equals") var isEquals: T? = null + + @SerialName("not_equals") var isNotEquals: T? = null + + @SerialName("in") var isIn: MutableCollection? = null + + @SerialName("not_in") var isNotIn: MutableCollection? = null + + override fun test(value: T?): Boolean { + if (value == null) return false + return (isEquals == null || isEquals == value) + && (isNotEquals == null || isNotEquals != value) + && (isIn == null || isIn!!.contains(value)) + && (isNotIn == null || !isNotIn!!.contains(value)) + } +} diff --git a/app/src/main/java/org/wikipedia/analytics/eventplatform/SamplingConfig.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/sampling/SampleConfig.kt similarity index 50% rename from app/src/main/java/org/wikipedia/analytics/eventplatform/SamplingConfig.kt rename to analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/sampling/SampleConfig.kt index ac925493d85..2e79a09225e 100644 --- a/app/src/main/java/org/wikipedia/analytics/eventplatform/SamplingConfig.kt +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/sampling/SampleConfig.kt @@ -1,14 +1,11 @@ -package org.wikipedia.analytics.eventplatform +package org.wikimedia.metricsplatform.config.sampling import kotlinx.serialization.Serializable -/** - * Represents the sampling config component of a stream configuration. - */ @Serializable -class SamplingConfig( - val rate: Double = 1.0, - val unit: String = UNIT_SESSION +class SampleConfig( + val rate: Double = 1.0, + val unit: String = UNIT_SESSION ) { companion object { const val UNIT_PAGEVIEW = "pageview" diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/AgentData.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/AgentData.kt new file mode 100644 index 00000000000..f492a5969a9 --- /dev/null +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/AgentData.kt @@ -0,0 +1,24 @@ +package org.wikimedia.metricsplatform.context + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable + +/** + * Agent context data fields. + * + * All fields are nullable, and boxed types are used in place of their equivalent primitive types to avoid + * unexpected default values from being used where the true value is null. + */ +@Serializable +class AgentData( + @SerialName("app_flavor") var appFlavor: String? = null, + @SerialName("app_install_id") var appInstallId: String? = null, + @SerialName("app_theme") var appTheme: String? = null, + @SerialName("app_version") var appVersion: Int? = null, + @SerialName("app_version_name") var appVersionName: String? = null, + @SerialName("client_platform") var clientPlatform: String? = null, + @SerialName("client_platform_family") var clientPlatformFamily: String? = null, + @SerialName("device_family") var deviceFamily: String? = null, + @SerialName("device_language") var deviceLanguage: String? = null, + @SerialName("release_status") var releaseStatus: String? = null, +) diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/ClientData.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/ClientData.kt new file mode 100644 index 00000000000..c93092a1c48 --- /dev/null +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/ClientData.kt @@ -0,0 +1,22 @@ +package org.wikimedia.metricsplatform.context + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable + +/** + * Client metadata context fields. + * + * ClientData includes immutable and mutable contextual data from the client. + * This metadata is added to every event submission when queued for processing. + * + * All fields of nested data objects are nullable, and boxed types are used in place of their equivalent primitive types + * to avoid unexpected default values from being used where the true value is null. + */ +@Serializable +open class ClientData ( + @SerialName("agent") val agentData: AgentData? = null, + @SerialName("page") val pageData: PageData? = null, + @SerialName("mediawiki") val mediawikiData: MediawikiData? = null, + @SerialName("performer") val performerData: PerformerData? = null, + @SerialName("domain") val domain: String? = null +) diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/ContextValue.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/ContextValue.kt new file mode 100644 index 00000000000..3c424ccda87 --- /dev/null +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/ContextValue.kt @@ -0,0 +1,38 @@ +package org.wikimedia.metricsplatform.context + +/** + * @see [Metrics Platform/Contextual attributes](https://wikitech.wikimedia.org/wiki/Metrics_Platform/Contextual_attributes) + */ +object ContextValue { + const val AGENT_APP_INSTALL_ID: String = "agent_app_install_id" + const val AGENT_CLIENT_PLATFORM: String = "agent_client_platform" + const val AGENT_CLIENT_PLATFORM_FAMILY: String = "agent_client_platform_family" + const val AGENT_APP_FLAVOR: String = "agent_app_flavor" + const val AGENT_APP_THEME: String = "agent_app_theme" + const val AGENT_APP_VERSION: String = "agent_app_version" + const val AGENT_APP_VERSION_NAME: String = "agent_app_version_name" + const val AGENT_DEVICE_FAMILY: String = "agent_device_family" + const val AGENT_DEVICE_LANGUAGE: String = "agent_device_language" + const val AGENT_RELEASE_STATUS: String = "agent_release_status" + + const val MEDIAWIKI_DATABASE: String = "mediawiki_database" + + const val PAGE_ID: String = "page_id" + const val PAGE_TITLE: String = "page_title" + const val PAGE_NAMESPACE_ID: String = "page_namespace_id" + const val PAGE_NAMESPACE_NAME: String = "page_namespace_name" + const val PAGE_REVISION_ID: String = "page_revision_id" + const val PAGE_WIKIDATA_QID: String = "page_wikidata_qid" + const val PAGE_CONTENT_LANGUAGE: String = "page_content_language" + + const val PERFORMER_ID: String = "performer_id" + const val PERFORMER_NAME: String = "performer_name" + const val PERFORMER_IS_LOGGED_IN: String = "performer_is_logged_in" + const val PERFORMER_IS_TEMP: String = "performer_is_temp" + const val PERFORMER_SESSION_ID: String = "performer_session_id" + const val PERFORMER_PAGEVIEW_ID: String = "performer_pageview_id" + const val PERFORMER_GROUPS: String = "performer_groups" + const val PERFORMER_LANGUAGE_GROUPS: String = "performer_language_groups" + const val PERFORMER_LANGUAGE_PRIMARY: String = "performer_language_primary" + const val PERFORMER_REGISTRATION_DT: String = "performer_registration_dt" +} diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/CustomData.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/CustomData.kt new file mode 100644 index 00000000000..0d069e1e5cd --- /dev/null +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/CustomData.kt @@ -0,0 +1,33 @@ +package org.wikimedia.metricsplatform.context + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable + +@Serializable +class CustomData ( + @SerialName("data_type") val type: CustomDataType? = null, + val value: String? = null +) { + companion object { + /** + * Return custom data, based on a generic object. + * + * @return formatted custom data + */ + fun of(value: Any?): CustomData { + var formattedValue = value.toString() + val formattedType: CustomDataType? + + if (value is Number) { + formattedType = CustomDataType.NUMBER + } else if (value is Boolean) { + formattedType = CustomDataType.BOOLEAN + formattedValue = if (value) "true" else "false" + } else { + formattedType = CustomDataType.STRING + } + + return CustomData(formattedType, formattedValue) + } + } +} diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/CustomDataType.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/CustomDataType.kt new file mode 100644 index 00000000000..5517db9d47f --- /dev/null +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/CustomDataType.kt @@ -0,0 +1,10 @@ +package org.wikimedia.metricsplatform.context + +import kotlinx.serialization.SerialName + +enum class CustomDataType { + @SerialName("number") NUMBER, + @SerialName("string") STRING, + @SerialName("boolean") BOOLEAN, + @SerialName("null") NULL +} diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/InstantSerializer.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/InstantSerializer.kt new file mode 100644 index 00000000000..89ea029c01a --- /dev/null +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/InstantSerializer.kt @@ -0,0 +1,16 @@ +package org.wikimedia.metricsplatform.context + +import kotlinx.serialization.KSerializer +import kotlinx.serialization.descriptors.PrimitiveKind +import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor +import kotlinx.serialization.descriptors.SerialDescriptor +import kotlinx.serialization.encoding.Decoder +import kotlinx.serialization.encoding.Encoder +import java.time.Instant + +object InstantSerializer : KSerializer { + override val descriptor: SerialDescriptor = + PrimitiveSerialDescriptor("java.time.Instant", PrimitiveKind.STRING) + override fun serialize(encoder: Encoder, value: Instant) = encoder.encodeString(value.toString()) + override fun deserialize(decoder: Decoder): Instant = Instant.parse(decoder.decodeString()) +} diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/InteractionData.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/InteractionData.kt new file mode 100644 index 00000000000..8cbad584d43 --- /dev/null +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/InteractionData.kt @@ -0,0 +1,21 @@ +package org.wikimedia.metricsplatform.context + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable + +/** + * Interaction data fields. + * + * Common interaction fields that describe the event being submitted. Most fields are nullable. + */ +@Serializable +class InteractionData( + @SerialName("action") val action: String? = null, + @SerialName("action_subtype") val actionSubtype: String? = null, + @SerialName("action_source") val actionSource: String? = null, + @SerialName("action_context") val actionContext: String? = null, + @SerialName("element_id") val elementId: String? = null, + @SerialName("element_friendly_name") val elementFriendlyName: String? = null, + @SerialName("funnel_entry_token") val funnelEntryToken: String? = null, + @SerialName("funnel_event_sequence_position") val funnelEventSequencePosition: Int? = null +) diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/MediawikiData.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/MediawikiData.kt new file mode 100644 index 00000000000..d1b3957e613 --- /dev/null +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/MediawikiData.kt @@ -0,0 +1,15 @@ +package org.wikimedia.metricsplatform.context + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable + +/** + * Mediawiki context data fields. + * + * All fields are nullable, and boxed types are used in place of their equivalent primitive types to avoid + * unexpected default values from being used where the true value is null. + */ +@Serializable +class MediawikiData( + @SerialName("database") var database: String? = null +) diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/PageData.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/PageData.kt new file mode 100644 index 00000000000..f67c9a32b17 --- /dev/null +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/PageData.kt @@ -0,0 +1,24 @@ +package org.wikimedia.metricsplatform.context + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable + +/** + * Page context data context fields. + * + * PageData are dynamic context fields that change with every request. PageData is submitted with each event + * by the client and then queued for processing by EventProcessor. + * + * All fields are nullable, and boxed types are used in place of their equivalent primitive types to avoid + * unexpected default values from being used where the true value is null. + */ +@Serializable +class PageData( + var id: Int? = null, + var title: String? = null, + @SerialName("namespace_id") var namespaceId: Int? = null, + @SerialName("namespace_name") var namespaceName: String? = null, + @SerialName("revision_id") var revisionId: Long? = null, + @SerialName("wikidata_qid") var wikidataItemQid: String? = null, + @SerialName("content_language") var contentLanguage: String? = null, +) diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/PerformerData.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/PerformerData.kt new file mode 100644 index 00000000000..3cf6b7a5397 --- /dev/null +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/PerformerData.kt @@ -0,0 +1,28 @@ +@file:UseSerializers(InstantSerializer::class) + +package org.wikimedia.metricsplatform.context + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import kotlinx.serialization.UseSerializers +import java.time.Instant + +/** + * Performer context data fields. + * + * All fields are nullable, and boxed types are used in place of their equivalent primitive types to avoid + * unexpected default values from being used where the true value is null. + */ +@Serializable +data class PerformerData ( + var id: Int? = null, + @SerialName("name") var name: String? = null, + @SerialName("is_logged_in") var isLoggedIn: Boolean? = null, + @SerialName("is_temp") var isTemp: Boolean? = null, + @SerialName("session_id") var sessionId: String? = null, + @SerialName("pageview_id") var pageviewId: String? = null, + @SerialName("groups") var groups: Collection? = null, + @SerialName("language_groups") var languageGroups: String? = null, + @SerialName("language_primary") var languagePrimary: String? = null, + @SerialName("registration_dt") var registrationDt: Instant? = null +) diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/event/Event.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/event/Event.kt new file mode 100644 index 00000000000..c4099a14d86 --- /dev/null +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/event/Event.kt @@ -0,0 +1,42 @@ +package org.wikimedia.metricsplatform.event + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import kotlinx.serialization.Transient +import org.wikimedia.metricsplatform.config.sampling.SampleConfig +import org.wikimedia.metricsplatform.context.ClientData +import org.wikimedia.metricsplatform.context.InteractionData + +@Serializable +open class Event(@Transient val _stream: String = "") { + var name: String? = null + + @SerialName("\$schema") + var schema: String = "" + + @SerialName("dt") var timestamp: String? = null + + @SerialName("custom_data") var customData: Map? = null + + protected val meta = Meta(_stream) + + @SerialName("client_data") var clientData: ClientData = ClientData() + + @SerialName("sample") var sample: SampleConfig? = null + + @SerialName("interaction_data") var interactionData: InteractionData = InteractionData() + + var stream + get() = meta.stream + set(value) { meta.stream = value } + + fun setDomain(domain: String?) { + meta.domain = domain + } + + @Serializable + protected class Meta( + var stream: String = "", + var domain: String? = null + ) +} diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/event/EventProcessed.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/event/EventProcessed.kt new file mode 100644 index 00000000000..3bbff824312 --- /dev/null +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/event/EventProcessed.kt @@ -0,0 +1,120 @@ +package org.wikimedia.metricsplatform.event + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import org.wikimedia.metricsplatform.config.sampling.SampleConfig +import org.wikimedia.metricsplatform.context.AgentData +import org.wikimedia.metricsplatform.context.ClientData +import org.wikimedia.metricsplatform.context.InteractionData +import org.wikimedia.metricsplatform.context.MediawikiData +import org.wikimedia.metricsplatform.context.PageData +import org.wikimedia.metricsplatform.context.PerformerData + +@Serializable +class EventProcessed : Event { + @SerialName("agent") var agentData: AgentData? = null + @SerialName("page") var pageData: PageData? = null + @SerialName("mediawiki") var mediawikiData: MediawikiData? = null + @SerialName("performer") var performerData: PerformerData? = null + + @SerialName("action") var action: String? = null + @SerialName("action_subtype") private var actionSubtype: String? = null + @SerialName("action_source") private var actionSource: String? = null + @SerialName("action_context") private var actionContext: String? = null + @SerialName("element_id") private var elementId: String? = null + @SerialName("element_friendly_name") private var elementFriendlyName: String? = null + @SerialName("funnel_entry_token") private var funnelEntryToken: String? = null + @SerialName("funnel_event_sequence_position") private var funnelEventSequencePosition: Int? = null + + /** + * Constructor for EventProcessed. + * + * @param schema schema id + * @param stream stream name + * @param name event name + * @param clientData agent, mediawiki, page, performer data + */ + constructor(schema: String, stream: String, name: String, clientData: ClientData) : super(stream) { + this.schema = schema + this.name = name + this.clientData = clientData + this.agentData = clientData.agentData + this.pageData = clientData.pageData + this.mediawikiData = clientData.mediawikiData + this.performerData = clientData.performerData + this.action = name + } + + /** + * Constructor for EventProcessed. + * + * @param schema schema id + * @param stream stream name + * @param name event name + * @param customData custom data + * @param clientData agent, mediawiki, page, performer data + * @param sample sample configuration + * @param interactionData contextual data of the interaction + * + * + * Although 'setInteractionData()' sets the 'action' property for the event, + * because 'action' is a nonnull property for both 'EventProcessed' and the + * 'InteractionData' data object, removing the redundant setting of 'action' + * triggers NP_NONNULL_FIELD_NOT_INITIALIZED_IN_CONSTRUCTOR in spotbugs so + * leaving it as is for the time being rather than suppressing the error. + */ + constructor( + schema: String, + stream: String, + name: String?, + customData: Map?, + clientData: ClientData, + sample: SampleConfig?, + interactionData: InteractionData + ) : super(stream) { + this.schema = schema + this.name = name + this.clientData = clientData + this.agentData = clientData.agentData + this.pageData = clientData.pageData + this.mediawikiData = clientData.mediawikiData + this.performerData = clientData.performerData + this.customData = customData + this.sample = sample + this.interactionData = interactionData + this.action = interactionData.action + } + + fun applyClientData(clientData: ClientData) { + agentData = clientData.agentData + pageData = clientData.pageData + mediawikiData = clientData.mediawikiData + performerData = clientData.performerData + this.clientData = clientData + } + + fun applyInteractionData(interactionData: InteractionData) { + this.action = interactionData.action + this.actionContext = interactionData.actionContext + this.actionSource = interactionData.actionSource + this.actionSubtype = interactionData.actionSubtype + this.elementId = interactionData.elementId + this.elementFriendlyName = interactionData.elementFriendlyName + this.funnelEntryToken = interactionData.funnelEntryToken + this.funnelEventSequencePosition = interactionData.funnelEventSequencePosition + } + + companion object { + fun fromEvent(event: Event): EventProcessed { + return EventProcessed( + event.schema, + event.stream, + event.name, + event.customData, + event.clientData, + event.sample, + event.interactionData + ) + } + } +} diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/event/EventProcessedSerializer.java b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/event/EventProcessedSerializer.java new file mode 100644 index 00000000000..015c68c4f11 --- /dev/null +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/event/EventProcessedSerializer.java @@ -0,0 +1,66 @@ +package org.wikimedia.metricsplatform.event; + +public class EventProcessedSerializer { + /* + private final Gson gson = new GsonBuilder() + .registerTypeAdapter(Instant.class, new InstantConverter()) + .create(); + + @Override + public JsonElement serialize(EventProcessed src, Type type, JsonSerializationContext jsonSerializationContext) { + JsonElement jsonElement = gson.toJsonTree(src); + JsonObject jsonObject = (JsonObject) jsonElement; + + * Custom data can be passed into the EventProcessed constructor as a map + * of key-value pairs. EventProcessed inherits from Event which contains a + * customData property that would be serialized by default. To validate + * successfully against the corresponding schema, custom data must be sent + * as top-level properties with the event. + + if (src.customData != null) { + int customDataAdded = 0; + int customDataCount = src.customData.size(); + for (Map.Entry entry : src.customData.entrySet()) { + if (entry.getValue() instanceof Number) + jsonObject.addProperty(entry.getKey(), (Number) entry.getValue()); + if (entry.getValue() instanceof String) + jsonObject.addProperty(entry.getKey(), entry.getValue().toString()); + if (entry.getValue() instanceof Boolean) + jsonObject.addProperty(entry.getKey(), (Boolean) entry.getValue()); + + if (jsonObject.has(entry.getKey())) + customDataAdded++; + } + + if (customDataAdded != customDataCount) { + log.log(INFO, "Only " + customDataAdded + " custom data key-value pairs were serialized " + + "but there are " + customDataCount + " total custom data items for this event."); + } + + jsonObject.remove("custom_data"); + jsonObject.remove("client_data"); + } + + * Removing a few properties here because it would require adding annotations on every Event and EventProcessed + * property just to exclude few properties from serialization. + * + * The "name" property is an Event property that eventually gets submitted as "action" with an event, but it is + * not included in the Metrics Platform base interactions schemas. Because the InteractionData data object can + * be a null parameter in the MetricsClient::submitMetricsEvent method, the "name" property is needed when + * clients send events without InteractionData (see @ToDo add link to MP schemas once they are merged). + * + * Once Metrics Platform core interactions schemas are updated to include the "sample" property, the line to + * remove it can be deleted here. + jsonObject.remove("name"); + jsonObject.remove("sample"); + + * Remove the top level data objects from EventProcessed which are + * inherited from its superclass Event. The values in "client_data" + * and "interaction_data" are set as top level properties in + * EventProcessed's constructor. + jsonObject.remove("client_data"); + jsonObject.remove("interaction_data"); + return jsonObject; + } + */ +} diff --git a/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/ConsistencyIT.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/ConsistencyIT.java new file mode 100644 index 00000000000..bfbabe25fdb --- /dev/null +++ b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/ConsistencyIT.java @@ -0,0 +1,143 @@ +package org.wikimedia.metricsplatform; + +import static java.util.Collections.singletonMap; +import static org.assertj.core.api.Assertions.assertThat; +import static org.wikimedia.metricsplatform.ConsistencyITClientData.createConsistencyTestClientData; +import static org.wikimedia.metricsplatform.config.StreamConfigFetcher.ANALYTICS_API_ENDPOINT; +import static org.wikimedia.metricsplatform.MetricsClient.METRICS_PLATFORM_SCHEMA_BASE; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.Reader; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicReference; + +import org.junit.jupiter.api.Test; +import org.wikimedia.metricsplatform.config.SourceConfig; +import org.wikimedia.metricsplatform.config.StreamConfig; +import org.wikimedia.metricsplatform.config.StreamConfigFetcher; +import org.wikimedia.metricsplatform.context.ClientData; +import org.wikimedia.metricsplatform.context.DataFixtures; +import org.wikimedia.metricsplatform.event.EventProcessed; +import org.wikimedia.metricsplatform.json.GsonHelper; + +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +import okhttp3.OkHttpClient; + +class ConsistencyIT { + private JsonObject expectedEvent; + + @Test void testConsistency() throws IOException { + Path pathStreamConfigs = Paths.get("../tests/consistency/stream_configs_apps.json"); + + try (BufferedReader reader = Files.newBufferedReader(pathStreamConfigs)) { + + // Init shared test config + variables for creating new metrics client + event processor. + Map testStreamConfigs = getTestStreamConfigs(reader); + SourceConfig sourceConfig = new SourceConfig(testStreamConfigs); + AtomicReference sourceConfigRef = new AtomicReference<>(); + sourceConfigRef.set(sourceConfig); + BlockingQueue eventQueue = new LinkedBlockingQueue<>(10); + ClientData consistencyTestClientData = createConsistencyTestClientData(); + SamplingController samplingController = new SamplingController(consistencyTestClientData, new SessionController()); + + EventProcessor consistencyTestEventProcessor = getTestEventProcessor( + sourceConfigRef, + samplingController, + eventQueue + ); + + MetricsClient consistencyTestMetricsClient = getTestMetricsClient( + consistencyTestClientData, + sourceConfigRef, + eventQueue + ); + + consistencyTestMetricsClient.submitMetricsEvent( + "test.consistency", + METRICS_PLATFORM_SCHEMA_BASE, + "test_consistency_event", + DataFixtures.getTestClientData(getExpectedEventJson().toString()), + singletonMap("test", "consistency") + ); + + EventProcessed queuedEvent = eventQueue.peek(); + consistencyTestEventProcessor.eventPassesCurationRules(queuedEvent, testStreamConfigs); + + // Adjust the queuedEvent and compare it against the expected event. + if (this.expectedEvent != null) { + Gson gson = GsonHelper.getGson(); + String queuedEventJsonStringRaw = gson.toJson(queuedEvent); + JsonObject queuedEventJsonObject = JsonParser.parseString(queuedEventJsonStringRaw).getAsJsonObject(); + // Remove the timestamp properties from the queued event to match the expected event json. + removeExtraProperties(queuedEventJsonObject); + + assertThat(queuedEventJsonObject) + .usingRecursiveComparison() + .ignoringCollectionOrder() + .isEqualTo(this.expectedEvent); + } + } + } + + private static MetricsClient getTestMetricsClient( + ClientData consistencyTestClientData, + AtomicReference sourceConfigRef, + BlockingQueue eventQueue + ) { + return MetricsClient.builder(consistencyTestClientData) + .sourceConfigRef(sourceConfigRef) + .eventQueue(eventQueue) + .build(); + } + + private static EventProcessor getTestEventProcessor( + AtomicReference sourceConfigRef, + SamplingController samplingController, + BlockingQueue eventQueue + ) { + ContextController contextController = new ContextController(); + CurationController curationController = new CurationController(); + EventSender eventSender = new TestEventSender(); + return new EventProcessor( + contextController, + curationController, + sourceConfigRef, + samplingController, + eventSender, + eventQueue, + true + ); + } + + private static Map getTestStreamConfigs(Reader reader) throws MalformedURLException { + StreamConfigFetcher streamConfigFetcher = new StreamConfigFetcher(new URL(ANALYTICS_API_ENDPOINT), new OkHttpClient(), GsonHelper.getGson()); + return streamConfigFetcher.parseConfig(reader); + } + + private static void removeExtraProperties(JsonObject eventJsonObject) { + eventJsonObject.remove("dt"); + eventJsonObject.remove("sample"); + eventJsonObject.getAsJsonObject("performer").remove("registration_dt"); + } + + private JsonObject getExpectedEventJson() throws IOException { + if (this.expectedEvent == null) { + Path pathExpectedEvent = Paths.get("../tests/consistency/expected_event_apps.json"); + try (BufferedReader expectedEventReader = Files.newBufferedReader(pathExpectedEvent)) { + this.expectedEvent = JsonParser.parseReader(expectedEventReader).getAsJsonObject(); + } + } + return this.expectedEvent; + } +} diff --git a/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/ConsistencyITClientData.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/ConsistencyITClientData.java new file mode 100644 index 00000000000..5f02ad69668 --- /dev/null +++ b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/ConsistencyITClientData.java @@ -0,0 +1,102 @@ +package org.wikimedia.metricsplatform; + +import java.io.BufferedReader; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collections; + +import org.wikimedia.metricsplatform.context.AgentData; +import org.wikimedia.metricsplatform.context.ClientData; +import org.wikimedia.metricsplatform.context.MediawikiData; +import org.wikimedia.metricsplatform.context.PageData; +import org.wikimedia.metricsplatform.context.PerformerData; + +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +public class ConsistencyITClientData extends ClientData { + public JsonObject agentJson; + public JsonObject pageJson; + public JsonObject mediawikiJson; + public JsonObject performerJson; + + public ConsistencyITClientData( + JsonObject agent, + JsonObject page, + JsonObject mediawiki, + JsonObject performer, + String domain + ) { + this.agentJson = agent; + this.pageJson = page; + this.mediawikiJson = mediawiki; + this.performerJson = performer; + + AgentData agentData = AgentData.builder() + .appInstallId(this.agentJson.get("app_install_id").getAsString()) + .clientPlatform(this.agentJson.get("client_platform").getAsString()) + .clientPlatformFamily(this.agentJson.get("client_platform_family").getAsString()) + .deviceFamily(this.agentJson.get("device_family").getAsString()) + .build(); + + PageData pageData = PageData.builder() + .id(this.pageJson.get("id").getAsInt()) + .title(this.pageJson.get("title").getAsString()) + .namespaceId(this.pageJson.get("namespace_id").getAsInt()) + .namespaceName(this.pageJson.get("namespace_name").getAsString()) + .revisionId(this.pageJson.get("revision_id").getAsLong()) + .wikidataItemQid(this.pageJson.get("wikidata_qid").getAsString()) + .contentLanguage(this.pageJson.get("content_language").getAsString()) + .build(); + MediawikiData mediawikiData = MediawikiData.builder() + .database(this.mediawikiJson.get("database").getAsString()) + .build(); + PerformerData performerData = PerformerData.builder() + .id(this.performerJson.get("id").getAsInt()) + .isLoggedIn(this.performerJson.get("is_logged_in").getAsBoolean()) + .sessionId(this.performerJson.get("session_id").getAsString()) + .pageviewId(this.performerJson.get("pageview_id").getAsString()) + .groups(Collections.singleton(this.performerJson.get("groups").getAsString())) + .languagePrimary(this.performerJson.get("language_primary").getAsString()) + .languageGroups(this.performerJson.get("language_groups").getAsString()) + .build(); + + this.setAgentData(agentData); + this.setPageData(pageData); + this.setMediawikiData(mediawikiData); + this.setPerformerData(performerData); + this.setDomain(domain); + } + + public static ConsistencyITClientData createConsistencyTestClientData() { + try { + JsonObject data = getIntegrationData(); + JsonObject agent = data.getAsJsonObject("agent"); + JsonObject page = data.getAsJsonObject("page"); + JsonObject mediawiki = data.getAsJsonObject("mediawiki"); + JsonObject performer = data.getAsJsonObject("performer"); + String domain = data.get("hostname").getAsString(); + + return new ConsistencyITClientData( + agent, + page, + mediawiki, + performer, + domain + ); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static JsonObject getIntegrationData() throws IOException { + Path pathIntegration = Paths.get("../tests/consistency/integration_data_apps.json"); + try (BufferedReader reader = Files.newBufferedReader(pathIntegration)) { + JsonElement jsonElement = JsonParser.parseReader(reader); + return jsonElement.getAsJsonObject(); + } + } +} diff --git a/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/ContextControllerTest.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/ContextControllerTest.java new file mode 100644 index 00000000000..7e817e31f49 --- /dev/null +++ b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/ContextControllerTest.java @@ -0,0 +1,64 @@ +package org.wikimedia.metricsplatform; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.wikimedia.metricsplatform.event.EventProcessed.fromEvent; + +import org.junit.jupiter.api.Test; +import org.wikimedia.metricsplatform.config.StreamConfig; +import org.wikimedia.metricsplatform.config.StreamConfigFixtures; +import org.wikimedia.metricsplatform.context.AgentData; +import org.wikimedia.metricsplatform.context.ClientData; +import org.wikimedia.metricsplatform.context.DataFixtures; +import org.wikimedia.metricsplatform.context.MediawikiData; +import org.wikimedia.metricsplatform.context.PageData; +import org.wikimedia.metricsplatform.context.PerformerData; +import org.wikimedia.metricsplatform.event.Event; +import org.wikimedia.metricsplatform.event.EventProcessed; + +class ContextControllerTest { + + @Test void testAddRequestedValues() { + ContextController contextController = new ContextController(); + Event eventBasic = new Event("test/event", "test.stream", "testEvent"); + EventProcessed event = fromEvent(eventBasic); + ClientData clientDataSample = DataFixtures.getTestClientData(); + event.setClientData(clientDataSample); + StreamConfig streamConfig = StreamConfigFixtures.STREAM_CONFIGS_WITH_EVENTS.get("test.stream"); + contextController.enrichEvent(event, streamConfig); + + AgentData agentData = event.getAgentData(); + MediawikiData mediawikiData = event.getMediawikiData(); + PageData pageData = event.getPageData(); + PerformerData performerData = event.getPerformerData(); + + assertThat(agentData.getAppFlavor()).isEqualTo("devdebug"); + assertThat(agentData.getAppInstallId()).isEqualTo("ffffffff-ffff-ffff-ffff-ffffffffffff"); + assertThat(agentData.getAppTheme()).isEqualTo("LIGHT"); + assertThat(agentData.getAppVersion()).isEqualTo(982734); + assertThat(agentData.getAppVersionName()).isEqualTo("2.7.50470-dev-2024-02-14"); + assertThat(agentData.getClientPlatform()).isEqualTo("android"); + assertThat(agentData.getClientPlatformFamily()).isEqualTo("app"); + assertThat(agentData.getDeviceFamily()).isEqualTo("Samsung SM-G960F"); + assertThat(agentData.getDeviceLanguage()).isEqualTo("en"); + assertThat(agentData.getReleaseStatus()).isEqualTo("dev"); + + assertThat(mediawikiData.getDatabase()).isEqualTo("enwiki"); + + assertThat(pageData.getId()).isEqualTo(1); + assertThat(pageData.getNamespaceId()).isEqualTo(0); + assertThat(pageData.getNamespaceName()).isEqualTo("Main"); + assertThat(pageData.getTitle()).isEqualTo("Test Page Title"); + assertThat(pageData.getRevisionId()).isEqualTo(1L); + assertThat(pageData.getContentLanguage()).isEqualTo("en"); + assertThat(pageData.getWikidataItemQid()).isEqualTo("Q123456"); + + assertThat(performerData.getId()).isEqualTo(1); + assertThat(performerData.getIsLoggedIn()).isTrue(); + assertThat(performerData.getIsTemp()).isFalse(); + assertThat(performerData.getName()).isEqualTo("TestPerformer"); + assertThat(performerData.getGroups()).containsExactly("*"); + assertThat(performerData.getRegistrationDt()).isEqualTo("2023-03-01T01:08:30Z"); + assertThat(performerData.getLanguageGroups()).isEqualTo("zh, en"); + assertThat(performerData.getLanguagePrimary()).isEqualTo("zh-tw"); + } +} diff --git a/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/CurationControllerTest.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/CurationControllerTest.java new file mode 100644 index 00000000000..20ce5c3b602 --- /dev/null +++ b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/CurationControllerTest.java @@ -0,0 +1,102 @@ +package org.wikimedia.metricsplatform; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.wikimedia.metricsplatform.event.EventFixtures.getEvent; + +import java.time.Instant; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.wikimedia.metricsplatform.config.StreamConfig; +import org.wikimedia.metricsplatform.config.StreamConfigFixtures; +import org.wikimedia.metricsplatform.context.PerformerData; +import org.wikimedia.metricsplatform.json.GsonHelper; +import org.wikimedia.metricsplatform.config.CurationFilter; +import org.wikimedia.metricsplatform.event.EventProcessed; + +import com.google.gson.Gson; + +class CurationControllerTest { + + private static StreamConfig streamConfig; + + private static CurationController curationController = new CurationController(); + + private static final List groups = Arrays.asList("user", "autoconfirmed", "steward"); + + @BeforeAll static void setUp() { + Gson gson = GsonHelper.getGson(); + String curationFilterJson = "{\"page_id\":{\"less_than\":500,\"not_equals\":42},\"page_namespace_name\":" + + "{\"equals\":\"Talk\"},\"performer_is_logged_in\":{\"equals\":true},\"performer_edit_count_bucket\":" + + "{\"in\":[\"100-999 edits\",\"1000+ edits\"]},\"performer_groups\":{\"contains_all\":" + + "[\"user\",\"autoconfirmed\"],\"does_not_contain\":\"sysop\"}}"; + CurationFilter curationFilter = gson.fromJson(curationFilterJson, CurationFilter.class); + + streamConfig = StreamConfigFixtures.streamConfig(curationFilter); + } + + @Test void testEventPasses() { + assertThat(curationController.shouldProduceEvent(getEvent(), streamConfig)).isTrue(); + } + + @Test void testEventFailsWrongPageId() { + EventProcessed event = getEvent(42, "Talk", groups, true, "1000+ edits"); + assertThat(curationController.shouldProduceEvent(event, streamConfig)).isFalse(); + } + + @Test void testEventFailsWrongPageNamespaceText() { + EventProcessed event = getEvent(1, "User", groups, true, "1000+ edits"); + assertThat(curationController.shouldProduceEvent(event, streamConfig)).isFalse(); + } + + @Test void testEventFailsWrongUserGroups() { + List wrongGroups = Arrays.asList("user", "autoconfirmed", "sysop"); + EventProcessed event = getEvent(1, "Talk", wrongGroups, true, "1000+ edits"); + assertThat(curationController.shouldProduceEvent(event, streamConfig)).isFalse(); + } + + @Test void testEventFailsNoUserGroups() { + EventProcessed event = getEvent(1, "Talk", Collections.emptyList(), true, "1000+ edits"); + assertThat(curationController.shouldProduceEvent(event, streamConfig)).isFalse(); + } + + @Test void testEventFailsNotLoggedIn() { + EventProcessed event = getEvent(1, "Talk", groups, false, "1000+ edits"); + assertThat(curationController.shouldProduceEvent(event, streamConfig)).isFalse(); + } + + @Test void testEventPassesPerformerRegistrationDtDeserializes() { + EventProcessed event = getEvent(); + event.setPerformerData( + PerformerData.builder() + .groups(groups) + .isLoggedIn(true) + .registrationDt(Instant.parse("2023-03-01T01:08:30Z")) + .build() + ); + assertThat(curationController.shouldProduceEvent(event, streamConfig)).isTrue(); + } + + @Test void testEventPassesCurationFilters() { + EventProcessed event = getEvent(1, "Talk", groups, true, "1000+ edits"); + assertThat(curationController.shouldProduceEvent(event, streamConfig)).isTrue(); + } + + @Test void testEventFailsEqualsRule() { + EventProcessed event = getEvent(1, "Main", groups, true, "1000+ edits"); + assertThat(curationController.shouldProduceEvent(event, streamConfig)).isFalse(); + } + + @Test void testEventFailsCollectionContainsAnyRule() { + EventProcessed event = getEvent(1, "Talk", Collections.singletonList("*"), true, "1000+ edits"); + assertThat(curationController.shouldProduceEvent(event, streamConfig)).isFalse(); + } + + @Test void testEventFailsCollectionDoesNotContainRule() { + EventProcessed event = getEvent(1, "Talk", Arrays.asList("foo", "bar"), true, "1000+ edits"); + assertThat(curationController.shouldProduceEvent(event, streamConfig)).isFalse(); + } +} diff --git a/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/DestinationEventServiceTest.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/DestinationEventServiceTest.java new file mode 100644 index 00000000000..619f6834a73 --- /dev/null +++ b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/DestinationEventServiceTest.java @@ -0,0 +1,42 @@ +package org.wikimedia.metricsplatform; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.net.MalformedURLException; +import java.net.URL; + +import org.junit.jupiter.api.Test; +import org.wikimedia.metricsplatform.config.DestinationEventService; + +class DestinationEventServiceTest { + + @Test void testDestinationEventServiceAnalytics() throws MalformedURLException { + DestinationEventService loggingService = DestinationEventService.ANALYTICS; + assertThat(loggingService.getBaseUri()).isEqualTo(new URL("https://intake-analytics.wikimedia.org/v1/events?hasty=true")); + } + + @Test void testDestinationEventServiceLogging() throws MalformedURLException { + DestinationEventService loggingService = DestinationEventService.ERROR_LOGGING; + assertThat(loggingService.getBaseUri()).isEqualTo(new URL("https://intake-logging.wikimedia.org/v1/events?hasty=true")); + } + + @Test void testDestinationEventServiceLocal() throws MalformedURLException { + DestinationEventService loggingService = DestinationEventService.LOCAL; + assertThat(loggingService.getBaseUri()).isEqualTo(new URL("http://localhost:8192/v1/events?hasty=true")); + } + + @Test void testDestinationEventServiceAnalyticsDev() throws MalformedURLException { + DestinationEventService loggingService = DestinationEventService.ANALYTICS; + assertThat(loggingService.getBaseUri(true)).isEqualTo(new URL("https://intake-analytics.wikimedia.org/v1/events")); + } + + @Test void testDestinationEventServiceLoggingDev() throws MalformedURLException { + DestinationEventService loggingService = DestinationEventService.ERROR_LOGGING; + assertThat(loggingService.getBaseUri(true)).isEqualTo(new URL("https://intake-logging.wikimedia.org/v1/events")); + } + + @Test void testDestinationEventServiceLocalDev() throws MalformedURLException { + DestinationEventService loggingService = DestinationEventService.LOCAL; + assertThat(loggingService.getBaseUri(true)).isEqualTo(new URL("http://localhost:8192/v1/events")); + } +} diff --git a/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/EndToEndIT.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/EndToEndIT.java new file mode 100644 index 00000000000..fff8ba18361 --- /dev/null +++ b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/EndToEndIT.java @@ -0,0 +1,199 @@ +package org.wikimedia.metricsplatform; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.equalToJson; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.post; +import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.client.WireMock.verify; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.awaitility.Awaitility.await; +import static org.wikimedia.metricsplatform.ConsistencyITClientData.createConsistencyTestClientData; +import static org.wikimedia.metricsplatform.context.DataFixtures.getTestCustomData; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.wikimedia.metricsplatform.context.ClientData; +import org.wikimedia.metricsplatform.context.DataFixtures; + +import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import com.google.common.io.Resources; + +@WireMockTest(httpPort = 8192) +public class EndToEndIT { + private String expectedEventClick; + private String expectedEventClickCustom; + private String expectedEventInteraction; + private String expectedEventView; + private byte[] localConfig; + private final ClientData testClientData = createConsistencyTestClientData(); + + @BeforeEach + void fetchStreamConfigs() throws IOException { + // Stub fetching the stream config from api endpoint. + stubStreamConfigFetch(); + } + + @Test void submitClickEvent(WireMockRuntimeInfo wireMockRuntimeInfo) throws IOException { + // Create the metrics client. + MetricsClient testMetricsClient = buildMetricsClient(wireMockRuntimeInfo); + await().atMost(10, SECONDS).until(testMetricsClient::isFullyInitialized); + + // Stub response from posting event to local eventgate logging service. + stubFor(post("/v1/events?hasty=true") + .willReturn(aResponse() + .withBody(getExpectedEventClick()))); + + testMetricsClient.submitClick( + DataFixtures.getTestStream("click"), + DataFixtures.getTestClientData(getExpectedEventClick()), + DataFixtures.getTestInteractionData("TestClick") + ); + + await().atMost(10, SECONDS).until(testMetricsClient::isEventQueueEmpty); + + verify(postRequestedFor(urlEqualTo("/v1/events?hasty=true")) + .withRequestBody(equalToJson(getExpectedEventClick(), true, true))); + } + + @Test void submitClickEventWithCustomData(WireMockRuntimeInfo wireMockRuntimeInfo) throws IOException { + // Create the metrics client. + MetricsClient testMetricsClient = buildMetricsClient(wireMockRuntimeInfo); + await().atMost(10, SECONDS).until(testMetricsClient::isFullyInitialized); + + // Stub response from posting event to local eventgate logging service. + stubFor(post("/v1/events?hasty=true") + .willReturn(aResponse() + .withBody(getExpectedEventClickCustom()))); + + testMetricsClient.submitClick( + DataFixtures.getTestStream("click_custom"), + "/analytics/product_metrics/app/click_custom/1.0.0", + "click.test_event_name_for_end_to_end_testing", + DataFixtures.getTestClientData(getExpectedEventClickCustom()), + getTestCustomData(), + DataFixtures.getTestInteractionData("TestClickCustom") + ); + + await().atMost(10, SECONDS).until(testMetricsClient::isEventQueueEmpty); + + verify(postRequestedFor(urlEqualTo("/v1/events?hasty=true")) + .withRequestBody(equalToJson(getExpectedEventClickCustom(), true, true))); + } + + @Test void submitViewEvent(WireMockRuntimeInfo wireMockRuntimeInfo) throws IOException { + // Create the metrics client. + MetricsClient testMetricsClient = buildMetricsClient(wireMockRuntimeInfo); + await().atMost(10, SECONDS).until(testMetricsClient::isFullyInitialized); + + // Stub response from posting event to local eventgate logging service. + stubFor(post("/v1/events?hasty=true") + .willReturn(aResponse() + .withBody(getExpectedEventView()))); + + testMetricsClient.submitView( + DataFixtures.getTestStream("view"), + DataFixtures.getTestClientData(getExpectedEventView()), + DataFixtures.getTestInteractionData("TestView") + ); + + await().atMost(10, SECONDS).until(testMetricsClient::isEventQueueEmpty); + + verify(postRequestedFor(urlEqualTo("/v1/events?hasty=true")) + .withRequestBody(equalToJson(getExpectedEventView(), true, true))); + } + + @Test void submitInteractionEvent(WireMockRuntimeInfo wireMockRuntimeInfo) throws IOException { + // Create the metrics client. + MetricsClient testMetricsClient = buildMetricsClient(wireMockRuntimeInfo); + await().atMost(10, SECONDS).until(testMetricsClient::isFullyInitialized); + + // Stub response from posting event to local eventgate logging service. + stubFor(post("/v1/events?hasty=true") + .willReturn(aResponse() + .withBody(getExpectedEventInteraction()))); + + testMetricsClient.submitInteraction( + DataFixtures.getTestStream("interaction"), + "interaction.test_event_name_for_end_to_end_testing", + DataFixtures.getTestClientData(getExpectedEventInteraction()), + DataFixtures.getTestInteractionData("TestInteraction") + ); + + await().atMost(15, SECONDS).until(testMetricsClient::isEventQueueEmpty); + + verify(postRequestedFor(urlEqualTo("/v1/events?hasty=true")) + .withRequestBody(equalToJson(getExpectedEventInteraction(), true, true))); + } + + private byte[] readConfig() throws IOException { + if (this.localConfig == null) { + this.localConfig = Resources.asByteSource( + Resources.getResource("org/wikimedia/metrics_platform/config/streamconfigs-local.json") + ).read(); + } + return this.localConfig; + } + + private String getExpectedEventClick() throws IOException { + if (this.expectedEventClick == null) { + this.expectedEventClick = Resources.asCharSource( + Resources.getResource("org/wikimedia/metrics_platform/event/expected_event_click.json"), + UTF_8 + ).read(); + } + return this.expectedEventClick; + } + + private String getExpectedEventClickCustom() throws IOException { + if (this.expectedEventClickCustom == null) { + this.expectedEventClickCustom = Resources.asCharSource( + Resources.getResource("org/wikimedia/metrics_platform/event/expected_event_click_custom.json"), + UTF_8 + ).read(); + } + return this.expectedEventClickCustom; + } + + private String getExpectedEventView() throws IOException { + if (this.expectedEventView == null) { + this.expectedEventView = Resources.asCharSource( + Resources.getResource("org/wikimedia/metrics_platform/event/expected_event_view.json"), + UTF_8 + ).read(); + } + return this.expectedEventView; + } + + private String getExpectedEventInteraction() throws IOException { + if (this.expectedEventInteraction == null) { + this.expectedEventInteraction = Resources.asCharSource( + Resources.getResource("org/wikimedia/metrics_platform/event/expected_event_interaction.json"), + UTF_8 + ).read(); + } + return this.expectedEventInteraction; + } + + private void stubStreamConfigFetch() throws IOException { + stubFor(get(urlEqualTo("/config")) + .willReturn(aResponse() + .withStatus(200) + .withBody(readConfig()))); + } + + private MetricsClient buildMetricsClient(WireMockRuntimeInfo wireMockRuntimeInfo) throws MalformedURLException { + return MetricsClient.builder(testClientData) + .streamConfigURL(new URL(wireMockRuntimeInfo.getHttpBaseUrl() + "/config")) + .isDebug(false) + .build(); + } +} diff --git a/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/EventProcessorTest.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/EventProcessorTest.java new file mode 100644 index 00000000000..b4c003a7cfa --- /dev/null +++ b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/EventProcessorTest.java @@ -0,0 +1,177 @@ +package org.wikimedia.metricsplatform; + +import static java.lang.Boolean.FALSE; +import static java.lang.Boolean.TRUE; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyCollection; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.wikimedia.metricsplatform.event.EventFixtures.minimalEventProcessed; + +import java.io.IOException; +import java.net.URL; +import java.net.UnknownHostException; +import java.util.Collection; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicReference; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.wikimedia.metricsplatform.config.SourceConfig; +import org.wikimedia.metricsplatform.config.SourceConfigFixtures; +import org.wikimedia.metricsplatform.config.StreamConfig; +import org.wikimedia.metricsplatform.context.ClientData; +import org.wikimedia.metricsplatform.event.EventProcessed; + +@ExtendWith(MockitoExtension.class) +class EventProcessorTest { + @Mock private EventSender mockEventSender; + @Mock private CurationController mockCurationController; + private final AtomicReference sourceConfig = new AtomicReference<>(); + private final BlockingQueue eventQueue = new LinkedBlockingQueue<>(10); + private final ClientData consistencyTestClientData = ConsistencyITClientData.createConsistencyTestClientData(); + private final SamplingController samplingController = new SamplingController(consistencyTestClientData, new SessionController()); + private EventProcessor eventProcessor; + + @BeforeEach void clearEventQueue() { + eventQueue.clear(); + } + + @BeforeEach void createEventProcessor() { + sourceConfig.set(SourceConfigFixtures.getTestSourceConfigMax()); + + eventProcessor = new EventProcessor( + new ContextController(), + mockCurationController, + sourceConfig, + samplingController, + mockEventSender, + eventQueue, + false + ); + } + + @Test void enqueuedEventsAreSent() throws IOException { + whenEventsArePassingCurationFilter(); + + eventQueue.offer(minimalEventProcessed()); + eventProcessor.sendEnqueuedEvents(); + verify(mockEventSender).sendEvents(any(URL.class), anyCollection()); + } + + @Test void eventsNotPassingCurationFiltersAreDropped() throws IOException { + whenEventsAreNotPassingCurationFilter(); + + eventQueue.offer(minimalEventProcessed()); + eventProcessor.sendEnqueuedEvents(); + + verify(mockEventSender, never()).sendEvents(any(URL.class), anyCollection()); + assertThat(eventQueue).isEmpty(); + } + + @Test void eventsAreRemovedFromQueueOnceSent() { + whenEventsArePassingCurationFilter(); + + eventQueue.offer(minimalEventProcessed()); + eventProcessor.sendEnqueuedEvents(); + + assertThat(eventQueue).isEmpty(); + } + + @Test void eventsRemainInOutputBufferOnFailure() throws IOException { + whenEventsArePassingCurationFilter(); + doThrow(UnknownHostException.class).when(mockEventSender).sendEvents(any(URL.class), anyCollection()); + + eventQueue.offer(minimalEventProcessed()); + eventProcessor.sendEnqueuedEvents(); + + assertThat(eventQueue).isNotEmpty(); + } + + @Test void eventsAreEnrichedBeforeBeingSent() throws IOException { + whenEventsArePassingCurationFilter(); + + eventQueue.offer(minimalEventProcessed()); + eventProcessor.sendEnqueuedEvents(); + + @SuppressWarnings("unchecked") + ArgumentCaptor> eventCaptor = ArgumentCaptor.forClass(Collection.class); + verify(mockEventSender).sendEvents(any(URL.class), eventCaptor.capture()); + + EventProcessed sentEvent = eventCaptor.getValue().iterator().next(); + + // Verify client data based on minimum provided values in StreamConfigFixtures. + assertThat(sentEvent.getAgentData().getClientPlatform()).isEqualTo("android"); + assertThat(sentEvent.getAgentData().getClientPlatformFamily()).isEqualTo("app"); + assertThat(sentEvent.getPageData().getTitle()).isEqualTo("Test Page Title"); + assertThat(sentEvent.getMediawikiData().getDatabase()).isEqualTo("enwiki"); + assertThat(sentEvent.getPerformerData().getSessionId()).isEqualTo("eeeeeeeeeeeeeeeeeeee"); + } + + @Test void eventsNotSentWhenFetchStreamConfigFails() { + sourceConfig.set(null); + + eventQueue.offer(minimalEventProcessed()); + eventProcessor.sendEnqueuedEvents(); + + assertThat(eventQueue).isNotEmpty(); + } + + @Test void testSentEventsHaveClientData() throws IOException { + whenEventsArePassingCurationFilter(); + + eventQueue.offer(minimalEventProcessed()); + eventProcessor.sendEnqueuedEvents(); + + @SuppressWarnings("unchecked") + ArgumentCaptor> eventCaptor = ArgumentCaptor.forClass(Collection.class); + verify(mockEventSender).sendEvents(any(URL.class), eventCaptor.capture()); + + EventProcessed sentEvent = eventCaptor.getValue().iterator().next(); + + // Verify client data based on extended provided values in StreamConfigFixtures. + assertThat(sentEvent.getAgentData().getAppFlavor()).isEqualTo("devdebug"); + assertThat(sentEvent.getAgentData().getAppInstallId()).isEqualTo("ffffffff-ffff-ffff-ffff-ffffffffffff"); + assertThat(sentEvent.getAgentData().getAppTheme()).isEqualTo("LIGHT"); + assertThat(sentEvent.getAgentData().getAppVersion()).isEqualTo(982734); + assertThat(sentEvent.getAgentData().getAppVersionName()).isEqualTo("2.7.50470-dev-2024-02-14"); + assertThat(sentEvent.getAgentData().getClientPlatform()).isEqualTo("android"); + assertThat(sentEvent.getAgentData().getClientPlatformFamily()).isEqualTo("app"); + assertThat(sentEvent.getAgentData().getDeviceLanguage()).isEqualTo("en"); + assertThat(sentEvent.getAgentData().getReleaseStatus()).isEqualTo("dev"); + + assertThat(sentEvent.getPageData().getId()).isEqualTo(1); + assertThat(sentEvent.getPageData().getNamespaceId()).isEqualTo(0); + assertThat(sentEvent.getPageData().getWikidataItemQid()).isEqualTo("Q123456"); + + assertThat(sentEvent.getPerformerData().getPageviewId()).isEqualTo("eeeeeeeeeeeeeeeeeeee"); + assertThat(sentEvent.getPerformerData().getLanguageGroups()).isEqualTo("zh, en"); + assertThat(sentEvent.getPerformerData().getLanguagePrimary()).isEqualTo("zh-tw"); + } + + private void whenEventsArePassingCurationFilter() { + when( + mockCurationController.shouldProduceEvent( + any(EventProcessed.class), + any(StreamConfig.class) + ) + ).thenReturn(TRUE); + } + + private void whenEventsAreNotPassingCurationFilter() { + when( + mockCurationController.shouldProduceEvent( + any(EventProcessed.class), + any(StreamConfig.class) + ) + ).thenReturn(FALSE); + } +} diff --git a/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/EventTest.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/EventTest.java new file mode 100644 index 00000000000..6fb08a5143a --- /dev/null +++ b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/EventTest.java @@ -0,0 +1,154 @@ +package org.wikimedia.metricsplatform; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.wikimedia.metricsplatform.MetricsClient.DATE_FORMAT; +import static org.wikimedia.metricsplatform.event.EventProcessed.fromEvent; + +import java.time.Instant; +import java.util.Collections; +import java.util.Locale; +import java.util.UUID; + +import org.junit.jupiter.api.Test; +import org.wikimedia.metricsplatform.context.AgentData; +import org.wikimedia.metricsplatform.context.ClientData; +import org.wikimedia.metricsplatform.context.DataFixtures; +import org.wikimedia.metricsplatform.event.Event; +import org.wikimedia.metricsplatform.event.EventProcessed; +import org.wikimedia.metricsplatform.json.GsonHelper; + +import com.google.gson.Gson; + +class EventTest { + + @Test void testEvent() { + Event eventBasic = new Event("test/event/1.0.0", "test.event", "testEvent"); + EventProcessed event = fromEvent(eventBasic); + String timestamp = DATE_FORMAT.format(Instant.EPOCH); + event.setTimestamp(timestamp); + + assertThat(event.getStream()).isEqualTo("test.event"); + assertThat(event.getTimestamp()).isEqualTo("1970-01-01T00:00:00Z"); + } + + @Test void testEventSerialization() { + String uuid = UUID.randomUUID().toString(); + Event eventBasic = new Event("test/event/1.0.0", "test.event", "testEvent"); + EventProcessed event = fromEvent(eventBasic); + + event.setTimestamp("2021-08-27T12:00:00Z"); + + event.setClientData(DataFixtures.getTestClientData()); + ClientData clientData = DataFixtures.getTestClientData(); + clientData.setAgentData( + AgentData.builder() + .appFlavor("flamingo") + .appInstallId(uuid) + .appTheme("giraffe") + .appVersion(123456789) + .clientPlatform("android") + .clientPlatformFamily("app") + .deviceLanguage("en") + .releaseStatus("beta") + .build() + ); + event.setClientData(clientData); + + assertThat(event.getStream()).isEqualTo("test.event"); + assertThat(event.getSchema()).isEqualTo("test/event/1.0.0"); + assertThat(event.getName()).isEqualTo("testEvent"); + assertThat(event.getAgentData().getAppFlavor()).isEqualTo("flamingo"); + assertThat(event.getAgentData().getAppInstallId()).isEqualTo(uuid); + assertThat(event.getAgentData().getAppTheme()).isEqualTo("giraffe"); + assertThat(event.getAgentData().getAppVersion()).isEqualTo(123456789); + assertThat(event.getTimestamp()).isEqualTo("2021-08-27T12:00:00Z"); + assertThat(event.getAgentData().getClientPlatform()).isEqualTo("android"); + assertThat(event.getAgentData().getClientPlatformFamily()).isEqualTo("app"); + assertThat(event.getAgentData().getDeviceLanguage()).isEqualTo("en"); + assertThat(event.getAgentData().getReleaseStatus()).isEqualTo("beta"); + + assertThat(event.getPageData().getId()).isEqualTo(1); + assertThat(event.getPageData().getTitle()).isEqualTo("Test Page Title"); + assertThat(event.getPageData().getNamespaceId()).isEqualTo(0); + assertThat(event.getPageData().getNamespaceName()).isEqualTo("Main"); + assertThat(event.getPageData().getRevisionId()).isEqualTo(1); + assertThat(event.getPageData().getWikidataItemQid()).isEqualTo("Q123456"); + assertThat(event.getPageData().getContentLanguage()).isEqualTo("en"); + + assertThat(event.getMediawikiData().getDatabase()).isEqualTo("enwiki"); + + assertThat(event.getPerformerData().getId()).isEqualTo(1); + assertThat(event.getPerformerData().getName()).isEqualTo("TestPerformer"); + assertThat(event.getPerformerData().getIsLoggedIn()).isTrue(); + assertThat(event.getPerformerData().getIsTemp()).isFalse(); + assertThat(event.getPerformerData().getSessionId()).isEqualTo("eeeeeeeeeeeeeeeeeeee"); + assertThat(event.getPerformerData().getPageviewId()).isEqualTo("eeeeeeeeeeeeeeeeeeee"); + assertThat(event.getPerformerData().getGroups()).isEqualTo(Collections.singletonList("*")); + assertThat(event.getPerformerData().getLanguageGroups()).isEqualTo("zh, en"); + assertThat(event.getPerformerData().getLanguagePrimary()).isEqualTo("zh-tw"); + assertThat(event.getPerformerData().getRegistrationDt()).isEqualTo("2023-03-01T01:08:30Z"); + + event.applyInteractionData(DataFixtures.getTestInteractionData("TestAction")); + + assertThat(event.getAction()).isEqualTo("TestAction"); + assertThat(event.getActionSource()).isEqualTo("TestActionSource"); + assertThat(event.getActionContext()).isEqualTo("TestActionContext"); + assertThat(event.getActionSubtype()).isEqualTo("TestActionSubtype"); + assertThat(event.getElementId()).isEqualTo("TestElementId"); + assertThat(event.getElementFriendlyName()).isEqualTo("TestElementFriendlyName"); + assertThat(event.getFunnelEntryToken()).isEqualTo("TestFunnelEntryToken"); + assertThat(event.getFunnelEventSequencePosition()).isEqualTo(8); + + Gson gson = GsonHelper.getGson(); + String json = gson.toJson(event); + assertThat(json).isEqualTo(String.format(Locale.ROOT, + "{" + + "\"agent\":{" + + "\"app_flavor\":\"flamingo\"," + + "\"app_install_id\":\"%s\"," + + "\"app_theme\":\"giraffe\"," + + "\"app_version\":123456789," + + "\"client_platform\":\"android\"," + + "\"client_platform_family\":\"app\"," + + "\"device_language\":\"en\"," + + "\"release_status\":\"beta\"" + + "}," + + "\"page\":{" + + "\"id\":1," + + "\"title\":\"Test Page Title\"," + + "\"namespace_id\":0," + + "\"namespace_name\":\"Main\"," + + "\"revision_id\":1," + + "\"wikidata_qid\":\"Q123456\"," + + "\"content_language\":\"en\"" + + "}," + + "\"mediawiki\":{" + + "\"database\":\"enwiki\"" + + "}," + + "\"performer\":{" + + "\"id\":1," + + "\"name\":\"TestPerformer\"," + + "\"is_logged_in\":true," + + "\"is_temp\":false," + + "\"session_id\":\"eeeeeeeeeeeeeeeeeeee\"," + + "\"pageview_id\":\"eeeeeeeeeeeeeeeeeeee\"," + + "\"groups\":[\"*\"]," + + "\"language_groups\":\"zh, en\"," + + "\"language_primary\":\"zh-tw\"," + + "\"registration_dt\":\"2023-03-01T01:08:30Z\"" + + "}," + + "\"action\":\"TestAction\"," + + "\"action_subtype\":\"TestActionSubtype\"," + + "\"action_source\":\"TestActionSource\"," + + "\"action_context\":\"TestActionContext\"," + + "\"element_id\":\"TestElementId\"," + + "\"element_friendly_name\":\"TestElementFriendlyName\"," + + "\"funnel_entry_token\":\"TestFunnelEntryToken\"," + + "\"funnel_event_sequence_position\":8," + + "\"$schema\":\"test/event/1.0.0\"," + + "\"dt\":\"2021-08-27T12:00:00Z\"," + + "\"meta\":{\"stream\":\"test.event\"}" + + "}", uuid, uuid)); + } + +} diff --git a/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/MetricsClientTest.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/MetricsClientTest.java new file mode 100644 index 00000000000..36d7b70e5bc --- /dev/null +++ b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/MetricsClientTest.java @@ -0,0 +1,269 @@ +package org.wikimedia.metricsplatform; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.wikimedia.metricsplatform.MetricsClient.METRICS_PLATFORM_SCHEMA_BASE; +import static org.wikimedia.metricsplatform.config.StreamConfigFixtures.streamConfig; +import static org.wikimedia.metricsplatform.context.DataFixtures.getTestClientData; +import static org.wikimedia.metricsplatform.context.DataFixtures.getTestCustomData; +import static org.wikimedia.metricsplatform.curation.CurationFilterFixtures.curationFilter; +import static org.wikimedia.metricsplatform.event.EventProcessed.fromEvent; + +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicReference; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.wikimedia.metricsplatform.config.SourceConfig; +import org.wikimedia.metricsplatform.config.SourceConfigFixtures; +import org.wikimedia.metricsplatform.config.StreamConfig; +import org.wikimedia.metricsplatform.context.ClientData; +import org.wikimedia.metricsplatform.context.DataFixtures; +import org.wikimedia.metricsplatform.context.InteractionData; +import org.wikimedia.metricsplatform.context.PageData; +import org.wikimedia.metricsplatform.context.PerformerData; +import org.wikimedia.metricsplatform.event.Event; +import org.wikimedia.metricsplatform.event.EventProcessed; + +@ExtendWith(MockitoExtension.class) +class MetricsClientTest { + + @Mock private ClientData clientData; + @Mock private SessionController mockSessionController; + @Mock private SamplingController mockSamplingController; + private MetricsClient client; + private BlockingQueue eventQueue; + private AtomicReference sourceConfig; + + @BeforeEach void createEventProcessorMetricsClient() { + eventQueue = new LinkedBlockingQueue<>(10); + sourceConfig = new AtomicReference<>(SourceConfigFixtures.getTestSourceConfigMin()); + clientData = DataFixtures.getTestClientData(); + + client = MetricsClient.builder(clientData) + .sessionController(mockSessionController) + .samplingController(mockSamplingController) + .sourceConfigRef(sourceConfig) + .eventQueue(eventQueue) + .build(); + } + + @Test void testSubmit() { + Event event = new Event(METRICS_PLATFORM_SCHEMA_BASE, "test_stream", "test_event"); + client.submit(event); + EventProcessed eventProcessed = eventQueue.peek(); + String stream = eventProcessed.getStream(); + assertThat(stream).isEqualTo("test_stream"); + } + + @Test void testSubmitMetricsEventWithoutClientData() { + when(mockSamplingController.isInSample(streamConfig(curationFilter()))).thenReturn(true); + + Map customDataMap = getTestCustomData(); + client.submitMetricsEvent("test_stream", METRICS_PLATFORM_SCHEMA_BASE, "test_event", customDataMap); + + assertThat(eventQueue).isNotEmpty(); + + EventProcessed queuedEvent = eventQueue.remove(); + + // Verify custom data + assertThat(queuedEvent.getName()).isEqualTo("test_event"); + Map customData = queuedEvent.getCustomData(); + assertThat(customData.get("font_size")).isEqualTo("small"); + assertThat(customData.get("is_full_width")).isEqualTo(true); + assertThat(customData.get("screen_size")).isEqualTo(1080); + + // Verify that client data is not included + assertThat(queuedEvent.getAgentData().getClientPlatform()).isNull(); + assertThat(queuedEvent.getPageData().getId()).isNull(); + assertThat(queuedEvent.getPageData().getTitle()).isNull(); + assertThat(queuedEvent.getPageData().getNamespaceId()).isNull(); + assertThat(queuedEvent.getPageData().getNamespaceName()).isNull(); + assertThat(queuedEvent.getPageData().getRevisionId()).isNull(); + assertThat(queuedEvent.getPageData().getWikidataItemQid()).isNull(); + assertThat(queuedEvent.getPageData().getContentLanguage()).isNull(); + assertThat(queuedEvent.getMediawikiData().getDatabase()).isNull(); + assertThat(queuedEvent.getPerformerData().getId()).isNull(); + } + + @Test void testSubmitMetricsEventWithClientData() { + when(mockSamplingController.isInSample(streamConfig(curationFilter()))).thenReturn(true); + + // Update a few client data members to confirm that the client data parameter during metrics client + // instantiation gets overridden with the client data sent with the event. + ClientData clientData = DataFixtures.getTestClientData(); + PageData pageData = PageData.builder() + .id(108) + .title("Revised Page Title") + .namespaceId(0) + .namespaceName("Main") + .revisionId(1L) + .wikidataItemQid("Q123456") + .contentLanguage("en") + .build(); + clientData.setPageData(pageData); + + client.submitMetricsEvent("test_stream", METRICS_PLATFORM_SCHEMA_BASE, "test_event", clientData, getTestCustomData()); + + assertThat(eventQueue).isNotEmpty(); + + EventProcessed queuedEvent = eventQueue.remove(); + + // Verify custom data + assertThat(queuedEvent.getName()).isEqualTo("test_event"); + Map customData = queuedEvent.getCustomData(); + assertThat(customData.get("font_size")).isEqualTo("small"); + assertThat(customData.get("is_full_width")).isEqualTo(true); + assertThat(customData.get("screen_size")).isEqualTo(1080); + + // Verify client data + assertThat(queuedEvent.getAgentData().getAppInstallId()).isEqualTo("ffffffff-ffff-ffff-ffff-ffffffffffff"); + assertThat(queuedEvent.getAgentData().getClientPlatform()).isEqualTo("android"); + assertThat(queuedEvent.getAgentData().getClientPlatformFamily()).isEqualTo("app"); + + assertThat(queuedEvent.getPageData().getId()).isEqualTo(108); + assertThat(queuedEvent.getPageData().getTitle()).isEqualTo("Revised Page Title"); + assertThat(queuedEvent.getPageData().getNamespaceId()).isEqualTo(0); + assertThat(queuedEvent.getPageData().getNamespaceName()).isEqualTo("Main"); + assertThat(queuedEvent.getPageData().getRevisionId()).isEqualTo(1L); + assertThat(queuedEvent.getPageData().getWikidataItemQid()).isEqualTo("Q123456"); + assertThat(queuedEvent.getPageData().getContentLanguage()).isEqualTo("en"); + + assertThat(queuedEvent.getMediawikiData().getDatabase()).isEqualTo("enwiki"); + + assertThat(queuedEvent.getPerformerData().getId()).isEqualTo(1); + assertThat(queuedEvent.getPerformerData().getName()).isEqualTo("TestPerformer"); + assertThat(queuedEvent.getPerformerData().getIsLoggedIn()).isTrue(); + assertThat(queuedEvent.getPerformerData().getIsTemp()).isFalse(); + assertThat(queuedEvent.getPerformerData().getPageviewId()).isEqualTo("eeeeeeeeeeeeeeeeeeee"); + assertThat(queuedEvent.getPerformerData().getGroups()).contains("*"); + assertThat(queuedEvent.getPerformerData().getLanguageGroups()).isEqualTo("zh, en"); + assertThat(queuedEvent.getPerformerData().getLanguagePrimary()).isEqualTo("zh-tw"); + assertThat(queuedEvent.getPerformerData().getRegistrationDt()).isEqualTo("2023-03-01T01:08:30Z"); + + assertThat(queuedEvent.getClientData().domain).isEqualTo("en.wikipedia.org"); + } + + @Test void testSubmitMetricsEventWithInteractionData() { + when(mockSamplingController.isInSample(streamConfig(curationFilter()))).thenReturn(true); + + ClientData clientData = DataFixtures.getTestClientData(); + Map customDataMap = getTestCustomData(); + InteractionData interactionData = DataFixtures.getTestInteractionData("TestAction"); + client.submitMetricsEvent("test_stream", METRICS_PLATFORM_SCHEMA_BASE, "test_event", clientData, customDataMap, interactionData); + + assertThat(eventQueue).isNotEmpty(); + + EventProcessed queuedEvent = eventQueue.remove(); + + assertThat(queuedEvent.getAction()).isEqualTo("TestAction"); + assertThat(queuedEvent.getActionSource()).isEqualTo("TestActionSource"); + assertThat(queuedEvent.getActionContext()).isEqualTo("TestActionContext"); + assertThat(queuedEvent.getActionSubtype()).isEqualTo("TestActionSubtype"); + assertThat(queuedEvent.getElementId()).isEqualTo("TestElementId"); + assertThat(queuedEvent.getElementFriendlyName()).isEqualTo("TestElementFriendlyName"); + assertThat(queuedEvent.getFunnelEntryToken()).isEqualTo("TestFunnelEntryToken"); + assertThat(queuedEvent.getFunnelEventSequencePosition()).isEqualTo(8); + } + + @Test void testSubmitMetricsEventIncludesSample() { + StreamConfig streamConfig = streamConfig(curationFilter()); + + when(mockSamplingController.isInSample(streamConfig)).thenReturn(true); + + Map customDataMap = getTestCustomData(); + client.submitMetricsEvent("test_stream", METRICS_PLATFORM_SCHEMA_BASE, "test_event", customDataMap); + + assertThat(eventQueue).isNotEmpty(); + + EventProcessed queuedEvent = eventQueue.remove(); + + assertThat(queuedEvent.getSample()).isEqualTo(streamConfig.getSampleConfig()); + } + + @Test void testSubmitWhenEventQueueIsFull() { + for (int i = 1; i <= 10; i++) { + Event event = new Event("test_schema" + i, "test_stream" + i, "test_event" + i); + EventProcessed eventProcessed = fromEvent(event); + eventQueue.add(eventProcessed); + } + EventProcessed oldestEvent = eventQueue.peek(); + + Event event11 = new Event("test_schema11", "test_stream11", "test_event11"); + EventProcessed eventProcessed11 = fromEvent(event11); + client.submit(eventProcessed11); + + assertThat(eventQueue).doesNotContain(oldestEvent); + + Boolean containsNewestEvent = eventQueue.stream().anyMatch(event -> event.getName().equals("test_event11")); + assertThat(containsNewestEvent).isTrue(); + } + + @Test void testTouchSessionOnAppPause() { + when(mockSamplingController.isInSample(streamConfig(curationFilter()))).thenReturn(true); + fillEventQueue(); + assertThat(eventQueue).isNotEmpty(); + + client.onAppPause(); + verify(mockSessionController).touchSession(); + } + + @Test void testResumeSessionOnAppResume() { + client.onAppResume(); + verify(mockSessionController).touchSession(); + } + + @Test void testResetSession() { + client.resetSession(); + verify(mockSessionController).beginSession(); + } + + @Test void testCloseSessionOnAppClose() { + when(mockSamplingController.isInSample(streamConfig(curationFilter()))).thenReturn(true); + fillEventQueue(); + assertThat(eventQueue).isNotEmpty(); + + client.onAppClose(); + verify(mockSessionController).closeSession(); + } + + @Test void testAddRequiredMetadata() { + Event event = new Event("test/event/1.0.0", "test_event", "testEvent"); + assertThat(event.getTimestamp()).isNull(); + + client.submit(event); + EventProcessed queuedEvent = eventQueue.remove(); + + assertThat(queuedEvent.getTimestamp()).isNotNull(); + verify(mockSessionController).getSessionId(); + } + + @Test void testPerformerDataLanguageGroups() { + Event event = new Event("test/event/1.0.0", "test_event", "testEvent"); + ClientData clientData = getTestClientData(); + PerformerData performerData = event.getClientData().getPerformerData(); + clientData.setPerformerData(PerformerData.builderFrom(performerData) + .languageGroups("[zh-hant, zh-hans, ja, en, zh-yue, ko, fr, de, it, es, pt, da, tr, ru, nl, sv, cs, " + + "fi, uk, el, pl, hu, vi, id, ca, mk, sl, ms, tl, avk, lt, sr-el, eu, nb, ceb, als, uz-latn, " + + "az, af, nn, et, eo, la, br, jv, io, bg, ro, nrm, pcd, tg-latn, lmo, gl, cy, sq, is, ha, gd, " + + "ku-latn, hr, lv, sk, bar, pms, lld, ga, war]") + .build()); + event.setClientData(clientData); + + client.submit(event); + EventProcessed queuedEvent = eventQueue.remove(); + assertThat(queuedEvent.getPerformerData().getLanguageGroups().length()).isEqualTo(255); + } + + private void fillEventQueue() { + for (int i = 1; i <= 10; i++) { + client.submitMetricsEvent("test_stream", METRICS_PLATFORM_SCHEMA_BASE, "test_event", getTestCustomData()); + } + } +} diff --git a/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/SamplingControllerTest.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/SamplingControllerTest.java new file mode 100644 index 00000000000..898ecfbbd5e --- /dev/null +++ b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/SamplingControllerTest.java @@ -0,0 +1,58 @@ +package org.wikimedia.metricsplatform; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.wikimedia.metricsplatform.config.sampling.SampleConfig.Identifier.DEVICE; +import static org.wikimedia.metricsplatform.config.sampling.SampleConfig.Identifier.SESSION; + +import org.junit.jupiter.api.Test; +import org.wikimedia.metricsplatform.config.sampling.SampleConfig; +import org.wikimedia.metricsplatform.config.StreamConfig; +import org.wikimedia.metricsplatform.context.DataFixtures; + +class SamplingControllerTest { + + private final SamplingController samplingController = new SamplingController( + DataFixtures.getTestClientData(), + new SessionController() + ); + + @Test void testGetSamplingValue() { + double deviceVal = samplingController.getSamplingValue(DEVICE); + assertThat(deviceVal).isBetween(0.0, 1.0); + } + + @Test void testGetSamplingId() { + assertThat(samplingController.getSamplingId(DEVICE)).isNotNull(); + assertThat(samplingController.getSamplingId(SESSION)).isNotNull(); + } + + @Test void testNoSamplingConfig() { + StreamConfig noSamplingConfig = new StreamConfig("foo", "bar", null, null, null); + assertThat(samplingController.isInSample(noSamplingConfig)).isTrue(); + } + + @Test void testAlwaysInSample() { + StreamConfig alwaysInSample = new StreamConfig("foo", "bar", null, + new StreamConfig.ProducerConfig(new StreamConfig.MetricsPlatformClientConfig( + null, + null, + null + )), + null + ); + assertThat(samplingController.isInSample(alwaysInSample)).isTrue(); + } + + @Test void testNeverInSample() { + StreamConfig neverInSample = new StreamConfig("foo", "bar", null, + new StreamConfig.ProducerConfig(new StreamConfig.MetricsPlatformClientConfig( + null, + null, + null + )), + new SampleConfig(0.0, SESSION) + ); + assertThat(samplingController.isInSample(neverInSample)).isFalse(); + } + +} diff --git a/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/SessionControllerTest.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/SessionControllerTest.java new file mode 100644 index 00000000000..1252a32d028 --- /dev/null +++ b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/SessionControllerTest.java @@ -0,0 +1,36 @@ +package org.wikimedia.metricsplatform; + +import static java.time.temporal.ChronoUnit.HOURS; +import static org.assertj.core.api.Assertions.assertThat; + +import java.time.Instant; + +import org.junit.jupiter.api.Test; + +class SessionControllerTest { + + @Test void testSessionExpiry() { + Instant oneHourAgo = Instant.now().minus(1, HOURS); + SessionController sessionController = new SessionController(oneHourAgo); + assertThat(sessionController.sessionExpired()).isTrue(); + } + + @Test void testTouchSession() { + Instant oneHourAgo = Instant.now().minus(1, HOURS); + SessionController sessionController = new SessionController(oneHourAgo); + String sessionId1 = sessionController.getSessionId(); + sessionController.touchSession(); + assertThat(sessionController.sessionExpired()).isFalse(); + + String sessionId2 = sessionController.getSessionId(); + + assertThat(sessionId1).isNotEqualTo(sessionId2); + } + + @Test void testSessionIdLength() { + Instant twoHoursAgo = Instant.now().minus(2, HOURS); + SessionController sessionController = new SessionController(twoHoursAgo); + String sessionId = sessionController.getSessionId(); + assertThat(sessionId.length()).isEqualTo(20); + } +} diff --git a/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/TestEventSender.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/TestEventSender.java new file mode 100644 index 00000000000..39fff38ff5e --- /dev/null +++ b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/TestEventSender.java @@ -0,0 +1,27 @@ +package org.wikimedia.metricsplatform; + +import java.io.IOException; +import java.net.URL; +import java.util.Collection; + +import org.wikimedia.metricsplatform.event.EventProcessed; + +class TestEventSender implements EventSender { + + private final boolean shouldFail; + + TestEventSender() { + this(false); + } + + TestEventSender(boolean shouldFail) { + this.shouldFail = shouldFail; + } + + @Override + public void sendEvents(URL baseUri, Collection events) throws IOException { + if (shouldFail) { + throw new IOException(); + } + } +} diff --git a/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/CurationFilterFixtures.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/CurationFilterFixtures.java new file mode 100644 index 00000000000..d3c6c9e61d5 --- /dev/null +++ b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/CurationFilterFixtures.java @@ -0,0 +1,25 @@ +package org.wikimedia.metricsplatform.config; + +import java.util.Arrays; + +import org.wikimedia.metricsplatform.config.curation.CollectionCurationRules; +import org.wikimedia.metricsplatform.config.curation.CurationRules; + +public final class CurationFilterFixtures { + + private CurationFilterFixtures() { + // Utility class - should never be instantiated + } + + public static CurationFilter getCurationFilter() { + return CurationFilter.builder() + .pageTitleRules(CurationRules.builder().isEquals("Test").build()) + .performerGroupsRules( + CollectionCurationRules.builder() + .doesNotContain("sysop") + .containsAny(Arrays.asList("steward", "bureaucrat")) + .build() + ) + .build(); + } +} diff --git a/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/SampleConfigTest.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/SampleConfigTest.java new file mode 100644 index 00000000000..8e53c18f714 --- /dev/null +++ b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/SampleConfigTest.java @@ -0,0 +1,22 @@ +package org.wikimedia.metricsplatform.config; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.wikimedia.metricsplatform.config.sampling.SampleConfig.Identifier.DEVICE; + +import org.junit.jupiter.api.Test; +import org.wikimedia.metricsplatform.config.sampling.SampleConfig; +import org.wikimedia.metricsplatform.json.GsonHelper; + +import com.google.gson.Gson; + +class SampleConfigTest { + + @Test void testSamplingConfigDeserialization() { + Gson gson = GsonHelper.getGson(); + String samplingConfigJson = "{\"rate\":0.25,\"identifier\":\"device\"}"; + SampleConfig sampleConfig = gson.fromJson(samplingConfigJson, SampleConfig.class); + assertThat(sampleConfig.getRate()).isEqualTo(0.25); + assertThat(sampleConfig.getIdentifier()).isEqualTo(DEVICE); + } + +} diff --git a/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/SourceConfigFixtures.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/SourceConfigFixtures.java new file mode 100644 index 00000000000..ff02615c4c9 --- /dev/null +++ b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/SourceConfigFixtures.java @@ -0,0 +1,22 @@ +package org.wikimedia.metricsplatform.config; + +public final class SourceConfigFixtures { + + private SourceConfigFixtures() { + // Utility class, should never be instantiated + } + + /** + * Convenience method for getting source config with minimum provided values. + */ + public static SourceConfig getTestSourceConfigMin() { + return new SourceConfig(StreamConfigFixtures.streamConfigMap(StreamConfigFixtures.provideValuesMinimum())); + } + + /** + * Convenience method for getting source config with extended provided values. + */ + public static SourceConfig getTestSourceConfigMax() { + return new SourceConfig(StreamConfigFixtures.streamConfigMap(StreamConfigFixtures.provideValuesExtended())); + } +} diff --git a/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/SourceConfigTest.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/SourceConfigTest.java new file mode 100644 index 00000000000..c7f878eb4ad --- /dev/null +++ b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/SourceConfigTest.java @@ -0,0 +1,31 @@ +package org.wikimedia.metricsplatform.config; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +class SourceConfigTest { + + private static SourceConfig sourceConfig; + + @Test void testConfig() { + sourceConfig = new SourceConfig(StreamConfigFixtures.STREAM_CONFIGS_WITH_EVENTS); + assertThat(sourceConfig.getStreamNamesByEvent("test.event")).containsExactly("test.stream"); + } + + @Test void testGetStreamNames() { + sourceConfig = new SourceConfig(StreamConfigFixtures.STREAM_CONFIGS_WITH_EVENTS); + assertThat(sourceConfig.getStreamNames()).containsExactly("test.stream"); + } + + @Test void testGetStreamConfigByName() { + sourceConfig = new SourceConfig(StreamConfigFixtures.STREAM_CONFIGS_WITH_EVENTS); + StreamConfig streamConfig = StreamConfigFixtures.sampleStreamConfig(true); + assertThat(streamConfig).isEqualTo(sourceConfig.getStreamConfigByName("test.stream")); + } + + @Test void testGetStreamNamesByEvent() { + sourceConfig = new SourceConfig(StreamConfigFixtures.STREAM_CONFIGS_WITH_EVENTS); + assertThat(sourceConfig.getStreamNamesByEvent("test.event")).containsExactly("test.stream"); + } +} diff --git a/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigFetcherTest.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigFetcherTest.java new file mode 100644 index 00000000000..293b173f08a --- /dev/null +++ b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigFetcherTest.java @@ -0,0 +1,50 @@ +package org.wikimedia.metricsplatform.config; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.assertj.core.api.Assertions.assertThat; +import static org.wikimedia.metricsplatform.config.DestinationEventService.LOCAL; +import static org.wikimedia.metricsplatform.config.StreamConfigFetcher.ANALYTICS_API_ENDPOINT; + +import java.io.IOException; +import java.io.Reader; +import java.net.URL; +import java.util.Map; + +import org.junit.jupiter.api.Test; +import org.wikimedia.metricsplatform.json.GsonHelper; + +import com.google.common.io.Resources; + +import okhttp3.OkHttpClient; + +class StreamConfigFetcherTest { + + @Test void parsingConfigFromJsonWorks() throws IOException { + try (Reader in = readConfigFile("streamconfigs.json")) { + StreamConfigFetcher streamConfigFetcher = new StreamConfigFetcher(new URL(ANALYTICS_API_ENDPOINT), new OkHttpClient(), GsonHelper.getGson()); + Map config = streamConfigFetcher.parseConfig(in); + assertThat(config).containsKey("mediawiki.visual_editor_feature_use"); + StreamConfig streamConfig = config.get("mediawiki.visual_editor_feature_use"); + String schemaTitle = streamConfig.getSchemaTitle(); + assertThat(schemaTitle).isEqualTo("analytics/mediawiki/client/metrics_event"); + assertThat(streamConfig.getStreamName()).isEqualTo("mediawiki.visual_editor_feature_use"); + } + } + + @Test void parsingLocalConfigFromJsonWorks() throws IOException { + try (Reader in = readConfigFile("streamconfigs-local.json")) { + StreamConfigFetcher streamConfigFetcher = new StreamConfigFetcher(new URL(ANALYTICS_API_ENDPOINT), new OkHttpClient(), GsonHelper.getGson()); + Map config = streamConfigFetcher.parseConfig(in); + assertThat(config).containsKey("mediawiki.visual_editor_feature_use"); + StreamConfig streamConfig = config.get("mediawiki.edit_attempt"); + assertThat(streamConfig.getDestinationEventService()).isEqualTo(LOCAL); + } + } + + private Reader readConfigFile(String filename) throws IOException { + return Resources.asCharSource( + Resources.getResource("org/wikimedia/metrics_platform/config/" + filename), + UTF_8 + ).openStream(); + } +} diff --git a/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigFixtures.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigFixtures.java new file mode 100644 index 00000000000..f34abe72f09 --- /dev/null +++ b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigFixtures.java @@ -0,0 +1,164 @@ +package org.wikimedia.metricsplatform.config; + +import static java.util.Collections.emptySet; +import static java.util.Collections.singletonList; +import static java.util.Collections.singletonMap; +import static org.wikimedia.metricsplatform.config.StreamConfigFetcher.METRICS_PLATFORM_SCHEMA_TITLE; +import static org.wikimedia.metricsplatform.context.ContextValue.AGENT_APP_INSTALL_ID; +import static org.wikimedia.metricsplatform.context.ContextValue.AGENT_APP_FLAVOR; +import static org.wikimedia.metricsplatform.context.ContextValue.AGENT_APP_THEME; +import static org.wikimedia.metricsplatform.context.ContextValue.AGENT_APP_VERSION; +import static org.wikimedia.metricsplatform.context.ContextValue.AGENT_DEVICE_LANGUAGE; +import static org.wikimedia.metricsplatform.context.ContextValue.AGENT_RELEASE_STATUS; +import static org.wikimedia.metricsplatform.context.ContextValue.AGENT_CLIENT_PLATFORM; +import static org.wikimedia.metricsplatform.context.ContextValue.AGENT_CLIENT_PLATFORM_FAMILY; +import static org.wikimedia.metricsplatform.context.ContextValue.MEDIAWIKI_DATABASE; +import static org.wikimedia.metricsplatform.context.ContextValue.PAGE_TITLE; +import static org.wikimedia.metricsplatform.context.ContextValue.PAGE_ID; +import static org.wikimedia.metricsplatform.context.ContextValue.PAGE_NAMESPACE_ID; +import static org.wikimedia.metricsplatform.context.ContextValue.PAGE_WIKIDATA_QID; +import static org.wikimedia.metricsplatform.context.ContextValue.PERFORMER_SESSION_ID; +import static org.wikimedia.metricsplatform.context.ContextValue.PERFORMER_PAGEVIEW_ID; +import static org.wikimedia.metricsplatform.context.ContextValue.PERFORMER_LANGUAGE_GROUPS; +import static org.wikimedia.metricsplatform.context.ContextValue.PERFORMER_LANGUAGE_PRIMARY; +import static org.wikimedia.metricsplatform.curation.CurationFilterFixtures.curationFilter; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.wikimedia.metricsplatform.config.sampling.SampleConfig; + +public final class StreamConfigFixtures { + + public static final Map STREAM_CONFIGS_WITH_EVENTS = new HashMap() {{ + put("test.stream", sampleStreamConfig(true)); + } + }; + + private StreamConfigFixtures() { + // Utility class, should never be instantiated + } + + public static StreamConfig sampleStreamConfig(boolean hasEvents) { + Set emptyEvents = emptySet(); + Set testEvents = new HashSet<>(singletonList("test.event")); + Set events = hasEvents ? testEvents : emptyEvents; + Set requestedValuesSet = new HashSet<>(Arrays.asList( + "agent_app_install_id", + "agent_client_platform", + "agent_client_platform_family", + "mediawiki_database", + "page_id", + "page_namespace_id", + "page_namespace_name", + "page_title", + "page_revision_id", + "page_content_language", + "page_wikidata_qid", + "performer_id", + "performer_is_logged_in", + "performer_is_temp", + "performer_name", + "performer_session_id", + "performer_pageview_id", + "performer_groups", + "performer_registration_dt", + "performer_language_groups", + "performer_language_primary" + )); + SampleConfig sampleConfig = new SampleConfig(1.0, SampleConfig.Identifier.PAGEVIEW); + + return new StreamConfig( + "test.stream", + "test/event", + DestinationEventService.ANALYTICS, + new StreamConfig.ProducerConfig( + new StreamConfig.MetricsPlatformClientConfig( + events, + requestedValuesSet, + CurationFilterFixtures.getCurationFilter() + ) + ), + sampleConfig + ); + } + + /** + * Convenience method for getting stream config. + */ + public static StreamConfig streamConfig(CurationFilter curationFilter, String[] provideValues) { + Set events = Collections.singleton("test_event"); + SampleConfig sampleConfig = new SampleConfig(1.0f, SampleConfig.Identifier.PAGEVIEW); + + return new StreamConfig( + "test_stream", + METRICS_PLATFORM_SCHEMA_TITLE, + DestinationEventService.LOCAL, + new StreamConfig.ProducerConfig( + new StreamConfig.MetricsPlatformClientConfig( + events, + new HashSet<>(List.of(provideValues)), + curationFilter + ) + ), + sampleConfig + ); + } + + /** + * Convenience method for getting stream config. + */ + public static StreamConfig streamConfig(CurationFilter curationFilter) { + return streamConfig(curationFilter, provideValuesMinimum()); + } + + /** + * Convenience method for getting a stream config map. + */ + public static Map streamConfigMap(String[] provideValues) { + return streamConfigMap(curationFilter(), provideValues); + } + + + public static Map streamConfigMap(CurationFilter curationFilter, String[] provideValues) { + StreamConfig streamConfig = streamConfig(curationFilter, provideValues); + return singletonMap(streamConfig.getStreamName(), streamConfig); + } + + public static String[] provideValuesMinimum() { + return new String[]{ + AGENT_CLIENT_PLATFORM, + AGENT_CLIENT_PLATFORM_FAMILY, + PAGE_TITLE, + MEDIAWIKI_DATABASE, + PERFORMER_SESSION_ID + }; + } + + public static String[] provideValuesExtended() { + return new String[]{ + AGENT_APP_INSTALL_ID, + AGENT_CLIENT_PLATFORM, + AGENT_CLIENT_PLATFORM_FAMILY, + AGENT_APP_FLAVOR, + AGENT_APP_THEME, + AGENT_APP_VERSION, + AGENT_DEVICE_LANGUAGE, + AGENT_RELEASE_STATUS, + PAGE_ID, + PAGE_TITLE, + PAGE_NAMESPACE_ID, + PAGE_WIKIDATA_QID, + MEDIAWIKI_DATABASE, + PERFORMER_SESSION_ID, + PERFORMER_PAGEVIEW_ID, + PERFORMER_LANGUAGE_GROUPS, + PERFORMER_LANGUAGE_PRIMARY, + }; + } +} diff --git a/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigIT.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigIT.java new file mode 100644 index 00000000000..015ea3cc500 --- /dev/null +++ b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigIT.java @@ -0,0 +1,48 @@ +package org.wikimedia.metricsplatform.config; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.net.URL; + +import org.junit.jupiter.api.Test; +import org.wikimedia.metricsplatform.json.GsonHelper; + +import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import com.google.common.io.Resources; + +import okhttp3.OkHttpClient; + +@WireMockTest +class StreamConfigIT { + + @Test void canLoadConfigOverHTTP(WireMockRuntimeInfo wmRuntimeInfo) throws IOException { + stubFor(get("/streamConfig").willReturn( + aResponse() + .withBody(loadConfigStream()) + ) + ); + + StreamConfigFetcher streamConfigFetcher = new StreamConfigFetcher( + new URL(wmRuntimeInfo.getHttpBaseUrl() + "/streamConfig"), + new OkHttpClient(), + GsonHelper.getGson() + ); + + SourceConfig sourceConfig = streamConfigFetcher.fetchStreamConfigs(); + + assertThat(sourceConfig).isNotNull(); + + } + + private byte[] loadConfigStream() throws IOException { + return Resources.asByteSource( + Resources.getResource("org/wikimedia/metrics_platform/config/streamconfigs.json") + ).read(); + } + +} diff --git a/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigTest.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigTest.java new file mode 100644 index 00000000000..34efa02c01f --- /dev/null +++ b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigTest.java @@ -0,0 +1,31 @@ +package org.wikimedia.metricsplatform.config; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.wikimedia.metricsplatform.config.sampling.SampleConfig.Identifier.SESSION; + +import java.util.Set; + +import org.junit.jupiter.api.Test; +import org.wikimedia.metricsplatform.json.GsonHelper; + +import com.google.gson.Gson; + +class StreamConfigTest { + + @Test void testStreamConfigDeserialization() { + Gson gson = GsonHelper.getGson(); + String streamConfigJson = "{\"stream\":\"test.event\",\"schema_title\":\"test/event\"," + + "\"destination_event_service\":\"eventgate-logging-local\"," + + "\"producers\":" + + "{\"metrics_platform_client\":{\"provide_values\":[\"page_id\",\"user_id\"]}}," + + "\"sample\":{\"rate\":0.5,\"identifier\":\"session\"}}"; + StreamConfig streamConfig = gson.fromJson(streamConfigJson, StreamConfig.class); + assertThat(streamConfig.getStreamName()).isEqualTo("test.event"); + assertThat(streamConfig.getSchemaTitle()).isEqualTo("test/event"); + assertThat(streamConfig.getSampleConfig().getRate()).isEqualTo(0.5); + assertThat(streamConfig.getSampleConfig().getIdentifier()).isEqualTo(SESSION); + assertThat(streamConfig.getProducerConfig().getMetricsPlatformClientConfig().getRequestedValues()) + .isEqualTo(Set.of("page_id", "user_id")); + assertThat(streamConfig.getDestinationEventService()).isEqualTo(DestinationEventService.LOCAL); + } +} diff --git a/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/context/AgentDataTest.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/context/AgentDataTest.java new file mode 100644 index 00000000000..5beddc82046 --- /dev/null +++ b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/context/AgentDataTest.java @@ -0,0 +1,52 @@ +package org.wikimedia.metricsplatform.context; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; +import org.wikimedia.metricsplatform.json.GsonHelper; + +import com.google.gson.Gson; + +public class AgentDataTest { + @Test + void testAgentData() { + AgentData agentData = AgentData.builder() + .appFlavor("flamingo") + .appInstallId("ffffffff-ffff-ffff-ffff-ffffffffffff") + .appTheme("giraffe") + .appVersion(123456789) + .appVersionName("2.7.50470-dev-2024-02-14") + .clientPlatform("android") + .clientPlatformFamily("app") + .deviceFamily("Samsung SM-G960F") + .deviceLanguage("en") + .releaseStatus("beta") + .build(); + + assertThat(agentData.getAppFlavor()).isEqualTo("flamingo"); + assertThat(agentData.getAppInstallId()).isEqualTo("ffffffff-ffff-ffff-ffff-ffffffffffff"); + assertThat(agentData.getAppTheme()).isEqualTo("giraffe"); + assertThat(agentData.getAppVersion()).isEqualTo(123456789); + assertThat(agentData.getAppVersionName()).isEqualTo("2.7.50470-dev-2024-02-14"); + assertThat(agentData.getClientPlatform()).isEqualTo("android"); + assertThat(agentData.getClientPlatformFamily()).isEqualTo("app"); + assertThat(agentData.getDeviceFamily()).isEqualTo("Samsung SM-G960F"); + assertThat(agentData.getDeviceLanguage()).isEqualTo("en"); + assertThat(agentData.getReleaseStatus()).isEqualTo("beta"); + + Gson gson = GsonHelper.getGson(); + String json = gson.toJson(agentData); + assertThat(json).isEqualTo("{" + + "\"app_flavor\":\"flamingo\"," + + "\"app_install_id\":\"ffffffff-ffff-ffff-ffff-ffffffffffff\"," + + "\"app_theme\":\"giraffe\"," + + "\"app_version\":123456789," + + "\"app_version_name\":\"2.7.50470-dev-2024-02-14\"," + + "\"client_platform\":\"android\"," + + "\"client_platform_family\":\"app\"," + + "\"device_family\":\"Samsung SM-G960F\"," + + "\"device_language\":\"en\"," + + "\"release_status\":\"beta\"" + + "}"); + } +} diff --git a/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/context/CustomDataTest.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/context/CustomDataTest.java new file mode 100644 index 00000000000..df66f8dc083 --- /dev/null +++ b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/context/CustomDataTest.java @@ -0,0 +1,54 @@ +package org.wikimedia.metricsplatform.context; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.wikimedia.metricsplatform.context.DataFixtures.getTestCustomData; + +import java.util.Map; + +import org.junit.jupiter.api.Test; +import org.wikimedia.metricsplatform.json.GsonHelper; + +import com.google.gson.Gson; + +class CustomDataTest { + +// @Test void testFormatCustomDataByCustomSerialization() { +// Map customData = getTestCustomDataFormatted(); +// +// Gson gson = GsonHelper.getGson(); +// String jsonCustomData = gson.toJson(customData); +// +// assertThat(jsonCustomData) +// .isEqualTo("{" + +// "\"is_full_width\":" + +// "{" + +// "\"data_type\":\"boolean\"," + +// "\"value\":\"true\"" + +// "}," + +// "\"font_size\":" + +// "{" + +// "\"data_type\":\"string\"," + +// "\"value\":\"small\"" + +// "}," + +// "\"screen_size\":" + +// "{" + +// "\"data_type\":\"number\"," + +// "\"value\":\"1080\"" + +// "}" + +// "}"); +// } + + @Test void testCustomDataSerialization() { + Map customData = getTestCustomData(); + + Gson gson = GsonHelper.getGson(); + String jsonCustomData = gson.toJson(customData); + + assertThat(jsonCustomData) + .isEqualTo("{" + + "\"is_full_width\":true," + + "\"font_size\":\"small\"," + + "\"screen_size\":1080" + + "}"); + } +} diff --git a/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/context/DataFixtures.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/context/DataFixtures.java new file mode 100644 index 00000000000..a8aa8fbf120 --- /dev/null +++ b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/context/DataFixtures.java @@ -0,0 +1,134 @@ +package org.wikimedia.metricsplatform.context; + +import java.time.Instant; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.wikimedia.metricsplatform.json.GsonHelper; + +import com.google.gson.Gson; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +public final class DataFixtures { + + private DataFixtures() { + // Utility class, should never be instantiated + } + + public static ClientData getTestClientData() { + return new ClientData( + getTestAgentData(), + getTestPageData(), + getTestMediawikiData(), + getTestPerformerData(), + "en.wikipedia.org" + ); + } + + public static ClientData getTestClientData(String expectedEvent) { + Map dataMap = new HashMap<>(); + + JsonElement jsonElement = JsonParser.parseString(expectedEvent); + JsonObject expectedEventJson = jsonElement.isJsonArray() ? jsonElement.getAsJsonArray().get(0).getAsJsonObject() : jsonElement.getAsJsonObject(); + + Set dataObjectNames = Stream.of("agent", "page", "mediawiki", "performer") + .collect(Collectors.toCollection(HashSet::new)); + + for (String dataObjectName : dataObjectNames) { + JsonObject metaData = expectedEventJson.getAsJsonObject(dataObjectName); + Set keys = metaData.keySet(); + Map dataMapEach = new HashMap<>(); + for (String key : keys) { + dataMapEach.put(key, metaData.get(key)); + } + dataMap.put(dataObjectName, dataMapEach); + } + + String domain = expectedEventJson.get("meta").getAsJsonObject().get("domain").getAsString(); + dataMap.put("domain", domain); + + Gson gson = GsonHelper.getGson(); + JsonElement jsonClientData = gson.toJsonTree(dataMap); + return gson.fromJson(jsonClientData, ClientData.class); + } + + public static AgentData getTestAgentData() { + return AgentData.builder() + .appInstallId("ffffffff-ffff-ffff-ffff-ffffffffffff") + .clientPlatform("android") + .clientPlatformFamily("app") + .appFlavor("devdebug") + .appVersion(982734) + .appVersionName("2.7.50470-dev-2024-02-14") + .appTheme("LIGHT") + .deviceFamily("Samsung SM-G960F") + .deviceLanguage("en") + .releaseStatus("dev") + .build(); + } + + public static PageData getTestPageData() { + return PageData.builder() + .id(1) + .title("Test Page Title") + .namespaceId(0) + .namespaceName("Main") + .revisionId(1L) + .wikidataItemQid("Q123456") + .contentLanguage("en") + .build(); + } + + public static MediawikiData getTestMediawikiData() { + return MediawikiData.builder() + .database("enwiki") + .build(); + } + + public static PerformerData getTestPerformerData() { + return PerformerData.builder() + .id(1) + .name("TestPerformer") + .isLoggedIn(true) + .isTemp(false) + .sessionId("eeeeeeeeeeeeeeeeeeee") + .pageviewId("eeeeeeeeeeeeeeeeeeee") + .groups(Collections.singletonList("*")) + .languageGroups("zh, en") + .languagePrimary("zh-tw") + .registrationDt(Instant.parse("2023-03-01T01:08:30Z")) + .build(); + } + + public static InteractionData getTestInteractionData(String action) { + return InteractionData.builder() + .action(action) + .actionSubtype("TestActionSubtype") + .actionSource("TestActionSource") + .actionContext("TestActionContext") + .elementId("TestElementId") + .elementFriendlyName("TestElementFriendlyName") + .funnelEntryToken("TestFunnelEntryToken") + .funnelEventSequencePosition(8) + .build(); + } + + public static Map getTestCustomData() { + Map customData = new HashMap(); + customData.put("font_size", "small"); + customData.put("is_full_width", true); + customData.put("screen_size", 1080); + return customData; + } + + public static String getTestStream(String streamNameFragment) { + return "mediawiki.metrics_platform." + streamNameFragment; + } +} diff --git a/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/context/MediawikiDataTest.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/context/MediawikiDataTest.java new file mode 100644 index 00000000000..e404848b854 --- /dev/null +++ b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/context/MediawikiDataTest.java @@ -0,0 +1,25 @@ +package org.wikimedia.metricsplatform.context; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; +import org.wikimedia.metricsplatform.json.GsonHelper; + +import com.google.gson.Gson; + +public class MediawikiDataTest { + @Test + void testMediawikiData() { + MediawikiData mediawikiData = MediawikiData.builder() + .database("enwiki") + .build(); + + assertThat(mediawikiData.getDatabase()).isEqualTo("enwiki"); + + Gson gson = GsonHelper.getGson(); + String json = gson.toJson(mediawikiData); + assertThat(json).isEqualTo("{" + + "\"database\":\"enwiki\"" + + "}"); + } +} diff --git a/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/context/PageDataTest.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/context/PageDataTest.java new file mode 100644 index 00000000000..493cb26adc7 --- /dev/null +++ b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/context/PageDataTest.java @@ -0,0 +1,43 @@ +package org.wikimedia.metricsplatform.context; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; +import org.wikimedia.metricsplatform.json.GsonHelper; + +import com.google.gson.Gson; + +class PageDataTest { + + @Test void testPageData() { + PageData pageData = PageData.builder() + .id(1) + .title("Test") + .namespaceId(0) + .namespaceName("") + .revisionId(1L) + .wikidataItemQid("Q1") + .contentLanguage("zh") + .build(); + + assertThat(pageData.getId()).isEqualTo(1); + assertThat(pageData.getNamespaceId()).isEqualTo(0); + assertThat(pageData.getNamespaceName()).isEmpty(); + assertThat(pageData.getTitle()).isEqualTo("Test"); + assertThat(pageData.getRevisionId()).isEqualTo(1); + assertThat(pageData.getWikidataItemQid()).isEqualTo("Q1"); + assertThat(pageData.getContentLanguage()).isEqualTo("zh"); + + Gson gson = GsonHelper.getGson(); + String json = gson.toJson(pageData); + assertThat(json).isEqualTo("{\"id\":1," + + "\"title\":\"Test\"," + + "\"namespace_id\":0," + + "\"namespace_name\":\"\"," + + "\"revision_id\":1," + + "\"wikidata_qid\":\"Q1\"," + + "\"content_language\":\"zh\"" + + "}"); + } + +} diff --git a/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/context/PerformerDataTest.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/context/PerformerDataTest.java new file mode 100644 index 00000000000..52af125e2a0 --- /dev/null +++ b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/context/PerformerDataTest.java @@ -0,0 +1,55 @@ +package org.wikimedia.metricsplatform.context; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.time.Instant; +import java.util.Collections; + +import org.junit.jupiter.api.Test; +import org.wikimedia.metricsplatform.json.GsonHelper; + +import com.google.gson.Gson; + +class PerformerDataTest { + + @Test void testPerformerData() { + PerformerData performerData = PerformerData.builder() + .id(1) + .name("TestPerformer") + .isLoggedIn(true) + .isTemp(false) + .sessionId("eeeeeeeeeeeeeeeeeeee") + .pageviewId("eeeeeeeeeeeeeeeeeeee") + .groups(Collections.singletonList("*")) + .languageGroups("zh, en") + .languagePrimary("zh-tw") + .registrationDt(Instant.parse("2023-03-01T01:08:30Z")) + .build(); + + assertThat(performerData.getId()).isEqualTo(1); + assertThat(performerData.getName()).isEqualTo("TestPerformer"); + assertThat(performerData.getIsLoggedIn()).isTrue(); + assertThat(performerData.getIsTemp()).isFalse(); + assertThat(performerData.getGroups()).isEqualTo(Collections.singletonList("*")); + assertThat(performerData.getLanguageGroups()).isEqualTo("zh, en"); + assertThat(performerData.getLanguagePrimary()).isEqualTo("zh-tw"); + assertThat(performerData.getRegistrationDt()).isEqualTo("2023-03-01T01:08:30Z"); + + Gson gson = GsonHelper.getGson(); + + String json = gson.toJson(performerData); + assertThat(json).isEqualTo("{" + + "\"id\":1," + + "\"name\":\"TestPerformer\"," + + "\"is_logged_in\":true," + + "\"is_temp\":false," + + "\"session_id\":\"eeeeeeeeeeeeeeeeeeee\"," + + "\"pageview_id\":\"eeeeeeeeeeeeeeeeeeee\"," + + "\"groups\":[\"*\"]," + + "\"language_groups\":\"zh, en\"," + + "\"language_primary\":\"zh-tw\"," + + "\"registration_dt\":\"2023-03-01T01:08:30Z\"" + + "}"); + } + +} diff --git a/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/curation/CurationFilterFixtures.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/curation/CurationFilterFixtures.java new file mode 100644 index 00000000000..ef96b93e95b --- /dev/null +++ b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/curation/CurationFilterFixtures.java @@ -0,0 +1,21 @@ +package org.wikimedia.metricsplatform.curation; + +import org.wikimedia.metricsplatform.json.GsonHelper; +import org.wikimedia.metricsplatform.config.CurationFilter; + +import com.google.gson.Gson; + +public final class CurationFilterFixtures { + private CurationFilterFixtures() { + // Utility class, should never be instantiated + } + + public static CurationFilter curationFilter() { + Gson gson = GsonHelper.getGson(); + String curationFilterJson = "{\"page_id\":{\"less_than\":500,\"not_equals\":42},\"page_namespace_text\":" + + "{\"equals\":\"Talk\"},\"user_is_logged_in\":{\"equals\":true},\"user_edit_count_bucket\":" + + "{\"in\":[\"100-999 edits\",\"1000+ edits\"]},\"user_groups\":{\"contains_all\":" + + "[\"user\",\"autoconfirmed\"],\"does_not_contain\":\"sysop\"}}"; + return gson.fromJson(curationFilterJson, CurationFilter.class); + } +} diff --git a/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/event/EventFixtures.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/event/EventFixtures.java new file mode 100644 index 00000000000..13e0970caa0 --- /dev/null +++ b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/event/EventFixtures.java @@ -0,0 +1,65 @@ +package org.wikimedia.metricsplatform.event; + +import static org.wikimedia.metricsplatform.context.DataFixtures.getTestClientData; +import static org.wikimedia.metricsplatform.event.EventProcessed.fromEvent; + +import java.util.Arrays; +import java.util.List; + +import org.wikimedia.metricsplatform.context.ClientData; +import org.wikimedia.metricsplatform.context.PageData; +import org.wikimedia.metricsplatform.context.PerformerData; + +public final class EventFixtures { + + private EventFixtures() { + // Utility class, should never be instantiated + } + public static Event minimalEvent() { + return new Event("test_schema", "test_stream", "test_event"); + } + + public static EventProcessed minimalEventProcessed() { + EventProcessed eventProcessed = fromEvent(minimalEvent()); + eventProcessed.setClientData(getTestClientData()); + return eventProcessed; + } + + public static EventProcessed getEvent() { + Event event = new Event("test/event", "test.event", "testEvent"); + ClientData clientData = new ClientData(); + clientData.setPageData(PageData.builder().id(1).namespaceName("Talk").build()); + + event.setClientData(clientData); + EventProcessed eventProcessed = fromEvent(event); + eventProcessed.setPerformerData( + PerformerData.builder() + .groups(Arrays.asList("user", "autoconfirmed", "steward")) + .isLoggedIn(true) + .build() + ); + return eventProcessed; + } + + public static EventProcessed getEvent( + Integer id, + String namespaceName, + List groups, + boolean isLoggedIn, + String editCount + ) { + Event event = new Event("test/event", "test.event", "testEvent"); + ClientData clientData = new ClientData(); + clientData.setPageData(PageData.builder().id(id).namespaceName(namespaceName).build()); + + event.setClientData(clientData); + EventProcessed eventProcessed = fromEvent(event); + eventProcessed.setPerformerData( + PerformerData.builder() + .groups(groups) + .isLoggedIn(isLoggedIn) + .build() + ); + return eventProcessed; + } +} diff --git a/analytics/metricsplatform/src/test/resources/org/wikimedia/metricsplatform/config/streamconfigs-local.json b/analytics/metricsplatform/src/test/resources/org/wikimedia/metricsplatform/config/streamconfigs-local.json new file mode 100644 index 00000000000..a721750f589 --- /dev/null +++ b/analytics/metricsplatform/src/test/resources/org/wikimedia/metricsplatform/config/streamconfigs-local.json @@ -0,0 +1,2288 @@ +{ + "streams": { + "eventlogging_CentralNoticeBannerHistory": { + "schema_title": "analytics/legacy/centralnoticebannerhistory", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_CentralNoticeBannerHistory", + "topics": [ + "eventlogging_CentralNoticeBannerHistory" + ] + }, + "eventlogging_CentralNoticeImpression": { + "schema_title": "analytics/legacy/centralnoticeimpression", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_CentralNoticeImpression", + "topics": [ + "eventlogging_CentralNoticeImpression" + ] + }, + "eventlogging_CentralNoticeTiming": { + "schema_title": "analytics/legacy/centralnoticetiming", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_CentralNoticeTiming", + "topics": [ + "eventlogging_CentralNoticeTiming" + ] + }, + "eventlogging_CodeMirrorUsage": { + "schema_title": "analytics/legacy/codemirrorusage", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_CodeMirrorUsage", + "topics": [ + "eventlogging_CodeMirrorUsage" + ] + }, + "eventlogging_ContentTranslationAbuseFilter": { + "schema_title": "analytics/legacy/contenttranslationabusefilter", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_ContentTranslationAbuseFilter", + "topics": [ + "eventlogging_ContentTranslationAbuseFilter" + ] + }, + "eventlogging_CpuBenchmark": { + "schema_title": "analytics/legacy/cpubenchmark", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_CpuBenchmark", + "topics": [ + "eventlogging_CpuBenchmark" + ] + }, + "eventlogging_DesktopWebUIActionsTracking": { + "schema_title": "analytics/legacy/desktopwebuiactionstracking", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_DesktopWebUIActionsTracking", + "topics": [ + "eventlogging_DesktopWebUIActionsTracking" + ] + }, + "eventlogging_ElementTiming": { + "schema_title": "analytics/legacy/elementtiming", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_ElementTiming", + "topics": [ + "eventlogging_ElementTiming" + ] + }, + "eventlogging_EditAttemptStep": { + "schema_title": "analytics/legacy/editattemptstep", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_EditAttemptStep", + "topics": [ + "eventlogging_EditAttemptStep" + ] + }, + "eventlogging_FeaturePolicyViolation": { + "schema_title": "analytics/legacy/featurepolicyviolation", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_FeaturePolicyViolation", + "topics": [ + "eventlogging_FeaturePolicyViolation" + ] + }, + "eventlogging_FirstInputTiming": { + "schema_title": "analytics/legacy/firstinputtiming", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_FirstInputTiming", + "topics": [ + "eventlogging_FirstInputTiming" + ] + }, + "eventlogging_HelpPanel": { + "schema_title": "analytics/legacy/helppanel", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_HelpPanel", + "topics": [ + "eventlogging_HelpPanel" + ] + }, + "eventlogging_HomepageModule": { + "schema_title": "analytics/legacy/homepagemodule", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_HomepageModule", + "topics": [ + "eventlogging_HomepageModule" + ] + }, + "eventlogging_HomepageVisit": { + "schema_title": "analytics/legacy/homepagevisit", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_HomepageVisit", + "topics": [ + "eventlogging_HomepageVisit" + ] + }, + "eventlogging_InukaPageView": { + "schema_title": "analytics/legacy/inukapageview", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_InukaPageView", + "topics": [ + "eventlogging_InukaPageView" + ] + }, + "eventlogging_KaiOSAppFirstRun": { + "schema_title": "analytics/legacy/kaiosappfirstrun", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_KaiOSAppFirstRun", + "topics": [ + "eventlogging_KaiOSAppFirstRun" + ] + }, + "eventlogging_KaiOSAppFeedback": { + "schema_title": "analytics/legacy/kaiosappfeedback", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_KaiOSAppFeedback", + "topics": [ + "eventlogging_KaiOSAppFeedback" + ] + }, + "eventlogging_LandingPageImpression": { + "schema_title": "analytics/legacy/landingpageimpression", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_LandingPageImpression", + "topics": [ + "eventlogging_LandingPageImpression" + ] + }, + "eventlogging_LayoutShift": { + "schema_title": "analytics/legacy/layoutshift", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_LayoutShift", + "topics": [ + "eventlogging_LayoutShift" + ] + }, + "eventlogging_MobileWebUIActionsTracking": { + "schema_title": "analytics/legacy/mobilewebuiactionstracking", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_MobileWebUIActionsTracking", + "topics": [ + "eventlogging_MobileWebUIActionsTracking" + ] + }, + "eventlogging_NavigationTiming": { + "schema_title": "analytics/legacy/navigationtiming", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_NavigationTiming", + "topics": [ + "eventlogging_NavigationTiming" + ] + }, + "eventlogging_NewcomerTask": { + "schema_title": "analytics/legacy/newcomertask", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_NewcomerTask", + "topics": [ + "eventlogging_NewcomerTask" + ] + }, + "eventlogging_PaintTiming": { + "schema_title": "analytics/legacy/painttiming", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_PaintTiming", + "topics": [ + "eventlogging_PaintTiming" + ] + }, + "eventlogging_PrefUpdate": { + "schema_title": "analytics/legacy/prefupdate", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_PrefUpdate", + "topics": [ + "eventlogging_PrefUpdate" + ] + }, + "eventlogging_WikipediaPortal": { + "schema_title": "analytics/legacy/wikipediaportal", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_WikipediaPortal", + "topics": [ + "eventlogging_WikipediaPortal" + ] + }, + "eventlogging_QuickSurveyInitiation": { + "schema_title": "analytics/legacy/quicksurveyinitiation", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_QuickSurveyInitiation", + "topics": [ + "eventlogging_QuickSurveyInitiation" + ] + }, + "eventlogging_QuickSurveysResponses": { + "schema_title": "analytics/legacy/quicksurveysresponses", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_QuickSurveysResponses", + "topics": [ + "eventlogging_QuickSurveysResponses" + ] + }, + "eventlogging_ReferencePreviewsBaseline": { + "schema_title": "analytics/legacy/referencepreviewsbaseline", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_ReferencePreviewsBaseline", + "topics": [ + "eventlogging_ReferencePreviewsBaseline" + ] + }, + "eventlogging_ReferencePreviewsCite": { + "schema_title": "analytics/legacy/referencepreviewscite", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_ReferencePreviewsCite", + "topics": [ + "eventlogging_ReferencePreviewsCite" + ] + }, + "eventlogging_ReferencePreviewsPopups": { + "schema_title": "analytics/legacy/referencepreviewspopups", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_ReferencePreviewsPopups", + "topics": [ + "eventlogging_ReferencePreviewsPopups" + ] + }, + "eventlogging_ResourceTiming": { + "schema_title": "analytics/legacy/resourcetiming", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_ResourceTiming", + "topics": [ + "eventlogging_ResourceTiming" + ] + }, + "eventlogging_RUMSpeedIndex": { + "schema_title": "analytics/legacy/rumspeedindex", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_RUMSpeedIndex", + "topics": [ + "eventlogging_RUMSpeedIndex" + ] + }, + "eventlogging_SaveTiming": { + "schema_title": "analytics/legacy/savetiming", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_SaveTiming", + "topics": [ + "eventlogging_SaveTiming" + ] + }, + "eventlogging_SearchSatisfaction": { + "schema_title": "analytics/legacy/searchsatisfaction", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_SearchSatisfaction", + "topics": [ + "eventlogging_SearchSatisfaction" + ] + }, + "eventlogging_ServerSideAccountCreation": { + "schema_title": "analytics/legacy/serversideaccountcreation", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_ServerSideAccountCreation", + "topics": [ + "eventlogging_ServerSideAccountCreation" + ] + }, + "eventlogging_SpecialInvestigate": { + "schema_title": "analytics/legacy/specialinvestigate", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_SpecialInvestigate", + "topics": [ + "eventlogging_SpecialInvestigate" + ] + }, + "eventlogging_SpecialMuteSubmit": { + "schema_title": "analytics/legacy/specialmutesubmit", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_SpecialMuteSubmit", + "topics": [ + "eventlogging_SpecialMuteSubmit" + ] + }, + "eventlogging_SuggestedTagsAction": { + "schema_title": "analytics/legacy/suggestedtagsaction", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_SuggestedTagsAction", + "topics": [ + "eventlogging_SuggestedTagsAction" + ] + }, + "eventlogging_TemplateDataApi": { + "schema_title": "analytics/legacy/templatedataapi", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_TemplateDataApi", + "topics": [ + "eventlogging_TemplateDataApi" + ] + }, + "eventlogging_TemplateDataEditor": { + "schema_title": "analytics/legacy/templatedataeditor", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_TemplateDataEditor", + "topics": [ + "eventlogging_TemplateDataEditor" + ] + }, + "eventlogging_TemplateWizard": { + "schema_title": "analytics/legacy/templatewizard", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_TemplateWizard", + "topics": [ + "eventlogging_TemplateWizard" + ] + }, + "eventlogging_Test": { + "schema_title": "analytics/legacy/test", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_Test", + "topics": [ + "eventlogging_Test" + ] + }, + "eventlogging_TranslationRecommendationUserAction": { + "schema_title": "analytics/legacy/translationrecommendationuseraction", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_TranslationRecommendationUserAction", + "topics": [ + "eventlogging_TranslationRecommendationUserAction" + ] + }, + "eventlogging_TranslationRecommendationUIRequests": { + "schema_title": "analytics/legacy/translationrecommendationuirequests", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_TranslationRecommendationUIRequests", + "topics": [ + "eventlogging_TranslationRecommendationUIRequests" + ] + }, + "eventlogging_TranslationRecommendationAPIRequests": { + "schema_title": "analytics/legacy/translationrecommendationapirequests", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_TranslationRecommendationAPIRequests", + "topics": [ + "eventlogging_TranslationRecommendationAPIRequests" + ] + }, + "eventlogging_TwoColConflictConflict": { + "schema_title": "analytics/legacy/twocolconflictconflict", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_TwoColConflictConflict", + "topics": [ + "eventlogging_TwoColConflictConflict" + ] + }, + "eventlogging_TwoColConflictExit": { + "schema_title": "analytics/legacy/twocolconflictexit", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_TwoColConflictExit", + "topics": [ + "eventlogging_TwoColConflictExit" + ] + }, + "eventlogging_UniversalLanguageSelector": { + "schema_title": "analytics/legacy/universallanguageselector", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_UniversalLanguageSelector", + "topics": [ + "eventlogging_UniversalLanguageSelector" + ] + }, + "eventlogging_VirtualPageView": { + "schema_title": "analytics/legacy/virtualpageview", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_VirtualPageView", + "topics": [ + "eventlogging_VirtualPageView" + ] + }, + "eventlogging_VisualEditorFeatureUse": { + "schema_title": "analytics/legacy/visualeditorfeatureuse", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_VisualEditorFeatureUse", + "topics": [ + "eventlogging_VisualEditorFeatureUse" + ] + }, + "eventlogging_VisualEditorTemplateDialogUse": { + "schema_title": "analytics/legacy/visualeditortemplatedialoguse", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_VisualEditorTemplateDialogUse", + "topics": [ + "eventlogging_VisualEditorTemplateDialogUse" + ] + }, + "eventlogging_WikibaseTermboxInteraction": { + "schema_title": "analytics/legacy/wikibasetermboxinteraction", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_WikibaseTermboxInteraction", + "topics": [ + "eventlogging_WikibaseTermboxInteraction" + ] + }, + "eventlogging_WikidataCompletionSearchClicks": { + "schema_title": "analytics/legacy/wikidatacompletionsearchclicks", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_WikidataCompletionSearchClicks", + "topics": [ + "eventlogging_WikidataCompletionSearchClicks" + ] + }, + "eventlogging_WMDEBannerEvents": { + "schema_title": "analytics/legacy/wmdebannerevents", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_WMDEBannerEvents", + "topics": [ + "eventlogging_WMDEBannerEvents" + ] + }, + "eventlogging_WMDEBannerInteractions": { + "schema_title": "analytics/legacy/wmdebannerinteractions", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_WMDEBannerInteractions", + "topics": [ + "eventlogging_WMDEBannerInteractions" + ] + }, + "eventlogging_WMDEBannerSizeIssue": { + "schema_title": "analytics/legacy/wmdebannersizeissue", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_WMDEBannerSizeIssue", + "topics": [ + "eventlogging_WMDEBannerSizeIssue" + ] + }, + "test.instrumentation": { + "schema_title": "analytics/test", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "test.instrumentation", + "topics": [ + "eqiad.test.instrumentation", + "codfw.test.instrumentation" + ] + }, + "test.instrumentation.sampled": { + "schema_title": "analytics/test", + "destination_event_service": "eventgate-analytics-external", + "sample": { + "rate": 0.5, + "unit": "session" + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "test.instrumentation.sampled", + "topics": [ + "eqiad.test.instrumentation.sampled", + "codfw.test.instrumentation.sampled" + ] + }, + "mediawiki.client.session_tick": { + "schema_title": "analytics/session_tick", + "destination_event_service": "eventgate-analytics-external", + "sample": { + "unit": "session", + "rate": 0.1 + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.client.session_tick", + "topics": [ + "eqiad.mediawiki.client.session_tick", + "codfw.mediawiki.client.session_tick" + ] + }, + "ios.edit_history_compare": { + "schema_title": "analytics/mobile_apps/ios_edit_history_compare", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "ios.edit_history_compare", + "topics": [ + "eqiad.ios.edit_history_compare", + "codfw.ios.edit_history_compare" + ] + }, + "ios.notification_interaction": { + "schema_title": "analytics/mobile_apps/ios_notification_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "ios.notification_interaction", + "topics": [ + "eqiad.ios.notification_interaction", + "codfw.ios.notification_interaction" + ] + }, + "android.user_contribution_screen": { + "schema_title": "analytics/mobile_apps/android_user_contribution_screen", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.user_contribution_screen", + "topics": [ + "eqiad.android.user_contribution_screen", + "codfw.android.user_contribution_screen" + ] + }, + "android.notification_interaction": { + "schema_title": "analytics/mobile_apps/android_notification_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.notification_interaction", + "topics": [ + "eqiad.android.notification_interaction", + "codfw.android.notification_interaction" + ] + }, + "android.image_recommendation_interaction": { + "schema_title": "analytics/mobile_apps/android_image_recommendation_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.image_recommendation_interaction", + "topics": [ + "eqiad.android.image_recommendation_interaction", + "codfw.android.image_recommendation_interaction" + ] + }, + "android.daily_stats": { + "schema_title": "analytics/mobile_apps/android_daily_stats", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.daily_stats", + "topics": [ + "eqiad.android.daily_stats", + "codfw.android.daily_stats" + ] + }, + "android.customize_toolbar_interaction": { + "schema_title": "analytics/mobile_apps/android_customize_toolbar_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.customize_toolbar_interaction", + "topics": [ + "eqiad.android.customize_toolbar_interaction", + "codfw.android.customize_toolbar_interaction" + ] + }, + "android.article_toolbar_interaction": { + "schema_title": "analytics/mobile_apps/android_article_toolbar_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.article_toolbar_interaction", + "topics": [ + "eqiad.android.article_toolbar_interaction", + "codfw.android.article_toolbar_interaction" + ] + }, + "android.edit_history_interaction": { + "schema_title": "analytics/mobile_apps/android_edit_history_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.edit_history_interaction", + "topics": [ + "eqiad.android.edit_history_interaction", + "codfw.android.edit_history_interaction" + ] + }, + "android.breadcrumbs_event": { + "schema_title": "analytics/mobile_apps/android_breadcrumbs_event", + "destination_event_service": "eventgate-analytics-external", + "sample": { + "unit": "device", + "rate": 0.5 + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.breadcrumbs_event", + "topics": [ + "eqiad.android.breadcrumbs_event", + "codfw.android.breadcrumbs_event" + ] + }, + "android.app_appearance_settings_interaction": { + "schema_title": "analytics/mobile_apps/android_app_appearance_settings_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.app_appearance_settings_interaction", + "topics": [ + "eqiad.android.app_appearance_settings_interaction", + "codfw.android.app_appearance_settings_interaction" + ] + }, + "android.article_link_preview_interaction": { + "schema_title": "analytics/mobile_apps/android_article_link_preview_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.article_link_preview_interaction", + "topics": [ + "eqiad.android.article_link_preview_interaction", + "codfw.android.article_link_preview_interaction" + ] + }, + "android.article_page_scroll_interaction": { + "schema_title": "analytics/mobile_apps/android_article_page_scroll_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.article_page_scroll_interaction", + "topics": [ + "eqiad.android.article_page_scroll_interaction", + "codfw.android.article_page_scroll_interaction" + ] + }, + "android.article_toc_interaction": { + "schema_title": "analytics/mobile_apps/android_article_toc_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.article_toc_interaction", + "topics": [ + "eqiad.android.article_toc_interaction", + "codfw.android.article_toc_interaction" + ] + }, + "android.find_in_page_interaction": { + "schema_title": "analytics/mobile_apps/android_find_in_page_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.find_in_page_interaction", + "topics": [ + "eqiad.android.find_in_page_interaction", + "codfw.android.find_in_page_interaction" + ] + }, + "android.product_metrics.article_link_preview_interaction": { + "schema_title": "analytics/product_metrics/app/base", + "destination_event_service": "eventgate-analytics-external", + "producers": { + "metrics_platform_client": { + "events": [ + "android.metrics_platform.article_link_preview_interaction" + ], + "provide_values": [ + "mediawiki_database", + "page_title", + "page_content_language", + "page_id", + "page_namespace_id", + "performer_is_logged_in", + "performer_session_id", + "performer_pageview_id", + "performer_language_groups", + "performer_language_primary", + "performer_groups", + ] + } + }, + "sample": { + "unit": "device", + "rate": 1 + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.product_metrics.article_link_preview_interaction", + "topics": [ + "eqiad.android.product_metrics.article_link_preview_interaction", + "codfw.android.product_metrics.article_link_preview_interaction" + ] + }, + "android.product_metrics.article_toc_interaction": { + "schema_title": "analytics/mobile_apps/product_metrics/android_article_toc_interaction", + "destination_event_service": "eventgate-analytics-external", + "producers": { + "metrics_platform_client": { + "events": [ + "android.metrics_platform.article_toc_interaction" + ], + "provide_values": [ + "mediawiki_database", + "page_title", + "page_content_language", + "page_id", + "page_namespace_id", + "performer_is_logged_in", + "performer_session_id", + "performer_pageview_id", + "performer_language_groups", + "performer_language_primary", + "performer_groups", + ] + } + }, + "sample": { + "unit": "device", + "rate": 1 + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.product_metrics.article_toc_interaction", + "topics": [ + "eqiad.android.product_metrics.article_toc_interaction", + "codfw.android.product_metrics.article_toc_interaction" + ] + }, + "android.product_metrics.article_toolbar_interaction": { + "schema_title": "analytics/product_metrics/app/base", + "destination_event_service": "eventgate-analytics-external", + "producers": { + "metrics_platform_client": { + "events": [ + "android.metrics_platform.article_toolbar_interaction" + ], + "provide_values": [ + "mediawiki_database", + "page_title", + "page_content_language", + "page_id", + "page_namespace_id", + "performer_is_logged_in", + "performer_session_id", + "performer_pageview_id", + "performer_language_groups", + "performer_language_primary", + "performer_groups", + ] + } + }, + "sample": { + "unit": "device", + "rate": 1 + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.product_metrics.article_toolbar_interaction", + "topics": [ + "eqiad.android.product_metrics.article_toolbar_interaction", + "codfw.android.product_metrics.article_toolbar_interaction" + ] + }, + "android.product_metrics.find_in_page_interaction": { + "schema_title": "analytics/mobile_apps/product_metrics/android_find_in_page_interaction", + "destination_event_service": "eventgate-analytics-external", + "producers": { + "metrics_platform_client": { + "events": [ + "android.metrics_platform.find_in_page_interaction" + ], + "provide_values": [ + "mediawiki_database", + "page_title", + "page_content_language", + "page_id", + "page_namespace_id", + "performer_is_logged_in", + "performer_session_id", + "performer_pageview_id", + "performer_language_groups", + "performer_language_primary", + "performer_groups", + ] + } + }, + "sample": { + "unit": "device", + "rate": 1 + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.product_metrics.find_in_page_interaction", + "topics": [ + "eqiad.android.product_metrics.find_in_page_interaction", + "codfw.android.product_metrics.find_in_page_interaction" + ] + }, + "mediawiki.mediasearch_interaction": { + "schema_title": "analytics/mediawiki/mediasearch_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.mediasearch_interaction", + "topics": [ + "eqiad.mediawiki.mediasearch_interaction", + "codfw.mediawiki.mediasearch_interaction" + ] + }, + "mediawiki.structured_task.article.link_suggestion_interaction": { + "schema_title": "analytics/mediawiki/structured_task/article/link_suggestion_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.structured_task.article.link_suggestion_interaction", + "topics": [ + "eqiad.mediawiki.structured_task.article.link_suggestion_interaction", + "codfw.mediawiki.structured_task.article.link_suggestion_interaction" + ] + }, + "mediawiki.structured_task.article.image_suggestion_interaction": { + "schema_title": "analytics/mediawiki/structured_task/article/image_suggestion_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.structured_task.article.image_suggestion_interaction", + "topics": [ + "eqiad.mediawiki.structured_task.article.image_suggestion_interaction", + "codfw.mediawiki.structured_task.article.image_suggestion_interaction" + ] + }, + "mediawiki.pref_diff": { + "schema_title": "analytics/pref_diff", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.pref_diff", + "topics": [ + "eqiad.mediawiki.pref_diff", + "codfw.mediawiki.pref_diff" + ] + }, + "mediawiki.skin_diff": { + "schema_title": "analytics/pref_diff", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.skin_diff", + "topics": [ + "eqiad.mediawiki.skin_diff", + "codfw.mediawiki.skin_diff" + ] + }, + "mediawiki.content_translation_event": { + "schema_title": "analytics/mediawiki/content_translation_event", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.content_translation_event", + "topics": [ + "eqiad.mediawiki.content_translation_event", + "codfw.mediawiki.content_translation_event" + ] + }, + "mediawiki.reading_depth": { + "schema_title": "analytics/mediawiki/web_ui_reading_depth", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.reading_depth", + "topics": [ + "eqiad.mediawiki.reading_depth", + "codfw.mediawiki.reading_depth" + ] + }, + "mediawiki.web_ab_test_enrollment": { + "schema_title": "analytics/mediawiki/web_ab_test_enrollment", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.web_ab_test_enrollment", + "topics": [ + "eqiad.mediawiki.web_ab_test_enrollment", + "codfw.mediawiki.web_ab_test_enrollment" + ] + }, + "mediawiki.web_ui_scroll": { + "schema_title": "analytics/mediawiki/web_ui_scroll", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.web_ui_scroll", + "topics": [ + "eqiad.mediawiki.web_ui_scroll", + "codfw.mediawiki.web_ui_scroll" + ] + }, + "mediawiki.ipinfo_interaction": { + "schema_title": "analytics/mediawiki/ipinfo_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.ipinfo_interaction", + "topics": [ + "eqiad.mediawiki.ipinfo_interaction", + "codfw.mediawiki.ipinfo_interaction" + ] + }, + "wd_propertysuggester.client_side_property_request": { + "schema_title": "analytics/mediawiki/wd_propertysuggester/client_side_property_request", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "wd_propertysuggester.client_side_property_request", + "topics": [ + "eqiad.wd_propertysuggester.client_side_property_request", + "codfw.wd_propertysuggester.client_side_property_request" + ] + }, + "wd_propertysuggester.server_side_property_request": { + "schema_title": "analytics/mediawiki/wd_propertysuggester/server_side_property_request", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "wd_propertysuggester.server_side_property_request", + "topics": [ + "eqiad.wd_propertysuggester.server_side_property_request", + "codfw.wd_propertysuggester.server_side_property_request" + ] + }, + "mediawiki.mentor_dashboard.visit": { + "schema_title": "analytics/mediawiki/mentor_dashboard/visit", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.mentor_dashboard.visit", + "topics": [ + "eqiad.mediawiki.mentor_dashboard.visit", + "codfw.mediawiki.mentor_dashboard.visit" + ] + }, + "mediawiki.welcomesurvey.interaction": { + "schema_title": "analytics/mediawiki/welcomesurvey/interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.welcomesurvey.interaction", + "topics": [ + "eqiad.mediawiki.welcomesurvey.interaction", + "codfw.mediawiki.welcomesurvey.interaction" + ] + }, + "mediawiki.editgrowthconfig": { + "schema_title": "analytics/mediawiki/editgrowthconfig", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.editgrowthconfig", + "topics": [ + "eqiad.mediawiki.editgrowthconfig", + "codfw.mediawiki.editgrowthconfig" + ] + }, + "mediawiki.accountcreation_block": { + "schema_title": "analytics/mediawiki/accountcreation/block", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.accountcreation_block", + "topics": [ + "eqiad.mediawiki.accountcreation_block", + "codfw.mediawiki.accountcreation_block" + ] + }, + "mediawiki.editattempt_block": { + "schema_title": "analytics/mediawiki/editattemptsblocked", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.editattempt_block", + "topics": [ + "eqiad.mediawiki.editattempt_block", + "codfw.mediawiki.editattempt_block" + ] + }, + "mediawiki.talk_page_edit": { + "stream": "mediawiki.talk_page_edit", + "schema_title": "analytics/mediawiki/talk_page_edit", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "topics": [ + "eqiad.mediawiki.talk_page_edit", + "codfw.mediawiki.talk_page_edit" + ] + }, + "mwcli.command_execute": { + "schema_title": "analytics/mwcli/command_execute", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mwcli.command_execute", + "topics": [ + "eqiad.mwcli.command_execute", + "codfw.mwcli.command_execute" + ] + }, + "mediawiki.web_ui.interactions": { + "schema_title": "analytics/mediawiki/client/metrics_event", + "destination_event_service": "eventgate-analytics-external", + "producers": { + "metrics_platform_client": { + "events": [ + "web.ui.", + "web_ui." + ], + "provide_values": [ + "page_namespace", + "performer_is_logged_in", + "performer_session_id", + "performer_pageview_id", + "performer_edit_count_bucket", + "mediawiki_skin", + "mediawiki_database" + ], + "curation": { + "mediawiki_skin": { + "in": [ + "minerva", + "vector", + "vector-2022" + ] + } + } + } + }, + "sample": { + "unit": "pageview", + "rate": 0 + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.web_ui.interactions", + "topics": [ + "eqiad.mediawiki.web_ui.interactions", + "codfw.mediawiki.web_ui.interactions" + ] + }, + "mediawiki.edit_attempt": { + "schema_title": "analytics/mediawiki/client/metrics_event", + "destination_event_service": "eventgate-logging-local", + "producers": { + "metrics_platform_client": { + "events": [ + "eas." + ], + "provide_values": [ + "agent_app_install_id", + "agent_client_platform_family", + "page_id", + "page_title", + "page_namespace_id", + "page_revision_id", + "mediawiki_database", + "performer_is_logged_in", + "performer_id", + "performer_session_id", + "performer_pageview_id" + ] + } + }, + "sample": { + "unit": "pageview", + "rate": 1 + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.edit_attempt", + "topics": [ + "eqiad.mediawiki.edit_attempt", + "codfw.mediawiki.edit_attempt" + ] + }, + "mediawiki.metrics_platform.click": { + "schema_title": "analytics/product_metrics/app/base", + "destination_event_service": "eventgate-logging-local", + "producers": { + "metrics_platform_client": { + "events": [ + "click", + "click." + ], + "provide_values": [ + "agent_app_install_id", + "agent_client_platform_family", + "page_id", + "page_title", + "page_namespace_id", + "page_revision_id", + "mediawiki_database", + "performer_is_logged_in", + "performer_id", + "performer_session_id", + "performer_pageview_id" + ] + } + }, + "sample": { + "unit": "pageview", + "rate": 1 + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.metrics_platform.click", + "topics": [ + "eqiad.mediawiki.metrics_platform.click", + "codfw.mediawiki.metrics_platform.click" + ] + }, + "mediawiki.metrics_platform.click_custom": { + "schema_title": "analytics/product_metrics/app/click_custom", + "destination_event_service": "eventgate-logging-local", + "producers": { + "metrics_platform_client": { + "events": [ + "click", + "click." + ], + "provide_values": [ + "agent_app_install_id", + "agent_client_platform_family", + "page_id", + "page_title", + "page_namespace_id", + "page_revision_id", + "mediawiki_database", + "performer_is_logged_in", + "performer_id", + "performer_session_id", + "performer_pageview_id" + ] + } + }, + "sample": { + "unit": "pageview", + "rate": 1 + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.metrics_platform.click_custom", + "topics": [ + "eqiad.mediawiki.metrics_platform.click_custom", + "codfw.mediawiki.metrics_platform.click_custom" + ] + }, + "mediawiki.metrics_platform.interaction": { + "schema_title": "analytics/product_metrics/app/base", + "destination_event_service": "eventgate-logging-local", + "producers": { + "metrics_platform_client": { + "events": [ + "interaction", + "interaction." + ], + "provide_values": [ + "agent_app_install_id", + "agent_client_platform_family", + "page_id", + "page_title", + "page_namespace_id", + "page_revision_id", + "mediawiki_database", + "performer_is_logged_in", + "performer_id", + "performer_session_id", + "performer_pageview_id" + ] + } + }, + "sample": { + "unit": "pageview", + "rate": 1 + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.metrics_platform.interaction", + "topics": [ + "eqiad.mediawiki.metrics_platform.interaction", + "codfw.mediawiki.metrics_platform.interaction" + ] + }, + "mediawiki.metrics_platform.view": { + "schema_title": "analytics/product_metrics/app/base", + "destination_event_service": "eventgate-logging-local", + "producers": { + "metrics_platform_client": { + "events": [ + "view", + "view." + ], + "provide_values": [ + "agent_app_install_id", + "agent_client_platform_family", + "page_id", + "page_title", + "page_namespace_id", + "page_revision_id", + "mediawiki_database", + "performer_is_logged_in", + "performer_id", + "performer_session_id", + "performer_pageview_id" + ] + } + }, + "sample": { + "unit": "pageview", + "rate": 1 + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.metrics_platform.view", + "topics": [ + "eqiad.mediawiki.metrics_platform.view", + "codfw.mediawiki.metrics_platform.view" + ] + }, + "mediawiki.visual_editor_feature_use": { + "schema_title": "analytics/mediawiki/client/metrics_event", + "destination_event_service": "eventgate-analytics-external", + "producers": { + "metrics_platform_client": { + "events": [ + "vefu." + ], + "provide_values": [ + "agent_client_platform_family", + "mediawiki_database", + "performer_id", + "performer_edit_count" + ] + } + }, + "sample": { + "unit": "pageview", + "rate": 0 + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.visual_editor_feature_use", + "topics": [ + "eqiad.mediawiki.visual_editor_feature_use", + "codfw.mediawiki.visual_editor_feature_use" + ] + }, + "mediawiki.wikistories_consumption_event": { + "schema_title": "analytics/mediawiki/wikistories_consumption_event", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.wikistories_consumption_event", + "topics": [ + "eqiad.mediawiki.wikistories_consumption_event", + "codfw.mediawiki.wikistories_consumption_event" + ] + }, + "mediawiki.wikistories_contribution_event": { + "schema_title": "analytics/mediawiki/wikistories_contribution_event", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.wikistories_contribution_event", + "topics": [ + "eqiad.mediawiki.wikistories_contribution_event", + "codfw.mediawiki.wikistories_contribution_event" + ] + }, + "rc0.mediawiki.page_change": { + "schema_title": "development/mediawiki/page/change", + "message_key_fields": { + "wiki_id": "wiki_id", + "page_id": "page.page_id" + }, + "destination_event_service": "eventgate-analytics-external", + "producers": { + "mediawiki_eventbus": { + "enabled": false + } + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "rc0.mediawiki.page_change", + "topics": [ + "eqiad.rc0.mediawiki.page_change", + "codfw.rc0.mediawiki.page_change" + ] + }, + "rc0.mediawiki.page_content_change": { + "schema_title": "development/mediawiki/page/change", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "rc0.mediawiki.page_content_change", + "topics": [ + "eqiad.rc0.mediawiki.page_content_change", + "codfw.rc0.mediawiki.page_content_change" + ] + }, + "mediawiki.maps_interaction": { + "schema_title": "analytics/mediawiki/maps/interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.maps_interaction", + "topics": [ + "eqiad.mediawiki.maps_interaction", + "codfw.mediawiki.maps_interaction" + ] + }, + "eventgate-analytics-external.test.event": { + "schema_title": "test/event", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "eventgate-analytics-external.test.event", + "topics": [ + "eqiad.eventgate-analytics-external.test.event", + "codfw.eventgate-analytics-external.test.event" + ] + }, + "eventgate-analytics-external.error.validation": { + "schema_title": "error", + "destination_event_service": "eventgate-analytics-external", + "canary_events_enabled": false, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "eventgate-analytics-external.error.validation", + "topics": [ + "eqiad.eventgate-analytics-external.error.validation", + "codfw.eventgate-analytics-external.error.validation" + ] + } + } +} diff --git a/analytics/metricsplatform/src/test/resources/org/wikimedia/metricsplatform/config/streamconfigs.json b/analytics/metricsplatform/src/test/resources/org/wikimedia/metricsplatform/config/streamconfigs.json new file mode 100644 index 00000000000..d54ea8a8626 --- /dev/null +++ b/analytics/metricsplatform/src/test/resources/org/wikimedia/metricsplatform/config/streamconfigs.json @@ -0,0 +1,1934 @@ +{ + "streams": { + "eventlogging_CentralNoticeBannerHistory": { + "schema_title": "analytics/legacy/centralnoticebannerhistory", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_CentralNoticeBannerHistory", + "topics": [ + "eventlogging_CentralNoticeBannerHistory" + ] + }, + "eventlogging_CentralNoticeImpression": { + "schema_title": "analytics/legacy/centralnoticeimpression", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_CentralNoticeImpression", + "topics": [ + "eventlogging_CentralNoticeImpression" + ] + }, + "eventlogging_CentralNoticeTiming": { + "schema_title": "analytics/legacy/centralnoticetiming", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_CentralNoticeTiming", + "topics": [ + "eventlogging_CentralNoticeTiming" + ] + }, + "eventlogging_CodeMirrorUsage": { + "schema_title": "analytics/legacy/codemirrorusage", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_CodeMirrorUsage", + "topics": [ + "eventlogging_CodeMirrorUsage" + ] + }, + "eventlogging_ContentTranslationAbuseFilter": { + "schema_title": "analytics/legacy/contenttranslationabusefilter", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_ContentTranslationAbuseFilter", + "topics": [ + "eventlogging_ContentTranslationAbuseFilter" + ] + }, + "eventlogging_CpuBenchmark": { + "schema_title": "analytics/legacy/cpubenchmark", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_CpuBenchmark", + "topics": [ + "eventlogging_CpuBenchmark" + ] + }, + "eventlogging_DesktopWebUIActionsTracking": { + "schema_title": "analytics/legacy/desktopwebuiactionstracking", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_DesktopWebUIActionsTracking", + "topics": [ + "eventlogging_DesktopWebUIActionsTracking" + ] + }, + "eventlogging_ElementTiming": { + "schema_title": "analytics/legacy/elementtiming", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_ElementTiming", + "topics": [ + "eventlogging_ElementTiming" + ] + }, + "eventlogging_EditAttemptStep": { + "schema_title": "analytics/legacy/editattemptstep", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_EditAttemptStep", + "topics": [ + "eventlogging_EditAttemptStep" + ] + }, + "eventlogging_FeaturePolicyViolation": { + "schema_title": "analytics/legacy/featurepolicyviolation", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_FeaturePolicyViolation", + "topics": [ + "eventlogging_FeaturePolicyViolation" + ] + }, + "eventlogging_FirstInputTiming": { + "schema_title": "analytics/legacy/firstinputtiming", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_FirstInputTiming", + "topics": [ + "eventlogging_FirstInputTiming" + ] + }, + "eventlogging_HelpPanel": { + "schema_title": "analytics/legacy/helppanel", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_HelpPanel", + "topics": [ + "eventlogging_HelpPanel" + ] + }, + "eventlogging_HomepageModule": { + "schema_title": "analytics/legacy/homepagemodule", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_HomepageModule", + "topics": [ + "eventlogging_HomepageModule" + ] + }, + "eventlogging_HomepageVisit": { + "schema_title": "analytics/legacy/homepagevisit", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_HomepageVisit", + "topics": [ + "eventlogging_HomepageVisit" + ] + }, + "eventlogging_InukaPageView": { + "schema_title": "analytics/legacy/inukapageview", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_InukaPageView", + "topics": [ + "eventlogging_InukaPageView" + ] + }, + "eventlogging_KaiOSAppFirstRun": { + "schema_title": "analytics/legacy/kaiosappfirstrun", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_KaiOSAppFirstRun", + "topics": [ + "eventlogging_KaiOSAppFirstRun" + ] + }, + "eventlogging_KaiOSAppFeedback": { + "schema_title": "analytics/legacy/kaiosappfeedback", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_KaiOSAppFeedback", + "topics": [ + "eventlogging_KaiOSAppFeedback" + ] + }, + "eventlogging_LandingPageImpression": { + "schema_title": "analytics/legacy/landingpageimpression", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_LandingPageImpression", + "topics": [ + "eventlogging_LandingPageImpression" + ] + }, + "eventlogging_LayoutShift": { + "schema_title": "analytics/legacy/layoutshift", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_LayoutShift", + "topics": [ + "eventlogging_LayoutShift" + ] + }, + "eventlogging_MobileWebUIActionsTracking": { + "schema_title": "analytics/legacy/mobilewebuiactionstracking", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_MobileWebUIActionsTracking", + "topics": [ + "eventlogging_MobileWebUIActionsTracking" + ] + }, + "eventlogging_NavigationTiming": { + "schema_title": "analytics/legacy/navigationtiming", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_NavigationTiming", + "topics": [ + "eventlogging_NavigationTiming" + ] + }, + "eventlogging_NewcomerTask": { + "schema_title": "analytics/legacy/newcomertask", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_NewcomerTask", + "topics": [ + "eventlogging_NewcomerTask" + ] + }, + "eventlogging_PaintTiming": { + "schema_title": "analytics/legacy/painttiming", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_PaintTiming", + "topics": [ + "eventlogging_PaintTiming" + ] + }, + "eventlogging_PrefUpdate": { + "schema_title": "analytics/legacy/prefupdate", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_PrefUpdate", + "topics": [ + "eventlogging_PrefUpdate" + ] + }, + "eventlogging_WikipediaPortal": { + "schema_title": "analytics/legacy/wikipediaportal", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_WikipediaPortal", + "topics": [ + "eventlogging_WikipediaPortal" + ] + }, + "eventlogging_QuickSurveyInitiation": { + "schema_title": "analytics/legacy/quicksurveyinitiation", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_QuickSurveyInitiation", + "topics": [ + "eventlogging_QuickSurveyInitiation" + ] + }, + "eventlogging_QuickSurveysResponses": { + "schema_title": "analytics/legacy/quicksurveysresponses", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_QuickSurveysResponses", + "topics": [ + "eventlogging_QuickSurveysResponses" + ] + }, + "eventlogging_ReferencePreviewsBaseline": { + "schema_title": "analytics/legacy/referencepreviewsbaseline", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_ReferencePreviewsBaseline", + "topics": [ + "eventlogging_ReferencePreviewsBaseline" + ] + }, + "eventlogging_ReferencePreviewsCite": { + "schema_title": "analytics/legacy/referencepreviewscite", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_ReferencePreviewsCite", + "topics": [ + "eventlogging_ReferencePreviewsCite" + ] + }, + "eventlogging_ReferencePreviewsPopups": { + "schema_title": "analytics/legacy/referencepreviewspopups", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_ReferencePreviewsPopups", + "topics": [ + "eventlogging_ReferencePreviewsPopups" + ] + }, + "eventlogging_ResourceTiming": { + "schema_title": "analytics/legacy/resourcetiming", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_ResourceTiming", + "topics": [ + "eventlogging_ResourceTiming" + ] + }, + "eventlogging_RUMSpeedIndex": { + "schema_title": "analytics/legacy/rumspeedindex", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_RUMSpeedIndex", + "topics": [ + "eventlogging_RUMSpeedIndex" + ] + }, + "eventlogging_SaveTiming": { + "schema_title": "analytics/legacy/savetiming", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_SaveTiming", + "topics": [ + "eventlogging_SaveTiming" + ] + }, + "eventlogging_SearchSatisfaction": { + "schema_title": "analytics/legacy/searchsatisfaction", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_SearchSatisfaction", + "topics": [ + "eventlogging_SearchSatisfaction" + ] + }, + "eventlogging_ServerSideAccountCreation": { + "schema_title": "analytics/legacy/serversideaccountcreation", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_ServerSideAccountCreation", + "topics": [ + "eventlogging_ServerSideAccountCreation" + ] + }, + "eventlogging_SpecialInvestigate": { + "schema_title": "analytics/legacy/specialinvestigate", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_SpecialInvestigate", + "topics": [ + "eventlogging_SpecialInvestigate" + ] + }, + "eventlogging_SpecialMuteSubmit": { + "schema_title": "analytics/legacy/specialmutesubmit", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_SpecialMuteSubmit", + "topics": [ + "eventlogging_SpecialMuteSubmit" + ] + }, + "eventlogging_SuggestedTagsAction": { + "schema_title": "analytics/legacy/suggestedtagsaction", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_SuggestedTagsAction", + "topics": [ + "eventlogging_SuggestedTagsAction" + ] + }, + "eventlogging_TemplateDataApi": { + "schema_title": "analytics/legacy/templatedataapi", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_TemplateDataApi", + "topics": [ + "eventlogging_TemplateDataApi" + ] + }, + "eventlogging_TemplateDataEditor": { + "schema_title": "analytics/legacy/templatedataeditor", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_TemplateDataEditor", + "topics": [ + "eventlogging_TemplateDataEditor" + ] + }, + "eventlogging_TemplateWizard": { + "schema_title": "analytics/legacy/templatewizard", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_TemplateWizard", + "topics": [ + "eventlogging_TemplateWizard" + ] + }, + "eventlogging_Test": { + "schema_title": "analytics/legacy/test", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_Test", + "topics": [ + "eventlogging_Test" + ] + }, + "eventlogging_TranslationRecommendationUserAction": { + "schema_title": "analytics/legacy/translationrecommendationuseraction", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_TranslationRecommendationUserAction", + "topics": [ + "eventlogging_TranslationRecommendationUserAction" + ] + }, + "eventlogging_TranslationRecommendationUIRequests": { + "schema_title": "analytics/legacy/translationrecommendationuirequests", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_TranslationRecommendationUIRequests", + "topics": [ + "eventlogging_TranslationRecommendationUIRequests" + ] + }, + "eventlogging_TranslationRecommendationAPIRequests": { + "schema_title": "analytics/legacy/translationrecommendationapirequests", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_TranslationRecommendationAPIRequests", + "topics": [ + "eventlogging_TranslationRecommendationAPIRequests" + ] + }, + "eventlogging_TwoColConflictConflict": { + "schema_title": "analytics/legacy/twocolconflictconflict", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_TwoColConflictConflict", + "topics": [ + "eventlogging_TwoColConflictConflict" + ] + }, + "eventlogging_TwoColConflictExit": { + "schema_title": "analytics/legacy/twocolconflictexit", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_TwoColConflictExit", + "topics": [ + "eventlogging_TwoColConflictExit" + ] + }, + "eventlogging_UniversalLanguageSelector": { + "schema_title": "analytics/legacy/universallanguageselector", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_UniversalLanguageSelector", + "topics": [ + "eventlogging_UniversalLanguageSelector" + ] + }, + "eventlogging_VirtualPageView": { + "schema_title": "analytics/legacy/virtualpageview", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_VirtualPageView", + "topics": [ + "eventlogging_VirtualPageView" + ] + }, + "eventlogging_VisualEditorFeatureUse": { + "schema_title": "analytics/legacy/visualeditorfeatureuse", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_VisualEditorFeatureUse", + "topics": [ + "eventlogging_VisualEditorFeatureUse" + ] + }, + "eventlogging_VisualEditorTemplateDialogUse": { + "schema_title": "analytics/legacy/visualeditortemplatedialoguse", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_VisualEditorTemplateDialogUse", + "topics": [ + "eventlogging_VisualEditorTemplateDialogUse" + ] + }, + "eventlogging_WikibaseTermboxInteraction": { + "schema_title": "analytics/legacy/wikibasetermboxinteraction", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_WikibaseTermboxInteraction", + "topics": [ + "eventlogging_WikibaseTermboxInteraction" + ] + }, + "eventlogging_WikidataCompletionSearchClicks": { + "schema_title": "analytics/legacy/wikidatacompletionsearchclicks", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_WikidataCompletionSearchClicks", + "topics": [ + "eventlogging_WikidataCompletionSearchClicks" + ] + }, + "eventlogging_WMDEBannerEvents": { + "schema_title": "analytics/legacy/wmdebannerevents", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_WMDEBannerEvents", + "topics": [ + "eventlogging_WMDEBannerEvents" + ] + }, + "eventlogging_WMDEBannerInteractions": { + "schema_title": "analytics/legacy/wmdebannerinteractions", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_WMDEBannerInteractions", + "topics": [ + "eventlogging_WMDEBannerInteractions" + ] + }, + "eventlogging_WMDEBannerSizeIssue": { + "schema_title": "analytics/legacy/wmdebannersizeissue", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_WMDEBannerSizeIssue", + "topics": [ + "eventlogging_WMDEBannerSizeIssue" + ] + }, + "test.instrumentation": { + "schema_title": "analytics/test", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "test.instrumentation", + "topics": [ + "eqiad.test.instrumentation", + "codfw.test.instrumentation" + ] + }, + "test.instrumentation.sampled": { + "schema_title": "analytics/test", + "destination_event_service": "eventgate-analytics-external", + "sample": { + "rate": 0.5, + "unit": "session" + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "test.instrumentation.sampled", + "topics": [ + "eqiad.test.instrumentation.sampled", + "codfw.test.instrumentation.sampled" + ] + }, + "mediawiki.client.session_tick": { + "schema_title": "analytics/session_tick", + "destination_event_service": "eventgate-analytics-external", + "sample": { + "unit": "session", + "rate": 0.1 + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.client.session_tick", + "topics": [ + "eqiad.mediawiki.client.session_tick", + "codfw.mediawiki.client.session_tick" + ] + }, + "ios.edit_history_compare": { + "schema_title": "analytics/mobile_apps/ios_edit_history_compare", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "ios.edit_history_compare", + "topics": [ + "eqiad.ios.edit_history_compare", + "codfw.ios.edit_history_compare" + ] + }, + "ios.notification_interaction": { + "schema_title": "analytics/mobile_apps/ios_notification_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "ios.notification_interaction", + "topics": [ + "eqiad.ios.notification_interaction", + "codfw.ios.notification_interaction" + ] + }, + "android.user_contribution_screen": { + "schema_title": "analytics/mobile_apps/android_user_contribution_screen", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.user_contribution_screen", + "topics": [ + "eqiad.android.user_contribution_screen", + "codfw.android.user_contribution_screen" + ] + }, + "android.notification_interaction": { + "schema_title": "analytics/mobile_apps/android_notification_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.notification_interaction", + "topics": [ + "eqiad.android.notification_interaction", + "codfw.android.notification_interaction" + ] + }, + "android.image_recommendation_interaction": { + "schema_title": "analytics/mobile_apps/android_image_recommendation_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.image_recommendation_interaction", + "topics": [ + "eqiad.android.image_recommendation_interaction", + "codfw.android.image_recommendation_interaction" + ] + }, + "android.daily_stats": { + "schema_title": "analytics/mobile_apps/android_daily_stats", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.daily_stats", + "topics": [ + "eqiad.android.daily_stats", + "codfw.android.daily_stats" + ] + }, + "android.customize_toolbar_interaction": { + "schema_title": "analytics/mobile_apps/android_customize_toolbar_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.customize_toolbar_interaction", + "topics": [ + "eqiad.android.customize_toolbar_interaction", + "codfw.android.customize_toolbar_interaction" + ] + }, + "android.article_toolbar_interaction": { + "schema_title": "analytics/mobile_apps/android_article_toolbar_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.article_toolbar_interaction", + "topics": [ + "eqiad.android.article_toolbar_interaction", + "codfw.android.article_toolbar_interaction" + ] + }, + "android.edit_history_interaction": { + "schema_title": "analytics/mobile_apps/android_edit_history_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.edit_history_interaction", + "topics": [ + "eqiad.android.edit_history_interaction", + "codfw.android.edit_history_interaction" + ] + }, + "android.breadcrumbs_event": { + "schema_title": "analytics/mobile_apps/android_breadcrumbs_event", + "destination_event_service": "eventgate-analytics-external", + "sample": { + "unit": "device", + "rate": 0.5 + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.breadcrumbs_event", + "topics": [ + "eqiad.android.breadcrumbs_event", + "codfw.android.breadcrumbs_event" + ] + }, + "android.app_appearance_settings_interaction": { + "schema_title": "analytics/mobile_apps/android_app_appearance_settings_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.app_appearance_settings_interaction", + "topics": [ + "eqiad.android.app_appearance_settings_interaction", + "codfw.android.app_appearance_settings_interaction" + ] + }, + "android.article_link_preview_interaction": { + "schema_title": "analytics/mobile_apps/android_article_link_preview_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.article_link_preview_interaction", + "topics": [ + "eqiad.android.article_link_preview_interaction", + "codfw.android.article_link_preview_interaction" + ] + }, + "android.article_page_scroll_interaction": { + "schema_title": "analytics/mobile_apps/android_article_page_scroll_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.article_page_scroll_interaction", + "topics": [ + "eqiad.android.article_page_scroll_interaction", + "codfw.android.article_page_scroll_interaction" + ] + }, + "android.article_toc_interaction": { + "schema_title": "analytics/mobile_apps/android_article_toc_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.article_toc_interaction", + "topics": [ + "eqiad.android.article_toc_interaction", + "codfw.android.article_toc_interaction" + ] + }, + "android.find_in_page_interaction": { + "schema_title": "analytics/mobile_apps/android_find_in_page_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.find_in_page_interaction", + "topics": [ + "eqiad.android.find_in_page_interaction", + "codfw.android.find_in_page_interaction" + ] + }, + "mediawiki.mediasearch_interaction": { + "schema_title": "analytics/mediawiki/mediasearch_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.mediasearch_interaction", + "topics": [ + "eqiad.mediawiki.mediasearch_interaction", + "codfw.mediawiki.mediasearch_interaction" + ] + }, + "mediawiki.structured_task.article.link_suggestion_interaction": { + "schema_title": "analytics/mediawiki/structured_task/article/link_suggestion_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.structured_task.article.link_suggestion_interaction", + "topics": [ + "eqiad.mediawiki.structured_task.article.link_suggestion_interaction", + "codfw.mediawiki.structured_task.article.link_suggestion_interaction" + ] + }, + "mediawiki.structured_task.article.image_suggestion_interaction": { + "schema_title": "analytics/mediawiki/structured_task/article/image_suggestion_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.structured_task.article.image_suggestion_interaction", + "topics": [ + "eqiad.mediawiki.structured_task.article.image_suggestion_interaction", + "codfw.mediawiki.structured_task.article.image_suggestion_interaction" + ] + }, + "mediawiki.pref_diff": { + "schema_title": "analytics/pref_diff", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.pref_diff", + "topics": [ + "eqiad.mediawiki.pref_diff", + "codfw.mediawiki.pref_diff" + ] + }, + "mediawiki.skin_diff": { + "schema_title": "analytics/pref_diff", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.skin_diff", + "topics": [ + "eqiad.mediawiki.skin_diff", + "codfw.mediawiki.skin_diff" + ] + }, + "mediawiki.content_translation_event": { + "schema_title": "analytics/mediawiki/content_translation_event", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.content_translation_event", + "topics": [ + "eqiad.mediawiki.content_translation_event", + "codfw.mediawiki.content_translation_event" + ] + }, + "mediawiki.reading_depth": { + "schema_title": "analytics/mediawiki/web_ui_reading_depth", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.reading_depth", + "topics": [ + "eqiad.mediawiki.reading_depth", + "codfw.mediawiki.reading_depth" + ] + }, + "mediawiki.web_ab_test_enrollment": { + "schema_title": "analytics/mediawiki/web_ab_test_enrollment", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.web_ab_test_enrollment", + "topics": [ + "eqiad.mediawiki.web_ab_test_enrollment", + "codfw.mediawiki.web_ab_test_enrollment" + ] + }, + "mediawiki.web_ui_scroll": { + "schema_title": "analytics/mediawiki/web_ui_scroll", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.web_ui_scroll", + "topics": [ + "eqiad.mediawiki.web_ui_scroll", + "codfw.mediawiki.web_ui_scroll" + ] + }, + "mediawiki.ipinfo_interaction": { + "schema_title": "analytics/mediawiki/ipinfo_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.ipinfo_interaction", + "topics": [ + "eqiad.mediawiki.ipinfo_interaction", + "codfw.mediawiki.ipinfo_interaction" + ] + }, + "wd_propertysuggester.client_side_property_request": { + "schema_title": "analytics/mediawiki/wd_propertysuggester/client_side_property_request", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "wd_propertysuggester.client_side_property_request", + "topics": [ + "eqiad.wd_propertysuggester.client_side_property_request", + "codfw.wd_propertysuggester.client_side_property_request" + ] + }, + "wd_propertysuggester.server_side_property_request": { + "schema_title": "analytics/mediawiki/wd_propertysuggester/server_side_property_request", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "wd_propertysuggester.server_side_property_request", + "topics": [ + "eqiad.wd_propertysuggester.server_side_property_request", + "codfw.wd_propertysuggester.server_side_property_request" + ] + }, + "mediawiki.mentor_dashboard.visit": { + "schema_title": "analytics/mediawiki/mentor_dashboard/visit", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.mentor_dashboard.visit", + "topics": [ + "eqiad.mediawiki.mentor_dashboard.visit", + "codfw.mediawiki.mentor_dashboard.visit" + ] + }, + "mediawiki.welcomesurvey.interaction": { + "schema_title": "analytics/mediawiki/welcomesurvey/interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.welcomesurvey.interaction", + "topics": [ + "eqiad.mediawiki.welcomesurvey.interaction", + "codfw.mediawiki.welcomesurvey.interaction" + ] + }, + "mediawiki.editgrowthconfig": { + "schema_title": "analytics/mediawiki/editgrowthconfig", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.editgrowthconfig", + "topics": [ + "eqiad.mediawiki.editgrowthconfig", + "codfw.mediawiki.editgrowthconfig" + ] + }, + "mediawiki.accountcreation_block": { + "schema_title": "analytics/mediawiki/accountcreation/block", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.accountcreation_block", + "topics": [ + "eqiad.mediawiki.accountcreation_block", + "codfw.mediawiki.accountcreation_block" + ] + }, + "mediawiki.editattempt_block": { + "schema_title": "analytics/mediawiki/editattemptsblocked", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.editattempt_block", + "topics": [ + "eqiad.mediawiki.editattempt_block", + "codfw.mediawiki.editattempt_block" + ] + }, + "mediawiki.talk_page_edit": { + "stream": "mediawiki.talk_page_edit", + "schema_title": "analytics/mediawiki/talk_page_edit", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "topics": [ + "eqiad.mediawiki.talk_page_edit", + "codfw.mediawiki.talk_page_edit" + ] + }, + "mwcli.command_execute": { + "schema_title": "analytics/mwcli/command_execute", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mwcli.command_execute", + "topics": [ + "eqiad.mwcli.command_execute", + "codfw.mwcli.command_execute" + ] + }, + "mediawiki.web_ui.interactions": { + "schema_title": "analytics/mediawiki/client/metrics_event", + "destination_event_service": "eventgate-analytics-external", + "producers": { + "metrics_platform_client": { + "events": [ + "web.ui.", + "web_ui." + ], + "provide_values": [ + "page_namespace", + "performer_is_logged_in", + "performer_session_id", + "performer_pageview_id", + "performer_edit_count_bucket", + "mediawiki_skin", + "mediawiki_database" + ], + "curation": { + "mediawiki_skin": { + "in": [ + "minerva", + "vector", + "vector-2022" + ] + } + } + } + }, + "sample": { + "unit": "pageview", + "rate": 0 + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.web_ui.interactions", + "topics": [ + "eqiad.mediawiki.web_ui.interactions", + "codfw.mediawiki.web_ui.interactions" + ] + }, + "mediawiki.edit_attempt": { + "schema_title": "analytics/mediawiki/client/metrics_event", + "destination_event_service": "eventgate-analytics-external", + "producers": { + "metrics_platform_client": { + "events": [ + "eas." + ], + "provide_values": [ + "agent_client_platform_family", + "page_id", + "page_title", + "page_namespace", + "page_revision_id", + "mediawiki_version", + "mediawiki_is_debug_mode", + "mediawiki_database", + "performer_is_logged_in", + "performer_id", + "performer_session_id", + "performer_pageview_id", + "performer_edit_count" + ] + } + }, + "sample": { + "unit": "pageview", + "rate": 1 + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.edit_attempt", + "topics": [ + "eqiad.mediawiki.edit_attempt", + "codfw.mediawiki.edit_attempt" + ] + }, + "mediawiki.visual_editor_feature_use": { + "schema_title": "analytics/mediawiki/client/metrics_event", + "destination_event_service": "eventgate-analytics-external", + "producers": { + "metrics_platform_client": { + "events": [ + "vefu." + ], + "provide_values": [ + "agent_client_platform_family", + "mediawiki_database", + "performer_id", + "performer_edit_count" + ] + } + }, + "sample": { + "unit": "pageview", + "rate": 0 + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.visual_editor_feature_use", + "topics": [ + "eqiad.mediawiki.visual_editor_feature_use", + "codfw.mediawiki.visual_editor_feature_use" + ] + }, + "mediawiki.wikistories_consumption_event": { + "schema_title": "analytics/mediawiki/wikistories_consumption_event", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.wikistories_consumption_event", + "topics": [ + "eqiad.mediawiki.wikistories_consumption_event", + "codfw.mediawiki.wikistories_consumption_event" + ] + }, + "mediawiki.wikistories_contribution_event": { + "schema_title": "analytics/mediawiki/wikistories_contribution_event", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.wikistories_contribution_event", + "topics": [ + "eqiad.mediawiki.wikistories_contribution_event", + "codfw.mediawiki.wikistories_contribution_event" + ] + }, + "rc0.mediawiki.page_change": { + "schema_title": "development/mediawiki/page/change", + "message_key_fields": { + "wiki_id": "wiki_id", + "page_id": "page.page_id" + }, + "destination_event_service": "eventgate-analytics-external", + "producers": { + "mediawiki_eventbus": { + "enabled": false + } + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "rc0.mediawiki.page_change", + "topics": [ + "eqiad.rc0.mediawiki.page_change", + "codfw.rc0.mediawiki.page_change" + ] + }, + "rc0.mediawiki.page_content_change": { + "schema_title": "development/mediawiki/page/change", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "rc0.mediawiki.page_content_change", + "topics": [ + "eqiad.rc0.mediawiki.page_content_change", + "codfw.rc0.mediawiki.page_content_change" + ] + }, + "mediawiki.maps_interaction": { + "schema_title": "analytics/mediawiki/maps/interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.maps_interaction", + "topics": [ + "eqiad.mediawiki.maps_interaction", + "codfw.mediawiki.maps_interaction" + ] + }, + "eventgate-analytics-external.test.event": { + "schema_title": "test/event", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "eventgate-analytics-external.test.event", + "topics": [ + "eqiad.eventgate-analytics-external.test.event", + "codfw.eventgate-analytics-external.test.event" + ] + }, + "eventgate-analytics-external.error.validation": { + "schema_title": "error", + "destination_event_service": "eventgate-analytics-external", + "canary_events_enabled": false, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "eventgate-analytics-external.error.validation", + "topics": [ + "eqiad.eventgate-analytics-external.error.validation", + "codfw.eventgate-analytics-external.error.validation" + ] + } + } +} \ No newline at end of file diff --git a/analytics/metricsplatform/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_click.json b/analytics/metricsplatform/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_click.json new file mode 100644 index 00000000000..c31e50ad32e --- /dev/null +++ b/analytics/metricsplatform/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_click.json @@ -0,0 +1,43 @@ +[ { + "$schema": "/analytics/product_metrics/app/base/1.2.2", + "meta": { + "stream": "mediawiki.metrics_platform.click", + "domain": "en.wikipedia.org" + }, + "agent": { + "app_flavor": "devdebug", + "app_install_id": "ffffffff-ffff-ffff-ffff-ffffffffffff", + "app_theme": "LIGHT", + "app_version": 123456, + "app_version_name": "2.7.50470-dev-2024-02-14", + "client_platform": "android", + "client_platform_family": "app", + "device_family": "samsung", + "device_language": "en", + "release_status": "dev" + }, + "page": { + "id": 30715282, + "title": "Dark_Souls", + "namespace_id": 0, + "revision_id": 1084793259 + }, + "mediawiki": { + "database": "enwiki" + }, + "performer": { + "is_logged_in": false, + "id": 0, + "session_id": "eeeeeeeeeeeeeeeeeeee", + "pageview_id": "eeeeeeeeeeeeeeeeeeee" + }, + "action" : "TestClick", + "action_subtype" : "TestActionSubtype", + "action_source" : "TestActionSource", + "action_context" : "TestActionContext", + "element_id" : "TestElementId", + "element_friendly_name" : "TestElementFriendlyName", + "funnel_entry_token" : "TestFunnelEntryToken", + "funnel_event_sequence_position" : 8, + "dt" : "${json-unit.ignore}" +} ] diff --git a/analytics/metricsplatform/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_click_custom.json b/analytics/metricsplatform/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_click_custom.json new file mode 100644 index 00000000000..01e3868996d --- /dev/null +++ b/analytics/metricsplatform/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_click_custom.json @@ -0,0 +1,46 @@ +[ { + "$schema": "/analytics/product_metrics/app/click_custom/1.0.0", + "meta": { + "stream": "mediawiki.metrics_platform.click_custom", + "domain": "en.wikipedia.org" + }, + "font_size": "small", + "is_full_width": true, + "screen_size": 1080, + "agent": { + "app_flavor": "devdebug", + "app_install_id": "ffffffff-ffff-ffff-ffff-ffffffffffff", + "app_theme": "LIGHT", + "app_version": 123456, + "app_version_name": "2.7.50470-dev-2024-02-14", + "client_platform": "android", + "client_platform_family": "app", + "device_family": "samsung", + "device_language": "en", + "release_status": "dev" + }, + "page": { + "id": 30715282, + "title": "Dark_Souls", + "namespace_id": 0, + "revision_id": 1084793259 + }, + "mediawiki": { + "database": "enwiki" + }, + "performer": { + "is_logged_in": false, + "id": 0, + "session_id": "eeeeeeeeeeeeeeeeeeee", + "pageview_id": "eeeeeeeeeeeeeeeeeeee" + }, + "action" : "TestClickCustom", + "action_subtype" : "TestActionSubtype", + "action_source" : "TestActionSource", + "action_context" : "TestActionContext", + "element_id" : "TestElementId", + "element_friendly_name" : "TestElementFriendlyName", + "funnel_entry_token" : "TestFunnelEntryToken", + "funnel_event_sequence_position" : 8, + "dt" : "${json-unit.ignore}" +} ] diff --git a/analytics/metricsplatform/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_interaction.json b/analytics/metricsplatform/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_interaction.json new file mode 100644 index 00000000000..76949a91bbc --- /dev/null +++ b/analytics/metricsplatform/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_interaction.json @@ -0,0 +1,43 @@ +[ { + "$schema": "/analytics/product_metrics/app/base/1.2.2", + "meta": { + "stream": "mediawiki.metrics_platform.interaction", + "domain": "en.wikipedia.org" + }, + "agent": { + "app_flavor": "devdebug", + "app_install_id": "ffffffff-ffff-ffff-ffff-ffffffffffff", + "app_theme": "LIGHT", + "app_version": 123456, + "app_version_name": "2.7.50470-dev-2024-02-14", + "client_platform": "android", + "client_platform_family": "app", + "device_family": "samsung", + "device_language": "en", + "release_status": "dev" + }, + "page": { + "id": 30715282, + "title": "Dark_Souls", + "namespace_id": 0, + "revision_id": 1084793259 + }, + "mediawiki": { + "database": "enwiki" + }, + "performer": { + "is_logged_in": false, + "id": 0, + "session_id": "eeeeeeeeeeeeeeeeeeee", + "pageview_id": "eeeeeeeeeeeeeeeeeeee" + }, + "action" : "TestInteraction", + "action_subtype" : "TestActionSubtype", + "action_source" : "TestActionSource", + "action_context" : "TestActionContext", + "element_id" : "TestElementId", + "element_friendly_name" : "TestElementFriendlyName", + "funnel_entry_token" : "TestFunnelEntryToken", + "funnel_event_sequence_position" : 8, + "dt" : "${json-unit.ignore}" +} ] diff --git a/analytics/metricsplatform/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_view.json b/analytics/metricsplatform/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_view.json new file mode 100644 index 00000000000..56dbfbf310f --- /dev/null +++ b/analytics/metricsplatform/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_view.json @@ -0,0 +1,43 @@ +[ { + "$schema": "/analytics/product_metrics/app/base/1.2.2", + "meta": { + "stream": "mediawiki.metrics_platform.view", + "domain": "en.wikipedia.org" + }, + "agent": { + "app_flavor": "devdebug", + "app_install_id": "ffffffff-ffff-ffff-ffff-ffffffffffff", + "app_theme": "LIGHT", + "app_version": 123456, + "app_version_name": "2.7.50470-dev-2024-02-14", + "client_platform": "android", + "client_platform_family": "app", + "device_family": "samsung", + "device_language": "en", + "release_status": "dev" + }, + "page": { + "id": 30715282, + "title": "Dark_Souls", + "namespace_id": 0, + "revision_id": 1084793259 + }, + "mediawiki": { + "database": "enwiki" + }, + "performer": { + "is_logged_in": false, + "id": 0, + "session_id": "eeeeeeeeeeeeeeeeeeee", + "pageview_id": "eeeeeeeeeeeeeeeeeeee" + }, + "action" : "TestView", + "action_subtype" : "TestActionSubtype", + "action_source" : "TestActionSource", + "action_context" : "TestActionContext", + "element_id" : "TestElementId", + "element_friendly_name" : "TestElementFriendlyName", + "funnel_entry_token" : "TestFunnelEntryToken", + "funnel_event_sequence_position" : 8, + "dt" : "${json-unit.ignore}" +} ] diff --git a/analytics/metricsplatform/src/test/resources/simplelogger.properties b/analytics/metricsplatform/src/test/resources/simplelogger.properties new file mode 100644 index 00000000000..906149cfb44 --- /dev/null +++ b/analytics/metricsplatform/src/test/resources/simplelogger.properties @@ -0,0 +1,2 @@ +org.slf4j.simpleLogger.defaultLogLevel=WARN +org.slf4j.simpleLogger.org.wikimedia.metrics_platform=DEBUG diff --git a/app/build.gradle b/app/build.gradle index 36c78ecea24..b5384882891 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -177,6 +177,8 @@ dependencies { coreLibraryDesugaring libs.desugar.jdk.libs + implementation project(':analytics:metricsplatform') + implementation libs.kotlin.stdlib.jdk8 implementation libs.kotlinx.coroutines.core implementation libs.kotlinx.coroutines.android @@ -197,7 +199,6 @@ dependencies { implementation libs.drawerlayout implementation libs.swiperefreshlayout implementation libs.work.runtime.ktx - implementation libs.metrics.platform implementation libs.okhttp.tls implementation libs.okhttp3.logging.interceptor diff --git a/app/src/main/java/org/wikipedia/analytics/eventplatform/EventPlatformClient.kt b/app/src/main/java/org/wikipedia/analytics/eventplatform/EventPlatformClient.kt index bf205817f7a..d6ff3646337 100644 --- a/app/src/main/java/org/wikipedia/analytics/eventplatform/EventPlatformClient.kt +++ b/app/src/main/java/org/wikipedia/analytics/eventplatform/EventPlatformClient.kt @@ -7,6 +7,8 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.MainScope import kotlinx.coroutines.launch +import org.wikimedia.metricsplatform.config.StreamConfig +import org.wikimedia.metricsplatform.config.sampling.SampleConfig import org.wikipedia.BuildConfig import org.wikipedia.WikipediaApp import org.wikipedia.dataclient.ServiceFactory @@ -328,7 +330,7 @@ object EventPlatformClient { return SAMPLING_CACHE[stream]!! } val streamConfig = getStreamConfig(stream) ?: return false - val samplingConfig = streamConfig.samplingConfig + val samplingConfig = streamConfig.sampleConfig if (samplingConfig == null || samplingConfig.rate == 1.0) { return true } @@ -350,13 +352,13 @@ object EventPlatformClient { } fun getSamplingId(unit: String): String { - if (unit == SamplingConfig.UNIT_SESSION) { + if (unit == SampleConfig.UNIT_SESSION) { return AssociationController.sessionId } - if (unit == SamplingConfig.UNIT_PAGEVIEW) { + if (unit == SampleConfig.UNIT_PAGEVIEW) { return AssociationController.pageViewId } - if (unit == SamplingConfig.UNIT_DEVICE) { + if (unit == SampleConfig.UNIT_DEVICE) { return WikipediaApp.instance.appInstallID } L.e("Bad identifier type") diff --git a/app/src/main/java/org/wikipedia/analytics/eventplatform/StreamConfig.kt b/app/src/main/java/org/wikipedia/analytics/eventplatform/StreamConfig.kt deleted file mode 100644 index 9b399b7b144..00000000000 --- a/app/src/main/java/org/wikipedia/analytics/eventplatform/StreamConfig.kt +++ /dev/null @@ -1,43 +0,0 @@ -package org.wikipedia.analytics.eventplatform - -import kotlinx.serialization.SerialName -import kotlinx.serialization.Serializable -import org.wikipedia.analytics.eventplatform.DestinationEventService.ANALYTICS -import java.lang.IllegalArgumentException - -@Serializable -class StreamConfig { - - constructor(streamName: String, samplingConfig: SamplingConfig?, destinationEventService: DestinationEventService?) { - this.streamName = streamName - this.samplingConfig = samplingConfig - this.destinationEventService = destinationEventService ?: ANALYTICS - } - - @SerialName("stream") - var streamName = "" - - @SerialName("canary_events_enabled") - var canaryEventsEnabled = false - - @SerialName("destination_event_service") - val destinationEventServiceKey: String = "eventgate-analytics-external" - - var destinationEventService: DestinationEventService = ANALYTICS - - @SerialName("schema_title") - val schemaTitle: String = "" - - @SerialName("topic_prefixes") - val topicPrefixes: List = emptyList() - val topics: List = emptyList() - - @SerialName("sample") - var samplingConfig: SamplingConfig? = null - - init { - try { - destinationEventService = DestinationEventService.valueOf(destinationEventServiceKey) - } catch (e: IllegalArgumentException) {} - } -} diff --git a/app/src/main/java/org/wikipedia/analytics/metricsplatform/ArticleEvent.kt b/app/src/main/java/org/wikipedia/analytics/metricsplatform/ArticleEvent.kt index ea81343cc32..c9a72647aec 100644 --- a/app/src/main/java/org/wikipedia/analytics/metricsplatform/ArticleEvent.kt +++ b/app/src/main/java/org/wikipedia/analytics/metricsplatform/ArticleEvent.kt @@ -2,7 +2,7 @@ package org.wikipedia.analytics.metricsplatform import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable -import org.wikimedia.metrics_platform.context.PageData +import org.wikimedia.metricsplatform.context.PageData import org.wikipedia.dataclient.page.PageSummary import org.wikipedia.json.JsonUtil import org.wikipedia.page.PageFragment diff --git a/app/src/main/java/org/wikipedia/analytics/metricsplatform/MetricsEvent.kt b/app/src/main/java/org/wikipedia/analytics/metricsplatform/MetricsEvent.kt index bf9b62f4abd..662c498433b 100644 --- a/app/src/main/java/org/wikipedia/analytics/metricsplatform/MetricsEvent.kt +++ b/app/src/main/java/org/wikipedia/analytics/metricsplatform/MetricsEvent.kt @@ -1,9 +1,9 @@ package org.wikipedia.analytics.metricsplatform -import org.wikimedia.metrics_platform.context.ClientData -import org.wikimedia.metrics_platform.context.InteractionData -import org.wikimedia.metrics_platform.context.PageData -import org.wikimedia.metrics_platform.context.PerformerData +import org.wikimedia.metricsplatform.context.ClientData +import org.wikimedia.metricsplatform.context.InteractionData +import org.wikimedia.metricsplatform.context.PageData +import org.wikimedia.metricsplatform.context.PerformerData import org.wikipedia.WikipediaApp import org.wikipedia.analytics.eventplatform.EventPlatformClient import org.wikipedia.auth.AccountUtil @@ -25,7 +25,7 @@ open class MetricsEvent { protected fun submitEvent( streamName: String, eventName: String, - interactionData: InteractionData?, + interactionData: InteractionData? = null, pageData: PageData? = null ) { MetricsPlatform.client.submitInteraction( diff --git a/app/src/main/java/org/wikipedia/analytics/metricsplatform/MetricsPlatform.kt b/app/src/main/java/org/wikipedia/analytics/metricsplatform/MetricsPlatform.kt index 81e9d712d72..c329ab2c0b0 100644 --- a/app/src/main/java/org/wikipedia/analytics/metricsplatform/MetricsPlatform.kt +++ b/app/src/main/java/org/wikipedia/analytics/metricsplatform/MetricsPlatform.kt @@ -1,16 +1,17 @@ package org.wikipedia.analytics.metricsplatform import android.os.Build -import org.wikimedia.metrics_platform.MetricsClient -import org.wikimedia.metrics_platform.context.AgentData -import org.wikimedia.metrics_platform.context.ClientData -import org.wikimedia.metrics_platform.context.MediawikiData +import org.wikimedia.metricsplatform.EventSenderDefault +import org.wikimedia.metricsplatform.MetricsClient +import org.wikimedia.metricsplatform.context.AgentData +import org.wikimedia.metricsplatform.context.ClientData +import org.wikimedia.metricsplatform.context.MediawikiData import org.wikipedia.BuildConfig import org.wikipedia.WikipediaApp import org.wikipedia.dataclient.okhttp.OkHttpConnectionFactory +import org.wikipedia.json.JsonUtil import org.wikipedia.settings.Prefs import org.wikipedia.util.ReleaseUtil -import java.time.Duration object MetricsPlatform { val agentData = AgentData( @@ -40,11 +41,12 @@ object MetricsPlatform { domain ) - val client: MetricsClient = MetricsClient.builder(clientData) - .httpClient(OkHttpConnectionFactory.client) - .eventQueueCapacity(Prefs.analyticsQueueSize) - .streamConfigFetchInterval(Duration.ofHours(12)) - .sendEventsInterval(Duration.ofSeconds(30)) - .isDebug(ReleaseUtil.isDevRelease) - .build() + val client: MetricsClient = MetricsClient( + clientData, + EventSenderDefault(JsonUtil.json, OkHttpConnectionFactory.client), + null, + + queueCapacity = Prefs.analyticsQueueSize, + isDebug = ReleaseUtil.isDevRelease + ) } diff --git a/app/src/main/java/org/wikipedia/dataclient/Service.kt b/app/src/main/java/org/wikipedia/dataclient/Service.kt index 3cc8dc704e5..7dd2715c966 100644 --- a/app/src/main/java/org/wikipedia/dataclient/Service.kt +++ b/app/src/main/java/org/wikipedia/dataclient/Service.kt @@ -1,5 +1,6 @@ package org.wikipedia.dataclient +import org.wikimedia.metricsplatform.config.StreamConfigCollection import org.wikipedia.captcha.Captcha import org.wikipedia.dataclient.discussiontools.DiscussionToolsEditResponse import org.wikipedia.dataclient.discussiontools.DiscussionToolsInfoResponse @@ -10,7 +11,6 @@ import org.wikipedia.dataclient.mwapi.CreateAccountResponse import org.wikipedia.dataclient.mwapi.MwParseResponse import org.wikipedia.dataclient.mwapi.MwPostResponse import org.wikipedia.dataclient.mwapi.MwQueryResponse -import org.wikipedia.dataclient.mwapi.MwStreamConfigsResponse import org.wikipedia.dataclient.mwapi.ParamInfoResponse import org.wikipedia.dataclient.mwapi.ShortenUrlResponse import org.wikipedia.dataclient.mwapi.SiteMatrix @@ -213,8 +213,8 @@ interface Service { @Field("token") token: String ): MwPostResponse - @GET(MW_API_PREFIX + "action=streamconfigs&format=json&constraints=destination_event_service=eventgate-analytics-external") - suspend fun getStreamConfigs(): MwStreamConfigsResponse + @GET(MW_API_PREFIX + "action=streamconfigs&format=json&constraints=destination_event_service%3Deventgate-analytics-external") + suspend fun getStreamConfigs(): StreamConfigCollection @GET(MW_API_PREFIX + "action=query&meta=allmessages&amenableparser=1") suspend fun getMessages( diff --git a/app/src/main/java/org/wikipedia/dataclient/ServiceFactory.kt b/app/src/main/java/org/wikipedia/dataclient/ServiceFactory.kt index b97dee61a73..166825e20bf 100644 --- a/app/src/main/java/org/wikipedia/dataclient/ServiceFactory.kt +++ b/app/src/main/java/org/wikipedia/dataclient/ServiceFactory.kt @@ -5,10 +5,10 @@ import okhttp3.Interceptor import okhttp3.MediaType.Companion.toMediaType import okhttp3.Response import okhttp3.logging.HttpLoggingInterceptor +import org.wikimedia.metricsplatform.config.DestinationEventService +import org.wikimedia.metricsplatform.config.StreamConfig import org.wikipedia.WikipediaApp -import org.wikipedia.analytics.eventplatform.DestinationEventService import org.wikipedia.analytics.eventplatform.EventService -import org.wikipedia.analytics.eventplatform.StreamConfig import org.wikipedia.dataclient.okhttp.OkHttpConnectionFactory import org.wikipedia.json.JsonUtil import org.wikipedia.settings.Prefs diff --git a/app/src/main/java/org/wikipedia/dataclient/mwapi/MwStreamConfigsResponse.kt b/app/src/main/java/org/wikipedia/dataclient/mwapi/MwStreamConfigsResponse.kt deleted file mode 100644 index 7b558aa142e..00000000000 --- a/app/src/main/java/org/wikipedia/dataclient/mwapi/MwStreamConfigsResponse.kt +++ /dev/null @@ -1,12 +0,0 @@ -package org.wikipedia.dataclient.mwapi - -import kotlinx.serialization.Serializable -import org.wikipedia.analytics.eventplatform.StreamConfig - -@Serializable -class MwStreamConfigsResponse : MwResponse() { - - private val streams: Map? = null - val streamConfigs: Map - get() = streams ?: emptyMap() -} diff --git a/app/src/main/java/org/wikipedia/settings/Prefs.kt b/app/src/main/java/org/wikipedia/settings/Prefs.kt index 072891986c8..c5ff9302e33 100644 --- a/app/src/main/java/org/wikipedia/settings/Prefs.kt +++ b/app/src/main/java/org/wikipedia/settings/Prefs.kt @@ -4,12 +4,12 @@ import android.location.Location import okhttp3.Cookie import okhttp3.HttpUrl.Companion.toHttpUrlOrNull import okhttp3.logging.HttpLoggingInterceptor +import org.wikimedia.metricsplatform.config.StreamConfig import org.wikipedia.BuildConfig import org.wikipedia.R import org.wikipedia.WikipediaApp import org.wikipedia.analytics.SessionData import org.wikipedia.analytics.eventplatform.AppSessionEvent -import org.wikipedia.analytics.eventplatform.StreamConfig import org.wikipedia.dataclient.WikiSite import org.wikipedia.donate.DonationResult import org.wikipedia.donate.donationreminder.DonationReminderConfig diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index d3c042a91da..f8c9c72437c 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -28,7 +28,6 @@ kotlinxSerializationJson = "1.9.0" kspPlugin = "2.2.10-2.0.2" leakCanaryVersion = "2.14" material = "1.12.0" -metricsVersion = "2.9" mlKitVersion = "17.0.6" mockitoVersion = "5.2.0" navigationCompose = "2.9.3" @@ -96,7 +95,6 @@ kotlinx-coroutines-core = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-c kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "kotlinxSerializationJson" } leakcanary-android = { module = "com.squareup.leakcanary:leakcanary-android", version.ref = "leakCanaryVersion" } material = { module = "com.google.android.material:material", version.ref = "material" } -metrics-platform = { module = "org.wikimedia.metrics:metrics-platform", version.ref = "metricsVersion" } mockito-inline = { module = "org.mockito:mockito-inline", version.ref = "mockitoVersion" } mockwebserver = { module = "com.squareup.okhttp3:mockwebserver", version.ref = "okHttpVersion" } okhttp-tls = { module = "com.squareup.okhttp3:okhttp-tls", version.ref = "okHttpVersion" } diff --git a/settings.gradle.kts b/settings.gradle.kts index 1a6a4962cae..0dcf117aae2 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -15,3 +15,4 @@ dependencyResolutionManagement { } include(":app") +include(":analytics:metricsplatform")