From 5ea3716eef91e6babd8503bc27a4b56f85013094 Mon Sep 17 00:00:00 2001 From: Dmitry Brant Date: Fri, 27 Jun 2025 12:16:16 -0400 Subject: [PATCH 1/9] First commit of MP module copied from client library. --- analytics/metrics-platform/.gitignore | 1 + analytics/metrics-platform/build.gradle | 66 + .../metrics_platform/ContextController.java | 201 ++ .../metrics_platform/CurationController.java | 15 + .../metrics_platform/EventProcessor.java | 122 + .../metrics_platform/EventSender.java | 19 + .../metrics_platform/EventSenderDefault.java | 53 + .../metrics_platform/MetricsClient.java | 544 ++++ .../metrics_platform/SamplingController.java | 78 + .../metrics_platform/SessionController.java | 80 + .../config/ConfigFetcherRunnable.java | 48 + .../config/CurationFilter.java | 86 + .../config/DestinationEventService.java | 38 + .../metrics_platform/config/SourceConfig.java | 63 + .../metrics_platform/config/StreamConfig.java | 127 + .../config/StreamConfigCollection.java | 16 + .../config/StreamConfigFetcher.java | 52 + .../curation/CollectionCurationRules.java | 30 + .../curation/ComparableCurationRules.java | 38 + .../config/curation/CurationRules.java | 30 + .../config/sampling/SampleConfig.java | 18 + .../metrics_platform/context/AgentData.kt | 24 + .../metrics_platform/context/ClientData.kt | 22 + .../metrics_platform/context/ContextValue.kt | 38 + .../metrics_platform/context/CustomData.kt | 33 + .../context/CustomDataType.kt | 10 + .../context/InstantSerializer.kt | 16 + .../context/InteractionData.kt | 21 + .../metrics_platform/context/MediawikiData.kt | 15 + .../metrics_platform/context/PageData.kt | 24 + .../metrics_platform/context/PerformerData.kt | 41 + .../metrics_platform/event/Event.java | 65 + .../event/EventProcessed.java | 170 ++ .../event/EventProcessedSerializer.java | 97 + .../metrics_platform/json/GsonHelper.java | 23 + .../metrics_platform/utils/Objects.java | 24 + .../metrics_platform/ConsistencyIT.java | 143 ++ .../ConsistencyITClientData.java | 102 + .../ContextControllerTest.java | 64 + .../CurationControllerTest.java | 102 + .../DestinationEventServiceTest.java | 42 + .../metrics_platform/EndToEndIT.java | 199 ++ .../metrics_platform/EventProcessorTest.java | 177 ++ .../wikimedia/metrics_platform/EventTest.java | 154 ++ .../metrics_platform/MetricsClientTest.java | 269 ++ .../SamplingControllerTest.java | 58 + .../SessionControllerTest.java | 36 + .../metrics_platform/TestEventSender.java | 27 + .../config/CurationFilterFixtures.java | 25 + .../config/SampleConfigTest.java | 22 + .../config/SourceConfigFixtures.java | 22 + .../config/SourceConfigTest.java | 31 + .../config/StreamConfigFetcherTest.java | 50 + .../config/StreamConfigFixtures.java | 164 ++ .../config/StreamConfigIT.java | 48 + .../config/StreamConfigTest.java | 31 + .../context/AgentDataTest.java | 52 + .../context/CustomDataTest.java | 54 + .../context/DataFixtures.java | 134 + .../context/MediawikiDataTest.java | 25 + .../context/PageDataTest.java | 43 + .../context/PerformerDataTest.java | 55 + .../curation/CurationFilterFixtures.java | 21 + .../metrics_platform/event/EventFixtures.java | 65 + .../config/streamconfigs-local.json | 2288 +++++++++++++++++ .../config/streamconfigs.json | 1934 ++++++++++++++ .../event/expected_event_click.json | 43 + .../event/expected_event_click_custom.json | 46 + .../event/expected_event_interaction.json | 43 + .../event/expected_event_view.json | 43 + .../test/resources/simplelogger.properties | 2 + app/build.gradle | 1 - .../java/org/wikipedia/dataclient/Service.kt | 2 +- gradle/libs.versions.toml | 2 - settings.gradle.kts | 1 + 75 files changed, 8964 insertions(+), 4 deletions(-) create mode 100644 analytics/metrics-platform/.gitignore create mode 100644 analytics/metrics-platform/build.gradle create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/ContextController.java create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/CurationController.java create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/EventProcessor.java create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/EventSender.java create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/EventSenderDefault.java create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/MetricsClient.java create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/SamplingController.java create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/SessionController.java create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/ConfigFetcherRunnable.java create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/CurationFilter.java create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/DestinationEventService.java create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/SourceConfig.java create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/StreamConfig.java create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/StreamConfigCollection.java create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/StreamConfigFetcher.java create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/CollectionCurationRules.java create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/ComparableCurationRules.java create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/CurationRules.java create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/sampling/SampleConfig.java create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/AgentData.kt create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/ClientData.kt create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/ContextValue.kt create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/CustomData.kt create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/CustomDataType.kt create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/InstantSerializer.kt create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/InteractionData.kt create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/MediawikiData.kt create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/PageData.kt create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/PerformerData.kt create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/event/Event.java create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/event/EventProcessed.java create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/event/EventProcessedSerializer.java create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/json/GsonHelper.java create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/utils/Objects.java create mode 100644 analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/ConsistencyIT.java create mode 100644 analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/ConsistencyITClientData.java create mode 100644 analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/ContextControllerTest.java create mode 100644 analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/CurationControllerTest.java create mode 100644 analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/DestinationEventServiceTest.java create mode 100644 analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/EndToEndIT.java create mode 100644 analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/EventProcessorTest.java create mode 100644 analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/EventTest.java create mode 100644 analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/MetricsClientTest.java create mode 100644 analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/SamplingControllerTest.java create mode 100644 analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/SessionControllerTest.java create mode 100644 analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/TestEventSender.java create mode 100644 analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/CurationFilterFixtures.java create mode 100644 analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/SampleConfigTest.java create mode 100644 analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/SourceConfigFixtures.java create mode 100644 analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/SourceConfigTest.java create mode 100644 analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/StreamConfigFetcherTest.java create mode 100644 analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/StreamConfigFixtures.java create mode 100644 analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/StreamConfigIT.java create mode 100644 analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/StreamConfigTest.java create mode 100644 analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/AgentDataTest.java create mode 100644 analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/CustomDataTest.java create mode 100644 analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/DataFixtures.java create mode 100644 analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/MediawikiDataTest.java create mode 100644 analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/PageDataTest.java create mode 100644 analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/PerformerDataTest.java create mode 100644 analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/curation/CurationFilterFixtures.java create mode 100644 analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/event/EventFixtures.java create mode 100644 analytics/metrics-platform/src/test/resources/org/wikimedia/metrics_platform/config/streamconfigs-local.json create mode 100644 analytics/metrics-platform/src/test/resources/org/wikimedia/metrics_platform/config/streamconfigs.json create mode 100644 analytics/metrics-platform/src/test/resources/org/wikimedia/metrics_platform/event/expected_event_click.json create mode 100644 analytics/metrics-platform/src/test/resources/org/wikimedia/metrics_platform/event/expected_event_click_custom.json create mode 100644 analytics/metrics-platform/src/test/resources/org/wikimedia/metrics_platform/event/expected_event_interaction.json create mode 100644 analytics/metrics-platform/src/test/resources/org/wikimedia/metrics_platform/event/expected_event_view.json create mode 100644 analytics/metrics-platform/src/test/resources/simplelogger.properties diff --git a/analytics/metrics-platform/.gitignore b/analytics/metrics-platform/.gitignore new file mode 100644 index 00000000000..796b96d1c40 --- /dev/null +++ b/analytics/metrics-platform/.gitignore @@ -0,0 +1 @@ +/build diff --git a/analytics/metrics-platform/build.gradle b/analytics/metrics-platform/build.gradle new file mode 100644 index 00000000000..17cbbf08ae1 --- /dev/null +++ b/analytics/metrics-platform/build.gradle @@ -0,0 +1,66 @@ +plugins { + id 'com.android.library' + id 'com.google.devtools.ksp' + id 'kotlin-android' + id 'kotlinx-serialization' +} + +final JavaVersion JAVA_VERSION = JavaVersion.VERSION_17 + +android { + namespace 'org.wikimedia.metrics_platform' + + compileOptions { + coreLibraryDesugaringEnabled true + + sourceCompatibility = JAVA_VERSION + targetCompatibility = JAVA_VERSION + } + + kotlinOptions { + jvmTarget = JAVA_VERSION + } + compileSdk 35 +} + +dependencies { + coreLibraryDesugaring libs.desugar.jdk.libs + + implementation libs.kotlin.stdlib.jdk8 + implementation libs.kotlinx.coroutines.core + implementation libs.kotlinx.coroutines.android + implementation libs.kotlinx.serialization.json + + implementation libs.material + implementation libs.appcompat + implementation libs.core.ktx + implementation libs.browser + implementation libs.constraintlayout + implementation libs.fragment.ktx + implementation libs.paging.runtime.ktx + implementation libs.palette.ktx + implementation libs.preference.ktx + implementation libs.recyclerview + implementation libs.viewpager2 + implementation libs.flexbox + implementation libs.drawerlayout + implementation libs.swiperefreshlayout + implementation libs.work.runtime.ktx + + implementation libs.okhttp.tls + implementation libs.okhttp3.logging.interceptor + implementation libs.retrofit + implementation libs.commons.lang3 + implementation libs.jsoup + implementation libs.photoview + implementation libs.balloon + implementation libs.retrofit2.kotlinx.serialization.converter + + implementation libs.android.sdk + implementation libs.android.plugin.annotation.v9 + + implementation libs.androidx.room.runtime + annotationProcessor libs.androidx.room.compiler + ksp libs.androidx.room.compiler + implementation libs.androidx.room.ktx +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/ContextController.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/ContextController.java new file mode 100644 index 00000000000..0deafb4dc33 --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/ContextController.java @@ -0,0 +1,201 @@ +package org.wikimedia.metrics_platform; + +import static org.wikimedia.metrics_platform.context.ContextValue.AGENT_APP_INSTALL_ID; +import static org.wikimedia.metrics_platform.context.ContextValue.AGENT_CLIENT_PLATFORM; +import static org.wikimedia.metrics_platform.context.ContextValue.AGENT_CLIENT_PLATFORM_FAMILY; +import static org.wikimedia.metrics_platform.context.ContextValue.AGENT_APP_FLAVOR; +import static org.wikimedia.metrics_platform.context.ContextValue.AGENT_APP_THEME; +import static org.wikimedia.metrics_platform.context.ContextValue.AGENT_APP_VERSION; +import static org.wikimedia.metrics_platform.context.ContextValue.AGENT_APP_VERSION_NAME; +import static org.wikimedia.metrics_platform.context.ContextValue.AGENT_DEVICE_FAMILY; +import static org.wikimedia.metrics_platform.context.ContextValue.AGENT_DEVICE_LANGUAGE; +import static org.wikimedia.metrics_platform.context.ContextValue.AGENT_RELEASE_STATUS; +import static org.wikimedia.metrics_platform.context.ContextValue.MEDIAWIKI_DATABASE; +import static org.wikimedia.metrics_platform.context.ContextValue.PAGE_ID; +import static org.wikimedia.metrics_platform.context.ContextValue.PAGE_TITLE; +import static org.wikimedia.metrics_platform.context.ContextValue.PAGE_NAMESPACE_ID; +import static org.wikimedia.metrics_platform.context.ContextValue.PAGE_NAMESPACE_NAME; +import static org.wikimedia.metrics_platform.context.ContextValue.PAGE_REVISION_ID; +import static org.wikimedia.metrics_platform.context.ContextValue.PAGE_WIKIDATA_QID; +import static org.wikimedia.metrics_platform.context.ContextValue.PAGE_CONTENT_LANGUAGE; +import static org.wikimedia.metrics_platform.context.ContextValue.PERFORMER_ID; +import static org.wikimedia.metrics_platform.context.ContextValue.PERFORMER_NAME; +import static org.wikimedia.metrics_platform.context.ContextValue.PERFORMER_IS_LOGGED_IN; +import static org.wikimedia.metrics_platform.context.ContextValue.PERFORMER_IS_TEMP; +import static org.wikimedia.metrics_platform.context.ContextValue.PERFORMER_SESSION_ID; +import static org.wikimedia.metrics_platform.context.ContextValue.PERFORMER_PAGEVIEW_ID; +import static org.wikimedia.metrics_platform.context.ContextValue.PERFORMER_GROUPS; +import static org.wikimedia.metrics_platform.context.ContextValue.PERFORMER_LANGUAGE_GROUPS; +import static org.wikimedia.metrics_platform.context.ContextValue.PERFORMER_LANGUAGE_PRIMARY; +import static org.wikimedia.metrics_platform.context.ContextValue.PERFORMER_REGISTRATION_DT; + +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Set; + +import javax.annotation.ParametersAreNonnullByDefault; +import javax.annotation.concurrent.ThreadSafe; + +import org.wikimedia.metrics_platform.config.StreamConfig; +import org.wikimedia.metrics_platform.context.AgentData; +import org.wikimedia.metrics_platform.context.ClientData; +import org.wikimedia.metrics_platform.context.MediawikiData; +import org.wikimedia.metrics_platform.context.PageData; +import org.wikimedia.metrics_platform.context.PerformerData; +import org.wikimedia.metrics_platform.event.EventProcessed; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + +@ThreadSafe +@ParametersAreNonnullByDefault +public class ContextController { + + /** + * @see Metrics Platform/Contextual attributes + */ + private static final Collection REQUIRED_PROPERTIES = List.of( + "agent_app_flavor", + "agent_app_install_id", + "agent_app_theme", + "agent_app_version", + "agent_app_version_name", + "agent_client_platform", + "agent_client_platform_family", + "agent_device_family", + "agent_device_language", + "agent_release_status" + ); + + public void enrichEvent(EventProcessed event, StreamConfig streamConfig) { + if (!streamConfig.hasRequestedContextValuesConfig()) { + return; + } + // Check stream config for which contextual values should be added to the event. + Collection requestedValuesFromConfig = streamConfig.getProducerConfig() + .getMetricsPlatformClientConfig().getRequestedValues(); + // Add required properties. + Set requestedValues = new HashSet<>(requestedValuesFromConfig); + requestedValues.addAll(REQUIRED_PROPERTIES); + ClientData filteredData = filterClientData(event.getClientData(), requestedValues); + event.setClientData(filteredData); + } + + @SuppressWarnings("checkstyle:CyclomaticComplexity") + @SuppressFBWarnings(value = "CC_CYCLOMATIC_COMPLEXITY", justification = "TODO: needs to be refactored") + private ClientData filterClientData(ClientData clientData, Collection requestedValues) { + AgentData.AgentDataBuilder agentBuilder = AgentData.builder(); + PageData.PageDataBuilder pageBuilder = PageData.builder(); + MediawikiData.MediawikiDataBuilder mediawikiBuilder = MediawikiData.builder(); + PerformerData.PerformerDataBuilder performerBuilder = PerformerData.builder(); + + AgentData agentData = clientData.getAgentData(); + PageData pageData = clientData.getPageData(); + MediawikiData mediawikiData = clientData.getMediawikiData(); + PerformerData performerData = clientData.getPerformerData(); + + for (String requestedValue : requestedValues) { + switch (requestedValue) { + case AGENT_APP_INSTALL_ID: + agentBuilder.appInstallId(agentData.getAppInstallId()); + break; + case AGENT_CLIENT_PLATFORM: + agentBuilder.clientPlatform(agentData.getClientPlatform()); + break; + case AGENT_CLIENT_PLATFORM_FAMILY: + agentBuilder.clientPlatformFamily(agentData.getClientPlatformFamily()); + break; + case AGENT_APP_FLAVOR: + agentBuilder.appFlavor(agentData.getAppFlavor()); + break; + case AGENT_APP_THEME: + agentBuilder.appTheme(agentData.getAppTheme()); + break; + case AGENT_APP_VERSION: + agentBuilder.appVersion(agentData.getAppVersion()); + break; + case AGENT_APP_VERSION_NAME: + agentBuilder.appVersionName(agentData.getAppVersionName()); + break; + case AGENT_DEVICE_FAMILY: + agentBuilder.deviceFamily(agentData.getDeviceFamily()); + break; + case AGENT_DEVICE_LANGUAGE: + agentBuilder.deviceLanguage(agentData.getDeviceLanguage()); + break; + case AGENT_RELEASE_STATUS: + agentBuilder.releaseStatus(agentData.getReleaseStatus()); + break; + case PAGE_ID: + pageBuilder.id(pageData.getId()); + break; + case PAGE_TITLE: + pageBuilder.title(pageData.getTitle()); + break; + case PAGE_NAMESPACE_ID: + pageBuilder.namespaceId(pageData.getNamespaceId()); + break; + case PAGE_NAMESPACE_NAME: + pageBuilder.namespaceName(pageData.getNamespaceName()); + break; + case PAGE_REVISION_ID: + pageBuilder.revisionId(pageData.getRevisionId()); + break; + case PAGE_WIKIDATA_QID: + pageBuilder.wikidataItemQid(pageData.getWikidataItemQid()); + break; + case PAGE_CONTENT_LANGUAGE: + pageBuilder.contentLanguage(pageData.getContentLanguage()); + break; + case MEDIAWIKI_DATABASE: + mediawikiBuilder.database(mediawikiData.getDatabase()); + break; + case PERFORMER_ID: + performerBuilder.id(performerData.getId()); + break; + case PERFORMER_NAME: + performerBuilder.name(performerData.getName()); + break; + case PERFORMER_IS_LOGGED_IN: + performerBuilder.isLoggedIn(performerData.getIsLoggedIn()); + break; + case PERFORMER_IS_TEMP: + performerBuilder.isTemp(performerData.getIsTemp()); + break; + case PERFORMER_SESSION_ID: + performerBuilder.sessionId(performerData.getSessionId()); + break; + case PERFORMER_PAGEVIEW_ID: + performerBuilder.pageviewId(performerData.getPageviewId()); + break; + case PERFORMER_GROUPS: + performerBuilder.groups(performerData.getGroups()); + break; + case PERFORMER_LANGUAGE_GROUPS: + var languageGroups = performerData.getLanguageGroups(); + if (languageGroups != null && languageGroups.length > 255) { + languageGroups = languageGroups.substring(0, 255); + } + performerBuilder.languageGroups(languageGroups); + break; + case PERFORMER_LANGUAGE_PRIMARY: + performerBuilder.languagePrimary(performerData.getLanguagePrimary()); + break; + case PERFORMER_REGISTRATION_DT: + performerBuilder.registrationDt(performerData.getRegistrationDt()); + break; + + default: + throw new IllegalArgumentException(String.format(Locale.ROOT, "Unknown property %s", requestedValue)); + } + } + + ClientData.ClientDataBuilder clientDataBuilder = ClientData.builder(); + clientDataBuilder.agentData(agentBuilder.build()); + clientDataBuilder.pageData(pageBuilder.build()); + clientDataBuilder.mediawikiData(mediawikiBuilder.build()); + clientDataBuilder.performerData(performerBuilder.build()); + return clientDataBuilder.build(); + } +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/CurationController.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/CurationController.java new file mode 100644 index 00000000000..e1cec9dfcc6 --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/CurationController.java @@ -0,0 +1,15 @@ +package org.wikimedia.metrics_platform; + +import org.wikimedia.metrics_platform.config.StreamConfig; +import org.wikimedia.metrics_platform.event.EventProcessed; + +import lombok.NonNull; + +public class CurationController { + public boolean shouldProduceEvent(@NonNull EventProcessed event, @NonNull StreamConfig streamConfig) { + if (!streamConfig.hasCurationFilter()) return true; + + return streamConfig.getCurationFilter().test(event); + } + +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/EventProcessor.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/EventProcessor.java new file mode 100644 index 00000000000..89eef5a1a6a --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/EventProcessor.java @@ -0,0 +1,122 @@ +package org.wikimedia.metrics_platform; + +import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.toList; + +import java.net.SocketTimeoutException; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.atomic.AtomicReference; +import java.util.logging.Level; + +import javax.annotation.concurrent.ThreadSafe; + +import org.wikimedia.metrics_platform.config.DestinationEventService; +import org.wikimedia.metrics_platform.config.SourceConfig; +import org.wikimedia.metrics_platform.config.StreamConfig; +import org.wikimedia.metrics_platform.event.EventProcessed; + +import lombok.extern.java.Log; + +@ThreadSafe +@Log +public class EventProcessor { + + /** + * Enriches event data with context data requested in the stream configuration. + */ + private final ContextController contextController; + private final CurationController curationController; + + private final AtomicReference sourceConfig; + private final SamplingController samplingController; + private final BlockingQueue eventQueue; + private final EventSender eventSender; + private final boolean isDebug; + + /** + * EventProcessor constructor. + */ + public EventProcessor( + ContextController contextController, + CurationController curationController, + AtomicReference sourceConfig, + SamplingController samplingController, + EventSender eventSender, + BlockingQueue eventQueue, + boolean isDebug + ) { + this.contextController = contextController; + this.curationController = curationController; + this.sourceConfig = sourceConfig; + this.samplingController = samplingController; + this.eventSender = eventSender; + this.eventQueue = eventQueue; + this.isDebug = isDebug; + } + + /** + * Send all events currently in the output buffer. + *

+ * A shallow clone of the output buffer is created and passed to the integration layer for + * submission by the client. If the event submission succeeds, the events are removed from the + * output buffer. (Note that the shallow copy created by clone() retains pointers to the original + * Event objects.) If the event submission fails, a client error is produced, and the events remain + * in buffer to be retried on the next submission attempt. + */ + public void sendEnqueuedEvents() { + SourceConfig config = sourceConfig.get(); + if (config == null) { + log.log(Level.FINE, "Configuration is missing, enqueued events are not sent."); + return; + } + + ArrayList pending = new ArrayList<>(); + this.eventQueue.drainTo(pending); + + Map streamConfigsMap = config.getStreamConfigsMap(); + + pending.stream() + .filter(event -> streamConfigsMap.containsKey(event.getStream())) + .filter(event -> { + StreamConfig cfg = streamConfigsMap.get(event.getStream()); + if (cfg.hasSampleConfig()) { + event.setSample(cfg.getSampleConfig()); + } + return samplingController.isInSample(cfg); + }) + .filter(event -> eventPassesCurationRules(event, streamConfigsMap)) + .collect(groupingBy(event -> destinationEventService(event, streamConfigsMap), toList())) + .forEach(this::sendEventsToDestination); + } + + protected boolean eventPassesCurationRules(EventProcessed event, Map streamConfigMap) { + StreamConfig streamConfig = streamConfigMap.get(event.getStream()); + contextController.enrichEvent(event, streamConfig); + + return curationController.shouldProduceEvent(event, streamConfig); + } + + private DestinationEventService destinationEventService(EventProcessed event, Map streamConfigMap) { + StreamConfig streamConfig = streamConfigMap.get(event.getStream()); + return streamConfig.getDestinationEventService(); + } + + @SuppressWarnings("checkstyle:IllegalCatch") + private void sendEventsToDestination( + DestinationEventService destinationEventService, + List pendingValidEvents + ) { + try { + eventSender.sendEvents(destinationEventService.getBaseUri(isDebug), pendingValidEvents); + } catch (UnknownHostException | SocketTimeoutException e) { + log.log(Level.WARNING, "Network error while sending " + pendingValidEvents.size() + " events. Adding back to queue.", e); + eventQueue.addAll(pendingValidEvents); + } catch (Exception e) { + log.log(Level.WARNING, "Failed to send " + pendingValidEvents.size() + " events.", e); + } + } +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/EventSender.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/EventSender.java new file mode 100644 index 00000000000..c1b2d34e9e9 --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/EventSender.java @@ -0,0 +1,19 @@ +package org.wikimedia.metrics_platform; + +import java.io.IOException; +import java.net.URL; +import java.util.Collection; + +import org.wikimedia.metrics_platform.event.EventProcessed; + +public interface EventSender { + + /** + * Transmit an event to a destination intake service. + * + * @param baseUri base uri of destination intake service + * @param events events to be sent + */ + + void sendEvents(URL baseUri, Collection events) throws IOException; +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/EventSenderDefault.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/EventSenderDefault.java new file mode 100644 index 00000000000..c6a64b2a132 --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/EventSenderDefault.java @@ -0,0 +1,53 @@ +package org.wikimedia.metrics_platform; + +import static java.util.logging.Level.INFO; + +import java.io.IOException; +import java.net.URL; +import java.util.Collection; + +import org.wikimedia.metrics_platform.event.EventProcessed; + +import com.google.gson.Gson; + +import lombok.extern.java.Log; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import okhttp3.ResponseBody; + +@Log +public class EventSenderDefault implements EventSender { + + private final Gson gson; + private final OkHttpClient httpClient; + + public EventSenderDefault(Gson gson, OkHttpClient httpClient) { + this.gson = gson; + this.httpClient = httpClient; + } + + @Override + public void sendEvents(URL baseUri, Collection events) throws IOException { + Request request = new Request.Builder() + .url(baseUri) + .header("Accept", "application/json") + .header("User-Agent", "Metrics Platform Client/Java " + MetricsClient.METRICS_PLATFORM_LIBRARY_VERSION) + .post(RequestBody.create(gson.toJson(events), okhttp3.MediaType.parse("application/json"))) + .build(); + + try (Response response = httpClient.newCall(request).execute()) { + int status = response.code(); + ResponseBody body = response.body(); + if (!response.isSuccessful() || status == 207) { + // In the case of a multi-status response (207), it likely means that one or more + // events were rejected. In such a case, the error is actually contained in + // the normal response body. + throw new IOException(body.string()); + } + + log.log(INFO, "Sent " + events.size() + " events successfully."); + } + } +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/MetricsClient.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/MetricsClient.java new file mode 100644 index 00000000000..d486d1ab723 --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/MetricsClient.java @@ -0,0 +1,544 @@ +package org.wikimedia.metrics_platform; + +import static java.lang.Math.max; +import static java.time.Instant.now; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.logging.Level.FINE; +import static java.util.stream.Collectors.toList; +import static org.wikimedia.metrics_platform.config.StreamConfigFetcher.ANALYTICS_API_ENDPOINT; +import static org.wikimedia.metrics_platform.event.EventProcessed.fromEvent; + +import java.net.URL; +import java.time.Duration; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.logging.Level; +import java.util.stream.Stream; + +import javax.annotation.Nullable; +import javax.annotation.ParametersAreNonnullByDefault; +import javax.annotation.concurrent.NotThreadSafe; + +import org.wikimedia.metrics_platform.config.ConfigFetcherRunnable; +import org.wikimedia.metrics_platform.config.SourceConfig; +import org.wikimedia.metrics_platform.config.StreamConfig; +import org.wikimedia.metrics_platform.config.StreamConfigFetcher; +import org.wikimedia.metrics_platform.context.ClientData; +import org.wikimedia.metrics_platform.context.InteractionData; +import org.wikimedia.metrics_platform.context.PerformerData; +import org.wikimedia.metrics_platform.event.Event; +import org.wikimedia.metrics_platform.event.EventProcessed; +import org.wikimedia.metrics_platform.json.GsonHelper; + +import com.google.gson.Gson; + +import lombok.Setter; +import lombok.SneakyThrows; +import lombok.experimental.Accessors; +import lombok.extern.java.Log; +import okhttp3.OkHttpClient; + +@Log +public final class MetricsClient { + + public static final DateTimeFormatter DATE_FORMAT = DateTimeFormatter + .ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ROOT) + .withZone(ZoneId.of("UTC")); + + private final ScheduledExecutorService executorService; + public static final String METRICS_PLATFORM_LIBRARY_VERSION = "2.8"; + public static final String METRICS_PLATFORM_BASE_VERSION = "1.2.2"; + public static final String METRICS_PLATFORM_SCHEMA_BASE = "/analytics/product_metrics/app/base/" + METRICS_PLATFORM_BASE_VERSION; + private final AtomicReference sourceConfig; + + /** + * Handles logging session management. A new session begins (and a new session ID is created) + * if the app has been inactive for 15 minutes or more. + */ + private final SessionController sessionController; + + /** + * Evaluates whether events for a given stream are in-sample based on the stream configuration. + */ + private final SamplingController samplingController; + + private final BlockingQueue eventQueue; + private final EventProcessor eventProcessor; + + /** + * MetricsClient constructor. + */ + private MetricsClient( + ScheduledExecutorService executorService, + SessionController sessionController, + SamplingController samplingController, + AtomicReference sourceConfig, + BlockingQueue eventQueue, + EventProcessor eventProcessor + ) { + this.executorService = executorService; + this.sessionController = sessionController; + this.samplingController = samplingController; + this.sourceConfig = sourceConfig; + this.eventQueue = eventQueue; + this.eventProcessor = eventProcessor; + } + + /** + * Submit an event to be enqueued and sent to the Event Platform. + *

+ * If stream configs are not yet fetched, the event will be held temporarily in the input + * buffer (provided there is space to do so). + *

+ * If stream configs are available, the event will be validated and enqueued for submission + * to the configured event platform intake service. + *

+ * Supplemental metadata is added immediately on intake, regardless of the presence or absence + * of stream configs, so that the event timestamp is recorded accurately. + * + * @param event event data + */ + public void submit(Event event) { + EventProcessed eventProcessed = fromEvent(event); + addRequiredMetadata(eventProcessed); + addToEventQueue(eventProcessed); + } + + /** + * Construct and submits a Metrics Platform Event from the event name and custom data for each + * stream specified. + *

+ * The Metrics Platform Event for a stream (S) is constructed by: first initializing the minimum + * valid event (E) that can be submitted to S; and, second mixing the context attributes requested + * in the configuration for S into E. + *

+ * The Metrics Platform Event is submitted to a stream (S) if: 1) S is in sample; and 2) the event + * is filtered due to the filtering rules for S. + *

+ * This particular submitMetricsEvent method accepts unformatted custom data and calls the following + * submitMetricsEvent method with the custom data properly formatted. + * + * @see Metrics Platform/Java API + * + * @param streamName stream name + * @param schemaId schema id + * @param eventName event name + * @param customData custom data + */ + public void submitMetricsEvent( + String streamName, + String schemaId, + String eventName, + Map customData + ) { + submitMetricsEvent(streamName, schemaId, eventName, null, customData, null); + } + + /** + * Construct and submits a Metrics Platform Event from the schema id, event name, page metadata, and custom data for + * the stream that is interested in those events. + * + * @param streamName stream name + * @param schemaId schema id + * @param eventName event name + * @param clientData client context data + * @param customData custom data + */ + public void submitMetricsEvent( + String streamName, + String schemaId, + String eventName, + ClientData clientData, + Map customData + ) { + submitMetricsEvent(streamName, schemaId, eventName, clientData, customData, null); + } + + /** + * Construct and submits a Metrics Platform Event from the schema id, stream name, event name, page metadata, custom + * data, and interaction data for the specified stream. + * + * @param streamName stream name + * @param schemaId schema id + * @param eventName event name + * @param clientData client context data + * @param customData custom data + * @param interactionData common data for an interaction schema + */ + public void submitMetricsEvent( + String streamName, + String schemaId, + String eventName, + ClientData clientData, + Map customData, + InteractionData interactionData + ) { + if (streamName == null) { + log.log(Level.FINE, "No stream has been specified, the submitMetricsEvent event is ignored and dropped."); + return; + } + + // If we already have stream configs, then we can pre-validate certain conditions and exclude the event from the queue entirely. + StreamConfig streamConfig = null; + if (sourceConfig.get() != null) { + streamConfig = sourceConfig.get().getStreamConfigByName(streamName); + if (streamConfig == null) { + log.log(Level.FINE, "No stream config exists for this stream, the submitMetricsEvent event is ignored and dropped."); + return; + } + if (!samplingController.isInSample(streamConfig)) { + log.log(Level.FINE, "Not in sample, the submitMetricsEvent event is ignored and dropped."); + return; + } + } + + Event event = new Event(schemaId, streamName, eventName); + event.setClientData(clientData); + + if (customData != null) { + event.setCustomData(customData); + } + + event.setInteractionData(interactionData); + + if (streamConfig != null && streamConfig.hasSampleConfig()) { + event.setSample(streamConfig.getSampleConfig()); + } + + submit(event); + } + + /** + * Submit an interaction event to a stream. + *

+ * An interaction event is meant to represent a basic interaction with some target or some event + * occurring, e.g. the user (**performer**) tapping/clicking a UI element, or an app notifying the + * server of its current state. + * + * @param streamName stream name + * @param eventName event name + * @param clientData client context data + * @param interactionData common data for the base interaction schema + * + * @see Metrics Platform/Java API + */ + public void submitInteraction( + String streamName, + String eventName, + ClientData clientData, + InteractionData interactionData + ) { + submitMetricsEvent(streamName, METRICS_PLATFORM_SCHEMA_BASE, eventName, clientData, null, interactionData); + } + + /** + * Submit an interaction event to a stream. + *

+ * See above - takes additional parameters (custom data + custom schema id) to submit an interaction event. + * + * @param streamName stream name + * @param schemaId schema id + * @param eventName event name + * @param clientData client context data + * @param interactionData common data for the base interaction schema + * @param customData custom data for the interaction + * + * @see Metrics Platform/Java API + */ + public void submitInteraction( + String streamName, + String schemaId, + String eventName, + ClientData clientData, + InteractionData interactionData, + Map customData + ) { + submitMetricsEvent(streamName, schemaId, eventName, clientData, customData, interactionData); + } + + /** + * Submit a click event to a stream. + * + * @param streamName stream name + * @param clientData client context data + * @param interactionData common data for the base interaction schema + * + * @see Metrics Platform/Java API + */ + public void submitClick( + String streamName, + ClientData clientData, + InteractionData interactionData + ) { + submitMetricsEvent(streamName, METRICS_PLATFORM_SCHEMA_BASE, "click", clientData, null, interactionData); + } + + /** + * Submit a click event to a stream with custom data. + * + * @param streamName stream name + * @param schemaId schema id + * @param eventName event name + * @param clientData client context data + * @param customData custom data for the interaction + * @param interactionData common data for the base interaction schema + * + * @see Metrics Platform/Java API + */ + public void submitClick( + String streamName, + String schemaId, + String eventName, + ClientData clientData, + Map customData, + InteractionData interactionData + ) { + submitMetricsEvent(streamName, schemaId, eventName, clientData, customData, interactionData); + } + + /** + * Submit a view event to a stream. + * + * @param streamName stream name + * @param clientData client context data + * @param interactionData common data for the base interaction schema + */ + public void submitView( + String streamName, + ClientData clientData, + InteractionData interactionData + ) { + submitMetricsEvent(streamName, METRICS_PLATFORM_SCHEMA_BASE, "view", clientData, null, interactionData); + } + + /** + * Submit a view event to a stream with custom data. + * + * @param streamName stream name + * @param schemaId schema id + * @param eventName event name + * @param clientData client context data + * @param customData custom data for the interaction + * @param interactionData common data for the base interaction schema + */ + public void submitView( + String streamName, + String schemaId, + String eventName, + ClientData clientData, + Map customData, + InteractionData interactionData + ) { + submitMetricsEvent(streamName, schemaId, eventName, clientData, customData, interactionData); + } + + /** + * Convenience method to be called when + * + * the onPause() activity lifecycle callback is called. + *

+ * Touches the session so that we can determine whether it's session has expired if and when the + * application is resumed. + */ + public void onAppPause() { + executorService.schedule(eventProcessor::sendEnqueuedEvents, 0, MILLISECONDS); + sessionController.touchSession(); + } + + /** + * Convenience method to be called when + * + * the onResume() activity lifecycle callback is called. + *

+ * Touches the session so that we can determine whether it has expired. + */ + public void onAppResume() { + sessionController.touchSession(); + } + + /** + * Closes the session. + */ + public void onAppClose() { + executorService.schedule(eventProcessor::sendEnqueuedEvents, 0, MILLISECONDS); + sessionController.closeSession(); + } + + /** + * Begins a new session and touches the session. + */ + public void resetSession() { + sessionController.beginSession(); + } + + /** + * Supplement the outgoing event with additional metadata. + * These include: + * - app_session_id: the current session ID + * - dt: ISO 8601 timestamp + * - domain: hostname + * + * @param event event + */ + private void addRequiredMetadata(EventProcessed event) { + event.setPerformerData( + PerformerData.builderFrom(event.getPerformerData()) + .sessionId(sessionController.getSessionId()) + .build()); + event.setTimestamp(DATE_FORMAT.format(now())); + event.setDomain(event.getClientData().domain); + } + + /** + * Append an enriched event to the queue. + * If the queue is full, we remove the oldest events from the queue to add the current event. + * Number of attempts to add to the queue is 1/50 of the number queue capacity but at least 10 + * + * @param event a processed event + */ + private void addToEventQueue(EventProcessed event) { + int eventQueueAppendAttempts = max(eventQueue.size() / 50, 10); + + while (!eventQueue.offer(event)) { + EventProcessed removedEvent = eventQueue.remove(); + if (removedEvent != null) { + log.log(FINE, removedEvent.getName() + " was dropped so that a newer event could be added to the queue."); + } + if (eventQueueAppendAttempts-- <= 0) break; + } + } + + public boolean isFullyInitialized() { + return sourceConfig.get() != null; + } + + public boolean isEventQueueEmpty() { + return eventQueue.isEmpty(); + } + + public static Builder builder(ClientData clientData) { + return new Builder(clientData); + } + + @NotThreadSafe @ParametersAreNonnullByDefault + @Setter @Accessors(fluent = true) + @SuppressWarnings("checkstyle:classfanoutcomplexity") // As the main builder for the application, this class has + // to fan out to almost everything. We could hide this by + // using an injection framework (Guice?), but the added + // dependency is probably not worth it. + public static final class Builder { + + private final ClientData clientData; + private final Duration streamConfigFetchRetryDelay = Duration.ofMinutes(1); + private AtomicReference sourceConfigRef = new AtomicReference<>(); + private BlockingQueue eventQueue = new LinkedBlockingQueue<>(10); + private SessionController sessionController = new SessionController(); + + private CurationController curationController = new CurationController(); + + @Nullable private SamplingController samplingController; + + private OkHttpClient httpClient = new OkHttpClient(); + private URL streamConfigURL = safeURL(ANALYTICS_API_ENDPOINT); + private Duration streamConfigFetchInitialDelay = Duration.ofSeconds(0); + private Duration streamConfigFetchInterval = Duration.ofSeconds(30); + private Duration sendEventsInitialDelay = Duration.ofSeconds(3); + private Duration sendEventsInterval = Duration.ofSeconds(30); + private boolean isDebug; + private Consumer sourceConfigConsumer = sourceConfig -> {}; + private ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1, new SimpleThreadFactory()); + + @Nullable private SourceConfig sourceConfig; + + public Builder(ClientData clientData) { + this.clientData = clientData; + } + + public Builder eventQueueCapacity(int capacity) { + eventQueue = new LinkedBlockingQueue<>(capacity); + return this; + } + + public MetricsClient build() { + if (sourceConfig != null) sourceConfigRef.set(sourceConfig); + + if (samplingController == null) { + samplingController = new SamplingController(clientData, sessionController); + } + + Gson gson = GsonHelper.getGson(); + + EventProcessor eventProcessor = new EventProcessor( + new ContextController(), + curationController, + sourceConfigRef, + samplingController, + new EventSenderDefault(gson, httpClient), + eventQueue, + isDebug + ); + + MetricsClient metricsClient = new MetricsClient( + executorService, + sessionController, + samplingController, + sourceConfigRef, + eventQueue, + eventProcessor); + + List> consumers = Stream.of(sourceConfigRef::set, sourceConfigConsumer) + .collect(toList()); + + StreamConfigFetcher streamConfigFetcher = new StreamConfigFetcher(streamConfigURL, httpClient, gson); + + ConfigFetcherRunnable configFetchRunnable = new ConfigFetcherRunnable(streamConfigFetchInterval, + streamConfigFetcher, + consumers, + executorService, streamConfigFetchRetryDelay); + + startScheduledOperations(eventProcessor, configFetchRunnable, executorService); + + return metricsClient; + } + + private void startScheduledOperations( + EventProcessor eventProcessor, + ConfigFetcherRunnable configFetchRunnable, + ScheduledExecutorService executorService + ) { + executorService.schedule(configFetchRunnable, streamConfigFetchInitialDelay.toMillis(), MILLISECONDS); + + executorService.scheduleAtFixedRate( + eventProcessor::sendEnqueuedEvents, + sendEventsInitialDelay.toMillis(), sendEventsInterval.toMillis(), MILLISECONDS); + } + + @SneakyThrows + private static URL safeURL(String url) { + return new URL(url); + } + + } + + private static final class SimpleThreadFactory implements ThreadFactory { + + private final AtomicLong counter = new AtomicLong(); + + @Override + public Thread newThread(Runnable r) { + return new Thread(r, "metrics-client-" + counter.incrementAndGet()); + } + } + +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/SamplingController.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/SamplingController.java new file mode 100644 index 00000000000..b0122328ace --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/SamplingController.java @@ -0,0 +1,78 @@ +package org.wikimedia.metrics_platform; + +import javax.annotation.concurrent.ThreadSafe; +import javax.annotation.Nonnull; +import javax.annotation.ParametersAreNonnullByDefault; + +import org.wikimedia.metrics_platform.config.sampling.SampleConfig; +import org.wikimedia.metrics_platform.config.StreamConfig; +import org.wikimedia.metrics_platform.context.ClientData; + +/** + * SamplingController: computes various sampling functions on the client + * + * Sampling is based on associative identifiers, each of which have a + * well-defined scope, and sampling config, which each stream provides as + * part of its configuration. + */ +@ThreadSafe +@ParametersAreNonnullByDefault +public class SamplingController { + + private final ClientData clientData; + private final SessionController sessionController; + + SamplingController(ClientData clientData, SessionController sessionController) { + this.clientData = clientData; + this.sessionController = sessionController; + } + + /** + * @param streamConfig stream config + * @return true if in sample or false otherwise + */ + boolean isInSample(StreamConfig streamConfig) { + if (!streamConfig.hasSampleConfig()) { + return true; + } + SampleConfig sampleConfig = streamConfig.getSampleConfig(); + if (sampleConfig.getRate() == 1.0) { + return true; + } + if (sampleConfig.getRate() == 0.0) { + return false; + } + return getSamplingValue(sampleConfig.getIdentifier()) < sampleConfig.getRate(); + } + + /** + * @param identifier identifier type from sampling config + * @return a floating point value between 0.0 and 1.0 (inclusive) + */ + double getSamplingValue(SampleConfig.Identifier identifier) { + String token = getSamplingId(identifier).substring(0, 8); + return (double) Long.parseLong(token, 16) / (double) 0xFFFFFFFFL; + } + + /** + * Returns the ID string to be used when evaluating presence in sample. + * The ID used is configured in stream config. + * + * @param identifier Identifier enum value + * @return the requested ID string + */ + @Nonnull + String getSamplingId(SampleConfig.Identifier identifier) { + switch (identifier) { + case SESSION: + return sessionController.getSessionId(); + case DEVICE: + return clientData.getAgentData().getAppInstallId(); + case PAGEVIEW: + return clientData.getPerformerData().getPageviewId(); + default: + throw new IllegalArgumentException("Bad identifier type: " + identifier); + } + } + +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/SessionController.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/SessionController.java new file mode 100644 index 00000000000..3eab9ff8e68 --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/SessionController.java @@ -0,0 +1,80 @@ +package org.wikimedia.metrics_platform; + +import java.security.SecureRandom; +import java.time.Duration; +import java.time.Instant; +import java.util.Locale; +import java.util.Random; + +import javax.annotation.Nonnull; +import javax.annotation.ParametersAreNonnullByDefault; +import javax.annotation.concurrent.GuardedBy; +import javax.annotation.concurrent.ThreadSafe; + +/** + * Manages sessions and session IDs for the Metrics Platform Client. + * + * A session begins when the application is launched and expires when the app is in the background + * for 30 minutes or more. + */ +@ThreadSafe +@ParametersAreNonnullByDefault +public class SessionController { + + private static final Duration SESSION_LENGTH = Duration.ofMinutes(30); + @GuardedBy("this") + @Nonnull + private String sessionId = generateSessionId(); + @GuardedBy("this") + @Nonnull + private Instant sessionTouched; + private static final Random RANDOM = new SecureRandom(); + + SessionController() { + this(Instant.now()); + } + + /** + * Constructor for testing. + * + * @param date session start time + */ + SessionController(Instant date) { + this.sessionTouched = date; + } + + @Nonnull + synchronized String getSessionId() { + return sessionId; + } + + synchronized void touchSession() { + if (sessionExpired()) { + sessionId = generateSessionId(); + } + + sessionTouched = Instant.now(); + } + + synchronized void beginSession() { + sessionId = generateSessionId(); + sessionTouched = Instant.now(); + } + + synchronized void closeSession() { + // @ToDo Determine how to close the session. + sessionTouched = Instant.now(); + } + + synchronized boolean sessionExpired() { + return Duration.between(sessionTouched, Instant.now()).compareTo(SESSION_LENGTH) >= 0; + } + + @Nonnull + private static String generateSessionId() { + Random random = RANDOM; + return String.format(Locale.US, "%08x", random.nextInt()) + + String.format(Locale.US, "%08x", random.nextInt()) + + String.format(Locale.US, "%04x", random.nextInt() & 0xFFFF); + } +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/ConfigFetcherRunnable.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/ConfigFetcherRunnable.java new file mode 100644 index 00000000000..baa4bb994e2 --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/ConfigFetcherRunnable.java @@ -0,0 +1,48 @@ +package org.wikimedia.metrics_platform.config; + +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.logging.Level.WARNING; + +import java.time.Duration; +import java.util.List; +import java.util.concurrent.ScheduledExecutorService; +import java.util.function.Consumer; + +public class ConfigFetcherRunnable implements Runnable { + + private final Duration streamConfigFetchInterval; + private final StreamConfigFetcher streamConfigFetcher; + private final List> consumers; + private final ScheduledExecutorService executorService; + private final Duration retryDelay; + + public ConfigFetcherRunnable( + Duration streamConfigFetchInterval, + StreamConfigFetcher streamConfigFetcher, + List> consumers, + ScheduledExecutorService executorService, + Duration retryDelay + ) { + this.streamConfigFetchInterval = streamConfigFetchInterval; + this.streamConfigFetcher = streamConfigFetcher; + this.consumers = consumers; + this.executorService = executorService; + this.retryDelay = retryDelay; + } + + public void run() { + Duration nextFetch = streamConfigFetchInterval; + try { + SourceConfig sourceConfig = streamConfigFetcher.fetchStreamConfigs(); + + for (Consumer consumer : consumers) { + consumer.accept(sourceConfig); + } + } catch (Exception e) { + log.log(WARNING, "Could not fetch configuration. Will retry sooner.", e); + nextFetch = retryDelay; + } finally { + executorService.schedule(this, nextFetch.toMillis(), MILLISECONDS); + } + } +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/CurationFilter.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/CurationFilter.java new file mode 100644 index 00000000000..b75e9d5f02c --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/CurationFilter.java @@ -0,0 +1,86 @@ +package org.wikimedia.metrics_platform.config; + +import java.time.Instant; +import java.util.function.Predicate; + +import org.wikimedia.metrics_platform.config.curation.CollectionCurationRules; +import org.wikimedia.metrics_platform.config.curation.ComparableCurationRules; +import org.wikimedia.metrics_platform.config.curation.CurationRules; +import org.wikimedia.metrics_platform.context.AgentData; +import org.wikimedia.metrics_platform.context.MediawikiData; +import org.wikimedia.metrics_platform.context.PageData; +import org.wikimedia.metrics_platform.context.PerformerData; +import org.wikimedia.metrics_platform.event.EventProcessed; + +import com.google.gson.annotations.SerializedName; + +public class CurationFilter implements Predicate { + @SerializedName("agent_app_install_id") CurationRules agentAppInstallIdRules; + @SerializedName("agent_client_platform") CurationRules agentClientPlatformRules; + @SerializedName("agent_client_platform_family") CurationRules agentClientPlatformFamilyRules; + + @SerializedName("mediawiki_database") CurationRules mediawikiDatabase; + + @SerializedName("page_id") ComparableCurationRules pageIdRules; + @SerializedName("page_namespace_id") ComparableCurationRules pageNamespaceIdRules; + @SerializedName("page_namespace_name") CurationRules pageNamespaceNameRules; + @SerializedName("page_title") CurationRules pageTitleRules; + @SerializedName("page_revision_id") ComparableCurationRules pageRevisionIdRules; + @SerializedName("page_wikidata_qid") CurationRules pageWikidataQidRules; + @SerializedName("page_content_language") CurationRules pageContentLanguageRules; + + @SerializedName("performer_id") ComparableCurationRules performerIdRules; + @SerializedName("performer_name") CurationRules performerNameRules; + @SerializedName("performer_session_id") CurationRules performerSessionIdRules; + @SerializedName("performer_pageview_id") CurationRules performerPageviewIdRules; + @SerializedName("performer_groups") CollectionCurationRules performerGroupsRules; + @SerializedName("performer_is_logged_in") CurationRules performerIsLoggedInRules; + @SerializedName("performer_is_temp") CurationRules performerIsTempRules; + @SerializedName("performer_registration_dt") ComparableCurationRules performerRegistrationDtRules; + @SerializedName("performer_language_groups") CurationRules performerLanguageGroupsRules; + @SerializedName("performer_language_primary") CurationRules performerLanguagePrimaryRules; + + public boolean test(EventProcessed event) { + return applyAgentRules(event.getAgentData()) + && applyMediaWikiRules(event.getMediawikiData()) + && applyPageRules(event.getPageData()) + && applyPerformerRules(event.getPerformerData()); + } + + private boolean applyAgentRules(@Nonnull AgentData data) { + return applyPredicate(this.agentAppInstallIdRules, data.getAppInstallId()) + && applyPredicate(this.agentClientPlatformRules, data.getClientPlatform()) + && applyPredicate(this.agentClientPlatformFamilyRules, data.getClientPlatformFamily()); + } + + private boolean applyMediaWikiRules(@Nonnull MediawikiData data) { + return applyPredicate(this.mediawikiDatabase, data.getDatabase()); + } + + private boolean applyPageRules(@Nonnull PageData data) { + return applyPredicate(this.pageIdRules, data.getId()) + && applyPredicate(this.pageNamespaceIdRules, data.getNamespaceId()) + && applyPredicate(this.pageNamespaceNameRules, data.getNamespaceName()) + && applyPredicate(this.pageTitleRules, data.getTitle()) + && applyPredicate(this.pageRevisionIdRules, data.getRevisionId()) + && applyPredicate(this.pageWikidataQidRules, data.getWikidataItemQid()) + && applyPredicate(this.pageContentLanguageRules, data.getContentLanguage()); + } + + private boolean applyPerformerRules(@Nonnull PerformerData data) { + return applyPredicate(this.performerIdRules, data.getId()) + && applyPredicate(this.performerNameRules, data.getName()) + && applyPredicate(this.performerSessionIdRules, data.getSessionId()) + && applyPredicate(this.performerPageviewIdRules, data.getPageviewId()) + && applyPredicate(this.performerGroupsRules, data.getGroups()) + && applyPredicate(this.performerIsLoggedInRules, data.getIsLoggedIn()) + && applyPredicate(this.performerIsTempRules, data.getIsTemp()) + && applyPredicate(this.performerRegistrationDtRules, data.getRegistrationDt()) + && applyPredicate(this.performerLanguageGroupsRules, data.getLanguageGroups()) + && applyPredicate(this.performerLanguagePrimaryRules, data.getLanguagePrimary()); + } + + private static boolean applyPredicate(@Nullable Predicate rules, @Nullable T value) { + return rules == null || rules.test(value); + } +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/DestinationEventService.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/DestinationEventService.java new file mode 100644 index 00000000000..43bcd3b462e --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/DestinationEventService.java @@ -0,0 +1,38 @@ +package org.wikimedia.metrics_platform.config; + +import java.net.MalformedURLException; +import java.net.URL; + +import com.google.gson.annotations.SerializedName; + +/** + * Possible event destination endpoints which can be specified in stream configurations. + * For now, we'll assume that we always want to send to ANALYTICS. + * + * https://wikitech.wikimedia.org/wiki/Event_Platform/EventGate#EventGate_clusters + */ +public enum DestinationEventService { + + @SerializedName("eventgate-analytics-external") + ANALYTICS("https://intake-analytics.wikimedia.org"), + + @SerializedName("eventgate-logging-external") + ERROR_LOGGING("https://intake-logging.wikimedia.org"), + + @SerializedName("eventgate-logging-local") + LOCAL("http://localhost:8192"); + + private final URL baseUri; + + DestinationEventService(String baseUri) { + this.baseUri = new URL(baseUri + "/v1/events"); + } + + public URL getBaseUri() throws MalformedURLException { + return getBaseUri(false); + } + + public URL getBaseUri(boolean isDebug) throws MalformedURLException { + return isDebug ? this.baseUri : new URL(baseUri + "?hasty=true"); + } +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/SourceConfig.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/SourceConfig.java new file mode 100644 index 00000000000..aafc52ace93 --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/SourceConfig.java @@ -0,0 +1,63 @@ +package org.wikimedia.metrics_platform.config; + +import static java.util.Collections.unmodifiableMap; +import static java.util.Collections.unmodifiableSet; +import static java.util.stream.Collectors.toSet; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +public class SourceConfig { + private final Map streamConfigs; + private final Set sourceConfigs; + + public SourceConfig(Map streamConfigs) { + this.streamConfigs = unmodifiableMap(streamConfigs); + this.sourceConfigs = unmodifiableSet(new HashSet<>(streamConfigs.values())); + } + + /** + * Get all stream configs' stream names. + */ + public Set getStreamNames() { + Set sourceConfigNamesSet = streamConfigs.keySet(); + return unmodifiableSet(sourceConfigNamesSet); + } + + /** + * Get all stream configs with stream name. + */ + public Map getStreamConfigsMap() { + return streamConfigs; + } + + /** + * Get all stream configs without stream name. + */ + public Set getStreamConfigsRaw() { + return sourceConfigs; + } + + /** + * Get stream config by stream name. + * + * @param streamName stream name + */ + public StreamConfig getStreamConfigByName(String streamName) { + return streamConfigs.get(streamName); + } + + /** + * Get stream names by event name. + * + * @param eventName event name + */ + public Set getStreamNamesByEvent(String eventName) { + return sourceConfigs.stream() + .filter(streamConfig -> streamConfig.isInterestedInEvent(eventName)) + .map(StreamConfig::getStreamName) + .collect(toSet()); + } + +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/StreamConfig.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/StreamConfig.java new file mode 100644 index 00000000000..9d48f840e3e --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/StreamConfig.java @@ -0,0 +1,127 @@ +package org.wikimedia.metrics_platform.config; + +import java.util.Collections; +import java.util.Set; + +import org.wikimedia.metrics_platform.config.sampling.SampleConfig; + +import com.google.gson.annotations.SerializedName; + +public class StreamConfig { + + @SerializedName("stream") String streamName; + + @SerializedName("schema_title") String schemaTitle; + + @SerializedName("destination_event_service") + DestinationEventService destinationEventService; + + @SerializedName("producers") ProducerConfig producerConfig; + + @SerializedName("sample") + SampleConfig sampleConfig; + + /** + * The context attributes that the Metrics Platform Client can add to an event. + */ + public static final String[] CONTEXTUAL_ATTRIBUTES = new String[] { + // Agent + "agent_app_install_id", + "agent_client_platform", + "agent_client_platform_family", + // Page + "page_id", + "page_title", + "page_namespace_id", + "page_namespace_name", + "page_revision_id", + "page_wikidata_qid", + "page_content_language", + // MediaWiki + "mediawiki_database", + // Performer + "performer_is_logged_in", + "performer_id", + "performer_name", + "performer_session_id", + "performer_pageview_id", + "performer_groups", + "performer_language_primary", + "performer_language_groups", + "performer_registration_dt", + }; + + public boolean hasRequestedContextValuesConfig() { + return producerConfig != null && + producerConfig.metricsPlatformClientConfig != null && + producerConfig.metricsPlatformClientConfig.requestedValues != null; + } + + public boolean hasSampleConfig() { + return producerConfig != null && + producerConfig.metricsPlatformClientConfig != null && + sampleConfig != null; + } + + /** + * Return whether this stream has any events it is interested in. + * + * @return true if the stream has events + */ + public boolean hasEvents() { + return producerConfig != null && + producerConfig.metricsPlatformClientConfig != null && + producerConfig.metricsPlatformClientConfig.events != null; + } + + /** + * Return the event names this stream is interested in. + * + * @return event names for the stream + */ + public Set getEvents() { + if (hasEvents()) { + return producerConfig.metricsPlatformClientConfig.events; + } + return Collections.emptySet(); + } + + public DestinationEventService getDestinationEventService() { + return destinationEventService != null ? destinationEventService : DestinationEventService.ANALYTICS; + } + + public boolean hasCurationFilter() { + return producerConfig != null && + producerConfig.metricsPlatformClientConfig != null && + producerConfig.metricsPlatformClientConfig.curationFilter != null; + } + + public CurationFilter getCurationFilter() { + if (hasCurationFilter()) { + return producerConfig.metricsPlatformClientConfig.curationFilter; + } + + return CurationFilter.builder().build(); + } + + public static class ProducerConfig { + @SerializedName("metrics_platform_client") + StreamConfig.MetricsPlatformClientConfig metricsPlatformClientConfig; + } + + public static class MetricsPlatformClientConfig { + @SerializedName("events") Set events; + @SerializedName("provide_values") Set requestedValues; + @SerializedName("curation") CurationFilter curationFilter; + } + + public boolean isInterestedInEvent(String eventName) { + for (String streamEventName : getEvents()) { + // Match string prefixes for event names of interested streams. + if (eventName.startsWith(streamEventName)) { + return true; + } + } + return false; + } +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/StreamConfigCollection.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/StreamConfigCollection.java new file mode 100644 index 00000000000..97f619fafc3 --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/StreamConfigCollection.java @@ -0,0 +1,16 @@ +package org.wikimedia.metrics_platform.config; + +import java.util.Map; + +import javax.annotation.ParametersAreNullableByDefault; +import javax.annotation.concurrent.ThreadSafe; + +import com.google.gson.annotations.SerializedName; + +import lombok.Value; + +@Value @ThreadSafe +@ParametersAreNullableByDefault +public class StreamConfigCollection { + @SerializedName("streams") public Map streamConfigs; +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/StreamConfigFetcher.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/StreamConfigFetcher.java new file mode 100644 index 00000000000..c815e792004 --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/StreamConfigFetcher.java @@ -0,0 +1,52 @@ +package org.wikimedia.metrics_platform.config; + +import java.io.IOException; +import java.io.Reader; +import java.net.URL; +import java.util.Map; +import java.util.stream.Collectors; + +import com.google.gson.Gson; + +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.ResponseBody; + +public class StreamConfigFetcher { + public static final String ANALYTICS_API_ENDPOINT = "https://meta.wikimedia.org/w/api.php?" + + "action=streamconfigs&format=json&formatversion=2&" + + "constraints=destination_event_service%3Deventgate-analytics-external"; + + public static final String METRICS_PLATFORM_SCHEMA_TITLE = "analytics/mediawiki/client/metrics_event"; + + private final URL url; + private final OkHttpClient httpClient; + private final Gson gson; + + public StreamConfigFetcher(URL url, OkHttpClient httpClient, Gson gson) { + this.url = url; + this.httpClient = httpClient; + this.gson = gson; + } + + /** + * Fetch stream configs from analytics endpoint. + */ + public SourceConfig fetchStreamConfigs() throws IOException { + Request request = new Request.Builder().url(url).build(); + try (Response response = httpClient.newCall(request).execute()) { + ResponseBody body = response.body(); + if (body == null) { + throw new IOException("Failed to fetch stream configs: " + response.message()); + } + return new SourceConfig(parseConfig(body.charStream())); + } + } + + // Visible For Testing + public Map parseConfig(Reader reader) { + return gson.fromJson(reader, StreamConfigCollection.class).streamConfigs.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/CollectionCurationRules.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/CollectionCurationRules.java new file mode 100644 index 00000000000..089aab66fb3 --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/CollectionCurationRules.java @@ -0,0 +1,30 @@ +package org.wikimedia.metrics_platform.config.curation; + +import java.util.Collection; +import java.util.function.Predicate; + +import javax.annotation.ParametersAreNullableByDefault; + +import com.google.gson.annotations.SerializedName; + +import lombok.Builder; +import lombok.Value; + +@Builder @Value +@ParametersAreNullableByDefault +public class CollectionCurationRules implements Predicate> { + T contains; + @SerializedName("does_not_contain") T doesNotContain; + @SerializedName("contains_all") Collection containsAll; + @SerializedName("contains_any") Collection containsAny; + + @Override + public boolean test(Collection value) { + if (value == null) return false; + + return (contains == null || value.contains(contains)) + && (doesNotContain == null || !value.contains(doesNotContain)) + && (containsAll == null || value.containsAll(containsAll)) + && (containsAny == null || value.stream().filter(v -> containsAny.contains(v)).count() > 0); + } +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/ComparableCurationRules.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/ComparableCurationRules.java new file mode 100644 index 00000000000..96a8495940e --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/ComparableCurationRules.java @@ -0,0 +1,38 @@ +package org.wikimedia.metrics_platform.config.curation; + +import java.util.Collection; +import java.util.function.Predicate; + +import javax.annotation.ParametersAreNullableByDefault; + +import com.google.gson.annotations.SerializedName; + +import lombok.Builder; +import lombok.Value; + +@Builder @Value +@ParametersAreNullableByDefault +public class ComparableCurationRules> implements Predicate { + @SerializedName("is_equals") T isEquals; + @SerializedName("not_equals") T isNotEquals; + @SerializedName("greater_than") T greaterThan; + @SerializedName("less_than") T lessThan; + @SerializedName("greater_than_or_equals") T greaterThanOrEquals; + @SerializedName("less_than_or_equals") T lessThanOrEquals; + Collection in; + @SerializedName("not_in") Collection notIn; + + @SuppressWarnings("CyclomaticComplexity") // Code is readable enough here + @Override + public boolean test(T value) { + if (value == null) return false; + + return (isEquals == null || isEquals.equals(value)) + && (isNotEquals == null || !isNotEquals.equals(value)) + && (greaterThan == null || greaterThan.compareTo(value) < 0) + && (lessThan == null || lessThan.compareTo(value) > 0) + && (greaterThanOrEquals == null || greaterThanOrEquals.compareTo(value) <= 0) + && (lessThanOrEquals == null || lessThanOrEquals.compareTo(value) >= 0) + && (notIn == null || !notIn.contains(value)); + } +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/CurationRules.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/CurationRules.java new file mode 100644 index 00000000000..e5ce0f62d14 --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/CurationRules.java @@ -0,0 +1,30 @@ +package org.wikimedia.metrics_platform.config.curation; + +import java.util.Collection; +import java.util.function.Predicate; + +import javax.annotation.ParametersAreNullableByDefault; + +import com.google.gson.annotations.SerializedName; + +import lombok.Builder; +import lombok.Value; + +@Builder @Value +@ParametersAreNullableByDefault +public class CurationRules implements Predicate { + @SerializedName("equals") T isEquals; + @SerializedName("not_equals") T isNotEquals; + Collection in; + @SerializedName("not_in") Collection notIn; + + @Override + public boolean test(T value) { + if (value == null) return false; + + return (isEquals == null || isEquals.equals(value)) + && (isNotEquals == null || !isNotEquals.equals(value)) + && (in == null || in.contains(value)) + && (notIn == null || !notIn.contains(value)); + } +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/sampling/SampleConfig.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/sampling/SampleConfig.java new file mode 100644 index 00000000000..16b42bebd22 --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/sampling/SampleConfig.java @@ -0,0 +1,18 @@ +package org.wikimedia.metrics_platform.config.sampling; + +import com.google.gson.annotations.SerializedName; + +public class SampleConfig { + + public enum Identifier { + @SerializedName("session") SESSION, + @SerializedName("device") DEVICE, + @SerializedName("pageview") PAGEVIEW + } + + /** Sampling rate. **/ + double rate; + + /** ID type to use for sampling. */ + Identifier identifier; +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/AgentData.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/AgentData.kt new file mode 100644 index 00000000000..259e87fbdf9 --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/AgentData.kt @@ -0,0 +1,24 @@ +package org.wikimedia.metrics_platform.context + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable + +/** + * Agent context data fields. + * + * All fields are nullable, and boxed types are used in place of their equivalent primitive types to avoid + * unexpected default values from being used where the true value is null. + */ +@Serializable +class AgentData( + @SerialName("app_flavor") val appFlavor: String? = null, + @SerialName("app_install_id") val appInstallId: String? = null, + @SerialName("app_theme") val appTheme: String? = null, + @SerialName("app_version") val appVersion: Int? = null, + @SerialName("app_version_name") val appVersionName: String? = null, + @SerialName("client_platform") val clientPlatform: String? = null, + @SerialName("client_platform_family") val clientPlatformFamily: String? = null, + @SerialName("device_family") val deviceFamily: String? = null, + @SerialName("device_language") val deviceLanguage: String? = null, + @SerialName("release_status") val releaseStatus: String? = null, +) diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/ClientData.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/ClientData.kt new file mode 100644 index 00000000000..af4f921c11c --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/ClientData.kt @@ -0,0 +1,22 @@ +package org.wikimedia.metrics_platform.context + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable + +/** + * Client metadata context fields. + * + * ClientData includes immutable and mutable contextual data from the client. + * This metadata is added to every event submission when queued for processing. + * + * All fields of nested data objects are nullable, and boxed types are used in place of their equivalent primitive types + * to avoid unexpected default values from being used where the true value is null. + */ +@Serializable +open class ClientData ( + @SerialName("agent") val agentData: AgentData = AgentData(), + @SerialName("page") val pageData: PageData = PageData(), + @SerialName("mediawiki") val mediawikiData: MediawikiData = MediawikiData(), + @SerialName("performer") val performerData: PerformerData = PerformerData(), + @SerialName("domain") val domain: String? = null +) diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/ContextValue.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/ContextValue.kt new file mode 100644 index 00000000000..478b6a74918 --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/ContextValue.kt @@ -0,0 +1,38 @@ +package org.wikimedia.metrics_platform.context + +/** + * @see [Metrics Platform/Contextual attributes](https://wikitech.wikimedia.org/wiki/Metrics_Platform/Contextual_attributes) + */ +object ContextValue { + const val AGENT_APP_INSTALL_ID: String = "agent_app_install_id" + const val AGENT_CLIENT_PLATFORM: String = "agent_client_platform" + const val AGENT_CLIENT_PLATFORM_FAMILY: String = "agent_client_platform_family" + const val AGENT_APP_FLAVOR: String = "agent_app_flavor" + const val AGENT_APP_THEME: String = "agent_app_theme" + const val AGENT_APP_VERSION: String = "agent_app_version" + const val AGENT_APP_VERSION_NAME: String = "agent_app_version_name" + const val AGENT_DEVICE_FAMILY: String = "agent_device_family" + const val AGENT_DEVICE_LANGUAGE: String = "agent_device_language" + const val AGENT_RELEASE_STATUS: String = "agent_release_status" + + const val MEDIAWIKI_DATABASE: String = "mediawiki_database" + + const val PAGE_ID: String = "page_id" + const val PAGE_TITLE: String = "page_title" + const val PAGE_NAMESPACE_ID: String = "page_namespace_id" + const val PAGE_NAMESPACE_NAME: String = "page_namespace_name" + const val PAGE_REVISION_ID: String = "page_revision_id" + const val PAGE_WIKIDATA_QID: String = "page_wikidata_qid" + const val PAGE_CONTENT_LANGUAGE: String = "page_content_language" + + const val PERFORMER_ID: String = "performer_id" + const val PERFORMER_NAME: String = "performer_name" + const val PERFORMER_IS_LOGGED_IN: String = "performer_is_logged_in" + const val PERFORMER_IS_TEMP: String = "performer_is_temp" + const val PERFORMER_SESSION_ID: String = "performer_session_id" + const val PERFORMER_PAGEVIEW_ID: String = "performer_pageview_id" + const val PERFORMER_GROUPS: String = "performer_groups" + const val PERFORMER_LANGUAGE_GROUPS: String = "performer_language_groups" + const val PERFORMER_LANGUAGE_PRIMARY: String = "performer_language_primary" + const val PERFORMER_REGISTRATION_DT: String = "performer_registration_dt" +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/CustomData.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/CustomData.kt new file mode 100644 index 00000000000..2f949252c6f --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/CustomData.kt @@ -0,0 +1,33 @@ +package org.wikimedia.metrics_platform.context + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable + +@Serializable +class CustomData ( + @SerialName("data_type") val type: CustomDataType? = null, + val value: String? = null +) { + companion object { + /** + * Return custom data, based on a generic object. + * + * @return formatted custom data + */ + fun of(value: Any?): CustomData { + var formattedValue = value.toString() + val formattedType: CustomDataType? + + if (value is Number) { + formattedType = CustomDataType.NUMBER + } else if (value is Boolean) { + formattedType = CustomDataType.BOOLEAN + formattedValue = if (value) "true" else "false" + } else { + formattedType = CustomDataType.STRING + } + + return CustomData(formattedType, formattedValue) + } + } +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/CustomDataType.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/CustomDataType.kt new file mode 100644 index 00000000000..50d6b45429e --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/CustomDataType.kt @@ -0,0 +1,10 @@ +package org.wikimedia.metrics_platform.context + +import kotlinx.serialization.SerialName + +enum class CustomDataType { + @SerialName("number") NUMBER, + @SerialName("string") STRING, + @SerialName("boolean") BOOLEAN, + @SerialName("null") NULL +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/InstantSerializer.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/InstantSerializer.kt new file mode 100644 index 00000000000..b75c9738651 --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/InstantSerializer.kt @@ -0,0 +1,16 @@ +package org.wikimedia.metrics_platform.context + +import kotlinx.serialization.KSerializer +import kotlinx.serialization.descriptors.PrimitiveKind +import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor +import kotlinx.serialization.descriptors.SerialDescriptor +import kotlinx.serialization.encoding.Decoder +import kotlinx.serialization.encoding.Encoder +import java.time.Instant + +object InstantSerializer : KSerializer { + override val descriptor: SerialDescriptor = + PrimitiveSerialDescriptor("java.time.Instant", PrimitiveKind.STRING) + override fun serialize(encoder: Encoder, value: Instant) = encoder.encodeString(value.toString()) + override fun deserialize(decoder: Decoder): Instant = Instant.parse(decoder.decodeString()) +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/InteractionData.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/InteractionData.kt new file mode 100644 index 00000000000..73d0d34bbc6 --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/InteractionData.kt @@ -0,0 +1,21 @@ +package org.wikimedia.metrics_platform.context + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable + +/** + * Interaction data fields. + * + * Common interaction fields that describe the event being submitted. Most fields are nullable. + */ +@Serializable +class InteractionData( + @SerialName("action") val action: String? = null, + @SerialName("action_subtype") val actionSubtype: String? = null, + @SerialName("action_source") val actionSource: String? = null, + @SerialName("action_context") val actionContext: String? = null, + @SerialName("element_id") val elementId: String? = null, + @SerialName("element_friendly_name") val elementFriendlyName: String? = null, + @SerialName("funnel_entry_token") val funnelEntryToken: String? = null, + @SerialName("funnel_event_sequence_position") val funnelEventSequencePosition: Int? = null +) diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/MediawikiData.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/MediawikiData.kt new file mode 100644 index 00000000000..1b9913da849 --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/MediawikiData.kt @@ -0,0 +1,15 @@ +package org.wikimedia.metrics_platform.context + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable + +/** + * Mediawiki context data fields. + * + * All fields are nullable, and boxed types are used in place of their equivalent primitive types to avoid + * unexpected default values from being used where the true value is null. + */ +@Serializable +class MediawikiData( + @SerialName("database") val database: String? = null +) diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/PageData.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/PageData.kt new file mode 100644 index 00000000000..8ef13068464 --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/PageData.kt @@ -0,0 +1,24 @@ +package org.wikimedia.metrics_platform.context + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable + +/** + * Page context data context fields. + * + * PageData are dynamic context fields that change with every request. PageData is submitted with each event + * by the client and then queued for processing by EventProcessor. + * + * All fields are nullable, and boxed types are used in place of their equivalent primitive types to avoid + * unexpected default values from being used where the true value is null. + */ +@Serializable +class PageData( + val id: Int? = null, + val title: String? = null, + @SerialName("namespace_id") val namespaceId: Int? = null, + @SerialName("namespace_name") val namespaceName: String? = null, + @SerialName("revision_id") val revisionId: Long? = null, + @SerialName("wikidata_qid") val wikidataItemQid: String? = null, + @SerialName("content_language") val contentLanguage: String? = null, +) diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/PerformerData.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/PerformerData.kt new file mode 100644 index 00000000000..683fb813f39 --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/PerformerData.kt @@ -0,0 +1,41 @@ +@file:UseSerializers(InstantSerializer::class) + +package org.wikimedia.metrics_platform.context + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import kotlinx.serialization.UseSerializers +import java.time.Instant + +/** + * Performer context data fields. + * + * All fields are nullable, and boxed types are used in place of their equivalent primitive types to avoid + * unexpected default values from being used where the true value is null. + */ +@Serializable +class PerformerData ( + private val id: Int? = null, + @SerialName("name") val name: String? = null, + @SerialName("is_logged_in") val isLoggedIn: Boolean? = null, + @SerialName("is_temp") val isTemp: Boolean? = null, + @SerialName("session_id") val sessionId: String? = null, + @SerialName("pageview_id") val pageviewId: String? = null, + @SerialName("groups") val groups: MutableCollection? = null, + @SerialName("language_groups") val languageGroups: String? = null, + @SerialName("language_primary") val languagePrimary: String? = null, + @SerialName("registration_dt") val registrationDt: Instant? = null +) { + class PerformerDataBuilder { + fun languageGroups(languageGroups: String?): PerformerDataBuilder { + // Ensure that a performer's language groups do not exceed 255 characters. See T361265. + + var languageGroups = languageGroups + if (languageGroups != null && languageGroups.length > 255) { + languageGroups = languageGroups.substring(0, 255) + } + + return this + } + } +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/event/Event.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/event/Event.java new file mode 100644 index 00000000000..dd88af9cb5d --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/event/Event.java @@ -0,0 +1,65 @@ +package org.wikimedia.metrics_platform.event; + +import java.util.Map; + +import org.wikimedia.metrics_platform.config.sampling.SampleConfig; +import org.wikimedia.metrics_platform.context.ClientData; +import org.wikimedia.metrics_platform.context.InteractionData; +import org.wikimedia.metrics_platform.utils.Objects; + +import com.google.gson.annotations.SerializedName; + +public class Event { + @SerializedName("$schema") protected String schema; + @SerializedName("name") protected final String name; + @SerializedName("dt") protected String timestamp; + @SerializedName("custom_data") protected Map customData; + protected final Meta meta; + @SerializedName("client_data") protected ClientData clientData; + @SerializedName("sample") protected SampleConfig sample; + @SerializedName("interaction_data") protected InteractionData interactionData; + + public Event(String schema, String stream, String name) { + this.schema = schema; + this.meta = new Meta(stream); + this.name = name; + } + + @Nullable + public String getStream() { + return meta.getStream(); + } + + public void setDomain(String domain) { + meta.domain = domain; + } + + @Nonnull + public ClientData getClientData() { + clientData = Objects.firstNonNull(clientData, ClientData::new); + return clientData; + } + + public void setCustomData(@Nonnull Map customData) { + this.customData = customData; + } + + public void setSample(@Nonnull SampleConfig sample) { + this.sample = sample; + } + + @Nonnull + public InteractionData getInteractionData() { + interactionData = Objects.firstNonNull(interactionData, InteractionData::new); + return interactionData; + } + + public void setInteractionData(@Nonnull InteractionData interactionData) { + this.interactionData = interactionData; + } + + protected static final class Meta { + private final String stream; + private String domain; + } +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/event/EventProcessed.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/event/EventProcessed.java new file mode 100644 index 00000000000..69b2a330865 --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/event/EventProcessed.java @@ -0,0 +1,170 @@ +package org.wikimedia.metrics_platform.event; + +import static org.wikimedia.metrics_platform.utils.Objects.firstNonNull; + +import java.util.Map; + +import org.wikimedia.metrics_platform.config.sampling.SampleConfig; +import org.wikimedia.metrics_platform.context.AgentData; +import org.wikimedia.metrics_platform.context.ClientData; +import org.wikimedia.metrics_platform.context.InteractionData; +import org.wikimedia.metrics_platform.context.MediawikiData; +import org.wikimedia.metrics_platform.context.PageData; +import org.wikimedia.metrics_platform.context.PerformerData; + +import com.google.gson.annotations.SerializedName; + +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.Setter; + +@EqualsAndHashCode +@Getter +@Setter +public class EventProcessed extends Event { + @SerializedName("agent") private AgentData agentData; + @SerializedName("page") private PageData pageData; + @SerializedName("mediawiki") private MediawikiData mediawikiData; + @SerializedName("performer") private PerformerData performerData; + @Nonnull + @SerializedName("action") private String action; + @SerializedName("action_subtype") private String actionSubtype; + @SerializedName("action_source") private String actionSource; + @SerializedName("action_context") private String actionContext; + @SerializedName("element_id") private String elementId; + @SerializedName("element_friendly_name") private String elementFriendlyName; + @SerializedName("funnel_entry_token") private String funnelEntryToken; + @SerializedName("funnel_event_sequence_position") private Integer funnelEventSequencePosition; + + /** + * Constructor for EventProcessed. + * + * @param schema schema id + * @param stream stream name + * @param name event name + * @param clientData agent, mediawiki, page, performer data + */ + public EventProcessed( + String schema, + String stream, + @Nonnull String name, + ClientData clientData + ) { + super(schema, stream, name); + this.clientData = clientData; + this.agentData = clientData.getAgentData(); + this.pageData = clientData.getPageData(); + this.mediawikiData = clientData.getMediawikiData(); + this.performerData = clientData.getPerformerData(); + this.action = name; + } + + /** + * Constructor for EventProcessed. + * + * @param schema schema id + * @param stream stream name + * @param name event name + * @param customData custom data + * @param clientData agent, mediawiki, page, performer data + * @param sample sample configuration + * @param interactionData contextual data of the interaction + *

+ * Although 'setInteractionData()' sets the 'action' property for the event, + * because 'action' is a nonnull property for both 'EventProcessed' and the + * 'InteractionData' data object, removing the redundant setting of 'action' + * triggers NP_NONNULL_FIELD_NOT_INITIALIZED_IN_CONSTRUCTOR in spotbugs so + * leaving it as is for the time being rather than suppressing the error. + */ + public EventProcessed( + String schema, + String stream, + String name, + Map customData, + ClientData clientData, + SampleConfig sample, + InteractionData interactionData + ) { + super(schema, stream, name); + this.clientData = clientData; + this.agentData = clientData.getAgentData(); + this.pageData = clientData.getPageData(); + this.mediawikiData = clientData.getMediawikiData(); + this.performerData = clientData.getPerformerData(); + this.setCustomData(customData); + this.sample = sample; + this.setInteractionData(interactionData); + this.action = interactionData.getAction(); + } + + @Nonnull + public static EventProcessed fromEvent(Event event) { + return new EventProcessed( + event.getSchema(), + event.getStream(), + event.getName(), + event.getCustomData(), + event.getClientData(), + event.getSample(), + event.getInteractionData() + ); + } + + @Nonnull + public AgentData getAgentData() { + agentData = firstNonNull(agentData, AgentData.NULL_AGENT_DATA); + return agentData; + } + + @Nonnull + public PageData getPageData() { + pageData = firstNonNull(pageData, PageData.NULL_PAGE_DATA); + return pageData; + } + + @Nonnull + public MediawikiData getMediawikiData() { + mediawikiData = firstNonNull(mediawikiData, MediawikiData.NULL_MEDIAWIKI_DATA); + return mediawikiData; + } + + @Nonnull + public PerformerData getPerformerData() { + performerData = firstNonNull(performerData, PerformerData.NULL_PERFORMER_DATA); + return performerData; + } + + @Override + public void setClientData(@Nonnull ClientData clientData) { + setAgentData(clientData.getAgentData()); + setPageData(clientData.getPageData()); + setMediawikiData(clientData.getMediawikiData()); + setPerformerData(clientData.getPerformerData()); + this.clientData = clientData; + } + + @Override + public final void setInteractionData(@Nonnull InteractionData interactionData) { + this.action = interactionData.getAction(); + this.actionContext = interactionData.getActionContext(); + this.actionSource = interactionData.getActionSource(); + this.actionSubtype = interactionData.getActionSubtype(); + this.elementId = interactionData.getElementId(); + this.elementFriendlyName = interactionData.getElementFriendlyName(); + this.funnelEntryToken = interactionData.getFunnelEntryToken(); + this.funnelEventSequencePosition = interactionData.getFunnelEventSequencePosition(); + } + + public void setAgentData(AgentData agentData) { + this.agentData = agentData; + } + public void setPageData(PageData pageData) { + this.pageData = pageData; + } + public void setMediawikiData(MediawikiData mediawikiData) { + this.mediawikiData = mediawikiData; + } + public void setPerformerData(PerformerData performerData) { + this.performerData = performerData; + } +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/event/EventProcessedSerializer.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/event/EventProcessedSerializer.java new file mode 100644 index 00000000000..056fd04f4fe --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/event/EventProcessedSerializer.java @@ -0,0 +1,97 @@ +package org.wikimedia.metrics_platform.event; + +import static java.util.logging.Level.INFO; + +import java.lang.reflect.Type; +import java.time.Instant; +import java.util.Map; + +import org.wikimedia.metrics_platform.context.InstantConverter; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonSerializer; +import com.google.gson.JsonSerializationContext; + +public class EventProcessedSerializer implements JsonSerializer { + + /** + * Add gson builder for handling dt properties inside internal serializer. + *

+ * Because this is a new serializer, including it as a new type adapter in the GsonHelper + * utility causes issues with circularity. + */ + private final Gson gson = new GsonBuilder() + .registerTypeAdapter(Instant.class, new InstantConverter()) + .create(); + + /** + * EventProcessed serializer + *

+ * Rather than serializing all properties individually, this serializer starts from default serialization + * and modifies a few properties (custom data, name, etc.). + */ + @Override + public JsonElement serialize(EventProcessed src, Type type, JsonSerializationContext jsonSerializationContext) { + JsonElement jsonElement = gson.toJsonTree(src); + JsonObject jsonObject = (JsonObject) jsonElement; + + /* + * Custom data can be passed into the EventProcessed constructor as a map + * of key-value pairs. EventProcessed inherits from Event which contains a + * customData property that would be serialized by default. To validate + * successfully against the corresponding schema, custom data must be sent + * as top-level properties with the event. + */ + if (src.customData != null) { + int customDataAdded = 0; + int customDataCount = src.customData.size(); + for (Map.Entry entry : src.customData.entrySet()) { + if (entry.getValue() instanceof Number) + jsonObject.addProperty(entry.getKey(), (Number) entry.getValue()); + if (entry.getValue() instanceof String) + jsonObject.addProperty(entry.getKey(), entry.getValue().toString()); + if (entry.getValue() instanceof Boolean) + jsonObject.addProperty(entry.getKey(), (Boolean) entry.getValue()); + + if (jsonObject.has(entry.getKey())) + customDataAdded++; + } + + if (customDataAdded != customDataCount) { + log.log(INFO, "Only " + customDataAdded + " custom data key-value pairs were serialized " + + "but there are " + customDataCount + " total custom data items for this event."); + } + + jsonObject.remove("custom_data"); + jsonObject.remove("client_data"); + } + + /* + * Removing a few properties here because it would require adding annotations on every Event and EventProcessed + * property just to exclude few properties from serialization. + * + * The "name" property is an Event property that eventually gets submitted as "action" with an event, but it is + * not included in the Metrics Platform base interactions schemas. Because the InteractionData data object can + * be a null parameter in the MetricsClient::submitMetricsEvent method, the "name" property is needed when + * clients send events without InteractionData (see @ToDo add link to MP schemas once they are merged). + * + * Once Metrics Platform core interactions schemas are updated to include the "sample" property, the line to + * remove it can be deleted here. + */ + jsonObject.remove("name"); + jsonObject.remove("sample"); + + /* + * Remove the top level data objects from EventProcessed which are + * inherited from its superclass Event. The values in "client_data" + * and "interaction_data" are set as top level properties in + * EventProcessed's constructor. + */ + jsonObject.remove("client_data"); + jsonObject.remove("interaction_data"); + return jsonObject; + } +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/json/GsonHelper.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/json/GsonHelper.java new file mode 100644 index 00000000000..9a01aceeb34 --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/json/GsonHelper.java @@ -0,0 +1,23 @@ +package org.wikimedia.metrics_platform.json; + +import java.time.Instant; + +import org.wikimedia.metrics_platform.context.InstantConverter; +import org.wikimedia.metrics_platform.event.EventProcessed; +import org.wikimedia.metrics_platform.event.EventProcessedSerializer; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +public final class GsonHelper { + private GsonHelper() { + // Utility class which should never be instantiated. + } + + public static Gson getGson() { + return new GsonBuilder() + .registerTypeAdapter(Instant.class, new InstantConverter()) + .registerTypeAdapter(EventProcessed.class, new EventProcessedSerializer()) + .create(); + } +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/utils/Objects.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/utils/Objects.java new file mode 100644 index 00000000000..8e67f72bf29 --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/utils/Objects.java @@ -0,0 +1,24 @@ +package org.wikimedia.metrics_platform.utils; + +import java.util.function.Supplier; + +public final class Objects { + + private Objects() { + // Utility class, should not be instantiated + } + + @Nonnull + public static T firstNonNull(@Nullable T first, @Nonnull T second) { + if (first != null) return first; + return second; + } + + @Nonnull + public static T firstNonNull(@Nullable T first, @Nonnull Supplier second) { + if (first != null) return first; + T result = second.get(); + if (result == null) throw new NullPointerException("'second' parameter should always create a non null object."); + return result; + } +} diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/ConsistencyIT.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/ConsistencyIT.java new file mode 100644 index 00000000000..10deb207828 --- /dev/null +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/ConsistencyIT.java @@ -0,0 +1,143 @@ +package org.wikimedia.metrics_platform; + +import static java.util.Collections.singletonMap; +import static org.assertj.core.api.Assertions.assertThat; +import static org.wikimedia.metrics_platform.ConsistencyITClientData.createConsistencyTestClientData; +import static org.wikimedia.metrics_platform.config.StreamConfigFetcher.ANALYTICS_API_ENDPOINT; +import static org.wikimedia.metrics_platform.MetricsClient.METRICS_PLATFORM_SCHEMA_BASE; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.Reader; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicReference; + +import org.junit.jupiter.api.Test; +import org.wikimedia.metrics_platform.config.SourceConfig; +import org.wikimedia.metrics_platform.config.StreamConfig; +import org.wikimedia.metrics_platform.config.StreamConfigFetcher; +import org.wikimedia.metrics_platform.context.ClientData; +import org.wikimedia.metrics_platform.context.DataFixtures; +import org.wikimedia.metrics_platform.event.EventProcessed; +import org.wikimedia.metrics_platform.json.GsonHelper; + +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +import okhttp3.OkHttpClient; + +class ConsistencyIT { + private JsonObject expectedEvent; + + @Test void testConsistency() throws IOException { + Path pathStreamConfigs = Paths.get("../tests/consistency/stream_configs_apps.json"); + + try (BufferedReader reader = Files.newBufferedReader(pathStreamConfigs)) { + + // Init shared test config + variables for creating new metrics client + event processor. + Map testStreamConfigs = getTestStreamConfigs(reader); + SourceConfig sourceConfig = new SourceConfig(testStreamConfigs); + AtomicReference sourceConfigRef = new AtomicReference<>(); + sourceConfigRef.set(sourceConfig); + BlockingQueue eventQueue = new LinkedBlockingQueue<>(10); + ClientData consistencyTestClientData = createConsistencyTestClientData(); + SamplingController samplingController = new SamplingController(consistencyTestClientData, new SessionController()); + + EventProcessor consistencyTestEventProcessor = getTestEventProcessor( + sourceConfigRef, + samplingController, + eventQueue + ); + + MetricsClient consistencyTestMetricsClient = getTestMetricsClient( + consistencyTestClientData, + sourceConfigRef, + eventQueue + ); + + consistencyTestMetricsClient.submitMetricsEvent( + "test.consistency", + METRICS_PLATFORM_SCHEMA_BASE, + "test_consistency_event", + DataFixtures.getTestClientData(getExpectedEventJson().toString()), + singletonMap("test", "consistency") + ); + + EventProcessed queuedEvent = eventQueue.peek(); + consistencyTestEventProcessor.eventPassesCurationRules(queuedEvent, testStreamConfigs); + + // Adjust the queuedEvent and compare it against the expected event. + if (this.expectedEvent != null) { + Gson gson = GsonHelper.getGson(); + String queuedEventJsonStringRaw = gson.toJson(queuedEvent); + JsonObject queuedEventJsonObject = JsonParser.parseString(queuedEventJsonStringRaw).getAsJsonObject(); + // Remove the timestamp properties from the queued event to match the expected event json. + removeExtraProperties(queuedEventJsonObject); + + assertThat(queuedEventJsonObject) + .usingRecursiveComparison() + .ignoringCollectionOrder() + .isEqualTo(this.expectedEvent); + } + } + } + + private static MetricsClient getTestMetricsClient( + ClientData consistencyTestClientData, + AtomicReference sourceConfigRef, + BlockingQueue eventQueue + ) { + return MetricsClient.builder(consistencyTestClientData) + .sourceConfigRef(sourceConfigRef) + .eventQueue(eventQueue) + .build(); + } + + private static EventProcessor getTestEventProcessor( + AtomicReference sourceConfigRef, + SamplingController samplingController, + BlockingQueue eventQueue + ) { + ContextController contextController = new ContextController(); + CurationController curationController = new CurationController(); + EventSender eventSender = new TestEventSender(); + return new EventProcessor( + contextController, + curationController, + sourceConfigRef, + samplingController, + eventSender, + eventQueue, + true + ); + } + + private static Map getTestStreamConfigs(Reader reader) throws MalformedURLException { + StreamConfigFetcher streamConfigFetcher = new StreamConfigFetcher(new URL(ANALYTICS_API_ENDPOINT), new OkHttpClient(), GsonHelper.getGson()); + return streamConfigFetcher.parseConfig(reader); + } + + private static void removeExtraProperties(JsonObject eventJsonObject) { + eventJsonObject.remove("dt"); + eventJsonObject.remove("sample"); + eventJsonObject.getAsJsonObject("performer").remove("registration_dt"); + } + + private JsonObject getExpectedEventJson() throws IOException { + if (this.expectedEvent == null) { + Path pathExpectedEvent = Paths.get("../tests/consistency/expected_event_apps.json"); + try (BufferedReader expectedEventReader = Files.newBufferedReader(pathExpectedEvent)) { + this.expectedEvent = JsonParser.parseReader(expectedEventReader).getAsJsonObject(); + } + } + return this.expectedEvent; + } +} diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/ConsistencyITClientData.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/ConsistencyITClientData.java new file mode 100644 index 00000000000..3aea3509379 --- /dev/null +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/ConsistencyITClientData.java @@ -0,0 +1,102 @@ +package org.wikimedia.metrics_platform; + +import java.io.BufferedReader; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collections; + +import org.wikimedia.metrics_platform.context.AgentData; +import org.wikimedia.metrics_platform.context.ClientData; +import org.wikimedia.metrics_platform.context.MediawikiData; +import org.wikimedia.metrics_platform.context.PageData; +import org.wikimedia.metrics_platform.context.PerformerData; + +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +public class ConsistencyITClientData extends ClientData { + public JsonObject agentJson; + public JsonObject pageJson; + public JsonObject mediawikiJson; + public JsonObject performerJson; + + public ConsistencyITClientData( + JsonObject agent, + JsonObject page, + JsonObject mediawiki, + JsonObject performer, + String domain + ) { + this.agentJson = agent; + this.pageJson = page; + this.mediawikiJson = mediawiki; + this.performerJson = performer; + + AgentData agentData = AgentData.builder() + .appInstallId(this.agentJson.get("app_install_id").getAsString()) + .clientPlatform(this.agentJson.get("client_platform").getAsString()) + .clientPlatformFamily(this.agentJson.get("client_platform_family").getAsString()) + .deviceFamily(this.agentJson.get("device_family").getAsString()) + .build(); + + PageData pageData = PageData.builder() + .id(this.pageJson.get("id").getAsInt()) + .title(this.pageJson.get("title").getAsString()) + .namespaceId(this.pageJson.get("namespace_id").getAsInt()) + .namespaceName(this.pageJson.get("namespace_name").getAsString()) + .revisionId(this.pageJson.get("revision_id").getAsLong()) + .wikidataItemQid(this.pageJson.get("wikidata_qid").getAsString()) + .contentLanguage(this.pageJson.get("content_language").getAsString()) + .build(); + MediawikiData mediawikiData = MediawikiData.builder() + .database(this.mediawikiJson.get("database").getAsString()) + .build(); + PerformerData performerData = PerformerData.builder() + .id(this.performerJson.get("id").getAsInt()) + .isLoggedIn(this.performerJson.get("is_logged_in").getAsBoolean()) + .sessionId(this.performerJson.get("session_id").getAsString()) + .pageviewId(this.performerJson.get("pageview_id").getAsString()) + .groups(Collections.singleton(this.performerJson.get("groups").getAsString())) + .languagePrimary(this.performerJson.get("language_primary").getAsString()) + .languageGroups(this.performerJson.get("language_groups").getAsString()) + .build(); + + this.setAgentData(agentData); + this.setPageData(pageData); + this.setMediawikiData(mediawikiData); + this.setPerformerData(performerData); + this.setDomain(domain); + } + + public static ConsistencyITClientData createConsistencyTestClientData() { + try { + JsonObject data = getIntegrationData(); + JsonObject agent = data.getAsJsonObject("agent"); + JsonObject page = data.getAsJsonObject("page"); + JsonObject mediawiki = data.getAsJsonObject("mediawiki"); + JsonObject performer = data.getAsJsonObject("performer"); + String domain = data.get("hostname").getAsString(); + + return new ConsistencyITClientData( + agent, + page, + mediawiki, + performer, + domain + ); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static JsonObject getIntegrationData() throws IOException { + Path pathIntegration = Paths.get("../tests/consistency/integration_data_apps.json"); + try (BufferedReader reader = Files.newBufferedReader(pathIntegration)) { + JsonElement jsonElement = JsonParser.parseReader(reader); + return jsonElement.getAsJsonObject(); + } + } +} diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/ContextControllerTest.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/ContextControllerTest.java new file mode 100644 index 00000000000..2eb564ed6a2 --- /dev/null +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/ContextControllerTest.java @@ -0,0 +1,64 @@ +package org.wikimedia.metrics_platform; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.wikimedia.metrics_platform.event.EventProcessed.fromEvent; + +import org.junit.jupiter.api.Test; +import org.wikimedia.metrics_platform.config.StreamConfig; +import org.wikimedia.metrics_platform.config.StreamConfigFixtures; +import org.wikimedia.metrics_platform.context.AgentData; +import org.wikimedia.metrics_platform.context.ClientData; +import org.wikimedia.metrics_platform.context.DataFixtures; +import org.wikimedia.metrics_platform.context.MediawikiData; +import org.wikimedia.metrics_platform.context.PageData; +import org.wikimedia.metrics_platform.context.PerformerData; +import org.wikimedia.metrics_platform.event.Event; +import org.wikimedia.metrics_platform.event.EventProcessed; + +class ContextControllerTest { + + @Test void testAddRequestedValues() { + ContextController contextController = new ContextController(); + Event eventBasic = new Event("test/event", "test.stream", "testEvent"); + EventProcessed event = fromEvent(eventBasic); + ClientData clientDataSample = DataFixtures.getTestClientData(); + event.setClientData(clientDataSample); + StreamConfig streamConfig = StreamConfigFixtures.STREAM_CONFIGS_WITH_EVENTS.get("test.stream"); + contextController.enrichEvent(event, streamConfig); + + AgentData agentData = event.getAgentData(); + MediawikiData mediawikiData = event.getMediawikiData(); + PageData pageData = event.getPageData(); + PerformerData performerData = event.getPerformerData(); + + assertThat(agentData.getAppFlavor()).isEqualTo("devdebug"); + assertThat(agentData.getAppInstallId()).isEqualTo("ffffffff-ffff-ffff-ffff-ffffffffffff"); + assertThat(agentData.getAppTheme()).isEqualTo("LIGHT"); + assertThat(agentData.getAppVersion()).isEqualTo(982734); + assertThat(agentData.getAppVersionName()).isEqualTo("2.7.50470-dev-2024-02-14"); + assertThat(agentData.getClientPlatform()).isEqualTo("android"); + assertThat(agentData.getClientPlatformFamily()).isEqualTo("app"); + assertThat(agentData.getDeviceFamily()).isEqualTo("Samsung SM-G960F"); + assertThat(agentData.getDeviceLanguage()).isEqualTo("en"); + assertThat(agentData.getReleaseStatus()).isEqualTo("dev"); + + assertThat(mediawikiData.getDatabase()).isEqualTo("enwiki"); + + assertThat(pageData.getId()).isEqualTo(1); + assertThat(pageData.getNamespaceId()).isEqualTo(0); + assertThat(pageData.getNamespaceName()).isEqualTo("Main"); + assertThat(pageData.getTitle()).isEqualTo("Test Page Title"); + assertThat(pageData.getRevisionId()).isEqualTo(1L); + assertThat(pageData.getContentLanguage()).isEqualTo("en"); + assertThat(pageData.getWikidataItemQid()).isEqualTo("Q123456"); + + assertThat(performerData.getId()).isEqualTo(1); + assertThat(performerData.getIsLoggedIn()).isTrue(); + assertThat(performerData.getIsTemp()).isFalse(); + assertThat(performerData.getName()).isEqualTo("TestPerformer"); + assertThat(performerData.getGroups()).containsExactly("*"); + assertThat(performerData.getRegistrationDt()).isEqualTo("2023-03-01T01:08:30Z"); + assertThat(performerData.getLanguageGroups()).isEqualTo("zh, en"); + assertThat(performerData.getLanguagePrimary()).isEqualTo("zh-tw"); + } +} diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/CurationControllerTest.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/CurationControllerTest.java new file mode 100644 index 00000000000..f11d3f6e458 --- /dev/null +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/CurationControllerTest.java @@ -0,0 +1,102 @@ +package org.wikimedia.metrics_platform; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.wikimedia.metrics_platform.event.EventFixtures.getEvent; + +import java.time.Instant; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.wikimedia.metrics_platform.config.StreamConfig; +import org.wikimedia.metrics_platform.config.StreamConfigFixtures; +import org.wikimedia.metrics_platform.context.PerformerData; +import org.wikimedia.metrics_platform.json.GsonHelper; +import org.wikimedia.metrics_platform.config.CurationFilter; +import org.wikimedia.metrics_platform.event.EventProcessed; + +import com.google.gson.Gson; + +class CurationControllerTest { + + private static StreamConfig streamConfig; + + private static CurationController curationController = new CurationController(); + + private static final List groups = Arrays.asList("user", "autoconfirmed", "steward"); + + @BeforeAll static void setUp() { + Gson gson = GsonHelper.getGson(); + String curationFilterJson = "{\"page_id\":{\"less_than\":500,\"not_equals\":42},\"page_namespace_name\":" + + "{\"equals\":\"Talk\"},\"performer_is_logged_in\":{\"equals\":true},\"performer_edit_count_bucket\":" + + "{\"in\":[\"100-999 edits\",\"1000+ edits\"]},\"performer_groups\":{\"contains_all\":" + + "[\"user\",\"autoconfirmed\"],\"does_not_contain\":\"sysop\"}}"; + CurationFilter curationFilter = gson.fromJson(curationFilterJson, CurationFilter.class); + + streamConfig = StreamConfigFixtures.streamConfig(curationFilter); + } + + @Test void testEventPasses() { + assertThat(curationController.shouldProduceEvent(getEvent(), streamConfig)).isTrue(); + } + + @Test void testEventFailsWrongPageId() { + EventProcessed event = getEvent(42, "Talk", groups, true, "1000+ edits"); + assertThat(curationController.shouldProduceEvent(event, streamConfig)).isFalse(); + } + + @Test void testEventFailsWrongPageNamespaceText() { + EventProcessed event = getEvent(1, "User", groups, true, "1000+ edits"); + assertThat(curationController.shouldProduceEvent(event, streamConfig)).isFalse(); + } + + @Test void testEventFailsWrongUserGroups() { + List wrongGroups = Arrays.asList("user", "autoconfirmed", "sysop"); + EventProcessed event = getEvent(1, "Talk", wrongGroups, true, "1000+ edits"); + assertThat(curationController.shouldProduceEvent(event, streamConfig)).isFalse(); + } + + @Test void testEventFailsNoUserGroups() { + EventProcessed event = getEvent(1, "Talk", Collections.emptyList(), true, "1000+ edits"); + assertThat(curationController.shouldProduceEvent(event, streamConfig)).isFalse(); + } + + @Test void testEventFailsNotLoggedIn() { + EventProcessed event = getEvent(1, "Talk", groups, false, "1000+ edits"); + assertThat(curationController.shouldProduceEvent(event, streamConfig)).isFalse(); + } + + @Test void testEventPassesPerformerRegistrationDtDeserializes() { + EventProcessed event = getEvent(); + event.setPerformerData( + PerformerData.builder() + .groups(groups) + .isLoggedIn(true) + .registrationDt(Instant.parse("2023-03-01T01:08:30Z")) + .build() + ); + assertThat(curationController.shouldProduceEvent(event, streamConfig)).isTrue(); + } + + @Test void testEventPassesCurationFilters() { + EventProcessed event = getEvent(1, "Talk", groups, true, "1000+ edits"); + assertThat(curationController.shouldProduceEvent(event, streamConfig)).isTrue(); + } + + @Test void testEventFailsEqualsRule() { + EventProcessed event = getEvent(1, "Main", groups, true, "1000+ edits"); + assertThat(curationController.shouldProduceEvent(event, streamConfig)).isFalse(); + } + + @Test void testEventFailsCollectionContainsAnyRule() { + EventProcessed event = getEvent(1, "Talk", Collections.singletonList("*"), true, "1000+ edits"); + assertThat(curationController.shouldProduceEvent(event, streamConfig)).isFalse(); + } + + @Test void testEventFailsCollectionDoesNotContainRule() { + EventProcessed event = getEvent(1, "Talk", Arrays.asList("foo", "bar"), true, "1000+ edits"); + assertThat(curationController.shouldProduceEvent(event, streamConfig)).isFalse(); + } +} diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/DestinationEventServiceTest.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/DestinationEventServiceTest.java new file mode 100644 index 00000000000..fab66eb7ecc --- /dev/null +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/DestinationEventServiceTest.java @@ -0,0 +1,42 @@ +package org.wikimedia.metrics_platform; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.net.MalformedURLException; +import java.net.URL; + +import org.junit.jupiter.api.Test; +import org.wikimedia.metrics_platform.config.DestinationEventService; + +class DestinationEventServiceTest { + + @Test void testDestinationEventServiceAnalytics() throws MalformedURLException { + DestinationEventService loggingService = DestinationEventService.ANALYTICS; + assertThat(loggingService.getBaseUri()).isEqualTo(new URL("https://intake-analytics.wikimedia.org/v1/events?hasty=true")); + } + + @Test void testDestinationEventServiceLogging() throws MalformedURLException { + DestinationEventService loggingService = DestinationEventService.ERROR_LOGGING; + assertThat(loggingService.getBaseUri()).isEqualTo(new URL("https://intake-logging.wikimedia.org/v1/events?hasty=true")); + } + + @Test void testDestinationEventServiceLocal() throws MalformedURLException { + DestinationEventService loggingService = DestinationEventService.LOCAL; + assertThat(loggingService.getBaseUri()).isEqualTo(new URL("http://localhost:8192/v1/events?hasty=true")); + } + + @Test void testDestinationEventServiceAnalyticsDev() throws MalformedURLException { + DestinationEventService loggingService = DestinationEventService.ANALYTICS; + assertThat(loggingService.getBaseUri(true)).isEqualTo(new URL("https://intake-analytics.wikimedia.org/v1/events")); + } + + @Test void testDestinationEventServiceLoggingDev() throws MalformedURLException { + DestinationEventService loggingService = DestinationEventService.ERROR_LOGGING; + assertThat(loggingService.getBaseUri(true)).isEqualTo(new URL("https://intake-logging.wikimedia.org/v1/events")); + } + + @Test void testDestinationEventServiceLocalDev() throws MalformedURLException { + DestinationEventService loggingService = DestinationEventService.LOCAL; + assertThat(loggingService.getBaseUri(true)).isEqualTo(new URL("http://localhost:8192/v1/events")); + } +} diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/EndToEndIT.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/EndToEndIT.java new file mode 100644 index 00000000000..38a20351c39 --- /dev/null +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/EndToEndIT.java @@ -0,0 +1,199 @@ +package org.wikimedia.metrics_platform; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.equalToJson; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.post; +import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.client.WireMock.verify; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.awaitility.Awaitility.await; +import static org.wikimedia.metrics_platform.ConsistencyITClientData.createConsistencyTestClientData; +import static org.wikimedia.metrics_platform.context.DataFixtures.getTestCustomData; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.wikimedia.metrics_platform.context.ClientData; +import org.wikimedia.metrics_platform.context.DataFixtures; + +import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import com.google.common.io.Resources; + +@WireMockTest(httpPort = 8192) +public class EndToEndIT { + private String expectedEventClick; + private String expectedEventClickCustom; + private String expectedEventInteraction; + private String expectedEventView; + private byte[] localConfig; + private final ClientData testClientData = createConsistencyTestClientData(); + + @BeforeEach + void fetchStreamConfigs() throws IOException { + // Stub fetching the stream config from api endpoint. + stubStreamConfigFetch(); + } + + @Test void submitClickEvent(WireMockRuntimeInfo wireMockRuntimeInfo) throws IOException { + // Create the metrics client. + MetricsClient testMetricsClient = buildMetricsClient(wireMockRuntimeInfo); + await().atMost(10, SECONDS).until(testMetricsClient::isFullyInitialized); + + // Stub response from posting event to local eventgate logging service. + stubFor(post("/v1/events?hasty=true") + .willReturn(aResponse() + .withBody(getExpectedEventClick()))); + + testMetricsClient.submitClick( + DataFixtures.getTestStream("click"), + DataFixtures.getTestClientData(getExpectedEventClick()), + DataFixtures.getTestInteractionData("TestClick") + ); + + await().atMost(10, SECONDS).until(testMetricsClient::isEventQueueEmpty); + + verify(postRequestedFor(urlEqualTo("/v1/events?hasty=true")) + .withRequestBody(equalToJson(getExpectedEventClick(), true, true))); + } + + @Test void submitClickEventWithCustomData(WireMockRuntimeInfo wireMockRuntimeInfo) throws IOException { + // Create the metrics client. + MetricsClient testMetricsClient = buildMetricsClient(wireMockRuntimeInfo); + await().atMost(10, SECONDS).until(testMetricsClient::isFullyInitialized); + + // Stub response from posting event to local eventgate logging service. + stubFor(post("/v1/events?hasty=true") + .willReturn(aResponse() + .withBody(getExpectedEventClickCustom()))); + + testMetricsClient.submitClick( + DataFixtures.getTestStream("click_custom"), + "/analytics/product_metrics/app/click_custom/1.0.0", + "click.test_event_name_for_end_to_end_testing", + DataFixtures.getTestClientData(getExpectedEventClickCustom()), + getTestCustomData(), + DataFixtures.getTestInteractionData("TestClickCustom") + ); + + await().atMost(10, SECONDS).until(testMetricsClient::isEventQueueEmpty); + + verify(postRequestedFor(urlEqualTo("/v1/events?hasty=true")) + .withRequestBody(equalToJson(getExpectedEventClickCustom(), true, true))); + } + + @Test void submitViewEvent(WireMockRuntimeInfo wireMockRuntimeInfo) throws IOException { + // Create the metrics client. + MetricsClient testMetricsClient = buildMetricsClient(wireMockRuntimeInfo); + await().atMost(10, SECONDS).until(testMetricsClient::isFullyInitialized); + + // Stub response from posting event to local eventgate logging service. + stubFor(post("/v1/events?hasty=true") + .willReturn(aResponse() + .withBody(getExpectedEventView()))); + + testMetricsClient.submitView( + DataFixtures.getTestStream("view"), + DataFixtures.getTestClientData(getExpectedEventView()), + DataFixtures.getTestInteractionData("TestView") + ); + + await().atMost(10, SECONDS).until(testMetricsClient::isEventQueueEmpty); + + verify(postRequestedFor(urlEqualTo("/v1/events?hasty=true")) + .withRequestBody(equalToJson(getExpectedEventView(), true, true))); + } + + @Test void submitInteractionEvent(WireMockRuntimeInfo wireMockRuntimeInfo) throws IOException { + // Create the metrics client. + MetricsClient testMetricsClient = buildMetricsClient(wireMockRuntimeInfo); + await().atMost(10, SECONDS).until(testMetricsClient::isFullyInitialized); + + // Stub response from posting event to local eventgate logging service. + stubFor(post("/v1/events?hasty=true") + .willReturn(aResponse() + .withBody(getExpectedEventInteraction()))); + + testMetricsClient.submitInteraction( + DataFixtures.getTestStream("interaction"), + "interaction.test_event_name_for_end_to_end_testing", + DataFixtures.getTestClientData(getExpectedEventInteraction()), + DataFixtures.getTestInteractionData("TestInteraction") + ); + + await().atMost(15, SECONDS).until(testMetricsClient::isEventQueueEmpty); + + verify(postRequestedFor(urlEqualTo("/v1/events?hasty=true")) + .withRequestBody(equalToJson(getExpectedEventInteraction(), true, true))); + } + + private byte[] readConfig() throws IOException { + if (this.localConfig == null) { + this.localConfig = Resources.asByteSource( + Resources.getResource("org/wikimedia/metrics_platform/config/streamconfigs-local.json") + ).read(); + } + return this.localConfig; + } + + private String getExpectedEventClick() throws IOException { + if (this.expectedEventClick == null) { + this.expectedEventClick = Resources.asCharSource( + Resources.getResource("org/wikimedia/metrics_platform/event/expected_event_click.json"), + UTF_8 + ).read(); + } + return this.expectedEventClick; + } + + private String getExpectedEventClickCustom() throws IOException { + if (this.expectedEventClickCustom == null) { + this.expectedEventClickCustom = Resources.asCharSource( + Resources.getResource("org/wikimedia/metrics_platform/event/expected_event_click_custom.json"), + UTF_8 + ).read(); + } + return this.expectedEventClickCustom; + } + + private String getExpectedEventView() throws IOException { + if (this.expectedEventView == null) { + this.expectedEventView = Resources.asCharSource( + Resources.getResource("org/wikimedia/metrics_platform/event/expected_event_view.json"), + UTF_8 + ).read(); + } + return this.expectedEventView; + } + + private String getExpectedEventInteraction() throws IOException { + if (this.expectedEventInteraction == null) { + this.expectedEventInteraction = Resources.asCharSource( + Resources.getResource("org/wikimedia/metrics_platform/event/expected_event_interaction.json"), + UTF_8 + ).read(); + } + return this.expectedEventInteraction; + } + + private void stubStreamConfigFetch() throws IOException { + stubFor(get(urlEqualTo("/config")) + .willReturn(aResponse() + .withStatus(200) + .withBody(readConfig()))); + } + + private MetricsClient buildMetricsClient(WireMockRuntimeInfo wireMockRuntimeInfo) throws MalformedURLException { + return MetricsClient.builder(testClientData) + .streamConfigURL(new URL(wireMockRuntimeInfo.getHttpBaseUrl() + "/config")) + .isDebug(false) + .build(); + } +} diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/EventProcessorTest.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/EventProcessorTest.java new file mode 100644 index 00000000000..cba16779ab3 --- /dev/null +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/EventProcessorTest.java @@ -0,0 +1,177 @@ +package org.wikimedia.metrics_platform; + +import static java.lang.Boolean.FALSE; +import static java.lang.Boolean.TRUE; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyCollection; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.wikimedia.metrics_platform.event.EventFixtures.minimalEventProcessed; + +import java.io.IOException; +import java.net.URL; +import java.net.UnknownHostException; +import java.util.Collection; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicReference; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.wikimedia.metrics_platform.config.SourceConfig; +import org.wikimedia.metrics_platform.config.SourceConfigFixtures; +import org.wikimedia.metrics_platform.config.StreamConfig; +import org.wikimedia.metrics_platform.context.ClientData; +import org.wikimedia.metrics_platform.event.EventProcessed; + +@ExtendWith(MockitoExtension.class) +class EventProcessorTest { + @Mock private EventSender mockEventSender; + @Mock private CurationController mockCurationController; + private final AtomicReference sourceConfig = new AtomicReference<>(); + private final BlockingQueue eventQueue = new LinkedBlockingQueue<>(10); + private final ClientData consistencyTestClientData = ConsistencyITClientData.createConsistencyTestClientData(); + private final SamplingController samplingController = new SamplingController(consistencyTestClientData, new SessionController()); + private EventProcessor eventProcessor; + + @BeforeEach void clearEventQueue() { + eventQueue.clear(); + } + + @BeforeEach void createEventProcessor() { + sourceConfig.set(SourceConfigFixtures.getTestSourceConfigMax()); + + eventProcessor = new EventProcessor( + new ContextController(), + mockCurationController, + sourceConfig, + samplingController, + mockEventSender, + eventQueue, + false + ); + } + + @Test void enqueuedEventsAreSent() throws IOException { + whenEventsArePassingCurationFilter(); + + eventQueue.offer(minimalEventProcessed()); + eventProcessor.sendEnqueuedEvents(); + verify(mockEventSender).sendEvents(any(URL.class), anyCollection()); + } + + @Test void eventsNotPassingCurationFiltersAreDropped() throws IOException { + whenEventsAreNotPassingCurationFilter(); + + eventQueue.offer(minimalEventProcessed()); + eventProcessor.sendEnqueuedEvents(); + + verify(mockEventSender, never()).sendEvents(any(URL.class), anyCollection()); + assertThat(eventQueue).isEmpty(); + } + + @Test void eventsAreRemovedFromQueueOnceSent() { + whenEventsArePassingCurationFilter(); + + eventQueue.offer(minimalEventProcessed()); + eventProcessor.sendEnqueuedEvents(); + + assertThat(eventQueue).isEmpty(); + } + + @Test void eventsRemainInOutputBufferOnFailure() throws IOException { + whenEventsArePassingCurationFilter(); + doThrow(UnknownHostException.class).when(mockEventSender).sendEvents(any(URL.class), anyCollection()); + + eventQueue.offer(minimalEventProcessed()); + eventProcessor.sendEnqueuedEvents(); + + assertThat(eventQueue).isNotEmpty(); + } + + @Test void eventsAreEnrichedBeforeBeingSent() throws IOException { + whenEventsArePassingCurationFilter(); + + eventQueue.offer(minimalEventProcessed()); + eventProcessor.sendEnqueuedEvents(); + + @SuppressWarnings("unchecked") + ArgumentCaptor> eventCaptor = ArgumentCaptor.forClass(Collection.class); + verify(mockEventSender).sendEvents(any(URL.class), eventCaptor.capture()); + + EventProcessed sentEvent = eventCaptor.getValue().iterator().next(); + + // Verify client data based on minimum provided values in StreamConfigFixtures. + assertThat(sentEvent.getAgentData().getClientPlatform()).isEqualTo("android"); + assertThat(sentEvent.getAgentData().getClientPlatformFamily()).isEqualTo("app"); + assertThat(sentEvent.getPageData().getTitle()).isEqualTo("Test Page Title"); + assertThat(sentEvent.getMediawikiData().getDatabase()).isEqualTo("enwiki"); + assertThat(sentEvent.getPerformerData().getSessionId()).isEqualTo("eeeeeeeeeeeeeeeeeeee"); + } + + @Test void eventsNotSentWhenFetchStreamConfigFails() { + sourceConfig.set(null); + + eventQueue.offer(minimalEventProcessed()); + eventProcessor.sendEnqueuedEvents(); + + assertThat(eventQueue).isNotEmpty(); + } + + @Test void testSentEventsHaveClientData() throws IOException { + whenEventsArePassingCurationFilter(); + + eventQueue.offer(minimalEventProcessed()); + eventProcessor.sendEnqueuedEvents(); + + @SuppressWarnings("unchecked") + ArgumentCaptor> eventCaptor = ArgumentCaptor.forClass(Collection.class); + verify(mockEventSender).sendEvents(any(URL.class), eventCaptor.capture()); + + EventProcessed sentEvent = eventCaptor.getValue().iterator().next(); + + // Verify client data based on extended provided values in StreamConfigFixtures. + assertThat(sentEvent.getAgentData().getAppFlavor()).isEqualTo("devdebug"); + assertThat(sentEvent.getAgentData().getAppInstallId()).isEqualTo("ffffffff-ffff-ffff-ffff-ffffffffffff"); + assertThat(sentEvent.getAgentData().getAppTheme()).isEqualTo("LIGHT"); + assertThat(sentEvent.getAgentData().getAppVersion()).isEqualTo(982734); + assertThat(sentEvent.getAgentData().getAppVersionName()).isEqualTo("2.7.50470-dev-2024-02-14"); + assertThat(sentEvent.getAgentData().getClientPlatform()).isEqualTo("android"); + assertThat(sentEvent.getAgentData().getClientPlatformFamily()).isEqualTo("app"); + assertThat(sentEvent.getAgentData().getDeviceLanguage()).isEqualTo("en"); + assertThat(sentEvent.getAgentData().getReleaseStatus()).isEqualTo("dev"); + + assertThat(sentEvent.getPageData().getId()).isEqualTo(1); + assertThat(sentEvent.getPageData().getNamespaceId()).isEqualTo(0); + assertThat(sentEvent.getPageData().getWikidataItemQid()).isEqualTo("Q123456"); + + assertThat(sentEvent.getPerformerData().getPageviewId()).isEqualTo("eeeeeeeeeeeeeeeeeeee"); + assertThat(sentEvent.getPerformerData().getLanguageGroups()).isEqualTo("zh, en"); + assertThat(sentEvent.getPerformerData().getLanguagePrimary()).isEqualTo("zh-tw"); + } + + private void whenEventsArePassingCurationFilter() { + when( + mockCurationController.shouldProduceEvent( + any(EventProcessed.class), + any(StreamConfig.class) + ) + ).thenReturn(TRUE); + } + + private void whenEventsAreNotPassingCurationFilter() { + when( + mockCurationController.shouldProduceEvent( + any(EventProcessed.class), + any(StreamConfig.class) + ) + ).thenReturn(FALSE); + } +} diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/EventTest.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/EventTest.java new file mode 100644 index 00000000000..a51070c2534 --- /dev/null +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/EventTest.java @@ -0,0 +1,154 @@ +package org.wikimedia.metrics_platform; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.wikimedia.metrics_platform.MetricsClient.DATE_FORMAT; +import static org.wikimedia.metrics_platform.event.EventProcessed.fromEvent; + +import java.time.Instant; +import java.util.Collections; +import java.util.Locale; +import java.util.UUID; + +import org.junit.jupiter.api.Test; +import org.wikimedia.metrics_platform.context.AgentData; +import org.wikimedia.metrics_platform.context.ClientData; +import org.wikimedia.metrics_platform.context.DataFixtures; +import org.wikimedia.metrics_platform.event.Event; +import org.wikimedia.metrics_platform.event.EventProcessed; +import org.wikimedia.metrics_platform.json.GsonHelper; + +import com.google.gson.Gson; + +class EventTest { + + @Test void testEvent() { + Event eventBasic = new Event("test/event/1.0.0", "test.event", "testEvent"); + EventProcessed event = fromEvent(eventBasic); + String timestamp = DATE_FORMAT.format(Instant.EPOCH); + event.setTimestamp(timestamp); + + assertThat(event.getStream()).isEqualTo("test.event"); + assertThat(event.getTimestamp()).isEqualTo("1970-01-01T00:00:00Z"); + } + + @Test void testEventSerialization() { + String uuid = UUID.randomUUID().toString(); + Event eventBasic = new Event("test/event/1.0.0", "test.event", "testEvent"); + EventProcessed event = fromEvent(eventBasic); + + event.setTimestamp("2021-08-27T12:00:00Z"); + + event.setClientData(DataFixtures.getTestClientData()); + ClientData clientData = DataFixtures.getTestClientData(); + clientData.setAgentData( + AgentData.builder() + .appFlavor("flamingo") + .appInstallId(uuid) + .appTheme("giraffe") + .appVersion(123456789) + .clientPlatform("android") + .clientPlatformFamily("app") + .deviceLanguage("en") + .releaseStatus("beta") + .build() + ); + event.setClientData(clientData); + + assertThat(event.getStream()).isEqualTo("test.event"); + assertThat(event.getSchema()).isEqualTo("test/event/1.0.0"); + assertThat(event.getName()).isEqualTo("testEvent"); + assertThat(event.getAgentData().getAppFlavor()).isEqualTo("flamingo"); + assertThat(event.getAgentData().getAppInstallId()).isEqualTo(uuid); + assertThat(event.getAgentData().getAppTheme()).isEqualTo("giraffe"); + assertThat(event.getAgentData().getAppVersion()).isEqualTo(123456789); + assertThat(event.getTimestamp()).isEqualTo("2021-08-27T12:00:00Z"); + assertThat(event.getAgentData().getClientPlatform()).isEqualTo("android"); + assertThat(event.getAgentData().getClientPlatformFamily()).isEqualTo("app"); + assertThat(event.getAgentData().getDeviceLanguage()).isEqualTo("en"); + assertThat(event.getAgentData().getReleaseStatus()).isEqualTo("beta"); + + assertThat(event.getPageData().getId()).isEqualTo(1); + assertThat(event.getPageData().getTitle()).isEqualTo("Test Page Title"); + assertThat(event.getPageData().getNamespaceId()).isEqualTo(0); + assertThat(event.getPageData().getNamespaceName()).isEqualTo("Main"); + assertThat(event.getPageData().getRevisionId()).isEqualTo(1); + assertThat(event.getPageData().getWikidataItemQid()).isEqualTo("Q123456"); + assertThat(event.getPageData().getContentLanguage()).isEqualTo("en"); + + assertThat(event.getMediawikiData().getDatabase()).isEqualTo("enwiki"); + + assertThat(event.getPerformerData().getId()).isEqualTo(1); + assertThat(event.getPerformerData().getName()).isEqualTo("TestPerformer"); + assertThat(event.getPerformerData().getIsLoggedIn()).isTrue(); + assertThat(event.getPerformerData().getIsTemp()).isFalse(); + assertThat(event.getPerformerData().getSessionId()).isEqualTo("eeeeeeeeeeeeeeeeeeee"); + assertThat(event.getPerformerData().getPageviewId()).isEqualTo("eeeeeeeeeeeeeeeeeeee"); + assertThat(event.getPerformerData().getGroups()).isEqualTo(Collections.singletonList("*")); + assertThat(event.getPerformerData().getLanguageGroups()).isEqualTo("zh, en"); + assertThat(event.getPerformerData().getLanguagePrimary()).isEqualTo("zh-tw"); + assertThat(event.getPerformerData().getRegistrationDt()).isEqualTo("2023-03-01T01:08:30Z"); + + event.setInteractionData(DataFixtures.getTestInteractionData("TestAction")); + + assertThat(event.getAction()).isEqualTo("TestAction"); + assertThat(event.getActionSource()).isEqualTo("TestActionSource"); + assertThat(event.getActionContext()).isEqualTo("TestActionContext"); + assertThat(event.getActionSubtype()).isEqualTo("TestActionSubtype"); + assertThat(event.getElementId()).isEqualTo("TestElementId"); + assertThat(event.getElementFriendlyName()).isEqualTo("TestElementFriendlyName"); + assertThat(event.getFunnelEntryToken()).isEqualTo("TestFunnelEntryToken"); + assertThat(event.getFunnelEventSequencePosition()).isEqualTo(8); + + Gson gson = GsonHelper.getGson(); + String json = gson.toJson(event); + assertThat(json).isEqualTo(String.format(Locale.ROOT, + "{" + + "\"agent\":{" + + "\"app_flavor\":\"flamingo\"," + + "\"app_install_id\":\"%s\"," + + "\"app_theme\":\"giraffe\"," + + "\"app_version\":123456789," + + "\"client_platform\":\"android\"," + + "\"client_platform_family\":\"app\"," + + "\"device_language\":\"en\"," + + "\"release_status\":\"beta\"" + + "}," + + "\"page\":{" + + "\"id\":1," + + "\"title\":\"Test Page Title\"," + + "\"namespace_id\":0," + + "\"namespace_name\":\"Main\"," + + "\"revision_id\":1," + + "\"wikidata_qid\":\"Q123456\"," + + "\"content_language\":\"en\"" + + "}," + + "\"mediawiki\":{" + + "\"database\":\"enwiki\"" + + "}," + + "\"performer\":{" + + "\"id\":1," + + "\"name\":\"TestPerformer\"," + + "\"is_logged_in\":true," + + "\"is_temp\":false," + + "\"session_id\":\"eeeeeeeeeeeeeeeeeeee\"," + + "\"pageview_id\":\"eeeeeeeeeeeeeeeeeeee\"," + + "\"groups\":[\"*\"]," + + "\"language_groups\":\"zh, en\"," + + "\"language_primary\":\"zh-tw\"," + + "\"registration_dt\":\"2023-03-01T01:08:30Z\"" + + "}," + + "\"action\":\"TestAction\"," + + "\"action_subtype\":\"TestActionSubtype\"," + + "\"action_source\":\"TestActionSource\"," + + "\"action_context\":\"TestActionContext\"," + + "\"element_id\":\"TestElementId\"," + + "\"element_friendly_name\":\"TestElementFriendlyName\"," + + "\"funnel_entry_token\":\"TestFunnelEntryToken\"," + + "\"funnel_event_sequence_position\":8," + + "\"$schema\":\"test/event/1.0.0\"," + + "\"dt\":\"2021-08-27T12:00:00Z\"," + + "\"meta\":{\"stream\":\"test.event\"}" + + "}", uuid, uuid)); + } + +} diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/MetricsClientTest.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/MetricsClientTest.java new file mode 100644 index 00000000000..d106f4393ac --- /dev/null +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/MetricsClientTest.java @@ -0,0 +1,269 @@ +package org.wikimedia.metrics_platform; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.wikimedia.metrics_platform.MetricsClient.METRICS_PLATFORM_SCHEMA_BASE; +import static org.wikimedia.metrics_platform.config.StreamConfigFixtures.streamConfig; +import static org.wikimedia.metrics_platform.context.DataFixtures.getTestClientData; +import static org.wikimedia.metrics_platform.context.DataFixtures.getTestCustomData; +import static org.wikimedia.metrics_platform.curation.CurationFilterFixtures.curationFilter; +import static org.wikimedia.metrics_platform.event.EventProcessed.fromEvent; + +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicReference; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.wikimedia.metrics_platform.config.SourceConfig; +import org.wikimedia.metrics_platform.config.SourceConfigFixtures; +import org.wikimedia.metrics_platform.config.StreamConfig; +import org.wikimedia.metrics_platform.context.ClientData; +import org.wikimedia.metrics_platform.context.DataFixtures; +import org.wikimedia.metrics_platform.context.InteractionData; +import org.wikimedia.metrics_platform.context.PageData; +import org.wikimedia.metrics_platform.context.PerformerData; +import org.wikimedia.metrics_platform.event.Event; +import org.wikimedia.metrics_platform.event.EventProcessed; + +@ExtendWith(MockitoExtension.class) +class MetricsClientTest { + + @Mock private ClientData clientData; + @Mock private SessionController mockSessionController; + @Mock private SamplingController mockSamplingController; + private MetricsClient client; + private BlockingQueue eventQueue; + private AtomicReference sourceConfig; + + @BeforeEach void createEventProcessorMetricsClient() { + eventQueue = new LinkedBlockingQueue<>(10); + sourceConfig = new AtomicReference<>(SourceConfigFixtures.getTestSourceConfigMin()); + clientData = DataFixtures.getTestClientData(); + + client = MetricsClient.builder(clientData) + .sessionController(mockSessionController) + .samplingController(mockSamplingController) + .sourceConfigRef(sourceConfig) + .eventQueue(eventQueue) + .build(); + } + + @Test void testSubmit() { + Event event = new Event(METRICS_PLATFORM_SCHEMA_BASE, "test_stream", "test_event"); + client.submit(event); + EventProcessed eventProcessed = eventQueue.peek(); + String stream = eventProcessed.getStream(); + assertThat(stream).isEqualTo("test_stream"); + } + + @Test void testSubmitMetricsEventWithoutClientData() { + when(mockSamplingController.isInSample(streamConfig(curationFilter()))).thenReturn(true); + + Map customDataMap = getTestCustomData(); + client.submitMetricsEvent("test_stream", METRICS_PLATFORM_SCHEMA_BASE, "test_event", customDataMap); + + assertThat(eventQueue).isNotEmpty(); + + EventProcessed queuedEvent = eventQueue.remove(); + + // Verify custom data + assertThat(queuedEvent.getName()).isEqualTo("test_event"); + Map customData = queuedEvent.getCustomData(); + assertThat(customData.get("font_size")).isEqualTo("small"); + assertThat(customData.get("is_full_width")).isEqualTo(true); + assertThat(customData.get("screen_size")).isEqualTo(1080); + + // Verify that client data is not included + assertThat(queuedEvent.getAgentData().getClientPlatform()).isNull(); + assertThat(queuedEvent.getPageData().getId()).isNull(); + assertThat(queuedEvent.getPageData().getTitle()).isNull(); + assertThat(queuedEvent.getPageData().getNamespaceId()).isNull(); + assertThat(queuedEvent.getPageData().getNamespaceName()).isNull(); + assertThat(queuedEvent.getPageData().getRevisionId()).isNull(); + assertThat(queuedEvent.getPageData().getWikidataItemQid()).isNull(); + assertThat(queuedEvent.getPageData().getContentLanguage()).isNull(); + assertThat(queuedEvent.getMediawikiData().getDatabase()).isNull(); + assertThat(queuedEvent.getPerformerData().getId()).isNull(); + } + + @Test void testSubmitMetricsEventWithClientData() { + when(mockSamplingController.isInSample(streamConfig(curationFilter()))).thenReturn(true); + + // Update a few client data members to confirm that the client data parameter during metrics client + // instantiation gets overridden with the client data sent with the event. + ClientData clientData = DataFixtures.getTestClientData(); + PageData pageData = PageData.builder() + .id(108) + .title("Revised Page Title") + .namespaceId(0) + .namespaceName("Main") + .revisionId(1L) + .wikidataItemQid("Q123456") + .contentLanguage("en") + .build(); + clientData.setPageData(pageData); + + client.submitMetricsEvent("test_stream", METRICS_PLATFORM_SCHEMA_BASE, "test_event", clientData, getTestCustomData()); + + assertThat(eventQueue).isNotEmpty(); + + EventProcessed queuedEvent = eventQueue.remove(); + + // Verify custom data + assertThat(queuedEvent.getName()).isEqualTo("test_event"); + Map customData = queuedEvent.getCustomData(); + assertThat(customData.get("font_size")).isEqualTo("small"); + assertThat(customData.get("is_full_width")).isEqualTo(true); + assertThat(customData.get("screen_size")).isEqualTo(1080); + + // Verify client data + assertThat(queuedEvent.getAgentData().getAppInstallId()).isEqualTo("ffffffff-ffff-ffff-ffff-ffffffffffff"); + assertThat(queuedEvent.getAgentData().getClientPlatform()).isEqualTo("android"); + assertThat(queuedEvent.getAgentData().getClientPlatformFamily()).isEqualTo("app"); + + assertThat(queuedEvent.getPageData().getId()).isEqualTo(108); + assertThat(queuedEvent.getPageData().getTitle()).isEqualTo("Revised Page Title"); + assertThat(queuedEvent.getPageData().getNamespaceId()).isEqualTo(0); + assertThat(queuedEvent.getPageData().getNamespaceName()).isEqualTo("Main"); + assertThat(queuedEvent.getPageData().getRevisionId()).isEqualTo(1L); + assertThat(queuedEvent.getPageData().getWikidataItemQid()).isEqualTo("Q123456"); + assertThat(queuedEvent.getPageData().getContentLanguage()).isEqualTo("en"); + + assertThat(queuedEvent.getMediawikiData().getDatabase()).isEqualTo("enwiki"); + + assertThat(queuedEvent.getPerformerData().getId()).isEqualTo(1); + assertThat(queuedEvent.getPerformerData().getName()).isEqualTo("TestPerformer"); + assertThat(queuedEvent.getPerformerData().getIsLoggedIn()).isTrue(); + assertThat(queuedEvent.getPerformerData().getIsTemp()).isFalse(); + assertThat(queuedEvent.getPerformerData().getPageviewId()).isEqualTo("eeeeeeeeeeeeeeeeeeee"); + assertThat(queuedEvent.getPerformerData().getGroups()).contains("*"); + assertThat(queuedEvent.getPerformerData().getLanguageGroups()).isEqualTo("zh, en"); + assertThat(queuedEvent.getPerformerData().getLanguagePrimary()).isEqualTo("zh-tw"); + assertThat(queuedEvent.getPerformerData().getRegistrationDt()).isEqualTo("2023-03-01T01:08:30Z"); + + assertThat(queuedEvent.getClientData().domain).isEqualTo("en.wikipedia.org"); + } + + @Test void testSubmitMetricsEventWithInteractionData() { + when(mockSamplingController.isInSample(streamConfig(curationFilter()))).thenReturn(true); + + ClientData clientData = DataFixtures.getTestClientData(); + Map customDataMap = getTestCustomData(); + InteractionData interactionData = DataFixtures.getTestInteractionData("TestAction"); + client.submitMetricsEvent("test_stream", METRICS_PLATFORM_SCHEMA_BASE, "test_event", clientData, customDataMap, interactionData); + + assertThat(eventQueue).isNotEmpty(); + + EventProcessed queuedEvent = eventQueue.remove(); + + assertThat(queuedEvent.getAction()).isEqualTo("TestAction"); + assertThat(queuedEvent.getActionSource()).isEqualTo("TestActionSource"); + assertThat(queuedEvent.getActionContext()).isEqualTo("TestActionContext"); + assertThat(queuedEvent.getActionSubtype()).isEqualTo("TestActionSubtype"); + assertThat(queuedEvent.getElementId()).isEqualTo("TestElementId"); + assertThat(queuedEvent.getElementFriendlyName()).isEqualTo("TestElementFriendlyName"); + assertThat(queuedEvent.getFunnelEntryToken()).isEqualTo("TestFunnelEntryToken"); + assertThat(queuedEvent.getFunnelEventSequencePosition()).isEqualTo(8); + } + + @Test void testSubmitMetricsEventIncludesSample() { + StreamConfig streamConfig = streamConfig(curationFilter()); + + when(mockSamplingController.isInSample(streamConfig)).thenReturn(true); + + Map customDataMap = getTestCustomData(); + client.submitMetricsEvent("test_stream", METRICS_PLATFORM_SCHEMA_BASE, "test_event", customDataMap); + + assertThat(eventQueue).isNotEmpty(); + + EventProcessed queuedEvent = eventQueue.remove(); + + assertThat(queuedEvent.getSample()).isEqualTo(streamConfig.getSampleConfig()); + } + + @Test void testSubmitWhenEventQueueIsFull() { + for (int i = 1; i <= 10; i++) { + Event event = new Event("test_schema" + i, "test_stream" + i, "test_event" + i); + EventProcessed eventProcessed = fromEvent(event); + eventQueue.add(eventProcessed); + } + EventProcessed oldestEvent = eventQueue.peek(); + + Event event11 = new Event("test_schema11", "test_stream11", "test_event11"); + EventProcessed eventProcessed11 = fromEvent(event11); + client.submit(eventProcessed11); + + assertThat(eventQueue).doesNotContain(oldestEvent); + + Boolean containsNewestEvent = eventQueue.stream().anyMatch(event -> event.getName().equals("test_event11")); + assertThat(containsNewestEvent).isTrue(); + } + + @Test void testTouchSessionOnAppPause() { + when(mockSamplingController.isInSample(streamConfig(curationFilter()))).thenReturn(true); + fillEventQueue(); + assertThat(eventQueue).isNotEmpty(); + + client.onAppPause(); + verify(mockSessionController).touchSession(); + } + + @Test void testResumeSessionOnAppResume() { + client.onAppResume(); + verify(mockSessionController).touchSession(); + } + + @Test void testResetSession() { + client.resetSession(); + verify(mockSessionController).beginSession(); + } + + @Test void testCloseSessionOnAppClose() { + when(mockSamplingController.isInSample(streamConfig(curationFilter()))).thenReturn(true); + fillEventQueue(); + assertThat(eventQueue).isNotEmpty(); + + client.onAppClose(); + verify(mockSessionController).closeSession(); + } + + @Test void testAddRequiredMetadata() { + Event event = new Event("test/event/1.0.0", "test_event", "testEvent"); + assertThat(event.getTimestamp()).isNull(); + + client.submit(event); + EventProcessed queuedEvent = eventQueue.remove(); + + assertThat(queuedEvent.getTimestamp()).isNotNull(); + verify(mockSessionController).getSessionId(); + } + + @Test void testPerformerDataLanguageGroups() { + Event event = new Event("test/event/1.0.0", "test_event", "testEvent"); + ClientData clientData = getTestClientData(); + PerformerData performerData = event.getClientData().getPerformerData(); + clientData.setPerformerData(PerformerData.builderFrom(performerData) + .languageGroups("[zh-hant, zh-hans, ja, en, zh-yue, ko, fr, de, it, es, pt, da, tr, ru, nl, sv, cs, " + + "fi, uk, el, pl, hu, vi, id, ca, mk, sl, ms, tl, avk, lt, sr-el, eu, nb, ceb, als, uz-latn, " + + "az, af, nn, et, eo, la, br, jv, io, bg, ro, nrm, pcd, tg-latn, lmo, gl, cy, sq, is, ha, gd, " + + "ku-latn, hr, lv, sk, bar, pms, lld, ga, war]") + .build()); + event.setClientData(clientData); + + client.submit(event); + EventProcessed queuedEvent = eventQueue.remove(); + assertThat(queuedEvent.getPerformerData().getLanguageGroups().length()).isEqualTo(255); + } + + private void fillEventQueue() { + for (int i = 1; i <= 10; i++) { + client.submitMetricsEvent("test_stream", METRICS_PLATFORM_SCHEMA_BASE, "test_event", getTestCustomData()); + } + } +} diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/SamplingControllerTest.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/SamplingControllerTest.java new file mode 100644 index 00000000000..80d937fd810 --- /dev/null +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/SamplingControllerTest.java @@ -0,0 +1,58 @@ +package org.wikimedia.metrics_platform; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.wikimedia.metrics_platform.config.sampling.SampleConfig.Identifier.DEVICE; +import static org.wikimedia.metrics_platform.config.sampling.SampleConfig.Identifier.SESSION; + +import org.junit.jupiter.api.Test; +import org.wikimedia.metrics_platform.config.sampling.SampleConfig; +import org.wikimedia.metrics_platform.config.StreamConfig; +import org.wikimedia.metrics_platform.context.DataFixtures; + +class SamplingControllerTest { + + private final SamplingController samplingController = new SamplingController( + DataFixtures.getTestClientData(), + new SessionController() + ); + + @Test void testGetSamplingValue() { + double deviceVal = samplingController.getSamplingValue(DEVICE); + assertThat(deviceVal).isBetween(0.0, 1.0); + } + + @Test void testGetSamplingId() { + assertThat(samplingController.getSamplingId(DEVICE)).isNotNull(); + assertThat(samplingController.getSamplingId(SESSION)).isNotNull(); + } + + @Test void testNoSamplingConfig() { + StreamConfig noSamplingConfig = new StreamConfig("foo", "bar", null, null, null); + assertThat(samplingController.isInSample(noSamplingConfig)).isTrue(); + } + + @Test void testAlwaysInSample() { + StreamConfig alwaysInSample = new StreamConfig("foo", "bar", null, + new StreamConfig.ProducerConfig(new StreamConfig.MetricsPlatformClientConfig( + null, + null, + null + )), + null + ); + assertThat(samplingController.isInSample(alwaysInSample)).isTrue(); + } + + @Test void testNeverInSample() { + StreamConfig neverInSample = new StreamConfig("foo", "bar", null, + new StreamConfig.ProducerConfig(new StreamConfig.MetricsPlatformClientConfig( + null, + null, + null + )), + new SampleConfig(0.0, SESSION) + ); + assertThat(samplingController.isInSample(neverInSample)).isFalse(); + } + +} diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/SessionControllerTest.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/SessionControllerTest.java new file mode 100644 index 00000000000..262dc106a31 --- /dev/null +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/SessionControllerTest.java @@ -0,0 +1,36 @@ +package org.wikimedia.metrics_platform; + +import static java.time.temporal.ChronoUnit.HOURS; +import static org.assertj.core.api.Assertions.assertThat; + +import java.time.Instant; + +import org.junit.jupiter.api.Test; + +class SessionControllerTest { + + @Test void testSessionExpiry() { + Instant oneHourAgo = Instant.now().minus(1, HOURS); + SessionController sessionController = new SessionController(oneHourAgo); + assertThat(sessionController.sessionExpired()).isTrue(); + } + + @Test void testTouchSession() { + Instant oneHourAgo = Instant.now().minus(1, HOURS); + SessionController sessionController = new SessionController(oneHourAgo); + String sessionId1 = sessionController.getSessionId(); + sessionController.touchSession(); + assertThat(sessionController.sessionExpired()).isFalse(); + + String sessionId2 = sessionController.getSessionId(); + + assertThat(sessionId1).isNotEqualTo(sessionId2); + } + + @Test void testSessionIdLength() { + Instant twoHoursAgo = Instant.now().minus(2, HOURS); + SessionController sessionController = new SessionController(twoHoursAgo); + String sessionId = sessionController.getSessionId(); + assertThat(sessionId.length()).isEqualTo(20); + } +} diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/TestEventSender.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/TestEventSender.java new file mode 100644 index 00000000000..1bb196bd012 --- /dev/null +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/TestEventSender.java @@ -0,0 +1,27 @@ +package org.wikimedia.metrics_platform; + +import java.io.IOException; +import java.net.URL; +import java.util.Collection; + +import org.wikimedia.metrics_platform.event.EventProcessed; + +class TestEventSender implements EventSender { + + private final boolean shouldFail; + + TestEventSender() { + this(false); + } + + TestEventSender(boolean shouldFail) { + this.shouldFail = shouldFail; + } + + @Override + public void sendEvents(URL baseUri, Collection events) throws IOException { + if (shouldFail) { + throw new IOException(); + } + } +} diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/CurationFilterFixtures.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/CurationFilterFixtures.java new file mode 100644 index 00000000000..17631cab12f --- /dev/null +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/CurationFilterFixtures.java @@ -0,0 +1,25 @@ +package org.wikimedia.metrics_platform.config; + +import java.util.Arrays; + +import org.wikimedia.metrics_platform.config.curation.CollectionCurationRules; +import org.wikimedia.metrics_platform.config.curation.CurationRules; + +public final class CurationFilterFixtures { + + private CurationFilterFixtures() { + // Utility class - should never be instantiated + } + + public static CurationFilter getCurationFilter() { + return CurationFilter.builder() + .pageTitleRules(CurationRules.builder().isEquals("Test").build()) + .performerGroupsRules( + CollectionCurationRules.builder() + .doesNotContain("sysop") + .containsAny(Arrays.asList("steward", "bureaucrat")) + .build() + ) + .build(); + } +} diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/SampleConfigTest.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/SampleConfigTest.java new file mode 100644 index 00000000000..9a6083a1d67 --- /dev/null +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/SampleConfigTest.java @@ -0,0 +1,22 @@ +package org.wikimedia.metrics_platform.config; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.wikimedia.metrics_platform.config.sampling.SampleConfig.Identifier.DEVICE; + +import org.junit.jupiter.api.Test; +import org.wikimedia.metrics_platform.config.sampling.SampleConfig; +import org.wikimedia.metrics_platform.json.GsonHelper; + +import com.google.gson.Gson; + +class SampleConfigTest { + + @Test void testSamplingConfigDeserialization() { + Gson gson = GsonHelper.getGson(); + String samplingConfigJson = "{\"rate\":0.25,\"identifier\":\"device\"}"; + SampleConfig sampleConfig = gson.fromJson(samplingConfigJson, SampleConfig.class); + assertThat(sampleConfig.getRate()).isEqualTo(0.25); + assertThat(sampleConfig.getIdentifier()).isEqualTo(DEVICE); + } + +} diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/SourceConfigFixtures.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/SourceConfigFixtures.java new file mode 100644 index 00000000000..506ca2d2a4d --- /dev/null +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/SourceConfigFixtures.java @@ -0,0 +1,22 @@ +package org.wikimedia.metrics_platform.config; + +public final class SourceConfigFixtures { + + private SourceConfigFixtures() { + // Utility class, should never be instantiated + } + + /** + * Convenience method for getting source config with minimum provided values. + */ + public static SourceConfig getTestSourceConfigMin() { + return new SourceConfig(StreamConfigFixtures.streamConfigMap(StreamConfigFixtures.provideValuesMinimum())); + } + + /** + * Convenience method for getting source config with extended provided values. + */ + public static SourceConfig getTestSourceConfigMax() { + return new SourceConfig(StreamConfigFixtures.streamConfigMap(StreamConfigFixtures.provideValuesExtended())); + } +} diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/SourceConfigTest.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/SourceConfigTest.java new file mode 100644 index 00000000000..e7c91b9519c --- /dev/null +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/SourceConfigTest.java @@ -0,0 +1,31 @@ +package org.wikimedia.metrics_platform.config; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +class SourceConfigTest { + + private static SourceConfig sourceConfig; + + @Test void testConfig() { + sourceConfig = new SourceConfig(StreamConfigFixtures.STREAM_CONFIGS_WITH_EVENTS); + assertThat(sourceConfig.getStreamNamesByEvent("test.event")).containsExactly("test.stream"); + } + + @Test void testGetStreamNames() { + sourceConfig = new SourceConfig(StreamConfigFixtures.STREAM_CONFIGS_WITH_EVENTS); + assertThat(sourceConfig.getStreamNames()).containsExactly("test.stream"); + } + + @Test void testGetStreamConfigByName() { + sourceConfig = new SourceConfig(StreamConfigFixtures.STREAM_CONFIGS_WITH_EVENTS); + StreamConfig streamConfig = StreamConfigFixtures.sampleStreamConfig(true); + assertThat(streamConfig).isEqualTo(sourceConfig.getStreamConfigByName("test.stream")); + } + + @Test void testGetStreamNamesByEvent() { + sourceConfig = new SourceConfig(StreamConfigFixtures.STREAM_CONFIGS_WITH_EVENTS); + assertThat(sourceConfig.getStreamNamesByEvent("test.event")).containsExactly("test.stream"); + } +} diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/StreamConfigFetcherTest.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/StreamConfigFetcherTest.java new file mode 100644 index 00000000000..fe9bdc3680e --- /dev/null +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/StreamConfigFetcherTest.java @@ -0,0 +1,50 @@ +package org.wikimedia.metrics_platform.config; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.assertj.core.api.Assertions.assertThat; +import static org.wikimedia.metrics_platform.config.DestinationEventService.LOCAL; +import static org.wikimedia.metrics_platform.config.StreamConfigFetcher.ANALYTICS_API_ENDPOINT; + +import java.io.IOException; +import java.io.Reader; +import java.net.URL; +import java.util.Map; + +import org.junit.jupiter.api.Test; +import org.wikimedia.metrics_platform.json.GsonHelper; + +import com.google.common.io.Resources; + +import okhttp3.OkHttpClient; + +class StreamConfigFetcherTest { + + @Test void parsingConfigFromJsonWorks() throws IOException { + try (Reader in = readConfigFile("streamconfigs.json")) { + StreamConfigFetcher streamConfigFetcher = new StreamConfigFetcher(new URL(ANALYTICS_API_ENDPOINT), new OkHttpClient(), GsonHelper.getGson()); + Map config = streamConfigFetcher.parseConfig(in); + assertThat(config).containsKey("mediawiki.visual_editor_feature_use"); + StreamConfig streamConfig = config.get("mediawiki.visual_editor_feature_use"); + String schemaTitle = streamConfig.getSchemaTitle(); + assertThat(schemaTitle).isEqualTo("analytics/mediawiki/client/metrics_event"); + assertThat(streamConfig.getStreamName()).isEqualTo("mediawiki.visual_editor_feature_use"); + } + } + + @Test void parsingLocalConfigFromJsonWorks() throws IOException { + try (Reader in = readConfigFile("streamconfigs-local.json")) { + StreamConfigFetcher streamConfigFetcher = new StreamConfigFetcher(new URL(ANALYTICS_API_ENDPOINT), new OkHttpClient(), GsonHelper.getGson()); + Map config = streamConfigFetcher.parseConfig(in); + assertThat(config).containsKey("mediawiki.visual_editor_feature_use"); + StreamConfig streamConfig = config.get("mediawiki.edit_attempt"); + assertThat(streamConfig.getDestinationEventService()).isEqualTo(LOCAL); + } + } + + private Reader readConfigFile(String filename) throws IOException { + return Resources.asCharSource( + Resources.getResource("org/wikimedia/metrics_platform/config/" + filename), + UTF_8 + ).openStream(); + } +} diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/StreamConfigFixtures.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/StreamConfigFixtures.java new file mode 100644 index 00000000000..cf260f7a872 --- /dev/null +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/StreamConfigFixtures.java @@ -0,0 +1,164 @@ +package org.wikimedia.metrics_platform.config; + +import static java.util.Collections.emptySet; +import static java.util.Collections.singletonList; +import static java.util.Collections.singletonMap; +import static org.wikimedia.metrics_platform.config.StreamConfigFetcher.METRICS_PLATFORM_SCHEMA_TITLE; +import static org.wikimedia.metrics_platform.context.ContextValue.AGENT_APP_INSTALL_ID; +import static org.wikimedia.metrics_platform.context.ContextValue.AGENT_APP_FLAVOR; +import static org.wikimedia.metrics_platform.context.ContextValue.AGENT_APP_THEME; +import static org.wikimedia.metrics_platform.context.ContextValue.AGENT_APP_VERSION; +import static org.wikimedia.metrics_platform.context.ContextValue.AGENT_DEVICE_LANGUAGE; +import static org.wikimedia.metrics_platform.context.ContextValue.AGENT_RELEASE_STATUS; +import static org.wikimedia.metrics_platform.context.ContextValue.AGENT_CLIENT_PLATFORM; +import static org.wikimedia.metrics_platform.context.ContextValue.AGENT_CLIENT_PLATFORM_FAMILY; +import static org.wikimedia.metrics_platform.context.ContextValue.MEDIAWIKI_DATABASE; +import static org.wikimedia.metrics_platform.context.ContextValue.PAGE_TITLE; +import static org.wikimedia.metrics_platform.context.ContextValue.PAGE_ID; +import static org.wikimedia.metrics_platform.context.ContextValue.PAGE_NAMESPACE_ID; +import static org.wikimedia.metrics_platform.context.ContextValue.PAGE_WIKIDATA_QID; +import static org.wikimedia.metrics_platform.context.ContextValue.PERFORMER_SESSION_ID; +import static org.wikimedia.metrics_platform.context.ContextValue.PERFORMER_PAGEVIEW_ID; +import static org.wikimedia.metrics_platform.context.ContextValue.PERFORMER_LANGUAGE_GROUPS; +import static org.wikimedia.metrics_platform.context.ContextValue.PERFORMER_LANGUAGE_PRIMARY; +import static org.wikimedia.metrics_platform.curation.CurationFilterFixtures.curationFilter; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.wikimedia.metrics_platform.config.sampling.SampleConfig; + +public final class StreamConfigFixtures { + + public static final Map STREAM_CONFIGS_WITH_EVENTS = new HashMap() {{ + put("test.stream", sampleStreamConfig(true)); + } + }; + + private StreamConfigFixtures() { + // Utility class, should never be instantiated + } + + public static StreamConfig sampleStreamConfig(boolean hasEvents) { + Set emptyEvents = emptySet(); + Set testEvents = new HashSet<>(singletonList("test.event")); + Set events = hasEvents ? testEvents : emptyEvents; + Set requestedValuesSet = new HashSet<>(Arrays.asList( + "agent_app_install_id", + "agent_client_platform", + "agent_client_platform_family", + "mediawiki_database", + "page_id", + "page_namespace_id", + "page_namespace_name", + "page_title", + "page_revision_id", + "page_content_language", + "page_wikidata_qid", + "performer_id", + "performer_is_logged_in", + "performer_is_temp", + "performer_name", + "performer_session_id", + "performer_pageview_id", + "performer_groups", + "performer_registration_dt", + "performer_language_groups", + "performer_language_primary" + )); + SampleConfig sampleConfig = new SampleConfig(1.0, SampleConfig.Identifier.PAGEVIEW); + + return new StreamConfig( + "test.stream", + "test/event", + DestinationEventService.ANALYTICS, + new StreamConfig.ProducerConfig( + new StreamConfig.MetricsPlatformClientConfig( + events, + requestedValuesSet, + CurationFilterFixtures.getCurationFilter() + ) + ), + sampleConfig + ); + } + + /** + * Convenience method for getting stream config. + */ + public static StreamConfig streamConfig(CurationFilter curationFilter, String[] provideValues) { + Set events = Collections.singleton("test_event"); + SampleConfig sampleConfig = new SampleConfig(1.0f, SampleConfig.Identifier.PAGEVIEW); + + return new StreamConfig( + "test_stream", + METRICS_PLATFORM_SCHEMA_TITLE, + DestinationEventService.LOCAL, + new StreamConfig.ProducerConfig( + new StreamConfig.MetricsPlatformClientConfig( + events, + new HashSet<>(List.of(provideValues)), + curationFilter + ) + ), + sampleConfig + ); + } + + /** + * Convenience method for getting stream config. + */ + public static StreamConfig streamConfig(CurationFilter curationFilter) { + return streamConfig(curationFilter, provideValuesMinimum()); + } + + /** + * Convenience method for getting a stream config map. + */ + public static Map streamConfigMap(String[] provideValues) { + return streamConfigMap(curationFilter(), provideValues); + } + + + public static Map streamConfigMap(CurationFilter curationFilter, String[] provideValues) { + StreamConfig streamConfig = streamConfig(curationFilter, provideValues); + return singletonMap(streamConfig.getStreamName(), streamConfig); + } + + public static String[] provideValuesMinimum() { + return new String[]{ + AGENT_CLIENT_PLATFORM, + AGENT_CLIENT_PLATFORM_FAMILY, + PAGE_TITLE, + MEDIAWIKI_DATABASE, + PERFORMER_SESSION_ID + }; + } + + public static String[] provideValuesExtended() { + return new String[]{ + AGENT_APP_INSTALL_ID, + AGENT_CLIENT_PLATFORM, + AGENT_CLIENT_PLATFORM_FAMILY, + AGENT_APP_FLAVOR, + AGENT_APP_THEME, + AGENT_APP_VERSION, + AGENT_DEVICE_LANGUAGE, + AGENT_RELEASE_STATUS, + PAGE_ID, + PAGE_TITLE, + PAGE_NAMESPACE_ID, + PAGE_WIKIDATA_QID, + MEDIAWIKI_DATABASE, + PERFORMER_SESSION_ID, + PERFORMER_PAGEVIEW_ID, + PERFORMER_LANGUAGE_GROUPS, + PERFORMER_LANGUAGE_PRIMARY, + }; + } +} diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/StreamConfigIT.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/StreamConfigIT.java new file mode 100644 index 00000000000..3721fa7b7a2 --- /dev/null +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/StreamConfigIT.java @@ -0,0 +1,48 @@ +package org.wikimedia.metrics_platform.config; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.net.URL; + +import org.junit.jupiter.api.Test; +import org.wikimedia.metrics_platform.json.GsonHelper; + +import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import com.google.common.io.Resources; + +import okhttp3.OkHttpClient; + +@WireMockTest +class StreamConfigIT { + + @Test void canLoadConfigOverHTTP(WireMockRuntimeInfo wmRuntimeInfo) throws IOException { + stubFor(get("/streamConfig").willReturn( + aResponse() + .withBody(loadConfigStream()) + ) + ); + + StreamConfigFetcher streamConfigFetcher = new StreamConfigFetcher( + new URL(wmRuntimeInfo.getHttpBaseUrl() + "/streamConfig"), + new OkHttpClient(), + GsonHelper.getGson() + ); + + SourceConfig sourceConfig = streamConfigFetcher.fetchStreamConfigs(); + + assertThat(sourceConfig).isNotNull(); + + } + + private byte[] loadConfigStream() throws IOException { + return Resources.asByteSource( + Resources.getResource("org/wikimedia/metrics_platform/config/streamconfigs.json") + ).read(); + } + +} diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/StreamConfigTest.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/StreamConfigTest.java new file mode 100644 index 00000000000..516025a0a0f --- /dev/null +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/StreamConfigTest.java @@ -0,0 +1,31 @@ +package org.wikimedia.metrics_platform.config; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.wikimedia.metrics_platform.config.sampling.SampleConfig.Identifier.SESSION; + +import java.util.Set; + +import org.junit.jupiter.api.Test; +import org.wikimedia.metrics_platform.json.GsonHelper; + +import com.google.gson.Gson; + +class StreamConfigTest { + + @Test void testStreamConfigDeserialization() { + Gson gson = GsonHelper.getGson(); + String streamConfigJson = "{\"stream\":\"test.event\",\"schema_title\":\"test/event\"," + + "\"destination_event_service\":\"eventgate-logging-local\"," + + "\"producers\":" + + "{\"metrics_platform_client\":{\"provide_values\":[\"page_id\",\"user_id\"]}}," + + "\"sample\":{\"rate\":0.5,\"identifier\":\"session\"}}"; + StreamConfig streamConfig = gson.fromJson(streamConfigJson, StreamConfig.class); + assertThat(streamConfig.getStreamName()).isEqualTo("test.event"); + assertThat(streamConfig.getSchemaTitle()).isEqualTo("test/event"); + assertThat(streamConfig.getSampleConfig().getRate()).isEqualTo(0.5); + assertThat(streamConfig.getSampleConfig().getIdentifier()).isEqualTo(SESSION); + assertThat(streamConfig.getProducerConfig().getMetricsPlatformClientConfig().getRequestedValues()) + .isEqualTo(Set.of("page_id", "user_id")); + assertThat(streamConfig.getDestinationEventService()).isEqualTo(DestinationEventService.LOCAL); + } +} diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/AgentDataTest.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/AgentDataTest.java new file mode 100644 index 00000000000..7bec1fc9b94 --- /dev/null +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/AgentDataTest.java @@ -0,0 +1,52 @@ +package org.wikimedia.metrics_platform.context; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; +import org.wikimedia.metrics_platform.json.GsonHelper; + +import com.google.gson.Gson; + +public class AgentDataTest { + @Test + void testAgentData() { + AgentData agentData = AgentData.builder() + .appFlavor("flamingo") + .appInstallId("ffffffff-ffff-ffff-ffff-ffffffffffff") + .appTheme("giraffe") + .appVersion(123456789) + .appVersionName("2.7.50470-dev-2024-02-14") + .clientPlatform("android") + .clientPlatformFamily("app") + .deviceFamily("Samsung SM-G960F") + .deviceLanguage("en") + .releaseStatus("beta") + .build(); + + assertThat(agentData.getAppFlavor()).isEqualTo("flamingo"); + assertThat(agentData.getAppInstallId()).isEqualTo("ffffffff-ffff-ffff-ffff-ffffffffffff"); + assertThat(agentData.getAppTheme()).isEqualTo("giraffe"); + assertThat(agentData.getAppVersion()).isEqualTo(123456789); + assertThat(agentData.getAppVersionName()).isEqualTo("2.7.50470-dev-2024-02-14"); + assertThat(agentData.getClientPlatform()).isEqualTo("android"); + assertThat(agentData.getClientPlatformFamily()).isEqualTo("app"); + assertThat(agentData.getDeviceFamily()).isEqualTo("Samsung SM-G960F"); + assertThat(agentData.getDeviceLanguage()).isEqualTo("en"); + assertThat(agentData.getReleaseStatus()).isEqualTo("beta"); + + Gson gson = GsonHelper.getGson(); + String json = gson.toJson(agentData); + assertThat(json).isEqualTo("{" + + "\"app_flavor\":\"flamingo\"," + + "\"app_install_id\":\"ffffffff-ffff-ffff-ffff-ffffffffffff\"," + + "\"app_theme\":\"giraffe\"," + + "\"app_version\":123456789," + + "\"app_version_name\":\"2.7.50470-dev-2024-02-14\"," + + "\"client_platform\":\"android\"," + + "\"client_platform_family\":\"app\"," + + "\"device_family\":\"Samsung SM-G960F\"," + + "\"device_language\":\"en\"," + + "\"release_status\":\"beta\"" + + "}"); + } +} diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/CustomDataTest.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/CustomDataTest.java new file mode 100644 index 00000000000..6e0867b20a9 --- /dev/null +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/CustomDataTest.java @@ -0,0 +1,54 @@ +package org.wikimedia.metrics_platform.context; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.wikimedia.metrics_platform.context.DataFixtures.getTestCustomData; + +import java.util.Map; + +import org.junit.jupiter.api.Test; +import org.wikimedia.metrics_platform.json.GsonHelper; + +import com.google.gson.Gson; + +class CustomDataTest { + +// @Test void testFormatCustomDataByCustomSerialization() { +// Map customData = getTestCustomDataFormatted(); +// +// Gson gson = GsonHelper.getGson(); +// String jsonCustomData = gson.toJson(customData); +// +// assertThat(jsonCustomData) +// .isEqualTo("{" + +// "\"is_full_width\":" + +// "{" + +// "\"data_type\":\"boolean\"," + +// "\"value\":\"true\"" + +// "}," + +// "\"font_size\":" + +// "{" + +// "\"data_type\":\"string\"," + +// "\"value\":\"small\"" + +// "}," + +// "\"screen_size\":" + +// "{" + +// "\"data_type\":\"number\"," + +// "\"value\":\"1080\"" + +// "}" + +// "}"); +// } + + @Test void testCustomDataSerialization() { + Map customData = getTestCustomData(); + + Gson gson = GsonHelper.getGson(); + String jsonCustomData = gson.toJson(customData); + + assertThat(jsonCustomData) + .isEqualTo("{" + + "\"is_full_width\":true," + + "\"font_size\":\"small\"," + + "\"screen_size\":1080" + + "}"); + } +} diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/DataFixtures.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/DataFixtures.java new file mode 100644 index 00000000000..e66b80b92c4 --- /dev/null +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/DataFixtures.java @@ -0,0 +1,134 @@ +package org.wikimedia.metrics_platform.context; + +import java.time.Instant; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.wikimedia.metrics_platform.json.GsonHelper; + +import com.google.gson.Gson; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +public final class DataFixtures { + + private DataFixtures() { + // Utility class, should never be instantiated + } + + public static ClientData getTestClientData() { + return new ClientData( + getTestAgentData(), + getTestPageData(), + getTestMediawikiData(), + getTestPerformerData(), + "en.wikipedia.org" + ); + } + + public static ClientData getTestClientData(String expectedEvent) { + Map dataMap = new HashMap<>(); + + JsonElement jsonElement = JsonParser.parseString(expectedEvent); + JsonObject expectedEventJson = jsonElement.isJsonArray() ? jsonElement.getAsJsonArray().get(0).getAsJsonObject() : jsonElement.getAsJsonObject(); + + Set dataObjectNames = Stream.of("agent", "page", "mediawiki", "performer") + .collect(Collectors.toCollection(HashSet::new)); + + for (String dataObjectName : dataObjectNames) { + JsonObject metaData = expectedEventJson.getAsJsonObject(dataObjectName); + Set keys = metaData.keySet(); + Map dataMapEach = new HashMap<>(); + for (String key : keys) { + dataMapEach.put(key, metaData.get(key)); + } + dataMap.put(dataObjectName, dataMapEach); + } + + String domain = expectedEventJson.get("meta").getAsJsonObject().get("domain").getAsString(); + dataMap.put("domain", domain); + + Gson gson = GsonHelper.getGson(); + JsonElement jsonClientData = gson.toJsonTree(dataMap); + return gson.fromJson(jsonClientData, ClientData.class); + } + + public static AgentData getTestAgentData() { + return AgentData.builder() + .appInstallId("ffffffff-ffff-ffff-ffff-ffffffffffff") + .clientPlatform("android") + .clientPlatformFamily("app") + .appFlavor("devdebug") + .appVersion(982734) + .appVersionName("2.7.50470-dev-2024-02-14") + .appTheme("LIGHT") + .deviceFamily("Samsung SM-G960F") + .deviceLanguage("en") + .releaseStatus("dev") + .build(); + } + + public static PageData getTestPageData() { + return PageData.builder() + .id(1) + .title("Test Page Title") + .namespaceId(0) + .namespaceName("Main") + .revisionId(1L) + .wikidataItemQid("Q123456") + .contentLanguage("en") + .build(); + } + + public static MediawikiData getTestMediawikiData() { + return MediawikiData.builder() + .database("enwiki") + .build(); + } + + public static PerformerData getTestPerformerData() { + return PerformerData.builder() + .id(1) + .name("TestPerformer") + .isLoggedIn(true) + .isTemp(false) + .sessionId("eeeeeeeeeeeeeeeeeeee") + .pageviewId("eeeeeeeeeeeeeeeeeeee") + .groups(Collections.singletonList("*")) + .languageGroups("zh, en") + .languagePrimary("zh-tw") + .registrationDt(Instant.parse("2023-03-01T01:08:30Z")) + .build(); + } + + public static InteractionData getTestInteractionData(String action) { + return InteractionData.builder() + .action(action) + .actionSubtype("TestActionSubtype") + .actionSource("TestActionSource") + .actionContext("TestActionContext") + .elementId("TestElementId") + .elementFriendlyName("TestElementFriendlyName") + .funnelEntryToken("TestFunnelEntryToken") + .funnelEventSequencePosition(8) + .build(); + } + + public static Map getTestCustomData() { + Map customData = new HashMap(); + customData.put("font_size", "small"); + customData.put("is_full_width", true); + customData.put("screen_size", 1080); + return customData; + } + + public static String getTestStream(String streamNameFragment) { + return "mediawiki.metrics_platform." + streamNameFragment; + } +} diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/MediawikiDataTest.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/MediawikiDataTest.java new file mode 100644 index 00000000000..20674527fec --- /dev/null +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/MediawikiDataTest.java @@ -0,0 +1,25 @@ +package org.wikimedia.metrics_platform.context; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; +import org.wikimedia.metrics_platform.json.GsonHelper; + +import com.google.gson.Gson; + +public class MediawikiDataTest { + @Test + void testMediawikiData() { + MediawikiData mediawikiData = MediawikiData.builder() + .database("enwiki") + .build(); + + assertThat(mediawikiData.getDatabase()).isEqualTo("enwiki"); + + Gson gson = GsonHelper.getGson(); + String json = gson.toJson(mediawikiData); + assertThat(json).isEqualTo("{" + + "\"database\":\"enwiki\"" + + "}"); + } +} diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/PageDataTest.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/PageDataTest.java new file mode 100644 index 00000000000..38c52c9eed6 --- /dev/null +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/PageDataTest.java @@ -0,0 +1,43 @@ +package org.wikimedia.metrics_platform.context; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; +import org.wikimedia.metrics_platform.json.GsonHelper; + +import com.google.gson.Gson; + +class PageDataTest { + + @Test void testPageData() { + PageData pageData = PageData.builder() + .id(1) + .title("Test") + .namespaceId(0) + .namespaceName("") + .revisionId(1L) + .wikidataItemQid("Q1") + .contentLanguage("zh") + .build(); + + assertThat(pageData.getId()).isEqualTo(1); + assertThat(pageData.getNamespaceId()).isEqualTo(0); + assertThat(pageData.getNamespaceName()).isEmpty(); + assertThat(pageData.getTitle()).isEqualTo("Test"); + assertThat(pageData.getRevisionId()).isEqualTo(1); + assertThat(pageData.getWikidataItemQid()).isEqualTo("Q1"); + assertThat(pageData.getContentLanguage()).isEqualTo("zh"); + + Gson gson = GsonHelper.getGson(); + String json = gson.toJson(pageData); + assertThat(json).isEqualTo("{\"id\":1," + + "\"title\":\"Test\"," + + "\"namespace_id\":0," + + "\"namespace_name\":\"\"," + + "\"revision_id\":1," + + "\"wikidata_qid\":\"Q1\"," + + "\"content_language\":\"zh\"" + + "}"); + } + +} diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/PerformerDataTest.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/PerformerDataTest.java new file mode 100644 index 00000000000..5d7e9cbd6e4 --- /dev/null +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/PerformerDataTest.java @@ -0,0 +1,55 @@ +package org.wikimedia.metrics_platform.context; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.time.Instant; +import java.util.Collections; + +import org.junit.jupiter.api.Test; +import org.wikimedia.metrics_platform.json.GsonHelper; + +import com.google.gson.Gson; + +class PerformerDataTest { + + @Test void testPerformerData() { + PerformerData performerData = PerformerData.builder() + .id(1) + .name("TestPerformer") + .isLoggedIn(true) + .isTemp(false) + .sessionId("eeeeeeeeeeeeeeeeeeee") + .pageviewId("eeeeeeeeeeeeeeeeeeee") + .groups(Collections.singletonList("*")) + .languageGroups("zh, en") + .languagePrimary("zh-tw") + .registrationDt(Instant.parse("2023-03-01T01:08:30Z")) + .build(); + + assertThat(performerData.getId()).isEqualTo(1); + assertThat(performerData.getName()).isEqualTo("TestPerformer"); + assertThat(performerData.getIsLoggedIn()).isTrue(); + assertThat(performerData.getIsTemp()).isFalse(); + assertThat(performerData.getGroups()).isEqualTo(Collections.singletonList("*")); + assertThat(performerData.getLanguageGroups()).isEqualTo("zh, en"); + assertThat(performerData.getLanguagePrimary()).isEqualTo("zh-tw"); + assertThat(performerData.getRegistrationDt()).isEqualTo("2023-03-01T01:08:30Z"); + + Gson gson = GsonHelper.getGson(); + + String json = gson.toJson(performerData); + assertThat(json).isEqualTo("{" + + "\"id\":1," + + "\"name\":\"TestPerformer\"," + + "\"is_logged_in\":true," + + "\"is_temp\":false," + + "\"session_id\":\"eeeeeeeeeeeeeeeeeeee\"," + + "\"pageview_id\":\"eeeeeeeeeeeeeeeeeeee\"," + + "\"groups\":[\"*\"]," + + "\"language_groups\":\"zh, en\"," + + "\"language_primary\":\"zh-tw\"," + + "\"registration_dt\":\"2023-03-01T01:08:30Z\"" + + "}"); + } + +} diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/curation/CurationFilterFixtures.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/curation/CurationFilterFixtures.java new file mode 100644 index 00000000000..c2820710bc0 --- /dev/null +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/curation/CurationFilterFixtures.java @@ -0,0 +1,21 @@ +package org.wikimedia.metrics_platform.curation; + +import org.wikimedia.metrics_platform.json.GsonHelper; +import org.wikimedia.metrics_platform.config.CurationFilter; + +import com.google.gson.Gson; + +public final class CurationFilterFixtures { + private CurationFilterFixtures() { + // Utility class, should never be instantiated + } + + public static CurationFilter curationFilter() { + Gson gson = GsonHelper.getGson(); + String curationFilterJson = "{\"page_id\":{\"less_than\":500,\"not_equals\":42},\"page_namespace_text\":" + + "{\"equals\":\"Talk\"},\"user_is_logged_in\":{\"equals\":true},\"user_edit_count_bucket\":" + + "{\"in\":[\"100-999 edits\",\"1000+ edits\"]},\"user_groups\":{\"contains_all\":" + + "[\"user\",\"autoconfirmed\"],\"does_not_contain\":\"sysop\"}}"; + return gson.fromJson(curationFilterJson, CurationFilter.class); + } +} diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/event/EventFixtures.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/event/EventFixtures.java new file mode 100644 index 00000000000..b9163e667af --- /dev/null +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/event/EventFixtures.java @@ -0,0 +1,65 @@ +package org.wikimedia.metrics_platform.event; + +import static org.wikimedia.metrics_platform.context.DataFixtures.getTestClientData; +import static org.wikimedia.metrics_platform.event.EventProcessed.fromEvent; + +import java.util.Arrays; +import java.util.List; + +import org.wikimedia.metrics_platform.context.ClientData; +import org.wikimedia.metrics_platform.context.PageData; +import org.wikimedia.metrics_platform.context.PerformerData; + +public final class EventFixtures { + + private EventFixtures() { + // Utility class, should never be instantiated + } + public static Event minimalEvent() { + return new Event("test_schema", "test_stream", "test_event"); + } + + public static EventProcessed minimalEventProcessed() { + EventProcessed eventProcessed = fromEvent(minimalEvent()); + eventProcessed.setClientData(getTestClientData()); + return eventProcessed; + } + + public static EventProcessed getEvent() { + Event event = new Event("test/event", "test.event", "testEvent"); + ClientData clientData = new ClientData(); + clientData.setPageData(PageData.builder().id(1).namespaceName("Talk").build()); + + event.setClientData(clientData); + EventProcessed eventProcessed = fromEvent(event); + eventProcessed.setPerformerData( + PerformerData.builder() + .groups(Arrays.asList("user", "autoconfirmed", "steward")) + .isLoggedIn(true) + .build() + ); + return eventProcessed; + } + + public static EventProcessed getEvent( + Integer id, + String namespaceName, + List groups, + boolean isLoggedIn, + String editCount + ) { + Event event = new Event("test/event", "test.event", "testEvent"); + ClientData clientData = new ClientData(); + clientData.setPageData(PageData.builder().id(id).namespaceName(namespaceName).build()); + + event.setClientData(clientData); + EventProcessed eventProcessed = fromEvent(event); + eventProcessed.setPerformerData( + PerformerData.builder() + .groups(groups) + .isLoggedIn(isLoggedIn) + .build() + ); + return eventProcessed; + } +} diff --git a/analytics/metrics-platform/src/test/resources/org/wikimedia/metrics_platform/config/streamconfigs-local.json b/analytics/metrics-platform/src/test/resources/org/wikimedia/metrics_platform/config/streamconfigs-local.json new file mode 100644 index 00000000000..a721750f589 --- /dev/null +++ b/analytics/metrics-platform/src/test/resources/org/wikimedia/metrics_platform/config/streamconfigs-local.json @@ -0,0 +1,2288 @@ +{ + "streams": { + "eventlogging_CentralNoticeBannerHistory": { + "schema_title": "analytics/legacy/centralnoticebannerhistory", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_CentralNoticeBannerHistory", + "topics": [ + "eventlogging_CentralNoticeBannerHistory" + ] + }, + "eventlogging_CentralNoticeImpression": { + "schema_title": "analytics/legacy/centralnoticeimpression", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_CentralNoticeImpression", + "topics": [ + "eventlogging_CentralNoticeImpression" + ] + }, + "eventlogging_CentralNoticeTiming": { + "schema_title": "analytics/legacy/centralnoticetiming", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_CentralNoticeTiming", + "topics": [ + "eventlogging_CentralNoticeTiming" + ] + }, + "eventlogging_CodeMirrorUsage": { + "schema_title": "analytics/legacy/codemirrorusage", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_CodeMirrorUsage", + "topics": [ + "eventlogging_CodeMirrorUsage" + ] + }, + "eventlogging_ContentTranslationAbuseFilter": { + "schema_title": "analytics/legacy/contenttranslationabusefilter", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_ContentTranslationAbuseFilter", + "topics": [ + "eventlogging_ContentTranslationAbuseFilter" + ] + }, + "eventlogging_CpuBenchmark": { + "schema_title": "analytics/legacy/cpubenchmark", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_CpuBenchmark", + "topics": [ + "eventlogging_CpuBenchmark" + ] + }, + "eventlogging_DesktopWebUIActionsTracking": { + "schema_title": "analytics/legacy/desktopwebuiactionstracking", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_DesktopWebUIActionsTracking", + "topics": [ + "eventlogging_DesktopWebUIActionsTracking" + ] + }, + "eventlogging_ElementTiming": { + "schema_title": "analytics/legacy/elementtiming", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_ElementTiming", + "topics": [ + "eventlogging_ElementTiming" + ] + }, + "eventlogging_EditAttemptStep": { + "schema_title": "analytics/legacy/editattemptstep", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_EditAttemptStep", + "topics": [ + "eventlogging_EditAttemptStep" + ] + }, + "eventlogging_FeaturePolicyViolation": { + "schema_title": "analytics/legacy/featurepolicyviolation", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_FeaturePolicyViolation", + "topics": [ + "eventlogging_FeaturePolicyViolation" + ] + }, + "eventlogging_FirstInputTiming": { + "schema_title": "analytics/legacy/firstinputtiming", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_FirstInputTiming", + "topics": [ + "eventlogging_FirstInputTiming" + ] + }, + "eventlogging_HelpPanel": { + "schema_title": "analytics/legacy/helppanel", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_HelpPanel", + "topics": [ + "eventlogging_HelpPanel" + ] + }, + "eventlogging_HomepageModule": { + "schema_title": "analytics/legacy/homepagemodule", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_HomepageModule", + "topics": [ + "eventlogging_HomepageModule" + ] + }, + "eventlogging_HomepageVisit": { + "schema_title": "analytics/legacy/homepagevisit", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_HomepageVisit", + "topics": [ + "eventlogging_HomepageVisit" + ] + }, + "eventlogging_InukaPageView": { + "schema_title": "analytics/legacy/inukapageview", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_InukaPageView", + "topics": [ + "eventlogging_InukaPageView" + ] + }, + "eventlogging_KaiOSAppFirstRun": { + "schema_title": "analytics/legacy/kaiosappfirstrun", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_KaiOSAppFirstRun", + "topics": [ + "eventlogging_KaiOSAppFirstRun" + ] + }, + "eventlogging_KaiOSAppFeedback": { + "schema_title": "analytics/legacy/kaiosappfeedback", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_KaiOSAppFeedback", + "topics": [ + "eventlogging_KaiOSAppFeedback" + ] + }, + "eventlogging_LandingPageImpression": { + "schema_title": "analytics/legacy/landingpageimpression", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_LandingPageImpression", + "topics": [ + "eventlogging_LandingPageImpression" + ] + }, + "eventlogging_LayoutShift": { + "schema_title": "analytics/legacy/layoutshift", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_LayoutShift", + "topics": [ + "eventlogging_LayoutShift" + ] + }, + "eventlogging_MobileWebUIActionsTracking": { + "schema_title": "analytics/legacy/mobilewebuiactionstracking", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_MobileWebUIActionsTracking", + "topics": [ + "eventlogging_MobileWebUIActionsTracking" + ] + }, + "eventlogging_NavigationTiming": { + "schema_title": "analytics/legacy/navigationtiming", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_NavigationTiming", + "topics": [ + "eventlogging_NavigationTiming" + ] + }, + "eventlogging_NewcomerTask": { + "schema_title": "analytics/legacy/newcomertask", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_NewcomerTask", + "topics": [ + "eventlogging_NewcomerTask" + ] + }, + "eventlogging_PaintTiming": { + "schema_title": "analytics/legacy/painttiming", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_PaintTiming", + "topics": [ + "eventlogging_PaintTiming" + ] + }, + "eventlogging_PrefUpdate": { + "schema_title": "analytics/legacy/prefupdate", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_PrefUpdate", + "topics": [ + "eventlogging_PrefUpdate" + ] + }, + "eventlogging_WikipediaPortal": { + "schema_title": "analytics/legacy/wikipediaportal", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_WikipediaPortal", + "topics": [ + "eventlogging_WikipediaPortal" + ] + }, + "eventlogging_QuickSurveyInitiation": { + "schema_title": "analytics/legacy/quicksurveyinitiation", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_QuickSurveyInitiation", + "topics": [ + "eventlogging_QuickSurveyInitiation" + ] + }, + "eventlogging_QuickSurveysResponses": { + "schema_title": "analytics/legacy/quicksurveysresponses", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_QuickSurveysResponses", + "topics": [ + "eventlogging_QuickSurveysResponses" + ] + }, + "eventlogging_ReferencePreviewsBaseline": { + "schema_title": "analytics/legacy/referencepreviewsbaseline", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_ReferencePreviewsBaseline", + "topics": [ + "eventlogging_ReferencePreviewsBaseline" + ] + }, + "eventlogging_ReferencePreviewsCite": { + "schema_title": "analytics/legacy/referencepreviewscite", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_ReferencePreviewsCite", + "topics": [ + "eventlogging_ReferencePreviewsCite" + ] + }, + "eventlogging_ReferencePreviewsPopups": { + "schema_title": "analytics/legacy/referencepreviewspopups", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_ReferencePreviewsPopups", + "topics": [ + "eventlogging_ReferencePreviewsPopups" + ] + }, + "eventlogging_ResourceTiming": { + "schema_title": "analytics/legacy/resourcetiming", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_ResourceTiming", + "topics": [ + "eventlogging_ResourceTiming" + ] + }, + "eventlogging_RUMSpeedIndex": { + "schema_title": "analytics/legacy/rumspeedindex", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_RUMSpeedIndex", + "topics": [ + "eventlogging_RUMSpeedIndex" + ] + }, + "eventlogging_SaveTiming": { + "schema_title": "analytics/legacy/savetiming", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_SaveTiming", + "topics": [ + "eventlogging_SaveTiming" + ] + }, + "eventlogging_SearchSatisfaction": { + "schema_title": "analytics/legacy/searchsatisfaction", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_SearchSatisfaction", + "topics": [ + "eventlogging_SearchSatisfaction" + ] + }, + "eventlogging_ServerSideAccountCreation": { + "schema_title": "analytics/legacy/serversideaccountcreation", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_ServerSideAccountCreation", + "topics": [ + "eventlogging_ServerSideAccountCreation" + ] + }, + "eventlogging_SpecialInvestigate": { + "schema_title": "analytics/legacy/specialinvestigate", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_SpecialInvestigate", + "topics": [ + "eventlogging_SpecialInvestigate" + ] + }, + "eventlogging_SpecialMuteSubmit": { + "schema_title": "analytics/legacy/specialmutesubmit", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_SpecialMuteSubmit", + "topics": [ + "eventlogging_SpecialMuteSubmit" + ] + }, + "eventlogging_SuggestedTagsAction": { + "schema_title": "analytics/legacy/suggestedtagsaction", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_SuggestedTagsAction", + "topics": [ + "eventlogging_SuggestedTagsAction" + ] + }, + "eventlogging_TemplateDataApi": { + "schema_title": "analytics/legacy/templatedataapi", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_TemplateDataApi", + "topics": [ + "eventlogging_TemplateDataApi" + ] + }, + "eventlogging_TemplateDataEditor": { + "schema_title": "analytics/legacy/templatedataeditor", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_TemplateDataEditor", + "topics": [ + "eventlogging_TemplateDataEditor" + ] + }, + "eventlogging_TemplateWizard": { + "schema_title": "analytics/legacy/templatewizard", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_TemplateWizard", + "topics": [ + "eventlogging_TemplateWizard" + ] + }, + "eventlogging_Test": { + "schema_title": "analytics/legacy/test", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_Test", + "topics": [ + "eventlogging_Test" + ] + }, + "eventlogging_TranslationRecommendationUserAction": { + "schema_title": "analytics/legacy/translationrecommendationuseraction", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_TranslationRecommendationUserAction", + "topics": [ + "eventlogging_TranslationRecommendationUserAction" + ] + }, + "eventlogging_TranslationRecommendationUIRequests": { + "schema_title": "analytics/legacy/translationrecommendationuirequests", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_TranslationRecommendationUIRequests", + "topics": [ + "eventlogging_TranslationRecommendationUIRequests" + ] + }, + "eventlogging_TranslationRecommendationAPIRequests": { + "schema_title": "analytics/legacy/translationrecommendationapirequests", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_TranslationRecommendationAPIRequests", + "topics": [ + "eventlogging_TranslationRecommendationAPIRequests" + ] + }, + "eventlogging_TwoColConflictConflict": { + "schema_title": "analytics/legacy/twocolconflictconflict", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_TwoColConflictConflict", + "topics": [ + "eventlogging_TwoColConflictConflict" + ] + }, + "eventlogging_TwoColConflictExit": { + "schema_title": "analytics/legacy/twocolconflictexit", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_TwoColConflictExit", + "topics": [ + "eventlogging_TwoColConflictExit" + ] + }, + "eventlogging_UniversalLanguageSelector": { + "schema_title": "analytics/legacy/universallanguageselector", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_UniversalLanguageSelector", + "topics": [ + "eventlogging_UniversalLanguageSelector" + ] + }, + "eventlogging_VirtualPageView": { + "schema_title": "analytics/legacy/virtualpageview", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_VirtualPageView", + "topics": [ + "eventlogging_VirtualPageView" + ] + }, + "eventlogging_VisualEditorFeatureUse": { + "schema_title": "analytics/legacy/visualeditorfeatureuse", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_VisualEditorFeatureUse", + "topics": [ + "eventlogging_VisualEditorFeatureUse" + ] + }, + "eventlogging_VisualEditorTemplateDialogUse": { + "schema_title": "analytics/legacy/visualeditortemplatedialoguse", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_VisualEditorTemplateDialogUse", + "topics": [ + "eventlogging_VisualEditorTemplateDialogUse" + ] + }, + "eventlogging_WikibaseTermboxInteraction": { + "schema_title": "analytics/legacy/wikibasetermboxinteraction", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_WikibaseTermboxInteraction", + "topics": [ + "eventlogging_WikibaseTermboxInteraction" + ] + }, + "eventlogging_WikidataCompletionSearchClicks": { + "schema_title": "analytics/legacy/wikidatacompletionsearchclicks", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_WikidataCompletionSearchClicks", + "topics": [ + "eventlogging_WikidataCompletionSearchClicks" + ] + }, + "eventlogging_WMDEBannerEvents": { + "schema_title": "analytics/legacy/wmdebannerevents", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_WMDEBannerEvents", + "topics": [ + "eventlogging_WMDEBannerEvents" + ] + }, + "eventlogging_WMDEBannerInteractions": { + "schema_title": "analytics/legacy/wmdebannerinteractions", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_WMDEBannerInteractions", + "topics": [ + "eventlogging_WMDEBannerInteractions" + ] + }, + "eventlogging_WMDEBannerSizeIssue": { + "schema_title": "analytics/legacy/wmdebannersizeissue", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_WMDEBannerSizeIssue", + "topics": [ + "eventlogging_WMDEBannerSizeIssue" + ] + }, + "test.instrumentation": { + "schema_title": "analytics/test", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "test.instrumentation", + "topics": [ + "eqiad.test.instrumentation", + "codfw.test.instrumentation" + ] + }, + "test.instrumentation.sampled": { + "schema_title": "analytics/test", + "destination_event_service": "eventgate-analytics-external", + "sample": { + "rate": 0.5, + "unit": "session" + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "test.instrumentation.sampled", + "topics": [ + "eqiad.test.instrumentation.sampled", + "codfw.test.instrumentation.sampled" + ] + }, + "mediawiki.client.session_tick": { + "schema_title": "analytics/session_tick", + "destination_event_service": "eventgate-analytics-external", + "sample": { + "unit": "session", + "rate": 0.1 + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.client.session_tick", + "topics": [ + "eqiad.mediawiki.client.session_tick", + "codfw.mediawiki.client.session_tick" + ] + }, + "ios.edit_history_compare": { + "schema_title": "analytics/mobile_apps/ios_edit_history_compare", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "ios.edit_history_compare", + "topics": [ + "eqiad.ios.edit_history_compare", + "codfw.ios.edit_history_compare" + ] + }, + "ios.notification_interaction": { + "schema_title": "analytics/mobile_apps/ios_notification_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "ios.notification_interaction", + "topics": [ + "eqiad.ios.notification_interaction", + "codfw.ios.notification_interaction" + ] + }, + "android.user_contribution_screen": { + "schema_title": "analytics/mobile_apps/android_user_contribution_screen", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.user_contribution_screen", + "topics": [ + "eqiad.android.user_contribution_screen", + "codfw.android.user_contribution_screen" + ] + }, + "android.notification_interaction": { + "schema_title": "analytics/mobile_apps/android_notification_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.notification_interaction", + "topics": [ + "eqiad.android.notification_interaction", + "codfw.android.notification_interaction" + ] + }, + "android.image_recommendation_interaction": { + "schema_title": "analytics/mobile_apps/android_image_recommendation_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.image_recommendation_interaction", + "topics": [ + "eqiad.android.image_recommendation_interaction", + "codfw.android.image_recommendation_interaction" + ] + }, + "android.daily_stats": { + "schema_title": "analytics/mobile_apps/android_daily_stats", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.daily_stats", + "topics": [ + "eqiad.android.daily_stats", + "codfw.android.daily_stats" + ] + }, + "android.customize_toolbar_interaction": { + "schema_title": "analytics/mobile_apps/android_customize_toolbar_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.customize_toolbar_interaction", + "topics": [ + "eqiad.android.customize_toolbar_interaction", + "codfw.android.customize_toolbar_interaction" + ] + }, + "android.article_toolbar_interaction": { + "schema_title": "analytics/mobile_apps/android_article_toolbar_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.article_toolbar_interaction", + "topics": [ + "eqiad.android.article_toolbar_interaction", + "codfw.android.article_toolbar_interaction" + ] + }, + "android.edit_history_interaction": { + "schema_title": "analytics/mobile_apps/android_edit_history_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.edit_history_interaction", + "topics": [ + "eqiad.android.edit_history_interaction", + "codfw.android.edit_history_interaction" + ] + }, + "android.breadcrumbs_event": { + "schema_title": "analytics/mobile_apps/android_breadcrumbs_event", + "destination_event_service": "eventgate-analytics-external", + "sample": { + "unit": "device", + "rate": 0.5 + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.breadcrumbs_event", + "topics": [ + "eqiad.android.breadcrumbs_event", + "codfw.android.breadcrumbs_event" + ] + }, + "android.app_appearance_settings_interaction": { + "schema_title": "analytics/mobile_apps/android_app_appearance_settings_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.app_appearance_settings_interaction", + "topics": [ + "eqiad.android.app_appearance_settings_interaction", + "codfw.android.app_appearance_settings_interaction" + ] + }, + "android.article_link_preview_interaction": { + "schema_title": "analytics/mobile_apps/android_article_link_preview_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.article_link_preview_interaction", + "topics": [ + "eqiad.android.article_link_preview_interaction", + "codfw.android.article_link_preview_interaction" + ] + }, + "android.article_page_scroll_interaction": { + "schema_title": "analytics/mobile_apps/android_article_page_scroll_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.article_page_scroll_interaction", + "topics": [ + "eqiad.android.article_page_scroll_interaction", + "codfw.android.article_page_scroll_interaction" + ] + }, + "android.article_toc_interaction": { + "schema_title": "analytics/mobile_apps/android_article_toc_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.article_toc_interaction", + "topics": [ + "eqiad.android.article_toc_interaction", + "codfw.android.article_toc_interaction" + ] + }, + "android.find_in_page_interaction": { + "schema_title": "analytics/mobile_apps/android_find_in_page_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.find_in_page_interaction", + "topics": [ + "eqiad.android.find_in_page_interaction", + "codfw.android.find_in_page_interaction" + ] + }, + "android.product_metrics.article_link_preview_interaction": { + "schema_title": "analytics/product_metrics/app/base", + "destination_event_service": "eventgate-analytics-external", + "producers": { + "metrics_platform_client": { + "events": [ + "android.metrics_platform.article_link_preview_interaction" + ], + "provide_values": [ + "mediawiki_database", + "page_title", + "page_content_language", + "page_id", + "page_namespace_id", + "performer_is_logged_in", + "performer_session_id", + "performer_pageview_id", + "performer_language_groups", + "performer_language_primary", + "performer_groups", + ] + } + }, + "sample": { + "unit": "device", + "rate": 1 + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.product_metrics.article_link_preview_interaction", + "topics": [ + "eqiad.android.product_metrics.article_link_preview_interaction", + "codfw.android.product_metrics.article_link_preview_interaction" + ] + }, + "android.product_metrics.article_toc_interaction": { + "schema_title": "analytics/mobile_apps/product_metrics/android_article_toc_interaction", + "destination_event_service": "eventgate-analytics-external", + "producers": { + "metrics_platform_client": { + "events": [ + "android.metrics_platform.article_toc_interaction" + ], + "provide_values": [ + "mediawiki_database", + "page_title", + "page_content_language", + "page_id", + "page_namespace_id", + "performer_is_logged_in", + "performer_session_id", + "performer_pageview_id", + "performer_language_groups", + "performer_language_primary", + "performer_groups", + ] + } + }, + "sample": { + "unit": "device", + "rate": 1 + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.product_metrics.article_toc_interaction", + "topics": [ + "eqiad.android.product_metrics.article_toc_interaction", + "codfw.android.product_metrics.article_toc_interaction" + ] + }, + "android.product_metrics.article_toolbar_interaction": { + "schema_title": "analytics/product_metrics/app/base", + "destination_event_service": "eventgate-analytics-external", + "producers": { + "metrics_platform_client": { + "events": [ + "android.metrics_platform.article_toolbar_interaction" + ], + "provide_values": [ + "mediawiki_database", + "page_title", + "page_content_language", + "page_id", + "page_namespace_id", + "performer_is_logged_in", + "performer_session_id", + "performer_pageview_id", + "performer_language_groups", + "performer_language_primary", + "performer_groups", + ] + } + }, + "sample": { + "unit": "device", + "rate": 1 + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.product_metrics.article_toolbar_interaction", + "topics": [ + "eqiad.android.product_metrics.article_toolbar_interaction", + "codfw.android.product_metrics.article_toolbar_interaction" + ] + }, + "android.product_metrics.find_in_page_interaction": { + "schema_title": "analytics/mobile_apps/product_metrics/android_find_in_page_interaction", + "destination_event_service": "eventgate-analytics-external", + "producers": { + "metrics_platform_client": { + "events": [ + "android.metrics_platform.find_in_page_interaction" + ], + "provide_values": [ + "mediawiki_database", + "page_title", + "page_content_language", + "page_id", + "page_namespace_id", + "performer_is_logged_in", + "performer_session_id", + "performer_pageview_id", + "performer_language_groups", + "performer_language_primary", + "performer_groups", + ] + } + }, + "sample": { + "unit": "device", + "rate": 1 + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.product_metrics.find_in_page_interaction", + "topics": [ + "eqiad.android.product_metrics.find_in_page_interaction", + "codfw.android.product_metrics.find_in_page_interaction" + ] + }, + "mediawiki.mediasearch_interaction": { + "schema_title": "analytics/mediawiki/mediasearch_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.mediasearch_interaction", + "topics": [ + "eqiad.mediawiki.mediasearch_interaction", + "codfw.mediawiki.mediasearch_interaction" + ] + }, + "mediawiki.structured_task.article.link_suggestion_interaction": { + "schema_title": "analytics/mediawiki/structured_task/article/link_suggestion_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.structured_task.article.link_suggestion_interaction", + "topics": [ + "eqiad.mediawiki.structured_task.article.link_suggestion_interaction", + "codfw.mediawiki.structured_task.article.link_suggestion_interaction" + ] + }, + "mediawiki.structured_task.article.image_suggestion_interaction": { + "schema_title": "analytics/mediawiki/structured_task/article/image_suggestion_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.structured_task.article.image_suggestion_interaction", + "topics": [ + "eqiad.mediawiki.structured_task.article.image_suggestion_interaction", + "codfw.mediawiki.structured_task.article.image_suggestion_interaction" + ] + }, + "mediawiki.pref_diff": { + "schema_title": "analytics/pref_diff", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.pref_diff", + "topics": [ + "eqiad.mediawiki.pref_diff", + "codfw.mediawiki.pref_diff" + ] + }, + "mediawiki.skin_diff": { + "schema_title": "analytics/pref_diff", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.skin_diff", + "topics": [ + "eqiad.mediawiki.skin_diff", + "codfw.mediawiki.skin_diff" + ] + }, + "mediawiki.content_translation_event": { + "schema_title": "analytics/mediawiki/content_translation_event", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.content_translation_event", + "topics": [ + "eqiad.mediawiki.content_translation_event", + "codfw.mediawiki.content_translation_event" + ] + }, + "mediawiki.reading_depth": { + "schema_title": "analytics/mediawiki/web_ui_reading_depth", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.reading_depth", + "topics": [ + "eqiad.mediawiki.reading_depth", + "codfw.mediawiki.reading_depth" + ] + }, + "mediawiki.web_ab_test_enrollment": { + "schema_title": "analytics/mediawiki/web_ab_test_enrollment", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.web_ab_test_enrollment", + "topics": [ + "eqiad.mediawiki.web_ab_test_enrollment", + "codfw.mediawiki.web_ab_test_enrollment" + ] + }, + "mediawiki.web_ui_scroll": { + "schema_title": "analytics/mediawiki/web_ui_scroll", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.web_ui_scroll", + "topics": [ + "eqiad.mediawiki.web_ui_scroll", + "codfw.mediawiki.web_ui_scroll" + ] + }, + "mediawiki.ipinfo_interaction": { + "schema_title": "analytics/mediawiki/ipinfo_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.ipinfo_interaction", + "topics": [ + "eqiad.mediawiki.ipinfo_interaction", + "codfw.mediawiki.ipinfo_interaction" + ] + }, + "wd_propertysuggester.client_side_property_request": { + "schema_title": "analytics/mediawiki/wd_propertysuggester/client_side_property_request", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "wd_propertysuggester.client_side_property_request", + "topics": [ + "eqiad.wd_propertysuggester.client_side_property_request", + "codfw.wd_propertysuggester.client_side_property_request" + ] + }, + "wd_propertysuggester.server_side_property_request": { + "schema_title": "analytics/mediawiki/wd_propertysuggester/server_side_property_request", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "wd_propertysuggester.server_side_property_request", + "topics": [ + "eqiad.wd_propertysuggester.server_side_property_request", + "codfw.wd_propertysuggester.server_side_property_request" + ] + }, + "mediawiki.mentor_dashboard.visit": { + "schema_title": "analytics/mediawiki/mentor_dashboard/visit", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.mentor_dashboard.visit", + "topics": [ + "eqiad.mediawiki.mentor_dashboard.visit", + "codfw.mediawiki.mentor_dashboard.visit" + ] + }, + "mediawiki.welcomesurvey.interaction": { + "schema_title": "analytics/mediawiki/welcomesurvey/interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.welcomesurvey.interaction", + "topics": [ + "eqiad.mediawiki.welcomesurvey.interaction", + "codfw.mediawiki.welcomesurvey.interaction" + ] + }, + "mediawiki.editgrowthconfig": { + "schema_title": "analytics/mediawiki/editgrowthconfig", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.editgrowthconfig", + "topics": [ + "eqiad.mediawiki.editgrowthconfig", + "codfw.mediawiki.editgrowthconfig" + ] + }, + "mediawiki.accountcreation_block": { + "schema_title": "analytics/mediawiki/accountcreation/block", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.accountcreation_block", + "topics": [ + "eqiad.mediawiki.accountcreation_block", + "codfw.mediawiki.accountcreation_block" + ] + }, + "mediawiki.editattempt_block": { + "schema_title": "analytics/mediawiki/editattemptsblocked", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.editattempt_block", + "topics": [ + "eqiad.mediawiki.editattempt_block", + "codfw.mediawiki.editattempt_block" + ] + }, + "mediawiki.talk_page_edit": { + "stream": "mediawiki.talk_page_edit", + "schema_title": "analytics/mediawiki/talk_page_edit", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "topics": [ + "eqiad.mediawiki.talk_page_edit", + "codfw.mediawiki.talk_page_edit" + ] + }, + "mwcli.command_execute": { + "schema_title": "analytics/mwcli/command_execute", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mwcli.command_execute", + "topics": [ + "eqiad.mwcli.command_execute", + "codfw.mwcli.command_execute" + ] + }, + "mediawiki.web_ui.interactions": { + "schema_title": "analytics/mediawiki/client/metrics_event", + "destination_event_service": "eventgate-analytics-external", + "producers": { + "metrics_platform_client": { + "events": [ + "web.ui.", + "web_ui." + ], + "provide_values": [ + "page_namespace", + "performer_is_logged_in", + "performer_session_id", + "performer_pageview_id", + "performer_edit_count_bucket", + "mediawiki_skin", + "mediawiki_database" + ], + "curation": { + "mediawiki_skin": { + "in": [ + "minerva", + "vector", + "vector-2022" + ] + } + } + } + }, + "sample": { + "unit": "pageview", + "rate": 0 + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.web_ui.interactions", + "topics": [ + "eqiad.mediawiki.web_ui.interactions", + "codfw.mediawiki.web_ui.interactions" + ] + }, + "mediawiki.edit_attempt": { + "schema_title": "analytics/mediawiki/client/metrics_event", + "destination_event_service": "eventgate-logging-local", + "producers": { + "metrics_platform_client": { + "events": [ + "eas." + ], + "provide_values": [ + "agent_app_install_id", + "agent_client_platform_family", + "page_id", + "page_title", + "page_namespace_id", + "page_revision_id", + "mediawiki_database", + "performer_is_logged_in", + "performer_id", + "performer_session_id", + "performer_pageview_id" + ] + } + }, + "sample": { + "unit": "pageview", + "rate": 1 + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.edit_attempt", + "topics": [ + "eqiad.mediawiki.edit_attempt", + "codfw.mediawiki.edit_attempt" + ] + }, + "mediawiki.metrics_platform.click": { + "schema_title": "analytics/product_metrics/app/base", + "destination_event_service": "eventgate-logging-local", + "producers": { + "metrics_platform_client": { + "events": [ + "click", + "click." + ], + "provide_values": [ + "agent_app_install_id", + "agent_client_platform_family", + "page_id", + "page_title", + "page_namespace_id", + "page_revision_id", + "mediawiki_database", + "performer_is_logged_in", + "performer_id", + "performer_session_id", + "performer_pageview_id" + ] + } + }, + "sample": { + "unit": "pageview", + "rate": 1 + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.metrics_platform.click", + "topics": [ + "eqiad.mediawiki.metrics_platform.click", + "codfw.mediawiki.metrics_platform.click" + ] + }, + "mediawiki.metrics_platform.click_custom": { + "schema_title": "analytics/product_metrics/app/click_custom", + "destination_event_service": "eventgate-logging-local", + "producers": { + "metrics_platform_client": { + "events": [ + "click", + "click." + ], + "provide_values": [ + "agent_app_install_id", + "agent_client_platform_family", + "page_id", + "page_title", + "page_namespace_id", + "page_revision_id", + "mediawiki_database", + "performer_is_logged_in", + "performer_id", + "performer_session_id", + "performer_pageview_id" + ] + } + }, + "sample": { + "unit": "pageview", + "rate": 1 + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.metrics_platform.click_custom", + "topics": [ + "eqiad.mediawiki.metrics_platform.click_custom", + "codfw.mediawiki.metrics_platform.click_custom" + ] + }, + "mediawiki.metrics_platform.interaction": { + "schema_title": "analytics/product_metrics/app/base", + "destination_event_service": "eventgate-logging-local", + "producers": { + "metrics_platform_client": { + "events": [ + "interaction", + "interaction." + ], + "provide_values": [ + "agent_app_install_id", + "agent_client_platform_family", + "page_id", + "page_title", + "page_namespace_id", + "page_revision_id", + "mediawiki_database", + "performer_is_logged_in", + "performer_id", + "performer_session_id", + "performer_pageview_id" + ] + } + }, + "sample": { + "unit": "pageview", + "rate": 1 + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.metrics_platform.interaction", + "topics": [ + "eqiad.mediawiki.metrics_platform.interaction", + "codfw.mediawiki.metrics_platform.interaction" + ] + }, + "mediawiki.metrics_platform.view": { + "schema_title": "analytics/product_metrics/app/base", + "destination_event_service": "eventgate-logging-local", + "producers": { + "metrics_platform_client": { + "events": [ + "view", + "view." + ], + "provide_values": [ + "agent_app_install_id", + "agent_client_platform_family", + "page_id", + "page_title", + "page_namespace_id", + "page_revision_id", + "mediawiki_database", + "performer_is_logged_in", + "performer_id", + "performer_session_id", + "performer_pageview_id" + ] + } + }, + "sample": { + "unit": "pageview", + "rate": 1 + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.metrics_platform.view", + "topics": [ + "eqiad.mediawiki.metrics_platform.view", + "codfw.mediawiki.metrics_platform.view" + ] + }, + "mediawiki.visual_editor_feature_use": { + "schema_title": "analytics/mediawiki/client/metrics_event", + "destination_event_service": "eventgate-analytics-external", + "producers": { + "metrics_platform_client": { + "events": [ + "vefu." + ], + "provide_values": [ + "agent_client_platform_family", + "mediawiki_database", + "performer_id", + "performer_edit_count" + ] + } + }, + "sample": { + "unit": "pageview", + "rate": 0 + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.visual_editor_feature_use", + "topics": [ + "eqiad.mediawiki.visual_editor_feature_use", + "codfw.mediawiki.visual_editor_feature_use" + ] + }, + "mediawiki.wikistories_consumption_event": { + "schema_title": "analytics/mediawiki/wikistories_consumption_event", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.wikistories_consumption_event", + "topics": [ + "eqiad.mediawiki.wikistories_consumption_event", + "codfw.mediawiki.wikistories_consumption_event" + ] + }, + "mediawiki.wikistories_contribution_event": { + "schema_title": "analytics/mediawiki/wikistories_contribution_event", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.wikistories_contribution_event", + "topics": [ + "eqiad.mediawiki.wikistories_contribution_event", + "codfw.mediawiki.wikistories_contribution_event" + ] + }, + "rc0.mediawiki.page_change": { + "schema_title": "development/mediawiki/page/change", + "message_key_fields": { + "wiki_id": "wiki_id", + "page_id": "page.page_id" + }, + "destination_event_service": "eventgate-analytics-external", + "producers": { + "mediawiki_eventbus": { + "enabled": false + } + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "rc0.mediawiki.page_change", + "topics": [ + "eqiad.rc0.mediawiki.page_change", + "codfw.rc0.mediawiki.page_change" + ] + }, + "rc0.mediawiki.page_content_change": { + "schema_title": "development/mediawiki/page/change", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "rc0.mediawiki.page_content_change", + "topics": [ + "eqiad.rc0.mediawiki.page_content_change", + "codfw.rc0.mediawiki.page_content_change" + ] + }, + "mediawiki.maps_interaction": { + "schema_title": "analytics/mediawiki/maps/interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.maps_interaction", + "topics": [ + "eqiad.mediawiki.maps_interaction", + "codfw.mediawiki.maps_interaction" + ] + }, + "eventgate-analytics-external.test.event": { + "schema_title": "test/event", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "eventgate-analytics-external.test.event", + "topics": [ + "eqiad.eventgate-analytics-external.test.event", + "codfw.eventgate-analytics-external.test.event" + ] + }, + "eventgate-analytics-external.error.validation": { + "schema_title": "error", + "destination_event_service": "eventgate-analytics-external", + "canary_events_enabled": false, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "eventgate-analytics-external.error.validation", + "topics": [ + "eqiad.eventgate-analytics-external.error.validation", + "codfw.eventgate-analytics-external.error.validation" + ] + } + } +} diff --git a/analytics/metrics-platform/src/test/resources/org/wikimedia/metrics_platform/config/streamconfigs.json b/analytics/metrics-platform/src/test/resources/org/wikimedia/metrics_platform/config/streamconfigs.json new file mode 100644 index 00000000000..d54ea8a8626 --- /dev/null +++ b/analytics/metrics-platform/src/test/resources/org/wikimedia/metrics_platform/config/streamconfigs.json @@ -0,0 +1,1934 @@ +{ + "streams": { + "eventlogging_CentralNoticeBannerHistory": { + "schema_title": "analytics/legacy/centralnoticebannerhistory", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_CentralNoticeBannerHistory", + "topics": [ + "eventlogging_CentralNoticeBannerHistory" + ] + }, + "eventlogging_CentralNoticeImpression": { + "schema_title": "analytics/legacy/centralnoticeimpression", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_CentralNoticeImpression", + "topics": [ + "eventlogging_CentralNoticeImpression" + ] + }, + "eventlogging_CentralNoticeTiming": { + "schema_title": "analytics/legacy/centralnoticetiming", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_CentralNoticeTiming", + "topics": [ + "eventlogging_CentralNoticeTiming" + ] + }, + "eventlogging_CodeMirrorUsage": { + "schema_title": "analytics/legacy/codemirrorusage", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_CodeMirrorUsage", + "topics": [ + "eventlogging_CodeMirrorUsage" + ] + }, + "eventlogging_ContentTranslationAbuseFilter": { + "schema_title": "analytics/legacy/contenttranslationabusefilter", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_ContentTranslationAbuseFilter", + "topics": [ + "eventlogging_ContentTranslationAbuseFilter" + ] + }, + "eventlogging_CpuBenchmark": { + "schema_title": "analytics/legacy/cpubenchmark", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_CpuBenchmark", + "topics": [ + "eventlogging_CpuBenchmark" + ] + }, + "eventlogging_DesktopWebUIActionsTracking": { + "schema_title": "analytics/legacy/desktopwebuiactionstracking", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_DesktopWebUIActionsTracking", + "topics": [ + "eventlogging_DesktopWebUIActionsTracking" + ] + }, + "eventlogging_ElementTiming": { + "schema_title": "analytics/legacy/elementtiming", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_ElementTiming", + "topics": [ + "eventlogging_ElementTiming" + ] + }, + "eventlogging_EditAttemptStep": { + "schema_title": "analytics/legacy/editattemptstep", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_EditAttemptStep", + "topics": [ + "eventlogging_EditAttemptStep" + ] + }, + "eventlogging_FeaturePolicyViolation": { + "schema_title": "analytics/legacy/featurepolicyviolation", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_FeaturePolicyViolation", + "topics": [ + "eventlogging_FeaturePolicyViolation" + ] + }, + "eventlogging_FirstInputTiming": { + "schema_title": "analytics/legacy/firstinputtiming", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_FirstInputTiming", + "topics": [ + "eventlogging_FirstInputTiming" + ] + }, + "eventlogging_HelpPanel": { + "schema_title": "analytics/legacy/helppanel", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_HelpPanel", + "topics": [ + "eventlogging_HelpPanel" + ] + }, + "eventlogging_HomepageModule": { + "schema_title": "analytics/legacy/homepagemodule", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_HomepageModule", + "topics": [ + "eventlogging_HomepageModule" + ] + }, + "eventlogging_HomepageVisit": { + "schema_title": "analytics/legacy/homepagevisit", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_HomepageVisit", + "topics": [ + "eventlogging_HomepageVisit" + ] + }, + "eventlogging_InukaPageView": { + "schema_title": "analytics/legacy/inukapageview", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_InukaPageView", + "topics": [ + "eventlogging_InukaPageView" + ] + }, + "eventlogging_KaiOSAppFirstRun": { + "schema_title": "analytics/legacy/kaiosappfirstrun", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_KaiOSAppFirstRun", + "topics": [ + "eventlogging_KaiOSAppFirstRun" + ] + }, + "eventlogging_KaiOSAppFeedback": { + "schema_title": "analytics/legacy/kaiosappfeedback", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_KaiOSAppFeedback", + "topics": [ + "eventlogging_KaiOSAppFeedback" + ] + }, + "eventlogging_LandingPageImpression": { + "schema_title": "analytics/legacy/landingpageimpression", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_LandingPageImpression", + "topics": [ + "eventlogging_LandingPageImpression" + ] + }, + "eventlogging_LayoutShift": { + "schema_title": "analytics/legacy/layoutshift", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_LayoutShift", + "topics": [ + "eventlogging_LayoutShift" + ] + }, + "eventlogging_MobileWebUIActionsTracking": { + "schema_title": "analytics/legacy/mobilewebuiactionstracking", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_MobileWebUIActionsTracking", + "topics": [ + "eventlogging_MobileWebUIActionsTracking" + ] + }, + "eventlogging_NavigationTiming": { + "schema_title": "analytics/legacy/navigationtiming", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_NavigationTiming", + "topics": [ + "eventlogging_NavigationTiming" + ] + }, + "eventlogging_NewcomerTask": { + "schema_title": "analytics/legacy/newcomertask", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_NewcomerTask", + "topics": [ + "eventlogging_NewcomerTask" + ] + }, + "eventlogging_PaintTiming": { + "schema_title": "analytics/legacy/painttiming", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_PaintTiming", + "topics": [ + "eventlogging_PaintTiming" + ] + }, + "eventlogging_PrefUpdate": { + "schema_title": "analytics/legacy/prefupdate", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_PrefUpdate", + "topics": [ + "eventlogging_PrefUpdate" + ] + }, + "eventlogging_WikipediaPortal": { + "schema_title": "analytics/legacy/wikipediaportal", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_WikipediaPortal", + "topics": [ + "eventlogging_WikipediaPortal" + ] + }, + "eventlogging_QuickSurveyInitiation": { + "schema_title": "analytics/legacy/quicksurveyinitiation", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_QuickSurveyInitiation", + "topics": [ + "eventlogging_QuickSurveyInitiation" + ] + }, + "eventlogging_QuickSurveysResponses": { + "schema_title": "analytics/legacy/quicksurveysresponses", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_QuickSurveysResponses", + "topics": [ + "eventlogging_QuickSurveysResponses" + ] + }, + "eventlogging_ReferencePreviewsBaseline": { + "schema_title": "analytics/legacy/referencepreviewsbaseline", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_ReferencePreviewsBaseline", + "topics": [ + "eventlogging_ReferencePreviewsBaseline" + ] + }, + "eventlogging_ReferencePreviewsCite": { + "schema_title": "analytics/legacy/referencepreviewscite", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_ReferencePreviewsCite", + "topics": [ + "eventlogging_ReferencePreviewsCite" + ] + }, + "eventlogging_ReferencePreviewsPopups": { + "schema_title": "analytics/legacy/referencepreviewspopups", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_ReferencePreviewsPopups", + "topics": [ + "eventlogging_ReferencePreviewsPopups" + ] + }, + "eventlogging_ResourceTiming": { + "schema_title": "analytics/legacy/resourcetiming", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_ResourceTiming", + "topics": [ + "eventlogging_ResourceTiming" + ] + }, + "eventlogging_RUMSpeedIndex": { + "schema_title": "analytics/legacy/rumspeedindex", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_RUMSpeedIndex", + "topics": [ + "eventlogging_RUMSpeedIndex" + ] + }, + "eventlogging_SaveTiming": { + "schema_title": "analytics/legacy/savetiming", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_SaveTiming", + "topics": [ + "eventlogging_SaveTiming" + ] + }, + "eventlogging_SearchSatisfaction": { + "schema_title": "analytics/legacy/searchsatisfaction", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_SearchSatisfaction", + "topics": [ + "eventlogging_SearchSatisfaction" + ] + }, + "eventlogging_ServerSideAccountCreation": { + "schema_title": "analytics/legacy/serversideaccountcreation", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_ServerSideAccountCreation", + "topics": [ + "eventlogging_ServerSideAccountCreation" + ] + }, + "eventlogging_SpecialInvestigate": { + "schema_title": "analytics/legacy/specialinvestigate", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_SpecialInvestigate", + "topics": [ + "eventlogging_SpecialInvestigate" + ] + }, + "eventlogging_SpecialMuteSubmit": { + "schema_title": "analytics/legacy/specialmutesubmit", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_SpecialMuteSubmit", + "topics": [ + "eventlogging_SpecialMuteSubmit" + ] + }, + "eventlogging_SuggestedTagsAction": { + "schema_title": "analytics/legacy/suggestedtagsaction", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_SuggestedTagsAction", + "topics": [ + "eventlogging_SuggestedTagsAction" + ] + }, + "eventlogging_TemplateDataApi": { + "schema_title": "analytics/legacy/templatedataapi", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_TemplateDataApi", + "topics": [ + "eventlogging_TemplateDataApi" + ] + }, + "eventlogging_TemplateDataEditor": { + "schema_title": "analytics/legacy/templatedataeditor", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_TemplateDataEditor", + "topics": [ + "eventlogging_TemplateDataEditor" + ] + }, + "eventlogging_TemplateWizard": { + "schema_title": "analytics/legacy/templatewizard", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_TemplateWizard", + "topics": [ + "eventlogging_TemplateWizard" + ] + }, + "eventlogging_Test": { + "schema_title": "analytics/legacy/test", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_Test", + "topics": [ + "eventlogging_Test" + ] + }, + "eventlogging_TranslationRecommendationUserAction": { + "schema_title": "analytics/legacy/translationrecommendationuseraction", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_TranslationRecommendationUserAction", + "topics": [ + "eventlogging_TranslationRecommendationUserAction" + ] + }, + "eventlogging_TranslationRecommendationUIRequests": { + "schema_title": "analytics/legacy/translationrecommendationuirequests", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_TranslationRecommendationUIRequests", + "topics": [ + "eventlogging_TranslationRecommendationUIRequests" + ] + }, + "eventlogging_TranslationRecommendationAPIRequests": { + "schema_title": "analytics/legacy/translationrecommendationapirequests", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_TranslationRecommendationAPIRequests", + "topics": [ + "eventlogging_TranslationRecommendationAPIRequests" + ] + }, + "eventlogging_TwoColConflictConflict": { + "schema_title": "analytics/legacy/twocolconflictconflict", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_TwoColConflictConflict", + "topics": [ + "eventlogging_TwoColConflictConflict" + ] + }, + "eventlogging_TwoColConflictExit": { + "schema_title": "analytics/legacy/twocolconflictexit", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_TwoColConflictExit", + "topics": [ + "eventlogging_TwoColConflictExit" + ] + }, + "eventlogging_UniversalLanguageSelector": { + "schema_title": "analytics/legacy/universallanguageselector", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_UniversalLanguageSelector", + "topics": [ + "eventlogging_UniversalLanguageSelector" + ] + }, + "eventlogging_VirtualPageView": { + "schema_title": "analytics/legacy/virtualpageview", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_VirtualPageView", + "topics": [ + "eventlogging_VirtualPageView" + ] + }, + "eventlogging_VisualEditorFeatureUse": { + "schema_title": "analytics/legacy/visualeditorfeatureuse", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_VisualEditorFeatureUse", + "topics": [ + "eventlogging_VisualEditorFeatureUse" + ] + }, + "eventlogging_VisualEditorTemplateDialogUse": { + "schema_title": "analytics/legacy/visualeditortemplatedialoguse", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_VisualEditorTemplateDialogUse", + "topics": [ + "eventlogging_VisualEditorTemplateDialogUse" + ] + }, + "eventlogging_WikibaseTermboxInteraction": { + "schema_title": "analytics/legacy/wikibasetermboxinteraction", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_WikibaseTermboxInteraction", + "topics": [ + "eventlogging_WikibaseTermboxInteraction" + ] + }, + "eventlogging_WikidataCompletionSearchClicks": { + "schema_title": "analytics/legacy/wikidatacompletionsearchclicks", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_WikidataCompletionSearchClicks", + "topics": [ + "eventlogging_WikidataCompletionSearchClicks" + ] + }, + "eventlogging_WMDEBannerEvents": { + "schema_title": "analytics/legacy/wmdebannerevents", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_WMDEBannerEvents", + "topics": [ + "eventlogging_WMDEBannerEvents" + ] + }, + "eventlogging_WMDEBannerInteractions": { + "schema_title": "analytics/legacy/wmdebannerinteractions", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_WMDEBannerInteractions", + "topics": [ + "eventlogging_WMDEBannerInteractions" + ] + }, + "eventlogging_WMDEBannerSizeIssue": { + "schema_title": "analytics/legacy/wmdebannersizeissue", + "topic_prefixes": null, + "destination_event_service": "eventgate-analytics-external", + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "eventlogging_legacy", + "enabled": true + } + }, + "canary_events_enabled": true, + "stream": "eventlogging_WMDEBannerSizeIssue", + "topics": [ + "eventlogging_WMDEBannerSizeIssue" + ] + }, + "test.instrumentation": { + "schema_title": "analytics/test", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "test.instrumentation", + "topics": [ + "eqiad.test.instrumentation", + "codfw.test.instrumentation" + ] + }, + "test.instrumentation.sampled": { + "schema_title": "analytics/test", + "destination_event_service": "eventgate-analytics-external", + "sample": { + "rate": 0.5, + "unit": "session" + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "test.instrumentation.sampled", + "topics": [ + "eqiad.test.instrumentation.sampled", + "codfw.test.instrumentation.sampled" + ] + }, + "mediawiki.client.session_tick": { + "schema_title": "analytics/session_tick", + "destination_event_service": "eventgate-analytics-external", + "sample": { + "unit": "session", + "rate": 0.1 + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.client.session_tick", + "topics": [ + "eqiad.mediawiki.client.session_tick", + "codfw.mediawiki.client.session_tick" + ] + }, + "ios.edit_history_compare": { + "schema_title": "analytics/mobile_apps/ios_edit_history_compare", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "ios.edit_history_compare", + "topics": [ + "eqiad.ios.edit_history_compare", + "codfw.ios.edit_history_compare" + ] + }, + "ios.notification_interaction": { + "schema_title": "analytics/mobile_apps/ios_notification_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "ios.notification_interaction", + "topics": [ + "eqiad.ios.notification_interaction", + "codfw.ios.notification_interaction" + ] + }, + "android.user_contribution_screen": { + "schema_title": "analytics/mobile_apps/android_user_contribution_screen", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.user_contribution_screen", + "topics": [ + "eqiad.android.user_contribution_screen", + "codfw.android.user_contribution_screen" + ] + }, + "android.notification_interaction": { + "schema_title": "analytics/mobile_apps/android_notification_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.notification_interaction", + "topics": [ + "eqiad.android.notification_interaction", + "codfw.android.notification_interaction" + ] + }, + "android.image_recommendation_interaction": { + "schema_title": "analytics/mobile_apps/android_image_recommendation_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.image_recommendation_interaction", + "topics": [ + "eqiad.android.image_recommendation_interaction", + "codfw.android.image_recommendation_interaction" + ] + }, + "android.daily_stats": { + "schema_title": "analytics/mobile_apps/android_daily_stats", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.daily_stats", + "topics": [ + "eqiad.android.daily_stats", + "codfw.android.daily_stats" + ] + }, + "android.customize_toolbar_interaction": { + "schema_title": "analytics/mobile_apps/android_customize_toolbar_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.customize_toolbar_interaction", + "topics": [ + "eqiad.android.customize_toolbar_interaction", + "codfw.android.customize_toolbar_interaction" + ] + }, + "android.article_toolbar_interaction": { + "schema_title": "analytics/mobile_apps/android_article_toolbar_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.article_toolbar_interaction", + "topics": [ + "eqiad.android.article_toolbar_interaction", + "codfw.android.article_toolbar_interaction" + ] + }, + "android.edit_history_interaction": { + "schema_title": "analytics/mobile_apps/android_edit_history_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.edit_history_interaction", + "topics": [ + "eqiad.android.edit_history_interaction", + "codfw.android.edit_history_interaction" + ] + }, + "android.breadcrumbs_event": { + "schema_title": "analytics/mobile_apps/android_breadcrumbs_event", + "destination_event_service": "eventgate-analytics-external", + "sample": { + "unit": "device", + "rate": 0.5 + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.breadcrumbs_event", + "topics": [ + "eqiad.android.breadcrumbs_event", + "codfw.android.breadcrumbs_event" + ] + }, + "android.app_appearance_settings_interaction": { + "schema_title": "analytics/mobile_apps/android_app_appearance_settings_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.app_appearance_settings_interaction", + "topics": [ + "eqiad.android.app_appearance_settings_interaction", + "codfw.android.app_appearance_settings_interaction" + ] + }, + "android.article_link_preview_interaction": { + "schema_title": "analytics/mobile_apps/android_article_link_preview_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.article_link_preview_interaction", + "topics": [ + "eqiad.android.article_link_preview_interaction", + "codfw.android.article_link_preview_interaction" + ] + }, + "android.article_page_scroll_interaction": { + "schema_title": "analytics/mobile_apps/android_article_page_scroll_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.article_page_scroll_interaction", + "topics": [ + "eqiad.android.article_page_scroll_interaction", + "codfw.android.article_page_scroll_interaction" + ] + }, + "android.article_toc_interaction": { + "schema_title": "analytics/mobile_apps/android_article_toc_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.article_toc_interaction", + "topics": [ + "eqiad.android.article_toc_interaction", + "codfw.android.article_toc_interaction" + ] + }, + "android.find_in_page_interaction": { + "schema_title": "analytics/mobile_apps/android_find_in_page_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "android.find_in_page_interaction", + "topics": [ + "eqiad.android.find_in_page_interaction", + "codfw.android.find_in_page_interaction" + ] + }, + "mediawiki.mediasearch_interaction": { + "schema_title": "analytics/mediawiki/mediasearch_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.mediasearch_interaction", + "topics": [ + "eqiad.mediawiki.mediasearch_interaction", + "codfw.mediawiki.mediasearch_interaction" + ] + }, + "mediawiki.structured_task.article.link_suggestion_interaction": { + "schema_title": "analytics/mediawiki/structured_task/article/link_suggestion_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.structured_task.article.link_suggestion_interaction", + "topics": [ + "eqiad.mediawiki.structured_task.article.link_suggestion_interaction", + "codfw.mediawiki.structured_task.article.link_suggestion_interaction" + ] + }, + "mediawiki.structured_task.article.image_suggestion_interaction": { + "schema_title": "analytics/mediawiki/structured_task/article/image_suggestion_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.structured_task.article.image_suggestion_interaction", + "topics": [ + "eqiad.mediawiki.structured_task.article.image_suggestion_interaction", + "codfw.mediawiki.structured_task.article.image_suggestion_interaction" + ] + }, + "mediawiki.pref_diff": { + "schema_title": "analytics/pref_diff", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.pref_diff", + "topics": [ + "eqiad.mediawiki.pref_diff", + "codfw.mediawiki.pref_diff" + ] + }, + "mediawiki.skin_diff": { + "schema_title": "analytics/pref_diff", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.skin_diff", + "topics": [ + "eqiad.mediawiki.skin_diff", + "codfw.mediawiki.skin_diff" + ] + }, + "mediawiki.content_translation_event": { + "schema_title": "analytics/mediawiki/content_translation_event", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.content_translation_event", + "topics": [ + "eqiad.mediawiki.content_translation_event", + "codfw.mediawiki.content_translation_event" + ] + }, + "mediawiki.reading_depth": { + "schema_title": "analytics/mediawiki/web_ui_reading_depth", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.reading_depth", + "topics": [ + "eqiad.mediawiki.reading_depth", + "codfw.mediawiki.reading_depth" + ] + }, + "mediawiki.web_ab_test_enrollment": { + "schema_title": "analytics/mediawiki/web_ab_test_enrollment", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.web_ab_test_enrollment", + "topics": [ + "eqiad.mediawiki.web_ab_test_enrollment", + "codfw.mediawiki.web_ab_test_enrollment" + ] + }, + "mediawiki.web_ui_scroll": { + "schema_title": "analytics/mediawiki/web_ui_scroll", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.web_ui_scroll", + "topics": [ + "eqiad.mediawiki.web_ui_scroll", + "codfw.mediawiki.web_ui_scroll" + ] + }, + "mediawiki.ipinfo_interaction": { + "schema_title": "analytics/mediawiki/ipinfo_interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.ipinfo_interaction", + "topics": [ + "eqiad.mediawiki.ipinfo_interaction", + "codfw.mediawiki.ipinfo_interaction" + ] + }, + "wd_propertysuggester.client_side_property_request": { + "schema_title": "analytics/mediawiki/wd_propertysuggester/client_side_property_request", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "wd_propertysuggester.client_side_property_request", + "topics": [ + "eqiad.wd_propertysuggester.client_side_property_request", + "codfw.wd_propertysuggester.client_side_property_request" + ] + }, + "wd_propertysuggester.server_side_property_request": { + "schema_title": "analytics/mediawiki/wd_propertysuggester/server_side_property_request", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "wd_propertysuggester.server_side_property_request", + "topics": [ + "eqiad.wd_propertysuggester.server_side_property_request", + "codfw.wd_propertysuggester.server_side_property_request" + ] + }, + "mediawiki.mentor_dashboard.visit": { + "schema_title": "analytics/mediawiki/mentor_dashboard/visit", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.mentor_dashboard.visit", + "topics": [ + "eqiad.mediawiki.mentor_dashboard.visit", + "codfw.mediawiki.mentor_dashboard.visit" + ] + }, + "mediawiki.welcomesurvey.interaction": { + "schema_title": "analytics/mediawiki/welcomesurvey/interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.welcomesurvey.interaction", + "topics": [ + "eqiad.mediawiki.welcomesurvey.interaction", + "codfw.mediawiki.welcomesurvey.interaction" + ] + }, + "mediawiki.editgrowthconfig": { + "schema_title": "analytics/mediawiki/editgrowthconfig", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.editgrowthconfig", + "topics": [ + "eqiad.mediawiki.editgrowthconfig", + "codfw.mediawiki.editgrowthconfig" + ] + }, + "mediawiki.accountcreation_block": { + "schema_title": "analytics/mediawiki/accountcreation/block", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.accountcreation_block", + "topics": [ + "eqiad.mediawiki.accountcreation_block", + "codfw.mediawiki.accountcreation_block" + ] + }, + "mediawiki.editattempt_block": { + "schema_title": "analytics/mediawiki/editattemptsblocked", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.editattempt_block", + "topics": [ + "eqiad.mediawiki.editattempt_block", + "codfw.mediawiki.editattempt_block" + ] + }, + "mediawiki.talk_page_edit": { + "stream": "mediawiki.talk_page_edit", + "schema_title": "analytics/mediawiki/talk_page_edit", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "topics": [ + "eqiad.mediawiki.talk_page_edit", + "codfw.mediawiki.talk_page_edit" + ] + }, + "mwcli.command_execute": { + "schema_title": "analytics/mwcli/command_execute", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mwcli.command_execute", + "topics": [ + "eqiad.mwcli.command_execute", + "codfw.mwcli.command_execute" + ] + }, + "mediawiki.web_ui.interactions": { + "schema_title": "analytics/mediawiki/client/metrics_event", + "destination_event_service": "eventgate-analytics-external", + "producers": { + "metrics_platform_client": { + "events": [ + "web.ui.", + "web_ui." + ], + "provide_values": [ + "page_namespace", + "performer_is_logged_in", + "performer_session_id", + "performer_pageview_id", + "performer_edit_count_bucket", + "mediawiki_skin", + "mediawiki_database" + ], + "curation": { + "mediawiki_skin": { + "in": [ + "minerva", + "vector", + "vector-2022" + ] + } + } + } + }, + "sample": { + "unit": "pageview", + "rate": 0 + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.web_ui.interactions", + "topics": [ + "eqiad.mediawiki.web_ui.interactions", + "codfw.mediawiki.web_ui.interactions" + ] + }, + "mediawiki.edit_attempt": { + "schema_title": "analytics/mediawiki/client/metrics_event", + "destination_event_service": "eventgate-analytics-external", + "producers": { + "metrics_platform_client": { + "events": [ + "eas." + ], + "provide_values": [ + "agent_client_platform_family", + "page_id", + "page_title", + "page_namespace", + "page_revision_id", + "mediawiki_version", + "mediawiki_is_debug_mode", + "mediawiki_database", + "performer_is_logged_in", + "performer_id", + "performer_session_id", + "performer_pageview_id", + "performer_edit_count" + ] + } + }, + "sample": { + "unit": "pageview", + "rate": 1 + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.edit_attempt", + "topics": [ + "eqiad.mediawiki.edit_attempt", + "codfw.mediawiki.edit_attempt" + ] + }, + "mediawiki.visual_editor_feature_use": { + "schema_title": "analytics/mediawiki/client/metrics_event", + "destination_event_service": "eventgate-analytics-external", + "producers": { + "metrics_platform_client": { + "events": [ + "vefu." + ], + "provide_values": [ + "agent_client_platform_family", + "mediawiki_database", + "performer_id", + "performer_edit_count" + ] + } + }, + "sample": { + "unit": "pageview", + "rate": 0 + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.visual_editor_feature_use", + "topics": [ + "eqiad.mediawiki.visual_editor_feature_use", + "codfw.mediawiki.visual_editor_feature_use" + ] + }, + "mediawiki.wikistories_consumption_event": { + "schema_title": "analytics/mediawiki/wikistories_consumption_event", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.wikistories_consumption_event", + "topics": [ + "eqiad.mediawiki.wikistories_consumption_event", + "codfw.mediawiki.wikistories_consumption_event" + ] + }, + "mediawiki.wikistories_contribution_event": { + "schema_title": "analytics/mediawiki/wikistories_contribution_event", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.wikistories_contribution_event", + "topics": [ + "eqiad.mediawiki.wikistories_contribution_event", + "codfw.mediawiki.wikistories_contribution_event" + ] + }, + "rc0.mediawiki.page_change": { + "schema_title": "development/mediawiki/page/change", + "message_key_fields": { + "wiki_id": "wiki_id", + "page_id": "page.page_id" + }, + "destination_event_service": "eventgate-analytics-external", + "producers": { + "mediawiki_eventbus": { + "enabled": false + } + }, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "rc0.mediawiki.page_change", + "topics": [ + "eqiad.rc0.mediawiki.page_change", + "codfw.rc0.mediawiki.page_change" + ] + }, + "rc0.mediawiki.page_content_change": { + "schema_title": "development/mediawiki/page/change", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "rc0.mediawiki.page_content_change", + "topics": [ + "eqiad.rc0.mediawiki.page_content_change", + "codfw.rc0.mediawiki.page_content_change" + ] + }, + "mediawiki.maps_interaction": { + "schema_title": "analytics/mediawiki/maps/interaction", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "mediawiki.maps_interaction", + "topics": [ + "eqiad.mediawiki.maps_interaction", + "codfw.mediawiki.maps_interaction" + ] + }, + "eventgate-analytics-external.test.event": { + "schema_title": "test/event", + "destination_event_service": "eventgate-analytics-external", + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "canary_events_enabled": true, + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "eventgate-analytics-external.test.event", + "topics": [ + "eqiad.eventgate-analytics-external.test.event", + "codfw.eventgate-analytics-external.test.event" + ] + }, + "eventgate-analytics-external.error.validation": { + "schema_title": "error", + "destination_event_service": "eventgate-analytics-external", + "canary_events_enabled": false, + "topic_prefixes": [ + "eqiad.", + "codfw." + ], + "consumers": { + "analytics_hadoop_ingestion": { + "job_name": "event_default", + "enabled": true + } + }, + "stream": "eventgate-analytics-external.error.validation", + "topics": [ + "eqiad.eventgate-analytics-external.error.validation", + "codfw.eventgate-analytics-external.error.validation" + ] + } + } +} \ No newline at end of file diff --git a/analytics/metrics-platform/src/test/resources/org/wikimedia/metrics_platform/event/expected_event_click.json b/analytics/metrics-platform/src/test/resources/org/wikimedia/metrics_platform/event/expected_event_click.json new file mode 100644 index 00000000000..c31e50ad32e --- /dev/null +++ b/analytics/metrics-platform/src/test/resources/org/wikimedia/metrics_platform/event/expected_event_click.json @@ -0,0 +1,43 @@ +[ { + "$schema": "/analytics/product_metrics/app/base/1.2.2", + "meta": { + "stream": "mediawiki.metrics_platform.click", + "domain": "en.wikipedia.org" + }, + "agent": { + "app_flavor": "devdebug", + "app_install_id": "ffffffff-ffff-ffff-ffff-ffffffffffff", + "app_theme": "LIGHT", + "app_version": 123456, + "app_version_name": "2.7.50470-dev-2024-02-14", + "client_platform": "android", + "client_platform_family": "app", + "device_family": "samsung", + "device_language": "en", + "release_status": "dev" + }, + "page": { + "id": 30715282, + "title": "Dark_Souls", + "namespace_id": 0, + "revision_id": 1084793259 + }, + "mediawiki": { + "database": "enwiki" + }, + "performer": { + "is_logged_in": false, + "id": 0, + "session_id": "eeeeeeeeeeeeeeeeeeee", + "pageview_id": "eeeeeeeeeeeeeeeeeeee" + }, + "action" : "TestClick", + "action_subtype" : "TestActionSubtype", + "action_source" : "TestActionSource", + "action_context" : "TestActionContext", + "element_id" : "TestElementId", + "element_friendly_name" : "TestElementFriendlyName", + "funnel_entry_token" : "TestFunnelEntryToken", + "funnel_event_sequence_position" : 8, + "dt" : "${json-unit.ignore}" +} ] diff --git a/analytics/metrics-platform/src/test/resources/org/wikimedia/metrics_platform/event/expected_event_click_custom.json b/analytics/metrics-platform/src/test/resources/org/wikimedia/metrics_platform/event/expected_event_click_custom.json new file mode 100644 index 00000000000..01e3868996d --- /dev/null +++ b/analytics/metrics-platform/src/test/resources/org/wikimedia/metrics_platform/event/expected_event_click_custom.json @@ -0,0 +1,46 @@ +[ { + "$schema": "/analytics/product_metrics/app/click_custom/1.0.0", + "meta": { + "stream": "mediawiki.metrics_platform.click_custom", + "domain": "en.wikipedia.org" + }, + "font_size": "small", + "is_full_width": true, + "screen_size": 1080, + "agent": { + "app_flavor": "devdebug", + "app_install_id": "ffffffff-ffff-ffff-ffff-ffffffffffff", + "app_theme": "LIGHT", + "app_version": 123456, + "app_version_name": "2.7.50470-dev-2024-02-14", + "client_platform": "android", + "client_platform_family": "app", + "device_family": "samsung", + "device_language": "en", + "release_status": "dev" + }, + "page": { + "id": 30715282, + "title": "Dark_Souls", + "namespace_id": 0, + "revision_id": 1084793259 + }, + "mediawiki": { + "database": "enwiki" + }, + "performer": { + "is_logged_in": false, + "id": 0, + "session_id": "eeeeeeeeeeeeeeeeeeee", + "pageview_id": "eeeeeeeeeeeeeeeeeeee" + }, + "action" : "TestClickCustom", + "action_subtype" : "TestActionSubtype", + "action_source" : "TestActionSource", + "action_context" : "TestActionContext", + "element_id" : "TestElementId", + "element_friendly_name" : "TestElementFriendlyName", + "funnel_entry_token" : "TestFunnelEntryToken", + "funnel_event_sequence_position" : 8, + "dt" : "${json-unit.ignore}" +} ] diff --git a/analytics/metrics-platform/src/test/resources/org/wikimedia/metrics_platform/event/expected_event_interaction.json b/analytics/metrics-platform/src/test/resources/org/wikimedia/metrics_platform/event/expected_event_interaction.json new file mode 100644 index 00000000000..76949a91bbc --- /dev/null +++ b/analytics/metrics-platform/src/test/resources/org/wikimedia/metrics_platform/event/expected_event_interaction.json @@ -0,0 +1,43 @@ +[ { + "$schema": "/analytics/product_metrics/app/base/1.2.2", + "meta": { + "stream": "mediawiki.metrics_platform.interaction", + "domain": "en.wikipedia.org" + }, + "agent": { + "app_flavor": "devdebug", + "app_install_id": "ffffffff-ffff-ffff-ffff-ffffffffffff", + "app_theme": "LIGHT", + "app_version": 123456, + "app_version_name": "2.7.50470-dev-2024-02-14", + "client_platform": "android", + "client_platform_family": "app", + "device_family": "samsung", + "device_language": "en", + "release_status": "dev" + }, + "page": { + "id": 30715282, + "title": "Dark_Souls", + "namespace_id": 0, + "revision_id": 1084793259 + }, + "mediawiki": { + "database": "enwiki" + }, + "performer": { + "is_logged_in": false, + "id": 0, + "session_id": "eeeeeeeeeeeeeeeeeeee", + "pageview_id": "eeeeeeeeeeeeeeeeeeee" + }, + "action" : "TestInteraction", + "action_subtype" : "TestActionSubtype", + "action_source" : "TestActionSource", + "action_context" : "TestActionContext", + "element_id" : "TestElementId", + "element_friendly_name" : "TestElementFriendlyName", + "funnel_entry_token" : "TestFunnelEntryToken", + "funnel_event_sequence_position" : 8, + "dt" : "${json-unit.ignore}" +} ] diff --git a/analytics/metrics-platform/src/test/resources/org/wikimedia/metrics_platform/event/expected_event_view.json b/analytics/metrics-platform/src/test/resources/org/wikimedia/metrics_platform/event/expected_event_view.json new file mode 100644 index 00000000000..56dbfbf310f --- /dev/null +++ b/analytics/metrics-platform/src/test/resources/org/wikimedia/metrics_platform/event/expected_event_view.json @@ -0,0 +1,43 @@ +[ { + "$schema": "/analytics/product_metrics/app/base/1.2.2", + "meta": { + "stream": "mediawiki.metrics_platform.view", + "domain": "en.wikipedia.org" + }, + "agent": { + "app_flavor": "devdebug", + "app_install_id": "ffffffff-ffff-ffff-ffff-ffffffffffff", + "app_theme": "LIGHT", + "app_version": 123456, + "app_version_name": "2.7.50470-dev-2024-02-14", + "client_platform": "android", + "client_platform_family": "app", + "device_family": "samsung", + "device_language": "en", + "release_status": "dev" + }, + "page": { + "id": 30715282, + "title": "Dark_Souls", + "namespace_id": 0, + "revision_id": 1084793259 + }, + "mediawiki": { + "database": "enwiki" + }, + "performer": { + "is_logged_in": false, + "id": 0, + "session_id": "eeeeeeeeeeeeeeeeeeee", + "pageview_id": "eeeeeeeeeeeeeeeeeeee" + }, + "action" : "TestView", + "action_subtype" : "TestActionSubtype", + "action_source" : "TestActionSource", + "action_context" : "TestActionContext", + "element_id" : "TestElementId", + "element_friendly_name" : "TestElementFriendlyName", + "funnel_entry_token" : "TestFunnelEntryToken", + "funnel_event_sequence_position" : 8, + "dt" : "${json-unit.ignore}" +} ] diff --git a/analytics/metrics-platform/src/test/resources/simplelogger.properties b/analytics/metrics-platform/src/test/resources/simplelogger.properties new file mode 100644 index 00000000000..906149cfb44 --- /dev/null +++ b/analytics/metrics-platform/src/test/resources/simplelogger.properties @@ -0,0 +1,2 @@ +org.slf4j.simpleLogger.defaultLogLevel=WARN +org.slf4j.simpleLogger.org.wikimedia.metrics_platform=DEBUG diff --git a/app/build.gradle b/app/build.gradle index fa1ba8fcd1f..27f26f5ccf1 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -194,7 +194,6 @@ dependencies { implementation libs.drawerlayout implementation libs.swiperefreshlayout implementation libs.work.runtime.ktx - implementation libs.metrics.platform implementation libs.okhttp.tls implementation libs.okhttp3.logging.interceptor diff --git a/app/src/main/java/org/wikipedia/dataclient/Service.kt b/app/src/main/java/org/wikipedia/dataclient/Service.kt index 3cc8dc704e5..6d8aa529c2d 100644 --- a/app/src/main/java/org/wikipedia/dataclient/Service.kt +++ b/app/src/main/java/org/wikipedia/dataclient/Service.kt @@ -213,7 +213,7 @@ interface Service { @Field("token") token: String ): MwPostResponse - @GET(MW_API_PREFIX + "action=streamconfigs&format=json&constraints=destination_event_service=eventgate-analytics-external") + @GET(MW_API_PREFIX + "action=streamconfigs&format=json&constraints=destination_event_service%3Deventgate-analytics-external") suspend fun getStreamConfigs(): MwStreamConfigsResponse @GET(MW_API_PREFIX + "action=query&meta=allmessages&amenableparser=1") diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 67e868631ac..ecde3e03e9b 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -28,7 +28,6 @@ kotlinxSerializationJson = "1.8.1" kspPlugin = "2.2.0-2.0.2" leakCanaryVersion = "2.14" material = "1.12.0" -metricsVersion = "2.9" mlKitVersion = "17.0.6" mockitoVersion = "5.2.0" navigationCompose = "2.9.0" @@ -96,7 +95,6 @@ kotlinx-coroutines-core = { module = "org.jetbrains.kotlinx:kotlinx-coroutines-c kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "kotlinxSerializationJson" } leakcanary-android = { module = "com.squareup.leakcanary:leakcanary-android", version.ref = "leakCanaryVersion" } material = { module = "com.google.android.material:material", version.ref = "material" } -metrics-platform = { module = "org.wikimedia.metrics:metrics-platform", version.ref = "metricsVersion" } mockito-inline = { module = "org.mockito:mockito-inline", version.ref = "mockitoVersion" } mockwebserver = { module = "com.squareup.okhttp3:mockwebserver", version.ref = "okHttpVersion" } okhttp-tls = { module = "com.squareup.okhttp3:okhttp-tls", version.ref = "okHttpVersion" } diff --git a/settings.gradle.kts b/settings.gradle.kts index 1a6a4962cae..d23137e25b6 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -15,3 +15,4 @@ dependencyResolutionManagement { } include(":app") +include(":analytics:metrics-platform") From c7a6cd2fab5bf339157aae69310d9d9302bb2e60 Mon Sep 17 00:00:00 2001 From: Dmitry Brant Date: Fri, 27 Jun 2025 20:59:14 -0400 Subject: [PATCH 2/9] Right... --- .../metrics_platform/ContextController.java | 2 - .../metrics_platform/CurationController.java | 15 -- .../metrics_platform/CurationController.kt | 11 ++ .../metrics_platform/EventSender.java | 19 -- .../wikimedia/metrics_platform/EventSender.kt | 14 ++ .../metrics_platform/MetricsClient.java | 19 +- .../metrics_platform/SamplingController.java | 78 -------- .../metrics_platform/SamplingController.kt | 57 ++++++ .../metrics_platform/SessionController.java | 80 --------- .../metrics_platform/SessionController.kt | 58 ++++++ .../config/CurationFilter.java | 86 --------- .../metrics_platform/config/CurationFilter.kt | 98 ++++++++++ .../metrics_platform/config/SourceConfig.java | 63 ------- .../metrics_platform/config/SourceConfig.kt | 35 ++++ .../metrics_platform/config/StreamConfig.java | 127 ------------- .../metrics_platform/config/StreamConfig.kt | 86 +++++++++ .../config/StreamConfigCollection.java | 16 -- .../config/StreamConfigCollection.kt | 9 + .../curation/CollectionCurationRules.java | 30 ---- .../curation/CollectionCurationRules.kt | 24 +++ .../curation/ComparableCurationRules.java | 38 ---- .../curation/ComparableCurationRules.kt | 36 ++++ .../config/curation/CurationRules.java | 30 ---- .../config/curation/CurationRules.kt | 24 +++ .../config/sampling/SampleConfig.java | 18 -- .../config/sampling/SampleConfig.kt | 15 ++ .../metrics_platform/context/PerformerData.kt | 2 +- .../metrics_platform/event/Event.java | 65 ------- .../wikimedia/metrics_platform/event/Event.kt | 40 +++++ .../event/EventProcessed.java | 170 ------------------ .../metrics_platform/event/EventProcessed.kt | 120 +++++++++++++ .../metrics_platform/utils/Objects.java | 24 --- 32 files changed, 629 insertions(+), 880 deletions(-) delete mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/CurationController.java create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/CurationController.kt delete mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/EventSender.java create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/EventSender.kt delete mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/SamplingController.java create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/SamplingController.kt delete mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/SessionController.java create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/SessionController.kt delete mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/CurationFilter.java create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/CurationFilter.kt delete mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/SourceConfig.java create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/SourceConfig.kt delete mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/StreamConfig.java create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/StreamConfig.kt delete mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/StreamConfigCollection.java create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/StreamConfigCollection.kt delete mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/CollectionCurationRules.java create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/CollectionCurationRules.kt delete mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/ComparableCurationRules.java create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/ComparableCurationRules.kt delete mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/CurationRules.java create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/CurationRules.kt delete mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/sampling/SampleConfig.java create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/sampling/SampleConfig.kt delete mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/event/Event.java create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/event/Event.kt delete mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/event/EventProcessed.java create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/event/EventProcessed.kt delete mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/utils/Objects.java diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/ContextController.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/ContextController.java index 0deafb4dc33..72d01973a7a 100644 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/ContextController.java +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/ContextController.java @@ -82,8 +82,6 @@ public void enrichEvent(EventProcessed event, StreamConfig streamConfig) { event.setClientData(filteredData); } - @SuppressWarnings("checkstyle:CyclomaticComplexity") - @SuppressFBWarnings(value = "CC_CYCLOMATIC_COMPLEXITY", justification = "TODO: needs to be refactored") private ClientData filterClientData(ClientData clientData, Collection requestedValues) { AgentData.AgentDataBuilder agentBuilder = AgentData.builder(); PageData.PageDataBuilder pageBuilder = PageData.builder(); diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/CurationController.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/CurationController.java deleted file mode 100644 index e1cec9dfcc6..00000000000 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/CurationController.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.wikimedia.metrics_platform; - -import org.wikimedia.metrics_platform.config.StreamConfig; -import org.wikimedia.metrics_platform.event.EventProcessed; - -import lombok.NonNull; - -public class CurationController { - public boolean shouldProduceEvent(@NonNull EventProcessed event, @NonNull StreamConfig streamConfig) { - if (!streamConfig.hasCurationFilter()) return true; - - return streamConfig.getCurationFilter().test(event); - } - -} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/CurationController.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/CurationController.kt new file mode 100644 index 00000000000..50c51a060a1 --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/CurationController.kt @@ -0,0 +1,11 @@ +package org.wikimedia.metrics_platform + +import org.wikimedia.metrics_platform.config.StreamConfig +import org.wikimedia.metrics_platform.event.EventProcessed + +class CurationController { + fun shouldProduceEvent(event: EventProcessed, streamConfig: StreamConfig): Boolean { + if (!streamConfig.hasCurationFilter()) return true + return streamConfig.curationFilter.test(event) + } +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/EventSender.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/EventSender.java deleted file mode 100644 index c1b2d34e9e9..00000000000 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/EventSender.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.wikimedia.metrics_platform; - -import java.io.IOException; -import java.net.URL; -import java.util.Collection; - -import org.wikimedia.metrics_platform.event.EventProcessed; - -public interface EventSender { - - /** - * Transmit an event to a destination intake service. - * - * @param baseUri base uri of destination intake service - * @param events events to be sent - */ - - void sendEvents(URL baseUri, Collection events) throws IOException; -} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/EventSender.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/EventSender.kt new file mode 100644 index 00000000000..fab2bef8edf --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/EventSender.kt @@ -0,0 +1,14 @@ +package org.wikimedia.metrics_platform + +import org.wikimedia.metrics_platform.event.EventProcessed +import java.net.URL + +fun interface EventSender { + /** + * Transmit an event to a destination intake service. + * + * @param baseUri base uri of destination intake service + * @param events events to be sent + */ + fun sendEvents(baseUri: URL, events: List) +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/MetricsClient.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/MetricsClient.java index d486d1ab723..f374bcbb505 100644 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/MetricsClient.java +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/MetricsClient.java @@ -6,7 +6,6 @@ import static java.util.logging.Level.FINE; import static java.util.stream.Collectors.toList; import static org.wikimedia.metrics_platform.config.StreamConfigFetcher.ANALYTICS_API_ENDPOINT; -import static org.wikimedia.metrics_platform.event.EventProcessed.fromEvent; import java.net.URL; import java.time.Duration; @@ -26,10 +25,6 @@ import java.util.logging.Level; import java.util.stream.Stream; -import javax.annotation.Nullable; -import javax.annotation.ParametersAreNonnullByDefault; -import javax.annotation.concurrent.NotThreadSafe; - import org.wikimedia.metrics_platform.config.ConfigFetcherRunnable; import org.wikimedia.metrics_platform.config.SourceConfig; import org.wikimedia.metrics_platform.config.StreamConfig; @@ -43,13 +38,8 @@ import com.google.gson.Gson; -import lombok.Setter; -import lombok.SneakyThrows; -import lombok.experimental.Accessors; -import lombok.extern.java.Log; import okhttp3.OkHttpClient; -@Log public final class MetricsClient { public static final DateTimeFormatter DATE_FORMAT = DateTimeFormatter @@ -110,7 +100,7 @@ private MetricsClient( * @param event event data */ public void submit(Event event) { - EventProcessed eventProcessed = fromEvent(event); + EventProcessed eventProcessed = EventProcessed.fromEvent(event); addRequiredMetadata(eventProcessed); addToEventQueue(eventProcessed); } @@ -431,12 +421,6 @@ public static Builder builder(ClientData clientData) { return new Builder(clientData); } - @NotThreadSafe @ParametersAreNonnullByDefault - @Setter @Accessors(fluent = true) - @SuppressWarnings("checkstyle:classfanoutcomplexity") // As the main builder for the application, this class has - // to fan out to almost everything. We could hide this by - // using an injection framework (Guice?), but the added - // dependency is probably not worth it. public static final class Builder { private final ClientData clientData; @@ -524,7 +508,6 @@ private void startScheduledOperations( sendEventsInitialDelay.toMillis(), sendEventsInterval.toMillis(), MILLISECONDS); } - @SneakyThrows private static URL safeURL(String url) { return new URL(url); } diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/SamplingController.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/SamplingController.java deleted file mode 100644 index b0122328ace..00000000000 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/SamplingController.java +++ /dev/null @@ -1,78 +0,0 @@ -package org.wikimedia.metrics_platform; - -import javax.annotation.concurrent.ThreadSafe; -import javax.annotation.Nonnull; -import javax.annotation.ParametersAreNonnullByDefault; - -import org.wikimedia.metrics_platform.config.sampling.SampleConfig; -import org.wikimedia.metrics_platform.config.StreamConfig; -import org.wikimedia.metrics_platform.context.ClientData; - -/** - * SamplingController: computes various sampling functions on the client - * - * Sampling is based on associative identifiers, each of which have a - * well-defined scope, and sampling config, which each stream provides as - * part of its configuration. - */ -@ThreadSafe -@ParametersAreNonnullByDefault -public class SamplingController { - - private final ClientData clientData; - private final SessionController sessionController; - - SamplingController(ClientData clientData, SessionController sessionController) { - this.clientData = clientData; - this.sessionController = sessionController; - } - - /** - * @param streamConfig stream config - * @return true if in sample or false otherwise - */ - boolean isInSample(StreamConfig streamConfig) { - if (!streamConfig.hasSampleConfig()) { - return true; - } - SampleConfig sampleConfig = streamConfig.getSampleConfig(); - if (sampleConfig.getRate() == 1.0) { - return true; - } - if (sampleConfig.getRate() == 0.0) { - return false; - } - return getSamplingValue(sampleConfig.getIdentifier()) < sampleConfig.getRate(); - } - - /** - * @param identifier identifier type from sampling config - * @return a floating point value between 0.0 and 1.0 (inclusive) - */ - double getSamplingValue(SampleConfig.Identifier identifier) { - String token = getSamplingId(identifier).substring(0, 8); - return (double) Long.parseLong(token, 16) / (double) 0xFFFFFFFFL; - } - - /** - * Returns the ID string to be used when evaluating presence in sample. - * The ID used is configured in stream config. - * - * @param identifier Identifier enum value - * @return the requested ID string - */ - @Nonnull - String getSamplingId(SampleConfig.Identifier identifier) { - switch (identifier) { - case SESSION: - return sessionController.getSessionId(); - case DEVICE: - return clientData.getAgentData().getAppInstallId(); - case PAGEVIEW: - return clientData.getPerformerData().getPageviewId(); - default: - throw new IllegalArgumentException("Bad identifier type: " + identifier); - } - } - -} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/SamplingController.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/SamplingController.kt new file mode 100644 index 00000000000..bed42a8f5aa --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/SamplingController.kt @@ -0,0 +1,57 @@ +package org.wikimedia.metrics_platform + +import org.wikimedia.metrics_platform.config.StreamConfig +import org.wikimedia.metrics_platform.config.sampling.SampleConfig +import org.wikimedia.metrics_platform.context.ClientData +import java.util.UUID + +/** + * SamplingController: computes various sampling functions on the client + * + * Sampling is based on associative identifiers, each of which have a + * well-defined scope, and sampling config, which each stream provides as + * part of its configuration. + */ +class SamplingController internal constructor( + private val clientData: ClientData, + private val sessionController: SessionController +) { + /** + * @param streamConfig stream config + * @return true if in sample or false otherwise + */ + fun isInSample(streamConfig: StreamConfig): Boolean { + if (!streamConfig.hasSampleConfig()) { + return true + } + val sampleConfig = streamConfig.sampleConfig!! + if (sampleConfig.rate == 1.0) { + return true + } + if (sampleConfig.rate == 0.0) { + return false + } + return getSamplingValue(sampleConfig.unit) < sampleConfig.rate + } + + fun getSamplingValue(unit: String): Double { + val token = getSamplingId(unit).substring(0, 8) + return token.toLong(16).toDouble() / 0xFFFFFFFFL.toDouble() + } + + /** + * Returns the ID string to be used when evaluating presence in sample. + * The ID used is configured in stream config. + * + * @param unit Identifier enum value + * @return the requested ID string + */ + fun getSamplingId(unit: String): String { + return when (unit) { + SampleConfig.UNIT_SESSION -> return sessionController.sessionId + SampleConfig.UNIT_DEVICE -> return clientData.agentData.appInstallId.orEmpty() + SampleConfig.UNIT_PAGEVIEW -> return clientData.performerData.pageviewId.orEmpty() + else -> UUID.randomUUID().toString() + } + } +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/SessionController.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/SessionController.java deleted file mode 100644 index 3eab9ff8e68..00000000000 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/SessionController.java +++ /dev/null @@ -1,80 +0,0 @@ -package org.wikimedia.metrics_platform; - -import java.security.SecureRandom; -import java.time.Duration; -import java.time.Instant; -import java.util.Locale; -import java.util.Random; - -import javax.annotation.Nonnull; -import javax.annotation.ParametersAreNonnullByDefault; -import javax.annotation.concurrent.GuardedBy; -import javax.annotation.concurrent.ThreadSafe; - -/** - * Manages sessions and session IDs for the Metrics Platform Client. - * - * A session begins when the application is launched and expires when the app is in the background - * for 30 minutes or more. - */ -@ThreadSafe -@ParametersAreNonnullByDefault -public class SessionController { - - private static final Duration SESSION_LENGTH = Duration.ofMinutes(30); - @GuardedBy("this") - @Nonnull - private String sessionId = generateSessionId(); - @GuardedBy("this") - @Nonnull - private Instant sessionTouched; - private static final Random RANDOM = new SecureRandom(); - - SessionController() { - this(Instant.now()); - } - - /** - * Constructor for testing. - * - * @param date session start time - */ - SessionController(Instant date) { - this.sessionTouched = date; - } - - @Nonnull - synchronized String getSessionId() { - return sessionId; - } - - synchronized void touchSession() { - if (sessionExpired()) { - sessionId = generateSessionId(); - } - - sessionTouched = Instant.now(); - } - - synchronized void beginSession() { - sessionId = generateSessionId(); - sessionTouched = Instant.now(); - } - - synchronized void closeSession() { - // @ToDo Determine how to close the session. - sessionTouched = Instant.now(); - } - - synchronized boolean sessionExpired() { - return Duration.between(sessionTouched, Instant.now()).compareTo(SESSION_LENGTH) >= 0; - } - - @Nonnull - private static String generateSessionId() { - Random random = RANDOM; - return String.format(Locale.US, "%08x", random.nextInt()) + - String.format(Locale.US, "%08x", random.nextInt()) + - String.format(Locale.US, "%04x", random.nextInt() & 0xFFFF); - } -} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/SessionController.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/SessionController.kt new file mode 100644 index 00000000000..c1b30844c78 --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/SessionController.kt @@ -0,0 +1,58 @@ +package org.wikimedia.metrics_platform + +import java.security.SecureRandom +import java.time.Duration +import java.time.Instant +import java.util.Locale +import java.util.Random + +/** + * Manages sessions and session IDs for the Metrics Platform Client. + * + * A session begins when the application is launched and expires when the app is in the background + * for 30 minutes or more. + */ +class SessionController internal constructor( + private var sessionTouched: Instant? = Instant.now() +) { + @get:Synchronized + var sessionId = generateSessionId() + private set + + @Synchronized + fun touchSession() { + if (sessionExpired()) { + sessionId = generateSessionId() + } + sessionTouched = Instant.now() + } + + @Synchronized + fun beginSession() { + sessionId = generateSessionId() + sessionTouched = Instant.now() + } + + @Synchronized + fun closeSession() { + // @ToDo Determine how to close the session. + sessionTouched = Instant.now() + } + + @Synchronized + fun sessionExpired(): Boolean { + return Duration.between(sessionTouched, Instant.now()) >= SESSION_LENGTH + } + + companion object { + private val SESSION_LENGTH = Duration.ofMinutes(30) + private val RANDOM = SecureRandom() + + private fun generateSessionId(): String { + val random: Random = RANDOM + return String.format(Locale.US, "%08x", random.nextInt()) + + String.format(Locale.US, "%08x", random.nextInt()) + + String.format(Locale.US, "%04x", random.nextInt() and 0xFFFF) + } + } +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/CurationFilter.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/CurationFilter.java deleted file mode 100644 index b75e9d5f02c..00000000000 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/CurationFilter.java +++ /dev/null @@ -1,86 +0,0 @@ -package org.wikimedia.metrics_platform.config; - -import java.time.Instant; -import java.util.function.Predicate; - -import org.wikimedia.metrics_platform.config.curation.CollectionCurationRules; -import org.wikimedia.metrics_platform.config.curation.ComparableCurationRules; -import org.wikimedia.metrics_platform.config.curation.CurationRules; -import org.wikimedia.metrics_platform.context.AgentData; -import org.wikimedia.metrics_platform.context.MediawikiData; -import org.wikimedia.metrics_platform.context.PageData; -import org.wikimedia.metrics_platform.context.PerformerData; -import org.wikimedia.metrics_platform.event.EventProcessed; - -import com.google.gson.annotations.SerializedName; - -public class CurationFilter implements Predicate { - @SerializedName("agent_app_install_id") CurationRules agentAppInstallIdRules; - @SerializedName("agent_client_platform") CurationRules agentClientPlatformRules; - @SerializedName("agent_client_platform_family") CurationRules agentClientPlatformFamilyRules; - - @SerializedName("mediawiki_database") CurationRules mediawikiDatabase; - - @SerializedName("page_id") ComparableCurationRules pageIdRules; - @SerializedName("page_namespace_id") ComparableCurationRules pageNamespaceIdRules; - @SerializedName("page_namespace_name") CurationRules pageNamespaceNameRules; - @SerializedName("page_title") CurationRules pageTitleRules; - @SerializedName("page_revision_id") ComparableCurationRules pageRevisionIdRules; - @SerializedName("page_wikidata_qid") CurationRules pageWikidataQidRules; - @SerializedName("page_content_language") CurationRules pageContentLanguageRules; - - @SerializedName("performer_id") ComparableCurationRules performerIdRules; - @SerializedName("performer_name") CurationRules performerNameRules; - @SerializedName("performer_session_id") CurationRules performerSessionIdRules; - @SerializedName("performer_pageview_id") CurationRules performerPageviewIdRules; - @SerializedName("performer_groups") CollectionCurationRules performerGroupsRules; - @SerializedName("performer_is_logged_in") CurationRules performerIsLoggedInRules; - @SerializedName("performer_is_temp") CurationRules performerIsTempRules; - @SerializedName("performer_registration_dt") ComparableCurationRules performerRegistrationDtRules; - @SerializedName("performer_language_groups") CurationRules performerLanguageGroupsRules; - @SerializedName("performer_language_primary") CurationRules performerLanguagePrimaryRules; - - public boolean test(EventProcessed event) { - return applyAgentRules(event.getAgentData()) - && applyMediaWikiRules(event.getMediawikiData()) - && applyPageRules(event.getPageData()) - && applyPerformerRules(event.getPerformerData()); - } - - private boolean applyAgentRules(@Nonnull AgentData data) { - return applyPredicate(this.agentAppInstallIdRules, data.getAppInstallId()) - && applyPredicate(this.agentClientPlatformRules, data.getClientPlatform()) - && applyPredicate(this.agentClientPlatformFamilyRules, data.getClientPlatformFamily()); - } - - private boolean applyMediaWikiRules(@Nonnull MediawikiData data) { - return applyPredicate(this.mediawikiDatabase, data.getDatabase()); - } - - private boolean applyPageRules(@Nonnull PageData data) { - return applyPredicate(this.pageIdRules, data.getId()) - && applyPredicate(this.pageNamespaceIdRules, data.getNamespaceId()) - && applyPredicate(this.pageNamespaceNameRules, data.getNamespaceName()) - && applyPredicate(this.pageTitleRules, data.getTitle()) - && applyPredicate(this.pageRevisionIdRules, data.getRevisionId()) - && applyPredicate(this.pageWikidataQidRules, data.getWikidataItemQid()) - && applyPredicate(this.pageContentLanguageRules, data.getContentLanguage()); - } - - private boolean applyPerformerRules(@Nonnull PerformerData data) { - return applyPredicate(this.performerIdRules, data.getId()) - && applyPredicate(this.performerNameRules, data.getName()) - && applyPredicate(this.performerSessionIdRules, data.getSessionId()) - && applyPredicate(this.performerPageviewIdRules, data.getPageviewId()) - && applyPredicate(this.performerGroupsRules, data.getGroups()) - && applyPredicate(this.performerIsLoggedInRules, data.getIsLoggedIn()) - && applyPredicate(this.performerIsTempRules, data.getIsTemp()) - && applyPredicate(this.performerRegistrationDtRules, data.getRegistrationDt()) - && applyPredicate(this.performerLanguageGroupsRules, data.getLanguageGroups()) - && applyPredicate(this.performerLanguagePrimaryRules, data.getLanguagePrimary()); - } - - private static boolean applyPredicate(@Nullable Predicate rules, @Nullable T value) { - return rules == null || rules.test(value); - } -} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/CurationFilter.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/CurationFilter.kt new file mode 100644 index 00000000000..62767522109 --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/CurationFilter.kt @@ -0,0 +1,98 @@ +@file:UseSerializers(InstantSerializer::class) + +package org.wikimedia.metrics_platform.config + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import kotlinx.serialization.UseSerializers +import org.wikimedia.metrics_platform.config.curation.CollectionCurationRules +import org.wikimedia.metrics_platform.config.curation.ComparableCurationRules +import org.wikimedia.metrics_platform.config.curation.CurationRules +import org.wikimedia.metrics_platform.context.AgentData +import org.wikimedia.metrics_platform.context.InstantSerializer +import org.wikimedia.metrics_platform.context.MediawikiData +import org.wikimedia.metrics_platform.context.PageData +import org.wikimedia.metrics_platform.context.PerformerData +import org.wikimedia.metrics_platform.event.EventProcessed +import java.time.Instant +import java.util.function.Predicate + +@Serializable +class CurationFilter : Predicate { + @SerialName("agent_app_install_id") var agentAppInstallIdRules: CurationRules? = null + @SerialName("agent_client_platform") var agentClientPlatformRules: CurationRules? = null + @SerialName("agent_client_platform_family") var agentClientPlatformFamilyRules: CurationRules? = null + @SerialName("mediawiki_database") var mediawikiDatabase: CurationRules? = null + @SerialName("page_id") var pageIdRules: ComparableCurationRules? = null + @SerialName("page_namespace_id") var pageNamespaceIdRules: ComparableCurationRules? = null + @SerialName("page_namespace_name") var pageNamespaceNameRules: CurationRules? = null + @SerialName("page_title") var pageTitleRules: CurationRules? = null + @SerialName("page_revision_id") var pageRevisionIdRules: ComparableCurationRules? = null + @SerialName("page_wikidata_qid") var pageWikidataQidRules: CurationRules? = null + @SerialName("page_content_language") var pageContentLanguageRules: CurationRules? = null + @SerialName("performer_id") var performerIdRules: ComparableCurationRules? = null + @SerialName("performer_name") var performerNameRules: CurationRules? = null + @SerialName("performer_session_id") var performerSessionIdRules: CurationRules? = null + @SerialName("performer_pageview_id") var performerPageviewIdRules: CurationRules? = null + @SerialName("performer_groups") var performerGroupsRules: CollectionCurationRules? = null + @SerialName("performer_is_logged_in") var performerIsLoggedInRules: CurationRules? = null + @SerialName("performer_is_temp") var performerIsTempRules: CurationRules? = null + @SerialName("performer_registration_dt") var performerRegistrationDtRules: ComparableCurationRules? = null + + @SerialName("performer_language_groups") + var performerLanguageGroupsRules: CurationRules? = null + + @SerialName("performer_language_primary") + var performerLanguagePrimaryRules: CurationRules? = null + + override fun test(event: EventProcessed): Boolean { + return applyAgentRules(event.agentData) + && applyMediaWikiRules(event.mediawikiData) + && applyPageRules(event.pageData) + && applyPerformerRules(event.performerData) + } + + private fun applyAgentRules(data: AgentData): Boolean { + return applyPredicate(this.agentAppInstallIdRules, data.appInstallId) + && applyPredicate(this.agentClientPlatformRules, data.clientPlatform) + && applyPredicate( + this.agentClientPlatformFamilyRules, + data.clientPlatformFamily + ) + } + + private fun applyMediaWikiRules(data: MediawikiData): Boolean { + return applyPredicate(this.mediawikiDatabase, data.database) + } + + private fun applyPageRules(data: PageData): Boolean { + return applyPredicate(this.pageIdRules, data.id) + && applyPredicate(this.pageNamespaceIdRules, data.namespaceId) + && applyPredicate(this.pageNamespaceNameRules, data.namespaceName) + && applyPredicate(this.pageTitleRules, data.title) + && applyPredicate(this.pageRevisionIdRules, data.revisionId) + && applyPredicate(this.pageWikidataQidRules, data.wikidataItemQid) + && applyPredicate(this.pageContentLanguageRules, data.contentLanguage) + } + + private fun applyPerformerRules(data: PerformerData): Boolean { + return applyPredicate(this.performerIdRules, data.id) + && applyPredicate(this.performerNameRules, data.name) + && applyPredicate(this.performerSessionIdRules, data.sessionId) + && applyPredicate(this.performerPageviewIdRules, data.pageviewId) + && applyPredicate(this.performerGroupsRules, data.groups) + && applyPredicate(this.performerIsLoggedInRules, data.isLoggedIn) + && applyPredicate(this.performerIsTempRules, data.isTemp) + && applyPredicate(this.performerRegistrationDtRules, data.registrationDt) + && applyPredicate(this.performerLanguageGroupsRules, data.languageGroups) + && applyPredicate(this.performerLanguagePrimaryRules, data.languagePrimary) + } + + companion object { + private fun applyPredicate(rules: Predicate?, value: T?): Boolean { + if (rules == null) return true + if (value == null) return false + return rules.test(value) + } + } +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/SourceConfig.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/SourceConfig.java deleted file mode 100644 index aafc52ace93..00000000000 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/SourceConfig.java +++ /dev/null @@ -1,63 +0,0 @@ -package org.wikimedia.metrics_platform.config; - -import static java.util.Collections.unmodifiableMap; -import static java.util.Collections.unmodifiableSet; -import static java.util.stream.Collectors.toSet; - -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -public class SourceConfig { - private final Map streamConfigs; - private final Set sourceConfigs; - - public SourceConfig(Map streamConfigs) { - this.streamConfigs = unmodifiableMap(streamConfigs); - this.sourceConfigs = unmodifiableSet(new HashSet<>(streamConfigs.values())); - } - - /** - * Get all stream configs' stream names. - */ - public Set getStreamNames() { - Set sourceConfigNamesSet = streamConfigs.keySet(); - return unmodifiableSet(sourceConfigNamesSet); - } - - /** - * Get all stream configs with stream name. - */ - public Map getStreamConfigsMap() { - return streamConfigs; - } - - /** - * Get all stream configs without stream name. - */ - public Set getStreamConfigsRaw() { - return sourceConfigs; - } - - /** - * Get stream config by stream name. - * - * @param streamName stream name - */ - public StreamConfig getStreamConfigByName(String streamName) { - return streamConfigs.get(streamName); - } - - /** - * Get stream names by event name. - * - * @param eventName event name - */ - public Set getStreamNamesByEvent(String eventName) { - return sourceConfigs.stream() - .filter(streamConfig -> streamConfig.isInterestedInEvent(eventName)) - .map(StreamConfig::getStreamName) - .collect(toSet()); - } - -} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/SourceConfig.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/SourceConfig.kt new file mode 100644 index 00000000000..733ad98721c --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/SourceConfig.kt @@ -0,0 +1,35 @@ +package org.wikimedia.metrics_platform.config + +class SourceConfig(streamConfigs: Map) { + /** + * Get all stream configs with stream name. + */ + val streamConfigsMap = streamConfigs + + /** + * Get all stream configs without stream name. + */ + val streamConfigsRaw = streamConfigs.values + + val streamNames get() = streamConfigsMap.keys + + /** + * Get stream config by stream name. + * + * @param streamName stream name + */ + fun getStreamConfigByName(streamName: String?): StreamConfig? { + return streamConfigsMap[streamName] + } + + /** + * Get stream names by event name. + * + * @param eventName event name + */ + fun getStreamNamesByEvent(eventName: String): Collection { + return streamConfigsRaw + .filter { streamConfig: StreamConfig -> streamConfig.isInterestedInEvent(eventName) } + .map { streamConfig: StreamConfig -> streamConfig.streamName } + } +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/StreamConfig.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/StreamConfig.java deleted file mode 100644 index 9d48f840e3e..00000000000 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/StreamConfig.java +++ /dev/null @@ -1,127 +0,0 @@ -package org.wikimedia.metrics_platform.config; - -import java.util.Collections; -import java.util.Set; - -import org.wikimedia.metrics_platform.config.sampling.SampleConfig; - -import com.google.gson.annotations.SerializedName; - -public class StreamConfig { - - @SerializedName("stream") String streamName; - - @SerializedName("schema_title") String schemaTitle; - - @SerializedName("destination_event_service") - DestinationEventService destinationEventService; - - @SerializedName("producers") ProducerConfig producerConfig; - - @SerializedName("sample") - SampleConfig sampleConfig; - - /** - * The context attributes that the Metrics Platform Client can add to an event. - */ - public static final String[] CONTEXTUAL_ATTRIBUTES = new String[] { - // Agent - "agent_app_install_id", - "agent_client_platform", - "agent_client_platform_family", - // Page - "page_id", - "page_title", - "page_namespace_id", - "page_namespace_name", - "page_revision_id", - "page_wikidata_qid", - "page_content_language", - // MediaWiki - "mediawiki_database", - // Performer - "performer_is_logged_in", - "performer_id", - "performer_name", - "performer_session_id", - "performer_pageview_id", - "performer_groups", - "performer_language_primary", - "performer_language_groups", - "performer_registration_dt", - }; - - public boolean hasRequestedContextValuesConfig() { - return producerConfig != null && - producerConfig.metricsPlatformClientConfig != null && - producerConfig.metricsPlatformClientConfig.requestedValues != null; - } - - public boolean hasSampleConfig() { - return producerConfig != null && - producerConfig.metricsPlatformClientConfig != null && - sampleConfig != null; - } - - /** - * Return whether this stream has any events it is interested in. - * - * @return true if the stream has events - */ - public boolean hasEvents() { - return producerConfig != null && - producerConfig.metricsPlatformClientConfig != null && - producerConfig.metricsPlatformClientConfig.events != null; - } - - /** - * Return the event names this stream is interested in. - * - * @return event names for the stream - */ - public Set getEvents() { - if (hasEvents()) { - return producerConfig.metricsPlatformClientConfig.events; - } - return Collections.emptySet(); - } - - public DestinationEventService getDestinationEventService() { - return destinationEventService != null ? destinationEventService : DestinationEventService.ANALYTICS; - } - - public boolean hasCurationFilter() { - return producerConfig != null && - producerConfig.metricsPlatformClientConfig != null && - producerConfig.metricsPlatformClientConfig.curationFilter != null; - } - - public CurationFilter getCurationFilter() { - if (hasCurationFilter()) { - return producerConfig.metricsPlatformClientConfig.curationFilter; - } - - return CurationFilter.builder().build(); - } - - public static class ProducerConfig { - @SerializedName("metrics_platform_client") - StreamConfig.MetricsPlatformClientConfig metricsPlatformClientConfig; - } - - public static class MetricsPlatformClientConfig { - @SerializedName("events") Set events; - @SerializedName("provide_values") Set requestedValues; - @SerializedName("curation") CurationFilter curationFilter; - } - - public boolean isInterestedInEvent(String eventName) { - for (String streamEventName : getEvents()) { - // Match string prefixes for event names of interested streams. - if (eventName.startsWith(streamEventName)) { - return true; - } - } - return false; - } -} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/StreamConfig.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/StreamConfig.kt new file mode 100644 index 00000000000..4e42b72e0e0 --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/StreamConfig.kt @@ -0,0 +1,86 @@ +package org.wikimedia.metrics_platform.config + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import org.wikimedia.metrics_platform.config.sampling.SampleConfig + +@Serializable +class StreamConfig { + @SerialName("stream") var streamName: String = "" + @SerialName("schema_title") var schemaTitle: String? = null + @SerialName("destination_event_service") var destinationEventService: DestinationEventService? = null + @SerialName("producers") var producerConfig: ProducerConfig? = null + @SerialName("sample") var sampleConfig: SampleConfig? = null + + fun hasRequestedContextValuesConfig(): Boolean { + return producerConfig?.metricsPlatformClientConfig?.requestedValues != null + } + + fun hasSampleConfig(): Boolean { + return producerConfig?.metricsPlatformClientConfig != null && sampleConfig != null + } + + val events + get() = producerConfig?.metricsPlatformClientConfig?.events.orEmpty() + + fun getDestinationEventService(): DestinationEventService { + return (if (destinationEventService != null) destinationEventService else DestinationEventService.ANALYTICS)!! + } + + fun hasCurationFilter(): Boolean { + return producerConfig?.metricsPlatformClientConfig?.curationFilter != null + } + + val curationFilter get() = producerConfig?.metricsPlatformClientConfig?.curationFilter ?: CurationFilter() + + @Serializable + class ProducerConfig { + @SerialName("metrics_platform_client") var metricsPlatformClientConfig: MetricsPlatformClientConfig? = null + } + + @Serializable + class MetricsPlatformClientConfig { + @SerialName("events") var events: List? = null + @SerialName("provide_values") var requestedValues: List? = null + @SerialName("curation") var curationFilter: CurationFilter? = null + } + + fun isInterestedInEvent(eventName: String): Boolean { + for (streamEventName in this.events) { + // Match string prefixes for event names of interested streams. + if (eventName.startsWith(streamEventName)) { + return true + } + } + return false + } + + companion object { + /** + * The context attributes that the Metrics Platform Client can add to an event. + */ + val CONTEXTUAL_ATTRIBUTES = arrayOf( + // Agent + "agent_app_install_id", + "agent_client_platform", + "agent_client_platform_family", // Page + "page_id", + "page_title", + "page_namespace_id", + "page_namespace_name", + "page_revision_id", + "page_wikidata_qid", + "page_content_language", // MediaWiki + "mediawiki_database", // Performer + "performer_is_logged_in", + "performer_id", + "performer_name", + "performer_session_id", + "performer_pageview_id", + "performer_groups", + "performer_language_primary", + "performer_language_groups", + "performer_registration_dt", + ) + } +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/StreamConfigCollection.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/StreamConfigCollection.java deleted file mode 100644 index 97f619fafc3..00000000000 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/StreamConfigCollection.java +++ /dev/null @@ -1,16 +0,0 @@ -package org.wikimedia.metrics_platform.config; - -import java.util.Map; - -import javax.annotation.ParametersAreNullableByDefault; -import javax.annotation.concurrent.ThreadSafe; - -import com.google.gson.annotations.SerializedName; - -import lombok.Value; - -@Value @ThreadSafe -@ParametersAreNullableByDefault -public class StreamConfigCollection { - @SerializedName("streams") public Map streamConfigs; -} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/StreamConfigCollection.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/StreamConfigCollection.kt new file mode 100644 index 00000000000..cf6352d75df --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/StreamConfigCollection.kt @@ -0,0 +1,9 @@ +package org.wikimedia.metrics_platform.config + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable + +@Serializable +class StreamConfigCollection { + @SerialName("streams") var streamConfigs: MutableMap? = null +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/CollectionCurationRules.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/CollectionCurationRules.java deleted file mode 100644 index 089aab66fb3..00000000000 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/CollectionCurationRules.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.wikimedia.metrics_platform.config.curation; - -import java.util.Collection; -import java.util.function.Predicate; - -import javax.annotation.ParametersAreNullableByDefault; - -import com.google.gson.annotations.SerializedName; - -import lombok.Builder; -import lombok.Value; - -@Builder @Value -@ParametersAreNullableByDefault -public class CollectionCurationRules implements Predicate> { - T contains; - @SerializedName("does_not_contain") T doesNotContain; - @SerializedName("contains_all") Collection containsAll; - @SerializedName("contains_any") Collection containsAny; - - @Override - public boolean test(Collection value) { - if (value == null) return false; - - return (contains == null || value.contains(contains)) - && (doesNotContain == null || !value.contains(doesNotContain)) - && (containsAll == null || value.containsAll(containsAll)) - && (containsAny == null || value.stream().filter(v -> containsAny.contains(v)).count() > 0); - } -} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/CollectionCurationRules.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/CollectionCurationRules.kt new file mode 100644 index 00000000000..53d8506cf08 --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/CollectionCurationRules.kt @@ -0,0 +1,24 @@ +package org.wikimedia.metrics_platform.config.curation + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import java.util.function.Predicate + +@Serializable +class CollectionCurationRules : Predicate?> { + var contains: T? = null + + @SerialName("does_not_contain") var doesNotContain: T? = null + + @SerialName("contains_all") var containsAll: MutableCollection? = null + + @SerialName("contains_any") var containsAny: MutableCollection? = null + + override fun test(value: MutableCollection?): Boolean { + if (value == null) return false + return (contains == null || value.contains(contains)) + && (doesNotContain == null || !value.contains(doesNotContain)) + && (containsAll == null || value.containsAll(containsAll!!)) + && (containsAny == null || value.count { v: T? -> containsAny!!.contains(v) } > 0) + } +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/ComparableCurationRules.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/ComparableCurationRules.java deleted file mode 100644 index 96a8495940e..00000000000 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/ComparableCurationRules.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.wikimedia.metrics_platform.config.curation; - -import java.util.Collection; -import java.util.function.Predicate; - -import javax.annotation.ParametersAreNullableByDefault; - -import com.google.gson.annotations.SerializedName; - -import lombok.Builder; -import lombok.Value; - -@Builder @Value -@ParametersAreNullableByDefault -public class ComparableCurationRules> implements Predicate { - @SerializedName("is_equals") T isEquals; - @SerializedName("not_equals") T isNotEquals; - @SerializedName("greater_than") T greaterThan; - @SerializedName("less_than") T lessThan; - @SerializedName("greater_than_or_equals") T greaterThanOrEquals; - @SerializedName("less_than_or_equals") T lessThanOrEquals; - Collection in; - @SerializedName("not_in") Collection notIn; - - @SuppressWarnings("CyclomaticComplexity") // Code is readable enough here - @Override - public boolean test(T value) { - if (value == null) return false; - - return (isEquals == null || isEquals.equals(value)) - && (isNotEquals == null || !isNotEquals.equals(value)) - && (greaterThan == null || greaterThan.compareTo(value) < 0) - && (lessThan == null || lessThan.compareTo(value) > 0) - && (greaterThanOrEquals == null || greaterThanOrEquals.compareTo(value) <= 0) - && (lessThanOrEquals == null || lessThanOrEquals.compareTo(value) >= 0) - && (notIn == null || !notIn.contains(value)); - } -} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/ComparableCurationRules.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/ComparableCurationRules.kt new file mode 100644 index 00000000000..fd8f23a7f99 --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/ComparableCurationRules.kt @@ -0,0 +1,36 @@ +package org.wikimedia.metrics_platform.config.curation + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import java.util.function.Predicate + +@Serializable +class ComparableCurationRules?> : Predicate { + @SerialName("is_equals") var isEquals: T? = null + + @SerialName("not_equals") var isNotEquals: T? = null + + @SerialName("greater_than") var greaterThan: T? = null + + @SerialName("less_than") var lessThan: T? = null + + @SerialName("greater_than_or_equals") var greaterThanOrEquals: T? = null + + @SerialName("less_than_or_equals") var lessThanOrEquals: T? = null + + @SerialName("in") var isIn: MutableCollection? = null + + @SerialName("not_in") var isNotIn: MutableCollection? = null + + override fun test(value: T): Boolean { + if (value == null) return false + return (isEquals == null || isEquals == value) + && (isNotEquals == null || isNotEquals != value) + && (greaterThan == null || greaterThan!!.compareTo(value) < 0) + && (lessThan == null || lessThan!!.compareTo(value) > 0) + && (greaterThanOrEquals == null || greaterThanOrEquals!!.compareTo(value) <= 0) + && (lessThanOrEquals == null || lessThanOrEquals!!.compareTo(value) >= 0) + && (isIn == null || isIn!!.contains(value)) + && (isNotIn == null || !isNotIn!!.contains(value)) + } +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/CurationRules.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/CurationRules.java deleted file mode 100644 index e5ce0f62d14..00000000000 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/CurationRules.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.wikimedia.metrics_platform.config.curation; - -import java.util.Collection; -import java.util.function.Predicate; - -import javax.annotation.ParametersAreNullableByDefault; - -import com.google.gson.annotations.SerializedName; - -import lombok.Builder; -import lombok.Value; - -@Builder @Value -@ParametersAreNullableByDefault -public class CurationRules implements Predicate { - @SerializedName("equals") T isEquals; - @SerializedName("not_equals") T isNotEquals; - Collection in; - @SerializedName("not_in") Collection notIn; - - @Override - public boolean test(T value) { - if (value == null) return false; - - return (isEquals == null || isEquals.equals(value)) - && (isNotEquals == null || !isNotEquals.equals(value)) - && (in == null || in.contains(value)) - && (notIn == null || !notIn.contains(value)); - } -} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/CurationRules.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/CurationRules.kt new file mode 100644 index 00000000000..ef352151547 --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/CurationRules.kt @@ -0,0 +1,24 @@ +package org.wikimedia.metrics_platform.config.curation + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import java.util.function.Predicate + +@Serializable +class CurationRules : Predicate { + @SerialName("equals") var isEquals: T? = null + + @SerialName("not_equals") var isNotEquals: T? = null + + @SerialName("in") var isIn: MutableCollection? = null + + @SerialName("not_in") var isNotIn: MutableCollection? = null + + override fun test(value: T?): Boolean { + if (value == null) return false + return (isEquals == null || isEquals == value) + && (isNotEquals == null || isNotEquals != value) + && (isIn == null || isIn!!.contains(value)) + && (isNotIn == null || !isNotIn!!.contains(value)) + } +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/sampling/SampleConfig.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/sampling/SampleConfig.java deleted file mode 100644 index 16b42bebd22..00000000000 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/sampling/SampleConfig.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.wikimedia.metrics_platform.config.sampling; - -import com.google.gson.annotations.SerializedName; - -public class SampleConfig { - - public enum Identifier { - @SerializedName("session") SESSION, - @SerializedName("device") DEVICE, - @SerializedName("pageview") PAGEVIEW - } - - /** Sampling rate. **/ - double rate; - - /** ID type to use for sampling. */ - Identifier identifier; -} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/sampling/SampleConfig.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/sampling/SampleConfig.kt new file mode 100644 index 00000000000..698c85ef586 --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/sampling/SampleConfig.kt @@ -0,0 +1,15 @@ +package org.wikimedia.metrics_platform.config.sampling + +import kotlinx.serialization.Serializable + +@Serializable +class SampleConfig( + val rate: Double = 1.0, + val unit: String = UNIT_SESSION +) { + companion object { + const val UNIT_PAGEVIEW = "pageview" + const val UNIT_SESSION = "session" + const val UNIT_DEVICE = "device" + } +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/PerformerData.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/PerformerData.kt index 683fb813f39..60dc1db928c 100644 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/PerformerData.kt +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/PerformerData.kt @@ -15,7 +15,7 @@ import java.time.Instant */ @Serializable class PerformerData ( - private val id: Int? = null, + val id: Int? = null, @SerialName("name") val name: String? = null, @SerialName("is_logged_in") val isLoggedIn: Boolean? = null, @SerialName("is_temp") val isTemp: Boolean? = null, diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/event/Event.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/event/Event.java deleted file mode 100644 index dd88af9cb5d..00000000000 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/event/Event.java +++ /dev/null @@ -1,65 +0,0 @@ -package org.wikimedia.metrics_platform.event; - -import java.util.Map; - -import org.wikimedia.metrics_platform.config.sampling.SampleConfig; -import org.wikimedia.metrics_platform.context.ClientData; -import org.wikimedia.metrics_platform.context.InteractionData; -import org.wikimedia.metrics_platform.utils.Objects; - -import com.google.gson.annotations.SerializedName; - -public class Event { - @SerializedName("$schema") protected String schema; - @SerializedName("name") protected final String name; - @SerializedName("dt") protected String timestamp; - @SerializedName("custom_data") protected Map customData; - protected final Meta meta; - @SerializedName("client_data") protected ClientData clientData; - @SerializedName("sample") protected SampleConfig sample; - @SerializedName("interaction_data") protected InteractionData interactionData; - - public Event(String schema, String stream, String name) { - this.schema = schema; - this.meta = new Meta(stream); - this.name = name; - } - - @Nullable - public String getStream() { - return meta.getStream(); - } - - public void setDomain(String domain) { - meta.domain = domain; - } - - @Nonnull - public ClientData getClientData() { - clientData = Objects.firstNonNull(clientData, ClientData::new); - return clientData; - } - - public void setCustomData(@Nonnull Map customData) { - this.customData = customData; - } - - public void setSample(@Nonnull SampleConfig sample) { - this.sample = sample; - } - - @Nonnull - public InteractionData getInteractionData() { - interactionData = Objects.firstNonNull(interactionData, InteractionData::new); - return interactionData; - } - - public void setInteractionData(@Nonnull InteractionData interactionData) { - this.interactionData = interactionData; - } - - protected static final class Meta { - private final String stream; - private String domain; - } -} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/event/Event.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/event/Event.kt new file mode 100644 index 00000000000..0bc8232aa05 --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/event/Event.kt @@ -0,0 +1,40 @@ +package org.wikimedia.metrics_platform.event + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import kotlinx.serialization.Transient +import org.wikimedia.metrics_platform.config.sampling.SampleConfig +import org.wikimedia.metrics_platform.context.ClientData +import org.wikimedia.metrics_platform.context.InteractionData + +@Serializable +open class Event(@Transient val _stream: String = "") { + val name: String? = null + + @SerialName("\$schema") + var schema: String? = null + + @SerialName("dt") protected var timestamp: String? = null + + @SerialName("custom_data") var customData: MutableMap? = null + + protected val meta = Meta(_stream) + + @SerialName("client_data") var clientData: ClientData = ClientData() + + @SerialName("sample") var sample: SampleConfig? = null + + @SerialName("interaction_data") var interactionData: InteractionData = InteractionData() + + val stream: String get() = meta.stream + + fun setDomain(domain: String?) { + meta.domain = domain + } + + @Serializable + protected class Meta( + val stream: String = "", + var domain: String? = null + ) +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/event/EventProcessed.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/event/EventProcessed.java deleted file mode 100644 index 69b2a330865..00000000000 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/event/EventProcessed.java +++ /dev/null @@ -1,170 +0,0 @@ -package org.wikimedia.metrics_platform.event; - -import static org.wikimedia.metrics_platform.utils.Objects.firstNonNull; - -import java.util.Map; - -import org.wikimedia.metrics_platform.config.sampling.SampleConfig; -import org.wikimedia.metrics_platform.context.AgentData; -import org.wikimedia.metrics_platform.context.ClientData; -import org.wikimedia.metrics_platform.context.InteractionData; -import org.wikimedia.metrics_platform.context.MediawikiData; -import org.wikimedia.metrics_platform.context.PageData; -import org.wikimedia.metrics_platform.context.PerformerData; - -import com.google.gson.annotations.SerializedName; - -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.Setter; - -@EqualsAndHashCode -@Getter -@Setter -public class EventProcessed extends Event { - @SerializedName("agent") private AgentData agentData; - @SerializedName("page") private PageData pageData; - @SerializedName("mediawiki") private MediawikiData mediawikiData; - @SerializedName("performer") private PerformerData performerData; - @Nonnull - @SerializedName("action") private String action; - @SerializedName("action_subtype") private String actionSubtype; - @SerializedName("action_source") private String actionSource; - @SerializedName("action_context") private String actionContext; - @SerializedName("element_id") private String elementId; - @SerializedName("element_friendly_name") private String elementFriendlyName; - @SerializedName("funnel_entry_token") private String funnelEntryToken; - @SerializedName("funnel_event_sequence_position") private Integer funnelEventSequencePosition; - - /** - * Constructor for EventProcessed. - * - * @param schema schema id - * @param stream stream name - * @param name event name - * @param clientData agent, mediawiki, page, performer data - */ - public EventProcessed( - String schema, - String stream, - @Nonnull String name, - ClientData clientData - ) { - super(schema, stream, name); - this.clientData = clientData; - this.agentData = clientData.getAgentData(); - this.pageData = clientData.getPageData(); - this.mediawikiData = clientData.getMediawikiData(); - this.performerData = clientData.getPerformerData(); - this.action = name; - } - - /** - * Constructor for EventProcessed. - * - * @param schema schema id - * @param stream stream name - * @param name event name - * @param customData custom data - * @param clientData agent, mediawiki, page, performer data - * @param sample sample configuration - * @param interactionData contextual data of the interaction - *

- * Although 'setInteractionData()' sets the 'action' property for the event, - * because 'action' is a nonnull property for both 'EventProcessed' and the - * 'InteractionData' data object, removing the redundant setting of 'action' - * triggers NP_NONNULL_FIELD_NOT_INITIALIZED_IN_CONSTRUCTOR in spotbugs so - * leaving it as is for the time being rather than suppressing the error. - */ - public EventProcessed( - String schema, - String stream, - String name, - Map customData, - ClientData clientData, - SampleConfig sample, - InteractionData interactionData - ) { - super(schema, stream, name); - this.clientData = clientData; - this.agentData = clientData.getAgentData(); - this.pageData = clientData.getPageData(); - this.mediawikiData = clientData.getMediawikiData(); - this.performerData = clientData.getPerformerData(); - this.setCustomData(customData); - this.sample = sample; - this.setInteractionData(interactionData); - this.action = interactionData.getAction(); - } - - @Nonnull - public static EventProcessed fromEvent(Event event) { - return new EventProcessed( - event.getSchema(), - event.getStream(), - event.getName(), - event.getCustomData(), - event.getClientData(), - event.getSample(), - event.getInteractionData() - ); - } - - @Nonnull - public AgentData getAgentData() { - agentData = firstNonNull(agentData, AgentData.NULL_AGENT_DATA); - return agentData; - } - - @Nonnull - public PageData getPageData() { - pageData = firstNonNull(pageData, PageData.NULL_PAGE_DATA); - return pageData; - } - - @Nonnull - public MediawikiData getMediawikiData() { - mediawikiData = firstNonNull(mediawikiData, MediawikiData.NULL_MEDIAWIKI_DATA); - return mediawikiData; - } - - @Nonnull - public PerformerData getPerformerData() { - performerData = firstNonNull(performerData, PerformerData.NULL_PERFORMER_DATA); - return performerData; - } - - @Override - public void setClientData(@Nonnull ClientData clientData) { - setAgentData(clientData.getAgentData()); - setPageData(clientData.getPageData()); - setMediawikiData(clientData.getMediawikiData()); - setPerformerData(clientData.getPerformerData()); - this.clientData = clientData; - } - - @Override - public final void setInteractionData(@Nonnull InteractionData interactionData) { - this.action = interactionData.getAction(); - this.actionContext = interactionData.getActionContext(); - this.actionSource = interactionData.getActionSource(); - this.actionSubtype = interactionData.getActionSubtype(); - this.elementId = interactionData.getElementId(); - this.elementFriendlyName = interactionData.getElementFriendlyName(); - this.funnelEntryToken = interactionData.getFunnelEntryToken(); - this.funnelEventSequencePosition = interactionData.getFunnelEventSequencePosition(); - } - - public void setAgentData(AgentData agentData) { - this.agentData = agentData; - } - public void setPageData(PageData pageData) { - this.pageData = pageData; - } - public void setMediawikiData(MediawikiData mediawikiData) { - this.mediawikiData = mediawikiData; - } - public void setPerformerData(PerformerData performerData) { - this.performerData = performerData; - } -} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/event/EventProcessed.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/event/EventProcessed.kt new file mode 100644 index 00000000000..dbe573d511d --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/event/EventProcessed.kt @@ -0,0 +1,120 @@ +package org.wikimedia.metrics_platform.event + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import org.wikimedia.metrics_platform.config.sampling.SampleConfig +import org.wikimedia.metrics_platform.context.AgentData +import org.wikimedia.metrics_platform.context.ClientData +import org.wikimedia.metrics_platform.context.InteractionData +import org.wikimedia.metrics_platform.context.MediawikiData +import org.wikimedia.metrics_platform.context.PageData +import org.wikimedia.metrics_platform.context.PerformerData + +@Serializable +class EventProcessed : Event { + @SerialName("agent") var agentData: AgentData = AgentData() + @SerialName("page") var pageData: PageData = PageData() + @SerialName("mediawiki") var mediawikiData: MediawikiData = MediawikiData() + @SerialName("performer") var performerData: PerformerData = PerformerData() + @SerialName("action") var action: String + @SerialName("action_subtype") private var actionSubtype: String? = null + @SerialName("action_source") private var actionSource: String? = null + @SerialName("action_context") private var actionContext: String? = null + @SerialName("element_id") private var elementId: String? = null + @SerialName("element_friendly_name") private var elementFriendlyName: String? = null + @SerialName("funnel_entry_token") private var funnelEntryToken: String? = null + @SerialName("funnel_event_sequence_position") private var funnelEventSequencePosition: Int? = null + + /** + * Constructor for EventProcessed. + * + * @param schema schema id + * @param stream stream name + * @param name event name + * @param clientData agent, mediawiki, page, performer data + */ + constructor( + schema: String?, + stream: String?, + @Nonnull name: String?, + clientData: ClientData + ) : super(schema!!, stream, name) { + this.clientData = clientData + this.agentData = clientData.agentData + this.pageData = clientData.pageData + this.mediawikiData = clientData.mediawikiData + this.performerData = clientData.performerData + this.action = name + } + + /** + * Constructor for EventProcessed. + * + * @param schema schema id + * @param stream stream name + * @param name event name + * @param customData custom data + * @param clientData agent, mediawiki, page, performer data + * @param sample sample configuration + * @param interactionData contextual data of the interaction + * + * + * Although 'setInteractionData()' sets the 'action' property for the event, + * because 'action' is a nonnull property for both 'EventProcessed' and the + * 'InteractionData' data object, removing the redundant setting of 'action' + * triggers NP_NONNULL_FIELD_NOT_INITIALIZED_IN_CONSTRUCTOR in spotbugs so + * leaving it as is for the time being rather than suppressing the error. + */ + constructor( + schema: String?, + stream: String?, + name: String?, + customData: MutableMap?, + clientData: ClientData, + sample: SampleConfig?, + interactionData: InteractionData + ) : super(schema!!, stream, name) { + this.clientData = clientData + this.agentData = clientData.agentData + this.pageData = clientData.pageData + this.mediawikiData = clientData.mediawikiData + this.performerData = clientData.performerData + this.customData = customData + this.sample = sample + this.interactionData = interactionData + this.action = interactionData.action + } + + public override fun setClientData(@Nonnull clientData: ClientData) { + setAgentData(clientData.agentData) + setPageData(clientData.pageData) + setMediawikiData(clientData.mediawikiData) + setPerformerData(clientData.performerData) + this.clientData = clientData + } + + public override fun setInteractionData(@Nonnull interactionData: InteractionData) { + this.action = interactionData.action + this.actionContext = interactionData.actionContext + this.actionSource = interactionData.actionSource + this.actionSubtype = interactionData.actionSubtype + this.elementId = interactionData.elementId + this.elementFriendlyName = interactionData.elementFriendlyName + this.funnelEntryToken = interactionData.funnelEntryToken + this.funnelEventSequencePosition = interactionData.funnelEventSequencePosition + } + + companion object { + fun fromEvent(event: Event): EventProcessed { + return EventProcessed( + event.schema, + event.stream, + event.name, + event.customData, + event.clientData, + event.sample, + event.interactionData + ) + } + } +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/utils/Objects.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/utils/Objects.java deleted file mode 100644 index 8e67f72bf29..00000000000 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/utils/Objects.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.wikimedia.metrics_platform.utils; - -import java.util.function.Supplier; - -public final class Objects { - - private Objects() { - // Utility class, should not be instantiated - } - - @Nonnull - public static T firstNonNull(@Nullable T first, @Nonnull T second) { - if (first != null) return first; - return second; - } - - @Nonnull - public static T firstNonNull(@Nullable T first, @Nonnull Supplier second) { - if (first != null) return first; - T result = second.get(); - if (result == null) throw new NullPointerException("'second' parameter should always create a non null object."); - return result; - } -} From fe954e41b46bf8a3b40b97a7d41e6f97d24031b0 Mon Sep 17 00:00:00 2001 From: Dmitry Brant Date: Mon, 30 Jun 2025 20:05:16 -0400 Subject: [PATCH 3/9] Rename. --- analytics/metrics-platform/build.gradle | 2 +- .../metrics_platform/ContextController.java | 199 ------- .../metrics_platform/EventProcessor.java | 122 ---- .../metrics_platform/EventSenderDefault.java | 53 -- .../metrics_platform/MetricsClient.java | 527 ------------------ .../config/ConfigFetcherRunnable.java | 48 -- .../config/DestinationEventService.java | 38 -- .../metrics_platform/context/AgentData.kt | 24 - .../metrics_platform/context/PerformerData.kt | 41 -- .../metrics_platform/json/GsonHelper.java | 23 - .../metricsplatform/ContextController.kt | 94 ++++ .../CurationController.kt | 6 +- .../metricsplatform/EventProcessor.kt | 116 ++++ .../EventSender.kt | 4 +- .../metricsplatform/EventSenderDefault.kt | 34 ++ .../metricsplatform/MetricsClient.kt | 493 ++++++++++++++++ .../SamplingController.kt | 8 +- .../SessionController.kt | 2 +- .../config/CurationFilter.kt | 20 +- .../config/DestinationEventService.kt | 31 ++ .../config/SourceConfig.kt | 2 +- .../config/StreamConfig.kt | 6 +- .../config/StreamConfigCollection.kt | 2 +- .../config/StreamConfigFetcher.java | 2 +- .../curation/CollectionCurationRules.kt | 2 +- .../curation/ComparableCurationRules.kt | 2 +- .../config/curation/CurationRules.kt | 2 +- .../config/sampling/SampleConfig.kt | 2 +- .../metricsplatform/context/AgentData.kt | 24 + .../context/ClientData.kt | 2 +- .../context/ContextValue.kt | 2 +- .../context/CustomData.kt | 2 +- .../context/CustomDataType.kt | 2 +- .../context/InstantSerializer.kt | 2 +- .../context/InteractionData.kt | 2 +- .../context/MediawikiData.kt | 4 +- .../context/PageData.kt | 16 +- .../metricsplatform/context/PerformerData.kt | 28 + .../event/Event.kt | 18 +- .../event/EventProcessed.kt | 38 +- .../event/EventProcessedSerializer.java | 41 +- .../ConsistencyIT.java | 22 +- .../ConsistencyITClientData.java | 12 +- .../ContextControllerTest.java | 24 +- .../CurationControllerTest.java | 16 +- .../DestinationEventServiceTest.java | 4 +- .../EndToEndIT.java | 10 +- .../EventProcessorTest.java | 14 +- .../EventTest.java | 18 +- .../MetricsClientTest.java | 34 +- .../SamplingControllerTest.java | 12 +- .../SessionControllerTest.java | 2 +- .../TestEventSender.java | 4 +- .../config/CurationFilterFixtures.java | 6 +- .../config/SampleConfigTest.java | 8 +- .../config/SourceConfigFixtures.java | 2 +- .../config/SourceConfigTest.java | 2 +- .../config/StreamConfigFetcherTest.java | 8 +- .../config/StreamConfigFixtures.java | 42 +- .../config/StreamConfigIT.java | 4 +- .../config/StreamConfigTest.java | 6 +- .../context/AgentDataTest.java | 4 +- .../context/CustomDataTest.java | 6 +- .../context/DataFixtures.java | 4 +- .../context/MediawikiDataTest.java | 4 +- .../context/PageDataTest.java | 4 +- .../context/PerformerDataTest.java | 4 +- .../curation/CurationFilterFixtures.java | 6 +- .../event/EventFixtures.java | 12 +- .../config/streamconfigs-local.json | 0 .../config/streamconfigs.json | 0 .../event/expected_event_click.json | 0 .../event/expected_event_click_custom.json | 0 .../event/expected_event_interaction.json | 0 .../event/expected_event_view.json | 0 app/build.gradle | 2 + .../analytics/metricsplatform/ArticleEvent.kt | 2 +- .../analytics/metricsplatform/MetricsEvent.kt | 8 +- .../metricsplatform/MetricsPlatform.kt | 8 +- 79 files changed, 1059 insertions(+), 1341 deletions(-) delete mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/ContextController.java delete mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/EventProcessor.java delete mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/EventSenderDefault.java delete mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/MetricsClient.java delete mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/ConfigFetcherRunnable.java delete mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/DestinationEventService.java delete mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/AgentData.kt delete mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/PerformerData.kt delete mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/json/GsonHelper.java create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/ContextController.kt rename analytics/metrics-platform/src/main/java/org/wikimedia/{metrics_platform => metricsplatform}/CurationController.kt (60%) create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/EventProcessor.kt rename analytics/metrics-platform/src/main/java/org/wikimedia/{metrics_platform => metricsplatform}/EventSender.kt (75%) create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/EventSenderDefault.kt create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/MetricsClient.kt rename analytics/metrics-platform/src/main/java/org/wikimedia/{metrics_platform => metricsplatform}/SamplingController.kt (88%) rename analytics/metrics-platform/src/main/java/org/wikimedia/{metrics_platform => metricsplatform}/SessionController.kt (97%) rename analytics/metrics-platform/src/main/java/org/wikimedia/{metrics_platform => metricsplatform}/config/CurationFilter.kt (88%) create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/DestinationEventService.kt rename analytics/metrics-platform/src/main/java/org/wikimedia/{metrics_platform => metricsplatform}/config/SourceConfig.kt (95%) rename analytics/metrics-platform/src/main/java/org/wikimedia/{metrics_platform => metricsplatform}/config/StreamConfig.kt (92%) rename analytics/metrics-platform/src/main/java/org/wikimedia/{metrics_platform => metricsplatform}/config/StreamConfigCollection.kt (82%) rename analytics/metrics-platform/src/main/java/org/wikimedia/{metrics_platform => metricsplatform}/config/StreamConfigFetcher.java (97%) rename analytics/metrics-platform/src/main/java/org/wikimedia/{metrics_platform => metricsplatform}/config/curation/CollectionCurationRules.kt (94%) rename analytics/metrics-platform/src/main/java/org/wikimedia/{metrics_platform => metricsplatform}/config/curation/ComparableCurationRules.kt (96%) rename analytics/metrics-platform/src/main/java/org/wikimedia/{metrics_platform => metricsplatform}/config/curation/CurationRules.kt (93%) rename analytics/metrics-platform/src/main/java/org/wikimedia/{metrics_platform => metricsplatform}/config/sampling/SampleConfig.kt (84%) create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/AgentData.kt rename analytics/metrics-platform/src/main/java/org/wikimedia/{metrics_platform => metricsplatform}/context/ClientData.kt (94%) rename analytics/metrics-platform/src/main/java/org/wikimedia/{metrics_platform => metricsplatform}/context/ContextValue.kt (97%) rename analytics/metrics-platform/src/main/java/org/wikimedia/{metrics_platform => metricsplatform}/context/CustomData.kt (95%) rename analytics/metrics-platform/src/main/java/org/wikimedia/{metrics_platform => metricsplatform}/context/CustomDataType.kt (81%) rename analytics/metrics-platform/src/main/java/org/wikimedia/{metrics_platform => metricsplatform}/context/InstantSerializer.kt (93%) rename analytics/metrics-platform/src/main/java/org/wikimedia/{metrics_platform => metricsplatform}/context/InteractionData.kt (94%) rename analytics/metrics-platform/src/main/java/org/wikimedia/{metrics_platform => metricsplatform}/context/MediawikiData.kt (76%) rename analytics/metrics-platform/src/main/java/org/wikimedia/{metrics_platform => metricsplatform}/context/PageData.kt (56%) create mode 100644 analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/PerformerData.kt rename analytics/metrics-platform/src/main/java/org/wikimedia/{metrics_platform => metricsplatform}/event/Event.kt (64%) rename analytics/metrics-platform/src/main/java/org/wikimedia/{metrics_platform => metricsplatform}/event/EventProcessed.kt (81%) rename analytics/metrics-platform/src/main/java/org/wikimedia/{metrics_platform => metricsplatform}/event/EventProcessedSerializer.java (74%) rename analytics/metrics-platform/src/test/java/org/wikimedia/{metrics_platform => metricsplatform}/ConsistencyIT.java (88%) rename analytics/metrics-platform/src/test/java/org/wikimedia/{metrics_platform => metricsplatform}/ConsistencyITClientData.java (92%) rename analytics/metrics-platform/src/test/java/org/wikimedia/{metrics_platform => metricsplatform}/ContextControllerTest.java (79%) rename analytics/metrics-platform/src/test/java/org/wikimedia/{metrics_platform => metricsplatform}/CurationControllerTest.java (89%) rename analytics/metrics-platform/src/test/java/org/wikimedia/{metrics_platform => metricsplatform}/DestinationEventServiceTest.java (94%) rename analytics/metrics-platform/src/test/java/org/wikimedia/{metrics_platform => metricsplatform}/EndToEndIT.java (96%) rename analytics/metrics-platform/src/test/java/org/wikimedia/{metrics_platform => metricsplatform}/EventProcessorTest.java (94%) rename analytics/metrics-platform/src/test/java/org/wikimedia/{metrics_platform => metricsplatform}/EventTest.java (93%) rename analytics/metrics-platform/src/test/java/org/wikimedia/{metrics_platform => metricsplatform}/MetricsClientTest.java (91%) rename analytics/metrics-platform/src/test/java/org/wikimedia/{metrics_platform => metricsplatform}/SamplingControllerTest.java (81%) rename analytics/metrics-platform/src/test/java/org/wikimedia/{metrics_platform => metricsplatform}/SessionControllerTest.java (96%) rename analytics/metrics-platform/src/test/java/org/wikimedia/{metrics_platform => metricsplatform}/TestEventSender.java (83%) rename analytics/metrics-platform/src/test/java/org/wikimedia/{metrics_platform => metricsplatform}/config/CurationFilterFixtures.java (78%) rename analytics/metrics-platform/src/test/java/org/wikimedia/{metrics_platform => metricsplatform}/config/SampleConfigTest.java (67%) rename analytics/metrics-platform/src/test/java/org/wikimedia/{metrics_platform => metricsplatform}/config/SourceConfigFixtures.java (93%) rename analytics/metrics-platform/src/test/java/org/wikimedia/{metrics_platform => metricsplatform}/config/SourceConfigTest.java (96%) rename analytics/metrics-platform/src/test/java/org/wikimedia/{metrics_platform => metricsplatform}/config/StreamConfigFetcherTest.java (88%) rename analytics/metrics-platform/src/test/java/org/wikimedia/{metrics_platform => metricsplatform}/config/StreamConfigFixtures.java (72%) rename analytics/metrics-platform/src/test/java/org/wikimedia/{metrics_platform => metricsplatform}/config/StreamConfigIT.java (93%) rename analytics/metrics-platform/src/test/java/org/wikimedia/{metrics_platform => metricsplatform}/config/StreamConfigTest.java (87%) rename analytics/metrics-platform/src/test/java/org/wikimedia/{metrics_platform => metricsplatform}/context/AgentDataTest.java (95%) rename analytics/metrics-platform/src/test/java/org/wikimedia/{metrics_platform => metricsplatform}/context/CustomDataTest.java (89%) rename analytics/metrics-platform/src/test/java/org/wikimedia/{metrics_platform => metricsplatform}/context/DataFixtures.java (97%) rename analytics/metrics-platform/src/test/java/org/wikimedia/{metrics_platform => metricsplatform}/context/MediawikiDataTest.java (85%) rename analytics/metrics-platform/src/test/java/org/wikimedia/{metrics_platform => metricsplatform}/context/PageDataTest.java (92%) rename analytics/metrics-platform/src/test/java/org/wikimedia/{metrics_platform => metricsplatform}/context/PerformerDataTest.java (95%) rename analytics/metrics-platform/src/test/java/org/wikimedia/{metrics_platform => metricsplatform}/curation/CurationFilterFixtures.java (81%) rename analytics/metrics-platform/src/test/java/org/wikimedia/{metrics_platform => metricsplatform}/event/EventFixtures.java (83%) rename analytics/metrics-platform/src/test/resources/org/wikimedia/{metrics_platform => metricsplatform}/config/streamconfigs-local.json (100%) rename analytics/metrics-platform/src/test/resources/org/wikimedia/{metrics_platform => metricsplatform}/config/streamconfigs.json (100%) rename analytics/metrics-platform/src/test/resources/org/wikimedia/{metrics_platform => metricsplatform}/event/expected_event_click.json (100%) rename analytics/metrics-platform/src/test/resources/org/wikimedia/{metrics_platform => metricsplatform}/event/expected_event_click_custom.json (100%) rename analytics/metrics-platform/src/test/resources/org/wikimedia/{metrics_platform => metricsplatform}/event/expected_event_interaction.json (100%) rename analytics/metrics-platform/src/test/resources/org/wikimedia/{metrics_platform => metricsplatform}/event/expected_event_view.json (100%) diff --git a/analytics/metrics-platform/build.gradle b/analytics/metrics-platform/build.gradle index 17cbbf08ae1..bf702040879 100644 --- a/analytics/metrics-platform/build.gradle +++ b/analytics/metrics-platform/build.gradle @@ -8,7 +8,7 @@ plugins { final JavaVersion JAVA_VERSION = JavaVersion.VERSION_17 android { - namespace 'org.wikimedia.metrics_platform' + namespace 'org.wikimedia.metricsplatform' compileOptions { coreLibraryDesugaringEnabled true diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/ContextController.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/ContextController.java deleted file mode 100644 index 72d01973a7a..00000000000 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/ContextController.java +++ /dev/null @@ -1,199 +0,0 @@ -package org.wikimedia.metrics_platform; - -import static org.wikimedia.metrics_platform.context.ContextValue.AGENT_APP_INSTALL_ID; -import static org.wikimedia.metrics_platform.context.ContextValue.AGENT_CLIENT_PLATFORM; -import static org.wikimedia.metrics_platform.context.ContextValue.AGENT_CLIENT_PLATFORM_FAMILY; -import static org.wikimedia.metrics_platform.context.ContextValue.AGENT_APP_FLAVOR; -import static org.wikimedia.metrics_platform.context.ContextValue.AGENT_APP_THEME; -import static org.wikimedia.metrics_platform.context.ContextValue.AGENT_APP_VERSION; -import static org.wikimedia.metrics_platform.context.ContextValue.AGENT_APP_VERSION_NAME; -import static org.wikimedia.metrics_platform.context.ContextValue.AGENT_DEVICE_FAMILY; -import static org.wikimedia.metrics_platform.context.ContextValue.AGENT_DEVICE_LANGUAGE; -import static org.wikimedia.metrics_platform.context.ContextValue.AGENT_RELEASE_STATUS; -import static org.wikimedia.metrics_platform.context.ContextValue.MEDIAWIKI_DATABASE; -import static org.wikimedia.metrics_platform.context.ContextValue.PAGE_ID; -import static org.wikimedia.metrics_platform.context.ContextValue.PAGE_TITLE; -import static org.wikimedia.metrics_platform.context.ContextValue.PAGE_NAMESPACE_ID; -import static org.wikimedia.metrics_platform.context.ContextValue.PAGE_NAMESPACE_NAME; -import static org.wikimedia.metrics_platform.context.ContextValue.PAGE_REVISION_ID; -import static org.wikimedia.metrics_platform.context.ContextValue.PAGE_WIKIDATA_QID; -import static org.wikimedia.metrics_platform.context.ContextValue.PAGE_CONTENT_LANGUAGE; -import static org.wikimedia.metrics_platform.context.ContextValue.PERFORMER_ID; -import static org.wikimedia.metrics_platform.context.ContextValue.PERFORMER_NAME; -import static org.wikimedia.metrics_platform.context.ContextValue.PERFORMER_IS_LOGGED_IN; -import static org.wikimedia.metrics_platform.context.ContextValue.PERFORMER_IS_TEMP; -import static org.wikimedia.metrics_platform.context.ContextValue.PERFORMER_SESSION_ID; -import static org.wikimedia.metrics_platform.context.ContextValue.PERFORMER_PAGEVIEW_ID; -import static org.wikimedia.metrics_platform.context.ContextValue.PERFORMER_GROUPS; -import static org.wikimedia.metrics_platform.context.ContextValue.PERFORMER_LANGUAGE_GROUPS; -import static org.wikimedia.metrics_platform.context.ContextValue.PERFORMER_LANGUAGE_PRIMARY; -import static org.wikimedia.metrics_platform.context.ContextValue.PERFORMER_REGISTRATION_DT; - -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Locale; -import java.util.Set; - -import javax.annotation.ParametersAreNonnullByDefault; -import javax.annotation.concurrent.ThreadSafe; - -import org.wikimedia.metrics_platform.config.StreamConfig; -import org.wikimedia.metrics_platform.context.AgentData; -import org.wikimedia.metrics_platform.context.ClientData; -import org.wikimedia.metrics_platform.context.MediawikiData; -import org.wikimedia.metrics_platform.context.PageData; -import org.wikimedia.metrics_platform.context.PerformerData; -import org.wikimedia.metrics_platform.event.EventProcessed; - -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; - -@ThreadSafe -@ParametersAreNonnullByDefault -public class ContextController { - - /** - * @see Metrics Platform/Contextual attributes - */ - private static final Collection REQUIRED_PROPERTIES = List.of( - "agent_app_flavor", - "agent_app_install_id", - "agent_app_theme", - "agent_app_version", - "agent_app_version_name", - "agent_client_platform", - "agent_client_platform_family", - "agent_device_family", - "agent_device_language", - "agent_release_status" - ); - - public void enrichEvent(EventProcessed event, StreamConfig streamConfig) { - if (!streamConfig.hasRequestedContextValuesConfig()) { - return; - } - // Check stream config for which contextual values should be added to the event. - Collection requestedValuesFromConfig = streamConfig.getProducerConfig() - .getMetricsPlatformClientConfig().getRequestedValues(); - // Add required properties. - Set requestedValues = new HashSet<>(requestedValuesFromConfig); - requestedValues.addAll(REQUIRED_PROPERTIES); - ClientData filteredData = filterClientData(event.getClientData(), requestedValues); - event.setClientData(filteredData); - } - - private ClientData filterClientData(ClientData clientData, Collection requestedValues) { - AgentData.AgentDataBuilder agentBuilder = AgentData.builder(); - PageData.PageDataBuilder pageBuilder = PageData.builder(); - MediawikiData.MediawikiDataBuilder mediawikiBuilder = MediawikiData.builder(); - PerformerData.PerformerDataBuilder performerBuilder = PerformerData.builder(); - - AgentData agentData = clientData.getAgentData(); - PageData pageData = clientData.getPageData(); - MediawikiData mediawikiData = clientData.getMediawikiData(); - PerformerData performerData = clientData.getPerformerData(); - - for (String requestedValue : requestedValues) { - switch (requestedValue) { - case AGENT_APP_INSTALL_ID: - agentBuilder.appInstallId(agentData.getAppInstallId()); - break; - case AGENT_CLIENT_PLATFORM: - agentBuilder.clientPlatform(agentData.getClientPlatform()); - break; - case AGENT_CLIENT_PLATFORM_FAMILY: - agentBuilder.clientPlatformFamily(agentData.getClientPlatformFamily()); - break; - case AGENT_APP_FLAVOR: - agentBuilder.appFlavor(agentData.getAppFlavor()); - break; - case AGENT_APP_THEME: - agentBuilder.appTheme(agentData.getAppTheme()); - break; - case AGENT_APP_VERSION: - agentBuilder.appVersion(agentData.getAppVersion()); - break; - case AGENT_APP_VERSION_NAME: - agentBuilder.appVersionName(agentData.getAppVersionName()); - break; - case AGENT_DEVICE_FAMILY: - agentBuilder.deviceFamily(agentData.getDeviceFamily()); - break; - case AGENT_DEVICE_LANGUAGE: - agentBuilder.deviceLanguage(agentData.getDeviceLanguage()); - break; - case AGENT_RELEASE_STATUS: - agentBuilder.releaseStatus(agentData.getReleaseStatus()); - break; - case PAGE_ID: - pageBuilder.id(pageData.getId()); - break; - case PAGE_TITLE: - pageBuilder.title(pageData.getTitle()); - break; - case PAGE_NAMESPACE_ID: - pageBuilder.namespaceId(pageData.getNamespaceId()); - break; - case PAGE_NAMESPACE_NAME: - pageBuilder.namespaceName(pageData.getNamespaceName()); - break; - case PAGE_REVISION_ID: - pageBuilder.revisionId(pageData.getRevisionId()); - break; - case PAGE_WIKIDATA_QID: - pageBuilder.wikidataItemQid(pageData.getWikidataItemQid()); - break; - case PAGE_CONTENT_LANGUAGE: - pageBuilder.contentLanguage(pageData.getContentLanguage()); - break; - case MEDIAWIKI_DATABASE: - mediawikiBuilder.database(mediawikiData.getDatabase()); - break; - case PERFORMER_ID: - performerBuilder.id(performerData.getId()); - break; - case PERFORMER_NAME: - performerBuilder.name(performerData.getName()); - break; - case PERFORMER_IS_LOGGED_IN: - performerBuilder.isLoggedIn(performerData.getIsLoggedIn()); - break; - case PERFORMER_IS_TEMP: - performerBuilder.isTemp(performerData.getIsTemp()); - break; - case PERFORMER_SESSION_ID: - performerBuilder.sessionId(performerData.getSessionId()); - break; - case PERFORMER_PAGEVIEW_ID: - performerBuilder.pageviewId(performerData.getPageviewId()); - break; - case PERFORMER_GROUPS: - performerBuilder.groups(performerData.getGroups()); - break; - case PERFORMER_LANGUAGE_GROUPS: - var languageGroups = performerData.getLanguageGroups(); - if (languageGroups != null && languageGroups.length > 255) { - languageGroups = languageGroups.substring(0, 255); - } - performerBuilder.languageGroups(languageGroups); - break; - case PERFORMER_LANGUAGE_PRIMARY: - performerBuilder.languagePrimary(performerData.getLanguagePrimary()); - break; - case PERFORMER_REGISTRATION_DT: - performerBuilder.registrationDt(performerData.getRegistrationDt()); - break; - - default: - throw new IllegalArgumentException(String.format(Locale.ROOT, "Unknown property %s", requestedValue)); - } - } - - ClientData.ClientDataBuilder clientDataBuilder = ClientData.builder(); - clientDataBuilder.agentData(agentBuilder.build()); - clientDataBuilder.pageData(pageBuilder.build()); - clientDataBuilder.mediawikiData(mediawikiBuilder.build()); - clientDataBuilder.performerData(performerBuilder.build()); - return clientDataBuilder.build(); - } -} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/EventProcessor.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/EventProcessor.java deleted file mode 100644 index 89eef5a1a6a..00000000000 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/EventProcessor.java +++ /dev/null @@ -1,122 +0,0 @@ -package org.wikimedia.metrics_platform; - -import static java.util.stream.Collectors.groupingBy; -import static java.util.stream.Collectors.toList; - -import java.net.SocketTimeoutException; -import java.net.UnknownHostException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.atomic.AtomicReference; -import java.util.logging.Level; - -import javax.annotation.concurrent.ThreadSafe; - -import org.wikimedia.metrics_platform.config.DestinationEventService; -import org.wikimedia.metrics_platform.config.SourceConfig; -import org.wikimedia.metrics_platform.config.StreamConfig; -import org.wikimedia.metrics_platform.event.EventProcessed; - -import lombok.extern.java.Log; - -@ThreadSafe -@Log -public class EventProcessor { - - /** - * Enriches event data with context data requested in the stream configuration. - */ - private final ContextController contextController; - private final CurationController curationController; - - private final AtomicReference sourceConfig; - private final SamplingController samplingController; - private final BlockingQueue eventQueue; - private final EventSender eventSender; - private final boolean isDebug; - - /** - * EventProcessor constructor. - */ - public EventProcessor( - ContextController contextController, - CurationController curationController, - AtomicReference sourceConfig, - SamplingController samplingController, - EventSender eventSender, - BlockingQueue eventQueue, - boolean isDebug - ) { - this.contextController = contextController; - this.curationController = curationController; - this.sourceConfig = sourceConfig; - this.samplingController = samplingController; - this.eventSender = eventSender; - this.eventQueue = eventQueue; - this.isDebug = isDebug; - } - - /** - * Send all events currently in the output buffer. - *

- * A shallow clone of the output buffer is created and passed to the integration layer for - * submission by the client. If the event submission succeeds, the events are removed from the - * output buffer. (Note that the shallow copy created by clone() retains pointers to the original - * Event objects.) If the event submission fails, a client error is produced, and the events remain - * in buffer to be retried on the next submission attempt. - */ - public void sendEnqueuedEvents() { - SourceConfig config = sourceConfig.get(); - if (config == null) { - log.log(Level.FINE, "Configuration is missing, enqueued events are not sent."); - return; - } - - ArrayList pending = new ArrayList<>(); - this.eventQueue.drainTo(pending); - - Map streamConfigsMap = config.getStreamConfigsMap(); - - pending.stream() - .filter(event -> streamConfigsMap.containsKey(event.getStream())) - .filter(event -> { - StreamConfig cfg = streamConfigsMap.get(event.getStream()); - if (cfg.hasSampleConfig()) { - event.setSample(cfg.getSampleConfig()); - } - return samplingController.isInSample(cfg); - }) - .filter(event -> eventPassesCurationRules(event, streamConfigsMap)) - .collect(groupingBy(event -> destinationEventService(event, streamConfigsMap), toList())) - .forEach(this::sendEventsToDestination); - } - - protected boolean eventPassesCurationRules(EventProcessed event, Map streamConfigMap) { - StreamConfig streamConfig = streamConfigMap.get(event.getStream()); - contextController.enrichEvent(event, streamConfig); - - return curationController.shouldProduceEvent(event, streamConfig); - } - - private DestinationEventService destinationEventService(EventProcessed event, Map streamConfigMap) { - StreamConfig streamConfig = streamConfigMap.get(event.getStream()); - return streamConfig.getDestinationEventService(); - } - - @SuppressWarnings("checkstyle:IllegalCatch") - private void sendEventsToDestination( - DestinationEventService destinationEventService, - List pendingValidEvents - ) { - try { - eventSender.sendEvents(destinationEventService.getBaseUri(isDebug), pendingValidEvents); - } catch (UnknownHostException | SocketTimeoutException e) { - log.log(Level.WARNING, "Network error while sending " + pendingValidEvents.size() + " events. Adding back to queue.", e); - eventQueue.addAll(pendingValidEvents); - } catch (Exception e) { - log.log(Level.WARNING, "Failed to send " + pendingValidEvents.size() + " events.", e); - } - } -} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/EventSenderDefault.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/EventSenderDefault.java deleted file mode 100644 index c6a64b2a132..00000000000 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/EventSenderDefault.java +++ /dev/null @@ -1,53 +0,0 @@ -package org.wikimedia.metrics_platform; - -import static java.util.logging.Level.INFO; - -import java.io.IOException; -import java.net.URL; -import java.util.Collection; - -import org.wikimedia.metrics_platform.event.EventProcessed; - -import com.google.gson.Gson; - -import lombok.extern.java.Log; -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.RequestBody; -import okhttp3.Response; -import okhttp3.ResponseBody; - -@Log -public class EventSenderDefault implements EventSender { - - private final Gson gson; - private final OkHttpClient httpClient; - - public EventSenderDefault(Gson gson, OkHttpClient httpClient) { - this.gson = gson; - this.httpClient = httpClient; - } - - @Override - public void sendEvents(URL baseUri, Collection events) throws IOException { - Request request = new Request.Builder() - .url(baseUri) - .header("Accept", "application/json") - .header("User-Agent", "Metrics Platform Client/Java " + MetricsClient.METRICS_PLATFORM_LIBRARY_VERSION) - .post(RequestBody.create(gson.toJson(events), okhttp3.MediaType.parse("application/json"))) - .build(); - - try (Response response = httpClient.newCall(request).execute()) { - int status = response.code(); - ResponseBody body = response.body(); - if (!response.isSuccessful() || status == 207) { - // In the case of a multi-status response (207), it likely means that one or more - // events were rejected. In such a case, the error is actually contained in - // the normal response body. - throw new IOException(body.string()); - } - - log.log(INFO, "Sent " + events.size() + " events successfully."); - } - } -} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/MetricsClient.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/MetricsClient.java deleted file mode 100644 index f374bcbb505..00000000000 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/MetricsClient.java +++ /dev/null @@ -1,527 +0,0 @@ -package org.wikimedia.metrics_platform; - -import static java.lang.Math.max; -import static java.time.Instant.now; -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.logging.Level.FINE; -import static java.util.stream.Collectors.toList; -import static org.wikimedia.metrics_platform.config.StreamConfigFetcher.ANALYTICS_API_ENDPOINT; - -import java.net.URL; -import java.time.Duration; -import java.time.ZoneId; -import java.time.format.DateTimeFormatter; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Consumer; -import java.util.logging.Level; -import java.util.stream.Stream; - -import org.wikimedia.metrics_platform.config.ConfigFetcherRunnable; -import org.wikimedia.metrics_platform.config.SourceConfig; -import org.wikimedia.metrics_platform.config.StreamConfig; -import org.wikimedia.metrics_platform.config.StreamConfigFetcher; -import org.wikimedia.metrics_platform.context.ClientData; -import org.wikimedia.metrics_platform.context.InteractionData; -import org.wikimedia.metrics_platform.context.PerformerData; -import org.wikimedia.metrics_platform.event.Event; -import org.wikimedia.metrics_platform.event.EventProcessed; -import org.wikimedia.metrics_platform.json.GsonHelper; - -import com.google.gson.Gson; - -import okhttp3.OkHttpClient; - -public final class MetricsClient { - - public static final DateTimeFormatter DATE_FORMAT = DateTimeFormatter - .ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ROOT) - .withZone(ZoneId.of("UTC")); - - private final ScheduledExecutorService executorService; - public static final String METRICS_PLATFORM_LIBRARY_VERSION = "2.8"; - public static final String METRICS_PLATFORM_BASE_VERSION = "1.2.2"; - public static final String METRICS_PLATFORM_SCHEMA_BASE = "/analytics/product_metrics/app/base/" + METRICS_PLATFORM_BASE_VERSION; - private final AtomicReference sourceConfig; - - /** - * Handles logging session management. A new session begins (and a new session ID is created) - * if the app has been inactive for 15 minutes or more. - */ - private final SessionController sessionController; - - /** - * Evaluates whether events for a given stream are in-sample based on the stream configuration. - */ - private final SamplingController samplingController; - - private final BlockingQueue eventQueue; - private final EventProcessor eventProcessor; - - /** - * MetricsClient constructor. - */ - private MetricsClient( - ScheduledExecutorService executorService, - SessionController sessionController, - SamplingController samplingController, - AtomicReference sourceConfig, - BlockingQueue eventQueue, - EventProcessor eventProcessor - ) { - this.executorService = executorService; - this.sessionController = sessionController; - this.samplingController = samplingController; - this.sourceConfig = sourceConfig; - this.eventQueue = eventQueue; - this.eventProcessor = eventProcessor; - } - - /** - * Submit an event to be enqueued and sent to the Event Platform. - *

- * If stream configs are not yet fetched, the event will be held temporarily in the input - * buffer (provided there is space to do so). - *

- * If stream configs are available, the event will be validated and enqueued for submission - * to the configured event platform intake service. - *

- * Supplemental metadata is added immediately on intake, regardless of the presence or absence - * of stream configs, so that the event timestamp is recorded accurately. - * - * @param event event data - */ - public void submit(Event event) { - EventProcessed eventProcessed = EventProcessed.fromEvent(event); - addRequiredMetadata(eventProcessed); - addToEventQueue(eventProcessed); - } - - /** - * Construct and submits a Metrics Platform Event from the event name and custom data for each - * stream specified. - *

- * The Metrics Platform Event for a stream (S) is constructed by: first initializing the minimum - * valid event (E) that can be submitted to S; and, second mixing the context attributes requested - * in the configuration for S into E. - *

- * The Metrics Platform Event is submitted to a stream (S) if: 1) S is in sample; and 2) the event - * is filtered due to the filtering rules for S. - *

- * This particular submitMetricsEvent method accepts unformatted custom data and calls the following - * submitMetricsEvent method with the custom data properly formatted. - * - * @see Metrics Platform/Java API - * - * @param streamName stream name - * @param schemaId schema id - * @param eventName event name - * @param customData custom data - */ - public void submitMetricsEvent( - String streamName, - String schemaId, - String eventName, - Map customData - ) { - submitMetricsEvent(streamName, schemaId, eventName, null, customData, null); - } - - /** - * Construct and submits a Metrics Platform Event from the schema id, event name, page metadata, and custom data for - * the stream that is interested in those events. - * - * @param streamName stream name - * @param schemaId schema id - * @param eventName event name - * @param clientData client context data - * @param customData custom data - */ - public void submitMetricsEvent( - String streamName, - String schemaId, - String eventName, - ClientData clientData, - Map customData - ) { - submitMetricsEvent(streamName, schemaId, eventName, clientData, customData, null); - } - - /** - * Construct and submits a Metrics Platform Event from the schema id, stream name, event name, page metadata, custom - * data, and interaction data for the specified stream. - * - * @param streamName stream name - * @param schemaId schema id - * @param eventName event name - * @param clientData client context data - * @param customData custom data - * @param interactionData common data for an interaction schema - */ - public void submitMetricsEvent( - String streamName, - String schemaId, - String eventName, - ClientData clientData, - Map customData, - InteractionData interactionData - ) { - if (streamName == null) { - log.log(Level.FINE, "No stream has been specified, the submitMetricsEvent event is ignored and dropped."); - return; - } - - // If we already have stream configs, then we can pre-validate certain conditions and exclude the event from the queue entirely. - StreamConfig streamConfig = null; - if (sourceConfig.get() != null) { - streamConfig = sourceConfig.get().getStreamConfigByName(streamName); - if (streamConfig == null) { - log.log(Level.FINE, "No stream config exists for this stream, the submitMetricsEvent event is ignored and dropped."); - return; - } - if (!samplingController.isInSample(streamConfig)) { - log.log(Level.FINE, "Not in sample, the submitMetricsEvent event is ignored and dropped."); - return; - } - } - - Event event = new Event(schemaId, streamName, eventName); - event.setClientData(clientData); - - if (customData != null) { - event.setCustomData(customData); - } - - event.setInteractionData(interactionData); - - if (streamConfig != null && streamConfig.hasSampleConfig()) { - event.setSample(streamConfig.getSampleConfig()); - } - - submit(event); - } - - /** - * Submit an interaction event to a stream. - *

- * An interaction event is meant to represent a basic interaction with some target or some event - * occurring, e.g. the user (**performer**) tapping/clicking a UI element, or an app notifying the - * server of its current state. - * - * @param streamName stream name - * @param eventName event name - * @param clientData client context data - * @param interactionData common data for the base interaction schema - * - * @see Metrics Platform/Java API - */ - public void submitInteraction( - String streamName, - String eventName, - ClientData clientData, - InteractionData interactionData - ) { - submitMetricsEvent(streamName, METRICS_PLATFORM_SCHEMA_BASE, eventName, clientData, null, interactionData); - } - - /** - * Submit an interaction event to a stream. - *

- * See above - takes additional parameters (custom data + custom schema id) to submit an interaction event. - * - * @param streamName stream name - * @param schemaId schema id - * @param eventName event name - * @param clientData client context data - * @param interactionData common data for the base interaction schema - * @param customData custom data for the interaction - * - * @see Metrics Platform/Java API - */ - public void submitInteraction( - String streamName, - String schemaId, - String eventName, - ClientData clientData, - InteractionData interactionData, - Map customData - ) { - submitMetricsEvent(streamName, schemaId, eventName, clientData, customData, interactionData); - } - - /** - * Submit a click event to a stream. - * - * @param streamName stream name - * @param clientData client context data - * @param interactionData common data for the base interaction schema - * - * @see Metrics Platform/Java API - */ - public void submitClick( - String streamName, - ClientData clientData, - InteractionData interactionData - ) { - submitMetricsEvent(streamName, METRICS_PLATFORM_SCHEMA_BASE, "click", clientData, null, interactionData); - } - - /** - * Submit a click event to a stream with custom data. - * - * @param streamName stream name - * @param schemaId schema id - * @param eventName event name - * @param clientData client context data - * @param customData custom data for the interaction - * @param interactionData common data for the base interaction schema - * - * @see Metrics Platform/Java API - */ - public void submitClick( - String streamName, - String schemaId, - String eventName, - ClientData clientData, - Map customData, - InteractionData interactionData - ) { - submitMetricsEvent(streamName, schemaId, eventName, clientData, customData, interactionData); - } - - /** - * Submit a view event to a stream. - * - * @param streamName stream name - * @param clientData client context data - * @param interactionData common data for the base interaction schema - */ - public void submitView( - String streamName, - ClientData clientData, - InteractionData interactionData - ) { - submitMetricsEvent(streamName, METRICS_PLATFORM_SCHEMA_BASE, "view", clientData, null, interactionData); - } - - /** - * Submit a view event to a stream with custom data. - * - * @param streamName stream name - * @param schemaId schema id - * @param eventName event name - * @param clientData client context data - * @param customData custom data for the interaction - * @param interactionData common data for the base interaction schema - */ - public void submitView( - String streamName, - String schemaId, - String eventName, - ClientData clientData, - Map customData, - InteractionData interactionData - ) { - submitMetricsEvent(streamName, schemaId, eventName, clientData, customData, interactionData); - } - - /** - * Convenience method to be called when - * - * the onPause() activity lifecycle callback is called. - *

- * Touches the session so that we can determine whether it's session has expired if and when the - * application is resumed. - */ - public void onAppPause() { - executorService.schedule(eventProcessor::sendEnqueuedEvents, 0, MILLISECONDS); - sessionController.touchSession(); - } - - /** - * Convenience method to be called when - * - * the onResume() activity lifecycle callback is called. - *

- * Touches the session so that we can determine whether it has expired. - */ - public void onAppResume() { - sessionController.touchSession(); - } - - /** - * Closes the session. - */ - public void onAppClose() { - executorService.schedule(eventProcessor::sendEnqueuedEvents, 0, MILLISECONDS); - sessionController.closeSession(); - } - - /** - * Begins a new session and touches the session. - */ - public void resetSession() { - sessionController.beginSession(); - } - - /** - * Supplement the outgoing event with additional metadata. - * These include: - * - app_session_id: the current session ID - * - dt: ISO 8601 timestamp - * - domain: hostname - * - * @param event event - */ - private void addRequiredMetadata(EventProcessed event) { - event.setPerformerData( - PerformerData.builderFrom(event.getPerformerData()) - .sessionId(sessionController.getSessionId()) - .build()); - event.setTimestamp(DATE_FORMAT.format(now())); - event.setDomain(event.getClientData().domain); - } - - /** - * Append an enriched event to the queue. - * If the queue is full, we remove the oldest events from the queue to add the current event. - * Number of attempts to add to the queue is 1/50 of the number queue capacity but at least 10 - * - * @param event a processed event - */ - private void addToEventQueue(EventProcessed event) { - int eventQueueAppendAttempts = max(eventQueue.size() / 50, 10); - - while (!eventQueue.offer(event)) { - EventProcessed removedEvent = eventQueue.remove(); - if (removedEvent != null) { - log.log(FINE, removedEvent.getName() + " was dropped so that a newer event could be added to the queue."); - } - if (eventQueueAppendAttempts-- <= 0) break; - } - } - - public boolean isFullyInitialized() { - return sourceConfig.get() != null; - } - - public boolean isEventQueueEmpty() { - return eventQueue.isEmpty(); - } - - public static Builder builder(ClientData clientData) { - return new Builder(clientData); - } - - public static final class Builder { - - private final ClientData clientData; - private final Duration streamConfigFetchRetryDelay = Duration.ofMinutes(1); - private AtomicReference sourceConfigRef = new AtomicReference<>(); - private BlockingQueue eventQueue = new LinkedBlockingQueue<>(10); - private SessionController sessionController = new SessionController(); - - private CurationController curationController = new CurationController(); - - @Nullable private SamplingController samplingController; - - private OkHttpClient httpClient = new OkHttpClient(); - private URL streamConfigURL = safeURL(ANALYTICS_API_ENDPOINT); - private Duration streamConfigFetchInitialDelay = Duration.ofSeconds(0); - private Duration streamConfigFetchInterval = Duration.ofSeconds(30); - private Duration sendEventsInitialDelay = Duration.ofSeconds(3); - private Duration sendEventsInterval = Duration.ofSeconds(30); - private boolean isDebug; - private Consumer sourceConfigConsumer = sourceConfig -> {}; - private ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1, new SimpleThreadFactory()); - - @Nullable private SourceConfig sourceConfig; - - public Builder(ClientData clientData) { - this.clientData = clientData; - } - - public Builder eventQueueCapacity(int capacity) { - eventQueue = new LinkedBlockingQueue<>(capacity); - return this; - } - - public MetricsClient build() { - if (sourceConfig != null) sourceConfigRef.set(sourceConfig); - - if (samplingController == null) { - samplingController = new SamplingController(clientData, sessionController); - } - - Gson gson = GsonHelper.getGson(); - - EventProcessor eventProcessor = new EventProcessor( - new ContextController(), - curationController, - sourceConfigRef, - samplingController, - new EventSenderDefault(gson, httpClient), - eventQueue, - isDebug - ); - - MetricsClient metricsClient = new MetricsClient( - executorService, - sessionController, - samplingController, - sourceConfigRef, - eventQueue, - eventProcessor); - - List> consumers = Stream.of(sourceConfigRef::set, sourceConfigConsumer) - .collect(toList()); - - StreamConfigFetcher streamConfigFetcher = new StreamConfigFetcher(streamConfigURL, httpClient, gson); - - ConfigFetcherRunnable configFetchRunnable = new ConfigFetcherRunnable(streamConfigFetchInterval, - streamConfigFetcher, - consumers, - executorService, streamConfigFetchRetryDelay); - - startScheduledOperations(eventProcessor, configFetchRunnable, executorService); - - return metricsClient; - } - - private void startScheduledOperations( - EventProcessor eventProcessor, - ConfigFetcherRunnable configFetchRunnable, - ScheduledExecutorService executorService - ) { - executorService.schedule(configFetchRunnable, streamConfigFetchInitialDelay.toMillis(), MILLISECONDS); - - executorService.scheduleAtFixedRate( - eventProcessor::sendEnqueuedEvents, - sendEventsInitialDelay.toMillis(), sendEventsInterval.toMillis(), MILLISECONDS); - } - - private static URL safeURL(String url) { - return new URL(url); - } - - } - - private static final class SimpleThreadFactory implements ThreadFactory { - - private final AtomicLong counter = new AtomicLong(); - - @Override - public Thread newThread(Runnable r) { - return new Thread(r, "metrics-client-" + counter.incrementAndGet()); - } - } - -} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/ConfigFetcherRunnable.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/ConfigFetcherRunnable.java deleted file mode 100644 index baa4bb994e2..00000000000 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/ConfigFetcherRunnable.java +++ /dev/null @@ -1,48 +0,0 @@ -package org.wikimedia.metrics_platform.config; - -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static java.util.logging.Level.WARNING; - -import java.time.Duration; -import java.util.List; -import java.util.concurrent.ScheduledExecutorService; -import java.util.function.Consumer; - -public class ConfigFetcherRunnable implements Runnable { - - private final Duration streamConfigFetchInterval; - private final StreamConfigFetcher streamConfigFetcher; - private final List> consumers; - private final ScheduledExecutorService executorService; - private final Duration retryDelay; - - public ConfigFetcherRunnable( - Duration streamConfigFetchInterval, - StreamConfigFetcher streamConfigFetcher, - List> consumers, - ScheduledExecutorService executorService, - Duration retryDelay - ) { - this.streamConfigFetchInterval = streamConfigFetchInterval; - this.streamConfigFetcher = streamConfigFetcher; - this.consumers = consumers; - this.executorService = executorService; - this.retryDelay = retryDelay; - } - - public void run() { - Duration nextFetch = streamConfigFetchInterval; - try { - SourceConfig sourceConfig = streamConfigFetcher.fetchStreamConfigs(); - - for (Consumer consumer : consumers) { - consumer.accept(sourceConfig); - } - } catch (Exception e) { - log.log(WARNING, "Could not fetch configuration. Will retry sooner.", e); - nextFetch = retryDelay; - } finally { - executorService.schedule(this, nextFetch.toMillis(), MILLISECONDS); - } - } -} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/DestinationEventService.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/DestinationEventService.java deleted file mode 100644 index 43bcd3b462e..00000000000 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/DestinationEventService.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.wikimedia.metrics_platform.config; - -import java.net.MalformedURLException; -import java.net.URL; - -import com.google.gson.annotations.SerializedName; - -/** - * Possible event destination endpoints which can be specified in stream configurations. - * For now, we'll assume that we always want to send to ANALYTICS. - * - * https://wikitech.wikimedia.org/wiki/Event_Platform/EventGate#EventGate_clusters - */ -public enum DestinationEventService { - - @SerializedName("eventgate-analytics-external") - ANALYTICS("https://intake-analytics.wikimedia.org"), - - @SerializedName("eventgate-logging-external") - ERROR_LOGGING("https://intake-logging.wikimedia.org"), - - @SerializedName("eventgate-logging-local") - LOCAL("http://localhost:8192"); - - private final URL baseUri; - - DestinationEventService(String baseUri) { - this.baseUri = new URL(baseUri + "/v1/events"); - } - - public URL getBaseUri() throws MalformedURLException { - return getBaseUri(false); - } - - public URL getBaseUri(boolean isDebug) throws MalformedURLException { - return isDebug ? this.baseUri : new URL(baseUri + "?hasty=true"); - } -} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/AgentData.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/AgentData.kt deleted file mode 100644 index 259e87fbdf9..00000000000 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/AgentData.kt +++ /dev/null @@ -1,24 +0,0 @@ -package org.wikimedia.metrics_platform.context - -import kotlinx.serialization.SerialName -import kotlinx.serialization.Serializable - -/** - * Agent context data fields. - * - * All fields are nullable, and boxed types are used in place of their equivalent primitive types to avoid - * unexpected default values from being used where the true value is null. - */ -@Serializable -class AgentData( - @SerialName("app_flavor") val appFlavor: String? = null, - @SerialName("app_install_id") val appInstallId: String? = null, - @SerialName("app_theme") val appTheme: String? = null, - @SerialName("app_version") val appVersion: Int? = null, - @SerialName("app_version_name") val appVersionName: String? = null, - @SerialName("client_platform") val clientPlatform: String? = null, - @SerialName("client_platform_family") val clientPlatformFamily: String? = null, - @SerialName("device_family") val deviceFamily: String? = null, - @SerialName("device_language") val deviceLanguage: String? = null, - @SerialName("release_status") val releaseStatus: String? = null, -) diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/PerformerData.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/PerformerData.kt deleted file mode 100644 index 60dc1db928c..00000000000 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/PerformerData.kt +++ /dev/null @@ -1,41 +0,0 @@ -@file:UseSerializers(InstantSerializer::class) - -package org.wikimedia.metrics_platform.context - -import kotlinx.serialization.SerialName -import kotlinx.serialization.Serializable -import kotlinx.serialization.UseSerializers -import java.time.Instant - -/** - * Performer context data fields. - * - * All fields are nullable, and boxed types are used in place of their equivalent primitive types to avoid - * unexpected default values from being used where the true value is null. - */ -@Serializable -class PerformerData ( - val id: Int? = null, - @SerialName("name") val name: String? = null, - @SerialName("is_logged_in") val isLoggedIn: Boolean? = null, - @SerialName("is_temp") val isTemp: Boolean? = null, - @SerialName("session_id") val sessionId: String? = null, - @SerialName("pageview_id") val pageviewId: String? = null, - @SerialName("groups") val groups: MutableCollection? = null, - @SerialName("language_groups") val languageGroups: String? = null, - @SerialName("language_primary") val languagePrimary: String? = null, - @SerialName("registration_dt") val registrationDt: Instant? = null -) { - class PerformerDataBuilder { - fun languageGroups(languageGroups: String?): PerformerDataBuilder { - // Ensure that a performer's language groups do not exceed 255 characters. See T361265. - - var languageGroups = languageGroups - if (languageGroups != null && languageGroups.length > 255) { - languageGroups = languageGroups.substring(0, 255) - } - - return this - } - } -} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/json/GsonHelper.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/json/GsonHelper.java deleted file mode 100644 index 9a01aceeb34..00000000000 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/json/GsonHelper.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.wikimedia.metrics_platform.json; - -import java.time.Instant; - -import org.wikimedia.metrics_platform.context.InstantConverter; -import org.wikimedia.metrics_platform.event.EventProcessed; -import org.wikimedia.metrics_platform.event.EventProcessedSerializer; - -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; - -public final class GsonHelper { - private GsonHelper() { - // Utility class which should never be instantiated. - } - - public static Gson getGson() { - return new GsonBuilder() - .registerTypeAdapter(Instant.class, new InstantConverter()) - .registerTypeAdapter(EventProcessed.class, new EventProcessedSerializer()) - .create(); - } -} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/ContextController.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/ContextController.kt new file mode 100644 index 00000000000..a41ce88f037 --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/ContextController.kt @@ -0,0 +1,94 @@ +package org.wikimedia.metricsplatform + +import org.wikimedia.metricsplatform.config.StreamConfig +import org.wikimedia.metricsplatform.context.AgentData +import org.wikimedia.metricsplatform.context.ClientData +import org.wikimedia.metricsplatform.context.ContextValue +import org.wikimedia.metricsplatform.context.MediawikiData +import org.wikimedia.metricsplatform.context.PageData +import org.wikimedia.metricsplatform.context.PerformerData +import org.wikimedia.metricsplatform.event.EventProcessed + +class ContextController { + fun enrichEvent(event: EventProcessed, streamConfig: StreamConfig) { + if (!streamConfig.hasRequestedContextValuesConfig()) { + return + } + // Check stream config for which contextual values should be added to the event. + val requestedValuesFromConfig = streamConfig.producerConfig?.metricsPlatformClientConfig?.requestedValues.orEmpty() + // Add required properties. + val requestedValues= mutableSetOf() + requestedValues.addAll(requestedValuesFromConfig) + requestedValues.addAll(REQUIRED_PROPERTIES) + val filteredData = filterClientData(event.clientData, requestedValues) + event.setClientData(filteredData) + } + + private fun filterClientData(clientData: ClientData, requestedValues: Collection): ClientData { + val newAgentData = AgentData() + val newPageData = PageData() + val newMediawikiData = MediawikiData() + val newPerformerData = PerformerData() + + for (requestedValue in requestedValues) { + when (requestedValue) { + ContextValue.AGENT_APP_INSTALL_ID -> newAgentData.appInstallId = clientData.agentData.appInstallId + ContextValue.AGENT_CLIENT_PLATFORM -> newAgentData.clientPlatform = clientData.agentData.clientPlatform + ContextValue.AGENT_CLIENT_PLATFORM_FAMILY -> newAgentData.clientPlatformFamily = clientData.agentData.clientPlatformFamily + ContextValue.AGENT_APP_FLAVOR -> newAgentData.appFlavor = clientData.agentData.appFlavor + ContextValue.AGENT_APP_THEME -> newAgentData.appTheme = clientData.agentData.appTheme + ContextValue.AGENT_APP_VERSION -> newAgentData.appVersion = clientData.agentData.appVersion + ContextValue.AGENT_APP_VERSION_NAME -> newAgentData.appVersionName = clientData.agentData.appVersionName + ContextValue.AGENT_DEVICE_FAMILY -> newAgentData.deviceFamily = clientData.agentData.deviceFamily + ContextValue.AGENT_DEVICE_LANGUAGE -> newAgentData.deviceLanguage = clientData.agentData.deviceLanguage + ContextValue.AGENT_RELEASE_STATUS -> newAgentData.releaseStatus = clientData.agentData.releaseStatus + ContextValue.PAGE_ID -> newPageData.id = clientData.pageData.id + ContextValue.PAGE_TITLE -> newPageData.title = clientData.pageData.title + ContextValue.PAGE_NAMESPACE_ID -> newPageData.namespaceId = clientData.pageData.namespaceId + ContextValue.PAGE_NAMESPACE_NAME -> newPageData.namespaceName = clientData.pageData.namespaceName + ContextValue.PAGE_REVISION_ID -> newPageData.revisionId = clientData.pageData.revisionId + ContextValue.PAGE_WIKIDATA_QID -> newPageData.wikidataItemQid = clientData.pageData.wikidataItemQid + ContextValue.PAGE_CONTENT_LANGUAGE -> newPageData.contentLanguage = clientData.pageData.contentLanguage + ContextValue.MEDIAWIKI_DATABASE -> newMediawikiData.database = clientData.mediawikiData.database + ContextValue.PERFORMER_ID -> newPerformerData.id = clientData.performerData.id + ContextValue.PERFORMER_NAME -> newPerformerData.name = clientData.performerData.name + ContextValue.PERFORMER_IS_LOGGED_IN -> newPerformerData.isLoggedIn = clientData.performerData.isLoggedIn + ContextValue.PERFORMER_IS_TEMP -> newPerformerData.isTemp = clientData.performerData.isTemp + ContextValue.PERFORMER_SESSION_ID -> newPerformerData.sessionId = clientData.performerData.sessionId + ContextValue.PERFORMER_PAGEVIEW_ID -> newPerformerData.pageviewId = clientData.performerData.pageviewId + ContextValue.PERFORMER_GROUPS -> newPerformerData.groups = clientData.performerData.groups + ContextValue.PERFORMER_LANGUAGE_GROUPS -> { + var languageGroups = clientData.performerData.languageGroups + if (languageGroups != null && languageGroups.length > 255) { + languageGroups = languageGroups.substring(0, 255) + } + newPerformerData.languageGroups = languageGroups + } + + ContextValue.PERFORMER_LANGUAGE_PRIMARY -> newPerformerData.languagePrimary = clientData.performerData.languagePrimary + ContextValue.PERFORMER_REGISTRATION_DT -> newPerformerData.registrationDt = clientData.performerData.registrationDt + else -> throw IllegalArgumentException("Unknown property: $requestedValue") + } + } + + return ClientData(newAgentData, newPageData, newMediawikiData, newPerformerData) + } + + companion object { + /** + * @see [Metrics Platform/Contextual attributes](https://wikitech.wikimedia.org/wiki/Metrics_Platform/Contextual_attributes) + */ + private val REQUIRED_PROPERTIES = listOf( + "agent_app_flavor", + "agent_app_install_id", + "agent_app_theme", + "agent_app_version", + "agent_app_version_name", + "agent_client_platform", + "agent_client_platform_family", + "agent_device_family", + "agent_device_language", + "agent_release_status" + ) + } +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/CurationController.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/CurationController.kt similarity index 60% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/CurationController.kt rename to analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/CurationController.kt index 50c51a060a1..adecb3908ad 100644 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/CurationController.kt +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/CurationController.kt @@ -1,7 +1,7 @@ -package org.wikimedia.metrics_platform +package org.wikimedia.metricsplatform -import org.wikimedia.metrics_platform.config.StreamConfig -import org.wikimedia.metrics_platform.event.EventProcessed +import org.wikimedia.metricsplatform.config.StreamConfig +import org.wikimedia.metricsplatform.event.EventProcessed class CurationController { fun shouldProduceEvent(event: EventProcessed, streamConfig: StreamConfig): Boolean { diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/EventProcessor.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/EventProcessor.kt new file mode 100644 index 00000000000..485e65b9597 --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/EventProcessor.kt @@ -0,0 +1,116 @@ +package org.wikimedia.metricsplatform + +import org.wikimedia.metricsplatform.config.DestinationEventService +import org.wikimedia.metricsplatform.config.SourceConfig +import org.wikimedia.metricsplatform.config.StreamConfig +import org.wikimedia.metricsplatform.event.EventProcessed +import java.net.SocketTimeoutException +import java.net.UnknownHostException +import java.util.concurrent.BlockingQueue +import java.util.concurrent.atomic.AtomicReference + +class EventProcessor( + /** + * Enriches event data with context data requested in the stream configuration. + */ + private val contextController: ContextController, + private val curationController: CurationController, + sourceConfig: AtomicReference, + samplingController: SamplingController, + eventSender: EventSender, + eventQueue: BlockingQueue, + isDebug: Boolean +) { + private val sourceConfig: AtomicReference + private val samplingController: SamplingController + private val eventQueue: BlockingQueue + private val eventSender: EventSender + private val isDebug: Boolean + + /** + * EventProcessor constructor. + */ + init { + this.sourceConfig = sourceConfig + this.samplingController = samplingController + this.eventSender = eventSender + this.eventQueue = eventQueue + this.isDebug = isDebug + } + + /** + * Send all events currently in the output buffer. + * + * + * A shallow clone of the output buffer is created and passed to the integration layer for + * submission by the client. If the event submission succeeds, the events are removed from the + * output buffer. (Note that the shallow copy created by clone() retains pointers to the original + * Event objects.) If the event submission fails, a client error is produced, and the events remain + * in buffer to be retried on the next submission attempt. + */ + fun sendEnqueuedEvents() { + val config: SourceConfig? = sourceConfig.get() + if (config == null) { + //log.log(Level.FINE, "Configuration is missing, enqueued events are not sent.") + return + } + + val pending: ArrayList = ArrayList() + this.eventQueue.drainTo(pending) + + val streamConfigsMap = config.streamConfigsMap + + val foo = pending.filter { event -> streamConfigsMap.containsKey(event.stream) } + .filter { event -> + val cfg = streamConfigsMap[event.stream] + if (cfg!!.hasSampleConfig()) { + event.sample = cfg.sampleConfig + } + samplingController.isInSample(cfg) + } + .filter { event -> eventPassesCurationRules(event, streamConfigsMap) } + .groupBy { event -> destinationEventService(event, streamConfigsMap) } + .forEach { (destinationEventService, pendingValidEvents) -> + this.sendEventsToDestination( + destinationEventService, pendingValidEvents + ) + } + } + + fun eventPassesCurationRules( + event: EventProcessed, + streamConfigMap: Map + ): Boolean { + val streamConfig = streamConfigMap[event.stream] + if (streamConfig == null) { + return false + } + contextController.enrichEvent(event, streamConfig) + return curationController.shouldProduceEvent(event, streamConfig) + } + + private fun destinationEventService( + event: EventProcessed, + streamConfigMap: Map + ): DestinationEventService { + val streamConfig = streamConfigMap[event.stream] + return streamConfig?.getDestinationEventService() ?: DestinationEventService.ANALYTICS + } + + private fun sendEventsToDestination( + destinationEventService: DestinationEventService, + pendingValidEvents: List + ) { + try { + eventSender.sendEvents(destinationEventService.getBaseUri(isDebug), pendingValidEvents) + } catch (e: UnknownHostException) { + //log.log(Level.WARNING, "Network error while sending " + pendingValidEvents.size + " events. Adding back to queue.", e) + eventQueue.addAll(pendingValidEvents) + } catch (e: SocketTimeoutException) { + //log.log(Level.WARNING, "Network error while sending " + pendingValidEvents.size + " events. Adding back to queue.", e) + eventQueue.addAll(pendingValidEvents) + } catch (e: Exception) { + //log.log(Level.WARNING, "Failed to send " + pendingValidEvents.size + " events.", e) + } + } +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/EventSender.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/EventSender.kt similarity index 75% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/EventSender.kt rename to analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/EventSender.kt index fab2bef8edf..e78aef7145c 100644 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/EventSender.kt +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/EventSender.kt @@ -1,6 +1,6 @@ -package org.wikimedia.metrics_platform +package org.wikimedia.metricsplatform -import org.wikimedia.metrics_platform.event.EventProcessed +import org.wikimedia.metricsplatform.event.EventProcessed import java.net.URL fun interface EventSender { diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/EventSenderDefault.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/EventSenderDefault.kt new file mode 100644 index 00000000000..9504c62871f --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/EventSenderDefault.kt @@ -0,0 +1,34 @@ +package org.wikimedia.metricsplatform + +import okhttp3.OkHttpClient +import okhttp3.RequestBody +import okhttp3.ResponseBody +import org.wikimedia.metricsplatform.event.EventProcessed + +class EventSenderDefault(private val gson: com.google.gson.Gson, + private val httpClient: OkHttpClient +) : EventSender { + override fun sendEvents(baseUri: java.net.URL, events: List) { + val request = okhttp3.Request.Builder() + .url(baseUri) + .header("Accept", "application/json") + .header( + "User-Agent", + "Metrics Platform Client/Java " + MetricsClient.METRICS_PLATFORM_LIBRARY_VERSION + ) + .post(RequestBody.create(gson.toJson(events), parse.parse("application/json"))) + .build() + + httpClient.newCall(request).execute().use { response -> + val status: kotlin.Int = response.code + val body: ResponseBody? = response.body + if (!response.isSuccessful || status == 207) { + // In the case of a multi-status response (207), it likely means that one or more + // events were rejected. In such a case, the error is actually contained in + // the normal response body. + throw java.io.IOException(body.string()) + } + //log.log(java.util.logging.Level.INFO, "Sent " + events.size + " events successfully.") + } + } +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/MetricsClient.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/MetricsClient.kt new file mode 100644 index 00000000000..fe448181164 --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/MetricsClient.kt @@ -0,0 +1,493 @@ +package org.wikimedia.metricsplatform + +import okhttp3.OkHttpClient +import org.wikimedia.metricsplatform.config.SourceConfig +import org.wikimedia.metricsplatform.config.StreamConfig +import org.wikimedia.metricsplatform.config.StreamConfigFetcher +import org.wikimedia.metricsplatform.context.ClientData +import org.wikimedia.metricsplatform.context.InteractionData +import org.wikimedia.metricsplatform.event.Event +import org.wikimedia.metricsplatform.event.EventProcessed +import java.net.URL +import java.time.Instant +import java.time.ZoneId +import java.time.format.DateTimeFormatter +import java.util.Locale +import java.util.concurrent.BlockingQueue +import java.util.concurrent.Executors +import java.util.concurrent.LinkedBlockingQueue +import java.util.concurrent.ScheduledExecutorService +import java.util.concurrent.ThreadFactory +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicLong +import java.util.concurrent.atomic.AtomicReference +import kotlin.math.max + +class MetricsClient private constructor( + private val executorService: ScheduledExecutorService, + /** + * Handles logging session management. A new session begins (and a new session ID is created) + * if the app has been inactive for 15 minutes or more. + */ + private val sessionController: SessionController, + /** + * Evaluates whether events for a given stream are in-sample based on the stream configuration. + */ + private val samplingController: SamplingController, + private val sourceConfig: AtomicReference, + eventQueue: BlockingQueue, + eventProcessor: EventProcessor +) { + + private val eventQueue: BlockingQueue + private val eventProcessor: EventProcessor + + /** + * MetricsClient constructor. + */ + init { + this.eventQueue = eventQueue + this.eventProcessor = eventProcessor + } + + /** + * Submit an event to be enqueued and sent to the Event Platform. + * + * + * If stream configs are not yet fetched, the event will be held temporarily in the input + * buffer (provided there is space to do so). + * + * + * If stream configs are available, the event will be validated and enqueued for submission + * to the configured event platform intake service. + * + * + * Supplemental metadata is added immediately on intake, regardless of the presence or absence + * of stream configs, so that the event timestamp is recorded accurately. + * + * @param event event data + */ + fun submit(event: Event) { + val eventProcessed: EventProcessed = EventProcessed.fromEvent(event) + addRequiredMetadata(eventProcessed) + addToEventQueue(eventProcessed) + } + + /** + * Construct and submits a Metrics Platform Event from the event name and custom data for each + * stream specified. + * + * + * The Metrics Platform Event for a stream (S) is constructed by: first initializing the minimum + * valid event (E) that can be submitted to S; and, second mixing the context attributes requested + * in the configuration for S into E. + * + * + * The Metrics Platform Event is submitted to a stream (S) if: 1) S is in sample; and 2) the event + * is filtered due to the filtering rules for S. + * + * + * This particular submitMetricsEvent method accepts unformatted custom data and calls the following + * submitMetricsEvent method with the custom data properly formatted. + * + * @see [Metrics Platform/Java API](https://wikitech.wikimedia.org/wiki/Metrics_Platform/Java_API) + * + * + * @param streamName stream name + * @param schemaId schema id + * @param eventName event name + * @param customData custom data + */ + fun submitMetricsEvent( + streamName: String?, + schemaId: String?, + eventName: String?, + customData: MutableMap? + ) { + submitMetricsEvent(streamName, schemaId, eventName, null, customData, null) + } + + /** + * Construct and submits a Metrics Platform Event from the schema id, stream name, event name, page metadata, custom + * data, and interaction data for the specified stream. + * + * @param streamName stream name + * @param schemaId schema id + * @param eventName event name + * @param clientData client context data + * @param customData custom data + * @param interactionData common data for an interaction schema + */ + /** + * Construct and submits a Metrics Platform Event from the schema id, event name, page metadata, and custom data for + * the stream that is interested in those events. + * + * @param streamName stream name + * @param schemaId schema id + * @param eventName event name + * @param clientData client context data + * @param customData custom data + */ + @JvmOverloads + fun submitMetricsEvent( + streamName: String?, + schemaId: String?, + eventName: String?, + clientData: ClientData, + customData: MutableMap?, + interactionData: InteractionData = null + ) { + if (streamName == null) { + //log.log(Level.FINE, "No stream has been specified, the submitMetricsEvent event is ignored and dropped.") + return + } + + // If we already have stream configs, then we can pre-validate certain conditions and exclude the event from the queue entirely. + var streamConfig: StreamConfig? = null + if (sourceConfig.get() != null) { + streamConfig = sourceConfig.get().getStreamConfigByName(streamName) + if (streamConfig == null) { + //log.log(Level.FINE, "No stream config exists for this stream, the submitMetricsEvent event is ignored and dropped.") + return + } + if (!samplingController.isInSample(streamConfig)) { + //log.log(Level.FINE, "Not in sample, the submitMetricsEvent event is ignored and dropped.") + return + } + } + + val event = Event(schemaId!!, streamName, eventName) + event.clientData = clientData + + if (customData != null) { + event.customData = customData + } + + event.interactionData = interactionData + + if (streamConfig != null && streamConfig.hasSampleConfig()) { + event.sample = streamConfig.sampleConfig + } + + submit(event) + } + + /** + * Submit an interaction event to a stream. + * + * + * An interaction event is meant to represent a basic interaction with some target or some event + * occurring, e.g. the user (**performer**) tapping/clicking a UI element, or an app notifying the + * server of its current state. + * + * @param streamName stream name + * @param eventName event name + * @param clientData client context data + * @param interactionData common data for the base interaction schema + * + * @see [Metrics Platform/Java API](https://wikitech.wikimedia.org/wiki/Metrics_Platform/Java_API) + */ + fun submitInteraction( + streamName: String?, + eventName: String?, + clientData: ClientData, + interactionData: InteractionData + ) { + submitMetricsEvent( + streamName, + METRICS_PLATFORM_SCHEMA_BASE, + eventName, + clientData, + null, + interactionData + ) + } + + /** + * Submit an interaction event to a stream. + * + * + * See above - takes additional parameters (custom data + custom schema id) to submit an interaction event. + * + * @param streamName stream name + * @param schemaId schema id + * @param eventName event name + * @param clientData client context data + * @param interactionData common data for the base interaction schema + * @param customData custom data for the interaction + * + * @see [Metrics Platform/Java API](https://wikitech.wikimedia.org/wiki/Metrics_Platform/Java_API) + */ + fun submitInteraction( + streamName: String?, + schemaId: String?, + eventName: String?, + clientData: ClientData, + interactionData: InteractionData, + customData: MutableMap? + ) { + submitMetricsEvent(streamName, schemaId, eventName, clientData, customData, interactionData) + } + + /** + * Submit a click event to a stream. + * + * @param streamName stream name + * @param clientData client context data + * @param interactionData common data for the base interaction schema + * + * @see [Metrics Platform/Java API](https://wikitech.wikimedia.org/wiki/Metrics_Platform/Java_API) + */ + fun submitClick( + streamName: String?, + clientData: ClientData, + interactionData: InteractionData + ) { + submitMetricsEvent( + streamName, + METRICS_PLATFORM_SCHEMA_BASE, + "click", + clientData, + null, + interactionData + ) + } + + /** + * Submit a click event to a stream with custom data. + * + * @param streamName stream name + * @param schemaId schema id + * @param eventName event name + * @param clientData client context data + * @param customData custom data for the interaction + * @param interactionData common data for the base interaction schema + * + * @see [Metrics Platform/Java API](https://wikitech.wikimedia.org/wiki/Metrics_Platform/Java_API) + */ + fun submitClick( + streamName: String?, + schemaId: String?, + eventName: String?, + clientData: ClientData, + customData: MutableMap?, + interactionData: InteractionData + ) { + submitMetricsEvent(streamName, schemaId, eventName, clientData, customData, interactionData) + } + + /** + * Submit a view event to a stream. + * + * @param streamName stream name + * @param clientData client context data + * @param interactionData common data for the base interaction schema + */ + fun submitView( + streamName: String?, + clientData: ClientData, + interactionData: InteractionData + ) { + submitMetricsEvent( + streamName, + METRICS_PLATFORM_SCHEMA_BASE, + "view", + clientData, + null, + interactionData + ) + } + + /** + * Submit a view event to a stream with custom data. + * + * @param streamName stream name + * @param schemaId schema id + * @param eventName event name + * @param clientData client context data + * @param customData custom data for the interaction + * @param interactionData common data for the base interaction schema + */ + fun submitView( + streamName: String?, + schemaId: String?, + eventName: String?, + clientData: ClientData, + customData: MutableMap?, + interactionData: InteractionData + ) { + submitMetricsEvent(streamName, schemaId, eventName, clientData, customData, interactionData) + } + + /** + * Convenience method to be called when + * [ + * the onPause() activity lifecycle callback](https://developer.android.com/guide/components/activities/activity-lifecycle#onpause) is called. + * + * + * Touches the session so that we can determine whether it's session has expired if and when the + * application is resumed. + */ + fun onAppPause() { + executorService.schedule( + Runnable { eventProcessor.sendEnqueuedEvents() }, + 0, + TimeUnit.MILLISECONDS + ) + sessionController.touchSession() + } + + /** + * Convenience method to be called when + * [ + * the onResume() activity lifecycle callback](https://developer.android.com/guide/components/activities/activity-lifecycle#onresume) is called. + * + * + * Touches the session so that we can determine whether it has expired. + */ + fun onAppResume() { + sessionController.touchSession() + } + + /** + * Closes the session. + */ + fun onAppClose() { + executorService.schedule( + Runnable { eventProcessor.sendEnqueuedEvents() }, + 0, + TimeUnit.MILLISECONDS + ) + sessionController.closeSession() + } + + /** + * Begins a new session and touches the session. + */ + fun resetSession() { + sessionController.beginSession() + } + + /** + * Supplement the outgoing event with additional metadata. + * These include: + * - app_session_id: the current session ID + * - dt: ISO 8601 timestamp + * - domain: hostname + * + * @param event event + */ + private fun addRequiredMetadata(event: EventProcessed) { + event.performerData.sessionId = sessionController.sessionId + event.timestamp = DATE_FORMAT.format(Instant.now()) + event.setDomain(event.clientData.domain) + } + + /** + * Append an enriched event to the queue. + * If the queue is full, we remove the oldest events from the queue to add the current event. + * Number of attempts to add to the queue is 1/50 of the number queue capacity but at least 10 + * + * @param event a processed event + */ + private fun addToEventQueue(event: EventProcessed?) { + var eventQueueAppendAttempts = max(eventQueue.size / 50, 10) + + while (!eventQueue.offer(event)) { + val removedEvent: EventProcessed? = eventQueue.remove() + if (removedEvent != null) { + //log.log(Level.FINE, removedEvent.name + " was dropped so that a newer event could be added to the queue.") + } + if (eventQueueAppendAttempts-- <= 0) break + } + } + + val isFullyInitialized get() = sourceConfig.get() != null + + val isEventQueueEmpty get() = eventQueue.isEmpty() + + class Builder(private val clientData: ClientData) { + private val sourceConfigRef = AtomicReference() + private var eventQueue = LinkedBlockingQueue(10) + private val sessionController = SessionController() + + private val curationController = CurationController() + + private var samplingController: SamplingController? = null + + private val httpClient: OkHttpClient = OkHttpClient() + + private val streamConfigURL: URL = safeURL(StreamConfigFetcher.ANALYTICS_API_ENDPOINT) + + private val isDebug = false + + private val executorService: ScheduledExecutorService = + Executors.newScheduledThreadPool(1, SimpleThreadFactory()) + + private val sourceConfig: SourceConfig? = null + + fun eventQueueCapacity(capacity: Int): Builder { + eventQueue = LinkedBlockingQueue(capacity) + return this + } + + fun build(): MetricsClient { + if (sourceConfig != null) sourceConfigRef.set(sourceConfig) + + if (samplingController == null) { + samplingController = SamplingController(clientData, sessionController) + } + + val eventProcessor = EventProcessor( + ContextController(), + curationController, + sourceConfigRef, + samplingController!!, + EventSenderDefault(gson, httpClient), + eventQueue, + isDebug + ) + + val metricsClient = MetricsClient( + executorService, + sessionController, + samplingController!!, + sourceConfigRef, + eventQueue, + eventProcessor + ) + + return metricsClient + } + + companion object { + private fun safeURL(url: String?): URL { + return URL(url) + } + } + } + + private class SimpleThreadFactory : ThreadFactory { + private val counter = AtomicLong() + + override fun newThread(r: Runnable?): Thread { + return Thread(r, "metrics-client-" + counter.incrementAndGet()) + } + } + + companion object { + @JvmField + val DATE_FORMAT: DateTimeFormatter = DateTimeFormatter + .ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ROOT) + .withZone(ZoneId.of("UTC")) + + const val METRICS_PLATFORM_LIBRARY_VERSION: String = "2.8" + const val METRICS_PLATFORM_BASE_VERSION: String = "1.2.2" + + const val METRICS_PLATFORM_SCHEMA_BASE: String = "/analytics/product_metrics/app/base/$METRICS_PLATFORM_BASE_VERSION" + + fun builder(clientData: ClientData): Builder { + return Builder(clientData) + } + } +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/SamplingController.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/SamplingController.kt similarity index 88% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/SamplingController.kt rename to analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/SamplingController.kt index bed42a8f5aa..3eb5a5373b2 100644 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/SamplingController.kt +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/SamplingController.kt @@ -1,8 +1,8 @@ -package org.wikimedia.metrics_platform +package org.wikimedia.metricsplatform -import org.wikimedia.metrics_platform.config.StreamConfig -import org.wikimedia.metrics_platform.config.sampling.SampleConfig -import org.wikimedia.metrics_platform.context.ClientData +import org.wikimedia.metricsplatform.config.StreamConfig +import org.wikimedia.metricsplatform.config.sampling.SampleConfig +import org.wikimedia.metricsplatform.context.ClientData import java.util.UUID /** diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/SessionController.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/SessionController.kt similarity index 97% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/SessionController.kt rename to analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/SessionController.kt index c1b30844c78..a10cf366c3f 100644 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/SessionController.kt +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/SessionController.kt @@ -1,4 +1,4 @@ -package org.wikimedia.metrics_platform +package org.wikimedia.metricsplatform import java.security.SecureRandom import java.time.Duration diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/CurationFilter.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/CurationFilter.kt similarity index 88% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/CurationFilter.kt rename to analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/CurationFilter.kt index 62767522109..4a226f9fc0c 100644 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/CurationFilter.kt +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/CurationFilter.kt @@ -1,19 +1,19 @@ @file:UseSerializers(InstantSerializer::class) -package org.wikimedia.metrics_platform.config +package org.wikimedia.metricsplatform.config import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable import kotlinx.serialization.UseSerializers -import org.wikimedia.metrics_platform.config.curation.CollectionCurationRules -import org.wikimedia.metrics_platform.config.curation.ComparableCurationRules -import org.wikimedia.metrics_platform.config.curation.CurationRules -import org.wikimedia.metrics_platform.context.AgentData -import org.wikimedia.metrics_platform.context.InstantSerializer -import org.wikimedia.metrics_platform.context.MediawikiData -import org.wikimedia.metrics_platform.context.PageData -import org.wikimedia.metrics_platform.context.PerformerData -import org.wikimedia.metrics_platform.event.EventProcessed +import org.wikimedia.metricsplatform.config.curation.CollectionCurationRules +import org.wikimedia.metricsplatform.config.curation.ComparableCurationRules +import org.wikimedia.metricsplatform.config.curation.CurationRules +import org.wikimedia.metricsplatform.context.AgentData +import org.wikimedia.metricsplatform.context.InstantSerializer +import org.wikimedia.metricsplatform.context.MediawikiData +import org.wikimedia.metricsplatform.context.PageData +import org.wikimedia.metricsplatform.context.PerformerData +import org.wikimedia.metricsplatform.event.EventProcessed import java.time.Instant import java.util.function.Predicate diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/DestinationEventService.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/DestinationEventService.kt new file mode 100644 index 00000000000..280632cbe07 --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/DestinationEventService.kt @@ -0,0 +1,31 @@ +package org.wikimedia.metricsplatform.config + +import com.google.gson.annotations.SerializedName +import java.net.URL + +/** + * Possible event destination endpoints which can be specified in stream configurations. + * For now, we'll assume that we always want to send to ANALYTICS. + * + * https://wikitech.wikimedia.org/wiki/Event_Platform/EventGate#EventGate_clusters + */ +enum class DestinationEventService(baseUri: String) { + @SerializedName("eventgate-analytics-external") + ANALYTICS("https://intake-analytics.wikimedia.org"), + + @SerializedName("eventgate-logging-external") + ERROR_LOGGING("https://intake-logging.wikimedia.org"), + + @SerializedName("eventgate-logging-local") + LOCAL("http://localhost:8192"); + + private val baseUri: URL = URL("$baseUri/v1/events") + + fun getBaseUri(): URL { + return getBaseUri(false) + } + + fun getBaseUri(isDebug: Boolean): URL { + return if (isDebug) this.baseUri else URL("$baseUri?hasty=true") + } +} diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/SourceConfig.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/SourceConfig.kt similarity index 95% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/SourceConfig.kt rename to analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/SourceConfig.kt index 733ad98721c..aca743606b3 100644 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/SourceConfig.kt +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/SourceConfig.kt @@ -1,4 +1,4 @@ -package org.wikimedia.metrics_platform.config +package org.wikimedia.metricsplatform.config class SourceConfig(streamConfigs: Map) { /** diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/StreamConfig.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfig.kt similarity index 92% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/StreamConfig.kt rename to analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfig.kt index 4e42b72e0e0..b87baa9efd9 100644 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/StreamConfig.kt +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfig.kt @@ -1,8 +1,8 @@ -package org.wikimedia.metrics_platform.config +package org.wikimedia.metricsplatform.config import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable -import org.wikimedia.metrics_platform.config.sampling.SampleConfig +import org.wikimedia.metricsplatform.config.sampling.SampleConfig @Serializable class StreamConfig { @@ -24,7 +24,7 @@ class StreamConfig { get() = producerConfig?.metricsPlatformClientConfig?.events.orEmpty() fun getDestinationEventService(): DestinationEventService { - return (if (destinationEventService != null) destinationEventService else DestinationEventService.ANALYTICS)!! + return if (destinationEventService != null) destinationEventService!! else DestinationEventService.ANALYTICS } fun hasCurationFilter(): Boolean { diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/StreamConfigCollection.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfigCollection.kt similarity index 82% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/StreamConfigCollection.kt rename to analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfigCollection.kt index cf6352d75df..bfed994e15c 100644 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/StreamConfigCollection.kt +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfigCollection.kt @@ -1,4 +1,4 @@ -package org.wikimedia.metrics_platform.config +package org.wikimedia.metricsplatform.config import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/StreamConfigFetcher.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfigFetcher.java similarity index 97% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/StreamConfigFetcher.java rename to analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfigFetcher.java index c815e792004..18f0486d982 100644 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/StreamConfigFetcher.java +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfigFetcher.java @@ -1,4 +1,4 @@ -package org.wikimedia.metrics_platform.config; +package org.wikimedia.metricsplatform.config; import java.io.IOException; import java.io.Reader; diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/CollectionCurationRules.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/curation/CollectionCurationRules.kt similarity index 94% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/CollectionCurationRules.kt rename to analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/curation/CollectionCurationRules.kt index 53d8506cf08..b65c97c5e05 100644 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/CollectionCurationRules.kt +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/curation/CollectionCurationRules.kt @@ -1,4 +1,4 @@ -package org.wikimedia.metrics_platform.config.curation +package org.wikimedia.metricsplatform.config.curation import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/ComparableCurationRules.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/curation/ComparableCurationRules.kt similarity index 96% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/ComparableCurationRules.kt rename to analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/curation/ComparableCurationRules.kt index fd8f23a7f99..632b9808f70 100644 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/ComparableCurationRules.kt +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/curation/ComparableCurationRules.kt @@ -1,4 +1,4 @@ -package org.wikimedia.metrics_platform.config.curation +package org.wikimedia.metricsplatform.config.curation import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/CurationRules.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/curation/CurationRules.kt similarity index 93% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/CurationRules.kt rename to analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/curation/CurationRules.kt index ef352151547..7a015958ea6 100644 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/curation/CurationRules.kt +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/curation/CurationRules.kt @@ -1,4 +1,4 @@ -package org.wikimedia.metrics_platform.config.curation +package org.wikimedia.metricsplatform.config.curation import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/sampling/SampleConfig.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/sampling/SampleConfig.kt similarity index 84% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/sampling/SampleConfig.kt rename to analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/sampling/SampleConfig.kt index 698c85ef586..2e79a09225e 100644 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/config/sampling/SampleConfig.kt +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/sampling/SampleConfig.kt @@ -1,4 +1,4 @@ -package org.wikimedia.metrics_platform.config.sampling +package org.wikimedia.metricsplatform.config.sampling import kotlinx.serialization.Serializable diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/AgentData.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/AgentData.kt new file mode 100644 index 00000000000..f492a5969a9 --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/AgentData.kt @@ -0,0 +1,24 @@ +package org.wikimedia.metricsplatform.context + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable + +/** + * Agent context data fields. + * + * All fields are nullable, and boxed types are used in place of their equivalent primitive types to avoid + * unexpected default values from being used where the true value is null. + */ +@Serializable +class AgentData( + @SerialName("app_flavor") var appFlavor: String? = null, + @SerialName("app_install_id") var appInstallId: String? = null, + @SerialName("app_theme") var appTheme: String? = null, + @SerialName("app_version") var appVersion: Int? = null, + @SerialName("app_version_name") var appVersionName: String? = null, + @SerialName("client_platform") var clientPlatform: String? = null, + @SerialName("client_platform_family") var clientPlatformFamily: String? = null, + @SerialName("device_family") var deviceFamily: String? = null, + @SerialName("device_language") var deviceLanguage: String? = null, + @SerialName("release_status") var releaseStatus: String? = null, +) diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/ClientData.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/ClientData.kt similarity index 94% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/ClientData.kt rename to analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/ClientData.kt index af4f921c11c..33c88b41921 100644 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/ClientData.kt +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/ClientData.kt @@ -1,4 +1,4 @@ -package org.wikimedia.metrics_platform.context +package org.wikimedia.metricsplatform.context import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/ContextValue.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/ContextValue.kt similarity index 97% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/ContextValue.kt rename to analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/ContextValue.kt index 478b6a74918..3c424ccda87 100644 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/ContextValue.kt +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/ContextValue.kt @@ -1,4 +1,4 @@ -package org.wikimedia.metrics_platform.context +package org.wikimedia.metricsplatform.context /** * @see [Metrics Platform/Contextual attributes](https://wikitech.wikimedia.org/wiki/Metrics_Platform/Contextual_attributes) diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/CustomData.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/CustomData.kt similarity index 95% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/CustomData.kt rename to analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/CustomData.kt index 2f949252c6f..0d069e1e5cd 100644 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/CustomData.kt +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/CustomData.kt @@ -1,4 +1,4 @@ -package org.wikimedia.metrics_platform.context +package org.wikimedia.metricsplatform.context import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/CustomDataType.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/CustomDataType.kt similarity index 81% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/CustomDataType.kt rename to analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/CustomDataType.kt index 50d6b45429e..5517db9d47f 100644 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/CustomDataType.kt +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/CustomDataType.kt @@ -1,4 +1,4 @@ -package org.wikimedia.metrics_platform.context +package org.wikimedia.metricsplatform.context import kotlinx.serialization.SerialName diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/InstantSerializer.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/InstantSerializer.kt similarity index 93% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/InstantSerializer.kt rename to analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/InstantSerializer.kt index b75c9738651..89ea029c01a 100644 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/InstantSerializer.kt +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/InstantSerializer.kt @@ -1,4 +1,4 @@ -package org.wikimedia.metrics_platform.context +package org.wikimedia.metricsplatform.context import kotlinx.serialization.KSerializer import kotlinx.serialization.descriptors.PrimitiveKind diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/InteractionData.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/InteractionData.kt similarity index 94% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/InteractionData.kt rename to analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/InteractionData.kt index 73d0d34bbc6..8cbad584d43 100644 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/InteractionData.kt +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/InteractionData.kt @@ -1,4 +1,4 @@ -package org.wikimedia.metrics_platform.context +package org.wikimedia.metricsplatform.context import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/MediawikiData.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/MediawikiData.kt similarity index 76% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/MediawikiData.kt rename to analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/MediawikiData.kt index 1b9913da849..d1b3957e613 100644 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/MediawikiData.kt +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/MediawikiData.kt @@ -1,4 +1,4 @@ -package org.wikimedia.metrics_platform.context +package org.wikimedia.metricsplatform.context import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable @@ -11,5 +11,5 @@ import kotlinx.serialization.Serializable */ @Serializable class MediawikiData( - @SerialName("database") val database: String? = null + @SerialName("database") var database: String? = null ) diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/PageData.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/PageData.kt similarity index 56% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/PageData.kt rename to analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/PageData.kt index 8ef13068464..f67c9a32b17 100644 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/context/PageData.kt +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/PageData.kt @@ -1,4 +1,4 @@ -package org.wikimedia.metrics_platform.context +package org.wikimedia.metricsplatform.context import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable @@ -14,11 +14,11 @@ import kotlinx.serialization.Serializable */ @Serializable class PageData( - val id: Int? = null, - val title: String? = null, - @SerialName("namespace_id") val namespaceId: Int? = null, - @SerialName("namespace_name") val namespaceName: String? = null, - @SerialName("revision_id") val revisionId: Long? = null, - @SerialName("wikidata_qid") val wikidataItemQid: String? = null, - @SerialName("content_language") val contentLanguage: String? = null, + var id: Int? = null, + var title: String? = null, + @SerialName("namespace_id") var namespaceId: Int? = null, + @SerialName("namespace_name") var namespaceName: String? = null, + @SerialName("revision_id") var revisionId: Long? = null, + @SerialName("wikidata_qid") var wikidataItemQid: String? = null, + @SerialName("content_language") var contentLanguage: String? = null, ) diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/PerformerData.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/PerformerData.kt new file mode 100644 index 00000000000..9c9aa4ec8f3 --- /dev/null +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/PerformerData.kt @@ -0,0 +1,28 @@ +@file:UseSerializers(InstantSerializer::class) + +package org.wikimedia.metricsplatform.context + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import kotlinx.serialization.UseSerializers +import java.time.Instant + +/** + * Performer context data fields. + * + * All fields are nullable, and boxed types are used in place of their equivalent primitive types to avoid + * unexpected default values from being used where the true value is null. + */ +@Serializable +data class PerformerData ( + var id: Int? = null, + @SerialName("name") var name: String? = null, + @SerialName("is_logged_in") var isLoggedIn: Boolean? = null, + @SerialName("is_temp") var isTemp: Boolean? = null, + @SerialName("session_id") var sessionId: String? = null, + @SerialName("pageview_id") var pageviewId: String? = null, + @SerialName("groups") var groups: MutableCollection? = null, + @SerialName("language_groups") var languageGroups: String? = null, + @SerialName("language_primary") var languagePrimary: String? = null, + @SerialName("registration_dt") var registrationDt: Instant? = null +) diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/event/Event.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/event/Event.kt similarity index 64% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/event/Event.kt rename to analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/event/Event.kt index 0bc8232aa05..b98486dfdf0 100644 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/event/Event.kt +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/event/Event.kt @@ -1,20 +1,20 @@ -package org.wikimedia.metrics_platform.event +package org.wikimedia.metricsplatform.event import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable import kotlinx.serialization.Transient -import org.wikimedia.metrics_platform.config.sampling.SampleConfig -import org.wikimedia.metrics_platform.context.ClientData -import org.wikimedia.metrics_platform.context.InteractionData +import org.wikimedia.metricsplatform.config.sampling.SampleConfig +import org.wikimedia.metricsplatform.context.ClientData +import org.wikimedia.metricsplatform.context.InteractionData @Serializable open class Event(@Transient val _stream: String = "") { - val name: String? = null + var name: String? = null @SerialName("\$schema") var schema: String? = null - @SerialName("dt") protected var timestamp: String? = null + @SerialName("dt") var timestamp: String? = null @SerialName("custom_data") var customData: MutableMap? = null @@ -26,7 +26,9 @@ open class Event(@Transient val _stream: String = "") { @SerialName("interaction_data") var interactionData: InteractionData = InteractionData() - val stream: String get() = meta.stream + var stream + get() = meta.stream + set(value) { meta.stream = value } fun setDomain(domain: String?) { meta.domain = domain @@ -34,7 +36,7 @@ open class Event(@Transient val _stream: String = "") { @Serializable protected class Meta( - val stream: String = "", + var stream: String = "", var domain: String? = null ) } diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/event/EventProcessed.kt b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/event/EventProcessed.kt similarity index 81% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/event/EventProcessed.kt rename to analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/event/EventProcessed.kt index dbe573d511d..973bebdb8b0 100644 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/event/EventProcessed.kt +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/event/EventProcessed.kt @@ -1,14 +1,14 @@ -package org.wikimedia.metrics_platform.event +package org.wikimedia.metricsplatform.event import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable -import org.wikimedia.metrics_platform.config.sampling.SampleConfig -import org.wikimedia.metrics_platform.context.AgentData -import org.wikimedia.metrics_platform.context.ClientData -import org.wikimedia.metrics_platform.context.InteractionData -import org.wikimedia.metrics_platform.context.MediawikiData -import org.wikimedia.metrics_platform.context.PageData -import org.wikimedia.metrics_platform.context.PerformerData +import org.wikimedia.metricsplatform.config.sampling.SampleConfig +import org.wikimedia.metricsplatform.context.AgentData +import org.wikimedia.metricsplatform.context.ClientData +import org.wikimedia.metricsplatform.context.InteractionData +import org.wikimedia.metricsplatform.context.MediawikiData +import org.wikimedia.metricsplatform.context.PageData +import org.wikimedia.metricsplatform.context.PerformerData @Serializable class EventProcessed : Event { @@ -16,7 +16,8 @@ class EventProcessed : Event { @SerialName("page") var pageData: PageData = PageData() @SerialName("mediawiki") var mediawikiData: MediawikiData = MediawikiData() @SerialName("performer") var performerData: PerformerData = PerformerData() - @SerialName("action") var action: String + + @SerialName("action") var action: String? = null @SerialName("action_subtype") private var actionSubtype: String? = null @SerialName("action_source") private var actionSource: String? = null @SerialName("action_context") private var actionContext: String? = null @@ -33,12 +34,9 @@ class EventProcessed : Event { * @param name event name * @param clientData agent, mediawiki, page, performer data */ - constructor( - schema: String?, - stream: String?, - @Nonnull name: String?, - clientData: ClientData - ) : super(schema!!, stream, name) { + constructor(schema: String?, stream: String, name: String, clientData: ClientData) : super(schema!!) { + this.stream = stream + this.name = name this.clientData = clientData this.agentData = clientData.agentData this.pageData = clientData.pageData @@ -67,13 +65,15 @@ class EventProcessed : Event { */ constructor( schema: String?, - stream: String?, + stream: String, name: String?, customData: MutableMap?, clientData: ClientData, sample: SampleConfig?, interactionData: InteractionData - ) : super(schema!!, stream, name) { + ) : super(schema!!) { + this.stream = stream + this.name = name this.clientData = clientData this.agentData = clientData.agentData this.pageData = clientData.pageData @@ -85,7 +85,7 @@ class EventProcessed : Event { this.action = interactionData.action } - public override fun setClientData(@Nonnull clientData: ClientData) { + fun setClientData(clientData: ClientData) { setAgentData(clientData.agentData) setPageData(clientData.pageData) setMediawikiData(clientData.mediawikiData) @@ -93,7 +93,7 @@ class EventProcessed : Event { this.clientData = clientData } - public override fun setInteractionData(@Nonnull interactionData: InteractionData) { + fun setInteractionData(interactionData: InteractionData) { this.action = interactionData.action this.actionContext = interactionData.actionContext this.actionSource = interactionData.actionSource diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/event/EventProcessedSerializer.java b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/event/EventProcessedSerializer.java similarity index 74% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/event/EventProcessedSerializer.java rename to analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/event/EventProcessedSerializer.java index 056fd04f4fe..015c68c4f11 100644 --- a/analytics/metrics-platform/src/main/java/org/wikimedia/metrics_platform/event/EventProcessedSerializer.java +++ b/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/event/EventProcessedSerializer.java @@ -1,50 +1,22 @@ -package org.wikimedia.metrics_platform.event; +package org.wikimedia.metricsplatform.event; -import static java.util.logging.Level.INFO; - -import java.lang.reflect.Type; -import java.time.Instant; -import java.util.Map; - -import org.wikimedia.metrics_platform.context.InstantConverter; - -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonSerializer; -import com.google.gson.JsonSerializationContext; - -public class EventProcessedSerializer implements JsonSerializer { - - /** - * Add gson builder for handling dt properties inside internal serializer. - *

- * Because this is a new serializer, including it as a new type adapter in the GsonHelper - * utility causes issues with circularity. - */ +public class EventProcessedSerializer { + /* private final Gson gson = new GsonBuilder() .registerTypeAdapter(Instant.class, new InstantConverter()) .create(); - /** - * EventProcessed serializer - *

- * Rather than serializing all properties individually, this serializer starts from default serialization - * and modifies a few properties (custom data, name, etc.). - */ @Override public JsonElement serialize(EventProcessed src, Type type, JsonSerializationContext jsonSerializationContext) { JsonElement jsonElement = gson.toJsonTree(src); JsonObject jsonObject = (JsonObject) jsonElement; - /* * Custom data can be passed into the EventProcessed constructor as a map * of key-value pairs. EventProcessed inherits from Event which contains a * customData property that would be serialized by default. To validate * successfully against the corresponding schema, custom data must be sent * as top-level properties with the event. - */ + if (src.customData != null) { int customDataAdded = 0; int customDataCount = src.customData.size(); @@ -69,7 +41,6 @@ public JsonElement serialize(EventProcessed src, Type type, JsonSerializationCon jsonObject.remove("client_data"); } - /* * Removing a few properties here because it would require adding annotations on every Event and EventProcessed * property just to exclude few properties from serialization. * @@ -80,18 +51,16 @@ public JsonElement serialize(EventProcessed src, Type type, JsonSerializationCon * * Once Metrics Platform core interactions schemas are updated to include the "sample" property, the line to * remove it can be deleted here. - */ jsonObject.remove("name"); jsonObject.remove("sample"); - /* * Remove the top level data objects from EventProcessed which are * inherited from its superclass Event. The values in "client_data" * and "interaction_data" are set as top level properties in * EventProcessed's constructor. - */ jsonObject.remove("client_data"); jsonObject.remove("interaction_data"); return jsonObject; } + */ } diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/ConsistencyIT.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/ConsistencyIT.java similarity index 88% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/ConsistencyIT.java rename to analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/ConsistencyIT.java index 10deb207828..bfbabe25fdb 100644 --- a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/ConsistencyIT.java +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/ConsistencyIT.java @@ -1,10 +1,10 @@ -package org.wikimedia.metrics_platform; +package org.wikimedia.metricsplatform; import static java.util.Collections.singletonMap; import static org.assertj.core.api.Assertions.assertThat; -import static org.wikimedia.metrics_platform.ConsistencyITClientData.createConsistencyTestClientData; -import static org.wikimedia.metrics_platform.config.StreamConfigFetcher.ANALYTICS_API_ENDPOINT; -import static org.wikimedia.metrics_platform.MetricsClient.METRICS_PLATFORM_SCHEMA_BASE; +import static org.wikimedia.metricsplatform.ConsistencyITClientData.createConsistencyTestClientData; +import static org.wikimedia.metricsplatform.config.StreamConfigFetcher.ANALYTICS_API_ENDPOINT; +import static org.wikimedia.metricsplatform.MetricsClient.METRICS_PLATFORM_SCHEMA_BASE; import java.io.BufferedReader; import java.io.IOException; @@ -20,13 +20,13 @@ import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.Test; -import org.wikimedia.metrics_platform.config.SourceConfig; -import org.wikimedia.metrics_platform.config.StreamConfig; -import org.wikimedia.metrics_platform.config.StreamConfigFetcher; -import org.wikimedia.metrics_platform.context.ClientData; -import org.wikimedia.metrics_platform.context.DataFixtures; -import org.wikimedia.metrics_platform.event.EventProcessed; -import org.wikimedia.metrics_platform.json.GsonHelper; +import org.wikimedia.metricsplatform.config.SourceConfig; +import org.wikimedia.metricsplatform.config.StreamConfig; +import org.wikimedia.metricsplatform.config.StreamConfigFetcher; +import org.wikimedia.metricsplatform.context.ClientData; +import org.wikimedia.metricsplatform.context.DataFixtures; +import org.wikimedia.metricsplatform.event.EventProcessed; +import org.wikimedia.metricsplatform.json.GsonHelper; import com.google.gson.Gson; import com.google.gson.JsonObject; diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/ConsistencyITClientData.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/ConsistencyITClientData.java similarity index 92% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/ConsistencyITClientData.java rename to analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/ConsistencyITClientData.java index 3aea3509379..5f02ad69668 100644 --- a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/ConsistencyITClientData.java +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/ConsistencyITClientData.java @@ -1,4 +1,4 @@ -package org.wikimedia.metrics_platform; +package org.wikimedia.metricsplatform; import java.io.BufferedReader; import java.io.IOException; @@ -7,11 +7,11 @@ import java.nio.file.Paths; import java.util.Collections; -import org.wikimedia.metrics_platform.context.AgentData; -import org.wikimedia.metrics_platform.context.ClientData; -import org.wikimedia.metrics_platform.context.MediawikiData; -import org.wikimedia.metrics_platform.context.PageData; -import org.wikimedia.metrics_platform.context.PerformerData; +import org.wikimedia.metricsplatform.context.AgentData; +import org.wikimedia.metricsplatform.context.ClientData; +import org.wikimedia.metricsplatform.context.MediawikiData; +import org.wikimedia.metricsplatform.context.PageData; +import org.wikimedia.metricsplatform.context.PerformerData; import com.google.gson.JsonElement; import com.google.gson.JsonObject; diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/ContextControllerTest.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/ContextControllerTest.java similarity index 79% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/ContextControllerTest.java rename to analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/ContextControllerTest.java index 2eb564ed6a2..7e817e31f49 100644 --- a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/ContextControllerTest.java +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/ContextControllerTest.java @@ -1,19 +1,19 @@ -package org.wikimedia.metrics_platform; +package org.wikimedia.metricsplatform; import static org.assertj.core.api.Assertions.assertThat; -import static org.wikimedia.metrics_platform.event.EventProcessed.fromEvent; +import static org.wikimedia.metricsplatform.event.EventProcessed.fromEvent; import org.junit.jupiter.api.Test; -import org.wikimedia.metrics_platform.config.StreamConfig; -import org.wikimedia.metrics_platform.config.StreamConfigFixtures; -import org.wikimedia.metrics_platform.context.AgentData; -import org.wikimedia.metrics_platform.context.ClientData; -import org.wikimedia.metrics_platform.context.DataFixtures; -import org.wikimedia.metrics_platform.context.MediawikiData; -import org.wikimedia.metrics_platform.context.PageData; -import org.wikimedia.metrics_platform.context.PerformerData; -import org.wikimedia.metrics_platform.event.Event; -import org.wikimedia.metrics_platform.event.EventProcessed; +import org.wikimedia.metricsplatform.config.StreamConfig; +import org.wikimedia.metricsplatform.config.StreamConfigFixtures; +import org.wikimedia.metricsplatform.context.AgentData; +import org.wikimedia.metricsplatform.context.ClientData; +import org.wikimedia.metricsplatform.context.DataFixtures; +import org.wikimedia.metricsplatform.context.MediawikiData; +import org.wikimedia.metricsplatform.context.PageData; +import org.wikimedia.metricsplatform.context.PerformerData; +import org.wikimedia.metricsplatform.event.Event; +import org.wikimedia.metricsplatform.event.EventProcessed; class ContextControllerTest { diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/CurationControllerTest.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/CurationControllerTest.java similarity index 89% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/CurationControllerTest.java rename to analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/CurationControllerTest.java index f11d3f6e458..20ce5c3b602 100644 --- a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/CurationControllerTest.java +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/CurationControllerTest.java @@ -1,7 +1,7 @@ -package org.wikimedia.metrics_platform; +package org.wikimedia.metricsplatform; import static org.assertj.core.api.Assertions.assertThat; -import static org.wikimedia.metrics_platform.event.EventFixtures.getEvent; +import static org.wikimedia.metricsplatform.event.EventFixtures.getEvent; import java.time.Instant; import java.util.Arrays; @@ -10,12 +10,12 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.wikimedia.metrics_platform.config.StreamConfig; -import org.wikimedia.metrics_platform.config.StreamConfigFixtures; -import org.wikimedia.metrics_platform.context.PerformerData; -import org.wikimedia.metrics_platform.json.GsonHelper; -import org.wikimedia.metrics_platform.config.CurationFilter; -import org.wikimedia.metrics_platform.event.EventProcessed; +import org.wikimedia.metricsplatform.config.StreamConfig; +import org.wikimedia.metricsplatform.config.StreamConfigFixtures; +import org.wikimedia.metricsplatform.context.PerformerData; +import org.wikimedia.metricsplatform.json.GsonHelper; +import org.wikimedia.metricsplatform.config.CurationFilter; +import org.wikimedia.metricsplatform.event.EventProcessed; import com.google.gson.Gson; diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/DestinationEventServiceTest.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/DestinationEventServiceTest.java similarity index 94% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/DestinationEventServiceTest.java rename to analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/DestinationEventServiceTest.java index fab66eb7ecc..619f6834a73 100644 --- a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/DestinationEventServiceTest.java +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/DestinationEventServiceTest.java @@ -1,4 +1,4 @@ -package org.wikimedia.metrics_platform; +package org.wikimedia.metricsplatform; import static org.assertj.core.api.Assertions.assertThat; @@ -6,7 +6,7 @@ import java.net.URL; import org.junit.jupiter.api.Test; -import org.wikimedia.metrics_platform.config.DestinationEventService; +import org.wikimedia.metricsplatform.config.DestinationEventService; class DestinationEventServiceTest { diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/EndToEndIT.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/EndToEndIT.java similarity index 96% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/EndToEndIT.java rename to analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/EndToEndIT.java index 38a20351c39..fff8ba18361 100644 --- a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/EndToEndIT.java +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/EndToEndIT.java @@ -1,4 +1,4 @@ -package org.wikimedia.metrics_platform; +package org.wikimedia.metricsplatform; import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; import static com.github.tomakehurst.wiremock.client.WireMock.equalToJson; @@ -11,8 +11,8 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.concurrent.TimeUnit.SECONDS; import static org.awaitility.Awaitility.await; -import static org.wikimedia.metrics_platform.ConsistencyITClientData.createConsistencyTestClientData; -import static org.wikimedia.metrics_platform.context.DataFixtures.getTestCustomData; +import static org.wikimedia.metricsplatform.ConsistencyITClientData.createConsistencyTestClientData; +import static org.wikimedia.metricsplatform.context.DataFixtures.getTestCustomData; import java.io.IOException; import java.net.MalformedURLException; @@ -20,8 +20,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.wikimedia.metrics_platform.context.ClientData; -import org.wikimedia.metrics_platform.context.DataFixtures; +import org.wikimedia.metricsplatform.context.ClientData; +import org.wikimedia.metricsplatform.context.DataFixtures; import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; import com.github.tomakehurst.wiremock.junit5.WireMockTest; diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/EventProcessorTest.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/EventProcessorTest.java similarity index 94% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/EventProcessorTest.java rename to analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/EventProcessorTest.java index cba16779ab3..b4c003a7cfa 100644 --- a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/EventProcessorTest.java +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/EventProcessorTest.java @@ -1,4 +1,4 @@ -package org.wikimedia.metrics_platform; +package org.wikimedia.metricsplatform; import static java.lang.Boolean.FALSE; import static java.lang.Boolean.TRUE; @@ -9,7 +9,7 @@ import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import static org.wikimedia.metrics_platform.event.EventFixtures.minimalEventProcessed; +import static org.wikimedia.metricsplatform.event.EventFixtures.minimalEventProcessed; import java.io.IOException; import java.net.URL; @@ -25,11 +25,11 @@ import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.wikimedia.metrics_platform.config.SourceConfig; -import org.wikimedia.metrics_platform.config.SourceConfigFixtures; -import org.wikimedia.metrics_platform.config.StreamConfig; -import org.wikimedia.metrics_platform.context.ClientData; -import org.wikimedia.metrics_platform.event.EventProcessed; +import org.wikimedia.metricsplatform.config.SourceConfig; +import org.wikimedia.metricsplatform.config.SourceConfigFixtures; +import org.wikimedia.metricsplatform.config.StreamConfig; +import org.wikimedia.metricsplatform.context.ClientData; +import org.wikimedia.metricsplatform.event.EventProcessed; @ExtendWith(MockitoExtension.class) class EventProcessorTest { diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/EventTest.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/EventTest.java similarity index 93% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/EventTest.java rename to analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/EventTest.java index a51070c2534..674cf7b6a61 100644 --- a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/EventTest.java +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/EventTest.java @@ -1,8 +1,8 @@ -package org.wikimedia.metrics_platform; +package org.wikimedia.metricsplatform; import static org.assertj.core.api.Assertions.assertThat; -import static org.wikimedia.metrics_platform.MetricsClient.DATE_FORMAT; -import static org.wikimedia.metrics_platform.event.EventProcessed.fromEvent; +import static org.wikimedia.metricsplatform.MetricsClient.DATE_FORMAT; +import static org.wikimedia.metricsplatform.event.EventProcessed.fromEvent; import java.time.Instant; import java.util.Collections; @@ -10,12 +10,12 @@ import java.util.UUID; import org.junit.jupiter.api.Test; -import org.wikimedia.metrics_platform.context.AgentData; -import org.wikimedia.metrics_platform.context.ClientData; -import org.wikimedia.metrics_platform.context.DataFixtures; -import org.wikimedia.metrics_platform.event.Event; -import org.wikimedia.metrics_platform.event.EventProcessed; -import org.wikimedia.metrics_platform.json.GsonHelper; +import org.wikimedia.metricsplatform.context.AgentData; +import org.wikimedia.metricsplatform.context.ClientData; +import org.wikimedia.metricsplatform.context.DataFixtures; +import org.wikimedia.metricsplatform.event.Event; +import org.wikimedia.metricsplatform.event.EventProcessed; +import org.wikimedia.metricsplatform.json.GsonHelper; import com.google.gson.Gson; diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/MetricsClientTest.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/MetricsClientTest.java similarity index 91% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/MetricsClientTest.java rename to analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/MetricsClientTest.java index d106f4393ac..36d7b70e5bc 100644 --- a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/MetricsClientTest.java +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/MetricsClientTest.java @@ -1,14 +1,14 @@ -package org.wikimedia.metrics_platform; +package org.wikimedia.metricsplatform; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import static org.wikimedia.metrics_platform.MetricsClient.METRICS_PLATFORM_SCHEMA_BASE; -import static org.wikimedia.metrics_platform.config.StreamConfigFixtures.streamConfig; -import static org.wikimedia.metrics_platform.context.DataFixtures.getTestClientData; -import static org.wikimedia.metrics_platform.context.DataFixtures.getTestCustomData; -import static org.wikimedia.metrics_platform.curation.CurationFilterFixtures.curationFilter; -import static org.wikimedia.metrics_platform.event.EventProcessed.fromEvent; +import static org.wikimedia.metricsplatform.MetricsClient.METRICS_PLATFORM_SCHEMA_BASE; +import static org.wikimedia.metricsplatform.config.StreamConfigFixtures.streamConfig; +import static org.wikimedia.metricsplatform.context.DataFixtures.getTestClientData; +import static org.wikimedia.metricsplatform.context.DataFixtures.getTestCustomData; +import static org.wikimedia.metricsplatform.curation.CurationFilterFixtures.curationFilter; +import static org.wikimedia.metricsplatform.event.EventProcessed.fromEvent; import java.util.Map; import java.util.concurrent.BlockingQueue; @@ -20,16 +20,16 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.wikimedia.metrics_platform.config.SourceConfig; -import org.wikimedia.metrics_platform.config.SourceConfigFixtures; -import org.wikimedia.metrics_platform.config.StreamConfig; -import org.wikimedia.metrics_platform.context.ClientData; -import org.wikimedia.metrics_platform.context.DataFixtures; -import org.wikimedia.metrics_platform.context.InteractionData; -import org.wikimedia.metrics_platform.context.PageData; -import org.wikimedia.metrics_platform.context.PerformerData; -import org.wikimedia.metrics_platform.event.Event; -import org.wikimedia.metrics_platform.event.EventProcessed; +import org.wikimedia.metricsplatform.config.SourceConfig; +import org.wikimedia.metricsplatform.config.SourceConfigFixtures; +import org.wikimedia.metricsplatform.config.StreamConfig; +import org.wikimedia.metricsplatform.context.ClientData; +import org.wikimedia.metricsplatform.context.DataFixtures; +import org.wikimedia.metricsplatform.context.InteractionData; +import org.wikimedia.metricsplatform.context.PageData; +import org.wikimedia.metricsplatform.context.PerformerData; +import org.wikimedia.metricsplatform.event.Event; +import org.wikimedia.metricsplatform.event.EventProcessed; @ExtendWith(MockitoExtension.class) class MetricsClientTest { diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/SamplingControllerTest.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/SamplingControllerTest.java similarity index 81% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/SamplingControllerTest.java rename to analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/SamplingControllerTest.java index 80d937fd810..898ecfbbd5e 100644 --- a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/SamplingControllerTest.java +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/SamplingControllerTest.java @@ -1,13 +1,13 @@ -package org.wikimedia.metrics_platform; +package org.wikimedia.metricsplatform; import static org.assertj.core.api.Assertions.assertThat; -import static org.wikimedia.metrics_platform.config.sampling.SampleConfig.Identifier.DEVICE; -import static org.wikimedia.metrics_platform.config.sampling.SampleConfig.Identifier.SESSION; +import static org.wikimedia.metricsplatform.config.sampling.SampleConfig.Identifier.DEVICE; +import static org.wikimedia.metricsplatform.config.sampling.SampleConfig.Identifier.SESSION; import org.junit.jupiter.api.Test; -import org.wikimedia.metrics_platform.config.sampling.SampleConfig; -import org.wikimedia.metrics_platform.config.StreamConfig; -import org.wikimedia.metrics_platform.context.DataFixtures; +import org.wikimedia.metricsplatform.config.sampling.SampleConfig; +import org.wikimedia.metricsplatform.config.StreamConfig; +import org.wikimedia.metricsplatform.context.DataFixtures; class SamplingControllerTest { diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/SessionControllerTest.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/SessionControllerTest.java similarity index 96% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/SessionControllerTest.java rename to analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/SessionControllerTest.java index 262dc106a31..1252a32d028 100644 --- a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/SessionControllerTest.java +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/SessionControllerTest.java @@ -1,4 +1,4 @@ -package org.wikimedia.metrics_platform; +package org.wikimedia.metricsplatform; import static java.time.temporal.ChronoUnit.HOURS; import static org.assertj.core.api.Assertions.assertThat; diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/TestEventSender.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/TestEventSender.java similarity index 83% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/TestEventSender.java rename to analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/TestEventSender.java index 1bb196bd012..39fff38ff5e 100644 --- a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/TestEventSender.java +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/TestEventSender.java @@ -1,10 +1,10 @@ -package org.wikimedia.metrics_platform; +package org.wikimedia.metricsplatform; import java.io.IOException; import java.net.URL; import java.util.Collection; -import org.wikimedia.metrics_platform.event.EventProcessed; +import org.wikimedia.metricsplatform.event.EventProcessed; class TestEventSender implements EventSender { diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/CurationFilterFixtures.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/CurationFilterFixtures.java similarity index 78% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/CurationFilterFixtures.java rename to analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/CurationFilterFixtures.java index 17631cab12f..d3c6c9e61d5 100644 --- a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/CurationFilterFixtures.java +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/CurationFilterFixtures.java @@ -1,9 +1,9 @@ -package org.wikimedia.metrics_platform.config; +package org.wikimedia.metricsplatform.config; import java.util.Arrays; -import org.wikimedia.metrics_platform.config.curation.CollectionCurationRules; -import org.wikimedia.metrics_platform.config.curation.CurationRules; +import org.wikimedia.metricsplatform.config.curation.CollectionCurationRules; +import org.wikimedia.metricsplatform.config.curation.CurationRules; public final class CurationFilterFixtures { diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/SampleConfigTest.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/SampleConfigTest.java similarity index 67% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/SampleConfigTest.java rename to analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/SampleConfigTest.java index 9a6083a1d67..8e53c18f714 100644 --- a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/SampleConfigTest.java +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/SampleConfigTest.java @@ -1,11 +1,11 @@ -package org.wikimedia.metrics_platform.config; +package org.wikimedia.metricsplatform.config; import static org.assertj.core.api.Assertions.assertThat; -import static org.wikimedia.metrics_platform.config.sampling.SampleConfig.Identifier.DEVICE; +import static org.wikimedia.metricsplatform.config.sampling.SampleConfig.Identifier.DEVICE; import org.junit.jupiter.api.Test; -import org.wikimedia.metrics_platform.config.sampling.SampleConfig; -import org.wikimedia.metrics_platform.json.GsonHelper; +import org.wikimedia.metricsplatform.config.sampling.SampleConfig; +import org.wikimedia.metricsplatform.json.GsonHelper; import com.google.gson.Gson; diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/SourceConfigFixtures.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/SourceConfigFixtures.java similarity index 93% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/SourceConfigFixtures.java rename to analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/SourceConfigFixtures.java index 506ca2d2a4d..ff02615c4c9 100644 --- a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/SourceConfigFixtures.java +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/SourceConfigFixtures.java @@ -1,4 +1,4 @@ -package org.wikimedia.metrics_platform.config; +package org.wikimedia.metricsplatform.config; public final class SourceConfigFixtures { diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/SourceConfigTest.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/SourceConfigTest.java similarity index 96% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/SourceConfigTest.java rename to analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/SourceConfigTest.java index e7c91b9519c..c7f878eb4ad 100644 --- a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/SourceConfigTest.java +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/SourceConfigTest.java @@ -1,4 +1,4 @@ -package org.wikimedia.metrics_platform.config; +package org.wikimedia.metricsplatform.config; import static org.assertj.core.api.Assertions.assertThat; diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/StreamConfigFetcherTest.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigFetcherTest.java similarity index 88% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/StreamConfigFetcherTest.java rename to analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigFetcherTest.java index fe9bdc3680e..293b173f08a 100644 --- a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/StreamConfigFetcherTest.java +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigFetcherTest.java @@ -1,9 +1,9 @@ -package org.wikimedia.metrics_platform.config; +package org.wikimedia.metricsplatform.config; import static java.nio.charset.StandardCharsets.UTF_8; import static org.assertj.core.api.Assertions.assertThat; -import static org.wikimedia.metrics_platform.config.DestinationEventService.LOCAL; -import static org.wikimedia.metrics_platform.config.StreamConfigFetcher.ANALYTICS_API_ENDPOINT; +import static org.wikimedia.metricsplatform.config.DestinationEventService.LOCAL; +import static org.wikimedia.metricsplatform.config.StreamConfigFetcher.ANALYTICS_API_ENDPOINT; import java.io.IOException; import java.io.Reader; @@ -11,7 +11,7 @@ import java.util.Map; import org.junit.jupiter.api.Test; -import org.wikimedia.metrics_platform.json.GsonHelper; +import org.wikimedia.metricsplatform.json.GsonHelper; import com.google.common.io.Resources; diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/StreamConfigFixtures.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigFixtures.java similarity index 72% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/StreamConfigFixtures.java rename to analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigFixtures.java index cf260f7a872..f34abe72f09 100644 --- a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/StreamConfigFixtures.java +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigFixtures.java @@ -1,27 +1,27 @@ -package org.wikimedia.metrics_platform.config; +package org.wikimedia.metricsplatform.config; import static java.util.Collections.emptySet; import static java.util.Collections.singletonList; import static java.util.Collections.singletonMap; -import static org.wikimedia.metrics_platform.config.StreamConfigFetcher.METRICS_PLATFORM_SCHEMA_TITLE; -import static org.wikimedia.metrics_platform.context.ContextValue.AGENT_APP_INSTALL_ID; -import static org.wikimedia.metrics_platform.context.ContextValue.AGENT_APP_FLAVOR; -import static org.wikimedia.metrics_platform.context.ContextValue.AGENT_APP_THEME; -import static org.wikimedia.metrics_platform.context.ContextValue.AGENT_APP_VERSION; -import static org.wikimedia.metrics_platform.context.ContextValue.AGENT_DEVICE_LANGUAGE; -import static org.wikimedia.metrics_platform.context.ContextValue.AGENT_RELEASE_STATUS; -import static org.wikimedia.metrics_platform.context.ContextValue.AGENT_CLIENT_PLATFORM; -import static org.wikimedia.metrics_platform.context.ContextValue.AGENT_CLIENT_PLATFORM_FAMILY; -import static org.wikimedia.metrics_platform.context.ContextValue.MEDIAWIKI_DATABASE; -import static org.wikimedia.metrics_platform.context.ContextValue.PAGE_TITLE; -import static org.wikimedia.metrics_platform.context.ContextValue.PAGE_ID; -import static org.wikimedia.metrics_platform.context.ContextValue.PAGE_NAMESPACE_ID; -import static org.wikimedia.metrics_platform.context.ContextValue.PAGE_WIKIDATA_QID; -import static org.wikimedia.metrics_platform.context.ContextValue.PERFORMER_SESSION_ID; -import static org.wikimedia.metrics_platform.context.ContextValue.PERFORMER_PAGEVIEW_ID; -import static org.wikimedia.metrics_platform.context.ContextValue.PERFORMER_LANGUAGE_GROUPS; -import static org.wikimedia.metrics_platform.context.ContextValue.PERFORMER_LANGUAGE_PRIMARY; -import static org.wikimedia.metrics_platform.curation.CurationFilterFixtures.curationFilter; +import static org.wikimedia.metricsplatform.config.StreamConfigFetcher.METRICS_PLATFORM_SCHEMA_TITLE; +import static org.wikimedia.metricsplatform.context.ContextValue.AGENT_APP_INSTALL_ID; +import static org.wikimedia.metricsplatform.context.ContextValue.AGENT_APP_FLAVOR; +import static org.wikimedia.metricsplatform.context.ContextValue.AGENT_APP_THEME; +import static org.wikimedia.metricsplatform.context.ContextValue.AGENT_APP_VERSION; +import static org.wikimedia.metricsplatform.context.ContextValue.AGENT_DEVICE_LANGUAGE; +import static org.wikimedia.metricsplatform.context.ContextValue.AGENT_RELEASE_STATUS; +import static org.wikimedia.metricsplatform.context.ContextValue.AGENT_CLIENT_PLATFORM; +import static org.wikimedia.metricsplatform.context.ContextValue.AGENT_CLIENT_PLATFORM_FAMILY; +import static org.wikimedia.metricsplatform.context.ContextValue.MEDIAWIKI_DATABASE; +import static org.wikimedia.metricsplatform.context.ContextValue.PAGE_TITLE; +import static org.wikimedia.metricsplatform.context.ContextValue.PAGE_ID; +import static org.wikimedia.metricsplatform.context.ContextValue.PAGE_NAMESPACE_ID; +import static org.wikimedia.metricsplatform.context.ContextValue.PAGE_WIKIDATA_QID; +import static org.wikimedia.metricsplatform.context.ContextValue.PERFORMER_SESSION_ID; +import static org.wikimedia.metricsplatform.context.ContextValue.PERFORMER_PAGEVIEW_ID; +import static org.wikimedia.metricsplatform.context.ContextValue.PERFORMER_LANGUAGE_GROUPS; +import static org.wikimedia.metricsplatform.context.ContextValue.PERFORMER_LANGUAGE_PRIMARY; +import static org.wikimedia.metricsplatform.curation.CurationFilterFixtures.curationFilter; import java.util.Arrays; import java.util.Collections; @@ -31,7 +31,7 @@ import java.util.Map; import java.util.Set; -import org.wikimedia.metrics_platform.config.sampling.SampleConfig; +import org.wikimedia.metricsplatform.config.sampling.SampleConfig; public final class StreamConfigFixtures { diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/StreamConfigIT.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigIT.java similarity index 93% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/StreamConfigIT.java rename to analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigIT.java index 3721fa7b7a2..015ea3cc500 100644 --- a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/StreamConfigIT.java +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigIT.java @@ -1,4 +1,4 @@ -package org.wikimedia.metrics_platform.config; +package org.wikimedia.metricsplatform.config; import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; import static com.github.tomakehurst.wiremock.client.WireMock.get; @@ -9,7 +9,7 @@ import java.net.URL; import org.junit.jupiter.api.Test; -import org.wikimedia.metrics_platform.json.GsonHelper; +import org.wikimedia.metricsplatform.json.GsonHelper; import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; import com.github.tomakehurst.wiremock.junit5.WireMockTest; diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/StreamConfigTest.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigTest.java similarity index 87% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/StreamConfigTest.java rename to analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigTest.java index 516025a0a0f..34efa02c01f 100644 --- a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/config/StreamConfigTest.java +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigTest.java @@ -1,12 +1,12 @@ -package org.wikimedia.metrics_platform.config; +package org.wikimedia.metricsplatform.config; import static org.assertj.core.api.Assertions.assertThat; -import static org.wikimedia.metrics_platform.config.sampling.SampleConfig.Identifier.SESSION; +import static org.wikimedia.metricsplatform.config.sampling.SampleConfig.Identifier.SESSION; import java.util.Set; import org.junit.jupiter.api.Test; -import org.wikimedia.metrics_platform.json.GsonHelper; +import org.wikimedia.metricsplatform.json.GsonHelper; import com.google.gson.Gson; diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/AgentDataTest.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/context/AgentDataTest.java similarity index 95% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/AgentDataTest.java rename to analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/context/AgentDataTest.java index 7bec1fc9b94..5beddc82046 100644 --- a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/AgentDataTest.java +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/context/AgentDataTest.java @@ -1,9 +1,9 @@ -package org.wikimedia.metrics_platform.context; +package org.wikimedia.metricsplatform.context; import static org.assertj.core.api.Assertions.assertThat; import org.junit.jupiter.api.Test; -import org.wikimedia.metrics_platform.json.GsonHelper; +import org.wikimedia.metricsplatform.json.GsonHelper; import com.google.gson.Gson; diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/CustomDataTest.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/context/CustomDataTest.java similarity index 89% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/CustomDataTest.java rename to analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/context/CustomDataTest.java index 6e0867b20a9..df66f8dc083 100644 --- a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/CustomDataTest.java +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/context/CustomDataTest.java @@ -1,12 +1,12 @@ -package org.wikimedia.metrics_platform.context; +package org.wikimedia.metricsplatform.context; import static org.assertj.core.api.Assertions.assertThat; -import static org.wikimedia.metrics_platform.context.DataFixtures.getTestCustomData; +import static org.wikimedia.metricsplatform.context.DataFixtures.getTestCustomData; import java.util.Map; import org.junit.jupiter.api.Test; -import org.wikimedia.metrics_platform.json.GsonHelper; +import org.wikimedia.metricsplatform.json.GsonHelper; import com.google.gson.Gson; diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/DataFixtures.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/context/DataFixtures.java similarity index 97% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/DataFixtures.java rename to analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/context/DataFixtures.java index e66b80b92c4..a8aa8fbf120 100644 --- a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/DataFixtures.java +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/context/DataFixtures.java @@ -1,4 +1,4 @@ -package org.wikimedia.metrics_platform.context; +package org.wikimedia.metricsplatform.context; import java.time.Instant; import java.util.Collections; @@ -9,7 +9,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import org.wikimedia.metrics_platform.json.GsonHelper; +import org.wikimedia.metricsplatform.json.GsonHelper; import com.google.gson.Gson; import com.google.gson.JsonElement; diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/MediawikiDataTest.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/context/MediawikiDataTest.java similarity index 85% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/MediawikiDataTest.java rename to analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/context/MediawikiDataTest.java index 20674527fec..e404848b854 100644 --- a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/MediawikiDataTest.java +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/context/MediawikiDataTest.java @@ -1,9 +1,9 @@ -package org.wikimedia.metrics_platform.context; +package org.wikimedia.metricsplatform.context; import static org.assertj.core.api.Assertions.assertThat; import org.junit.jupiter.api.Test; -import org.wikimedia.metrics_platform.json.GsonHelper; +import org.wikimedia.metricsplatform.json.GsonHelper; import com.google.gson.Gson; diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/PageDataTest.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/context/PageDataTest.java similarity index 92% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/PageDataTest.java rename to analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/context/PageDataTest.java index 38c52c9eed6..493cb26adc7 100644 --- a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/PageDataTest.java +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/context/PageDataTest.java @@ -1,9 +1,9 @@ -package org.wikimedia.metrics_platform.context; +package org.wikimedia.metricsplatform.context; import static org.assertj.core.api.Assertions.assertThat; import org.junit.jupiter.api.Test; -import org.wikimedia.metrics_platform.json.GsonHelper; +import org.wikimedia.metricsplatform.json.GsonHelper; import com.google.gson.Gson; diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/PerformerDataTest.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/context/PerformerDataTest.java similarity index 95% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/PerformerDataTest.java rename to analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/context/PerformerDataTest.java index 5d7e9cbd6e4..52af125e2a0 100644 --- a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/context/PerformerDataTest.java +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/context/PerformerDataTest.java @@ -1,4 +1,4 @@ -package org.wikimedia.metrics_platform.context; +package org.wikimedia.metricsplatform.context; import static org.assertj.core.api.Assertions.assertThat; @@ -6,7 +6,7 @@ import java.util.Collections; import org.junit.jupiter.api.Test; -import org.wikimedia.metrics_platform.json.GsonHelper; +import org.wikimedia.metricsplatform.json.GsonHelper; import com.google.gson.Gson; diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/curation/CurationFilterFixtures.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/curation/CurationFilterFixtures.java similarity index 81% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/curation/CurationFilterFixtures.java rename to analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/curation/CurationFilterFixtures.java index c2820710bc0..ef96b93e95b 100644 --- a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/curation/CurationFilterFixtures.java +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/curation/CurationFilterFixtures.java @@ -1,7 +1,7 @@ -package org.wikimedia.metrics_platform.curation; +package org.wikimedia.metricsplatform.curation; -import org.wikimedia.metrics_platform.json.GsonHelper; -import org.wikimedia.metrics_platform.config.CurationFilter; +import org.wikimedia.metricsplatform.json.GsonHelper; +import org.wikimedia.metricsplatform.config.CurationFilter; import com.google.gson.Gson; diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/event/EventFixtures.java b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/event/EventFixtures.java similarity index 83% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/event/EventFixtures.java rename to analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/event/EventFixtures.java index b9163e667af..13e0970caa0 100644 --- a/analytics/metrics-platform/src/test/java/org/wikimedia/metrics_platform/event/EventFixtures.java +++ b/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/event/EventFixtures.java @@ -1,14 +1,14 @@ -package org.wikimedia.metrics_platform.event; +package org.wikimedia.metricsplatform.event; -import static org.wikimedia.metrics_platform.context.DataFixtures.getTestClientData; -import static org.wikimedia.metrics_platform.event.EventProcessed.fromEvent; +import static org.wikimedia.metricsplatform.context.DataFixtures.getTestClientData; +import static org.wikimedia.metricsplatform.event.EventProcessed.fromEvent; import java.util.Arrays; import java.util.List; -import org.wikimedia.metrics_platform.context.ClientData; -import org.wikimedia.metrics_platform.context.PageData; -import org.wikimedia.metrics_platform.context.PerformerData; +import org.wikimedia.metricsplatform.context.ClientData; +import org.wikimedia.metricsplatform.context.PageData; +import org.wikimedia.metricsplatform.context.PerformerData; public final class EventFixtures { diff --git a/analytics/metrics-platform/src/test/resources/org/wikimedia/metrics_platform/config/streamconfigs-local.json b/analytics/metrics-platform/src/test/resources/org/wikimedia/metricsplatform/config/streamconfigs-local.json similarity index 100% rename from analytics/metrics-platform/src/test/resources/org/wikimedia/metrics_platform/config/streamconfigs-local.json rename to analytics/metrics-platform/src/test/resources/org/wikimedia/metricsplatform/config/streamconfigs-local.json diff --git a/analytics/metrics-platform/src/test/resources/org/wikimedia/metrics_platform/config/streamconfigs.json b/analytics/metrics-platform/src/test/resources/org/wikimedia/metricsplatform/config/streamconfigs.json similarity index 100% rename from analytics/metrics-platform/src/test/resources/org/wikimedia/metrics_platform/config/streamconfigs.json rename to analytics/metrics-platform/src/test/resources/org/wikimedia/metricsplatform/config/streamconfigs.json diff --git a/analytics/metrics-platform/src/test/resources/org/wikimedia/metrics_platform/event/expected_event_click.json b/analytics/metrics-platform/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_click.json similarity index 100% rename from analytics/metrics-platform/src/test/resources/org/wikimedia/metrics_platform/event/expected_event_click.json rename to analytics/metrics-platform/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_click.json diff --git a/analytics/metrics-platform/src/test/resources/org/wikimedia/metrics_platform/event/expected_event_click_custom.json b/analytics/metrics-platform/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_click_custom.json similarity index 100% rename from analytics/metrics-platform/src/test/resources/org/wikimedia/metrics_platform/event/expected_event_click_custom.json rename to analytics/metrics-platform/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_click_custom.json diff --git a/analytics/metrics-platform/src/test/resources/org/wikimedia/metrics_platform/event/expected_event_interaction.json b/analytics/metrics-platform/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_interaction.json similarity index 100% rename from analytics/metrics-platform/src/test/resources/org/wikimedia/metrics_platform/event/expected_event_interaction.json rename to analytics/metrics-platform/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_interaction.json diff --git a/analytics/metrics-platform/src/test/resources/org/wikimedia/metrics_platform/event/expected_event_view.json b/analytics/metrics-platform/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_view.json similarity index 100% rename from analytics/metrics-platform/src/test/resources/org/wikimedia/metrics_platform/event/expected_event_view.json rename to analytics/metrics-platform/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_view.json diff --git a/app/build.gradle b/app/build.gradle index 27f26f5ccf1..e11287cf0f6 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -174,6 +174,8 @@ dependencies { coreLibraryDesugaring libs.desugar.jdk.libs + implementation project(':analytics') + implementation libs.kotlin.stdlib.jdk8 implementation libs.kotlinx.coroutines.core implementation libs.kotlinx.coroutines.android diff --git a/app/src/main/java/org/wikipedia/analytics/metricsplatform/ArticleEvent.kt b/app/src/main/java/org/wikipedia/analytics/metricsplatform/ArticleEvent.kt index ea81343cc32..c9a72647aec 100644 --- a/app/src/main/java/org/wikipedia/analytics/metricsplatform/ArticleEvent.kt +++ b/app/src/main/java/org/wikipedia/analytics/metricsplatform/ArticleEvent.kt @@ -2,7 +2,7 @@ package org.wikipedia.analytics.metricsplatform import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable -import org.wikimedia.metrics_platform.context.PageData +import org.wikimedia.metricsplatform.context.PageData import org.wikipedia.dataclient.page.PageSummary import org.wikipedia.json.JsonUtil import org.wikipedia.page.PageFragment diff --git a/app/src/main/java/org/wikipedia/analytics/metricsplatform/MetricsEvent.kt b/app/src/main/java/org/wikipedia/analytics/metricsplatform/MetricsEvent.kt index bf9b62f4abd..c1394bda042 100644 --- a/app/src/main/java/org/wikipedia/analytics/metricsplatform/MetricsEvent.kt +++ b/app/src/main/java/org/wikipedia/analytics/metricsplatform/MetricsEvent.kt @@ -1,9 +1,9 @@ package org.wikipedia.analytics.metricsplatform -import org.wikimedia.metrics_platform.context.ClientData -import org.wikimedia.metrics_platform.context.InteractionData -import org.wikimedia.metrics_platform.context.PageData -import org.wikimedia.metrics_platform.context.PerformerData +import org.wikimedia.metricsplatform.context.ClientData +import org.wikimedia.metricsplatform.context.InteractionData +import org.wikimedia.metricsplatform.context.PageData +import org.wikimedia.metricsplatform.context.PerformerData import org.wikipedia.WikipediaApp import org.wikipedia.analytics.eventplatform.EventPlatformClient import org.wikipedia.auth.AccountUtil diff --git a/app/src/main/java/org/wikipedia/analytics/metricsplatform/MetricsPlatform.kt b/app/src/main/java/org/wikipedia/analytics/metricsplatform/MetricsPlatform.kt index 81e9d712d72..ce14eeac1e3 100644 --- a/app/src/main/java/org/wikipedia/analytics/metricsplatform/MetricsPlatform.kt +++ b/app/src/main/java/org/wikipedia/analytics/metricsplatform/MetricsPlatform.kt @@ -1,10 +1,10 @@ package org.wikipedia.analytics.metricsplatform import android.os.Build -import org.wikimedia.metrics_platform.MetricsClient -import org.wikimedia.metrics_platform.context.AgentData -import org.wikimedia.metrics_platform.context.ClientData -import org.wikimedia.metrics_platform.context.MediawikiData +import org.wikimedia.metricsplatform.MetricsClient +import org.wikimedia.metricsplatform.context.AgentData +import org.wikimedia.metricsplatform.context.ClientData +import org.wikimedia.metricsplatform.context.MediawikiData import org.wikipedia.BuildConfig import org.wikipedia.WikipediaApp import org.wikipedia.dataclient.okhttp.OkHttpConnectionFactory From e92b07af2ffa5ec93ad2fe4d5561c4347387778f Mon Sep 17 00:00:00 2001 From: Dmitry Brant Date: Mon, 30 Jun 2025 20:08:31 -0400 Subject: [PATCH 4/9] Rename. --- analytics/{metrics-platform => metricsplatform}/.gitignore | 0 analytics/{metrics-platform => metricsplatform}/build.gradle | 0 .../java/org/wikimedia/metricsplatform/ContextController.kt | 0 .../java/org/wikimedia/metricsplatform/CurationController.kt | 0 .../main/java/org/wikimedia/metricsplatform/EventProcessor.kt | 0 .../src/main/java/org/wikimedia/metricsplatform/EventSender.kt | 0 .../java/org/wikimedia/metricsplatform/EventSenderDefault.kt | 0 .../main/java/org/wikimedia/metricsplatform/MetricsClient.kt | 0 .../java/org/wikimedia/metricsplatform/SamplingController.kt | 0 .../java/org/wikimedia/metricsplatform/SessionController.kt | 0 .../java/org/wikimedia/metricsplatform/config/CurationFilter.kt | 0 .../wikimedia/metricsplatform/config/DestinationEventService.kt | 0 .../java/org/wikimedia/metricsplatform/config/SourceConfig.kt | 0 .../java/org/wikimedia/metricsplatform/config/StreamConfig.kt | 0 .../wikimedia/metricsplatform/config/StreamConfigCollection.kt | 0 .../wikimedia/metricsplatform/config/StreamConfigFetcher.java | 0 .../metricsplatform/config/curation/CollectionCurationRules.kt | 0 .../metricsplatform/config/curation/ComparableCurationRules.kt | 0 .../wikimedia/metricsplatform/config/curation/CurationRules.kt | 0 .../wikimedia/metricsplatform/config/sampling/SampleConfig.kt | 0 .../java/org/wikimedia/metricsplatform/context/AgentData.kt | 0 .../java/org/wikimedia/metricsplatform/context/ClientData.kt | 0 .../java/org/wikimedia/metricsplatform/context/ContextValue.kt | 0 .../java/org/wikimedia/metricsplatform/context/CustomData.kt | 0 .../org/wikimedia/metricsplatform/context/CustomDataType.kt | 0 .../org/wikimedia/metricsplatform/context/InstantSerializer.kt | 0 .../org/wikimedia/metricsplatform/context/InteractionData.kt | 0 .../java/org/wikimedia/metricsplatform/context/MediawikiData.kt | 0 .../main/java/org/wikimedia/metricsplatform/context/PageData.kt | 0 .../java/org/wikimedia/metricsplatform/context/PerformerData.kt | 0 .../src/main/java/org/wikimedia/metricsplatform/event/Event.kt | 0 .../java/org/wikimedia/metricsplatform/event/EventProcessed.kt | 0 .../metricsplatform/event/EventProcessedSerializer.java | 0 .../test/java/org/wikimedia/metricsplatform/ConsistencyIT.java | 0 .../org/wikimedia/metricsplatform/ConsistencyITClientData.java | 0 .../org/wikimedia/metricsplatform/ContextControllerTest.java | 0 .../org/wikimedia/metricsplatform/CurationControllerTest.java | 0 .../wikimedia/metricsplatform/DestinationEventServiceTest.java | 0 .../src/test/java/org/wikimedia/metricsplatform/EndToEndIT.java | 0 .../java/org/wikimedia/metricsplatform/EventProcessorTest.java | 0 .../src/test/java/org/wikimedia/metricsplatform/EventTest.java | 0 .../java/org/wikimedia/metricsplatform/MetricsClientTest.java | 0 .../org/wikimedia/metricsplatform/SamplingControllerTest.java | 0 .../org/wikimedia/metricsplatform/SessionControllerTest.java | 0 .../java/org/wikimedia/metricsplatform/TestEventSender.java | 0 .../metricsplatform/config/CurationFilterFixtures.java | 0 .../org/wikimedia/metricsplatform/config/SampleConfigTest.java | 0 .../wikimedia/metricsplatform/config/SourceConfigFixtures.java | 0 .../org/wikimedia/metricsplatform/config/SourceConfigTest.java | 0 .../metricsplatform/config/StreamConfigFetcherTest.java | 0 .../wikimedia/metricsplatform/config/StreamConfigFixtures.java | 0 .../org/wikimedia/metricsplatform/config/StreamConfigIT.java | 0 .../org/wikimedia/metricsplatform/config/StreamConfigTest.java | 0 .../org/wikimedia/metricsplatform/context/AgentDataTest.java | 0 .../org/wikimedia/metricsplatform/context/CustomDataTest.java | 0 .../org/wikimedia/metricsplatform/context/DataFixtures.java | 0 .../wikimedia/metricsplatform/context/MediawikiDataTest.java | 0 .../org/wikimedia/metricsplatform/context/PageDataTest.java | 0 .../wikimedia/metricsplatform/context/PerformerDataTest.java | 0 .../metricsplatform/curation/CurationFilterFixtures.java | 0 .../java/org/wikimedia/metricsplatform/event/EventFixtures.java | 0 .../wikimedia/metricsplatform/config/streamconfigs-local.json | 0 .../org/wikimedia/metricsplatform/config/streamconfigs.json | 0 .../wikimedia/metricsplatform/event/expected_event_click.json | 0 .../metricsplatform/event/expected_event_click_custom.json | 0 .../metricsplatform/event/expected_event_interaction.json | 0 .../wikimedia/metricsplatform/event/expected_event_view.json | 0 .../src/test/resources/simplelogger.properties | 0 settings.gradle.kts | 2 +- 69 files changed, 1 insertion(+), 1 deletion(-) rename analytics/{metrics-platform => metricsplatform}/.gitignore (100%) rename analytics/{metrics-platform => metricsplatform}/build.gradle (100%) rename analytics/{metrics-platform => metricsplatform}/src/main/java/org/wikimedia/metricsplatform/ContextController.kt (100%) rename analytics/{metrics-platform => metricsplatform}/src/main/java/org/wikimedia/metricsplatform/CurationController.kt (100%) rename analytics/{metrics-platform => metricsplatform}/src/main/java/org/wikimedia/metricsplatform/EventProcessor.kt (100%) rename analytics/{metrics-platform => metricsplatform}/src/main/java/org/wikimedia/metricsplatform/EventSender.kt (100%) rename analytics/{metrics-platform => metricsplatform}/src/main/java/org/wikimedia/metricsplatform/EventSenderDefault.kt (100%) rename analytics/{metrics-platform => metricsplatform}/src/main/java/org/wikimedia/metricsplatform/MetricsClient.kt (100%) rename analytics/{metrics-platform => metricsplatform}/src/main/java/org/wikimedia/metricsplatform/SamplingController.kt (100%) rename analytics/{metrics-platform => metricsplatform}/src/main/java/org/wikimedia/metricsplatform/SessionController.kt (100%) rename analytics/{metrics-platform => metricsplatform}/src/main/java/org/wikimedia/metricsplatform/config/CurationFilter.kt (100%) rename analytics/{metrics-platform => metricsplatform}/src/main/java/org/wikimedia/metricsplatform/config/DestinationEventService.kt (100%) rename analytics/{metrics-platform => metricsplatform}/src/main/java/org/wikimedia/metricsplatform/config/SourceConfig.kt (100%) rename analytics/{metrics-platform => metricsplatform}/src/main/java/org/wikimedia/metricsplatform/config/StreamConfig.kt (100%) rename analytics/{metrics-platform => metricsplatform}/src/main/java/org/wikimedia/metricsplatform/config/StreamConfigCollection.kt (100%) rename analytics/{metrics-platform => metricsplatform}/src/main/java/org/wikimedia/metricsplatform/config/StreamConfigFetcher.java (100%) rename analytics/{metrics-platform => metricsplatform}/src/main/java/org/wikimedia/metricsplatform/config/curation/CollectionCurationRules.kt (100%) rename analytics/{metrics-platform => metricsplatform}/src/main/java/org/wikimedia/metricsplatform/config/curation/ComparableCurationRules.kt (100%) rename analytics/{metrics-platform => metricsplatform}/src/main/java/org/wikimedia/metricsplatform/config/curation/CurationRules.kt (100%) rename analytics/{metrics-platform => metricsplatform}/src/main/java/org/wikimedia/metricsplatform/config/sampling/SampleConfig.kt (100%) rename analytics/{metrics-platform => metricsplatform}/src/main/java/org/wikimedia/metricsplatform/context/AgentData.kt (100%) rename analytics/{metrics-platform => metricsplatform}/src/main/java/org/wikimedia/metricsplatform/context/ClientData.kt (100%) rename analytics/{metrics-platform => metricsplatform}/src/main/java/org/wikimedia/metricsplatform/context/ContextValue.kt (100%) rename analytics/{metrics-platform => metricsplatform}/src/main/java/org/wikimedia/metricsplatform/context/CustomData.kt (100%) rename analytics/{metrics-platform => metricsplatform}/src/main/java/org/wikimedia/metricsplatform/context/CustomDataType.kt (100%) rename analytics/{metrics-platform => metricsplatform}/src/main/java/org/wikimedia/metricsplatform/context/InstantSerializer.kt (100%) rename analytics/{metrics-platform => metricsplatform}/src/main/java/org/wikimedia/metricsplatform/context/InteractionData.kt (100%) rename analytics/{metrics-platform => metricsplatform}/src/main/java/org/wikimedia/metricsplatform/context/MediawikiData.kt (100%) rename analytics/{metrics-platform => metricsplatform}/src/main/java/org/wikimedia/metricsplatform/context/PageData.kt (100%) rename analytics/{metrics-platform => metricsplatform}/src/main/java/org/wikimedia/metricsplatform/context/PerformerData.kt (100%) rename analytics/{metrics-platform => metricsplatform}/src/main/java/org/wikimedia/metricsplatform/event/Event.kt (100%) rename analytics/{metrics-platform => metricsplatform}/src/main/java/org/wikimedia/metricsplatform/event/EventProcessed.kt (100%) rename analytics/{metrics-platform => metricsplatform}/src/main/java/org/wikimedia/metricsplatform/event/EventProcessedSerializer.java (100%) rename analytics/{metrics-platform => metricsplatform}/src/test/java/org/wikimedia/metricsplatform/ConsistencyIT.java (100%) rename analytics/{metrics-platform => metricsplatform}/src/test/java/org/wikimedia/metricsplatform/ConsistencyITClientData.java (100%) rename analytics/{metrics-platform => metricsplatform}/src/test/java/org/wikimedia/metricsplatform/ContextControllerTest.java (100%) rename analytics/{metrics-platform => metricsplatform}/src/test/java/org/wikimedia/metricsplatform/CurationControllerTest.java (100%) rename analytics/{metrics-platform => metricsplatform}/src/test/java/org/wikimedia/metricsplatform/DestinationEventServiceTest.java (100%) rename analytics/{metrics-platform => metricsplatform}/src/test/java/org/wikimedia/metricsplatform/EndToEndIT.java (100%) rename analytics/{metrics-platform => metricsplatform}/src/test/java/org/wikimedia/metricsplatform/EventProcessorTest.java (100%) rename analytics/{metrics-platform => metricsplatform}/src/test/java/org/wikimedia/metricsplatform/EventTest.java (100%) rename analytics/{metrics-platform => metricsplatform}/src/test/java/org/wikimedia/metricsplatform/MetricsClientTest.java (100%) rename analytics/{metrics-platform => metricsplatform}/src/test/java/org/wikimedia/metricsplatform/SamplingControllerTest.java (100%) rename analytics/{metrics-platform => metricsplatform}/src/test/java/org/wikimedia/metricsplatform/SessionControllerTest.java (100%) rename analytics/{metrics-platform => metricsplatform}/src/test/java/org/wikimedia/metricsplatform/TestEventSender.java (100%) rename analytics/{metrics-platform => metricsplatform}/src/test/java/org/wikimedia/metricsplatform/config/CurationFilterFixtures.java (100%) rename analytics/{metrics-platform => metricsplatform}/src/test/java/org/wikimedia/metricsplatform/config/SampleConfigTest.java (100%) rename analytics/{metrics-platform => metricsplatform}/src/test/java/org/wikimedia/metricsplatform/config/SourceConfigFixtures.java (100%) rename analytics/{metrics-platform => metricsplatform}/src/test/java/org/wikimedia/metricsplatform/config/SourceConfigTest.java (100%) rename analytics/{metrics-platform => metricsplatform}/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigFetcherTest.java (100%) rename analytics/{metrics-platform => metricsplatform}/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigFixtures.java (100%) rename analytics/{metrics-platform => metricsplatform}/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigIT.java (100%) rename analytics/{metrics-platform => metricsplatform}/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigTest.java (100%) rename analytics/{metrics-platform => metricsplatform}/src/test/java/org/wikimedia/metricsplatform/context/AgentDataTest.java (100%) rename analytics/{metrics-platform => metricsplatform}/src/test/java/org/wikimedia/metricsplatform/context/CustomDataTest.java (100%) rename analytics/{metrics-platform => metricsplatform}/src/test/java/org/wikimedia/metricsplatform/context/DataFixtures.java (100%) rename analytics/{metrics-platform => metricsplatform}/src/test/java/org/wikimedia/metricsplatform/context/MediawikiDataTest.java (100%) rename analytics/{metrics-platform => metricsplatform}/src/test/java/org/wikimedia/metricsplatform/context/PageDataTest.java (100%) rename analytics/{metrics-platform => metricsplatform}/src/test/java/org/wikimedia/metricsplatform/context/PerformerDataTest.java (100%) rename analytics/{metrics-platform => metricsplatform}/src/test/java/org/wikimedia/metricsplatform/curation/CurationFilterFixtures.java (100%) rename analytics/{metrics-platform => metricsplatform}/src/test/java/org/wikimedia/metricsplatform/event/EventFixtures.java (100%) rename analytics/{metrics-platform => metricsplatform}/src/test/resources/org/wikimedia/metricsplatform/config/streamconfigs-local.json (100%) rename analytics/{metrics-platform => metricsplatform}/src/test/resources/org/wikimedia/metricsplatform/config/streamconfigs.json (100%) rename analytics/{metrics-platform => metricsplatform}/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_click.json (100%) rename analytics/{metrics-platform => metricsplatform}/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_click_custom.json (100%) rename analytics/{metrics-platform => metricsplatform}/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_interaction.json (100%) rename analytics/{metrics-platform => metricsplatform}/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_view.json (100%) rename analytics/{metrics-platform => metricsplatform}/src/test/resources/simplelogger.properties (100%) diff --git a/analytics/metrics-platform/.gitignore b/analytics/metricsplatform/.gitignore similarity index 100% rename from analytics/metrics-platform/.gitignore rename to analytics/metricsplatform/.gitignore diff --git a/analytics/metrics-platform/build.gradle b/analytics/metricsplatform/build.gradle similarity index 100% rename from analytics/metrics-platform/build.gradle rename to analytics/metricsplatform/build.gradle diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/ContextController.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/ContextController.kt similarity index 100% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/ContextController.kt rename to analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/ContextController.kt diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/CurationController.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/CurationController.kt similarity index 100% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/CurationController.kt rename to analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/CurationController.kt diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/EventProcessor.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventProcessor.kt similarity index 100% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/EventProcessor.kt rename to analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventProcessor.kt diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/EventSender.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventSender.kt similarity index 100% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/EventSender.kt rename to analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventSender.kt diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/EventSenderDefault.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventSenderDefault.kt similarity index 100% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/EventSenderDefault.kt rename to analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventSenderDefault.kt diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/MetricsClient.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/MetricsClient.kt similarity index 100% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/MetricsClient.kt rename to analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/MetricsClient.kt diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/SamplingController.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/SamplingController.kt similarity index 100% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/SamplingController.kt rename to analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/SamplingController.kt diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/SessionController.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/SessionController.kt similarity index 100% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/SessionController.kt rename to analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/SessionController.kt diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/CurationFilter.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/CurationFilter.kt similarity index 100% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/CurationFilter.kt rename to analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/CurationFilter.kt diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/DestinationEventService.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/DestinationEventService.kt similarity index 100% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/DestinationEventService.kt rename to analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/DestinationEventService.kt diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/SourceConfig.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/SourceConfig.kt similarity index 100% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/SourceConfig.kt rename to analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/SourceConfig.kt diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfig.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfig.kt similarity index 100% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfig.kt rename to analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfig.kt diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfigCollection.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfigCollection.kt similarity index 100% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfigCollection.kt rename to analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfigCollection.kt diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfigFetcher.java b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfigFetcher.java similarity index 100% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfigFetcher.java rename to analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfigFetcher.java diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/curation/CollectionCurationRules.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/curation/CollectionCurationRules.kt similarity index 100% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/curation/CollectionCurationRules.kt rename to analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/curation/CollectionCurationRules.kt diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/curation/ComparableCurationRules.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/curation/ComparableCurationRules.kt similarity index 100% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/curation/ComparableCurationRules.kt rename to analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/curation/ComparableCurationRules.kt diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/curation/CurationRules.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/curation/CurationRules.kt similarity index 100% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/curation/CurationRules.kt rename to analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/curation/CurationRules.kt diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/sampling/SampleConfig.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/sampling/SampleConfig.kt similarity index 100% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/config/sampling/SampleConfig.kt rename to analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/sampling/SampleConfig.kt diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/AgentData.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/AgentData.kt similarity index 100% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/AgentData.kt rename to analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/AgentData.kt diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/ClientData.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/ClientData.kt similarity index 100% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/ClientData.kt rename to analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/ClientData.kt diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/ContextValue.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/ContextValue.kt similarity index 100% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/ContextValue.kt rename to analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/ContextValue.kt diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/CustomData.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/CustomData.kt similarity index 100% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/CustomData.kt rename to analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/CustomData.kt diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/CustomDataType.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/CustomDataType.kt similarity index 100% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/CustomDataType.kt rename to analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/CustomDataType.kt diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/InstantSerializer.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/InstantSerializer.kt similarity index 100% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/InstantSerializer.kt rename to analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/InstantSerializer.kt diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/InteractionData.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/InteractionData.kt similarity index 100% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/InteractionData.kt rename to analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/InteractionData.kt diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/MediawikiData.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/MediawikiData.kt similarity index 100% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/MediawikiData.kt rename to analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/MediawikiData.kt diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/PageData.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/PageData.kt similarity index 100% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/PageData.kt rename to analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/PageData.kt diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/PerformerData.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/PerformerData.kt similarity index 100% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/context/PerformerData.kt rename to analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/PerformerData.kt diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/event/Event.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/event/Event.kt similarity index 100% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/event/Event.kt rename to analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/event/Event.kt diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/event/EventProcessed.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/event/EventProcessed.kt similarity index 100% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/event/EventProcessed.kt rename to analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/event/EventProcessed.kt diff --git a/analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/event/EventProcessedSerializer.java b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/event/EventProcessedSerializer.java similarity index 100% rename from analytics/metrics-platform/src/main/java/org/wikimedia/metricsplatform/event/EventProcessedSerializer.java rename to analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/event/EventProcessedSerializer.java diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/ConsistencyIT.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/ConsistencyIT.java similarity index 100% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/ConsistencyIT.java rename to analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/ConsistencyIT.java diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/ConsistencyITClientData.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/ConsistencyITClientData.java similarity index 100% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/ConsistencyITClientData.java rename to analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/ConsistencyITClientData.java diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/ContextControllerTest.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/ContextControllerTest.java similarity index 100% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/ContextControllerTest.java rename to analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/ContextControllerTest.java diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/CurationControllerTest.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/CurationControllerTest.java similarity index 100% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/CurationControllerTest.java rename to analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/CurationControllerTest.java diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/DestinationEventServiceTest.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/DestinationEventServiceTest.java similarity index 100% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/DestinationEventServiceTest.java rename to analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/DestinationEventServiceTest.java diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/EndToEndIT.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/EndToEndIT.java similarity index 100% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/EndToEndIT.java rename to analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/EndToEndIT.java diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/EventProcessorTest.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/EventProcessorTest.java similarity index 100% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/EventProcessorTest.java rename to analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/EventProcessorTest.java diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/EventTest.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/EventTest.java similarity index 100% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/EventTest.java rename to analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/EventTest.java diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/MetricsClientTest.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/MetricsClientTest.java similarity index 100% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/MetricsClientTest.java rename to analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/MetricsClientTest.java diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/SamplingControllerTest.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/SamplingControllerTest.java similarity index 100% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/SamplingControllerTest.java rename to analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/SamplingControllerTest.java diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/SessionControllerTest.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/SessionControllerTest.java similarity index 100% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/SessionControllerTest.java rename to analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/SessionControllerTest.java diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/TestEventSender.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/TestEventSender.java similarity index 100% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/TestEventSender.java rename to analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/TestEventSender.java diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/CurationFilterFixtures.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/CurationFilterFixtures.java similarity index 100% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/CurationFilterFixtures.java rename to analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/CurationFilterFixtures.java diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/SampleConfigTest.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/SampleConfigTest.java similarity index 100% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/SampleConfigTest.java rename to analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/SampleConfigTest.java diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/SourceConfigFixtures.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/SourceConfigFixtures.java similarity index 100% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/SourceConfigFixtures.java rename to analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/SourceConfigFixtures.java diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/SourceConfigTest.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/SourceConfigTest.java similarity index 100% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/SourceConfigTest.java rename to analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/SourceConfigTest.java diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigFetcherTest.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigFetcherTest.java similarity index 100% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigFetcherTest.java rename to analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigFetcherTest.java diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigFixtures.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigFixtures.java similarity index 100% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigFixtures.java rename to analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigFixtures.java diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigIT.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigIT.java similarity index 100% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigIT.java rename to analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigIT.java diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigTest.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigTest.java similarity index 100% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigTest.java rename to analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/config/StreamConfigTest.java diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/context/AgentDataTest.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/context/AgentDataTest.java similarity index 100% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/context/AgentDataTest.java rename to analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/context/AgentDataTest.java diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/context/CustomDataTest.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/context/CustomDataTest.java similarity index 100% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/context/CustomDataTest.java rename to analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/context/CustomDataTest.java diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/context/DataFixtures.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/context/DataFixtures.java similarity index 100% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/context/DataFixtures.java rename to analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/context/DataFixtures.java diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/context/MediawikiDataTest.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/context/MediawikiDataTest.java similarity index 100% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/context/MediawikiDataTest.java rename to analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/context/MediawikiDataTest.java diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/context/PageDataTest.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/context/PageDataTest.java similarity index 100% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/context/PageDataTest.java rename to analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/context/PageDataTest.java diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/context/PerformerDataTest.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/context/PerformerDataTest.java similarity index 100% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/context/PerformerDataTest.java rename to analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/context/PerformerDataTest.java diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/curation/CurationFilterFixtures.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/curation/CurationFilterFixtures.java similarity index 100% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/curation/CurationFilterFixtures.java rename to analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/curation/CurationFilterFixtures.java diff --git a/analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/event/EventFixtures.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/event/EventFixtures.java similarity index 100% rename from analytics/metrics-platform/src/test/java/org/wikimedia/metricsplatform/event/EventFixtures.java rename to analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/event/EventFixtures.java diff --git a/analytics/metrics-platform/src/test/resources/org/wikimedia/metricsplatform/config/streamconfigs-local.json b/analytics/metricsplatform/src/test/resources/org/wikimedia/metricsplatform/config/streamconfigs-local.json similarity index 100% rename from analytics/metrics-platform/src/test/resources/org/wikimedia/metricsplatform/config/streamconfigs-local.json rename to analytics/metricsplatform/src/test/resources/org/wikimedia/metricsplatform/config/streamconfigs-local.json diff --git a/analytics/metrics-platform/src/test/resources/org/wikimedia/metricsplatform/config/streamconfigs.json b/analytics/metricsplatform/src/test/resources/org/wikimedia/metricsplatform/config/streamconfigs.json similarity index 100% rename from analytics/metrics-platform/src/test/resources/org/wikimedia/metricsplatform/config/streamconfigs.json rename to analytics/metricsplatform/src/test/resources/org/wikimedia/metricsplatform/config/streamconfigs.json diff --git a/analytics/metrics-platform/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_click.json b/analytics/metricsplatform/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_click.json similarity index 100% rename from analytics/metrics-platform/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_click.json rename to analytics/metricsplatform/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_click.json diff --git a/analytics/metrics-platform/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_click_custom.json b/analytics/metricsplatform/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_click_custom.json similarity index 100% rename from analytics/metrics-platform/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_click_custom.json rename to analytics/metricsplatform/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_click_custom.json diff --git a/analytics/metrics-platform/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_interaction.json b/analytics/metricsplatform/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_interaction.json similarity index 100% rename from analytics/metrics-platform/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_interaction.json rename to analytics/metricsplatform/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_interaction.json diff --git a/analytics/metrics-platform/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_view.json b/analytics/metricsplatform/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_view.json similarity index 100% rename from analytics/metrics-platform/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_view.json rename to analytics/metricsplatform/src/test/resources/org/wikimedia/metricsplatform/event/expected_event_view.json diff --git a/analytics/metrics-platform/src/test/resources/simplelogger.properties b/analytics/metricsplatform/src/test/resources/simplelogger.properties similarity index 100% rename from analytics/metrics-platform/src/test/resources/simplelogger.properties rename to analytics/metricsplatform/src/test/resources/simplelogger.properties diff --git a/settings.gradle.kts b/settings.gradle.kts index d23137e25b6..0dcf117aae2 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -15,4 +15,4 @@ dependencyResolutionManagement { } include(":app") -include(":analytics:metrics-platform") +include(":analytics:metricsplatform") From 4c9c71be0fb977d1da8e4d76cf83aa1746def181 Mon Sep 17 00:00:00 2001 From: Dmitry Brant Date: Tue, 1 Jul 2025 09:07:53 -0400 Subject: [PATCH 5/9] Proceed... --- .../metricsplatform/EventSenderDefault.kt | 8 ++-- .../metricsplatform/MetricsClient.kt | 42 +++++++++---------- .../wikimedia/metricsplatform/event/Event.kt | 2 +- app/build.gradle | 2 +- 4 files changed, 25 insertions(+), 29 deletions(-) diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventSenderDefault.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventSenderDefault.kt index 9504c62871f..05ec3e6f88a 100644 --- a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventSenderDefault.kt +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventSenderDefault.kt @@ -1,15 +1,15 @@ package org.wikimedia.metricsplatform import okhttp3.OkHttpClient +import okhttp3.Request import okhttp3.RequestBody -import okhttp3.ResponseBody import org.wikimedia.metricsplatform.event.EventProcessed class EventSenderDefault(private val gson: com.google.gson.Gson, private val httpClient: OkHttpClient ) : EventSender { override fun sendEvents(baseUri: java.net.URL, events: List) { - val request = okhttp3.Request.Builder() + val request = Request.Builder() .url(baseUri) .header("Accept", "application/json") .header( @@ -20,8 +20,8 @@ class EventSenderDefault(private val gson: com.google.gson.Gson, .build() httpClient.newCall(request).execute().use { response -> - val status: kotlin.Int = response.code - val body: ResponseBody? = response.body + val status = response.code + val body = response.body if (!response.isSuccessful || status == 207) { // In the case of a multi-status response (207), it likely means that one or more // events were rejected. In such a case, the error is actually contained in diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/MetricsClient.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/MetricsClient.kt index fe448181164..fc4b0c93450 100644 --- a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/MetricsClient.kt +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/MetricsClient.kt @@ -10,9 +10,7 @@ import org.wikimedia.metricsplatform.event.Event import org.wikimedia.metricsplatform.event.EventProcessed import java.net.URL import java.time.Instant -import java.time.ZoneId import java.time.format.DateTimeFormatter -import java.util.Locale import java.util.concurrent.BlockingQueue import java.util.concurrent.Executors import java.util.concurrent.LinkedBlockingQueue @@ -102,7 +100,7 @@ class MetricsClient private constructor( streamName: String?, schemaId: String?, eventName: String?, - customData: MutableMap? + customData: Map? ) { submitMetricsEvent(streamName, schemaId, eventName, null, customData, null) } @@ -133,9 +131,9 @@ class MetricsClient private constructor( streamName: String?, schemaId: String?, eventName: String?, - clientData: ClientData, - customData: MutableMap?, - interactionData: InteractionData = null + clientData: ClientData?, + customData: Map?, + interactionData: InteractionData? = null ) { if (streamName == null) { //log.log(Level.FINE, "No stream has been specified, the submitMetricsEvent event is ignored and dropped.") @@ -156,19 +154,21 @@ class MetricsClient private constructor( } } - val event = Event(schemaId!!, streamName, eventName) - event.clientData = clientData - + val event = Event(schemaId!!) + event.stream = streamName + event.name = eventName + if (clientData != null) { + event.clientData = clientData + } if (customData != null) { event.customData = customData } - - event.interactionData = interactionData - + if (interactionData != null) { + event.interactionData = interactionData + } if (streamConfig != null && streamConfig.hasSampleConfig()) { event.sample = streamConfig.sampleConfig } - submit(event) } @@ -224,7 +224,7 @@ class MetricsClient private constructor( eventName: String?, clientData: ClientData, interactionData: InteractionData, - customData: MutableMap? + customData: Map? ) { submitMetricsEvent(streamName, schemaId, eventName, clientData, customData, interactionData) } @@ -270,7 +270,7 @@ class MetricsClient private constructor( schemaId: String?, eventName: String?, clientData: ClientData, - customData: MutableMap?, + customData: Map?, interactionData: InteractionData ) { submitMetricsEvent(streamName, schemaId, eventName, clientData, customData, interactionData) @@ -313,7 +313,7 @@ class MetricsClient private constructor( schemaId: String?, eventName: String?, clientData: ClientData, - customData: MutableMap?, + customData: Map?, interactionData: InteractionData ) { submitMetricsEvent(streamName, schemaId, eventName, clientData, customData, interactionData) @@ -353,8 +353,7 @@ class MetricsClient private constructor( * Closes the session. */ fun onAppClose() { - executorService.schedule( - Runnable { eventProcessor.sendEnqueuedEvents() }, + executorService.schedule({ eventProcessor.sendEnqueuedEvents() }, 0, TimeUnit.MILLISECONDS ) @@ -443,7 +442,7 @@ class MetricsClient private constructor( curationController, sourceConfigRef, samplingController!!, - EventSenderDefault(gson, httpClient), + EventSenderDefault(httpClient), eventQueue, isDebug ) @@ -476,10 +475,7 @@ class MetricsClient private constructor( } companion object { - @JvmField - val DATE_FORMAT: DateTimeFormatter = DateTimeFormatter - .ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ROOT) - .withZone(ZoneId.of("UTC")) + val DATE_FORMAT: DateTimeFormatter = DateTimeFormatter.ISO_DATE_TIME const val METRICS_PLATFORM_LIBRARY_VERSION: String = "2.8" const val METRICS_PLATFORM_BASE_VERSION: String = "1.2.2" diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/event/Event.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/event/Event.kt index b98486dfdf0..07e5dfb2067 100644 --- a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/event/Event.kt +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/event/Event.kt @@ -16,7 +16,7 @@ open class Event(@Transient val _stream: String = "") { @SerialName("dt") var timestamp: String? = null - @SerialName("custom_data") var customData: MutableMap? = null + @SerialName("custom_data") var customData: Map? = null protected val meta = Meta(_stream) diff --git a/app/build.gradle b/app/build.gradle index e11287cf0f6..b0eb450075c 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -174,7 +174,7 @@ dependencies { coreLibraryDesugaring libs.desugar.jdk.libs - implementation project(':analytics') + implementation project(':analytics:metricsplatform') implementation libs.kotlin.stdlib.jdk8 implementation libs.kotlinx.coroutines.core From d6090b422ddb425d1d46a26b87b8fe3316f6518e Mon Sep 17 00:00:00 2001 From: Dmitry Brant Date: Tue, 1 Jul 2025 12:52:52 -0400 Subject: [PATCH 6/9] Make it build. --- .../metricsplatform/ContextController.kt | 58 +++++++------- .../metricsplatform/EventProcessor.kt | 2 +- .../wikimedia/metricsplatform/EventSender.kt | 4 +- .../metricsplatform/EventSenderDefault.kt | 19 +++-- .../metricsplatform/MetricsClient.kt | 75 +++++++------------ .../metricsplatform/SamplingController.kt | 4 +- .../metricsplatform/config/CurationFilter.kt | 65 ++++++++-------- .../config/DestinationEventService.kt | 11 +-- .../metricsplatform/config/StreamConfig.kt | 4 - .../config/StreamConfigFetcher.java | 52 ------------- .../curation/CollectionCurationRules.kt | 8 +- .../metricsplatform/context/ClientData.kt | 8 +- .../metricsplatform/context/PerformerData.kt | 2 +- .../metricsplatform/event/EventProcessed.kt | 22 +++--- .../wikimedia/metricsplatform/EventTest.java | 2 +- .../analytics/metricsplatform/MetricsEvent.kt | 2 +- .../metricsplatform/MetricsPlatform.kt | 7 +- 17 files changed, 133 insertions(+), 212 deletions(-) delete mode 100644 analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfigFetcher.java diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/ContextController.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/ContextController.kt index a41ce88f037..2b8c90a7bbf 100644 --- a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/ContextController.kt +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/ContextController.kt @@ -21,7 +21,7 @@ class ContextController { requestedValues.addAll(requestedValuesFromConfig) requestedValues.addAll(REQUIRED_PROPERTIES) val filteredData = filterClientData(event.clientData, requestedValues) - event.setClientData(filteredData) + event.applyClientData(filteredData) } private fun filterClientData(clientData: ClientData, requestedValues: Collection): ClientData { @@ -32,41 +32,41 @@ class ContextController { for (requestedValue in requestedValues) { when (requestedValue) { - ContextValue.AGENT_APP_INSTALL_ID -> newAgentData.appInstallId = clientData.agentData.appInstallId - ContextValue.AGENT_CLIENT_PLATFORM -> newAgentData.clientPlatform = clientData.agentData.clientPlatform - ContextValue.AGENT_CLIENT_PLATFORM_FAMILY -> newAgentData.clientPlatformFamily = clientData.agentData.clientPlatformFamily - ContextValue.AGENT_APP_FLAVOR -> newAgentData.appFlavor = clientData.agentData.appFlavor - ContextValue.AGENT_APP_THEME -> newAgentData.appTheme = clientData.agentData.appTheme - ContextValue.AGENT_APP_VERSION -> newAgentData.appVersion = clientData.agentData.appVersion - ContextValue.AGENT_APP_VERSION_NAME -> newAgentData.appVersionName = clientData.agentData.appVersionName - ContextValue.AGENT_DEVICE_FAMILY -> newAgentData.deviceFamily = clientData.agentData.deviceFamily - ContextValue.AGENT_DEVICE_LANGUAGE -> newAgentData.deviceLanguage = clientData.agentData.deviceLanguage - ContextValue.AGENT_RELEASE_STATUS -> newAgentData.releaseStatus = clientData.agentData.releaseStatus - ContextValue.PAGE_ID -> newPageData.id = clientData.pageData.id - ContextValue.PAGE_TITLE -> newPageData.title = clientData.pageData.title - ContextValue.PAGE_NAMESPACE_ID -> newPageData.namespaceId = clientData.pageData.namespaceId - ContextValue.PAGE_NAMESPACE_NAME -> newPageData.namespaceName = clientData.pageData.namespaceName - ContextValue.PAGE_REVISION_ID -> newPageData.revisionId = clientData.pageData.revisionId - ContextValue.PAGE_WIKIDATA_QID -> newPageData.wikidataItemQid = clientData.pageData.wikidataItemQid - ContextValue.PAGE_CONTENT_LANGUAGE -> newPageData.contentLanguage = clientData.pageData.contentLanguage - ContextValue.MEDIAWIKI_DATABASE -> newMediawikiData.database = clientData.mediawikiData.database - ContextValue.PERFORMER_ID -> newPerformerData.id = clientData.performerData.id - ContextValue.PERFORMER_NAME -> newPerformerData.name = clientData.performerData.name - ContextValue.PERFORMER_IS_LOGGED_IN -> newPerformerData.isLoggedIn = clientData.performerData.isLoggedIn - ContextValue.PERFORMER_IS_TEMP -> newPerformerData.isTemp = clientData.performerData.isTemp - ContextValue.PERFORMER_SESSION_ID -> newPerformerData.sessionId = clientData.performerData.sessionId - ContextValue.PERFORMER_PAGEVIEW_ID -> newPerformerData.pageviewId = clientData.performerData.pageviewId - ContextValue.PERFORMER_GROUPS -> newPerformerData.groups = clientData.performerData.groups + ContextValue.AGENT_APP_INSTALL_ID -> newAgentData.appInstallId = clientData.agentData?.appInstallId + ContextValue.AGENT_CLIENT_PLATFORM -> newAgentData.clientPlatform = clientData.agentData?.clientPlatform + ContextValue.AGENT_CLIENT_PLATFORM_FAMILY -> newAgentData.clientPlatformFamily = clientData.agentData?.clientPlatformFamily + ContextValue.AGENT_APP_FLAVOR -> newAgentData.appFlavor = clientData.agentData?.appFlavor + ContextValue.AGENT_APP_THEME -> newAgentData.appTheme = clientData.agentData?.appTheme + ContextValue.AGENT_APP_VERSION -> newAgentData.appVersion = clientData.agentData?.appVersion + ContextValue.AGENT_APP_VERSION_NAME -> newAgentData.appVersionName = clientData.agentData?.appVersionName + ContextValue.AGENT_DEVICE_FAMILY -> newAgentData.deviceFamily = clientData.agentData?.deviceFamily + ContextValue.AGENT_DEVICE_LANGUAGE -> newAgentData.deviceLanguage = clientData.agentData?.deviceLanguage + ContextValue.AGENT_RELEASE_STATUS -> newAgentData.releaseStatus = clientData.agentData?.releaseStatus + ContextValue.PAGE_ID -> newPageData.id = clientData.pageData?.id + ContextValue.PAGE_TITLE -> newPageData.title = clientData.pageData?.title + ContextValue.PAGE_NAMESPACE_ID -> newPageData.namespaceId = clientData.pageData?.namespaceId + ContextValue.PAGE_NAMESPACE_NAME -> newPageData.namespaceName = clientData.pageData?.namespaceName + ContextValue.PAGE_REVISION_ID -> newPageData.revisionId = clientData.pageData?.revisionId + ContextValue.PAGE_WIKIDATA_QID -> newPageData.wikidataItemQid = clientData.pageData?.wikidataItemQid + ContextValue.PAGE_CONTENT_LANGUAGE -> newPageData.contentLanguage = clientData.pageData?.contentLanguage + ContextValue.MEDIAWIKI_DATABASE -> newMediawikiData.database = clientData.mediawikiData?.database + ContextValue.PERFORMER_ID -> newPerformerData.id = clientData.performerData?.id + ContextValue.PERFORMER_NAME -> newPerformerData.name = clientData.performerData?.name + ContextValue.PERFORMER_IS_LOGGED_IN -> newPerformerData.isLoggedIn = clientData.performerData?.isLoggedIn + ContextValue.PERFORMER_IS_TEMP -> newPerformerData.isTemp = clientData.performerData?.isTemp + ContextValue.PERFORMER_SESSION_ID -> newPerformerData.sessionId = clientData.performerData?.sessionId + ContextValue.PERFORMER_PAGEVIEW_ID -> newPerformerData.pageviewId = clientData.performerData?.pageviewId + ContextValue.PERFORMER_GROUPS -> newPerformerData.groups = clientData.performerData?.groups ContextValue.PERFORMER_LANGUAGE_GROUPS -> { - var languageGroups = clientData.performerData.languageGroups + var languageGroups = clientData.performerData?.languageGroups if (languageGroups != null && languageGroups.length > 255) { languageGroups = languageGroups.substring(0, 255) } newPerformerData.languageGroups = languageGroups } - ContextValue.PERFORMER_LANGUAGE_PRIMARY -> newPerformerData.languagePrimary = clientData.performerData.languagePrimary - ContextValue.PERFORMER_REGISTRATION_DT -> newPerformerData.registrationDt = clientData.performerData.registrationDt + ContextValue.PERFORMER_LANGUAGE_PRIMARY -> newPerformerData.languagePrimary = clientData.performerData?.languagePrimary + ContextValue.PERFORMER_REGISTRATION_DT -> newPerformerData.registrationDt = clientData.performerData?.registrationDt else -> throw IllegalArgumentException("Unknown property: $requestedValue") } } diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventProcessor.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventProcessor.kt index 485e65b9597..7e2df419f3e 100644 --- a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventProcessor.kt +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventProcessor.kt @@ -94,7 +94,7 @@ class EventProcessor( streamConfigMap: Map ): DestinationEventService { val streamConfig = streamConfigMap[event.stream] - return streamConfig?.getDestinationEventService() ?: DestinationEventService.ANALYTICS + return streamConfig?.destinationEventService ?: DestinationEventService.ANALYTICS } private fun sendEventsToDestination( diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventSender.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventSender.kt index e78aef7145c..caeb22948b7 100644 --- a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventSender.kt +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventSender.kt @@ -1,7 +1,7 @@ package org.wikimedia.metricsplatform +import android.net.Uri import org.wikimedia.metricsplatform.event.EventProcessed -import java.net.URL fun interface EventSender { /** @@ -10,5 +10,5 @@ fun interface EventSender { * @param baseUri base uri of destination intake service * @param events events to be sent */ - fun sendEvents(baseUri: URL, events: List) + fun sendEvents(baseUri: Uri, events: List) } diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventSenderDefault.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventSenderDefault.kt index 05ec3e6f88a..83caec284d4 100644 --- a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventSenderDefault.kt +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventSenderDefault.kt @@ -1,22 +1,27 @@ package org.wikimedia.metricsplatform +import android.net.Uri +import kotlinx.serialization.json.Json +import okhttp3.MediaType.Companion.toMediaTypeOrNull import okhttp3.OkHttpClient import okhttp3.Request -import okhttp3.RequestBody +import okhttp3.RequestBody.Companion.toRequestBody import org.wikimedia.metricsplatform.event.EventProcessed +import java.io.IOException -class EventSenderDefault(private val gson: com.google.gson.Gson, - private val httpClient: OkHttpClient +class EventSenderDefault( + private val json: Json, + private val httpClient: OkHttpClient ) : EventSender { - override fun sendEvents(baseUri: java.net.URL, events: List) { + override fun sendEvents(baseUri: Uri, events: List) { val request = Request.Builder() - .url(baseUri) + .url(baseUri.toString()) .header("Accept", "application/json") .header( "User-Agent", "Metrics Platform Client/Java " + MetricsClient.METRICS_PLATFORM_LIBRARY_VERSION ) - .post(RequestBody.create(gson.toJson(events), parse.parse("application/json"))) + .post(json.encodeToString(events).toRequestBody("application/json".toMediaTypeOrNull())) .build() httpClient.newCall(request).execute().use { response -> @@ -26,7 +31,7 @@ class EventSenderDefault(private val gson: com.google.gson.Gson, // In the case of a multi-status response (207), it likely means that one or more // events were rejected. In such a case, the error is actually contained in // the normal response body. - throw java.io.IOException(body.string()) + throw IOException(body?.string().orEmpty()) } //log.log(java.util.logging.Level.INFO, "Sent " + events.size + " events successfully.") } diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/MetricsClient.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/MetricsClient.kt index fc4b0c93450..c7988774524 100644 --- a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/MetricsClient.kt +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/MetricsClient.kt @@ -1,28 +1,19 @@ package org.wikimedia.metricsplatform -import okhttp3.OkHttpClient import org.wikimedia.metricsplatform.config.SourceConfig import org.wikimedia.metricsplatform.config.StreamConfig -import org.wikimedia.metricsplatform.config.StreamConfigFetcher import org.wikimedia.metricsplatform.context.ClientData import org.wikimedia.metricsplatform.context.InteractionData import org.wikimedia.metricsplatform.event.Event import org.wikimedia.metricsplatform.event.EventProcessed -import java.net.URL import java.time.Instant import java.time.format.DateTimeFormatter import java.util.concurrent.BlockingQueue -import java.util.concurrent.Executors import java.util.concurrent.LinkedBlockingQueue -import java.util.concurrent.ScheduledExecutorService -import java.util.concurrent.ThreadFactory -import java.util.concurrent.TimeUnit -import java.util.concurrent.atomic.AtomicLong import java.util.concurrent.atomic.AtomicReference import kotlin.math.max class MetricsClient private constructor( - private val executorService: ScheduledExecutorService, /** * Handles logging session management. A new session begins (and a new session ID is created) * if the app has been inactive for 15 minutes or more. @@ -126,13 +117,12 @@ class MetricsClient private constructor( * @param clientData client context data * @param customData custom data */ - @JvmOverloads fun submitMetricsEvent( streamName: String?, schemaId: String?, eventName: String?, clientData: ClientData?, - customData: Map?, + customData: Map?, interactionData: InteractionData? = null ) { if (streamName == null) { @@ -161,7 +151,7 @@ class MetricsClient private constructor( event.clientData = clientData } if (customData != null) { - event.customData = customData + event.customData = customData.mapValues { it.value.toString() } } if (interactionData != null) { event.interactionData = interactionData @@ -191,7 +181,7 @@ class MetricsClient private constructor( streamName: String?, eventName: String?, clientData: ClientData, - interactionData: InteractionData + interactionData: InteractionData? ) { submitMetricsEvent( streamName, @@ -223,8 +213,8 @@ class MetricsClient private constructor( schemaId: String?, eventName: String?, clientData: ClientData, - interactionData: InteractionData, - customData: Map? + interactionData: InteractionData?, + customData: Map? ) { submitMetricsEvent(streamName, schemaId, eventName, clientData, customData, interactionData) } @@ -329,12 +319,9 @@ class MetricsClient private constructor( * application is resumed. */ fun onAppPause() { - executorService.schedule( - Runnable { eventProcessor.sendEnqueuedEvents() }, - 0, - TimeUnit.MILLISECONDS - ) sessionController.touchSession() + + //eventProcessor.sendEnqueuedEvents() } /** @@ -353,11 +340,9 @@ class MetricsClient private constructor( * Closes the session. */ fun onAppClose() { - executorService.schedule({ eventProcessor.sendEnqueuedEvents() }, - 0, - TimeUnit.MILLISECONDS - ) sessionController.closeSession() + + //eventProcessor.sendEnqueuedEvents() } /** @@ -377,7 +362,7 @@ class MetricsClient private constructor( * @param event event */ private fun addRequiredMetadata(event: EventProcessed) { - event.performerData.sessionId = sessionController.sessionId + event.performerData?.let { it.sessionId = sessionController.sessionId } event.timestamp = DATE_FORMAT.format(Instant.now()) event.setDomain(event.clientData.domain) } @@ -393,7 +378,7 @@ class MetricsClient private constructor( var eventQueueAppendAttempts = max(eventQueue.size / 50, 10) while (!eventQueue.offer(event)) { - val removedEvent: EventProcessed? = eventQueue.remove() + val removedEvent = eventQueue.remove() if (removedEvent != null) { //log.log(Level.FINE, removedEvent.name + " was dropped so that a newer event could be added to the queue.") } @@ -414,22 +399,27 @@ class MetricsClient private constructor( private var samplingController: SamplingController? = null - private val httpClient: OkHttpClient = OkHttpClient() - - private val streamConfigURL: URL = safeURL(StreamConfigFetcher.ANALYTICS_API_ENDPOINT) - - private val isDebug = false - - private val executorService: ScheduledExecutorService = - Executors.newScheduledThreadPool(1, SimpleThreadFactory()) + private var isDebug = false private val sourceConfig: SourceConfig? = null + private var eventSender: EventSender? = null + fun eventQueueCapacity(capacity: Int): Builder { eventQueue = LinkedBlockingQueue(capacity) return this } + fun eventSender(eventSender: EventSender): Builder { + this.eventSender = eventSender + return this + } + + fun isDebug(isDebug: Boolean): Builder { + this.isDebug = isDebug + return this + } + fun build(): MetricsClient { if (sourceConfig != null) sourceConfigRef.set(sourceConfig) @@ -442,13 +432,12 @@ class MetricsClient private constructor( curationController, sourceConfigRef, samplingController!!, - EventSenderDefault(httpClient), + eventSender!!, eventQueue, isDebug ) val metricsClient = MetricsClient( - executorService, sessionController, samplingController!!, sourceConfigRef, @@ -458,20 +447,6 @@ class MetricsClient private constructor( return metricsClient } - - companion object { - private fun safeURL(url: String?): URL { - return URL(url) - } - } - } - - private class SimpleThreadFactory : ThreadFactory { - private val counter = AtomicLong() - - override fun newThread(r: Runnable?): Thread { - return Thread(r, "metrics-client-" + counter.incrementAndGet()) - } } companion object { diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/SamplingController.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/SamplingController.kt index 3eb5a5373b2..bc9a53a72af 100644 --- a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/SamplingController.kt +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/SamplingController.kt @@ -49,8 +49,8 @@ class SamplingController internal constructor( fun getSamplingId(unit: String): String { return when (unit) { SampleConfig.UNIT_SESSION -> return sessionController.sessionId - SampleConfig.UNIT_DEVICE -> return clientData.agentData.appInstallId.orEmpty() - SampleConfig.UNIT_PAGEVIEW -> return clientData.performerData.pageviewId.orEmpty() + SampleConfig.UNIT_DEVICE -> return clientData.agentData?.appInstallId.orEmpty() + SampleConfig.UNIT_PAGEVIEW -> return clientData.performerData?.pageviewId.orEmpty() else -> UUID.randomUUID().toString() } } diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/CurationFilter.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/CurationFilter.kt index 4a226f9fc0c..587f5c31cd5 100644 --- a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/CurationFilter.kt +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/CurationFilter.kt @@ -19,9 +19,9 @@ import java.util.function.Predicate @Serializable class CurationFilter : Predicate { - @SerialName("agent_app_install_id") var agentAppInstallIdRules: CurationRules? = null - @SerialName("agent_client_platform") var agentClientPlatformRules: CurationRules? = null - @SerialName("agent_client_platform_family") var agentClientPlatformFamilyRules: CurationRules? = null + @SerialName("agent_app_install_id") var agentAppInstallIdRules: CurationRules? = null + @SerialName("agent_client_platform") var agentClientPlatformRules: CurationRules? = null + @SerialName("agent_client_platform_family") var agentClientPlatformFamilyRules: CurationRules? = null @SerialName("mediawiki_database") var mediawikiDatabase: CurationRules? = null @SerialName("page_id") var pageIdRules: ComparableCurationRules? = null @SerialName("page_namespace_id") var pageNamespaceIdRules: ComparableCurationRules? = null @@ -37,13 +37,13 @@ class CurationFilter : Predicate { @SerialName("performer_groups") var performerGroupsRules: CollectionCurationRules? = null @SerialName("performer_is_logged_in") var performerIsLoggedInRules: CurationRules? = null @SerialName("performer_is_temp") var performerIsTempRules: CurationRules? = null - @SerialName("performer_registration_dt") var performerRegistrationDtRules: ComparableCurationRules? = null + @SerialName("performer_registration_dt") var performerRegistrationDtRules: ComparableCurationRules? = null @SerialName("performer_language_groups") - var performerLanguageGroupsRules: CurationRules? = null + var performerLanguageGroupsRules: CurationRules? = null @SerialName("performer_language_primary") - var performerLanguagePrimaryRules: CurationRules? = null + var performerLanguagePrimaryRules: CurationRules? = null override fun test(event: EventProcessed): Boolean { return applyAgentRules(event.agentData) @@ -52,40 +52,37 @@ class CurationFilter : Predicate { && applyPerformerRules(event.performerData) } - private fun applyAgentRules(data: AgentData): Boolean { - return applyPredicate(this.agentAppInstallIdRules, data.appInstallId) - && applyPredicate(this.agentClientPlatformRules, data.clientPlatform) - && applyPredicate( - this.agentClientPlatformFamilyRules, - data.clientPlatformFamily - ) + private fun applyAgentRules(data: AgentData?): Boolean { + return applyPredicate(this.agentAppInstallIdRules, data?.appInstallId) + && applyPredicate(this.agentClientPlatformRules, data?.clientPlatform) + && applyPredicate(this.agentClientPlatformFamilyRules, data?.clientPlatformFamily) } - private fun applyMediaWikiRules(data: MediawikiData): Boolean { - return applyPredicate(this.mediawikiDatabase, data.database) + private fun applyMediaWikiRules(data: MediawikiData?): Boolean { + return applyPredicate(this.mediawikiDatabase, data?.database) } - private fun applyPageRules(data: PageData): Boolean { - return applyPredicate(this.pageIdRules, data.id) - && applyPredicate(this.pageNamespaceIdRules, data.namespaceId) - && applyPredicate(this.pageNamespaceNameRules, data.namespaceName) - && applyPredicate(this.pageTitleRules, data.title) - && applyPredicate(this.pageRevisionIdRules, data.revisionId) - && applyPredicate(this.pageWikidataQidRules, data.wikidataItemQid) - && applyPredicate(this.pageContentLanguageRules, data.contentLanguage) + private fun applyPageRules(data: PageData?): Boolean { + return applyPredicate(this.pageIdRules, data?.id) + && applyPredicate(this.pageNamespaceIdRules, data?.namespaceId) + && applyPredicate(this.pageNamespaceNameRules, data?.namespaceName) + && applyPredicate(this.pageTitleRules, data?.title) + && applyPredicate(this.pageRevisionIdRules, data?.revisionId) + && applyPredicate(this.pageWikidataQidRules, data?.wikidataItemQid) + && applyPredicate(this.pageContentLanguageRules, data?.contentLanguage) } - private fun applyPerformerRules(data: PerformerData): Boolean { - return applyPredicate(this.performerIdRules, data.id) - && applyPredicate(this.performerNameRules, data.name) - && applyPredicate(this.performerSessionIdRules, data.sessionId) - && applyPredicate(this.performerPageviewIdRules, data.pageviewId) - && applyPredicate(this.performerGroupsRules, data.groups) - && applyPredicate(this.performerIsLoggedInRules, data.isLoggedIn) - && applyPredicate(this.performerIsTempRules, data.isTemp) - && applyPredicate(this.performerRegistrationDtRules, data.registrationDt) - && applyPredicate(this.performerLanguageGroupsRules, data.languageGroups) - && applyPredicate(this.performerLanguagePrimaryRules, data.languagePrimary) + private fun applyPerformerRules(data: PerformerData?): Boolean { + return applyPredicate(this.performerIdRules, data?.id) + && applyPredicate(this.performerNameRules, data?.name) + && applyPredicate(this.performerSessionIdRules, data?.sessionId) + && applyPredicate(this.performerPageviewIdRules, data?.pageviewId) + && applyPredicate(this.performerGroupsRules, data?.groups) + && applyPredicate(this.performerIsLoggedInRules, data?.isLoggedIn) + && applyPredicate(this.performerIsTempRules, data?.isTemp) + && applyPredicate(this.performerRegistrationDtRules, data?.registrationDt) + && applyPredicate(this.performerLanguageGroupsRules, data?.languageGroups) + && applyPredicate(this.performerLanguagePrimaryRules, data?.languagePrimary) } companion object { diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/DestinationEventService.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/DestinationEventService.kt index 280632cbe07..1c80090be0e 100644 --- a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/DestinationEventService.kt +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/DestinationEventService.kt @@ -1,7 +1,8 @@ package org.wikimedia.metricsplatform.config +import android.net.Uri import com.google.gson.annotations.SerializedName -import java.net.URL +import androidx.core.net.toUri /** * Possible event destination endpoints which can be specified in stream configurations. @@ -19,13 +20,13 @@ enum class DestinationEventService(baseUri: String) { @SerializedName("eventgate-logging-local") LOCAL("http://localhost:8192"); - private val baseUri: URL = URL("$baseUri/v1/events") + private val baseUri = "$baseUri/v1/events".toUri() - fun getBaseUri(): URL { + fun getBaseUri(): Uri { return getBaseUri(false) } - fun getBaseUri(isDebug: Boolean): URL { - return if (isDebug) this.baseUri else URL("$baseUri?hasty=true") + fun getBaseUri(isDebug: Boolean): Uri { + return if (isDebug) this.baseUri else "$baseUri?hasty=true".toUri() } } diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfig.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfig.kt index b87baa9efd9..4f3f559ed8e 100644 --- a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfig.kt +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfig.kt @@ -23,10 +23,6 @@ class StreamConfig { val events get() = producerConfig?.metricsPlatformClientConfig?.events.orEmpty() - fun getDestinationEventService(): DestinationEventService { - return if (destinationEventService != null) destinationEventService!! else DestinationEventService.ANALYTICS - } - fun hasCurationFilter(): Boolean { return producerConfig?.metricsPlatformClientConfig?.curationFilter != null } diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfigFetcher.java b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfigFetcher.java deleted file mode 100644 index 18f0486d982..00000000000 --- a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfigFetcher.java +++ /dev/null @@ -1,52 +0,0 @@ -package org.wikimedia.metricsplatform.config; - -import java.io.IOException; -import java.io.Reader; -import java.net.URL; -import java.util.Map; -import java.util.stream.Collectors; - -import com.google.gson.Gson; - -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.Response; -import okhttp3.ResponseBody; - -public class StreamConfigFetcher { - public static final String ANALYTICS_API_ENDPOINT = "https://meta.wikimedia.org/w/api.php?" + - "action=streamconfigs&format=json&formatversion=2&" + - "constraints=destination_event_service%3Deventgate-analytics-external"; - - public static final String METRICS_PLATFORM_SCHEMA_TITLE = "analytics/mediawiki/client/metrics_event"; - - private final URL url; - private final OkHttpClient httpClient; - private final Gson gson; - - public StreamConfigFetcher(URL url, OkHttpClient httpClient, Gson gson) { - this.url = url; - this.httpClient = httpClient; - this.gson = gson; - } - - /** - * Fetch stream configs from analytics endpoint. - */ - public SourceConfig fetchStreamConfigs() throws IOException { - Request request = new Request.Builder().url(url).build(); - try (Response response = httpClient.newCall(request).execute()) { - ResponseBody body = response.body(); - if (body == null) { - throw new IOException("Failed to fetch stream configs: " + response.message()); - } - return new SourceConfig(parseConfig(body.charStream())); - } - } - - // Visible For Testing - public Map parseConfig(Reader reader) { - return gson.fromJson(reader, StreamConfigCollection.class).streamConfigs.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - } -} diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/curation/CollectionCurationRules.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/curation/CollectionCurationRules.kt index b65c97c5e05..e0c31917ca2 100644 --- a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/curation/CollectionCurationRules.kt +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/curation/CollectionCurationRules.kt @@ -5,16 +5,16 @@ import kotlinx.serialization.Serializable import java.util.function.Predicate @Serializable -class CollectionCurationRules : Predicate?> { +class CollectionCurationRules : Predicate?> { var contains: T? = null @SerialName("does_not_contain") var doesNotContain: T? = null - @SerialName("contains_all") var containsAll: MutableCollection? = null + @SerialName("contains_all") var containsAll: Collection? = null - @SerialName("contains_any") var containsAny: MutableCollection? = null + @SerialName("contains_any") var containsAny: Collection? = null - override fun test(value: MutableCollection?): Boolean { + override fun test(value: Collection?): Boolean { if (value == null) return false return (contains == null || value.contains(contains)) && (doesNotContain == null || !value.contains(doesNotContain)) diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/ClientData.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/ClientData.kt index 33c88b41921..c93092a1c48 100644 --- a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/ClientData.kt +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/ClientData.kt @@ -14,9 +14,9 @@ import kotlinx.serialization.Serializable */ @Serializable open class ClientData ( - @SerialName("agent") val agentData: AgentData = AgentData(), - @SerialName("page") val pageData: PageData = PageData(), - @SerialName("mediawiki") val mediawikiData: MediawikiData = MediawikiData(), - @SerialName("performer") val performerData: PerformerData = PerformerData(), + @SerialName("agent") val agentData: AgentData? = null, + @SerialName("page") val pageData: PageData? = null, + @SerialName("mediawiki") val mediawikiData: MediawikiData? = null, + @SerialName("performer") val performerData: PerformerData? = null, @SerialName("domain") val domain: String? = null ) diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/PerformerData.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/PerformerData.kt index 9c9aa4ec8f3..3cf6b7a5397 100644 --- a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/PerformerData.kt +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/context/PerformerData.kt @@ -21,7 +21,7 @@ data class PerformerData ( @SerialName("is_temp") var isTemp: Boolean? = null, @SerialName("session_id") var sessionId: String? = null, @SerialName("pageview_id") var pageviewId: String? = null, - @SerialName("groups") var groups: MutableCollection? = null, + @SerialName("groups") var groups: Collection? = null, @SerialName("language_groups") var languageGroups: String? = null, @SerialName("language_primary") var languagePrimary: String? = null, @SerialName("registration_dt") var registrationDt: Instant? = null diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/event/EventProcessed.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/event/EventProcessed.kt index 973bebdb8b0..9061771b104 100644 --- a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/event/EventProcessed.kt +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/event/EventProcessed.kt @@ -12,10 +12,10 @@ import org.wikimedia.metricsplatform.context.PerformerData @Serializable class EventProcessed : Event { - @SerialName("agent") var agentData: AgentData = AgentData() - @SerialName("page") var pageData: PageData = PageData() - @SerialName("mediawiki") var mediawikiData: MediawikiData = MediawikiData() - @SerialName("performer") var performerData: PerformerData = PerformerData() + @SerialName("agent") var agentData: AgentData? = null + @SerialName("page") var pageData: PageData? = null + @SerialName("mediawiki") var mediawikiData: MediawikiData? = null + @SerialName("performer") var performerData: PerformerData? = null @SerialName("action") var action: String? = null @SerialName("action_subtype") private var actionSubtype: String? = null @@ -67,7 +67,7 @@ class EventProcessed : Event { schema: String?, stream: String, name: String?, - customData: MutableMap?, + customData: Map?, clientData: ClientData, sample: SampleConfig?, interactionData: InteractionData @@ -85,15 +85,15 @@ class EventProcessed : Event { this.action = interactionData.action } - fun setClientData(clientData: ClientData) { - setAgentData(clientData.agentData) - setPageData(clientData.pageData) - setMediawikiData(clientData.mediawikiData) - setPerformerData(clientData.performerData) + fun applyClientData(clientData: ClientData) { + agentData = clientData.agentData + pageData = clientData.pageData + mediawikiData = clientData.mediawikiData + performerData = clientData.performerData this.clientData = clientData } - fun setInteractionData(interactionData: InteractionData) { + fun applyInteractionData(interactionData: InteractionData) { this.action = interactionData.action this.actionContext = interactionData.actionContext this.actionSource = interactionData.actionSource diff --git a/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/EventTest.java b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/EventTest.java index 674cf7b6a61..6fb08a5143a 100644 --- a/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/EventTest.java +++ b/analytics/metricsplatform/src/test/java/org/wikimedia/metricsplatform/EventTest.java @@ -88,7 +88,7 @@ class EventTest { assertThat(event.getPerformerData().getLanguagePrimary()).isEqualTo("zh-tw"); assertThat(event.getPerformerData().getRegistrationDt()).isEqualTo("2023-03-01T01:08:30Z"); - event.setInteractionData(DataFixtures.getTestInteractionData("TestAction")); + event.applyInteractionData(DataFixtures.getTestInteractionData("TestAction")); assertThat(event.getAction()).isEqualTo("TestAction"); assertThat(event.getActionSource()).isEqualTo("TestActionSource"); diff --git a/app/src/main/java/org/wikipedia/analytics/metricsplatform/MetricsEvent.kt b/app/src/main/java/org/wikipedia/analytics/metricsplatform/MetricsEvent.kt index c1394bda042..662c498433b 100644 --- a/app/src/main/java/org/wikipedia/analytics/metricsplatform/MetricsEvent.kt +++ b/app/src/main/java/org/wikipedia/analytics/metricsplatform/MetricsEvent.kt @@ -25,7 +25,7 @@ open class MetricsEvent { protected fun submitEvent( streamName: String, eventName: String, - interactionData: InteractionData?, + interactionData: InteractionData? = null, pageData: PageData? = null ) { MetricsPlatform.client.submitInteraction( diff --git a/app/src/main/java/org/wikipedia/analytics/metricsplatform/MetricsPlatform.kt b/app/src/main/java/org/wikipedia/analytics/metricsplatform/MetricsPlatform.kt index ce14eeac1e3..ff84ef44c21 100644 --- a/app/src/main/java/org/wikipedia/analytics/metricsplatform/MetricsPlatform.kt +++ b/app/src/main/java/org/wikipedia/analytics/metricsplatform/MetricsPlatform.kt @@ -1,6 +1,7 @@ package org.wikipedia.analytics.metricsplatform import android.os.Build +import org.wikimedia.metricsplatform.EventSenderDefault import org.wikimedia.metricsplatform.MetricsClient import org.wikimedia.metricsplatform.context.AgentData import org.wikimedia.metricsplatform.context.ClientData @@ -8,9 +9,9 @@ import org.wikimedia.metricsplatform.context.MediawikiData import org.wikipedia.BuildConfig import org.wikipedia.WikipediaApp import org.wikipedia.dataclient.okhttp.OkHttpConnectionFactory +import org.wikipedia.json.JsonUtil import org.wikipedia.settings.Prefs import org.wikipedia.util.ReleaseUtil -import java.time.Duration object MetricsPlatform { val agentData = AgentData( @@ -41,10 +42,8 @@ object MetricsPlatform { ) val client: MetricsClient = MetricsClient.builder(clientData) - .httpClient(OkHttpConnectionFactory.client) + .eventSender(EventSenderDefault(JsonUtil.json, OkHttpConnectionFactory.client)) .eventQueueCapacity(Prefs.analyticsQueueSize) - .streamConfigFetchInterval(Duration.ofHours(12)) - .sendEventsInterval(Duration.ofSeconds(30)) .isDebug(ReleaseUtil.isDevRelease) .build() } From 98ce0c784106ece07fa07e4c327e4e278a84cf90 Mon Sep 17 00:00:00 2001 From: Dmitry Brant Date: Tue, 1 Jul 2025 15:30:10 -0400 Subject: [PATCH 7/9] Make it run. --- .../metricsplatform/MetricsClient.kt | 55 +++++++++---------- .../wikimedia/metricsplatform/event/Event.kt | 2 +- .../metricsplatform/event/EventProcessed.kt | 10 ++-- 3 files changed, 32 insertions(+), 35 deletions(-) diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/MetricsClient.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/MetricsClient.kt index c7988774524..8a6e7cfe8e3 100644 --- a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/MetricsClient.kt +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/MetricsClient.kt @@ -6,7 +6,8 @@ import org.wikimedia.metricsplatform.context.ClientData import org.wikimedia.metricsplatform.context.InteractionData import org.wikimedia.metricsplatform.event.Event import org.wikimedia.metricsplatform.event.EventProcessed -import java.time.Instant +import java.time.ZoneId +import java.time.ZonedDateTime import java.time.format.DateTimeFormatter import java.util.concurrent.BlockingQueue import java.util.concurrent.LinkedBlockingQueue @@ -57,7 +58,7 @@ class MetricsClient private constructor( * @param event event data */ fun submit(event: Event) { - val eventProcessed: EventProcessed = EventProcessed.fromEvent(event) + val eventProcessed = EventProcessed.fromEvent(event) addRequiredMetadata(eventProcessed) addToEventQueue(eventProcessed) } @@ -88,9 +89,9 @@ class MetricsClient private constructor( * @param customData custom data */ fun submitMetricsEvent( - streamName: String?, - schemaId: String?, - eventName: String?, + streamName: String, + schemaId: String, + eventName: String, customData: Map? ) { submitMetricsEvent(streamName, schemaId, eventName, null, customData, null) @@ -118,18 +119,13 @@ class MetricsClient private constructor( * @param customData custom data */ fun submitMetricsEvent( - streamName: String?, - schemaId: String?, - eventName: String?, + streamName: String, + schemaId: String, + eventName: String, clientData: ClientData?, customData: Map?, interactionData: InteractionData? = null ) { - if (streamName == null) { - //log.log(Level.FINE, "No stream has been specified, the submitMetricsEvent event is ignored and dropped.") - return - } - // If we already have stream configs, then we can pre-validate certain conditions and exclude the event from the queue entirely. var streamConfig: StreamConfig? = null if (sourceConfig.get() != null) { @@ -144,8 +140,8 @@ class MetricsClient private constructor( } } - val event = Event(schemaId!!) - event.stream = streamName + val event = Event(streamName) + event.schema = schemaId event.name = eventName if (clientData != null) { event.clientData = clientData @@ -178,8 +174,8 @@ class MetricsClient private constructor( * @see [Metrics Platform/Java API](https://wikitech.wikimedia.org/wiki/Metrics_Platform/Java_API) */ fun submitInteraction( - streamName: String?, - eventName: String?, + streamName: String, + eventName: String, clientData: ClientData, interactionData: InteractionData? ) { @@ -209,9 +205,9 @@ class MetricsClient private constructor( * @see [Metrics Platform/Java API](https://wikitech.wikimedia.org/wiki/Metrics_Platform/Java_API) */ fun submitInteraction( - streamName: String?, - schemaId: String?, - eventName: String?, + streamName: String, + schemaId: String, + eventName: String, clientData: ClientData, interactionData: InteractionData?, customData: Map? @@ -229,7 +225,7 @@ class MetricsClient private constructor( * @see [Metrics Platform/Java API](https://wikitech.wikimedia.org/wiki/Metrics_Platform/Java_API) */ fun submitClick( - streamName: String?, + streamName: String, clientData: ClientData, interactionData: InteractionData ) { @@ -256,9 +252,9 @@ class MetricsClient private constructor( * @see [Metrics Platform/Java API](https://wikitech.wikimedia.org/wiki/Metrics_Platform/Java_API) */ fun submitClick( - streamName: String?, - schemaId: String?, - eventName: String?, + streamName: String, + schemaId: String, + eventName: String, clientData: ClientData, customData: Map?, interactionData: InteractionData @@ -274,7 +270,7 @@ class MetricsClient private constructor( * @param interactionData common data for the base interaction schema */ fun submitView( - streamName: String?, + streamName: String, clientData: ClientData, interactionData: InteractionData ) { @@ -299,9 +295,9 @@ class MetricsClient private constructor( * @param interactionData common data for the base interaction schema */ fun submitView( - streamName: String?, - schemaId: String?, - eventName: String?, + streamName: String, + schemaId: String, + eventName: String, clientData: ClientData, customData: Map?, interactionData: InteractionData @@ -363,7 +359,7 @@ class MetricsClient private constructor( */ private fun addRequiredMetadata(event: EventProcessed) { event.performerData?.let { it.sessionId = sessionController.sessionId } - event.timestamp = DATE_FORMAT.format(Instant.now()) + event.timestamp = DATE_FORMAT.format(ZonedDateTime.now(ZONE_Z)) event.setDomain(event.clientData.domain) } @@ -451,6 +447,7 @@ class MetricsClient private constructor( companion object { val DATE_FORMAT: DateTimeFormatter = DateTimeFormatter.ISO_DATE_TIME + val ZONE_Z = ZoneId.of("Z") const val METRICS_PLATFORM_LIBRARY_VERSION: String = "2.8" const val METRICS_PLATFORM_BASE_VERSION: String = "1.2.2" diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/event/Event.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/event/Event.kt index 07e5dfb2067..c4099a14d86 100644 --- a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/event/Event.kt +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/event/Event.kt @@ -12,7 +12,7 @@ open class Event(@Transient val _stream: String = "") { var name: String? = null @SerialName("\$schema") - var schema: String? = null + var schema: String = "" @SerialName("dt") var timestamp: String? = null diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/event/EventProcessed.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/event/EventProcessed.kt index 9061771b104..3bbff824312 100644 --- a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/event/EventProcessed.kt +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/event/EventProcessed.kt @@ -34,8 +34,8 @@ class EventProcessed : Event { * @param name event name * @param clientData agent, mediawiki, page, performer data */ - constructor(schema: String?, stream: String, name: String, clientData: ClientData) : super(schema!!) { - this.stream = stream + constructor(schema: String, stream: String, name: String, clientData: ClientData) : super(stream) { + this.schema = schema this.name = name this.clientData = clientData this.agentData = clientData.agentData @@ -64,15 +64,15 @@ class EventProcessed : Event { * leaving it as is for the time being rather than suppressing the error. */ constructor( - schema: String?, + schema: String, stream: String, name: String?, customData: Map?, clientData: ClientData, sample: SampleConfig?, interactionData: InteractionData - ) : super(schema!!) { - this.stream = stream + ) : super(stream) { + this.schema = schema this.name = name this.clientData = clientData this.agentData = clientData.agentData From 76f1cc3a717513843b2324080cf8bc78155c9b7a Mon Sep 17 00:00:00 2001 From: Dmitry Brant Date: Tue, 1 Jul 2025 20:40:21 -0400 Subject: [PATCH 8/9] Stay on target. --- .../metricsplatform/EventProcessor.kt | 14 +-- .../metricsplatform/MetricsClient.kt | 108 ++++-------------- .../metricsplatform/SamplingController.kt | 2 +- .../metricsplatform/config/StreamConfig.kt | 30 ++++- .../metricsplatform/MetricsPlatform.kt | 13 ++- 5 files changed, 64 insertions(+), 103 deletions(-) diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventProcessor.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventProcessor.kt index 7e2df419f3e..afa5ad1c5d8 100644 --- a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventProcessor.kt +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventProcessor.kt @@ -60,20 +60,20 @@ class EventProcessor( val streamConfigsMap = config.streamConfigsMap - val foo = pending.filter { event -> streamConfigsMap.containsKey(event.stream) } + pending.filter { event -> streamConfigsMap.containsKey(event.stream) } .filter { event -> val cfg = streamConfigsMap[event.stream] - if (cfg!!.hasSampleConfig()) { - event.sample = cfg.sampleConfig + if (cfg == null) { + false + } else { + cfg.sampleConfig?.let { event.sample = it } + samplingController.isInSample(cfg) } - samplingController.isInSample(cfg) } .filter { event -> eventPassesCurationRules(event, streamConfigsMap) } .groupBy { event -> destinationEventService(event, streamConfigsMap) } .forEach { (destinationEventService, pendingValidEvents) -> - this.sendEventsToDestination( - destinationEventService, pendingValidEvents - ) + this.sendEventsToDestination(destinationEventService, pendingValidEvents) } } diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/MetricsClient.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/MetricsClient.kt index 8a6e7cfe8e3..8d448814a9f 100644 --- a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/MetricsClient.kt +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/MetricsClient.kt @@ -14,31 +14,38 @@ import java.util.concurrent.LinkedBlockingQueue import java.util.concurrent.atomic.AtomicReference import kotlin.math.max -class MetricsClient private constructor( +class MetricsClient( + private val clientData: ClientData, + eventSender: EventSender, + sourceConfigInit: SourceConfig? = null, + queueCapacity: Int = 100, + val isDebug: Boolean = false +) { + + private val sourceConfig = AtomicReference(sourceConfigInit) + /** * Handles logging session management. A new session begins (and a new session ID is created) * if the app has been inactive for 15 minutes or more. */ - private val sessionController: SessionController, + private val sessionController = SessionController() + /** * Evaluates whether events for a given stream are in-sample based on the stream configuration. */ - private val samplingController: SamplingController, - private val sourceConfig: AtomicReference, - eventQueue: BlockingQueue, - eventProcessor: EventProcessor -) { + private val samplingController =SamplingController(clientData, sessionController) - private val eventQueue: BlockingQueue - private val eventProcessor: EventProcessor + private val eventQueue: BlockingQueue = LinkedBlockingQueue(queueCapacity) - /** - * MetricsClient constructor. - */ - init { - this.eventQueue = eventQueue - this.eventProcessor = eventProcessor - } + private val eventProcessor: EventProcessor = EventProcessor( + ContextController(), + CurationController(), + sourceConfig, + samplingController, + eventSender, + eventQueue, + isDebug + ) /** * Submit an event to be enqueued and sent to the Event Platform. @@ -152,7 +159,7 @@ class MetricsClient private constructor( if (interactionData != null) { event.interactionData = interactionData } - if (streamConfig != null && streamConfig.hasSampleConfig()) { + if (streamConfig?.sampleConfig != null) { event.sample = streamConfig.sampleConfig } submit(event) @@ -382,69 +389,6 @@ class MetricsClient private constructor( } } - val isFullyInitialized get() = sourceConfig.get() != null - - val isEventQueueEmpty get() = eventQueue.isEmpty() - - class Builder(private val clientData: ClientData) { - private val sourceConfigRef = AtomicReference() - private var eventQueue = LinkedBlockingQueue(10) - private val sessionController = SessionController() - - private val curationController = CurationController() - - private var samplingController: SamplingController? = null - - private var isDebug = false - - private val sourceConfig: SourceConfig? = null - - private var eventSender: EventSender? = null - - fun eventQueueCapacity(capacity: Int): Builder { - eventQueue = LinkedBlockingQueue(capacity) - return this - } - - fun eventSender(eventSender: EventSender): Builder { - this.eventSender = eventSender - return this - } - - fun isDebug(isDebug: Boolean): Builder { - this.isDebug = isDebug - return this - } - - fun build(): MetricsClient { - if (sourceConfig != null) sourceConfigRef.set(sourceConfig) - - if (samplingController == null) { - samplingController = SamplingController(clientData, sessionController) - } - - val eventProcessor = EventProcessor( - ContextController(), - curationController, - sourceConfigRef, - samplingController!!, - eventSender!!, - eventQueue, - isDebug - ) - - val metricsClient = MetricsClient( - sessionController, - samplingController!!, - sourceConfigRef, - eventQueue, - eventProcessor - ) - - return metricsClient - } - } - companion object { val DATE_FORMAT: DateTimeFormatter = DateTimeFormatter.ISO_DATE_TIME val ZONE_Z = ZoneId.of("Z") @@ -453,9 +397,5 @@ class MetricsClient private constructor( const val METRICS_PLATFORM_BASE_VERSION: String = "1.2.2" const val METRICS_PLATFORM_SCHEMA_BASE: String = "/analytics/product_metrics/app/base/$METRICS_PLATFORM_BASE_VERSION" - - fun builder(clientData: ClientData): Builder { - return Builder(clientData) - } } } diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/SamplingController.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/SamplingController.kt index bc9a53a72af..d7cf470da45 100644 --- a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/SamplingController.kt +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/SamplingController.kt @@ -21,7 +21,7 @@ class SamplingController internal constructor( * @return true if in sample or false otherwise */ fun isInSample(streamConfig: StreamConfig): Boolean { - if (!streamConfig.hasSampleConfig()) { + if (streamConfig.sampleConfig == null) { return true } val sampleConfig = streamConfig.sampleConfig!! diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfig.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfig.kt index 4f3f559ed8e..bbcce12dfe0 100644 --- a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfig.kt +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfig.kt @@ -6,20 +6,32 @@ import org.wikimedia.metricsplatform.config.sampling.SampleConfig @Serializable class StreamConfig { - @SerialName("stream") var streamName: String = "" + @SerialName("stream") + var streamName: String = "" + + @SerialName("canary_events_enabled") + var canaryEventsEnabled = false + + @SerialName("destination_event_service") + val destinationEventServiceKey: String = "eventgate-analytics-external" + + var destinationEventService: DestinationEventService = DestinationEventService.ANALYTICS + @SerialName("schema_title") var schemaTitle: String? = null - @SerialName("destination_event_service") var destinationEventService: DestinationEventService? = null + @SerialName("producers") var producerConfig: ProducerConfig? = null + @SerialName("sample") var sampleConfig: SampleConfig? = null + @SerialName("topic_prefixes") + val topicPrefixes: List = emptyList() + + val topics: List = emptyList() + fun hasRequestedContextValuesConfig(): Boolean { return producerConfig?.metricsPlatformClientConfig?.requestedValues != null } - fun hasSampleConfig(): Boolean { - return producerConfig?.metricsPlatformClientConfig != null && sampleConfig != null - } - val events get() = producerConfig?.metricsPlatformClientConfig?.events.orEmpty() @@ -29,6 +41,12 @@ class StreamConfig { val curationFilter get() = producerConfig?.metricsPlatformClientConfig?.curationFilter ?: CurationFilter() + init { + try { + destinationEventService = DestinationEventService.valueOf(destinationEventServiceKey) + } catch (_: Exception) { } + } + @Serializable class ProducerConfig { @SerialName("metrics_platform_client") var metricsPlatformClientConfig: MetricsPlatformClientConfig? = null diff --git a/app/src/main/java/org/wikipedia/analytics/metricsplatform/MetricsPlatform.kt b/app/src/main/java/org/wikipedia/analytics/metricsplatform/MetricsPlatform.kt index ff84ef44c21..c329ab2c0b0 100644 --- a/app/src/main/java/org/wikipedia/analytics/metricsplatform/MetricsPlatform.kt +++ b/app/src/main/java/org/wikipedia/analytics/metricsplatform/MetricsPlatform.kt @@ -41,9 +41,12 @@ object MetricsPlatform { domain ) - val client: MetricsClient = MetricsClient.builder(clientData) - .eventSender(EventSenderDefault(JsonUtil.json, OkHttpConnectionFactory.client)) - .eventQueueCapacity(Prefs.analyticsQueueSize) - .isDebug(ReleaseUtil.isDevRelease) - .build() + val client: MetricsClient = MetricsClient( + clientData, + EventSenderDefault(JsonUtil.json, OkHttpConnectionFactory.client), + null, + + queueCapacity = Prefs.analyticsQueueSize, + isDebug = ReleaseUtil.isDevRelease + ) } From 089f6fa2cd27b0e7e241e7031e0217a46b539355 Mon Sep 17 00:00:00 2001 From: Dmitry Brant Date: Mon, 21 Jul 2025 19:52:49 -0400 Subject: [PATCH 9/9] Start deduplicating. --- analytics/metricsplatform/build.gradle | 11 ++ .../metricsplatform/EventProcessor.kt | 41 ++----- .../metricsplatform/MetricsClient.kt | 110 +++--------------- .../config/DestinationEventService.kt | 27 +---- .../metricsplatform/config/SourceConfig.kt | 34 +----- .../metricsplatform/config/StreamConfig.kt | 2 + .../config/StreamConfigCollection.kt | 3 +- .../eventplatform/DestinationEventService.kt | 17 --- .../eventplatform/EventPlatformClient.kt | 10 +- .../analytics/eventplatform/SamplingConfig.kt | 18 --- .../analytics/eventplatform/StreamConfig.kt | 43 ------- .../java/org/wikipedia/dataclient/Service.kt | 4 +- .../wikipedia/dataclient/ServiceFactory.kt | 4 +- .../mwapi/MwStreamConfigsResponse.kt | 12 -- .../main/java/org/wikipedia/settings/Prefs.kt | 2 +- 15 files changed, 63 insertions(+), 275 deletions(-) delete mode 100644 app/src/main/java/org/wikipedia/analytics/eventplatform/DestinationEventService.kt delete mode 100644 app/src/main/java/org/wikipedia/analytics/eventplatform/SamplingConfig.kt delete mode 100644 app/src/main/java/org/wikipedia/analytics/eventplatform/StreamConfig.kt delete mode 100644 app/src/main/java/org/wikipedia/dataclient/mwapi/MwStreamConfigsResponse.kt diff --git a/analytics/metricsplatform/build.gradle b/analytics/metricsplatform/build.gradle index bf702040879..e74625f1781 100644 --- a/analytics/metricsplatform/build.gradle +++ b/analytics/metricsplatform/build.gradle @@ -21,6 +21,17 @@ android { jvmTarget = JAVA_VERSION } compileSdk 35 + + defaultConfig { + targetSdk 35 + + buildConfigField "String", "EVENTGATE_ANALYTICS_EXTERNAL_BASE_URI", '"https://intake-analytics.wikimedia.org"' + buildConfigField "String", "EVENTGATE_LOGGING_EXTERNAL_BASE_URI", '"https://intake-logging.wikimedia.org"' + } + + buildFeatures { + buildConfig true + } } dependencies { diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventProcessor.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventProcessor.kt index afa5ad1c5d8..f9f9cb645e2 100644 --- a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventProcessor.kt +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/EventProcessor.kt @@ -10,33 +10,13 @@ import java.util.concurrent.BlockingQueue import java.util.concurrent.atomic.AtomicReference class EventProcessor( - /** - * Enriches event data with context data requested in the stream configuration. - */ private val contextController: ContextController, private val curationController: CurationController, - sourceConfig: AtomicReference, - samplingController: SamplingController, - eventSender: EventSender, - eventQueue: BlockingQueue, - isDebug: Boolean -) { - private val sourceConfig: AtomicReference - private val samplingController: SamplingController + private val sourceConfig: AtomicReference, + private val samplingController: SamplingController, + private val eventSender: EventSender, private val eventQueue: BlockingQueue - private val eventSender: EventSender - private val isDebug: Boolean - - /** - * EventProcessor constructor. - */ - init { - this.sourceConfig = sourceConfig - this.samplingController = samplingController - this.eventSender = eventSender - this.eventQueue = eventQueue - this.isDebug = isDebug - } +) { /** * Send all events currently in the output buffer. @@ -55,10 +35,12 @@ class EventProcessor( return } - val pending: ArrayList = ArrayList() - this.eventQueue.drainTo(pending) + val pending = mutableListOf() + synchronized(eventQueue) { + eventQueue.drainTo(pending) + } - val streamConfigsMap = config.streamConfigsMap + val streamConfigsMap = config.streamConfigs pending.filter { event -> streamConfigsMap.containsKey(event.stream) } .filter { event -> @@ -73,7 +55,7 @@ class EventProcessor( .filter { event -> eventPassesCurationRules(event, streamConfigsMap) } .groupBy { event -> destinationEventService(event, streamConfigsMap) } .forEach { (destinationEventService, pendingValidEvents) -> - this.sendEventsToDestination(destinationEventService, pendingValidEvents) + sendEventsToDestination(destinationEventService, pendingValidEvents) } } @@ -102,7 +84,8 @@ class EventProcessor( pendingValidEvents: List ) { try { - eventSender.sendEvents(destinationEventService.getBaseUri(isDebug), pendingValidEvents) + //TODO! + //eventSender.sendEvents(destinationEventService.getBaseUri(isDebug), pendingValidEvents) } catch (e: UnknownHostException) { //log.log(Level.WARNING, "Network error while sending " + pendingValidEvents.size + " events. Adding back to queue.", e) eventQueue.addAll(pendingValidEvents) diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/MetricsClient.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/MetricsClient.kt index 8d448814a9f..67b74d000a1 100644 --- a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/MetricsClient.kt +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/MetricsClient.kt @@ -15,11 +15,10 @@ import java.util.concurrent.atomic.AtomicReference import kotlin.math.max class MetricsClient( - private val clientData: ClientData, + clientData: ClientData, eventSender: EventSender, sourceConfigInit: SourceConfig? = null, - queueCapacity: Int = 100, - val isDebug: Boolean = false + val queueCapacity: Int = 100 ) { private val sourceConfig = AtomicReference(sourceConfigInit) @@ -33,7 +32,7 @@ class MetricsClient( /** * Evaluates whether events for a given stream are in-sample based on the stream configuration. */ - private val samplingController =SamplingController(clientData, sessionController) + private val samplingController = SamplingController(clientData, sessionController) private val eventQueue: BlockingQueue = LinkedBlockingQueue(queueCapacity) @@ -43,8 +42,7 @@ class MetricsClient( sourceConfig, samplingController, eventSender, - eventQueue, - isDebug + eventQueue ) /** @@ -70,51 +68,6 @@ class MetricsClient( addToEventQueue(eventProcessed) } - /** - * Construct and submits a Metrics Platform Event from the event name and custom data for each - * stream specified. - * - * - * The Metrics Platform Event for a stream (S) is constructed by: first initializing the minimum - * valid event (E) that can be submitted to S; and, second mixing the context attributes requested - * in the configuration for S into E. - * - * - * The Metrics Platform Event is submitted to a stream (S) if: 1) S is in sample; and 2) the event - * is filtered due to the filtering rules for S. - * - * - * This particular submitMetricsEvent method accepts unformatted custom data and calls the following - * submitMetricsEvent method with the custom data properly formatted. - * - * @see [Metrics Platform/Java API](https://wikitech.wikimedia.org/wiki/Metrics_Platform/Java_API) - * - * - * @param streamName stream name - * @param schemaId schema id - * @param eventName event name - * @param customData custom data - */ - fun submitMetricsEvent( - streamName: String, - schemaId: String, - eventName: String, - customData: Map? - ) { - submitMetricsEvent(streamName, schemaId, eventName, null, customData, null) - } - - /** - * Construct and submits a Metrics Platform Event from the schema id, stream name, event name, page metadata, custom - * data, and interaction data for the specified stream. - * - * @param streamName stream name - * @param schemaId schema id - * @param eventName event name - * @param clientData client context data - * @param customData custom data - * @param interactionData common data for an interaction schema - */ /** * Construct and submits a Metrics Platform Event from the schema id, event name, page metadata, and custom data for * the stream that is interested in those events. @@ -129,8 +82,8 @@ class MetricsClient( streamName: String, schemaId: String, eventName: String, - clientData: ClientData?, - customData: Map?, + clientData: ClientData? = null, + customData: Map? = null, interactionData: InteractionData? = null ) { // If we already have stream configs, then we can pre-validate certain conditions and exclude the event from the queue entirely. @@ -165,37 +118,6 @@ class MetricsClient( submit(event) } - /** - * Submit an interaction event to a stream. - * - * - * An interaction event is meant to represent a basic interaction with some target or some event - * occurring, e.g. the user (**performer**) tapping/clicking a UI element, or an app notifying the - * server of its current state. - * - * @param streamName stream name - * @param eventName event name - * @param clientData client context data - * @param interactionData common data for the base interaction schema - * - * @see [Metrics Platform/Java API](https://wikitech.wikimedia.org/wiki/Metrics_Platform/Java_API) - */ - fun submitInteraction( - streamName: String, - eventName: String, - clientData: ClientData, - interactionData: InteractionData? - ) { - submitMetricsEvent( - streamName, - METRICS_PLATFORM_SCHEMA_BASE, - eventName, - clientData, - null, - interactionData - ) - } - /** * Submit an interaction event to a stream. * @@ -213,11 +135,11 @@ class MetricsClient( */ fun submitInteraction( streamName: String, - schemaId: String, eventName: String, - clientData: ClientData, - interactionData: InteractionData?, - customData: Map? + schemaId: String = METRICS_PLATFORM_SCHEMA_BASE, + clientData: ClientData? = null, + interactionData: InteractionData? = null, + customData: Map? = null ) { submitMetricsEvent(streamName, schemaId, eventName, clientData, customData, interactionData) } @@ -323,8 +245,7 @@ class MetricsClient( */ fun onAppPause() { sessionController.touchSession() - - //eventProcessor.sendEnqueuedEvents() + eventProcessor.sendEnqueuedEvents() } /** @@ -344,8 +265,7 @@ class MetricsClient( */ fun onAppClose() { sessionController.closeSession() - - //eventProcessor.sendEnqueuedEvents() + eventProcessor.sendEnqueuedEvents() } /** @@ -380,6 +300,10 @@ class MetricsClient( private fun addToEventQueue(event: EventProcessed?) { var eventQueueAppendAttempts = max(eventQueue.size / 50, 10) + if (eventQueue.size > queueCapacity / 2) { + eventProcessor.sendEnqueuedEvents() + } + while (!eventQueue.offer(event)) { val removedEvent = eventQueue.remove() if (removedEvent != null) { @@ -391,7 +315,7 @@ class MetricsClient( companion object { val DATE_FORMAT: DateTimeFormatter = DateTimeFormatter.ISO_DATE_TIME - val ZONE_Z = ZoneId.of("Z") + val ZONE_Z: ZoneId? = ZoneId.of("Z") const val METRICS_PLATFORM_LIBRARY_VERSION: String = "2.8" const val METRICS_PLATFORM_BASE_VERSION: String = "1.2.2" diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/DestinationEventService.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/DestinationEventService.kt index 1c80090be0e..6fd93bacd73 100644 --- a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/DestinationEventService.kt +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/DestinationEventService.kt @@ -1,8 +1,6 @@ package org.wikimedia.metricsplatform.config -import android.net.Uri -import com.google.gson.annotations.SerializedName -import androidx.core.net.toUri +import org.wikimedia.metricsplatform.BuildConfig /** * Possible event destination endpoints which can be specified in stream configurations. @@ -10,23 +8,8 @@ import androidx.core.net.toUri * * https://wikitech.wikimedia.org/wiki/Event_Platform/EventGate#EventGate_clusters */ -enum class DestinationEventService(baseUri: String) { - @SerializedName("eventgate-analytics-external") - ANALYTICS("https://intake-analytics.wikimedia.org"), - - @SerializedName("eventgate-logging-external") - ERROR_LOGGING("https://intake-logging.wikimedia.org"), - - @SerializedName("eventgate-logging-local") - LOCAL("http://localhost:8192"); - - private val baseUri = "$baseUri/v1/events".toUri() - - fun getBaseUri(): Uri { - return getBaseUri(false) - } - - fun getBaseUri(isDebug: Boolean): Uri { - return if (isDebug) this.baseUri else "$baseUri?hasty=true".toUri() - } +enum class DestinationEventService(val id: String, val baseUri: String) { + ANALYTICS("eventgate-analytics-external", BuildConfig.EVENTGATE_ANALYTICS_EXTERNAL_BASE_URI), + LOGGING("eventgate-logging-external", BuildConfig.EVENTGATE_LOGGING_EXTERNAL_BASE_URI), + LOCAL("eventgate-logging-local", "http://localhost:8192"); } diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/SourceConfig.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/SourceConfig.kt index aca743606b3..78be1055211 100644 --- a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/SourceConfig.kt +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/SourceConfig.kt @@ -1,35 +1,7 @@ package org.wikimedia.metricsplatform.config -class SourceConfig(streamConfigs: Map) { - /** - * Get all stream configs with stream name. - */ - val streamConfigsMap = streamConfigs - - /** - * Get all stream configs without stream name. - */ - val streamConfigsRaw = streamConfigs.values - - val streamNames get() = streamConfigsMap.keys - - /** - * Get stream config by stream name. - * - * @param streamName stream name - */ - fun getStreamConfigByName(streamName: String?): StreamConfig? { - return streamConfigsMap[streamName] - } - - /** - * Get stream names by event name. - * - * @param eventName event name - */ - fun getStreamNamesByEvent(eventName: String): Collection { - return streamConfigsRaw - .filter { streamConfig: StreamConfig -> streamConfig.isInterestedInEvent(eventName) } - .map { streamConfig: StreamConfig -> streamConfig.streamName } +class SourceConfig(val streamConfigs: Map) { + fun getStreamConfigByName(streamName: String): StreamConfig? { + return streamConfigs[streamName] } } diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfig.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfig.kt index bbcce12dfe0..89227b1c4e7 100644 --- a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfig.kt +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfig.kt @@ -2,6 +2,7 @@ package org.wikimedia.metricsplatform.config import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable +import kotlinx.serialization.Transient import org.wikimedia.metricsplatform.config.sampling.SampleConfig @Serializable @@ -15,6 +16,7 @@ class StreamConfig { @SerialName("destination_event_service") val destinationEventServiceKey: String = "eventgate-analytics-external" + @Transient var destinationEventService: DestinationEventService = DestinationEventService.ANALYTICS @SerialName("schema_title") var schemaTitle: String? = null diff --git a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfigCollection.kt b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfigCollection.kt index bfed994e15c..fdcfa025239 100644 --- a/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfigCollection.kt +++ b/analytics/metricsplatform/src/main/java/org/wikimedia/metricsplatform/config/StreamConfigCollection.kt @@ -3,7 +3,8 @@ package org.wikimedia.metricsplatform.config import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable +// TODO: make this inherit from MwResponse, and properly handle MW errors @Serializable class StreamConfigCollection { - @SerialName("streams") var streamConfigs: MutableMap? = null + @SerialName("streams") var streamConfigs: Map = emptyMap() } diff --git a/app/src/main/java/org/wikipedia/analytics/eventplatform/DestinationEventService.kt b/app/src/main/java/org/wikipedia/analytics/eventplatform/DestinationEventService.kt deleted file mode 100644 index 02f7c9eb962..00000000000 --- a/app/src/main/java/org/wikipedia/analytics/eventplatform/DestinationEventService.kt +++ /dev/null @@ -1,17 +0,0 @@ -package org.wikipedia.analytics.eventplatform - -import org.wikipedia.BuildConfig - -/** - * Possible event destination endpoints which can be specified in stream configurations. - * https://wikitech.wikimedia.org/wiki/Event_Platform/EventGate#EventGate_clusters - * - * N.B. Currently our streamconfigs API request is filtering for streams with their destination - * event service configured as eventgate-analytics-external. However, that will likely change in - * the future, so flexible destination event service support is added optimistically now. - */ -enum class DestinationEventService(val id: String, val baseUri: String) { - - ANALYTICS("eventgate-analytics-external", BuildConfig.EVENTGATE_ANALYTICS_EXTERNAL_BASE_URI), - LOGGING("eventgate-logging-external", BuildConfig.EVENTGATE_LOGGING_EXTERNAL_BASE_URI) -} diff --git a/app/src/main/java/org/wikipedia/analytics/eventplatform/EventPlatformClient.kt b/app/src/main/java/org/wikipedia/analytics/eventplatform/EventPlatformClient.kt index bf205817f7a..d6ff3646337 100644 --- a/app/src/main/java/org/wikipedia/analytics/eventplatform/EventPlatformClient.kt +++ b/app/src/main/java/org/wikipedia/analytics/eventplatform/EventPlatformClient.kt @@ -7,6 +7,8 @@ import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.MainScope import kotlinx.coroutines.launch +import org.wikimedia.metricsplatform.config.StreamConfig +import org.wikimedia.metricsplatform.config.sampling.SampleConfig import org.wikipedia.BuildConfig import org.wikipedia.WikipediaApp import org.wikipedia.dataclient.ServiceFactory @@ -328,7 +330,7 @@ object EventPlatformClient { return SAMPLING_CACHE[stream]!! } val streamConfig = getStreamConfig(stream) ?: return false - val samplingConfig = streamConfig.samplingConfig + val samplingConfig = streamConfig.sampleConfig if (samplingConfig == null || samplingConfig.rate == 1.0) { return true } @@ -350,13 +352,13 @@ object EventPlatformClient { } fun getSamplingId(unit: String): String { - if (unit == SamplingConfig.UNIT_SESSION) { + if (unit == SampleConfig.UNIT_SESSION) { return AssociationController.sessionId } - if (unit == SamplingConfig.UNIT_PAGEVIEW) { + if (unit == SampleConfig.UNIT_PAGEVIEW) { return AssociationController.pageViewId } - if (unit == SamplingConfig.UNIT_DEVICE) { + if (unit == SampleConfig.UNIT_DEVICE) { return WikipediaApp.instance.appInstallID } L.e("Bad identifier type") diff --git a/app/src/main/java/org/wikipedia/analytics/eventplatform/SamplingConfig.kt b/app/src/main/java/org/wikipedia/analytics/eventplatform/SamplingConfig.kt deleted file mode 100644 index ac925493d85..00000000000 --- a/app/src/main/java/org/wikipedia/analytics/eventplatform/SamplingConfig.kt +++ /dev/null @@ -1,18 +0,0 @@ -package org.wikipedia.analytics.eventplatform - -import kotlinx.serialization.Serializable - -/** - * Represents the sampling config component of a stream configuration. - */ -@Serializable -class SamplingConfig( - val rate: Double = 1.0, - val unit: String = UNIT_SESSION -) { - companion object { - const val UNIT_PAGEVIEW = "pageview" - const val UNIT_SESSION = "session" - const val UNIT_DEVICE = "device" - } -} diff --git a/app/src/main/java/org/wikipedia/analytics/eventplatform/StreamConfig.kt b/app/src/main/java/org/wikipedia/analytics/eventplatform/StreamConfig.kt deleted file mode 100644 index 9b399b7b144..00000000000 --- a/app/src/main/java/org/wikipedia/analytics/eventplatform/StreamConfig.kt +++ /dev/null @@ -1,43 +0,0 @@ -package org.wikipedia.analytics.eventplatform - -import kotlinx.serialization.SerialName -import kotlinx.serialization.Serializable -import org.wikipedia.analytics.eventplatform.DestinationEventService.ANALYTICS -import java.lang.IllegalArgumentException - -@Serializable -class StreamConfig { - - constructor(streamName: String, samplingConfig: SamplingConfig?, destinationEventService: DestinationEventService?) { - this.streamName = streamName - this.samplingConfig = samplingConfig - this.destinationEventService = destinationEventService ?: ANALYTICS - } - - @SerialName("stream") - var streamName = "" - - @SerialName("canary_events_enabled") - var canaryEventsEnabled = false - - @SerialName("destination_event_service") - val destinationEventServiceKey: String = "eventgate-analytics-external" - - var destinationEventService: DestinationEventService = ANALYTICS - - @SerialName("schema_title") - val schemaTitle: String = "" - - @SerialName("topic_prefixes") - val topicPrefixes: List = emptyList() - val topics: List = emptyList() - - @SerialName("sample") - var samplingConfig: SamplingConfig? = null - - init { - try { - destinationEventService = DestinationEventService.valueOf(destinationEventServiceKey) - } catch (e: IllegalArgumentException) {} - } -} diff --git a/app/src/main/java/org/wikipedia/dataclient/Service.kt b/app/src/main/java/org/wikipedia/dataclient/Service.kt index 6d8aa529c2d..7dd2715c966 100644 --- a/app/src/main/java/org/wikipedia/dataclient/Service.kt +++ b/app/src/main/java/org/wikipedia/dataclient/Service.kt @@ -1,5 +1,6 @@ package org.wikipedia.dataclient +import org.wikimedia.metricsplatform.config.StreamConfigCollection import org.wikipedia.captcha.Captcha import org.wikipedia.dataclient.discussiontools.DiscussionToolsEditResponse import org.wikipedia.dataclient.discussiontools.DiscussionToolsInfoResponse @@ -10,7 +11,6 @@ import org.wikipedia.dataclient.mwapi.CreateAccountResponse import org.wikipedia.dataclient.mwapi.MwParseResponse import org.wikipedia.dataclient.mwapi.MwPostResponse import org.wikipedia.dataclient.mwapi.MwQueryResponse -import org.wikipedia.dataclient.mwapi.MwStreamConfigsResponse import org.wikipedia.dataclient.mwapi.ParamInfoResponse import org.wikipedia.dataclient.mwapi.ShortenUrlResponse import org.wikipedia.dataclient.mwapi.SiteMatrix @@ -214,7 +214,7 @@ interface Service { ): MwPostResponse @GET(MW_API_PREFIX + "action=streamconfigs&format=json&constraints=destination_event_service%3Deventgate-analytics-external") - suspend fun getStreamConfigs(): MwStreamConfigsResponse + suspend fun getStreamConfigs(): StreamConfigCollection @GET(MW_API_PREFIX + "action=query&meta=allmessages&amenableparser=1") suspend fun getMessages( diff --git a/app/src/main/java/org/wikipedia/dataclient/ServiceFactory.kt b/app/src/main/java/org/wikipedia/dataclient/ServiceFactory.kt index b97dee61a73..166825e20bf 100644 --- a/app/src/main/java/org/wikipedia/dataclient/ServiceFactory.kt +++ b/app/src/main/java/org/wikipedia/dataclient/ServiceFactory.kt @@ -5,10 +5,10 @@ import okhttp3.Interceptor import okhttp3.MediaType.Companion.toMediaType import okhttp3.Response import okhttp3.logging.HttpLoggingInterceptor +import org.wikimedia.metricsplatform.config.DestinationEventService +import org.wikimedia.metricsplatform.config.StreamConfig import org.wikipedia.WikipediaApp -import org.wikipedia.analytics.eventplatform.DestinationEventService import org.wikipedia.analytics.eventplatform.EventService -import org.wikipedia.analytics.eventplatform.StreamConfig import org.wikipedia.dataclient.okhttp.OkHttpConnectionFactory import org.wikipedia.json.JsonUtil import org.wikipedia.settings.Prefs diff --git a/app/src/main/java/org/wikipedia/dataclient/mwapi/MwStreamConfigsResponse.kt b/app/src/main/java/org/wikipedia/dataclient/mwapi/MwStreamConfigsResponse.kt deleted file mode 100644 index 7b558aa142e..00000000000 --- a/app/src/main/java/org/wikipedia/dataclient/mwapi/MwStreamConfigsResponse.kt +++ /dev/null @@ -1,12 +0,0 @@ -package org.wikipedia.dataclient.mwapi - -import kotlinx.serialization.Serializable -import org.wikipedia.analytics.eventplatform.StreamConfig - -@Serializable -class MwStreamConfigsResponse : MwResponse() { - - private val streams: Map? = null - val streamConfigs: Map - get() = streams ?: emptyMap() -} diff --git a/app/src/main/java/org/wikipedia/settings/Prefs.kt b/app/src/main/java/org/wikipedia/settings/Prefs.kt index d8d17369bf1..f8f357ef15a 100644 --- a/app/src/main/java/org/wikipedia/settings/Prefs.kt +++ b/app/src/main/java/org/wikipedia/settings/Prefs.kt @@ -4,12 +4,12 @@ import android.location.Location import okhttp3.Cookie import okhttp3.HttpUrl.Companion.toHttpUrlOrNull import okhttp3.logging.HttpLoggingInterceptor +import org.wikimedia.metricsplatform.config.StreamConfig import org.wikipedia.BuildConfig import org.wikipedia.R import org.wikipedia.WikipediaApp import org.wikipedia.analytics.SessionData import org.wikipedia.analytics.eventplatform.AppSessionEvent -import org.wikipedia.analytics.eventplatform.StreamConfig import org.wikipedia.dataclient.WikiSite import org.wikipedia.donate.DonationResult import org.wikipedia.games.onthisday.OnThisDayGameNotificationState