Migrate Clearcut client-side usage metrics (#3600)

* Migrate from Clearcut to Firelog
* Add a safeguard
* Fix test
* Update copyright header

Author: chanseokoh

kokoro/ubuntu/release.sh

@@ -17,11 +17,13 @@ gcloud components install app-engine-java --quiet
 
 echo "OAUTH_CLIENT_ID: ${OAUTH_CLIENT_ID}"
 echo "OAUTH_CLIENT_SECRET: ${OAUTH_CLIENT_SECRET}"
+echo "FIRELOG_API_KEY: ${FIRELOG_API_KEY}"
 echo "PRODUCT_VERSION_SUFFIX: "${PRODUCT_VERSION_SUFFIX}
 
 # Exit if undefined (zero-length).
 test -n "${OAUTH_CLIENT_ID}"
 test -n "${OAUTH_CLIENT_SECRET}"
+test -n "${FIRELOG_API_KEY}"
 
 cd git/google-cloud-eclipse
 
@@ -34,5 +36,6 @@ TMPDIR= xvfb-run \
   mvn -V -B \
       -Doauth.client.id="${OAUTH_CLIENT_ID}" \
       -Doauth.client.secret="${OAUTH_CLIENT_SECRET}" \
+      -Dfirelog.api.key="${FIRELOG_API_KEY}" \
       ${PRODUCT_VERSION_SUFFIX:+-Dproduct.version.qualifier.suffix="'${PRODUCT_VERSION_SUFFIX}'"} \
     clean verify

plugins/com.google.cloud.tools.eclipse.usagetracker.test/src/com/google/cloud/tools/eclipse/usagetracker/AnalyticsPingManagerPluginTest.java

@@ -60,25 +60,24 @@ public void testBuildJson() {
     PingEvent event = new PingEvent("SomeEvent", metadata, null);
     String json = pingManager.jsonEncode(event);
 
-    Type singletonRootType = new TypeToken<List<Map<String, ?>>>(){}.getType();
-    List<Map<String, ?>> singletonRoot = gson.fromJson(json, singletonRootType);
-    Map<String, ?> root = singletonRoot.get(0);
+    Type mapType = new TypeToken<Map<String, ?>>(){}.getType();
+    Map<String, ?> root = gson.fromJson(json, mapType);
 
-    Map<String, ?> clientInfo = ((List<Map<String, ?>>) root.get("client_info")).get(0);
+    Map<String, ?> clientInfo = (Map<String, ?>) root.get("client_info");
     Assert.assertEquals("DESKTOP", clientInfo.get("client_type"));  
-    Assert.assertEquals("CONCORD", root.get("log_source_name"));  
+    Assert.assertEquals("CONCORD", root.get("log_source"));
 
     long requestTimeMs = ((Double) root.get("request_time_ms")).longValue();
     Assert.assertTrue(requestTimeMs >= 1000000);
 
     Map<String, String> desktopClientInfo =
-        ((List<Map<String, String>>) clientInfo.get("desktop_client_info")).get(0);
+        (Map<String, String>) clientInfo.get("desktop_client_info");
     Assert.assertTrue(desktopClientInfo.get("os").length() > 1);
 
-    List<Object> logEvents = ((List<List<Object>>) root.get("log_event")).get(0);
+    List<Map<String, Object>> logEvents = (List<Map<String, Object>>) root.get("log_event");
     Assert.assertEquals(1, logEvents.size());
 
-    Map<String, Object> logEvent = (Map<String, Object>) logEvents.get(0);
+    Map<String, Object> logEvent = logEvents.get(0);
     long eventTimeMs = ((Double) logEvent.get("event_time_ms")).longValue();
     Assert.assertTrue(eventTimeMs >= 1000000);
 
@@ -88,7 +87,7 @@ public void testBuildJson() {
     Type sourceExtensionJsonType = new TypeToken<Map<String, ?>>(){}.getType();
     Map<String, ?> source = gson.fromJson(sourceExtensionJson, sourceExtensionJsonType);
     Assert.assertEquals("CLOUD_TOOLS_FOR_ECLIPSE", source.get("console_type"));
-    Assert.assertEquals("clientId", source.get("client_machine_id"));
+    Assert.assertEquals("clientId", source.get("client_install_id"));
     Assert.assertEquals("SomeEvent", source.get("event_name"));
 
     List<Map<String, String>> eventMetadata =

plugins/com.google.cloud.tools.eclipse.usagetracker/.classpath

@@ -3,5 +3,6 @@
 	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8"/>
 	<classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins"/>
 	<classpathentry kind="src" path="src"/>
+	<classpathentry kind="src" path="generated-sources"/>
 	<classpathentry kind="output" path="target/classes"/>
 </classpath>

plugins/com.google.cloud.tools.eclipse.usagetracker/build.properties

@@ -3,4 +3,5 @@ bin.includes = META-INF/,\
                plugin.xml,\
                plugin.properties,\
                .
-source.. = src/
+source.. = src/,\
+           generated-sources/

plugins/com.google.cloud.tools.eclipse.usagetracker/java-templates/com/google/cloud/tools/eclipse/usagetracker/Constants.java

@@ -0,0 +1,25 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.tools.eclipse.usagetracker;
+
+/**
+ * Placeholder constants initialized at compile-time.
+ */
+public class Constants {
+
+  public static final String FIRELOG_API_KEY = "@firelog.api.key@";
+}

plugins/com.google.cloud.tools.eclipse.usagetracker/pom.xml

@@ -10,4 +10,26 @@
   <artifactId>com.google.cloud.tools.eclipse.usagetracker</artifactId>
   <version>0.1.0-SNAPSHOT</version>
   <packaging>eclipse-plugin</packaging>
+
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>templating-maven-plugin</artifactId>
+        <version>1.0.0</version>
+        <configuration>
+          <sourceDirectory>${basedir}/java-templates</sourceDirectory>
+          <outputDirectory>${basedir}/generated-sources</outputDirectory>
+        </configuration>
+        <executions>
+          <execution>
+            <id>filter-src</id>
+            <goals>
+              <goal>filter-sources</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
 </project>

plugins/com.google.cloud.tools.eclipse.usagetracker/src/com/google/cloud/tools/eclipse/usagetracker/AnalyticsPingManager.java

@@ -53,12 +53,12 @@ public class AnalyticsPingManager {
 
   private static final Logger logger = Logger.getLogger(AnalyticsPingManager.class.getName());
 
-  private static final String CLEAR_CUT_COLLECTION_URL = "https://play.google.com/log";
+  private static final String FIRELOG_COLLECTION_URL =
+      "https://firebaselogging-pa.googleapis.com/v1/firelog/legacy/log";
 
   private static AnalyticsPingManager instance;
 
-  // analytics services
-  private final String clearCutUrl;
+  private final String collectionUrl;
 
   // Preference store (should be configuration scoped) from which we get UUID, opt-in status, etc.
   private final IEclipsePreferences preferences;
@@ -81,21 +81,20 @@ protected IStatus run(IProgressMonitor monitor) {
   private int sequencePosition = 0;
 
   @VisibleForTesting
-  AnalyticsPingManager(String clearCutUrl, IEclipsePreferences preferences,
+  AnalyticsPingManager(String collectionUrl, IEclipsePreferences preferences,
       ConcurrentLinkedQueue<PingEvent> concurrentLinkedQueue) {
-    this.clearCutUrl = clearCutUrl;
+    this.collectionUrl = collectionUrl;
     this.preferences = Preconditions.checkNotNull(preferences);
     pingEventQueue = concurrentLinkedQueue;
   }
 
   public static synchronized AnalyticsPingManager getInstance() {
     if (instance == null) {
-      String clearCutUrl = null;
-      if (!Platform.inDevelopmentMode()) {
-        // Enable only in production environment.
-        clearCutUrl = CLEAR_CUT_COLLECTION_URL;
+      String collectionUrl = null;
+      if (!Platform.inDevelopmentMode() && !Constants.FIRELOG_API_KEY.startsWith("@")) {
+        collectionUrl = FIRELOG_COLLECTION_URL + "?key=" + Constants.FIRELOG_API_KEY;
       }
-      instance = new AnalyticsPingManager(clearCutUrl, AnalyticsPreferences.getPreferenceNode(),
+      instance = new AnalyticsPingManager(collectionUrl, AnalyticsPreferences.getPreferenceNode(),
           new ConcurrentLinkedQueue<PingEvent>());
     }
     return instance;
@@ -180,7 +179,7 @@ private void sendPingOnShell(Shell parentShell, String eventName, Map<String, St
     Preconditions.checkArgument(!Strings.isNullOrEmpty(eventName), "eventName null or empty");
     Preconditions.checkNotNull(metadata);
 
-    if (clearCutUrl != null) {
+    if (collectionUrl != null) {
       // Note: always enqueue if a user has not seen the opt-in dialog yet; enqueuing itself
       // doesn't mean that the event ping will be posted.
       if (userHasOptedIn() || !userHasRegisteredOptInStatus()) {
@@ -199,7 +198,7 @@ private void sendPing(PingEvent pingEvent) {
     if (userHasOptedIn()) {
       try {
         String json = jsonEncode(pingEvent);
-        int resultCode = HttpUtil.sendPost(clearCutUrl, json, "application/json");
+        int resultCode = HttpUtil.sendPost(collectionUrl, json, "application/json");
         if (resultCode >= 300) {
           logger.log(Level.FINE, "Failed to POST to Concord with HTTP result " + resultCode);
         }
@@ -293,11 +292,11 @@ String jsonEncode(PingEvent event) {
 
     Map<String, Object> clientInfo = new HashMap<>();
     clientInfo.put("client_type", "DESKTOP");
-    clientInfo.put("desktop_client_info", Collections.singletonList(desktopClientInfo));
+    clientInfo.put("desktop_client_info", desktopClientInfo);
 
     // logs/proto/cloud/concord/concord_event.proto
     Map<String, Object> sourceExtension = new HashMap<>();
-    sourceExtension.put("client_machine_id", getAnonymizedClientId());
+    sourceExtension.put("client_install_id", getAnonymizedClientId());
     sourceExtension.put("console_type", CloudToolsInfo.CONSOLE_TYPE);
     sourceExtension.put("event_name", event.eventName);
 
@@ -321,11 +320,11 @@ String jsonEncode(PingEvent event) {
     logEvent.put("source_extension_json", sourceExtensionJsonString);
 
     Map<String, Object> root = new HashMap<>();
-    root.put("log_source_name", "CONCORD");
+    root.put("log_source", "CONCORD");
     root.put("request_time_ms", System.currentTimeMillis());
-    root.put("client_info", Collections.singletonList(clientInfo));
-    root.put("log_event", Collections.singletonList(Collections.singletonList(logEvent)));
+    root.put("client_info", clientInfo);
+    root.put("log_event", Collections.singletonList(logEvent));
 
-    return gson.toJson(Collections.singletonList(root));
+    return gson.toJson(root);
   }
 }