Add hypertrace config (#82)

* Add hypertrace config

Signed-off-by: Pavol Loffay <[email protected]>

* Init submodules

Signed-off-by: Pavol Loffay <[email protected]>

Author: pavolloffay

.circleci/config.yml

@@ -8,6 +8,7 @@ commands:
       - run:
           name: Generate cache key
           command: find . -type f -name "*.gradle*" -o -name "gradle-wrapper*" -exec shasum {} + | sort > /tmp/checksum.txt && cat /tmp/checksum.txt
+      - run: make init-submodules
       - restore_cache:
           keys:
             - v1-dependencies-{{ checksum "/tmp/checksum.txt" }}

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "javaagent-core/src/main/proto"]
+	path = javaagent-core/src/main/proto
+	url = [email protected]:hypertrace/agent-config.git

Makefile

@@ -20,3 +20,7 @@ format:
 .PHONY: clean
 clean:
 	./gradlew clean
+
+.PHONY: init-submodules
+init-submodules:
+	git submodule update --init --recursive

README.md

@@ -26,25 +26,26 @@ make build
 
 The final artifact is in `javaagent/build/libs/hypertrace-agent-<version>-all.jar`
 
-## Run
+## Run & Configure
 
 ```bash
-OTEL_EXPORTER=zipkin java -javaagent:javaagent/build/libs/hypertrace-agent-0.0.1-all.jar -jar app.jar
+HT_EXPORTING_ADDRESS=http://localhost:9411/api/v2/spans java -javaagent:javaagent/build/libs/hypertrace-agent-<version>-all.jar -jar app.jar
 ```
 
+By default the agent uses Zipkin exporter.
+
 The configuration precedence order 
 1. OpenTelemetry Agent's trace config file `OTEL_TRACE_CONFIG`/`otel.trace.config`
-3. OpenTelemetry system properties and env variables
+2. OpenTelemetry system properties and env variables
+3. Hypertrace configuration with the following precedence order:
+   1. system properties 
+   2. environment variables, TODO add link to agent-config repo
+   3. [configuration file](./example-config.yaml), specified `HT_CONFIG_FILE=example-config.yaml`
 
 Hypertrace body/headers capture can be disabled by:
-* `HYPERTRACE_INTEGRATION_ALL_ENABLED` - disables capture for all instrumentations
+* `HT_DATA_CAPTURE_HTTP_BODY_REQUEST` - disables additional data capture for all instrumentations
 * `HYPERTRACE_INTEGRATION_<integration>_ENABLED` - disables capture for a specified instrumentation e.g. `servlet`, `servlet-3`
 
-### Disable request/response body capture
-
-Request and response body capture can be disabled by `-Dotel.integration.body.enabled=false` or
-`-Dotel.integration.<instrumentation>-body.enabled=false`.
-
 ## Test
 
 Tests use docker via Testcontainers.org.

example-config.yaml

@@ -0,0 +1,3 @@
+serviceName: service_name
+reporting:
+  address: http://localhost:9411/api/v2/spans

instrumentation/grpc-1.5/src/test/java/io/opentelemetry/instrumentation/hypertrace/grpc/v1_5/GrpcBodyTest.java

@@ -34,7 +34,8 @@
 import java.io.IOException;
 import java.util.List;
 import java.util.concurrent.TimeoutException;
-import org.hypertrace.agent.core.DynamicConfig;
+import org.hypertrace.agent.core.EnvironmentConfig;
+import org.hypertrace.agent.core.HypertraceConfig;
 import org.hypertrace.agent.core.HypertraceSemanticAttributes;
 import org.hypertrace.agent.testing.AbstractInstrumenterTest;
 import org.hypertrace.example.GreeterGrpc;
@@ -107,7 +108,8 @@ public static void close() {
 
   @AfterEach
   public void afterEach() {
-    System.clearProperty(DynamicConfig.ENABLED_ALL_PROPERTY_NAME);
+    System.clearProperty(EnvironmentConfig.CAPTURE_HTTP_BODY_PREFIX + "request");
+    HypertraceConfig.reset();
   }
 
   @Test
@@ -168,7 +170,7 @@ public void serverRequestBlocking() throws TimeoutException, InterruptedExceptio
   @Test
   public void disabledInstrumentation_dynamicConfig()
       throws TimeoutException, InterruptedException {
-    System.setProperty(DynamicConfig.ENABLED_ALL_PROPERTY_NAME, "false");
+    System.setProperty(EnvironmentConfig.CAPTURE_HTTP_BODY_PREFIX + "request", "false");
 
     GreeterBlockingStub blockingStub = GreeterGrpc.newBlockingStub(CHANNEL);
     Response response = blockingStub.sayHello(REQUEST);

javaagent-core/build.gradle.kts

@@ -1,8 +1,33 @@
+import com.google.protobuf.gradle.*
+
 plugins {
     `java-library`
+    idea
+    id("com.google.protobuf") version "0.8.13"
+}
+
+protobuf {
+    protoc {
+        // The artifact spec for the Protobuf Compiler
+        artifact = "com.google.protobuf:protoc:3.13.0"
+    }
+    generateProtoTasks {
+    }
+}
+
+idea {
+    module {
+        sourceDirs.add(file("${projectDir}/build/generated/source/proto/main/proto"))
+    }
 }
 
 dependencies {
     implementation("io.opentelemetry:opentelemetry-api:0.9.1")
-}
 
+    api("com.google.protobuf:protobuf-java:3.13.0")
+    api("com.google.protobuf:protobuf-java-util:3.13.0")
+    // convert yaml to json, since java protobuf impl supports only json
+    implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.11.3")
+
+    testImplementation("org.junit-pioneer:junit-pioneer:1.0.0")
+}

javaagent-core/src/main/java/org/hypertrace/agent/core/DynamicConfig.java

@@ -20,25 +20,18 @@ public class DynamicConfig {
 
   private DynamicConfig() {}
 
-  public static final String ENABLED_ALL_PROPERTY_NAME = "hypertrace.integration.all.enabled";
-
   public static boolean isEnabled(String[] instrumentationNames) {
-    String integrationEnabled = getProperty(ENABLED_ALL_PROPERTY_NAME);
-    if (integrationEnabled != null && "false".equals(integrationEnabled.toLowerCase())) {
+    if (!HypertraceConfig.get().getDataCapture().getHttpBody().getRequest().getValue()) {
       return false;
     }
 
     for (String name : instrumentationNames) {
-      integrationEnabled = getProperty("hypertrace.integration." + name + ".enabled");
+      String integrationEnabled =
+          EnvironmentConfig.getProperty("hypertrace.integration." + name + ".enabled");
       if (integrationEnabled != null && "false".equals(integrationEnabled.toLowerCase())) {
         return false;
       }
     }
     return true;
   }
-
-  public static String getProperty(String name) {
-    /** * TODO this is just a mock impl */
-    return System.getProperty(name, System.getenv(name.replaceAll("\\.", "_").toUpperCase()));
-  }
 }

javaagent-core/src/main/java/org/hypertrace/agent/core/EnvironmentConfig.java

@@ -0,0 +1,100 @@
+/*
+ * Copyright The Hypertrace Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.hypertrace.agent.core;
+
+import com.google.protobuf.BoolValue;
+import com.google.protobuf.StringValue;
+import org.hypertrace.agent.config.Config.AgentConfig;
+import org.hypertrace.agent.config.Config.DataCapture;
+import org.hypertrace.agent.config.Config.Message;
+import org.hypertrace.agent.config.Config.Reporting;
+
+public class EnvironmentConfig {
+
+  private EnvironmentConfig() {}
+
+  private static final String HT_PREFIX = "ht.";
+
+  static final String CONFIG_FILE_PROPERTY = HT_PREFIX + "config.file";
+  static final String SERVICE_NAME = HT_PREFIX + "service.name";
+
+  private static final String REPORTING_PREFIX = HT_PREFIX + "reporting.";
+  static final String REPORTING_ADDRESS = REPORTING_PREFIX + "address";
+  static final String REPORTING_SECURE = REPORTING_PREFIX + "secure";
+
+  private static final String CAPTURE_PREFIX = HT_PREFIX + "data.capture.";
+  public static final String CAPTURE_HTTP_HEADERS_PREFIX = CAPTURE_PREFIX + "http.headers.";
+  public static final String CAPTURE_HTTP_BODY_PREFIX = CAPTURE_PREFIX + "http.body.";
+  public static final String CAPTURE_RPC_METADATA_PREFIX = CAPTURE_PREFIX + "rpc.metadata.";
+  public static final String CAPTURE_RPC_BODY_PREFIX = CAPTURE_PREFIX + "rpc.body.";
+
+  public static AgentConfig.Builder applyPropertiesAndEnvVars(AgentConfig.Builder builder) {
+    String serviceName = getProperty(SERVICE_NAME);
+    if (serviceName != null) {
+      builder.setServiceName(serviceName);
+    }
+
+    Reporting.Builder reportingBuilder = applyReporting(builder.getReporting().toBuilder());
+    builder.setReporting(reportingBuilder);
+
+    DataCapture.Builder dataCaptureBuilder =
+        setDefaultsToDataCapture(builder.getDataCapture().toBuilder());
+    builder.setDataCapture(dataCaptureBuilder);
+    return builder;
+  }
+
+  private static Reporting.Builder applyReporting(Reporting.Builder builder) {
+    String reporterAddress = getProperty(REPORTING_ADDRESS);
+    if (reporterAddress != null) {
+      builder.setAddress(StringValue.newBuilder().setValue(reporterAddress).build());
+    }
+    String secure = getProperty(REPORTING_SECURE);
+    if (secure != null) {
+      builder.setSecure(BoolValue.newBuilder().setValue(Boolean.valueOf(secure)).build());
+    }
+
+    return builder;
+  }
+
+  private static DataCapture.Builder setDefaultsToDataCapture(DataCapture.Builder builder) {
+    builder.setHttpHeaders(
+        applyMessageDefaults(builder.getHttpHeaders().toBuilder(), CAPTURE_HTTP_HEADERS_PREFIX));
+    builder.setHttpBody(
+        applyMessageDefaults(builder.getHttpBody().toBuilder(), CAPTURE_HTTP_BODY_PREFIX));
+    builder.setRpcMetadata(
+        applyMessageDefaults(builder.getRpcMetadata().toBuilder(), CAPTURE_RPC_METADATA_PREFIX));
+    builder.setRpcBody(
+        applyMessageDefaults(builder.getRpcBody().toBuilder(), CAPTURE_RPC_BODY_PREFIX));
+    return builder;
+  }
+
+  private static Message.Builder applyMessageDefaults(Message.Builder builder, String prefix) {
+    String request = getProperty(prefix + "request");
+    if (request != null) {
+      builder.setRequest(BoolValue.newBuilder().setValue(Boolean.valueOf(request)).build());
+    }
+    String response = getProperty(prefix + "response");
+    if (request != null) {
+      builder.setResponse(BoolValue.newBuilder().setValue(Boolean.valueOf(response)).build());
+    }
+    return builder;
+  }
+
+  public static String getProperty(String name) {
+    return System.getProperty(name, System.getenv(name.replaceAll("\\.", "_").toUpperCase()));
+  }
+}

javaagent-core/src/main/java/org/hypertrace/agent/core/HypertraceConfig.java

@@ -0,0 +1,137 @@
+/*
+ * Copyright The Hypertrace Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.hypertrace.agent.core;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.protobuf.BoolValue;
+import com.google.protobuf.StringValue;
+import com.google.protobuf.util.JsonFormat;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import org.hypertrace.agent.config.Config.AgentConfig;
+import org.hypertrace.agent.config.Config.DataCapture;
+import org.hypertrace.agent.config.Config.Message;
+import org.hypertrace.agent.config.Config.Reporting;
+
+/** {@link HypertraceConfig} loads a yaml config from file. */
+public class HypertraceConfig {
+
+  private HypertraceConfig() {}
+
+  private static AgentConfig agentConfig;
+
+  static final String DEFAULT_REPORTING_ADDRESS = "http://localhost:9411/api/v2/spans";
+  static final String DEFAULT_SERVICE_NAME = "default_service_name";
+
+  public static AgentConfig get() {
+    if (agentConfig == null) {
+      synchronized (HypertraceConfig.class) {
+        if (agentConfig == null) {
+          try {
+            agentConfig = load();
+          } catch (IOException e) {
+            throw new RuntimeException("Could not load config", e);
+          }
+        }
+      }
+    }
+    return agentConfig;
+  }
+
+  /** Reset the config, use only in tests. */
+  public static void reset() {
+    agentConfig = null;
+  }
+
+  private static AgentConfig load() throws IOException {
+    String configFile = EnvironmentConfig.getProperty(EnvironmentConfig.CONFIG_FILE_PROPERTY);
+    if (configFile == null) {
+      return EnvironmentConfig.applyPropertiesAndEnvVars(applyDefaults(AgentConfig.newBuilder()))
+          .build();
+    }
+    return load(configFile);
+  }
+
+  @VisibleForTesting
+  static AgentConfig load(String filename) throws IOException {
+    File configFile = new File(filename);
+    if (!configFile.exists() || configFile.isDirectory() || !configFile.canRead()) {
+      throw new IllegalArgumentException(
+          String.format("Config file %s either does not exist or cannot be read", configFile));
+    }
+
+    InputStream fileInputStream = new FileInputStream(configFile);
+    String json = convertYamlToJson(fileInputStream);
+
+    AgentConfig.Builder configBuilder = AgentConfig.newBuilder();
+    JsonFormat.parser().ignoringUnknownFields().merge(json, configBuilder);
+
+    return EnvironmentConfig.applyPropertiesAndEnvVars(applyDefaults(configBuilder)).build();
+  }
+
+  private static AgentConfig.Builder applyDefaults(AgentConfig.Builder configBuilder) {
+    if (configBuilder.getServiceName().isEmpty()) {
+      configBuilder.setServiceName(DEFAULT_SERVICE_NAME);
+    }
+
+    Reporting.Builder reportingBuilder =
+        applyReportingDefaults(configBuilder.getReporting().toBuilder());
+    configBuilder.setReporting(reportingBuilder);
+
+    DataCapture.Builder dataCaptureBuilder =
+        setDefaultsToDataCapture(configBuilder.getDataCapture().toBuilder());
+    configBuilder.setDataCapture(dataCaptureBuilder);
+    return configBuilder;
+  }
+
+  private static Reporting.Builder applyReportingDefaults(Reporting.Builder builder) {
+    if (!builder.hasAddress()) {
+      builder.setAddress(StringValue.newBuilder().setValue(DEFAULT_REPORTING_ADDRESS).build());
+    }
+    return builder;
+  }
+
+  private static DataCapture.Builder setDefaultsToDataCapture(DataCapture.Builder builder) {
+    builder.setHttpHeaders(applyMessageDefaults(builder.getHttpHeaders().toBuilder()));
+    builder.setHttpBody(applyMessageDefaults(builder.getHttpBody().toBuilder()));
+    builder.setRpcMetadata(applyMessageDefaults(builder.getRpcMetadata().toBuilder()));
+    builder.setRpcBody(applyMessageDefaults(builder.getRpcBody().toBuilder()));
+    return builder;
+  }
+
+  private static Message.Builder applyMessageDefaults(Message.Builder builder) {
+    if (!builder.hasRequest()) {
+      builder.setRequest(BoolValue.newBuilder().setValue(true).build());
+    }
+    if (!builder.hasResponse()) {
+      builder.setResponse(BoolValue.newBuilder().setValue(true).build());
+    }
+    return builder;
+  }
+
+  private static String convertYamlToJson(InputStream yaml) throws IOException {
+    ObjectMapper yamlReader = new ObjectMapper(new YAMLFactory());
+    Object obj = yamlReader.readValue(yaml, Object.class);
+
+    ObjectMapper jsonWriter = new ObjectMapper();
+    return jsonWriter.writeValueAsString(obj);
+  }
+}