Add config source reading from Test Optimization remote environment (#9701)

Author: nikita-tkachenko-datadog

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilityServices.java

@@ -1,20 +1,16 @@
 package datadog.trace.civisibility;
 
-import com.squareup.moshi.JsonAdapter;
-import com.squareup.moshi.Moshi;
-import com.squareup.moshi.Types;
 import datadog.communication.BackendApi;
 import datadog.communication.BackendApiFactory;
 import datadog.communication.ddagent.SharedCommunicationObjects;
-import datadog.communication.http.HttpRetryPolicy;
-import datadog.communication.http.OkHttpUtils;
 import datadog.communication.util.IOUtils;
 import datadog.trace.api.Config;
 import datadog.trace.api.civisibility.telemetry.CiVisibilityCountMetric;
 import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector;
 import datadog.trace.api.civisibility.telemetry.tag.Command;
 import datadog.trace.api.git.GitInfoProvider;
 import datadog.trace.api.intake.Intake;
+import datadog.trace.bootstrap.config.provider.civisibility.CiEnvironmentVariables;
 import datadog.trace.civisibility.ci.CIProviderInfoFactory;
 import datadog.trace.civisibility.ci.env.CiEnvironment;
 import datadog.trace.civisibility.ci.env.CiEnvironmentImpl;
@@ -33,21 +29,22 @@
 import datadog.trace.civisibility.source.ByteCodeLinesResolver;
 import datadog.trace.civisibility.source.CompilerAidedLinesResolver;
 import datadog.trace.civisibility.source.LinesResolver;
-import datadog.trace.civisibility.source.index.*;
+import datadog.trace.civisibility.source.index.CachingRepoIndexBuilderFactory;
+import datadog.trace.civisibility.source.index.ConventionBasedResourceResolver;
+import datadog.trace.civisibility.source.index.PackageResolver;
+import datadog.trace.civisibility.source.index.PackageResolverImpl;
+import datadog.trace.civisibility.source.index.RepoIndexFetcher;
+import datadog.trace.civisibility.source.index.RepoIndexProvider;
+import datadog.trace.civisibility.source.index.ResourceResolver;
 import datadog.trace.civisibility.utils.ShellCommandExecutor;
 import java.io.File;
-import java.lang.reflect.Type;
 import java.net.InetSocketAddress;
 import java.nio.file.FileSystem;
 import java.nio.file.FileSystems;
 import java.nio.file.Path;
-import java.util.Collections;
 import java.util.Map;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
-import okhttp3.HttpUrl;
-import okhttp3.OkHttpClient;
-import okhttp3.Request;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -61,8 +58,6 @@ public class CiVisibilityServices {
 
   private static final String GIT_FOLDER_NAME = ".git";
 
-  static final String DD_ENV_VARS_PROVIDER_KEY_HEADER = "DD-Env-Vars-Provider-Key";
-
   final ProcessHierarchy processHierarchy;
   final Config config;
   final CiVisibilityMetricCollector metricCollector;
@@ -90,7 +85,7 @@ public class CiVisibilityServices {
     this.jvmInfoFactory = new CachingJvmInfoFactory(config, new JvmInfoFactoryImpl());
     this.gitClientFactory = buildGitClientFactory(config, metricCollector);
 
-    this.environment = buildCiEnvironment(config, sco);
+    this.environment = buildCiEnvironment();
     this.ciProviderInfoFactory = new CIProviderInfoFactory(config, environment);
     this.linesResolver =
         new BestEffortLinesResolver(new CompilerAidedLinesResolver(), new ByteCodeLinesResolver());
@@ -145,50 +140,13 @@ private static GitClient.Factory buildGitClientFactory(
   }
 
   @Nonnull
-  private static CiEnvironment buildCiEnvironment(Config config, SharedCommunicationObjects sco) {
-    CiEnvironment localEnvironment = CiEnvironmentImpl.local();
-    String remoteEnvVarsProviderUrl = config.getCiVisibilityRemoteEnvVarsProviderUrl();
-    if (remoteEnvVarsProviderUrl != null) {
-      String remoteEnvVarsProviderKey = config.getCiVisibilityRemoteEnvVarsProviderKey();
-      CiEnvironment remoteEnvironment =
-          new CiEnvironmentImpl(
-              getRemoteEnvironment(
-                  remoteEnvVarsProviderUrl, remoteEnvVarsProviderKey, sco.agentHttpClient));
-      return new CompositeCiEnvironment(remoteEnvironment, localEnvironment);
+  private static CiEnvironment buildCiEnvironment() {
+    Map<String, String> remoteEnvironment = CiEnvironmentVariables.getAll();
+    if (remoteEnvironment != null) {
+      return new CompositeCiEnvironment(
+          new CiEnvironmentImpl(remoteEnvironment), CiEnvironmentImpl.local());
     } else {
-      return localEnvironment;
-    }
-  }
-
-  static Map<String, String> getRemoteEnvironment(String url, String key, OkHttpClient httpClient) {
-    HttpRetryPolicy.Factory retryPolicyFactory = new HttpRetryPolicy.Factory(5, 100, 2.0, true);
-
-    HttpUrl httpUrl = HttpUrl.get(url);
-    Request request =
-        new Request.Builder()
-            .url(httpUrl)
-            .header(DD_ENV_VARS_PROVIDER_KEY_HEADER, key)
-            .get()
-            .build();
-    try (okhttp3.Response response =
-        OkHttpUtils.sendWithRetries(httpClient, retryPolicyFactory, request)) {
-
-      if (response.isSuccessful()) {
-        Moshi moshi = new Moshi.Builder().build();
-        Type type = Types.newParameterizedType(Map.class, String.class, String.class);
-        JsonAdapter<Map<String, String>> adapter = moshi.adapter(type);
-        return adapter.fromJson(response.body().source());
-      } else {
-        logger.warn(
-            "Could not get remote CI environment (HTTP code {}) {}",
-            response.code(),
-            response.body() != null ? ": " + response.body().string() : "");
-        return Collections.emptyMap();
-      }
-
-    } catch (Exception e) {
-      logger.warn("Could not get remote CI environment", e);
-      return Collections.emptyMap();
+      return CiEnvironmentImpl.local();
     }
   }
 

dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/CiVisibilityServicesTest.groovy

@@ -76,10 +76,6 @@ public final class CiVisibilityConfig {
       "civisibility.rum.flush.wait.millis";
   public static final String CIVISIBILITY_AUTO_INSTRUMENTATION_PROVIDER =
       "civisibility.auto.instrumentation.provider";
-  public static final String CIVISIBILITY_REMOTE_ENV_VARS_PROVIDER_URL =
-      "civisibility.remote.env.vars.provider.url";
-  public static final String CIVISIBILITY_REMOTE_ENV_VARS_PROVIDER_KEY =
-      "civisibility.remote.env.vars.provider.key";
   public static final String CIVISIBILITY_TEST_ORDER = "civisibility.test.order";
   public static final String CIVISIBILITY_SCALATEST_FORK_MONITOR_ENABLED =
       "civisibility.scalatest.fork.monitor.enabled";

dd-trace-api/src/main/java/datadog/trace/api/config/CiVisibilityConfig.java

@@ -265,8 +265,6 @@
 import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_JVM_INFO_CACHE_SIZE;
 import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_KNOWN_TESTS_REQUEST_ENABLED;
 import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_MODULE_NAME;
-import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_REMOTE_ENV_VARS_PROVIDER_KEY;
-import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_REMOTE_ENV_VARS_PROVIDER_URL;
 import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_REPO_INDEX_DUPLICATE_KEY_CHECK_ENABLED;
 import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_REPO_INDEX_FOLLOW_SYMLINKS;
 import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_RESOURCE_FOLDER_NAMES;
@@ -1056,8 +1054,6 @@ public static String getHostName() {
   private final boolean ciVisibilityTelemetryEnabled;
   private final long ciVisibilityRumFlushWaitMillis;
   private final boolean ciVisibilityAutoInjected;
-  private final String ciVisibilityRemoteEnvVarsProviderUrl;
-  private final String ciVisibilityRemoteEnvVarsProviderKey;
   private final String ciVisibilityTestOrder;
   private final boolean ciVisibilityTestManagementEnabled;
   private final Integer ciVisibilityTestManagementAttemptToFixRetries;
@@ -2360,10 +2356,6 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment())
         configProvider.getLong(CIVISIBILITY_RUM_FLUSH_WAIT_MILLIS, 500);
     ciVisibilityAutoInjected =
         Strings.isNotBlank(configProvider.getString(CIVISIBILITY_AUTO_INSTRUMENTATION_PROVIDER));
-    ciVisibilityRemoteEnvVarsProviderUrl =
-        configProvider.getString(CIVISIBILITY_REMOTE_ENV_VARS_PROVIDER_URL);
-    ciVisibilityRemoteEnvVarsProviderKey =
-        configProvider.getString(CIVISIBILITY_REMOTE_ENV_VARS_PROVIDER_KEY);
     ciVisibilityTestOrder = configProvider.getString(CIVISIBILITY_TEST_ORDER);
     ciVisibilityTestManagementEnabled = configProvider.getBoolean(TEST_MANAGEMENT_ENABLED, true);
     ciVisibilityTestManagementAttemptToFixRetries =
@@ -4043,14 +4035,6 @@ public boolean isCiVisibilityAutoInjected() {
     return ciVisibilityAutoInjected;
   }
 
-  public String getCiVisibilityRemoteEnvVarsProviderUrl() {
-    return ciVisibilityRemoteEnvVarsProviderUrl;
-  }
-
-  public String getCiVisibilityRemoteEnvVarsProviderKey() {
-    return ciVisibilityRemoteEnvVarsProviderKey;
-  }
-
   public String getCiVisibilityTestOrder() {
     return ciVisibilityTestOrder;
   }

internal-api/src/main/java/datadog/trace/api/Config.java

@@ -16,6 +16,7 @@ val excludedClassesCoverage by extra(
     // tested in internal-api
     "datadog.trace.api.telemetry.OtelEnvMetricCollectorProvider",
     "datadog.trace.api.telemetry.ConfigInversionMetricCollectorProvider",
+    "datadog.trace.bootstrap.config.provider.civisibility.CiEnvironmentVariables",
     "datadog.trace.bootstrap.config.provider.CapturedEnvironmentConfigSource",
     "datadog.trace.bootstrap.config.provider.ConfigConverter.ValueOfLookup",
     // tested in internal-api
@@ -25,6 +26,7 @@ val excludedClassesCoverage by extra(
     "datadog.trace.bootstrap.config.provider.ConfigProvider.Singleton",
     "datadog.trace.bootstrap.config.provider.ConfigProvider.Source",
     "datadog.trace.bootstrap.config.provider.EnvironmentConfigSource",
+    "datadog.trace.bootstrap.config.provider.MapConfigSource",
     // tested in internal-api
     "datadog.trace.bootstrap.config.provider.OtelEnvironmentConfigSource",
     "datadog.trace.bootstrap.config.provider.stableconfig.Selector",
@@ -58,4 +60,5 @@ dependencies {
 
   testImplementation(project(":utils:test-utils"))
   testImplementation("org.snakeyaml:snakeyaml-engine:2.9")
+  testImplementation("com.squareup.okhttp3:mockwebserver:${libs.versions.okhttp.legacy.get()}")
 }

utils/config-utils/build.gradle.kts

@@ -7,6 +7,8 @@
 import datadog.environment.SystemProperties;
 import datadog.trace.api.ConfigCollector;
 import datadog.trace.api.ConfigOrigin;
+import datadog.trace.bootstrap.config.provider.civisibility.CiEnvironmentVariables;
+import datadog.trace.util.ConfigStrings;
 import de.thetaphi.forbiddenapis.SuppressForbidden;
 import java.io.File;
 import java.io.FileNotFoundException;
@@ -20,6 +22,7 @@
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Properties;
 import java.util.Set;
 import org.slf4j.Logger;
@@ -495,24 +498,32 @@ public static ConfigProvider createDefault() {
     Properties configProperties =
         loadConfigurationFile(
             new ConfigProvider(new SystemPropertiesConfigSource(), new EnvironmentConfigSource()));
-    if (configProperties.isEmpty()) {
-      return new ConfigProvider(
-          new SystemPropertiesConfigSource(),
-          StableConfigSource.FLEET,
-          new EnvironmentConfigSource(),
-          new OtelEnvironmentConfigSource(),
-          StableConfigSource.LOCAL,
-          new CapturedEnvironmentConfigSource());
-    } else {
-      return new ConfigProvider(
-          new SystemPropertiesConfigSource(),
-          StableConfigSource.FLEET,
-          new EnvironmentConfigSource(),
-          new PropertiesConfigSource(configProperties, true),
-          new OtelEnvironmentConfigSource(configProperties),
-          StableConfigSource.LOCAL,
-          new CapturedEnvironmentConfigSource());
-    }
+    ConfigProvider.Source propertiesSource =
+        !configProperties.isEmpty() ? new PropertiesConfigSource(configProperties, true) : null;
+
+    Map<String, String> ciEnvironmentVariables = CiEnvironmentVariables.getAll();
+    ConfigProvider.Source ciEnvironmentSource =
+        ciEnvironmentVariables != null
+            ? new MapConfigSource(
+                ciEnvironmentVariables,
+                ConfigStrings::propertyNameToEnvironmentVariableName,
+                ConfigOrigin.ENV)
+            : null;
+
+    return new ConfigProvider(
+        filterNonNull(
+            new SystemPropertiesConfigSource(),
+            StableConfigSource.FLEET,
+            ciEnvironmentSource,
+            new EnvironmentConfigSource(),
+            propertiesSource,
+            new OtelEnvironmentConfigSource(),
+            StableConfigSource.LOCAL,
+            new CapturedEnvironmentConfigSource()));
+  }
+
+  private static ConfigProvider.Source[] filterNonNull(ConfigProvider.Source... values) {
+    return Arrays.stream(values).filter(Objects::nonNull).toArray(Source[]::new);
   }
 
   public static ConfigProvider withoutCollector() {

utils/config-utils/src/main/java/datadog/trace/bootstrap/config/provider/ConfigProvider.java

@@ -0,0 +1,31 @@
+package datadog.trace.bootstrap.config.provider;
+
+import datadog.trace.api.ConfigOrigin;
+import java.util.Map;
+import java.util.function.Function;
+
+final class MapConfigSource extends ConfigProvider.Source {
+
+  private final Map<String, String> properties;
+  private final Function<String, String> keyTransformer;
+  private final ConfigOrigin origin;
+
+  MapConfigSource(
+      Map<String, String> properties,
+      Function<String, String> keyTransformer,
+      ConfigOrigin origin) {
+    this.properties = properties;
+    this.keyTransformer = keyTransformer;
+    this.origin = origin;
+  }
+
+  @Override
+  protected String get(String key) {
+    return properties.get(keyTransformer.apply(key));
+  }
+
+  @Override
+  public ConfigOrigin origin() {
+    return origin;
+  }
+}