Skip to content

Commit def8a8d

Browse files
nikita-tkachenko-datadognikita-tkachenko-datadog
committed
Add config source reading from Test Optimization remote environment
1 parent 0d75905 commit def8a8d

File tree

8 files changed

+219
-128
lines changed

8 files changed

+219
-128
lines changed

‎components/environment/build.gradle.kts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,3 +29,11 @@ val excludedClassesCoverage by extra {
2929
val excludedClassesBranchCoverage by extra {
3030
listOf("datadog.environment.CommandLine") // tested using forked process
3131
}
32+
33+
dependencies {
34+
implementation(libs.slf4j)
35+
implementation(libs.okio)
36+
implementation(libs.moshi)
37+
38+
testImplementation("com.squareup.okhttp3:mockwebserver:${libs.versions.okhttp.legacy.get()}")
39+
}

‎components/environment/src/main/java/datadog/environment/CiEnvironmentVariables.java‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
package datadog.environment;
2+
3+
import com.squareup.moshi.JsonAdapter;
4+
import com.squareup.moshi.Moshi;
5+
import com.squareup.moshi.Types;
6+
import java.io.BufferedReader;
7+
import java.io.IOException;
8+
import java.io.InputStreamReader;
9+
import java.lang.reflect.Type;
10+
import java.net.HttpURLConnection;
11+
import java.net.URL;
12+
import java.nio.charset.StandardCharsets;
13+
import java.util.Collections;
14+
import java.util.Map;
15+
import java.util.concurrent.Callable;
16+
import java.util.stream.Collectors;
17+
import javax.annotation.Nullable;
18+
import okio.BufferedSource;
19+
import okio.Okio;
20+
import org.slf4j.Logger;
21+
import org.slf4j.LoggerFactory;
22+
23+
public class CiEnvironmentVariables {
24+
25+
private static final Logger logger = LoggerFactory.getLogger(CiEnvironmentVariables.class);
26+
27+
static final String CIVISIBILITY_REMOTE_ENV_VARS_PROVIDER_URL =
28+
"dd.civisibility.remote.env.vars.provider.url";
29+
static final String CIVISIBILITY_REMOTE_ENV_VARS_PROVIDER_KEY =
30+
"dd.civisibility.remote.env.vars.provider.key";
31+
32+
static final String DD_ENV_VARS_PROVIDER_KEY_HEADER = "DD-Env-Vars-Provider-Key";
33+
34+
private static final int CONNECT_TIMEOUT_MILLIS = 5000;
35+
private static final int READ_TIMEOUT_MILLIS = 10000;
36+
37+
private static final Map<String, String> REMOTE_ENVIRONMENT;
38+
39+
static {
40+
String url = getConfigValue(CIVISIBILITY_REMOTE_ENV_VARS_PROVIDER_URL);
41+
String key = getConfigValue(CIVISIBILITY_REMOTE_ENV_VARS_PROVIDER_KEY);
42+
if (url != null && key != null) {
43+
REMOTE_ENVIRONMENT = doWithBackoffRetries(() -> getRemoteEnvironment(url, key), null);
44+
} else {
45+
REMOTE_ENVIRONMENT = null;
46+
}
47+
}
48+
49+
private static String getConfigValue(String propertyName) {
50+
String propertyValue = System.getProperty(propertyName);
51+
if (propertyValue != null) {
52+
return propertyValue;
53+
}
54+
return EnvironmentVariables.get(toEnvVar(propertyName));
55+
}
56+
57+
private static String toEnvVar(String string) {
58+
return string.replace('.', '_').replace('-', '_').toUpperCase();
59+
}
60+
61+
private static <T> T doWithBackoffRetries(Callable<T> action, T fallbackValue) {
62+
long delayMillis = 500;
63+
for (int i = 0; i < 5; i++) {
64+
if (Thread.currentThread().isInterrupted()) {
65+
logger.warn("Interrupted while trying to read remote environment");
66+
return fallbackValue;
67+
}
68+
try {
69+
return action.call();
70+
71+
} catch (Exception e) {
72+
logger.warn("Error while trying to read remote environment", e);
73+
sleep(delayMillis);
74+
delayMillis *= 2;
75+
}
76+
}
77+
return fallbackValue;
78+
}
79+
80+
private static void sleep(long delayMillis) {
81+
try {
82+
Thread.sleep(delayMillis);
83+
} catch (InterruptedException e) {
84+
Thread.currentThread().interrupt();
85+
}
86+
}
87+
88+
private static Map<String, String> getRemoteEnvironment(String url, String key) throws Exception {
89+
HttpURLConnection conn = null;
90+
try {
91+
URL target = new URL(url);
92+
conn = (HttpURLConnection) target.openConnection();
93+
conn.setRequestMethod("GET");
94+
conn.setRequestProperty(DD_ENV_VARS_PROVIDER_KEY_HEADER, key);
95+
conn.setConnectTimeout(CONNECT_TIMEOUT_MILLIS);
96+
conn.setReadTimeout(READ_TIMEOUT_MILLIS);
97+
98+
int code = conn.getResponseCode();
99+
if (code >= 200 && code < 300) {
100+
Moshi moshi = new Moshi.Builder().build();
101+
Type type = Types.newParameterizedType(Map.class, String.class, String.class);
102+
JsonAdapter<Map<String, String>> adapter = moshi.adapter(type);
103+
try (BufferedSource source = Okio.buffer(Okio.source(conn.getInputStream()))) {
104+
return adapter.fromJson(source);
105+
}
106+
107+
} else {
108+
try (BufferedReader r =
109+
new BufferedReader(
110+
new InputStreamReader(conn.getErrorStream(), StandardCharsets.UTF_8))) {
111+
String body = r.lines().collect(Collectors.joining("\n"));
112+
throw new IOException(
113+
String.format("Remote environment request failed (HTTP %d) %s", code, body));
114+
}
115+
}
116+
117+
} finally {
118+
if (conn != null) {
119+
conn.disconnect();
120+
}
121+
}
122+
}
123+
124+
@Nullable
125+
public static String get(String name) {
126+
if (REMOTE_ENVIRONMENT == null) {
127+
return null;
128+
}
129+
return REMOTE_ENVIRONMENT.get(name);
130+
}
131+
132+
@Nullable
133+
public static Map<String, String> getAll() {
134+
return REMOTE_ENVIRONMENT != null ? Collections.unmodifiableMap(REMOTE_ENVIRONMENT) : null;
135+
}
136+
}

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilityServices.java‎

Lines changed: 15 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,10 @@
11
package datadog.trace.civisibility;
22

3-
import com.squareup.moshi.JsonAdapter;
4-
import com.squareup.moshi.Moshi;
5-
import com.squareup.moshi.Types;
63
import datadog.communication.BackendApi;
74
import datadog.communication.BackendApiFactory;
85
import datadog.communication.ddagent.SharedCommunicationObjects;
9-
import datadog.communication.http.HttpRetryPolicy;
10-
import datadog.communication.http.OkHttpUtils;
116
import datadog.communication.util.IOUtils;
7+
import datadog.environment.CiEnvironmentVariables;
128
import datadog.trace.api.Config;
139
import datadog.trace.api.civisibility.telemetry.CiVisibilityCountMetric;
1410
import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector;
@@ -33,21 +29,22 @@
3329
import datadog.trace.civisibility.source.ByteCodeLinesResolver;
3430
import datadog.trace.civisibility.source.CompilerAidedLinesResolver;
3531
import datadog.trace.civisibility.source.LinesResolver;
36-
import datadog.trace.civisibility.source.index.*;
32+
import datadog.trace.civisibility.source.index.CachingRepoIndexBuilderFactory;
33+
import datadog.trace.civisibility.source.index.ConventionBasedResourceResolver;
34+
import datadog.trace.civisibility.source.index.PackageResolver;
35+
import datadog.trace.civisibility.source.index.PackageResolverImpl;
36+
import datadog.trace.civisibility.source.index.RepoIndexFetcher;
37+
import datadog.trace.civisibility.source.index.RepoIndexProvider;
38+
import datadog.trace.civisibility.source.index.ResourceResolver;
3739
import datadog.trace.civisibility.utils.ShellCommandExecutor;
3840
import java.io.File;
39-
import java.lang.reflect.Type;
4041
import java.net.InetSocketAddress;
4142
import java.nio.file.FileSystem;
4243
import java.nio.file.FileSystems;
4344
import java.nio.file.Path;
44-
import java.util.Collections;
4545
import java.util.Map;
4646
import javax.annotation.Nonnull;
4747
import javax.annotation.Nullable;
48-
import okhttp3.HttpUrl;
49-
import okhttp3.OkHttpClient;
50-
import okhttp3.Request;
5148
import org.slf4j.Logger;
5249
import org.slf4j.LoggerFactory;
5350

@@ -61,8 +58,6 @@ public class CiVisibilityServices {
6158

6259
private static final String GIT_FOLDER_NAME = ".git";
6360

64-
static final String DD_ENV_VARS_PROVIDER_KEY_HEADER = "DD-Env-Vars-Provider-Key";
65-
6661
final ProcessHierarchy processHierarchy;
6762
final Config config;
6863
final CiVisibilityMetricCollector metricCollector;
@@ -90,7 +85,7 @@ public class CiVisibilityServices {
9085
this.jvmInfoFactory = new CachingJvmInfoFactory(config, new JvmInfoFactoryImpl());
9186
this.gitClientFactory = buildGitClientFactory(config, metricCollector);
9287

93-
this.environment = buildCiEnvironment(config, sco);
88+
this.environment = buildCiEnvironment();
9489
this.ciProviderInfoFactory = new CIProviderInfoFactory(config, environment);
9590
this.linesResolver =
9691
new BestEffortLinesResolver(new CompilerAidedLinesResolver(), new ByteCodeLinesResolver());
@@ -145,50 +140,13 @@ private static GitClient.Factory buildGitClientFactory(
145140
}
146141

147142
@Nonnull
148-
private static CiEnvironment buildCiEnvironment(Config config, SharedCommunicationObjects sco) {
149-
CiEnvironment localEnvironment = CiEnvironmentImpl.local();
150-
String remoteEnvVarsProviderUrl = config.getCiVisibilityRemoteEnvVarsProviderUrl();
151-
if (remoteEnvVarsProviderUrl != null) {
152-
String remoteEnvVarsProviderKey = config.getCiVisibilityRemoteEnvVarsProviderKey();
153-
CiEnvironment remoteEnvironment =
154-
new CiEnvironmentImpl(
155-
getRemoteEnvironment(
156-
remoteEnvVarsProviderUrl, remoteEnvVarsProviderKey, sco.agentHttpClient));
157-
return new CompositeCiEnvironment(remoteEnvironment, localEnvironment);
143+
private static CiEnvironment buildCiEnvironment() {
144+
Map<String, String> remoteEnvironment = CiEnvironmentVariables.getAll();
145+
if (remoteEnvironment != null) {
146+
return new CompositeCiEnvironment(
147+
new CiEnvironmentImpl(remoteEnvironment), CiEnvironmentImpl.local());
158148
} else {
159-
return localEnvironment;
160-
}
161-
}
162-
163-
static Map<String, String> getRemoteEnvironment(String url, String key, OkHttpClient httpClient) {
164-
HttpRetryPolicy.Factory retryPolicyFactory = new HttpRetryPolicy.Factory(5, 100, 2.0, true);
165-
166-
HttpUrl httpUrl = HttpUrl.get(url);
167-
Request request =
168-
new Request.Builder()
169-
.url(httpUrl)
170-
.header(DD_ENV_VARS_PROVIDER_KEY_HEADER, key)
171-
.get()
172-
.build();
173-
try (okhttp3.Response response =
174-
OkHttpUtils.sendWithRetries(httpClient, retryPolicyFactory, request)) {
175-
176-
if (response.isSuccessful()) {
177-
Moshi moshi = new Moshi.Builder().build();
178-
Type type = Types.newParameterizedType(Map.class, String.class, String.class);
179-
JsonAdapter<Map<String, String>> adapter = moshi.adapter(type);
180-
return adapter.fromJson(response.body().source());
181-
} else {
182-
logger.warn(
183-
"Could not get remote CI environment (HTTP code {}) {}",
184-
response.code(),
185-
response.body() != null ? ": " + response.body().string() : "");
186-
return Collections.emptyMap();
187-
}
188-
189-
} catch (Exception e) {
190-
logger.warn("Could not get remote CI environment", e);
191-
return Collections.emptyMap();
149+
return CiEnvironmentImpl.local();
192150
}
193151
}
194152

‎dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/CiVisibilityServicesTest.groovy‎

Lines changed: 0 additions & 33 deletions
This file was deleted.

‎dd-trace-api/src/main/java/datadog/trace/api/config/CiVisibilityConfig.java‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -76,10 +76,6 @@ public final class CiVisibilityConfig {
7676
"civisibility.rum.flush.wait.millis";
7777
public static final String CIVISIBILITY_AUTO_INSTRUMENTATION_PROVIDER =
7878
"civisibility.auto.instrumentation.provider";
79-
public static final String CIVISIBILITY_REMOTE_ENV_VARS_PROVIDER_URL =
80-
"civisibility.remote.env.vars.provider.url";
81-
public static final String CIVISIBILITY_REMOTE_ENV_VARS_PROVIDER_KEY =
82-
"civisibility.remote.env.vars.provider.key";
8379
public static final String CIVISIBILITY_TEST_ORDER = "civisibility.test.order";
8480
public static final String CIVISIBILITY_SCALATEST_FORK_MONITOR_ENABLED =
8581
"civisibility.scalatest.fork.monitor.enabled";

‎internal-api/src/main/java/datadog/trace/api/Config.java‎

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -264,8 +264,6 @@
264264
import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_JVM_INFO_CACHE_SIZE;
265265
import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_KNOWN_TESTS_REQUEST_ENABLED;
266266
import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_MODULE_NAME;
267-
import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_REMOTE_ENV_VARS_PROVIDER_KEY;
268-
import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_REMOTE_ENV_VARS_PROVIDER_URL;
269267
import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_REPO_INDEX_DUPLICATE_KEY_CHECK_ENABLED;
270268
import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_REPO_INDEX_FOLLOW_SYMLINKS;
271269
import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_RESOURCE_FOLDER_NAMES;
@@ -1054,8 +1052,6 @@ public static String getHostName() {
10541052
private final boolean ciVisibilityTelemetryEnabled;
10551053
private final long ciVisibilityRumFlushWaitMillis;
10561054
private final boolean ciVisibilityAutoInjected;
1057-
private final String ciVisibilityRemoteEnvVarsProviderUrl;
1058-
private final String ciVisibilityRemoteEnvVarsProviderKey;
10591055
private final String ciVisibilityTestOrder;
10601056
private final boolean ciVisibilityTestManagementEnabled;
10611057
private final Integer ciVisibilityTestManagementAttemptToFixRetries;
@@ -2363,10 +2359,6 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment())
23632359
configProvider.getLong(CIVISIBILITY_RUM_FLUSH_WAIT_MILLIS, 500);
23642360
ciVisibilityAutoInjected =
23652361
Strings.isNotBlank(configProvider.getString(CIVISIBILITY_AUTO_INSTRUMENTATION_PROVIDER));
2366-
ciVisibilityRemoteEnvVarsProviderUrl =
2367-
configProvider.getString(CIVISIBILITY_REMOTE_ENV_VARS_PROVIDER_URL);
2368-
ciVisibilityRemoteEnvVarsProviderKey =
2369-
configProvider.getString(CIVISIBILITY_REMOTE_ENV_VARS_PROVIDER_KEY);
23702362
ciVisibilityTestOrder = configProvider.getString(CIVISIBILITY_TEST_ORDER);
23712363
ciVisibilityTestManagementEnabled = configProvider.getBoolean(TEST_MANAGEMENT_ENABLED, true);
23722364
ciVisibilityTestManagementAttemptToFixRetries =
@@ -4039,14 +4031,6 @@ public boolean isCiVisibilityAutoInjected() {
40394031
return ciVisibilityAutoInjected;
40404032
}
40414033

4042-
public String getCiVisibilityRemoteEnvVarsProviderUrl() {
4043-
return ciVisibilityRemoteEnvVarsProviderUrl;
4044-
}
4045-
4046-
public String getCiVisibilityRemoteEnvVarsProviderKey() {
4047-
return ciVisibilityRemoteEnvVarsProviderKey;
4048-
}
4049-
40504034
public String getCiVisibilityTestOrder() {
40514035
return ciVisibilityTestOrder;
40524036
}

0 commit comments

Comments
 (0)