Skip to content

Commit 7ea7b54

Browse files
vjkoskelaBrandonArp
vjkoskela
authored and
BrandonArp
committed
Upgrade the metrics client libraries in order to integration test pre-aggregation. (#167)
1 parent 1cf314d commit 7ea7b54

File tree

16 files changed

+138
-79
lines changed

16 files changed

+138
-79
lines changed

‎.jdkw‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
JDKW_RELEASE=latest
22
JDKW_DIST=zulu
3-
JDKW_BUILD=8.31.0.1
4-
JDKW_VERSION=8.0.181
3+
JDKW_BUILD=8.38.0.13
4+
JDKW_VERSION=8.0.212
55
JDKW_VERBOSE=true

‎README.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -329,6 +329,14 @@ To debug the server during integration tests on port 9000:
329329

330330
metrics-aggregator-daemon> ./jdk-wrapper.sh ./mvnw -Ddebug=true verify
331331

332+
To launch the service and its dependencies in Docker:
333+
334+
metrics-aggregator-daemon> ./jdk-wrapper.sh ./mvnw initialize docker:start
335+
336+
To launch the service with remote debugging and its dependencies in Docker:
337+
338+
metrics-aggregator-daemon> ./jdk-wrapper.sh ./mvnw -Ddebug=true initialize docker:start
339+
332340
To execute performance tests:
333341

334342
metrics-aggregator-daemon> ./jdk-wrapper.sh ./mvnw -PperformanceTest test

‎config/pipelines/pipeline.conf‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,14 @@ name=example_pipeline
3737

3838
# Sources
3939
# ~~~~
40+
kafka_source_bootstrap_host="localhost"
41+
kafka_source_bootstrap_host=${?KAFKA_SOURCE_BOOTSTRAP_HOST}
42+
kafka_source_bootstrap_port="9092"
43+
kafka_source_bootstrap_port=${?KAFKA_SOURCE_BOOTSTRAP_PORT}
44+
kafka_source_consumer_id="default"
45+
kafka_source_consumer_id=${?HOSTNAME}
46+
kafka_source_consumer_id=${?KAFKA_SOURCE_BOOTSTRAP_CONSUMER_ID}
47+
4048
sources=[
4149
{
4250
type="com.arpnetworking.metrics.common.sources.ClientHttpSourceV1"
@@ -137,9 +145,9 @@ sources=[
137145
topics=["topic"]
138146
configs={
139147
# Set any properties defined at: https://kafka.apache.org/documentation/#consumerconfigs
140-
"bootstrap.servers"="localhost:9092"
141-
"group.id"="group0"
142-
"client.id"="consumer0"
148+
"bootstrap.servers"=${kafka_source_bootstrap_host}":"${kafka_source_bootstrap_port}
149+
"group.id"="metrics-aggregator-daemon"
150+
"client.id"=${kafka_source_consumer_id}
143151
"key.deserializer"="org.apache.kafka.common.serialization.ByteArrayDeserializer"
144152
"value.deserializer"="org.apache.kafka.common.serialization.ByteArrayDeserializer"
145153
"auto.offset.reset"="earliest"

‎jdk-wrapper.sh‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -99,15 +99,13 @@ safe_command "rm \"${l_fifo}\""
9999
command=
100100
cmd_configuration=
101101
for arg in "$@"; do
102-
if [ -z ${in_command} ]; then
103-
jdkw_arg=$(echo "${arg}" | grep '^JDKW_.*')
104-
jdkw_base_dir_arg=$(echo "${arg}" | grep '^JDKW_BASE_DIR.*')
105-
if [ -n "${jdkw_base_dir_arg}" ]; then
106-
eval ${arg}
107-
fi
108-
if [ -n "${jdkw_arg}" ]; then
109-
cmd_configuration="${cmd_configuration}${arg} "
110-
fi
102+
jdkw_arg=$(echo "${arg}" | grep '^JDKW_.*')
103+
jdkw_base_dir_arg=$(echo "${arg}" | grep '^JDKW_BASE_DIR.*')
104+
if [ -n "${jdkw_base_dir_arg}" ]; then
105+
eval ${arg}
106+
fi
107+
if [ -n "${jdkw_arg}" ]; then
108+
cmd_configuration="${cmd_configuration}${arg} "
111109
fi
112110
case "${arg}" in
113111
*\'*)

‎maven/maven-wrapper.properties‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#Maven download properties
2-
#Sun Jan 13 13:44:30 PST 2019
2+
#Tue Sep 17 09:50:02 PDT 2019
33
checksumAlgorithm=SHA1
44
verifyDownload=true
55
distributionUrl=https\://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.5.4/apache-maven-3.5.4-bin.zip

‎pom.xml‎

Lines changed: 74 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
<parent>
1919
<groupId>com.arpnetworking.build</groupId>
2020
<artifactId>arpnetworking-parent-pom</artifactId>
21-
<version>2.0.0</version>
21+
<version>2.0.1</version>
2222
<relativePath />
2323
</parent>
2424

@@ -92,10 +92,10 @@
9292
<log4j.over.slf4j.version>1.7.25</log4j.over.slf4j.version>
9393
<metrics.aggregator.protocol.prometheus.version>1.0.0</metrics.aggregator.protocol.prometheus.version>
9494
<metrics.aggregator.protocol.version>1.0.6</metrics.aggregator.protocol.version>
95-
<metrics.client.version>0.10.1</metrics.client.version>
96-
<metrics.client.http.version>0.9.4</metrics.client.http.version>
97-
<metrics.client.incubator.version>0.6.3</metrics.client.incubator.version>
98-
<metrics.jvm.extra.version>0.10.1</metrics.jvm.extra.version>
95+
<metrics.client.version>0.11.0</metrics.client.version>
96+
<metrics.client.http.version>0.11.0</metrics.client.http.version>
97+
<metrics.client.incubator.version>0.11.0</metrics.client.incubator.version>
98+
<metrics.jvm.extra.version>0.11.0</metrics.jvm.extra.version>
9999
<oval.version>1.90</oval.version>
100100
<protobuf.version>3.8.0</protobuf.version>
101101
<scala.version>2.11</scala.version>
@@ -111,7 +111,7 @@
111111
<hamcrest.version>2.1</hamcrest.version>
112112
<junit.benchmarks.version>0.7.2</junit.benchmarks.version>
113113
<junit.version>4.12</junit.version>
114-
<metrics.generator.version>1.2.3</metrics.generator.version>
114+
<metrics.generator.version>1.3.0</metrics.generator.version>
115115
<mockito.version>2.23.4</mockito.version>
116116
<performance.test.version>1.2.1</performance.test.version>
117117
<statsd.client.timgroup>3.1.0</statsd.client.timgroup>
@@ -120,9 +120,11 @@
120120
<!-- Additional Plugins -->
121121
<aspectjtools.maven.plugin>1.9.2</aspectjtools.maven.plugin>
122122
<aspectj.maven.plugin.version>1.8</aspectj.maven.plugin.version>
123-
<docker.maven.plugin.version>0.26.0</docker.maven.plugin.version>
123+
<build.helper.plugin.version>3.0.0</build.helper.plugin.version>
124+
<docker.maven.plugin.version>0.27.2</docker.maven.plugin.version>
124125
<javassist.maven.plugin.version>0.2.2</javassist.maven.plugin.version>
125126
<maven.assembly.plugin.version>2.6</maven.assembly.plugin.version>
127+
<maven.properties.plugin.version>1.0.0</maven.properties.plugin.version>
126128
<rpm.maven.plugin.version>2.1.5</rpm.maven.plugin.version>
127129

128130
<!-- Code Coverage -->
@@ -321,6 +323,23 @@
321323
</dependency>
322324
</dependencies>
323325
</plugin>
326+
<plugin>
327+
<groupId>org.codehaus.mojo</groupId>
328+
<artifactId>build-helper-maven-plugin</artifactId>
329+
<version>${build.helper.plugin.version}</version>
330+
<executions>
331+
<execution>
332+
<id>get-local-ip</id>
333+
<phase>initialize</phase>
334+
<goals>
335+
<goal>local-ip</goal>
336+
</goals>
337+
<configuration>
338+
<localIpProperty>kafka.advertised.address</localIpProperty>
339+
</configuration>
340+
</execution>
341+
</executions>
342+
</plugin>
324343
<plugin>
325344
<groupId>io.fabric8</groupId>
326345
<artifactId>docker-maven-plugin</artifactId>
@@ -366,53 +385,70 @@
366385
</execution>
367386
</executions>
368387
<configuration>
388+
<autoCreateCustomNetworks>true</autoCreateCustomNetworks>
369389
<showLogs>true</showLogs>
370390
<images>
371391
<image>
372392
<name>arpnetworking/mad/kafka:${project.version}</name>
393+
<alias>kafka</alias>
373394
<build>
374395
<dockerFile>${project.basedir}/src/main/docker/kafka/Dockerfile</dockerFile>
375396
<tags>
376397
<tag>${buildNumber}</tag>
377398
</tags>
378399
</build>
379400
<run>
401+
<network>
402+
<mode>custom</mode>
403+
<name>mad-net</name>
404+
<alias>kafka</alias>
405+
</network>
406+
<ports>
407+
<port>+kafka.ip:2181:2181</port>
408+
<port>+kafka.ip:9092:9092</port>
409+
</ports>
380410
<env>
381-
<AUTO_CREATE_TOPICS>true</AUTO_CREATE_TOPICS>
382-
<ADVERTISED_HOST>localhost</ADVERTISED_HOST>
411+
<ADVERTISED_HOST>${kafka.advertised.address}</ADVERTISED_HOST>
383412
<ADVERTISED_PORT>9092</ADVERTISED_PORT>
413+
<AUTO_CREATE_TOPICS>true</AUTO_CREATE_TOPICS>
384414
</env>
385-
<ports>
386-
<port>2181:2181</port>
387-
<port>9092:9092</port>
388-
</ports>
389415
</run>
390416
</image>
391417
<image>
392418
<name>arpnetworking/mad:${project.version}</name>
419+
<alias>mad</alias>
393420
<build>
394421
<dockerFile>${project.build.directory}/docker-assembly/Dockerfile</dockerFile>
395422
<tags>
396423
<tag>${buildNumber}</tag>
397424
</tags>
398425
</build>
399426
<run>
400-
<env>
401-
<ADDITIONAL_JAVA_OPTS>${debugJavaOptions}</ADDITIONAL_JAVA_OPTS>
402-
</env>
427+
<dependsOn>
428+
<container>kafka</container>
429+
</dependsOn>
430+
<network>
431+
<mode>custom</mode>
432+
<name>mad-net</name>
433+
<alias>mad</alias>
434+
</network>
403435
<ports>
404-
<port>${debugJavaPort}:${debugJavaPort}</port>
405-
<port>7090:7090</port>
406-
<port>8125:8125/udp</port>
436+
<port>+mad.ip:${debugJavaPort}:${debugJavaPort}</port>
437+
<port>+mad.ip:7090:7090</port>
438+
<port>+mad.ip:8125:8125/udp</port>
407439
</ports>
440+
<env>
441+
<JAVA_OPTS>${debugJavaOptions}</JAVA_OPTS>
442+
<KAFKA_SOURCE_BOOTSTRAP_HOST>kafka</KAFKA_SOURCE_BOOTSTRAP_HOST>
443+
</env>
408444
<volumes>
409445
<bind>
410446
<volume>${project.basedir}/logs/docker:/opt/mad/logs</volume>
411447
</bind>
412448
</volumes>
413449
<wait>
414450
<http>
415-
<url>http://${docker.host.address}:7090/ping</url>
451+
<url>http://${mad.ip}:7090/ping</url>
416452
<method>GET</method>
417453
<status>200</status>
418454
</http>
@@ -426,6 +462,24 @@
426462
<skip>${skipDockerBuild}</skip>
427463
</configuration>
428464
</plugin>
465+
<plugin>
466+
<groupId>org.codehaus.mojo</groupId>
467+
<artifactId>properties-maven-plugin</artifactId>
468+
<version>${maven.properties.plugin.version}</version>
469+
<executions>
470+
<execution>
471+
<phase>pre-integration-test</phase>
472+
<goals>
473+
<goal>write-project-properties</goal>
474+
</goals>
475+
<configuration>
476+
<outputFile>
477+
${project.build.directory}/pre-integration-test.properties
478+
</outputFile>
479+
</configuration>
480+
</execution>
481+
</executions>
482+
</plugin>
429483
</plugins>
430484
</build>
431485

‎src/main/java/com/arpnetworking/http/Routes.java‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,6 @@
3939
import akka.stream.javadsl.Source;
4040
import akka.util.ByteString;
4141
import akka.util.Timeout;
42-
import com.arpnetworking.metrics.Units;
4342
import com.arpnetworking.metrics.common.sources.ClientHttpSourceV1;
4443
import com.arpnetworking.metrics.common.sources.ClientHttpSourceV2;
4544
import com.arpnetworking.metrics.common.sources.ClientHttpSourceV3;
@@ -120,8 +119,7 @@ public CompletionStage<HttpResponse> apply(final HttpRequest request) {
120119
final Stopwatch requestTimer = Stopwatch.createStarted();
121120
_metrics.recordGauge(
122121
createMetricName(request, BODY_SIZE_METRIC),
123-
request.entity().getContentLengthOption().orElse(0L),
124-
Optional.of(Units.BYTE));
122+
request.entity().getContentLengthOption().orElse(0L));
125123
final UUID requestId = UUID.randomUUID();
126124
if (LOGGER.isTraceEnabled()) {
127125
LOGGER.trace()
@@ -149,7 +147,7 @@ public CompletionStage<HttpResponse> apply(final HttpRequest request) {
149147
_metrics.recordTimer(
150148
createMetricName(request, REQUEST_METRIC),
151149
requestTimer.elapsed(TimeUnit.NANOSECONDS),
152-
Optional.of(Units.NANOSECOND));
150+
Optional.of(TimeUnit.NANOSECONDS));
153151

154152
final int responseStatus;
155153
if (response != null) {

‎src/main/java/com/arpnetworking/metrics/common/sources/KafkaSource.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@
1616
package com.arpnetworking.metrics.common.sources;
1717

1818
import com.arpnetworking.logback.annotations.LogValue;
19-
import com.arpnetworking.metrics.Units;
2019
import com.arpnetworking.metrics.common.kafka.ConsumerListener;
2120
import com.arpnetworking.metrics.common.kafka.RunnableConsumer;
2221
import com.arpnetworking.metrics.common.kafka.RunnableConsumerImpl;
@@ -191,7 +190,7 @@ public void run() {
191190
records = _parser.parse(value);
192191
parsingTimer.stop();
193192
_periodicMetrics.recordTimer(_parsingTimeMetricName,
194-
parsingTimer.elapsed(TimeUnit.NANOSECONDS), Optional.of(Units.NANOSECOND));
193+
parsingTimer.elapsed(TimeUnit.NANOSECONDS), Optional.of(TimeUnit.NANOSECONDS));
195194
} catch (final ParsingException e) {
196195
_periodicMetrics.recordCounter(_parsingExceptionCountMetricName, 1);
197196
_logger.error()

‎src/main/java/com/arpnetworking/metrics/mad/Main.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -255,7 +255,7 @@ private Injector launchGuice(final ActorSystem actorSystem) {
255255
// Instantiate the metrics factory
256256
final URI sinkUrl = URI.create(
257257
"http://" + _configuration.getMetricsClientHost() + ":"
258-
+ _configuration.getMetricsClientPort() + "/metrics/v2/application");
258+
+ _configuration.getMetricsClientPort() + "/metrics/v3/application");
259259
final MetricsFactory metricsFactory = new TsdMetricsFactory.Builder()
260260
.setClusterName(_configuration.getMonitoringCluster())
261261
.setServiceName(_configuration.getMonitoringService())

‎src/main/java/com/arpnetworking/metrics/mad/model/statistics/HistogramStatistic.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -289,7 +289,7 @@ long truncateToLong(final double val) {
289289

290290
long pack(final double val) {
291291
final long truncated = truncateToLong(val);
292-
final long shifted = truncated >> (52 - _precision);
292+
final long shifted = truncated >> (MANTISSA_BITS - _precision);
293293
return shifted & _packMask;
294294
}
295295

0 commit comments

Comments
 (0)