Port improvements to Hocon parsing and HTTP route metrics to master from master-2.0 (#142)

Author: vjkoskela

src/main/java/com/arpnetworking/configuration/jackson/HoconFileSource.java

@@ -23,6 +23,7 @@
 import com.typesafe.config.Config;
 import com.typesafe.config.ConfigFactory;
 import com.typesafe.config.ConfigRenderOptions;
+import com.typesafe.config.ConfigResolveOptions;
 import net.sf.oval.constraint.NotNull;
 
 import java.io.File;
@@ -69,8 +70,12 @@ private HoconFileSource(final Builder builder) {
         if (_file.canRead()) {
             try {
                 final Config config = ConfigFactory.parseFile(_file);
-                final String hoconAsJson = config.resolve().root().render(ConfigRenderOptions.concise());
-                jsonNode = _objectMapper.readTree(hoconAsJson);
+                final String configHoconAsJson = config.withFallback(ConfigFactory.systemProperties())
+                        .resolve(ConfigResolveOptions.defaults())
+                        .root()
+                        .render(ConfigRenderOptions.concise());
+
+                jsonNode = _objectMapper.readTree(configHoconAsJson);
             } catch (final IOException e) {
                 throw new RuntimeException(e);
             }

src/main/java/com/arpnetworking/http/Routes.java

@@ -62,9 +62,12 @@
 
 import java.util.Objects;
 import java.util.Optional;
+import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionStage;
 import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
 
 /**
  * Http server routes.
@@ -117,7 +120,21 @@ public CompletionStage<HttpResponse> apply(final HttpRequest request) {
                 createMetricName(request, BODY_SIZE_METRIC),
                 request.entity().getContentLengthOption().orElse(0L),
                 Optional.of(Units.BYTE));
-        // TODO(vkoskela): Add a request UUID and include in MDC. [MAI-462]
+        final UUID requestId = UUID.randomUUID();
+        if (LOGGER.isTraceEnabled()) {
+            LOGGER.trace()
+                    .setEvent("http.in.start")
+                    .addContext("requestId", requestId)
+                    .addData("method", request.method().toString())
+                    .addData("url", request.getUri().toString())
+                    .addData(
+                            "headers",
+                            StreamSupport.stream(request.getHeaders().spliterator(), false)
+                                    .map(h -> h.name() + "=" + h.value())
+                                    .collect(Collectors.toList()))
+                    .log();
+        }
+        // TODO(ville): Add a request UUID and include in MDC.
         LOGGER.trace()
                 .setEvent("http.in.start")
                 .addData("method", request.method())
@@ -132,23 +149,41 @@ public CompletionStage<HttpResponse> apply(final HttpRequest request) {
                             requestTimer.elapsed(TimeUnit.NANOSECONDS),
                             Optional.of(Units.NANOSECOND));
 
-                    final int responseStatusClass = response.status().intValue() / 100;
+                    final int responseStatus;
+                    if (response != null) {
+                        responseStatus = response.status().intValue();
+                    } else {
+                        // TODO(ville): Figure out how to intercept post-exception mapping.
+                        responseStatus = 599;
+                    }
+                    final int responseStatusClass = responseStatus / 100;
                     for (final int i : STATUS_CLASSES) {
                         _metrics.recordCounter(
                                 createMetricName(request, String.format("%s/%dxx", STATUS_METRIC, i)),
                                 responseStatusClass == i ? 1 : 0);
                     }
 
-                    final LogBuilder log = LOGGER.trace()
-                            .setEvent("http.in")
-                            .addData("method", request.method())
-                            .addData("url", request.getUri())
-                            .addData("status", response.status().intValue())
-                            .addData("headers", request.getHeaders());
-                    if (failure != null) {
-                        log.setEvent("http.in.error").addData("exception", failure);
+                    final LogBuilder log;
+                    if (failure != null || responseStatusClass == 5) {
+                        log = LOGGER.info().setEvent("http.in.failure");
+                        if (failure != null) {
+                            log.setThrowable(failure);
+                        }
+                        if (!LOGGER.isTraceEnabled() && LOGGER.isInfoEnabled()) {
+                            log.addData("method", request.method().toString())
+                                    .addData("url", request.getUri().toString())
+                                    .addData(
+                                            "headers",
+                                            StreamSupport.stream(request.getHeaders().spliterator(), false)
+                                                    .map(h -> h.name() + "=" + h.value())
+                                                    .collect(Collectors.toList()));
+                        }
+                    } else {
+                        log = LOGGER.trace().setEvent("http.in.complete");
                     }
-                    log.log();
+                    log.addContext("requestId", requestId)
+                            .addData("status", responseStatus)
+                            .log();
                 });
     }
 

src/main/java/com/arpnetworking/metrics/mad/Aggregator.java

@@ -161,7 +161,7 @@ private List<PeriodWorker> createPeriodWorkers(final Key key) {
             periodWorkerList.add(periodWorker);
             _periodWorkerExecutor.execute(periodWorker);
         }
-        LOGGER.info()
+        LOGGER.debug()
                 .setMessage("Created period workers")
                 .addData("key", key)
                 .addData("periodWorkersSize", periodWorkerList.size())

src/test/java/com/arpnetworking/configuration/jackson/HoconFileSourceTest.java

@@ -17,6 +17,7 @@
 
 import com.google.common.base.Charsets;
 import com.google.common.collect.ImmutableSet;
+import com.typesafe.config.impl.ConfigImpl;
 import org.junit.Assert;
 import org.junit.BeforeClass;
 import org.junit.Test;
@@ -36,6 +37,9 @@ public class HoconFileSourceTest {
     @BeforeClass
     public static void setUpClass() throws IOException {
         Files.createDirectories(new File("./target/tmp/filter/HoconFileSourceTest").toPath());
+        System.setProperty("HoconFileSourceTest_testSystemPropertyDirect_foo", "bar");
+        System.setProperty("HoconFileSourceTest_testSystemPropertyReference_foo", "bar");
+        ConfigImpl.reloadSystemPropertiesConfig();
     }
 
     @Test
@@ -70,6 +74,28 @@ public void testValidHocon() throws IOException {
         Assert.assertFalse(source.getValue("does-not-exist").isPresent());
     }
 
+    @Test
+    public void testSystemPropertyDirect() throws IOException {
+        final File file = new File("./target/tmp/filter/HoconFileSourceTest/testSystemPropertyDirect.hocon");
+        Files.write(file.toPath(), "".getBytes(Charsets.UTF_8));
+        final HoconFileSource source = new HoconFileSource.Builder()
+                .setFile(file)
+                .build();
+        Assert.assertTrue(source.getJsonNode().isPresent());
+        Assert.assertEquals("bar", source.getValue("HoconFileSourceTest_testSystemPropertyDirect_foo").get().textValue());
+    }
+
+    @Test
+    public void testSystemPropertyReference() throws IOException {
+        final File file = new File("./target/tmp/filter/HoconFileSourceTest/testSystemPropertyReference.hocon");
+        Files.write(file.toPath(), "foo:${HoconFileSourceTest_testSystemPropertyReference_foo}".getBytes(Charsets.UTF_8));
+        final HoconFileSource source = new HoconFileSource.Builder()
+                .setFile(file)
+                .build();
+        Assert.assertTrue(source.getJsonNode().isPresent());
+        Assert.assertEquals("bar", source.getValue("foo").get().textValue());
+    }
+
     @Test(expected = RuntimeException.class)
     public void testInvalidHocon() throws IOException {
         final File file = new File("./target/tmp/filter/HoconFileSourceTest/testInvalidHocon.json");