Merge pull request #37 from bliblidotcom/feature/newrelic-module

add new relic reactor instrumentation module

Author: khannedy

blibli-backend-framework-newrelic/README.md

@@ -0,0 +1,37 @@
+# Reactor New Relic Instrumentation
+
+A plugin to enable new relic instrumentation in Spring Boot 2 Webflux App.
+
+List of supported modules:
+- command-plugin: anything that implement `Command` interface.
+- spring-reactive-mongo: ReactiveMongoRepository all inherited method and custom query.
+
+Planned future modules:
+- Reactive relational DB (via R2DBC)
+- Reactive elastic search
+- Reactive redis
+
+## How to use
+
+1. Include the library in your POM.
+
+    ```
+    <dependency>
+        <groupId>com.blibli.oss</groupId>
+        <artifactId>blibli-backend-framework-newrelic</artifactId>
+    </dependency>
+    ```
+
+And viola! See your app instrumented in New Relic.
+
+![New Relic instrumentation](docs/newrelic-sample.png)
+
+## How it works
+
+1. Inject new relic Token and Transaction for every web request in reactor Context. See `NewRelicTokenInjectorFilter` filter class.
+2. When code is entering a known module (targeted by our AOP pointcut), then append additional reactor operator to time those method execution using New Relic `Transaction.startSegment()`.
+
+## Known issues
+
+- Command timing report will include ReactiveMongoRepository timing.
+- There are still "Remainder" time on New Relic report. It may caused by the time the thread waiting until executed by Spring thread pool because of traffic congestion.

blibli-backend-framework-newrelic/docs/newrelic-sample.png

@@ -0,0 +1,66 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <parent>
+    <artifactId>blibli-backend-framework</artifactId>
+    <groupId>com.blibli.oss</groupId>
+    <version>0.0.6-SNAPSHOT</version>
+  </parent>
+  <modelVersion>4.0.0</modelVersion>
+
+  <artifactId>blibli-backend-framework-newrelic</artifactId>
+
+  <dependencies>
+    <dependency>
+      <groupId>com.blibli.oss</groupId>
+      <artifactId>blibli-backend-framework-command</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.springframework.boot</groupId>
+      <artifactId>spring-boot-starter-webflux</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.springframework.boot</groupId>
+      <artifactId>spring-boot-starter-aop</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.springframework.boot</groupId>
+      <artifactId>spring-boot-configuration-processor</artifactId>
+      <optional>true</optional>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.newrelic.agent.java</groupId>
+      <artifactId>newrelic-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>io.projectreactor</groupId>
+      <artifactId>reactor-core</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.springframework.boot</groupId>
+      <artifactId>spring-boot-starter-data-mongodb-reactive</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.projectlombok</groupId>
+      <artifactId>lombok</artifactId>
+      <optional>true</optional>
+    </dependency>
+    <dependency>
+      <groupId>org.springframework.boot</groupId>
+      <artifactId>spring-boot-starter-test</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>io.projectreactor</groupId>
+      <artifactId>reactor-test</artifactId>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+
+</project>

blibli-backend-framework-newrelic/pom.xml

@@ -0,0 +1,41 @@
+package com.blibli.oss.backend.newrelic.aspect;
+
+import com.blibli.oss.backend.command.Command;
+import com.blibli.oss.backend.newrelic.aspect.service.AspectModifyService;
+import com.blibli.oss.backend.newrelic.aspect.service.util.SegmentType;
+import lombok.AllArgsConstructor;
+import org.aspectj.lang.ProceedingJoinPoint;
+import org.aspectj.lang.annotation.Around;
+import org.aspectj.lang.annotation.Aspect;
+import org.aspectj.lang.annotation.Pointcut;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Configuration;
+
+/**
+ * Aspect to target Blibli Command plugin for timing with New Relic Segment.
+ */
+@Aspect
+@Configuration
+@ConditionalOnClass(Command.class)
+@ConditionalOnProperty(
+    prefix = "blibli.newrelic.command",
+    name = "enabled",
+    matchIfMissing = true
+)
+@AllArgsConstructor
+public class CommandAspect {
+
+  private AspectModifyService aspectModifyService;
+
+  @Pointcut("execution(* com.blibli.oss.backend.command.Command.execute(..))")
+  private void commandExecute() {}
+
+  @Around(value = "commandExecute()")
+  public Object afterCommandExecute(ProceedingJoinPoint proceedingJoinPoint) throws Throwable {
+    return aspectModifyService.modifyRetValWithTiming(
+        proceedingJoinPoint, SegmentType.COMMAND
+    );
+  }
+
+}

blibli-backend-framework-newrelic/src/main/java/com/blibli/oss/backend/newrelic/aspect/CommandAspect.java

@@ -0,0 +1,46 @@
+package com.blibli.oss.backend.newrelic.aspect;
+
+import com.blibli.oss.backend.newrelic.aspect.service.AspectModifyService;
+import com.blibli.oss.backend.newrelic.aspect.service.util.SegmentType;
+import lombok.AllArgsConstructor;
+import org.aspectj.lang.ProceedingJoinPoint;
+import org.aspectj.lang.annotation.Around;
+import org.aspectj.lang.annotation.Aspect;
+import org.aspectj.lang.annotation.Pointcut;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.boot.autoconfigure.mongo.MongoProperties;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.data.mongodb.core.ReactiveMongoTemplate;
+import org.springframework.data.mongodb.repository.ReactiveMongoRepository;
+
+/**
+ * Aspect to target Spring Reactive MongoDB for timing with New Relic Segment.
+ */
+@Aspect
+@Configuration
+@ConditionalOnClass({
+    ReactiveMongoRepository.class,
+    ReactiveMongoTemplate.class,
+    MongoProperties.class
+})
+@ConditionalOnProperty(
+    prefix = "blibli.newrelic.reactive-mongodb",
+    name = "enabled",
+    matchIfMissing = true
+)
+@AllArgsConstructor
+public class ReactiveMongoDbAspect {
+
+  private AspectModifyService aspectModifyService;
+
+  @Pointcut("execution(public * *.*(..)) && within(org.springframework.data.mongodb.repository.ReactiveMongoRepository+)")
+  void mongoRepositoryInterface() {}
+
+  @Around(value = "mongoRepositoryInterface()")
+  public Object afterCommandExecute(ProceedingJoinPoint proceedingJoinPoint) throws Throwable {
+    return aspectModifyService.modifyRetValWithTiming(
+        proceedingJoinPoint, SegmentType.REACTIVE_MONGODB
+    );
+  }
+}

blibli-backend-framework-newrelic/src/main/java/com/blibli/oss/backend/newrelic/aspect/ReactiveMongoDbAspect.java

@@ -0,0 +1,12 @@
+package com.blibli.oss.backend.newrelic.aspect.service;
+
+import com.blibli.oss.backend.newrelic.aspect.service.util.SegmentType;
+import org.aspectj.lang.ProceedingJoinPoint;
+
+public interface AspectModifyService {
+
+  Object modifyRetValWithTiming(
+    ProceedingJoinPoint proceedingJoinPoint, SegmentType segmentType
+  ) throws Throwable;
+
+}

blibli-backend-framework-newrelic/src/main/java/com/blibli/oss/backend/newrelic/aspect/service/AspectModifyService.java

@@ -0,0 +1,96 @@
+package com.blibli.oss.backend.newrelic.aspect.service.impl;
+
+import com.blibli.oss.backend.newrelic.aspect.service.AspectModifyService;
+import com.blibli.oss.backend.newrelic.aspect.service.util.AspectHelper;
+import com.blibli.oss.backend.newrelic.aspect.service.util.RetValType;
+import com.blibli.oss.backend.newrelic.aspect.service.util.SegmentType;
+import com.blibli.oss.backend.newrelic.injector.NewRelicTokenInjectorFilter;
+import com.blibli.oss.backend.newrelic.reporter.ExternalReporter;
+import com.blibli.oss.backend.newrelic.reporter.helper.ExternalReporterHelper;
+import com.newrelic.api.agent.Segment;
+import lombok.Setter;
+import lombok.extern.slf4j.Slf4j;
+import org.aspectj.lang.JoinPoint;
+import org.aspectj.lang.ProceedingJoinPoint;
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.ApplicationContextAware;
+import reactor.core.publisher.Flux;
+import reactor.core.publisher.Mono;
+import reactor.core.publisher.Signal;
+import reactor.util.context.Context;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+
+@Slf4j
+public class AspectModifyServiceImpl implements AspectModifyService, ApplicationContextAware, InitializingBean {
+
+  @Setter
+  private ApplicationContext applicationContext;
+
+  private Map<SegmentType, List<ExternalReporter>> externalReporters;
+
+  @Override
+  public void afterPropertiesSet() {
+    externalReporters = ExternalReporterHelper.getExternalReporters(applicationContext);
+  }
+
+  @Override
+  public Object modifyRetValWithTiming(
+    ProceedingJoinPoint proceedingJoinPoint, SegmentType segmentType) throws Throwable {
+    Object retVal = proceedingJoinPoint.proceed(proceedingJoinPoint.getArgs());
+    if (!AspectHelper.retValIsType(retVal, segmentType.getReturnValueType())) {
+      return retVal;
+    }
+
+    AtomicReference<Segment> segmentRef = new AtomicReference<>();
+
+    if (AspectHelper.retValIsType(retVal, RetValType.MONO)) {
+      retVal = ((Mono<Object>) retVal)
+        .subscriberContext(ctx -> this.startSegment(segmentType, ctx, segmentRef, proceedingJoinPoint))
+        .doOnEach(signal -> this.stopSegment(signal, segmentRef));
+    } else if (AspectHelper.retValIsType(retVal, RetValType.FLUX)) {
+      retVal = ((Flux<Object>) retVal)
+        .subscriberContext(ctx -> this.startSegment(segmentType, ctx, segmentRef, proceedingJoinPoint))
+        .doOnEach(signal -> this.stopSegment(signal, segmentRef));
+    }
+    return retVal;
+  }
+
+  Context startSegment(SegmentType segmentType, Context ctx, AtomicReference<Segment> segmentRef, JoinPoint jp) {
+    // TODO starting segment through `subscriberContext` may make the timing longer than the actual execution block.
+    // Especially if we have long/blocking Mono operation before Command layer.
+    // Not likely though, because we discourage fat controller pattern.
+    // This is a hack for this issue https://github.com/reactor/reactor-core/issues/1526
+    NewRelicTokenInjectorFilter.getTransaction(ctx)
+      .ifPresent(t -> {
+        String segmentName = AspectHelper.constructSegmentName(jp, segmentType);
+        Segment segment = t.startSegment(segmentName);
+        segmentRef.set(segment);
+
+        startExternalReporters(segmentType, segment, jp);
+      });
+    return ctx;
+  }
+
+  private void startExternalReporters(SegmentType segmentType, Segment segment, JoinPoint jp) {
+    List<ExternalReporter> reporters = externalReporters.getOrDefault(segmentType, Collections.EMPTY_LIST);
+
+    reporters.forEach(
+      reporter -> reporter.report(segment, jp)
+    );
+  }
+
+  private void stopSegment(Signal signal, AtomicReference<Segment> segmentRef) {
+    if (signal.isOnComplete()) {
+      if (segmentRef.get() != null) {
+        segmentRef.get().end();
+      } else {
+        log.warn("New Relic segment does not exist!");
+      }
+    }
+  }
+}

blibli-backend-framework-newrelic/src/main/java/com/blibli/oss/backend/newrelic/aspect/service/impl/AspectModifyServiceImpl.java

@@ -0,0 +1,34 @@
+package com.blibli.oss.backend.newrelic.aspect.service.util;
+
+import lombok.experimental.UtilityClass;
+import lombok.extern.slf4j.Slf4j;
+import org.aspectj.lang.JoinPoint;
+import reactor.core.publisher.Flux;
+import reactor.core.publisher.Mono;
+
+@Slf4j
+@UtilityClass
+public class AspectHelper {
+
+  public static String constructSegmentName(JoinPoint joinPoint, SegmentType segmentType) {
+    return String.format(
+        segmentType.getStringFormat(),
+        joinPoint.getTarget().getClass().getSimpleName(),
+        joinPoint.getSignature().toShortString()
+    );
+  }
+
+  public static boolean retValIsType(Object retVal, RetValType type) {
+    switch (type) {
+      case FLUX:
+        return retVal instanceof Flux;
+      case MONO:
+        return retVal instanceof Mono;
+      case MONO_OR_FLUX:
+        return retVal instanceof Flux || retVal instanceof Mono;
+      default:
+        throw new IllegalArgumentException("RetValType is unknown");
+    }
+  }
+
+}

blibli-backend-framework-newrelic/src/main/java/com/blibli/oss/backend/newrelic/aspect/service/util/AspectHelper.java

@@ -0,0 +1,33 @@
+package com.blibli.oss.backend.newrelic.aspect.service.util;
+
+import org.springframework.boot.autoconfigure.mongo.MongoProperties;
+
+import java.util.Optional;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class MongoUriParser {
+
+  //    mongodb://[username:password@]host1[:port1][,...hostN[:portN]][/[defaultauthdb][?options]]
+  private static final String MONGO_CREDENTIALS_PATTERN = "([^:]+:[^@]+@)?";
+  private static final String MONGO_HOST_PORT_PATTERN = "[^@:,/]+(:[\\d]+)?";
+  private static final String MONGO_HOST_PORT_OTHERS_PATTERN = "(,[^@:,/]+(:[\\d]+)?)*";
+  private static final String MONGO_DB_OPTIONS_PATTERN = "(/.*)?";
+
+  private static final Pattern MONGO_URI_PATTERN = Pattern.compile(String.format("^mongodb://%s(%s%s)%s$",
+    MONGO_CREDENTIALS_PATTERN, MONGO_HOST_PORT_PATTERN, MONGO_HOST_PORT_OTHERS_PATTERN, MONGO_DB_OPTIONS_PATTERN));
+
+  public static String[] getHosts(String mongoUri) {
+    Matcher m = MONGO_URI_PATTERN.matcher(mongoUri);
+
+    if (m.find()) {
+      return Optional.ofNullable(m.group(2))
+        .map(hosts -> hosts.split(","))
+        .orElseThrow(() -> new RuntimeException("unsupported mongo uri"));
+    } else {
+      throw new RuntimeException("unsupported mongo uri");
+    }
+  }
+
+
+}

blibli-backend-framework-newrelic/src/main/java/com/blibli/oss/backend/newrelic/aspect/service/util/MongoUriParser.java

@@ -0,0 +1,5 @@
+package com.blibli.oss.backend.newrelic.aspect.service.util;
+
+public enum RetValType {
+  MONO, FLUX, MONO_OR_FLUX;
+}