Adding pause/resume functionality to reduce agent overhead on stress (#1040)

Author: eyalkoren

CHANGELOG.asciidoc

@@ -33,7 +33,9 @@ endif::[]
   unless <<config-use-elastic-traceparent-header, `use_elastic_traceparent_header`>> is set to false.
 * Creating spans for slow methods with the help of the sampling profiler https://github.com/jvm-profiling-tools/async-profiler[async-profiler].
   This is a low-overhead way of seeing which methods make your transactions slow and a replacement for the `trace_methods` configuration option.
-  See <<supported-java-methods>> for more details.
+  See <<supported-java-methods>> for more details
+* Adding a Circuit Breaker to pause the agent when stress is detected on the system and resume when the stress is relieved.
+  See <<circuit-breaker>> and {pull}1040[#1040] for more info.
 
 [float]
 ===== Bug fixes

apm-agent-benchmarks/src/main/java/co/elastic/apm/agent/benchmark/AbstractMockApmServerBenchmark.java

@@ -82,7 +82,7 @@ public void setUp(Blackhole blackhole) throws IOException {
                 .addConfigSource(new SimpleSource()
                     .add(CoreConfiguration.SERVICE_NAME, "benchmark")
                     .add(CoreConfiguration.INSTRUMENT, Boolean.toString(apmEnabled))
-                    .add(CoreConfiguration.ACTIVE, Boolean.toString(apmEnabled))
+                    .add("active", Boolean.toString(apmEnabled))
                     .add("api_request_size", "10mb")
                     .add("capture_headers", "false")
 //                     .add("profiling_inferred_spans", "true")

apm-agent-core/src/main/java/co/elastic/apm/agent/bci/MatcherTimerLifecycleListener.java

@@ -11,9 +11,9 @@
  * the Apache License, Version 2.0 (the "License"); you may
  * not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * 
+ *
  *   http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -25,21 +25,17 @@
 package co.elastic.apm.agent.bci;
 
 import co.elastic.apm.agent.bci.bytebuddy.MatcherTimer;
-import co.elastic.apm.agent.context.LifecycleListener;
+import co.elastic.apm.agent.context.AbstractLifecycleListener;
 import co.elastic.apm.agent.impl.ElasticApmTracer;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.util.ArrayList;
 import java.util.Collections;
 
-public class MatcherTimerLifecycleListener implements LifecycleListener {
+public class MatcherTimerLifecycleListener extends AbstractLifecycleListener {
     private static final Logger logger = LoggerFactory.getLogger(MatcherTimerLifecycleListener.class);
 
-    @Override
-    public void start(ElasticApmTracer tracer) {
-    }
-
     @Override
     public void stop() {
         if (logger.isDebugEnabled()) {

apm-agent-core/src/main/java/co/elastic/apm/agent/bci/OsgiBootDelegationEnabler.java

@@ -11,9 +11,9 @@
  * the Apache License, Version 2.0 (the "License"); you may
  * not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * 
+ *
  *   http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -25,7 +25,7 @@
 package co.elastic.apm.agent.bci;
 
 import co.elastic.apm.agent.configuration.CoreConfiguration;
-import co.elastic.apm.agent.context.LifecycleListener;
+import co.elastic.apm.agent.context.AbstractLifecycleListener;
 import co.elastic.apm.agent.impl.ElasticApmTracer;
 
 import javax.annotation.Nullable;
@@ -41,7 +41,7 @@
  * See {@link AgentMain#init(String, java.lang.instrument.Instrumentation)}
  * </p>
  */
-public class OsgiBootDelegationEnabler implements LifecycleListener {
+public class OsgiBootDelegationEnabler extends AbstractLifecycleListener {
     private static final String APM_BASE_PACKAGE = "co.elastic.apm.agent.*";
     // see https://confluence.atlassian.com/jirakb/using-javaagent-with-jira-790793295.html#UsingjavaagentwithJIRA-Resolution
     private static final String ATLASSIAN_BOOTDELEGATION_DEFAULTS = "META-INF.services,com.yourkit,com.singularity.*,com.jprofiler," +
@@ -70,9 +70,4 @@ private static void appendToSystemProperty(String propertyName, @Nullable String
             System.setProperty(propertyName, append);
         }
     }
-
-    @Override
-    public void stop() {
-        // noop
-    }
 }

apm-agent-core/src/main/java/co/elastic/apm/agent/bci/methodmatching/TraceMethodInstrumentation.java

@@ -74,9 +74,10 @@ public static void onMethodEnter(@Advice.Origin Class<?> clazz,
         if (tracer != null) {
             final TraceContextHolder<?> parent = tracer.getActive();
             if (parent == null) {
-                span = tracer.startRootTransaction(clazz.getClassLoader())
-                    .withName(signature)
-                    .activate();
+                span = tracer.startRootTransaction(clazz.getClassLoader());
+                if (span != null) {
+                    span.withName(signature).activate();
+                }
             } else if (parent.isSampled()) {
                 span = parent.createSpan()
                     .withName(signature)

apm-agent-core/src/main/java/co/elastic/apm/agent/configuration/ApmServerConfigurationSource.java

@@ -11,9 +11,9 @@
  * the Apache License, Version 2.0 (the "License"); you may
  * not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * 
+ *
  *   http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -227,6 +227,16 @@ public String getName() {
         return "APM Server";
     }
 
+    @Override
+    public void pause() {
+        // Keep polling for remote config changes, in case the user wants to resume a paused agent or change the stress
+        // monitoring configurations.
+    }
+
+    @Override
+    public void resume() {
+    }
+
     @Override
     public void stop() {
         if (this.threadPool != null) {

apm-agent-core/src/main/java/co/elastic/apm/agent/configuration/CoreConfiguration.java

@@ -53,28 +53,14 @@
 
 public class CoreConfiguration extends ConfigurationOptionProvider {
 
-    public static final String ACTIVE = "active";
     public static final String INSTRUMENT = "instrument";
     public static final String SERVICE_NAME = "service_name";
     public static final String SERVICE_NODE_NAME = "service_node_name";
     public static final String SAMPLE_RATE = "transaction_sample_rate";
-    private static final String CORE_CATEGORY = "Core";
+    public static final String CORE_CATEGORY = "Core";
     private static final String DEFAULT_CONFIG_FILE = AGENT_HOME_PLACEHOLDER + "/elasticapm.properties";
     public static final String CONFIG_FILE = "config_file";
 
-    private final ConfigurationOption<Boolean> active = ConfigurationOption.booleanOption()
-        .key(ACTIVE)
-        .configurationCategory(CORE_CATEGORY)
-        .description("A boolean specifying if the agent should be active or not.\n" +
-            "When active, the agent instruments incoming HTTP requests, tracks errors and collects and sends metrics.\n" +
-            "When inactive, the agent works as a noop, not collecting data and not communicating with the APM sever.\n" +
-            "As this is a reversible switch, agent threads are not being killed when inactivated, but they will be \n" +
-            "mostly idle in this state, so the overhead should be negligible.\n" +
-            "\n" +
-            "You can use this setting to dynamically disable Elastic APM at runtime.")
-        .dynamic(true)
-        .buildWithDefault(true);
-
     private final ConfigurationOption<Boolean> instrument = ConfigurationOption.booleanOption()
         .key(INSTRUMENT)
         .configurationCategory(CORE_CATEGORY)
@@ -526,10 +512,6 @@ public class CoreConfiguration extends ConfigurationOptionProvider {
         .tags("internal")
         .buildWithDefault(4096);
 
-    public boolean isActive() {
-        return active.get();
-    }
-
     public boolean isInstrument() {
         return instrument.get();
     }

apm-agent-core/src/main/java/co/elastic/apm/agent/configuration/StartupInfo.java

@@ -11,9 +11,9 @@
  * the Apache License, Version 2.0 (the "License"); you may
  * not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * 
+ *
  *   http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -25,7 +25,7 @@
 package co.elastic.apm.agent.configuration;
 
 import co.elastic.apm.agent.configuration.converter.TimeDuration;
-import co.elastic.apm.agent.context.LifecycleListener;
+import co.elastic.apm.agent.context.AbstractLifecycleListener;
 import co.elastic.apm.agent.impl.ElasticApmTracer;
 import co.elastic.apm.agent.impl.stacktrace.StacktraceConfiguration;
 import co.elastic.apm.agent.util.VersionUtils;
@@ -43,7 +43,7 @@
  * under Apache license 2.0.
  * </p>
  */
-public class StartupInfo implements LifecycleListener {
+public class StartupInfo extends AbstractLifecycleListener {
 
     private static final Logger logger = LoggerFactory.getLogger(StartupInfo.class);
     private final String elasticApmVersion;
@@ -105,9 +105,4 @@ private void logConfigWithNonDefaultValue(Logger logger, ConfigurationOption<?>
                 option.getNameOfCurrentConfigurationSource());
         }
     }
-
-    @Override
-    public void stop() throws Exception {
-        // noop
-    }
 }

apm-agent-core/src/main/java/co/elastic/apm/agent/context/AbstractLifecycleListener.java

@@ -11,9 +11,9 @@
  * the Apache License, Version 2.0 (the "License"); you may
  * not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * 
+ *
  *   http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing,
  * software distributed under the License is distributed on an
  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -27,14 +27,17 @@
 import co.elastic.apm.agent.impl.ElasticApmTracer;
 
 public abstract class AbstractLifecycleListener implements LifecycleListener {
-    protected final ElasticApmTracer tracer;
 
-    protected AbstractLifecycleListener(ElasticApmTracer tracer) {
-        this.tracer = tracer;
+    @Override
+    public void start(ElasticApmTracer tracer) {
     }
 
     @Override
-    public void start(ElasticApmTracer tracer) {
+    public void pause() throws Exception {
+    }
+
+    @Override
+    public void resume() throws Exception {
     }
 
     @Override

apm-agent-core/src/main/java/co/elastic/apm/agent/context/ClosableLifecycleListenerAdapter.java

@@ -0,0 +1,45 @@
+/*-
+ * #%L
+ * Elastic APM Java agent
+ * %%
+ * Copyright (C) 2018 - 2020 Elastic and contributors
+ * %%
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * #L%
+ */
+package co.elastic.apm.agent.context;
+
+import java.io.Closeable;
+
+public class ClosableLifecycleListenerAdapter extends AbstractLifecycleListener {
+
+    private final Closeable closeable;
+
+    public static LifecycleListener of(Closeable closeable) {
+        return new ClosableLifecycleListenerAdapter(closeable);
+    }
+
+    private ClosableLifecycleListenerAdapter(Closeable closeable) {
+        this.closeable = closeable;
+    }
+
+    @Override
+    public void stop() throws Exception {
+        closeable.close();
+    }
+}