Merge pull request #2153 from digma-ai/extended-observability-experiment

extended-observability-experiment

Author: shalom938

build.gradle.kts

@@ -249,6 +249,18 @@ tasks {
             "org.digma.plugin.enable.devtools" to "true",
 
 //            "idea.ProcessCanceledException" to "disabled"
+
+
+            //to use a local file for digma-agent or digma extension,
+            // usually when developing and we want to test the plugin
+            //see org.digma.intellij.plugin.idea.execution.OtelAgentPathProvider
+            //don't forget to comment when done testing !
+            //"digma.agent.override.path" to "/home/shalom/workspace/digma/digma-agent/build/libs/digma-agent-1.0.10-SNAPSHOT.jar"
+            //"digma.otel.extension.override.path" to "/home/shalom/workspace/digma/otel-java-instrumentation/agent-extension/build/libs/digma-otel-agent-extension-0.8.12.jar"
+            //can also change the url from where the jar is downloaded when IDE starts
+            //"org.digma.otel.extensionUrl" to "some url
+            //"org.digma.otel.digmaAgentUrl" to "some url
+
         )
 
 

ide-common/src/main/java/org/digma/intellij/plugin/settings/ProjectSettings.java

@@ -46,7 +46,8 @@ public boolean isModified() {
                 isJaegerLinkModeChanged(settings) ||
                 isSpringBootObservabilityModeChanged(settings) ||
                 isRuntimeObservabilityBackendUrlChanged(settings) ||
-                isExtendedObservabilityChanged(settings);
+                isExtendedObservabilityChanged(settings) ||
+                isExtendedObservabilityExcludeChanged(settings);
     }
 
     private boolean isRefreshDelayChanged(SettingsState settings) {
@@ -85,6 +86,10 @@ private boolean isExtendedObservabilityChanged(SettingsState settings) {
         return !Objects.equals(settings.extendedObservability, mySettingsComponent.getExtendedObservability());
     }
 
+    private boolean isExtendedObservabilityExcludeChanged(SettingsState settings) {
+        return !Objects.equals(settings.extendedObservabilityExcludes, mySettingsComponent.getExtendedObservabilityExclude());
+    }
+
     @Override
     public void apply() throws ConfigurationException {
         SettingsState settings = SettingsState.getInstance();
@@ -149,6 +154,7 @@ public void apply() throws ConfigurationException {
         settings.springBootObservabilityMode = mySettingsComponent.getSpringBootObservabilityMode();
         settings.runtimeObservabilityBackendUrl = mySettingsComponent.getRuntimeObservabilityBackendUrl();
         settings.extendedObservability = mySettingsComponent.getExtendedObservability();
+        settings.extendedObservabilityExcludes = mySettingsComponent.getExtendedObservabilityExclude();
         settings.fireChanged();
     }
 
@@ -164,6 +170,7 @@ public void reset() {
         mySettingsComponent.setSpringBootObservabilityMode(settings.springBootObservabilityMode);
         mySettingsComponent.setRuntimeObservabilityBackendUrl(settings.runtimeObservabilityBackendUrl);
         mySettingsComponent.setExtendedObservability(settings.extendedObservability);
+        mySettingsComponent.setExtendedObservabilityExclude(settings.extendedObservabilityExcludes);
     }
 
 

ide-common/src/main/java/org/digma/intellij/plugin/settings/SettingsComponent.java

@@ -30,9 +30,14 @@ public class SettingsComponent {
     private final JBLabel myRuntimeObservabilityBackendUrlLabel = new JBLabel("Runtime observability backend URL:");
     private final JBTextField myRuntimeObservabilityBackendUrlText = new JBTextField();
     private final JBTextField extendedObservabilityTextBox = new JBTextField();
+    private final JBTextField extendedObservabilityExcludeTextBox = new JBTextField();
 
     public SettingsComponent() {
 
+        extendedObservabilityTextBox.setToolTipText("packages names in format 'my.pkg1;my.pkg2");
+        extendedObservabilityExcludeTextBox.setToolTipText("class/method names to exclude in format 'MyClass;MyOtherClass.myOtherMethod;*get");
+
+
         var defaultLabelForeground = JBColor.foreground();
 
         var myUrlLabel = new JBLabel("Digma API URL: ");
@@ -156,6 +161,7 @@ public boolean verify(JComponent input) {
                 .addLabeledComponent(mySpringBootObservabilityModeLabel, mySpringBootObservabilityModeComboBox, 1, false)
                 .addLabeledComponent(myRuntimeObservabilityBackendUrlLabel, myRuntimeObservabilityBackendUrlText, 1, false)
                 .addLabeledComponent("Extended Observability (beta)", extendedObservabilityTextBox, 1, false)
+                .addLabeledComponent("Extended Observability Exclude (beta)", extendedObservabilityExcludeTextBox, 1, false)
                 .addComponent(resetButton)
                 .addComponentFillVertically(new JPanel(), 0)
                 .getPanel();
@@ -278,6 +284,16 @@ public String getExtendedObservability() {
         return extendedObservabilityTextBox.getText();
     }
 
+    public void setExtendedObservabilityExclude(@Nullable String extendedObservabilityExclude) {
+        extendedObservabilityExcludeTextBox.setText(extendedObservabilityExclude);
+    }
+
+    @Nullable
+    public String getExtendedObservabilityExclude() {
+        return extendedObservabilityExcludeTextBox.getText();
+    }
+
+
     private void resetToDefaults() {
         this.setApiUrlText(SettingsState.DEFAULT_API_URL);
         this.setApiToken(null);
@@ -289,5 +305,6 @@ private void resetToDefaults() {
         this.setSpringBootObservabilityMode(SettingsState.DEFAULT_SPRING_BOOT_OBSERVABILITY_MODE);
         this.setRuntimeObservabilityBackendUrl(SettingsState.DEFAULT_RUNTIME_OBSERVABILITY_BACKEND_URL);
         this.setExtendedObservability(null);
+        this.setExtendedObservabilityExclude(null);
     }
 }

ide-common/src/main/java/org/digma/intellij/plugin/settings/SettingsState.java

@@ -51,6 +51,8 @@ public class SettingsState implements PersistentStateComponent<SettingsState>, D
     public String posthogToken;
     @Nullable
     public String extendedObservability;
+    @Nullable
+    public String extendedObservabilityExcludes;
 
 
     private final List<SettingsChangeListener> listeners = new ArrayList<>();

ide-common/src/main/kotlin/org/digma/intellij/plugin/posthog/SettingsChangeTracker.kt

@@ -77,6 +77,7 @@ class SettingsChangeTracker {
         myTrackedSettings["refreshDelay"] = SettingsState.getInstance().refreshDelay.toString()
         myTrackedSettings["springBootObservabilityMode"] = SettingsState.getInstance().springBootObservabilityMode.name
         myTrackedSettings["extendedObservability"] = SettingsState.getInstance().extendedObservability.toString()
+        myTrackedSettings["extendedObservabilityExcludes"] = SettingsState.getInstance().extendedObservabilityExcludes.toString()
     }
 
 

jvm-common/build.gradle.kts

@@ -39,14 +39,29 @@ tasks {
         src(
             listOf(
                 properties.getProperty("otel-agent"),
-                properties.getProperty("digma-extension")
+                properties.getProperty("digma-extension"),
+                properties.getProperty("digma-agent")
             )
         )
 
-        logger.lifecycle("copying jars $properties")
+        logger.lifecycle("jars to download $properties")
 
         dest(File(project.sourceSets.main.get().output.resourcesDir, "otelJars"))
         overwrite(true)
+        //if a jar is downloaded with version then its name needs to change. it may happen
+        // in development if the url for some of the jars is changed to download from somewhere else.
+        //usually latest jar is downloaded without version
+        eachFile {
+            name = if (name.startsWith("digma-otel-agent-extension", true)) {
+                "digma-otel-agent-extension.jar"
+            } else if (name.startsWith("digma-agent", true)) {
+                "digma-agent.jar"
+            } else if (name.startsWith("opentelemetry-javaagent", true)) {
+                "opentelemetry-javaagent.jar"
+            } else {
+                name
+            }
+        }
     }
 
     processResources {

jvm-common/src/main/kotlin/org/digma/intellij/plugin/idea/execution/JavaParametersMerger.kt

@@ -95,9 +95,10 @@ open class JavaParametersMerger(
     protected fun smartMergeJavaToolOptions(myJavaToolOptions: String, currentJavaToolOptions: String): String {
 
         //merge two java tool options strings. values from myJavaToolOptions override values from currentJavaToolOptions
+        //we want to preserve the order , existing java tool options properties should be first
 
-        val myOptions = javaToolOptionsToMap(myJavaToolOptions)
         val currentOptions = javaToolOptionsToMap(currentJavaToolOptions)
+        val myOptions = javaToolOptionsToMap(myJavaToolOptions)
 
         val result = mutableMapOf<String, String?>()
         result.putAll(currentOptions)

jvm-common/src/main/kotlin/org/digma/intellij/plugin/idea/execution/JavaToolOptionsBuilder.kt

@@ -14,6 +14,7 @@ open class JavaToolOptionsBuilder(
 
     private val otelAgentPathProvider = OtelAgentPathProvider(configuration)
 
+    @Suppress("MemberVisibilityCanBePrivate")
     protected val javaToolOptions = StringBuilder(" ")
 
 
@@ -39,12 +40,29 @@ open class JavaToolOptionsBuilder(
     }
 
 
-    open fun withOtelAgent(useAgent: Boolean): JavaToolOptionsBuilder {
+    open fun withOtelAgent(useAgent: Boolean, parametersExtractor: ParametersExtractor): JavaToolOptionsBuilder {
 
         if (useAgent) {
+
             if (!otelAgentPathProvider.hasAgentPath()) {
-                throw JavaToolOptionsBuilderException("useAgent is true but can't find agent paths")
+                throw JavaToolOptionsBuilderException("useAgent is true but can't find agent or extension paths")
+            }
+
+            if (isExtendedObservabilityConfigured()) {
+
+                if (!otelAgentPathProvider.hasDigmaAgentPath()) {
+                    throw JavaToolOptionsBuilderException("can't find digma-agent path")
+                }
+
+                javaToolOptions
+                    .append("-javaagent:${otelAgentPathProvider.digmaAgentPath}")
+                    .append(" ")
+
+                withDigmaAgentDebug()
+
+
             }
+
             javaToolOptions
                 .append("-javaagent:${otelAgentPathProvider.otelAgentPath}")
                 .append(" ")
@@ -141,12 +159,33 @@ open class JavaToolOptionsBuilder(
     }
 
 
+    open fun withDigmaAgentDebug(): JavaToolOptionsBuilder {
+        if (java.lang.Boolean.getBoolean("digma.agent.debug") ||
+            java.lang.Boolean.valueOf(System.getenv("DIGMA_AGENT_DEBUG"))
+        ) {
+            javaToolOptions
+                .append("-Ddigma.agent.debug=true")
+                .append(" ")
+        }
+        return this
+    }
+
+
     open fun withExtendedObservability(): JavaToolOptionsBuilder {
-        if (!SettingsState.getInstance().extendedObservability.isNullOrBlank()) {
+
+        if (isExtendedObservabilityConfigured()) {
+
             javaToolOptions
                 .append("-Ddigma.autoinstrument.packages=${SettingsState.getInstance().extendedObservability}")
                 .append(" ")
+
+            if (!SettingsState.getInstance().extendedObservabilityExcludes.isNullOrBlank()) {
+                javaToolOptions
+                    .append("-Ddigma.autoinstrument.packages.exclude.names=${SettingsState.getInstance().extendedObservabilityExcludes}")
+                    .append(" ")
+            }
         }
+
         return this
     }
 
@@ -254,14 +293,34 @@ open class JavaToolOptionsBuilder(
 
 
     open fun build(): String {
+
+        //always disable extended observability extension because we changed to digma-agent
+        disableExtendedObservabilityExtension()
+
         return javaToolOptions.toString()
     }
 
 
+    private fun disableExtendedObservabilityExtension() {
+        //todo: need to disable the extended observability extension completely in the otel agent extension
+        // project after some time that we use digma-agent and all users upgraded the plugin.
+        // can't disable it yet because some users may still use older version of the plugin and are using extended observability.
+        javaToolOptions
+            .append("-Dotel.instrumentation.digma-methods.enabled=false")
+            .append(" ")
+    }
+
+
     open fun getExporterUrl(): String {
         return SettingsState.getInstance().runtimeObservabilityBackendUrl
     }
 
+
+    @Suppress("MemberVisibilityCanBePrivate")
+    fun isExtendedObservabilityConfigured(): Boolean {
+        return !SettingsState.getInstance().extendedObservability.isNullOrBlank()
+    }
+
 }
 
 

jvm-common/src/main/kotlin/org/digma/intellij/plugin/idea/execution/OTELJarProvider.kt

@@ -17,9 +17,27 @@ private const val RESOURCE_LOCATION = "otelJars"
 
 private const val OTEL_AGENT_JAR_PROP_NAME = "otel-agent"
 private const val DIGMA_EXTENSION_JAR_PROP_NAME = "digma-extension"
+private const val DIGMA_AGENT_JAR_PROP_NAME = "digma-agent"
 private const val OTEL_AGENT_JAR_NAME = "opentelemetry-javaagent.jar"
 private const val DIGMA_AGENT_EXTENSION_JAR_NAME = "digma-otel-agent-extension.jar"
-
+private const val DIGMA_AGENT_JAR_NAME = "digma-agent.jar"
+
+
+/**
+ * Downloads and provides the instrumentation jars.
+ *
+ * jars urls and path may be overridden for development to another download url or a local path,
+ * for example when developing the digma-agent or otel extension.
+ * system properties can be injected in runIde task or in idea custom properties.
+ *
+ * to override the download url see the method tryDownloadLatest, it will try to get the url from system
+ * property if exists. this is useful for example for downloading a different version of some of the jars.
+ *
+ * to override the final path see org.digma.intellij.plugin.idea.execution.OtelAgentPathProvider, it will
+ * first check if system property exists and use it instead of the default. this is useful when developing
+ * the digma agent or otel extension and you want to use the local build jar.
+ *
+ */
 
 //Do not change to light service because it will always register.
 // we want it to register only in Idea.
@@ -63,9 +81,9 @@ class OTELJarProvider {
 
     fun getDigmaAgentExtensionJarPath(): String? {
         ensureFilesExist()
-        val digmaJar = getDigmaAgentExtensionJar()
-        if (digmaJar.exists()) {
-            return digmaJar.absolutePath
+        val digmaExtensionJar = getDigmaAgentExtensionJar()
+        if (digmaExtensionJar.exists()) {
+            return digmaExtensionJar.absolutePath
         }
         return null
     }
@@ -74,6 +92,19 @@ class OTELJarProvider {
         return File(downloadDir, DIGMA_AGENT_EXTENSION_JAR_NAME)
     }
 
+    fun getDigmaAgentJarPath(): String? {
+        ensureFilesExist()
+        val digmaAgentJar = getDigmaAgentJar()
+        if (digmaAgentJar.exists()) {
+            return digmaAgentJar.absolutePath
+        }
+        return null
+    }
+
+    private fun getDigmaAgentJar(): File {
+        return File(downloadDir, DIGMA_AGENT_JAR_NAME)
+    }
+
 
     private fun ensureFilesExist() {
 
@@ -89,8 +120,9 @@ class OTELJarProvider {
 
     private fun filesExist(): Boolean {
         val otelJar = getOtelAgentJar()
-        val digmaJar = getDigmaAgentExtensionJar()
-        return otelJar.exists() && digmaJar.exists()
+        val digmaExtensionJar = getDigmaAgentExtensionJar()
+        val digmaAgentJar = getDigmaAgentJar()
+        return otelJar.exists() && digmaExtensionJar.exists() && digmaAgentJar.exists()
     }
 
 
@@ -109,6 +141,7 @@ class OTELJarProvider {
                 if (downloadDir.exists()) {
                     copyFileFromResource(OTEL_AGENT_JAR_NAME)
                     copyFileFromResource(DIGMA_AGENT_EXTENSION_JAR_NAME)
+                    copyFileFromResource(DIGMA_AGENT_JAR_NAME)
                     Log.log(logger::info, "otel agent jars unpacked to {}", downloadDir)
                 }
             } catch (e: Exception) {
@@ -143,9 +176,11 @@ class OTELJarProvider {
             try {
                 val otelUrl = System.getProperty("org.digma.otel.agentUrl", jarsUrls.getProperty(OTEL_AGENT_JAR_PROP_NAME))
                 val extensionUrl = System.getProperty("org.digma.otel.extensionUrl", jarsUrls.getProperty(DIGMA_EXTENSION_JAR_PROP_NAME))
+                val digmaAgentUrl = System.getProperty("org.digma.otel.digmaAgentUrl", jarsUrls.getProperty(DIGMA_AGENT_JAR_PROP_NAME))
 
                 downloadAndCopyJar(URL(otelUrl), getOtelAgentJar())
                 downloadAndCopyJar(URL(extensionUrl), getDigmaAgentExtensionJar())
+                downloadAndCopyJar(URL(digmaAgentUrl), getDigmaAgentJar())
             } catch (e: Exception) {
                 Log.warnWithException(logger, e, "could not download latest otel jars")
             }