extended-observability-experiment

Author: shalom938

build.gradle.kts

@@ -249,6 +249,18 @@ tasks {
             "org.digma.plugin.enable.devtools" to "true",
 
 //            "idea.ProcessCanceledException" to "disabled"
+
+
+            //to use a local file for digma-agent or digma extension,
+            // usually when developing and we want to test the plugin
+            //see org.digma.intellij.plugin.idea.execution.OtelAgentPathProvider
+            //don't forget to comment when done testing !
+            //"digma.agent.override.path" to "/home/shalom/workspace/digma/digma-agent/build/libs/digma-agent-1.0.2.jar"
+            //"digma.otel.extension.override.path" to "/home/shalom/workspace/digma/otel-java-instrumentation/agent-extension/build/libs/digma-otel-agent-extension-0.8.12.jar"
+            //can also change the url from where the jar is downloaded when IDE starts
+            //"org.digma.otel.extensionUrl" to "some url
+            //"org.digma.otel.digmaAgentUrl" to "some url
+
         )
 
 

ide-common/src/main/java/org/digma/intellij/plugin/settings/ProjectSettings.java

@@ -46,7 +46,8 @@ public boolean isModified() {
                 isJaegerLinkModeChanged(settings) ||
                 isSpringBootObservabilityModeChanged(settings) ||
                 isRuntimeObservabilityBackendUrlChanged(settings) ||
-                isExtendedObservabilityChanged(settings);
+                isExtendedObservabilityChanged(settings) ||
+                isExtendedObservabilityExcludeChanged(settings);
     }
 
     private boolean isRefreshDelayChanged(SettingsState settings) {
@@ -85,6 +86,10 @@ private boolean isExtendedObservabilityChanged(SettingsState settings) {
         return !Objects.equals(settings.extendedObservability, mySettingsComponent.getExtendedObservability());
     }
 
+    private boolean isExtendedObservabilityExcludeChanged(SettingsState settings) {
+        return !Objects.equals(settings.extendedObservabilityExcludes, mySettingsComponent.getExtendedObservabilityExclude());
+    }
+
     @Override
     public void apply() throws ConfigurationException {
         SettingsState settings = SettingsState.getInstance();
@@ -149,6 +154,7 @@ public void apply() throws ConfigurationException {
         settings.springBootObservabilityMode = mySettingsComponent.getSpringBootObservabilityMode();
         settings.runtimeObservabilityBackendUrl = mySettingsComponent.getRuntimeObservabilityBackendUrl();
         settings.extendedObservability = mySettingsComponent.getExtendedObservability();
+        settings.extendedObservabilityExcludes = mySettingsComponent.getExtendedObservabilityExclude();
         settings.fireChanged();
     }
 
@@ -164,6 +170,7 @@ public void reset() {
         mySettingsComponent.setSpringBootObservabilityMode(settings.springBootObservabilityMode);
         mySettingsComponent.setRuntimeObservabilityBackendUrl(settings.runtimeObservabilityBackendUrl);
         mySettingsComponent.setExtendedObservability(settings.extendedObservability);
+        mySettingsComponent.setExtendedObservabilityExclude(settings.extendedObservabilityExcludes);
     }
 
 

ide-common/src/main/java/org/digma/intellij/plugin/settings/SettingsComponent.java

@@ -30,9 +30,14 @@ public class SettingsComponent {
     private final JBLabel myRuntimeObservabilityBackendUrlLabel = new JBLabel("Runtime observability backend URL:");
     private final JBTextField myRuntimeObservabilityBackendUrlText = new JBTextField();
     private final JBTextField extendedObservabilityTextBox = new JBTextField();
+    private final JBTextField extendedObservabilityExcludeTextBox = new JBTextField();
 
     public SettingsComponent() {
 
+        extendedObservabilityTextBox.setToolTipText("packages names in format 'my.pkg1;my.pkg2");
+        extendedObservabilityExcludeTextBox.setToolTipText("class/method names to exclude in format 'MyClass;MyOtherClass.myOtherMethod;*get");
+
+
         var defaultLabelForeground = JBColor.foreground();
 
         var myUrlLabel = new JBLabel("Digma API URL: ");
@@ -156,6 +161,7 @@ public boolean verify(JComponent input) {
                 .addLabeledComponent(mySpringBootObservabilityModeLabel, mySpringBootObservabilityModeComboBox, 1, false)
                 .addLabeledComponent(myRuntimeObservabilityBackendUrlLabel, myRuntimeObservabilityBackendUrlText, 1, false)
                 .addLabeledComponent("Extended Observability (beta)", extendedObservabilityTextBox, 1, false)
+                .addLabeledComponent("Extended Observability Exclude (beta)", extendedObservabilityExcludeTextBox, 1, false)
                 .addComponent(resetButton)
                 .addComponentFillVertically(new JPanel(), 0)
                 .getPanel();
@@ -278,6 +284,16 @@ public String getExtendedObservability() {
         return extendedObservabilityTextBox.getText();
     }
 
+    public void setExtendedObservabilityExclude(@Nullable String extendedObservabilityExclude) {
+        extendedObservabilityExcludeTextBox.setText(extendedObservabilityExclude);
+    }
+
+    @Nullable
+    public String getExtendedObservabilityExclude() {
+        return extendedObservabilityExcludeTextBox.getText();
+    }
+
+
     private void resetToDefaults() {
         this.setApiUrlText(SettingsState.DEFAULT_API_URL);
         this.setApiToken(null);
@@ -289,5 +305,6 @@ private void resetToDefaults() {
         this.setSpringBootObservabilityMode(SettingsState.DEFAULT_SPRING_BOOT_OBSERVABILITY_MODE);
         this.setRuntimeObservabilityBackendUrl(SettingsState.DEFAULT_RUNTIME_OBSERVABILITY_BACKEND_URL);
         this.setExtendedObservability(null);
+        this.setExtendedObservabilityExclude(null);
     }
 }

ide-common/src/main/java/org/digma/intellij/plugin/settings/SettingsState.java

@@ -51,6 +51,8 @@ public class SettingsState implements PersistentStateComponent<SettingsState>, D
     public String posthogToken;
     @Nullable
     public String extendedObservability;
+    @Nullable
+    public String extendedObservabilityExcludes;
 
 
     private final List<SettingsChangeListener> listeners = new ArrayList<>();

ide-common/src/main/kotlin/org/digma/intellij/plugin/posthog/SettingsChangeTracker.kt

@@ -77,6 +77,7 @@ class SettingsChangeTracker {
         myTrackedSettings["refreshDelay"] = SettingsState.getInstance().refreshDelay.toString()
         myTrackedSettings["springBootObservabilityMode"] = SettingsState.getInstance().springBootObservabilityMode.name
         myTrackedSettings["extendedObservability"] = SettingsState.getInstance().extendedObservability.toString()
+        myTrackedSettings["extendedObservabilityExcludes"] = SettingsState.getInstance().extendedObservabilityExcludes.toString()
     }
 
 

jvm-common/build.gradle.kts

@@ -39,14 +39,29 @@ tasks {
         src(
             listOf(
                 properties.getProperty("otel-agent"),
-                properties.getProperty("digma-extension")
+                properties.getProperty("digma-extension"),
+                properties.getProperty("digma-agent")
             )
         )
 
-        logger.lifecycle("copying jars $properties")
+        logger.lifecycle("jars to download $properties")
 
         dest(File(project.sourceSets.main.get().output.resourcesDir, "otelJars"))
         overwrite(true)
+        //if a jar is downloaded with version then its name needs to change. it may happen
+        // in development if the url for some of the jars is changed to download from somewhere else.
+        //usually latest jar is downloaded without version
+        eachFile {
+            name = if (name.startsWith("digma-otel-agent-extension", true)) {
+                "digma-otel-agent-extension.jar"
+            } else if (name.startsWith("digma-agent", true)) {
+                "digma-agent.jar"
+            } else if (name.startsWith("opentelemetry-javaagent", true)) {
+                "opentelemetry-javaagent.jar"
+            } else {
+                name
+            }
+        }
     }
 
     processResources {

jvm-common/src/main/kotlin/org/digma/intellij/plugin/idea/execution/JavaToolOptionsBuilder.kt

@@ -46,6 +46,8 @@ open class JavaToolOptionsBuilder(
                 throw JavaToolOptionsBuilderException("useAgent is true but can't find agent paths")
             }
             javaToolOptions
+                .append("-javaagent:${otelAgentPathProvider.digmaAgentPath}")
+                .append(" ")
                 .append("-javaagent:${otelAgentPathProvider.otelAgentPath}")
                 .append(" ")
                 .append("-Dotel.javaagent.extensions=${otelAgentPathProvider.digmaExtensionPath}")
@@ -147,6 +149,12 @@ open class JavaToolOptionsBuilder(
                 .append("-Ddigma.autoinstrument.packages=${SettingsState.getInstance().extendedObservability}")
                 .append(" ")
         }
+
+        if (!SettingsState.getInstance().extendedObservabilityExcludes.isNullOrBlank()) {
+            javaToolOptions
+                .append("-Ddigma.autoinstrument.packages.exclude.names=${SettingsState.getInstance().extendedObservabilityExcludes}")
+                .append(" ")
+        }
         return this
     }
 
@@ -254,9 +262,18 @@ open class JavaToolOptionsBuilder(
 
 
     open fun build(): String {
+        disableExtendedObservabilityExtension()
         return javaToolOptions.toString()
     }
 
+    //todo: disabled in this branch because it uses Digma agent that injects @WithSpan instead of the
+    // digma-methods instrumentation module.
+    private fun disableExtendedObservabilityExtension() {
+        javaToolOptions
+            .append("-Dotel.instrumentation.digma-methods.enabled=false")
+            .append(" ")
+    }
+
 
     open fun getExporterUrl(): String {
         return SettingsState.getInstance().runtimeObservabilityBackendUrl

jvm-common/src/main/kotlin/org/digma/intellij/plugin/idea/execution/OTELJarProvider.kt

@@ -17,9 +17,27 @@ private const val RESOURCE_LOCATION = "otelJars"
 
 private const val OTEL_AGENT_JAR_PROP_NAME = "otel-agent"
 private const val DIGMA_EXTENSION_JAR_PROP_NAME = "digma-extension"
+private const val DIGMA_AGENT_JAR_PROP_NAME = "digma-agent"
 private const val OTEL_AGENT_JAR_NAME = "opentelemetry-javaagent.jar"
 private const val DIGMA_AGENT_EXTENSION_JAR_NAME = "digma-otel-agent-extension.jar"
-
+private const val DIGMA_AGENT_JAR_NAME = "digma-agent.jar"
+
+
+/**
+ * Downloads and provides the instrumentation jars.
+ *
+ * jars urls and path may be overridden for development to another download url or a local path,
+ * for example when developing the digma-agent or otel extension.
+ * system properties can be injected in runIde task or in idea custom properties.
+ *
+ * to override the download url see the method tryDownloadLatest, it will try to get the url from system
+ * property if exists. this is useful for example for downloading a different version of some of the jars.
+ *
+ * to override the final path see org.digma.intellij.plugin.idea.execution.OtelAgentPathProvider, it will
+ * first check if system property exists and use it instead of the default. this is useful when developing
+ * the digma agent or otel extension and you want to use the local build jar.
+ *
+ */
 
 //Do not change to light service because it will always register.
 // we want it to register only in Idea.
@@ -63,9 +81,9 @@ class OTELJarProvider {
 
     fun getDigmaAgentExtensionJarPath(): String? {
         ensureFilesExist()
-        val digmaJar = getDigmaAgentExtensionJar()
-        if (digmaJar.exists()) {
-            return digmaJar.absolutePath
+        val digmaExtensionJar = getDigmaAgentExtensionJar()
+        if (digmaExtensionJar.exists()) {
+            return digmaExtensionJar.absolutePath
         }
         return null
     }
@@ -74,6 +92,19 @@ class OTELJarProvider {
         return File(downloadDir, DIGMA_AGENT_EXTENSION_JAR_NAME)
     }
 
+    fun getDigmaAgentJarPath(): String? {
+        ensureFilesExist()
+        val digmaAgentJar = getDigmaAgentJar()
+        if (digmaAgentJar.exists()) {
+            return digmaAgentJar.absolutePath
+        }
+        return null
+    }
+
+    private fun getDigmaAgentJar(): File {
+        return File(downloadDir, DIGMA_AGENT_JAR_NAME)
+    }
+
 
     private fun ensureFilesExist() {
 
@@ -89,8 +120,9 @@ class OTELJarProvider {
 
     private fun filesExist(): Boolean {
         val otelJar = getOtelAgentJar()
-        val digmaJar = getDigmaAgentExtensionJar()
-        return otelJar.exists() && digmaJar.exists()
+        val digmaExtensionJar = getDigmaAgentExtensionJar()
+        val digmaAgentJar = getDigmaAgentJar()
+        return otelJar.exists() && digmaExtensionJar.exists() && digmaAgentJar.exists()
     }
 
 
@@ -109,6 +141,7 @@ class OTELJarProvider {
                 if (downloadDir.exists()) {
                     copyFileFromResource(OTEL_AGENT_JAR_NAME)
                     copyFileFromResource(DIGMA_AGENT_EXTENSION_JAR_NAME)
+                    copyFileFromResource(DIGMA_AGENT_JAR_NAME)
                     Log.log(logger::info, "otel agent jars unpacked to {}", downloadDir)
                 }
             } catch (e: Exception) {
@@ -143,9 +176,11 @@ class OTELJarProvider {
             try {
                 val otelUrl = System.getProperty("org.digma.otel.agentUrl", jarsUrls.getProperty(OTEL_AGENT_JAR_PROP_NAME))
                 val extensionUrl = System.getProperty("org.digma.otel.extensionUrl", jarsUrls.getProperty(DIGMA_EXTENSION_JAR_PROP_NAME))
+                val digmaAgentUrl = System.getProperty("org.digma.otel.digmaAgentUrl", jarsUrls.getProperty(DIGMA_AGENT_JAR_PROP_NAME))
 
                 downloadAndCopyJar(URL(otelUrl), getOtelAgentJar())
                 downloadAndCopyJar(URL(extensionUrl), getDigmaAgentExtensionJar())
+                downloadAndCopyJar(URL(digmaAgentUrl), getDigmaAgentJar())
             } catch (e: Exception) {
                 Log.warnWithException(logger, e, "could not download latest otel jars")
             }

jvm-common/src/main/kotlin/org/digma/intellij/plugin/idea/execution/OtelAgentPathProvider.kt

@@ -13,21 +13,30 @@ class OtelAgentPathProvider(configuration: RunConfiguration) {
 
     val otelAgentPath: String?
     val digmaExtensionPath: String?
+    val digmaAgentPath: String?
 
     init {
-        val tmpAgentPath = service<OTELJarProvider>().getOtelAgentJarPath()
-        val tmpDigmaExtensionPath = service<OTELJarProvider>().getDigmaAgentExtensionJarPath()
-        if (tmpAgentPath != null && tmpDigmaExtensionPath != null) {
+
+        //paths can be overrider with system properties for development purposes
+        val tmpOtelAgentPath = System.getProperty("digma.otel.agent.override.path", service<OTELJarProvider>().getOtelAgentJarPath())
+        val tmpDigmaExtensionPath =
+            System.getProperty("digma.otel.extension.override.path", service<OTELJarProvider>().getDigmaAgentExtensionJarPath())
+        val tmpDigmaAgentPath = System.getProperty("digma.agent.override.path", service<OTELJarProvider>().getDigmaAgentJarPath())
+
+        if (tmpOtelAgentPath != null && tmpDigmaExtensionPath != null && tmpDigmaAgentPath != null) {
             if (isWsl(configuration)) {
-                otelAgentPath = FileUtils.convertWinToWslPath(tmpAgentPath)
+                otelAgentPath = FileUtils.convertWinToWslPath(tmpOtelAgentPath)
                 digmaExtensionPath = FileUtils.convertWinToWslPath(tmpDigmaExtensionPath)
+                digmaAgentPath = FileUtils.convertWinToWslPath(tmpDigmaAgentPath)
             } else {
-                otelAgentPath = tmpAgentPath
+                otelAgentPath = tmpOtelAgentPath
                 digmaExtensionPath = tmpDigmaExtensionPath
+                digmaAgentPath = tmpDigmaAgentPath
             }
         } else {
             otelAgentPath = null
             digmaExtensionPath = null
+            digmaAgentPath = null
         }
 
     }
@@ -67,7 +76,7 @@ class OtelAgentPathProvider(configuration: RunConfiguration) {
 
 
     fun hasAgentPath(): Boolean {
-        return otelAgentPath != null && digmaExtensionPath != null
+        return otelAgentPath != null && digmaExtensionPath != null && digmaAgentPath != null
     }
 
 }

jvm-common/src/main/resources/jars-urls.properties

@@ -1,2 +1,3 @@
 otel-agent=https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v2.1.0/opentelemetry-javaagent.jar
-digma-extension=https://github.com/digma-ai/otel-java-instrumentation/releases/latest/download/digma-otel-agent-extension.jar
+digma-extension=https://github.com/digma-ai/otel-java-instrumentation/releases/latest/download/digma-otel-agent-extension.jar
+digma-agent=https://github.com/digma-ai/digma-agent/releases/latest/download/digma-agent.jar