[Lambda Java v2.11.x] Merge All Code Changes from v1.33.x Branch into v2.11.x (#1114)

This change merges all private Lambda Java updates from the v1.33 branch
into the v2.11.x branch. I performed a 'git rebase v2.11' on the v1.33
branch, reviewed all changes, and completed the build and testing
process. The resulting Lambda layer generated trace data identical to
the version built directly from the v2.11.x branch (excluding this PR).

Here is the list of all migrated PRs:

Build layer during CI/CD workflows + some minor refactoring #989 

support java11 runtime for lambda #1001 

Unique artifact names for upload and merge for download #1014 

Bug fixes] Lambda - duplicate lambda spans + appsignals from unsampled
spans #1000

Fix: Lambda Topology Issue (#1016)
 
Fix: Lambda Topology Issue (#1016) #1085
 
feat: Support microservice span in Lambda Java environment. #1053

Test
Tested Java11, 17, and 21 Lambda functions. Manually tested PR-1000 and
PR-1053. Both work as expected in the v2.11 branch. MicroService
(SpringBoot) support works well. I verified attribute
Trace.lambda.multiple server can be found in the Lambda server span,
once we have Servlet instrumentation enabled with
OTEL_INSTRUMENTATION_SERVLET_ENABLED.

Note: The changes in the patch files are not included in this PR. They
should have been reviewed and incorporated as part of this migration:

Upgrade Java Lambda Layer to 2.x #1076 

Lambda with SpringBoot MicroService:
<img width="1367" alt="lambda"
src="https://github.com/user-attachments/assets/5cf5be29-4986-454c-b61b-773d6cde3848"
/>

Service Map and added microservice attribute 'Trace.lambda.multiple
server'.
<img width="1864" alt="traceMap"
src="https://github.com/user-attachments/assets/f7ff1771-61f0-4013-b571-90370a726aa9"
/>

AppSignals
<img width="1875" alt="appSignals"
src="https://github.com/user-attachments/assets/24f1b3a8-851c-4c97-bb50-087ee275b86d"
/>



By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache 2.0 license.

Author: lukeina2z

.github/workflows/main-build.yml

@@ -256,7 +256,7 @@ jobs:
           aws s3 cp ./build/distributions/aws-opentelemetry-java-layer.zip s3://adot-main-build-staging-jar/adot-java-lambda-layer-${{ github.run_id }}.zip
 
   application-signals-e2e-test:
-    needs: [build]
+    needs: [build, application-signals-lambda-layer-build]
     uses: ./.github/workflows/application-signals-e2e-test.yml
     secrets: inherit
     with:

.github/workflows/release-lambda.yml

@@ -109,7 +109,7 @@ jobs:
             aws lambda publish-layer-version \
               --layer-name ${{ env.LAYER_NAME }} \
               --content S3Bucket=${{ env.BUCKET_NAME }},S3Key=aws-opentelemetry-java-layer.zip \
-              --compatible-runtimes java17 java21 \
+              --compatible-runtimes java11 java17 java21 \
               --compatible-architectures "arm64" "x86_64" \
               --license-info "Apache-2.0" \
               --description "AWS Distro of OpenTelemetry Lambda Layer for Java Runtime" \
@@ -140,7 +140,7 @@ jobs:
         if: ${{ success() }}
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ env.LAYER_NAME }}
+          name: ${{ env.LAYER_NAME }}-${{ matrix.aws_region }}
           path: ${{ env.LAYER_NAME }}/${{ matrix.aws_region }}
 
       - name: clean s3
@@ -154,20 +154,24 @@ jobs:
     steps:
       - name: Checkout Repo @ SHA - ${{ github.sha }}
         uses: actions/checkout@v4
+
       - uses: hashicorp/setup-terraform@v2
+
       - name: download layerARNs
         uses: actions/download-artifact@v4
         with:
           pattern: ${{ env.LAYER_NAME }}-*
           path: ${{ env.LAYER_NAME }}
           merge-multiple: true
+
       - name: show layerARNs
         run: |
           for file in ${{ env.LAYER_NAME }}/*
           do
           echo $file
           cat $file
           done
+
       - name: generate layer-note
         working-directory: ${{ env.LAYER_NAME }}
         run: |

awsagentprovider/src/main/java/software/amazon/opentelemetry/javaagent/providers/AwsApplicationSignalsCustomizerProvider.java

@@ -283,6 +283,10 @@ private Sampler customizeSampler(Sampler sampler, ConfigProperties configProps)
 
   private SdkTracerProviderBuilder customizeTracerProviderBuilder(
       SdkTracerProviderBuilder tracerProviderBuilder, ConfigProperties configProps) {
+    if (isLambdaEnvironment()) {
+      tracerProviderBuilder.addSpanProcessor(new AwsLambdaSpanProcessor());
+    }
+
     if (isApplicationSignalsEnabled(configProps)) {
       logger.info("AWS Application Signals enabled");
       Duration exportInterval =
@@ -294,9 +298,27 @@ private SdkTracerProviderBuilder customizeTracerProviderBuilder(
 
       // If running on Lambda, we just need to export 100% spans and skip generating any Application
       // Signals metrics.
-      if (isLambdaEnvironment()) {
+      if (isLambdaEnvironment()
+          && System.getenv(OTEL_EXPORTER_OTLP_TRACES_ENDPOINT_CONFIG) == null) {
+        String tracesEndpoint =
+            Optional.ofNullable(System.getenv(AWS_XRAY_DAEMON_ADDRESS_CONFIG))
+                .orElse(DEFAULT_UDP_ENDPOINT);
+        SpanExporter spanExporter =
+            new OtlpUdpSpanExporterBuilder()
+                .setPayloadSampleDecision(TracePayloadSampleDecision.UNSAMPLED)
+                .setEndpoint(tracesEndpoint)
+                .build();
+
+        // Wrap the udp exporter with the AwsMetricsAttributesSpanExporter to add Application
+        // Signals attributes to unsampled spans too
+        SpanExporter appSignalsSpanExporter =
+            AwsMetricAttributesSpanExporterBuilder.create(
+                    spanExporter, ResourceHolder.getResource())
+                .build();
+
         tracerProviderBuilder.addSpanProcessor(
             AwsUnsampledOnlySpanProcessorBuilder.create()
+                .setSpanExporter(appSignalsSpanExporter)
                 .setMaxExportBatchSize(LAMBDA_SPAN_EXPORT_BATCH_SIZE)
                 .build());
         return tracerProviderBuilder;

awsagentprovider/src/main/java/software/amazon/opentelemetry/javaagent/providers/AwsAttributeKeys.java

@@ -97,4 +97,7 @@ private AwsAttributeKeys() {}
       AttributeKey.stringKey("aws.bedrock.guardrail.id");
   static final AttributeKey<String> AWS_GUARDRAIL_ARN =
       AttributeKey.stringKey("aws.bedrock.guardrail.arn");
+
+  static final AttributeKey<Boolean> AWS_TRACE_LAMBDA_MULTIPLE_SERVER =
+      AttributeKey.booleanKey("aws.trace.lambda.multiple-server");
 }

awsagentprovider/src/main/java/software/amazon/opentelemetry/javaagent/providers/AwsLambdaSpanProcessor.java

@@ -0,0 +1,55 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.opentelemetry.javaagent.providers;
+
+import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.context.Context;
+import io.opentelemetry.sdk.trace.ReadWriteSpan;
+import io.opentelemetry.sdk.trace.ReadableSpan;
+import io.opentelemetry.sdk.trace.SpanProcessor;
+import javax.annotation.concurrent.Immutable;
+
+@Immutable
+public final class AwsLambdaSpanProcessor implements SpanProcessor {
+  @Override
+  public void onStart(Context parentContext, ReadWriteSpan span) {
+    if (AwsSpanProcessingUtil.isServletServerSpan(span)) {
+      Span parentSpan = Span.fromContextOrNull(parentContext);
+      if (parentSpan == null || !(parentSpan instanceof ReadWriteSpan)) {
+        return;
+      }
+
+      ReadWriteSpan parentReadWriteSpan = (ReadWriteSpan) parentSpan;
+      if (!AwsSpanProcessingUtil.isLambdaServerSpan(parentReadWriteSpan)) {
+        return;
+      }
+      parentReadWriteSpan.setAttribute(AwsAttributeKeys.AWS_TRACE_LAMBDA_MULTIPLE_SERVER, true);
+    }
+  }
+
+  @Override
+  public boolean isStartRequired() {
+    return true;
+  }
+
+  @Override
+  public void onEnd(ReadableSpan span) {}
+
+  @Override
+  public boolean isEndRequired() {
+    return false;
+  }
+}

awsagentprovider/src/main/java/software/amazon/opentelemetry/javaagent/providers/AwsSpanProcessingUtil.java

@@ -37,6 +37,7 @@
 import io.opentelemetry.api.trace.SpanContext;
 import io.opentelemetry.api.trace.SpanKind;
 import io.opentelemetry.sdk.common.InstrumentationScopeInfo;
+import io.opentelemetry.sdk.trace.ReadableSpan;
 import io.opentelemetry.sdk.trace.data.SpanData;
 import java.io.IOException;
 import java.io.InputStream;
@@ -68,6 +69,10 @@ final class AwsSpanProcessingUtil {
 
   private static final String SQL_DIALECT_KEYWORDS_JSON = "configuration/sql_dialect_keywords.json";
 
+  static final AttributeKey<String> OTEL_SCOPE_NAME = AttributeKey.stringKey("otel.scope.name");
+  static final String LAMBDA_SCOPE_PREFIX = "io.opentelemetry.aws-lambda-";
+  static final String SERVLET_SCOPE_PREFIX = "io.opentelemetry.servlet-";
+
   static List<String> getDialectKeywords() {
     try (InputStream jsonFile =
         AwsSpanProcessingUtil.class
@@ -91,6 +96,10 @@ static List<String> getDialectKeywords() {
    */
   static String getIngressOperation(SpanData span) {
     if (isLambdaEnvironment()) {
+      String op = generateIngressOperation(span);
+      if (!op.equals(UNKNOWN_OPERATION)) {
+        return op;
+      }
       return System.getenv(AWS_LAMBDA_FUNCTION_NAME_CONFIG) + "/FunctionHandler";
     }
     String operation = span.getName();
@@ -248,4 +257,30 @@ static boolean isDBSpan(SpanData span) {
         || isKeyPresent(span, DB_OPERATION)
         || isKeyPresent(span, DB_STATEMENT);
   }
+
+  static boolean isLambdaServerSpan(ReadableSpan span) {
+    String scopeName = null;
+    if (span != null
+        && span.toSpanData() != null
+        && span.toSpanData().getInstrumentationScopeInfo() != null) {
+      scopeName = span.toSpanData().getInstrumentationScopeInfo().getName();
+    }
+
+    return scopeName != null
+        && scopeName.startsWith(LAMBDA_SCOPE_PREFIX)
+        && SpanKind.SERVER == span.getKind();
+  }
+
+  static boolean isServletServerSpan(ReadableSpan span) {
+    String scopeName = null;
+    if (span != null
+        && span.toSpanData() != null
+        && span.toSpanData().getInstrumentationScopeInfo() != null) {
+      scopeName = span.toSpanData().getInstrumentationScopeInfo().getName();
+    }
+
+    return scopeName != null
+        && scopeName.startsWith(SERVLET_SCOPE_PREFIX)
+        && SpanKind.SERVER == span.getKind();
+  }
 }

awsagentprovider/src/test/java/software/amazon/opentelemetry/javaagent/providers/AwsLambdaSpanProcessorTest.java

@@ -0,0 +1,140 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.opentelemetry.javaagent.providers;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.mockito.Mockito.*;
+
+import io.opentelemetry.api.common.AttributeKey;
+import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.api.trace.SpanContext;
+import io.opentelemetry.api.trace.SpanKind;
+import io.opentelemetry.api.trace.Tracer;
+import io.opentelemetry.context.Context;
+import io.opentelemetry.sdk.common.InstrumentationScopeInfo;
+import io.opentelemetry.sdk.trace.ReadWriteSpan;
+import io.opentelemetry.sdk.trace.ReadableSpan;
+import io.opentelemetry.sdk.trace.SdkTracerProvider;
+import io.opentelemetry.sdk.trace.data.SpanData;
+import java.util.Map;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+class AwsLambdaSpanProcessorTest {
+
+  private AwsLambdaSpanProcessor processor;
+  private ReadWriteSpan mockLambdaServerSpan;
+  private SpanData mockLambdaSpanData;
+  private InstrumentationScopeInfo mockLambdaScopeInfo;
+  private Map<AttributeKey<?>, Object> attributeMapForLambdaSpan;
+  private SpanContext mockSpanContext;
+
+  private ReadWriteSpan mockServletServerSpan;
+  private SpanData mockServletSpanData;
+  private InstrumentationScopeInfo mockServletScopeInfo;
+
+  private Tracer lambdaTracer;
+  private Tracer servletTracer;
+  private Tracer otherTracer;
+
+  @BeforeEach
+  public void setup() {
+    processor = new AwsLambdaSpanProcessor();
+    lambdaTracer =
+        SdkTracerProvider.builder()
+            .addSpanProcessor(processor)
+            .build()
+            .get(AwsSpanProcessingUtil.LAMBDA_SCOPE_PREFIX + "core-1.0");
+
+    servletTracer =
+        SdkTracerProvider.builder()
+            .addSpanProcessor(processor)
+            .build()
+            .get(AwsSpanProcessingUtil.SERVLET_SCOPE_PREFIX + "lib-3.0");
+
+    otherTracer =
+        SdkTracerProvider.builder().addSpanProcessor(processor).build().get("other-lib-2.0");
+  }
+
+  @Test
+  void testOnStart_servletServerSpan_withLambdaServerSpan() {
+    Span parentSpan =
+        lambdaTracer.spanBuilder("parent-lambda").setSpanKind(SpanKind.SERVER).startSpan();
+    servletTracer
+        .spanBuilder("child-servlet")
+        .setSpanKind(SpanKind.SERVER)
+        .setParent(Context.current().with(parentSpan))
+        .startSpan();
+
+    ReadableSpan parentReadableSpan = (ReadableSpan) parentSpan;
+    assertThat(parentReadableSpan.getAttribute(AwsAttributeKeys.AWS_TRACE_LAMBDA_MULTIPLE_SERVER))
+        .isEqualTo(true);
+  }
+
+  @Test
+  void testOnStart_servletInternalSpan_withLambdaServerSpan() {
+    Span parentSpan =
+        lambdaTracer.spanBuilder("parent-lambda").setSpanKind(SpanKind.SERVER).startSpan();
+
+    servletTracer
+        .spanBuilder("child-servlet")
+        .setSpanKind(SpanKind.INTERNAL)
+        .setParent(Context.current().with(parentSpan))
+        .startSpan();
+
+    ReadableSpan parentReadableSpan = (ReadableSpan) parentSpan;
+    assertNull(parentReadableSpan.getAttribute(AwsAttributeKeys.AWS_TRACE_LAMBDA_MULTIPLE_SERVER));
+  }
+
+  @Test
+  void testOnStart_servletServerSpan_withLambdaInternalSpan() {
+    Span parentSpan =
+        lambdaTracer.spanBuilder("parent-lambda").setSpanKind(SpanKind.INTERNAL).startSpan();
+
+    servletTracer
+        .spanBuilder("child-servlet")
+        .setSpanKind(SpanKind.SERVER)
+        .setParent(Context.current().with(parentSpan))
+        .startSpan();
+
+    ReadableSpan parentReadableSpan = (ReadableSpan) parentSpan;
+    assertNull(parentReadableSpan.getAttribute(AwsAttributeKeys.AWS_TRACE_LAMBDA_MULTIPLE_SERVER));
+  }
+
+  @Test
+  void testOnStart_servletServerSpan_withLambdaServerSpanAsGrandParent() {
+    Span grandParentSpan =
+        lambdaTracer.spanBuilder("grandparent-lambda").setSpanKind(SpanKind.SERVER).startSpan();
+
+    Span parentSpan =
+        otherTracer
+            .spanBuilder("parent-other")
+            .setSpanKind(SpanKind.SERVER)
+            .setParent(Context.current().with(grandParentSpan))
+            .startSpan();
+
+    servletTracer
+        .spanBuilder("child-servlet")
+        .setSpanKind(SpanKind.SERVER)
+        .setParent(Context.current().with(parentSpan))
+        .startSpan();
+
+    ReadableSpan grandParentReadableSpan = (ReadableSpan) grandParentSpan;
+    assertNull(
+        grandParentReadableSpan.getAttribute(AwsAttributeKeys.AWS_TRACE_LAMBDA_MULTIPLE_SERVER));
+  }
+}