Skip to content

Commit 7b0f431

Browse files
bardabunbardabun
authored
Feat(lambda): Refactor Extension to a Unified Collector-Based Layer (#10)
* feat(lambda): Refactor extension to embedded collector model * Refactor Extension to a Unified Collector-Based Layer
1 parent cc170a4 commit 7b0f431

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+3615
-89
lines changed

‎.github/workflows/e2e-go.yml‎

Lines changed: 217 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,217 @@
1+
name: E2E - Go Layer
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
logzio_api_url:
7+
description: "Logz.io API base URL (default https://api.logz.io)"
8+
required: false
9+
default: "https://api.logz.io"
10+
aws_region:
11+
description: "AWS Region"
12+
required: false
13+
default: "us-east-1"
14+
15+
permissions:
16+
contents: read
17+
18+
env:
19+
AWS_REGION: ${{ inputs.aws_region || 'us-east-1' }}
20+
AWS_DEFAULT_REGION: ${{ inputs.aws_region || 'us-east-1' }}
21+
ARCHITECTURE: amd64
22+
FUNCTION_NAME: one-layer-e2e-test-go
23+
LAYER_BASE_NAME: otel-go-extension-e2e
24+
SERVICE_NAME: logzio-e2e-go-service
25+
LOGZIO_REGION: us
26+
27+
jobs:
28+
build-layer:
29+
runs-on: ubuntu-latest
30+
steps:
31+
- name: Checkout
32+
uses: actions/checkout@v5
33+
34+
- name: Set up Go (for Collector)
35+
uses: actions/setup-go@v5
36+
with:
37+
go-version-file: collector/go.mod
38+
39+
- name: Build combined Go layer (amd64)
40+
run: |
41+
cd go
42+
ARCHITECTURE=${ARCHITECTURE} ./build-combined.sh
43+
44+
- name: Upload layer artifact
45+
uses: actions/upload-artifact@v4
46+
with:
47+
name: otel-go-extension-layer.zip
48+
path: go/build/otel-go-extension-layer.zip
49+
50+
publish-update-invoke:
51+
runs-on: ubuntu-latest
52+
needs: build-layer
53+
outputs:
54+
layer_arn: ${{ steps.publish.outputs.layer_arn }}
55+
e2e_label: ${{ steps.vars.outputs.e2e_label }}
56+
steps:
57+
- name: Checkout
58+
uses: actions/checkout@v5
59+
60+
- name: Download layer artifact
61+
uses: actions/download-artifact@v4
62+
with:
63+
name: otel-go-extension-layer.zip
64+
65+
- name: Configure AWS (User Credentials)
66+
uses: aws-actions/configure-aws-credentials@v4
67+
with:
68+
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
69+
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
70+
aws-region: ${{ env.AWS_REGION }}
71+
72+
- name: Publish layer version
73+
id: publish
74+
shell: bash
75+
run: |
76+
set -euo pipefail
77+
LAYER_NAME="${LAYER_BASE_NAME}-amd64"
78+
ARN=$(aws lambda publish-layer-version \
79+
--layer-name "$LAYER_NAME" \
80+
--license-info "Apache-2.0" \
81+
--compatible-architectures x86_64 \
82+
--compatible-runtimes provided provided.al2 \
83+
--zip-file fileb://otel-go-extension-layer.zip \
84+
--query 'LayerVersionArn' --output text)
85+
echo "layer_arn=$ARN" >> "$GITHUB_OUTPUT"
86+
87+
- name: Prepare variables
88+
id: vars
89+
run: |
90+
echo "e2e_label=go-e2e-${GITHUB_RUN_ID}" >> "$GITHUB_OUTPUT"
91+
92+
- name: Check function exists and get current config
93+
run: |
94+
echo "Checking if function exists and its current configuration..."
95+
aws lambda get-function-configuration --function-name "${FUNCTION_NAME}" --query '{Role:Role,KMSKeyArn:KMSKeyArn,State:State,LastUpdateStatus:LastUpdateStatus}' --output table || {
96+
echo "❌ Function ${FUNCTION_NAME} does not exist or is not accessible."
97+
exit 1
98+
}
99+
100+
echo "Current environment variables:"
101+
aws lambda get-function-configuration --function-name "${FUNCTION_NAME}" --query 'Environment.Variables' --output json || echo "No environment variables set"
102+
103+
- name: Update Lambda configuration with current run's layer and env vars
104+
run: |
105+
echo "Updating function configuration with this run's published layer and environment variables..."
106+
aws lambda update-function-configuration \
107+
--function-name "${FUNCTION_NAME}" \
108+
--layers "${{ steps.publish.outputs.layer_arn }}" \
109+
--environment "Variables={OPENTELEMETRY_COLLECTOR_CONFIG_URI=/opt/collector-config/config.e2e.yaml,OTEL_SERVICE_NAME=${SERVICE_NAME},OTEL_TRACES_SAMPLER=always_on,OTEL_RESOURCE_ATTRIBUTES=deployment.environment=${{ steps.vars.outputs.e2e_label }},ENVIRONMENT=${{ steps.vars.outputs.e2e_label }},LOGZIO_REGION=${LOGZIO_REGION},LOGZIO_LOGS_TOKEN=${{ secrets.LOGZIO_LOGS_TOKEN }},LOGZIO_TRACES_TOKEN=${{ secrets.LOGZIO_TRACES_TOKEN }},LOGZIO_METRICS_TOKEN=${{ secrets.LOGZIO_METRICS_TOKEN }}}"
110+
111+
echo "Waiting for function update to complete..."
112+
aws lambda wait function-updated --function-name "${FUNCTION_NAME}"
113+
114+
echo "Updated configuration (layers and environment variables):"
115+
aws lambda get-function-configuration --function-name "${FUNCTION_NAME}" --query '{Layers:Layers[].Arn,Environment:Environment.Variables}' --output json
116+
117+
- name: Invoke function multiple times
118+
run: |
119+
echo "Invoking function first time..."
120+
aws lambda invoke --function-name "${FUNCTION_NAME}" --payload '{}' --cli-binary-format raw-in-base64-out response1.json
121+
echo "First invocation response:"
122+
cat response1.json
123+
echo ""
124+
125+
echo "Invoking function second time..."
126+
aws lambda invoke --function-name "${FUNCTION_NAME}" --payload '{}' --cli-binary-format raw-in-base64-out response2.json
127+
echo "Second invocation response:"
128+
cat response2.json
129+
echo ""
130+
131+
echo "Sleeping for 5 seconds before additional invocations..."
132+
sleep 5
133+
134+
echo "Invoking function third time..."
135+
aws lambda invoke --function-name "${FUNCTION_NAME}" --payload '{}' --cli-binary-format raw-in-base64-out response3.json
136+
echo "Third invocation response:"
137+
cat response3.json
138+
echo ""
139+
140+
echo "Invoking function fourth time..."
141+
aws lambda invoke --function-name "${FUNCTION_NAME}" --payload '{}' --cli-binary-format raw-in-base64-out response4.json
142+
echo "Fourth invocation response:"
143+
cat response4.json
144+
echo ""
145+
146+
echo "Invoking function fifth time..."
147+
aws lambda invoke --function-name "${FUNCTION_NAME}" --payload '{}' --cli-binary-format raw-in-base64-out response5.json
148+
echo "Fifth invocation response:"
149+
cat response5.json
150+
echo ""
151+
152+
- name: Check CloudWatch logs
153+
run: |
154+
echo "Checking recent CloudWatch logs for the function..."
155+
LOG_GROUP_NAME="/aws/lambda/${FUNCTION_NAME}"
156+
157+
# Get recent log events (last 5 minutes)
158+
aws logs filter-log-events \
159+
--log-group-name "$LOG_GROUP_NAME" \
160+
--start-time $(date -d '5 minutes ago' +%s)000 \
161+
--query 'events[].message' \
162+
--output text || {
163+
echo "❌ Could not fetch CloudWatch logs. Log group might not exist or no recent logs."
164+
echo "Checking if log group exists..."
165+
aws logs describe-log-groups --log-group-name-prefix "$LOG_GROUP_NAME" --query 'logGroups[].logGroupName' --output text
166+
}
167+
168+
verify-e2e:
169+
runs-on: ubuntu-latest
170+
needs: publish-update-invoke
171+
steps:
172+
- name: Checkout
173+
uses: actions/checkout@v5
174+
175+
- name: Set up Go
176+
uses: actions/setup-go@v5
177+
with:
178+
go-version: '1.21'
179+
180+
- name: Run E2E verification tests
181+
working-directory: e2e_tests
182+
env:
183+
LOGZIO_API_KEY: ${{ secrets.LOGZIO_API_KEY }}
184+
LOGZIO_API_URL: ${{ inputs.logzio_api_url || 'https://api.logz.io' }}
185+
LOGZIO_API_METRICS_KEY: ${{ secrets.LOGZIO_API_METRICS_KEY }}
186+
LOGZIO_METRICS_QUERY_URL: ${{ inputs.logzio_api_url || 'https://api.logz.io' }}
187+
LOGZIO_API_TRACES_KEY: ${{ secrets.LOGZIO_API_TRACES_KEY }}
188+
E2E_TEST_ENVIRONMENT_LABEL: ${{ needs.publish-update-invoke.outputs.e2e_label }}
189+
EXPECTED_LAMBDA_FUNCTION_NAME: one-layer-e2e-test-go
190+
EXPECTED_SERVICE_NAME: ${{ env.SERVICE_NAME }}
191+
GITHUB_RUN_ID: ${{ github.run_id }}
192+
AWS_REGION: ${{ env.AWS_REGION }}
193+
run: |
194+
go mod tidy
195+
go test ./... -v -tags=e2e -run TestE2ERunner
196+
197+
cleanup:
198+
if: always()
199+
runs-on: ubuntu-latest
200+
needs: [publish-update-invoke, verify-e2e]
201+
steps:
202+
- name: Configure AWS (User Credentials)
203+
uses: aws-actions/configure-aws-credentials@v4
204+
with:
205+
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
206+
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
207+
aws-region: ${{ inputs.aws_region || 'us-east-1' }}
208+
- name: Delete published layer version
209+
if: ${{ needs.publish-update-invoke.outputs.layer_arn != '' }}
210+
shell: bash
211+
run: |
212+
ARN="${{ needs.publish-update-invoke.outputs.layer_arn }}"
213+
LAYER_NAME=$(echo "$ARN" | cut -d: -f7)
214+
LAYER_VERSION=$(echo "$ARN" | cut -d: -f8)
215+
aws lambda delete-layer-version --layer-name "$LAYER_NAME" --version-number "$LAYER_VERSION" || echo "Failed to delete layer version."
216+
217+

0 commit comments

Comments
 (0)