Skip to content

Commit 2a22ab0

Browse files
prateekchaudhryprateekchaudhry
authored
Add Placeholder for building Version 3.0 (#990)
* update buildspec.yml - rebase * add placeholder for al2023 image in load tests * add placeholder for al2023 image in integ tests
1 parent 53982a6 commit 2a22ab0

File tree

3 files changed

+156
-82
lines changed

3 files changed

+156
-82
lines changed

‎buildspec.yml

Lines changed: 71 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -7,57 +7,83 @@ phases:
77
pre_build:
88
commands:
99
- echo Building the AWS for Fluent Bit image
10+
# Check BUILD_VERSION environment variable and set default if not provided
11+
- |
12+
if [ -z "$BUILD_VERSION" ]; then
13+
echo "BUILD_VERSION environment variable not provided, defaulting to BUILD_VERSION=2"
14+
export BUILD_VERSION=2
15+
else
16+
echo "BUILD_VERSION is set to: $BUILD_VERSION"
17+
fi
1018
build:
1119
commands:
12-
# Check latest image versions from dockerhub and from GitHub source file
13-
- './scripts/publish.sh cicd-check-image-version'
14-
# Disable buildkit features
15-
- export DOCKER_BUILDKIT=0
16-
# Command to build debug image
17-
- make debug
18-
# Command to build your project
19-
- make release
20+
# Execute build commands based on BUILD_VERSION
21+
- |
22+
if [ "$BUILD_VERSION" = "2" ]; then
23+
echo "Building AWS for Fluent Bit version 2 (current behavior)"
24+
25+
# Check latest image versions from dockerhub and from GitHub source file
26+
./scripts/publish.sh cicd-check-image-version
27+
28+
# Disable buildkit features
29+
export DOCKER_BUILDKIT=0
2030
21-
# List the docker images
22-
- docker images
31+
# Command to build debug image
32+
make debug
33+
# Command to build your project
34+
make release
2335
24-
# Push the image to ECR with corresponding architecture as the tag.
25-
- aws ecr get-login-password --region ${AWS_REGION}| docker login --username AWS --password-stdin ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com
26-
- aws ecr create-repository --repository-name amazon/aws-for-fluent-bit-test --image-scanning-configuration scanOnPush=true --region ${AWS_REGION} || true
27-
- architecture=$(docker inspect --format='{{.Architecture}}' amazon/aws-for-fluent-bit:latest-al2)
28-
- docker tag amazon/aws-for-fluent-bit:latest-al2 ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:$architecture
29-
- docker tag amazon/aws-for-fluent-bit:debug-al2 ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:$architecture-debug
30-
- docker push ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:$architecture
31-
- docker push ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:$architecture-debug
32-
- './scripts/publish.sh cicd-verify-ecr-image-scan ${AWS_REGION} amazon/aws-for-fluent-bit-test $architecture'
33-
- './scripts/publish.sh cicd-verify-ecr-image-scan ${AWS_REGION} amazon/aws-for-fluent-bit-test $architecture-debug'
34-
# Image with Init Process
35-
- docker tag amazon/aws-for-fluent-bit:init-latest-al2 ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:init-$architecture
36-
- docker tag amazon/aws-for-fluent-bit:init-debug-al2 ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:init-$architecture-debug
37-
- docker push ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:init-$architecture
38-
- docker push ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:init-$architecture-debug
39-
- './scripts/publish.sh cicd-verify-ecr-image-scan ${AWS_REGION} amazon/aws-for-fluent-bit-test init-$architecture'
40-
- './scripts/publish.sh cicd-verify-ecr-image-scan ${AWS_REGION} amazon/aws-for-fluent-bit-test init-$architecture-debug'
36+
# List the docker images
37+
docker images
4138
42-
# Create manifest list
43-
- export DOCKER_CLI_EXPERIMENTAL=enabled
44-
- docker manifest create ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:latest ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:arm64 ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:amd64 || true
45-
- docker manifest annotate --arch arm64 ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:latest ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:arm64 || true
46-
- docker manifest annotate --arch amd64 ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:latest ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:amd64 || true
47-
# Image with Init Process
48-
- docker manifest create ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:init-latest ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:init-arm64 ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:init-amd64 || true
49-
- docker manifest annotate --arch arm64 ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:init-latest ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:init-arm64 || true
50-
- docker manifest annotate --arch amd64 ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:init-latest ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:init-amd64 || true
51-
52-
# Sanity check for the debug log
53-
- docker manifest inspect ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:latest || true
54-
# Image with Init Process
55-
- docker manifest inspect ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:init-latest || true
39+
# Push the image to ECR with corresponding architecture as the tag.
40+
aws ecr get-login-password --region ${AWS_REGION}| docker login --username AWS --password-stdin ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com
41+
aws ecr create-repository --repository-name amazon/aws-for-fluent-bit-test --image-scanning-configuration scanOnPush=true --region ${AWS_REGION} || true
42+
architecture=$(docker inspect --format='{{.Architecture}}' amazon/aws-for-fluent-bit)
43+
docker tag amazon/aws-for-fluent-bit:latest ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:$architecture
44+
docker tag amazon/aws-for-fluent-bit:debug ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:$architecture-debug
45+
docker push ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:$architecture
46+
docker push ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:$architecture-debug
47+
./scripts/publish.sh cicd-verify-ecr-image-scan ${AWS_REGION} amazon/aws-for-fluent-bit-test $architecture
48+
./scripts/publish.sh cicd-verify-ecr-image-scan ${AWS_REGION} amazon/aws-for-fluent-bit-test $architecture-debug
49+
# Image with Init Process
50+
docker tag amazon/aws-for-fluent-bit:init-latest ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:init-$architecture
51+
docker tag amazon/aws-for-fluent-bit:init-debug ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:init-$architecture-debug
52+
docker push ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:init-$architecture
53+
docker push ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:init-$architecture-debug
54+
./scripts/publish.sh cicd-verify-ecr-image-scan ${AWS_REGION} amazon/aws-for-fluent-bit-test init-$architecture
55+
./scripts/publish.sh cicd-verify-ecr-image-scan ${AWS_REGION} amazon/aws-for-fluent-bit-test init-$architecture-debug
5656
57-
# Push manifest list
58-
- docker manifest push ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:latest || true
59-
# Image with Init Process
60-
- docker manifest push ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:init-latest || true
57+
# Create manifest list
58+
export DOCKER_CLI_EXPERIMENTAL=enabled
59+
docker manifest create ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:latest ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:arm64 ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:amd64 || true
60+
docker manifest annotate --arch arm64 ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:latest ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:arm64 || true
61+
docker manifest annotate --arch amd64 ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:latest ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:amd64 || true
62+
# Image with Init Process
63+
docker manifest create ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:init-latest ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:init-arm64 ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:init-amd64 || true
64+
docker manifest annotate --arch arm64 ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:init-latest ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:init-arm64 || true
65+
docker manifest annotate --arch amd64 ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:init-latest ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:init-amd64 || true
66+
67+
# Sanity check for the debug log
68+
docker manifest inspect ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:latest || true
69+
# Image with Init Process
70+
docker manifest inspect ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:init-latest || true
71+
72+
# Push manifest list
73+
docker manifest push ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:latest || true
74+
# Image with Init Process
75+
docker manifest push ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:init-latest || true
76+
77+
elif [ "$BUILD_VERSION" = "3" ]; then
78+
echo "Building AWS for Fluent Bit version 3 (no-op for now)"
79+
echo "Version 3 build logic will be implemented later"
80+
# Placeholder for future BUILD_VERSION=3 implementation
81+
82+
else
83+
echo "Unsupported BUILD_VERSION: $BUILD_VERSION"
84+
echo "Supported versions are: 2, 3"
85+
exit 1
86+
fi
6187
6288
artifacts:
6389
files:

‎buildspec_integ.yml

Lines changed: 41 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -8,26 +8,50 @@ phases:
88
- echo Running the integration test
99
build:
1010
commands:
11-
# Enforce STS regional endpoints
12-
- export AWS_STS_REGIONAL_ENDPOINTS=regional
13-
- export CW_INTEG_VALIDATOR_IMAGE_BASE=${AWS_ACCOUNT}.dkr.ecr.us-west-2.amazonaws.com/cw-integ-validator
14-
- export S3_INTEG_VALIDATOR_IMAGE_BASE=${AWS_ACCOUNT}.dkr.ecr.us-west-2.amazonaws.com/s3-integ-validator
15-
# Get the default credentials and set as environment variables
16-
- 'CREDS=`curl 169.254.170.2$AWS_CONTAINER_CREDENTIALS_RELATIVE_URI`'
17-
- 'export AWS_ACCESS_KEY_ID=`echo $CREDS | jq -r .AccessKeyId`'
18-
- 'export AWS_SECRET_ACCESS_KEY=`echo $CREDS | jq -r .SecretAccessKey`'
19-
- 'export AWS_SESSION_TOKEN=`echo $CREDS | jq -r .Token`'
11+
# Check BUILD_VERSION environment variable and set default if not provided
12+
- |
13+
if [ -z "$BUILD_VERSION" ]; then
14+
echo "BUILD_VERSION environment variable not provided, defaulting to BUILD_VERSION=2"
15+
export BUILD_VERSION=2
16+
else
17+
echo "BUILD_VERSION is set to: $BUILD_VERSION"
18+
fi
19+
# Execute integration test commands based on BUILD_VERSION
20+
- |
21+
if [ "$BUILD_VERSION" = "2" ]; then
22+
echo "Running integration tests for AWS for Fluent Bit version 2 (current behavior)"
23+
24+
# Enforce STS regional endpoints
25+
export AWS_STS_REGIONAL_ENDPOINTS=regional
26+
export CW_INTEG_VALIDATOR_IMAGE_BASE=${AWS_ACCOUNT}.dkr.ecr.us-west-2.amazonaws.com/cw-integ-validator
27+
export S3_INTEG_VALIDATOR_IMAGE_BASE=${AWS_ACCOUNT}.dkr.ecr.us-west-2.amazonaws.com/s3-integ-validator
28+
# Get the default credentials and set as environment variables
29+
CREDS=`curl 169.254.170.2$AWS_CONTAINER_CREDENTIALS_RELATIVE_URI`
30+
export AWS_ACCESS_KEY_ID=`echo $CREDS | jq -r .AccessKeyId`
31+
export AWS_SECRET_ACCESS_KEY=`echo $CREDS | jq -r .SecretAccessKey`
32+
export AWS_SESSION_TOKEN=`echo $CREDS | jq -r .Token`
2033
21-
# Pull the image that we built and pushed in the `Build` stage
22-
- aws ecr get-login-password --region ${AWS_REGION} | docker login --username AWS --password-stdin ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com
23-
- docker pull ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:latest
24-
- docker tag ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test amazon/aws-for-fluent-bit:latest
34+
# Pull the image that we built and pushed in the `Build` stage
35+
aws ecr get-login-password --region ${AWS_REGION} | docker login --username AWS --password-stdin ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com
36+
docker pull ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test:latest
37+
docker tag ${AWS_ACCOUNT}.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-for-fluent-bit-test amazon/aws-for-fluent-bit:latest
2538
26-
# List the images to do a double check
27-
- docker images
39+
# List the images to do a double check
40+
docker images
2841
29-
# Command to run the integration test
30-
- make integ
42+
# Command to run the integration test
43+
make integ
44+
45+
elif [ "$BUILD_VERSION" = "3" ]; then
46+
echo "Running integration tests for AWS for Fluent Bit version 3 (no-op for now)"
47+
echo "Version 3 integration test logic will be implemented later"
48+
# Placeholder for future BUILD_VERSION=3 implementation
49+
50+
else
51+
echo "Unsupported BUILD_VERSION: $BUILD_VERSION"
52+
echo "Supported versions are: 2, 3"
53+
exit 1
54+
fi
3155
artifacts:
3256
files:
3357
- '**/*'

‎buildspec_load_test.yml

Lines changed: 44 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -48,30 +48,54 @@ phases:
4848
- aws configure set default.region us-west-2
4949
build:
5050
commands:
51-
# Activate Python virtual environment and install the AWS CDK core dependencies
52-
- python -m venv venv
53-
- source venv/bin/activate
54-
- pip install -r ./load_tests/requirements.txt
51+
# Check BUILD_VERSION environment variable and set default if not provided
5552
- |
56-
if [ "${PLATFORM}" == "EKS" ]; then
57-
aws eks update-kubeconfig --name $EKS_CLUSTER_NAME
53+
if [ -z "$BUILD_VERSION" ]; then
54+
echo "BUILD_VERSION environment variable not provided, defaulting to BUILD_VERSION=2"
55+
export BUILD_VERSION=2
56+
else
57+
echo "BUILD_VERSION is set to: $BUILD_VERSION"
58+
fi
59+
# Execute load test commands based on BUILD_VERSION
60+
- |
61+
if [ "$BUILD_VERSION" = "2" ]; then
62+
echo "Running load tests for AWS for Fluent Bit version 2 (current behavior)"
63+
64+
# Activate Python virtual environment and install the AWS CDK core dependencies
65+
python -m venv venv
66+
source venv/bin/activate
67+
pip install -r ./load_tests/requirements.txt
68+
if [ "${PLATFORM}" == "EKS" ]; then
69+
aws eks update-kubeconfig --name $EKS_CLUSTER_NAME
70+
fi
71+
CREDS=$(aws sts assume-role --role-arn $LOAD_TEST_CFN_ROLE_ARN --role-session-name load-test-cfn --duration-seconds 3600)
72+
export CREDS
73+
export AWS_ACCESS_KEY_ID=$(echo "${CREDS}" | jq -r '.Credentials.AccessKeyId')
74+
export AWS_SECRET_ACCESS_KEY=$(echo "${CREDS}" | jq -r '.Credentials.SecretAccessKey')
75+
export AWS_SESSION_TOKEN=$(echo "${CREDS}" | jq -r '.Credentials.SessionToken')
76+
# Create and set up related testing resources
77+
python ./load_tests/load_test.py create_testing_resources
78+
# TODO: Modify setup_test_environment.sh to handle BUILD_VERSION=3 appropriately
79+
source ./load_tests/setup_test_environment.sh
80+
export AWS_ACCESS_KEY_ID=""
81+
export AWS_SECRET_ACCESS_KEY=""
82+
export AWS_SESSION_TOKEN=""
83+
# Run load tests on corresponding platform
84+
python ./load_tests/load_test.py ${PLATFORM}
85+
86+
elif [ "$BUILD_VERSION" = "3" ]; then
87+
echo "Running load tests for AWS for Fluent Bit version 3 (no-op for now)"
88+
echo "Version 3 load test logic will be implemented later"
89+
# Placeholder for future BUILD_VERSION=3 implementation
90+
91+
else
92+
echo "Unsupported BUILD_VERSION: $BUILD_VERSION"
93+
echo "Supported versions are: 2, 3"
94+
exit 1
5895
fi
59-
- CREDS=$(aws sts assume-role --role-arn $LOAD_TEST_CFN_ROLE_ARN --role-session-name load-test-cfn --duration-seconds 3600)
60-
- export CREDS
61-
- export AWS_ACCESS_KEY_ID=$(echo "${CREDS}" | jq -r '.Credentials.AccessKeyId')
62-
- export AWS_SECRET_ACCESS_KEY=$(echo "${CREDS}" | jq -r '.Credentials.SecretAccessKey')
63-
- export AWS_SESSION_TOKEN=$(echo "${CREDS}" | jq -r '.Credentials.SessionToken')
64-
# Create and set up related testing resources
65-
- python ./load_tests/load_test.py create_testing_resources
66-
- source ./load_tests/setup_test_environment.sh
67-
- export AWS_ACCESS_KEY_ID=""
68-
- export AWS_SECRET_ACCESS_KEY=""
69-
- export AWS_SESSION_TOKEN=""
70-
# Run load tests on corresponding platform
71-
- python ./load_tests/load_test.py ${PLATFORM}
7296
finally:
7397
# Clear up testing resources
7498
- python ./load_tests/load_test.py delete_testing_resources
7599
artifacts:
76100
files:
77-
- '**/*'
101+
- '**/*'

0 commit comments

Comments
 (0)