integ: re-building validator images for every test run is inefficient

Signed-off-by: Wesley Pettit <[email protected]>

Author: PettitWesley

README.md

@@ -26,7 +26,7 @@
 - [Using the AWS Plugins outside of a container](#using-the-aws-plugins-outside-of-a-container)
 - [Running aws-for-fluent-bit Windows containers](#running-aws-for-fluent-bit-windows-containers)
 - [Development](#development)
-    - [Releasing](#releasing)
+    - [Local testing](#local-testing)
     - [Developing Features in the AWS Plugins](#developing-features-in-the-aws-plugins)
 - [Fluent Bit Examples](#fluent-bit-examples)
 - [License](#license)
@@ -326,12 +326,17 @@ For more details about running Fluent Bit Windows containers in Amazon ECS, plea
 
 ### Development
 
-#### Releasing
+#### Local testing
 
 Use `make release` to build the image.
+
 To run the integration tests, run `make integ-dev`. The `make integ-dev` command will run the integration tests for all of our plugins-
 kinesis streams, kinesis firehose, and cloudwatch.
 
+The integ tests require the following env vars to be set:
+* `CW_INTEG_VALIDATOR_IMAGE`: Build the [integ/validate_cloudwatch/](integ/validate_cloudwatch/) folder with `docker build` and set the resulting image as the value of this env var.
+* `S3_INTEG_VALIDATOR_IMAGE`: Build the [integ/s3/](integ/s3/) folder with `docker build` and set the resulting image as the value of this env var.
+
 To run integration tests separately, execute `make integ-cloudwatch` or `make integ-kinesis` or `make integ-firehose`.
 
 [Documentation on GitHub steps for releases](Release_Process.md).

buildspec_integ.yml

@@ -10,6 +10,8 @@ phases:
     commands:
       # Enforce STS regional endpoints
       - export AWS_STS_REGIONAL_ENDPOINTS=regional
+      - export CW_INTEG_VALIDATOR_IMAGE=${AWS_ACCOUNT}.dkr.ecr.us-west-2.amazonaws.com/cw-integ-validator:latest
+      - export S3_INTEG_VALIDATOR_IMAGE=${AWS_ACCOUNT}.dkr.ecr.us-west-2.amazonaws.com/s3-integ-validator:latest
       # Get the default credentials and set as environment variables
       - 'CREDS=`curl 169.254.170.2$AWS_CONTAINER_CREDENTIALS_RELATIVE_URI`'
       - 'export AWS_ACCESS_KEY_ID=`echo $CREDS | jq -r .AccessKeyId`'

integ/integ.sh

@@ -2,7 +2,6 @@
 export AWS_REGION="us-west-2"
 export PROJECT_ROOT="$(pwd)"
 export VOLUME_MOUNT_CONTAINER="/out"
-export ValidateS3Dockerfile="Dockerfile"
 
 test_cloudwatch() {
 	export ARCHITECTURE=$(uname -m)

integ/run-integ.ps1

@@ -75,12 +75,13 @@ $env:FLUENT_BIT_IMAGE = $FluentBitImage
 $env:TAG = -join ((65..90) + (97..122) | Get-Random -Count 10 | % {[char]$_})
 # AWS_FOR_FLUENT_BIT_CONTAINER_NAME is the name for the fluent-bit container ran for each service.
 $env:AWS_FOR_FLUENT_BIT_CONTAINER_NAME = "aws-for-fluent-bit-$($env:TAG)"
+$env:CW_INTEG_VALIDATOR_IMAGE = "906394416424.dkr.ecr.us-west-2.amazonaws.com/cw-integ-validator:latest"
+$env:S3_INTEG_VALIDATOR_IMAGE = "906394416424.dkr.ecr.us-west-2.amazonaws.com/s3-integ-validator:latest"
 
 # Windows specific settings.
 $env:ARCHITECTURE= "x86-64"
 $env:LOG_GROUP_NAME="fluent-bit-integ-test-$Platform-$env:ARCHITECTURE"
 $env:VOLUME_MOUNT_CONTAINER="C:/out"
-$env:ValidateS3Dockerfile = "Dockerfile.windows"
 # For Windows, we need to specify a static IP address for the fluent-bit container.
 # This is because fluent-bit container would be started first and then the other containers
 # would need to connect to the fluent-bit container over a TCP socket.

integ/s3/updating_validator_image.md

@@ -0,0 +1,15 @@
+## CW Validator Image
+
+The S3 Validator image exists in our account as:
+
+```
+906394416424.dkr.ecr.us-west-2.amazonaws.com/s3-integ-validator:latest
+```
+
+If we need to perform a change or dependency update, build the image and push it to the account with that name.
+
+The image is set as an env var `S3_INTEG_VALIDATOR_IMAGE` in:
+- Linux: buildspec_integ.yml
+- Windows: integ/run-integ.ps1
+
+The env var is used in the `docker-compose.validate-and-clean.yml` files for firehose, kinesis, and S3 tests.

integ/test_cloudwatch/docker-compose.validate.yml

@@ -2,7 +2,7 @@ version: "2"
 
 services:
     cloudwatch_validator:
-        build: ${PROJECT_ROOT}/integ/validate_cloudwatch
+        image: ${CW_INTEG_VALIDATOR_IMAGE}
         environment:
             - "AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}"
             - "AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}"

integ/test_firehose/docker-compose.validate-and-clean-s3.yml

@@ -2,9 +2,7 @@ version: "2"
 
 services:
     validate-s3:
-        build:
-            context: ${PROJECT_ROOT}/integ/s3
-            dockerfile: ${ValidateS3Dockerfile}
+        image: ${S3_INTEG_VALIDATOR_IMAGE}
         environment:
             - "AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}"
             - "AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}"

integ/test_kinesis/docker-compose.validate-and-clean-s3.yml

@@ -2,9 +2,7 @@ version: "2"
 
 services:
     validate-s3:
-        build:
-            context: ${PROJECT_ROOT}/integ/s3
-            dockerfile: ${ValidateS3Dockerfile}
+        image: ${S3_INTEG_VALIDATOR_IMAGE}
         environment:
             - "AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}"
             - "AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}"

integ/test_s3/docker-compose.validate-s3-multipart.yml

@@ -2,9 +2,7 @@ version: "2"
 
 services:
     validate-s3-multipart:
-        build:
-            context: ${PROJECT_ROOT}/integ/s3
-            dockerfile: ${ValidateS3Dockerfile}
+        image: ${S3_INTEG_VALIDATOR_IMAGE}
         environment:
             - "AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}"
             - "AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}"

integ/test_s3/docker-compose.validate-s3-putobject.yml

@@ -2,9 +2,7 @@ version: "2"
 
 services:
     validate-s3-put-object:
-        build:
-            context: ${PROJECT_ROOT}/integ/s3
-            dockerfile: ${ValidateS3Dockerfile}
+        image: ${S3_INTEG_VALIDATOR_IMAGE}
         environment:
             - "AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}"
             - "AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}"