Skip to content

Commit 65258e7

Browse files
darkowlzzdarkowlzz
authored
Merge pull request #387 from fluxcd/update-int-tests-for-ci
Update integration tests and run them against Azure and GCP
2 parents 9b48346 + 5a86c4d commit 65258e7

File tree

13 files changed

+429
-108
lines changed

13 files changed

+429
-108
lines changed

‎.github/workflows/integration-azure.yaml‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
name: integration-azure
2+
3+
on:
4+
workflow_dispatch:
5+
# schedule:
6+
# - cron: "0 6 * * *"
7+
# push:
8+
# branches:
9+
# - main
10+
11+
permissions:
12+
contents: read
13+
14+
jobs:
15+
test:
16+
runs-on: ubuntu-latest
17+
defaults:
18+
run:
19+
working-directory: ./tests/integration
20+
steps:
21+
- name: Checkout
22+
uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
23+
- name: Set up Go
24+
uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
25+
with:
26+
go-version: 1.20.x
27+
cache-dependency-path: tests/integration/go.sum
28+
- name: Authenticate to Azure
29+
uses: Azure/login@92a5484dfaf04ca78a94597f4f19fea633851fa2 # v1.4.6
30+
with:
31+
creds: '{"clientId":"${{ secrets.IRC_E2E_AZ_ARM_CLIENT_ID }}","clientSecret":"${{ secrets.IRC_E2E_AZ_ARM_CLIENT_SECRET }}","subscriptionId":"${{ secrets.IRC_E2E_AZ_ARM_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.IRC_E2E_AZ_ARM_TENANT_ID }}"}'
32+
- name: Set up QEMU
33+
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v2.1.0
34+
- name: Set up Docker Buildx
35+
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v2.5.0
36+
- name: Cache Docker layers
37+
uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
38+
id: cache
39+
with:
40+
path: /tmp/.buildx-cache
41+
key: ${{ runner.os }}-buildx-ghcache-${{ github.sha }}
42+
restore-keys: |
43+
${{ runner.os }}-buildx-ghcache-
44+
- name: Set dynamic variables in .env
45+
run: |
46+
cat > .env <<EOF
47+
export TF_VAR_tags='{"environment"="github", "ci"="true", "repo"="image-reflector-controller", "createdat"="$(date -u +x%Y-%m-%d_%Hh%Mm%Ss)"}'
48+
EOF
49+
- name: Print .env for dynamic tag value reference
50+
run: cat .env
51+
- name: Build controller
52+
run: |
53+
make docker-build IMG=fluxcd/image-reflector-controller:dev \
54+
BUILD_PLATFORMS=linux/amd64 \
55+
BUILD_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache \
56+
--cache-to=type=local,dest=/tmp/.buildx-cache-new,mode=max"
57+
working-directory: ./
58+
- # Temp fix
59+
# https://github.com/docker/build-push-action/issues/252
60+
# https://github.com/moby/buildkit/issues/1896
61+
name: Move cache
62+
run: |
63+
rm -rf /tmp/.buildx-cache
64+
mv /tmp/.buildx-cache-new /tmp/.buildx-cache
65+
- name: Run tests
66+
run: . .env && make test-azure TEST_IMG=fluxcd/image-reflector-controller:dev
67+
env:
68+
ARM_CLIENT_ID: ${{ secrets.IRC_E2E_AZ_ARM_CLIENT_ID }}
69+
ARM_CLIENT_SECRET: ${{ secrets.IRC_E2E_AZ_ARM_CLIENT_SECRET }}
70+
ARM_SUBSCRIPTION_ID: ${{ secrets.IRC_E2E_AZ_ARM_SUBSCRIPTION_ID }}
71+
ARM_TENANT_ID: ${{ secrets.IRC_E2E_AZ_ARM_TENANT_ID }}
72+
TF_VAR_azure_location: ${{ vars.TF_VAR_azure_location }}

‎.github/workflows/integration-gcp.yaml‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
name: integration-gcp
2+
3+
on:
4+
workflow_dispatch:
5+
schedule:
6+
- cron: "0 6 * * *"
7+
# push:
8+
# branches:
9+
# - main
10+
11+
permissions:
12+
contents: read
13+
14+
jobs:
15+
test:
16+
runs-on: ubuntu-latest
17+
defaults:
18+
run:
19+
working-directory: ./tests/integration
20+
steps:
21+
- name: Checkout
22+
uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2
23+
- name: Set up Go
24+
uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
25+
with:
26+
go-version: 1.20.x
27+
cache-dependency-path: tests/integration/go.sum
28+
- name: Authenticate to Google Cloud
29+
uses: google-github-actions/auth@35b0e87d162680511bf346c299f71c9c5c379033 # v1.1.1
30+
id: 'auth'
31+
with:
32+
credentials_json: '${{ secrets.IRC_E2E_GOOGLE_CREDENTIALS }}'
33+
token_format: 'access_token'
34+
- name: Set up gcloud
35+
uses: google-github-actions/setup-gcloud@e30db14379863a8c79331b04a9969f4c1e225e0b # v1.1.1
36+
- name: Set up QEMU
37+
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v2.1.0
38+
- name: Set up Docker Buildx
39+
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v2.5.0
40+
- name: Cache Docker layers
41+
uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 # v3.3.1
42+
id: cache
43+
with:
44+
path: /tmp/.buildx-cache
45+
key: ${{ runner.os }}-buildx-ghcache-${{ github.sha }}
46+
restore-keys: |
47+
${{ runner.os }}-buildx-ghcache-
48+
- name: Log into gcr.io
49+
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v2.1.0
50+
with:
51+
registry: gcr.io
52+
username: oauth2accesstoken
53+
password: ${{ steps.auth.outputs.access_token }}
54+
- name: Log into us-central1-docker.pkg.dev
55+
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v2.1.0
56+
with:
57+
registry: us-central1-docker.pkg.dev
58+
username: oauth2accesstoken
59+
password: ${{ steps.auth.outputs.access_token }}
60+
- name: Set dynamic variables in .env
61+
run: |
62+
cat > .env <<EOF
63+
export TF_VAR_tags='{"environment"="github", "ci"="true", "repo"="image-reflector-controller", "createdat"="$(date -u +x%Y-%m-%d_%Hh%Mm%Ss)"}'
64+
EOF
65+
- name: Print .env for dynamic tag value reference
66+
run: cat .env
67+
- name: Build controller
68+
run: |
69+
make docker-build IMG=fluxcd/image-reflector-controller:dev \
70+
BUILD_PLATFORMS=linux/amd64 \
71+
BUILD_ARGS="--cache-from=type=local,src=/tmp/.buildx-cache \
72+
--cache-to=type=local,dest=/tmp/.buildx-cache-new,mode=max"
73+
working-directory: ./
74+
- # Temp fix
75+
# https://github.com/docker/build-push-action/issues/252
76+
# https://github.com/moby/buildkit/issues/1896
77+
name: Move cache
78+
run: |
79+
rm -rf /tmp/.buildx-cache
80+
mv /tmp/.buildx-cache-new /tmp/.buildx-cache
81+
- name: Run tests
82+
run: . .env && make test-gcp TEST_IMG=fluxcd/image-reflector-controller:dev
83+
env:
84+
TF_VAR_gcp_project_id: ${{ vars.TF_VAR_gcp_project_id }}
85+
TF_VAR_gcp_region: ${{ vars.TF_VAR_gcp_region }}
86+
TF_VAR_gcp_zone: ${{ vars.TF_VAR_gcp_zone }}

‎tests/integration/.env.sample‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,40 @@
22
# export AWS_ACCESS_KEY_ID=
33
# export AWS_SECRET_ACCESS_KEY=
44
# export AWS_REGION=us-east-2
5+
## This random value is needed for AWS only to prevent
6+
## https://github.com/hashicorp/terraform-provider-aws/issues/19583 which
7+
## happens when using dynamic "name" value in presence of more than one tag.
8+
# export TF_VAR_rand=${RANDOM}
59

610
## Azure
711
# export TF_VAR_azure_location=eastus
12+
## Set the following only when authenticating using Service Principal (suited
13+
## for CI environment).
14+
# export ARM_CLIENT_ID=
15+
# export ARM_CLIENT_SECRET=
16+
# export ARM_SUBSCRIPTION_ID=
17+
# export ARM_TENANT_ID=
818

919
## GCP
1020
# export TF_VAR_gcp_project_id=
1121
# export TF_VAR_gcp_region=us-central1
1222
# export TF_VAR_gcp_zone=us-central1-c
1323
## Leave GCR region empty to use gcr.io. Else set it to `us`, `eu` or `asia`.
1424
# export TF_VAR_gcr_region=
25+
## Set the following only when using service account.
26+
## Provide absolute path to the service account JSON key file.
27+
# export GOOGLE_APPLICATION_CREDENTIALS=
28+
29+
## Common variables
30+
# export TF_VAR_tags='{"environment"="dev"}'
31+
#
32+
## WARNING: For AWS, also set the "createdat" tag to overwrite the default
33+
## timestamp and use a static value. Dynamic tag value causes the issue
34+
## https://github.com/hashicorp/terraform-provider-aws/issues/19583.
35+
## The date format is based on the format defined in
36+
## fluxcd/test-infra/tf-modules/utils/tags tf-module that's compatible with the
37+
## tags/labels value in all the cloud providers.
38+
## Also, since "createdat" is a dynamic value, its value changes on subsequent
39+
## apply. Overriding it with a static value helps avoid modifying the resource
40+
## tags during development when the configurations are applied frequently.
41+
# export TF_VAR_tags='{"environment"="dev", "createdat"='"\"$(date -u +x%Y-%m-%d_%Hh%Mm%Ss)\""'}'

‎tests/integration/README.md‎

Lines changed: 69 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,23 +15,89 @@
1515
- Azure account with an active subscription to be able to create AKS and ACR,
1616
and permission to assign roles. Role assignment is required for allowing AKS
1717
workloads to access ACR.
18-
- Azure CLI, need to be logged in using `az login`.
18+
- Azure CLI, need to be logged in using `az login` as a User (not a Service
19+
Principal).
20+
21+
**NOTE:** To use Service Principal (for example in CI environment), set the
22+
`ARM-*` variables in `.env`, source it and authenticate Azure CLI with:
23+
```console
24+
$ az login --service-principal -u $ARM_CLIENT_ID -p $ARM_CLIENT_SECRET --tenant $ARM_TENANT_ID
25+
```
26+
In this case, the AzureRM client in terraform uses the Service Principal to
27+
authenticate and the Azure CLI is used only for authenticating with ACR for
28+
logging in and pushing container images. Attempting to authenticate terraform
29+
using Azure CLI with Service Principal results in the following error:
30+
> Authenticating using the Azure CLI is only supported as a User (not a Service Principal).
1931
- Docker CLI for registry login.
2032
- kubectl for applying certain install manifests.
2133

34+
#### Permissions
35+
36+
Following permissions are needed for provisioning the infrastructure and running
37+
the tests:
38+
- `Microsoft.Kubernetes/*`
39+
- `Microsoft.Resources/*`
40+
- `Microsoft.Authorization/roleAssignments/{Read,Write,Delete}`
41+
- `Microsoft.ContainerRegistry/*`
42+
- `Microsoft.ContainerService/*`
43+
2244
### Google Cloud Platform
2345

2446
- GCP account with project and GKE, GCR and Artifact Registry services enabled
2547
in the project.
26-
- gcloud CLI, need to be logged in using `gcloud auth login`.
48+
- gcloud CLI, need to be logged in using `gcloud auth login` as a User (not a
49+
Service Account), configure application default credentials with `gcloud auth
50+
application-default login` and docker credential helper with `gcloud auth configure-docker`.
51+
52+
**NOTE:** To use Service Account (for example in CI environment), set
53+
`GOOGLE_APPLICATION_CREDENTIALS` variable in `.env` with the path to the JSON
54+
key file, source it and authenticate gcloud CLI with:
55+
```console
56+
$ gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS
57+
```
58+
Depending on the Container/Artifact Registry host used in the test, authenticate
59+
docker accordingly
60+
```console
61+
$ gcloud auth print-access-token | docker login -u oauth2accesstoken --password-stdin https://us-central1-docker.pkg.dev
62+
$ gcloud auth print-access-token | docker login -u oauth2accesstoken --password-stdin https://gcr.io
63+
```
64+
In this case, the GCP client in terraform uses the Service Account to
65+
authenticate and the gcloud CLI is used only to authenticate with Google
66+
Container Registry and Google Artifact Registry.
67+
68+
**NOTE FOR CI USAGE:** When saving the JSON key file as a CI secret, compress
69+
the file content with
70+
```console
71+
$ cat key.json | jq -r tostring
72+
```
73+
to prevent aggressive masking in the logs. Refer
74+
[aggressive replacement in logs](https://github.com/google-github-actions/auth/blob/v1.1.0/docs/TROUBLESHOOTING.md#aggressive--replacement-in-logs)
75+
for more details.
2776
- Docker CLI for registry login.
2877
- kubectl for applying certain install manifests.
2978

3079
**NOTE:** Unlike ECR, ACR and Google Artifact Registry, Google Container
3180
Registry tests don't create a new registry. It pushes to an existing registry
3281
host in a project, for example `gcr.io`. Due to this, the test images pushed to
3382
GCR aren't cleaned up automatically at the end of the test and have to be
34-
deleted manually.
83+
deleted manually. [`gcrgc`](https://github.com/graillus/gcrgc) can be used to
84+
automatically delete all the GCR images.
85+
```console
86+
$ gcrgc gcr.io/<project-name>
87+
```
88+
89+
#### Permissions
90+
91+
Following roles are needed for provisioning the infrastructure and running the
92+
tests:
93+
- `Artifact Registry Administrator`
94+
- `Compute Instance Admin (v1)`
95+
- `Compute Storage Admin`
96+
- `Kubernetes Engine Admin`
97+
- `Service Account Admin`
98+
- `Service Account Token Creator`
99+
- `Service Account User`
100+
- `Storage Admin`
35101

36102
## Test setup
37103

‎tests/integration/go.mod‎

Lines changed: 29 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,14 @@ replace github.com/fluxcd/image-reflector-controller/api => ../../api
66

77
require (
88
github.com/fluxcd/image-reflector-controller/api v0.0.0
9-
github.com/fluxcd/test-infra/tftestenv v0.0.0-20230214200258-f19d6aa97a3f
9+
github.com/fluxcd/test-infra/tftestenv v0.0.0-20230530120643-bdcf7573fb2f
1010
github.com/hashicorp/terraform-json v0.15.0
11-
github.com/onsi/gomega v1.27.2
12-
k8s.io/apimachinery v0.26.3
11+
github.com/onsi/gomega v1.27.7
12+
k8s.io/apimachinery v0.27.2
1313
)
1414

1515
require (
16+
github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
1617
github.com/containerd/stargz-snapshotter/estargz v0.12.0 // indirect
1718
github.com/davecgh/go-spew v1.1.1 // indirect
1819
github.com/docker/cli v20.10.17+incompatible // indirect
@@ -22,28 +23,29 @@ require (
2223
github.com/emicklei/go-restful/v3 v3.9.0 // indirect
2324
github.com/evanphx/json-patch/v5 v5.6.0 // indirect
2425
github.com/fluxcd/pkg/apis/acl v0.1.0 // indirect
25-
github.com/fluxcd/pkg/apis/meta v1.0.0 // indirect
26+
github.com/fluxcd/pkg/apis/meta v1.1.0 // indirect
2627
github.com/go-logr/logr v1.2.4 // indirect
27-
github.com/go-openapi/jsonpointer v0.19.5 // indirect
28-
github.com/go-openapi/jsonreference v0.20.0 // indirect
29-
github.com/go-openapi/swag v0.19.14 // indirect
28+
github.com/go-openapi/jsonpointer v0.19.6 // indirect
29+
github.com/go-openapi/jsonreference v0.20.1 // indirect
30+
github.com/go-openapi/swag v0.22.3 // indirect
3031
github.com/gogo/protobuf v1.3.2 // indirect
31-
github.com/golang/protobuf v1.5.2 // indirect
32+
github.com/golang/protobuf v1.5.3 // indirect
3233
github.com/google/gnostic v0.5.7-v3refs // indirect
3334
github.com/google/go-cmp v0.5.9 // indirect
3435
github.com/google/go-containerregistry v0.11.0 // indirect
3536
github.com/google/gofuzz v1.2.0 // indirect
37+
github.com/google/uuid v1.3.0 // indirect
3638
github.com/hashicorp/errwrap v1.0.0 // indirect
3739
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
3840
github.com/hashicorp/go-multierror v1.1.1 // indirect
3941
github.com/hashicorp/go-version v1.6.0 // indirect
40-
github.com/hashicorp/hc-install v0.3.2 // indirect
41-
github.com/hashicorp/terraform-exec v0.16.1 // indirect
42+
github.com/hashicorp/hc-install v0.5.0 // indirect
43+
github.com/hashicorp/terraform-exec v0.18.1 // indirect
4244
github.com/imdario/mergo v0.3.12 // indirect
4345
github.com/josharian/intern v1.0.0 // indirect
4446
github.com/json-iterator/go v1.1.12 // indirect
4547
github.com/klauspost/compress v1.15.8 // indirect
46-
github.com/mailru/easyjson v0.7.6 // indirect
48+
github.com/mailru/easyjson v0.7.7 // indirect
4749
github.com/mitchellh/go-homedir v1.1.0 // indirect
4850
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
4951
github.com/modern-go/reflect2 v1.0.2 // indirect
@@ -54,26 +56,27 @@ require (
5456
github.com/sirupsen/logrus v1.9.0 // indirect
5557
github.com/spf13/pflag v1.0.5 // indirect
5658
github.com/vbatts/tar-split v0.11.2 // indirect
57-
github.com/zclconf/go-cty v1.10.0 // indirect
58-
golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd // indirect
59-
golang.org/x/net v0.8.0 // indirect
60-
golang.org/x/oauth2 v0.0.0-20220718184931-c8730f7fcb92 // indirect
61-
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect
62-
golang.org/x/sys v0.6.0 // indirect
63-
golang.org/x/term v0.6.0 // indirect
64-
golang.org/x/text v0.8.0 // indirect
59+
github.com/zclconf/go-cty v1.13.0 // indirect
60+
golang.org/x/crypto v0.5.0 // indirect
61+
golang.org/x/mod v0.8.0 // indirect
62+
golang.org/x/net v0.10.0 // indirect
63+
golang.org/x/oauth2 v0.5.0 // indirect
64+
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
65+
golang.org/x/sys v0.8.0 // indirect
66+
golang.org/x/term v0.8.0 // indirect
67+
golang.org/x/text v0.9.0 // indirect
6568
golang.org/x/time v0.3.0 // indirect
6669
google.golang.org/appengine v1.6.7 // indirect
67-
google.golang.org/protobuf v1.28.1 // indirect
70+
google.golang.org/protobuf v1.30.0 // indirect
6871
gopkg.in/inf.v0 v0.9.1 // indirect
6972
gopkg.in/yaml.v2 v2.4.0 // indirect
7073
gopkg.in/yaml.v3 v3.0.1 // indirect
71-
k8s.io/api v0.26.3 // indirect
72-
k8s.io/client-go v0.26.1 // indirect
73-
k8s.io/klog/v2 v2.90.1 // indirect
74-
k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect
75-
k8s.io/utils v0.0.0-20230313181309-38a27ef9d749 // indirect
76-
sigs.k8s.io/controller-runtime v0.14.6 // indirect
74+
k8s.io/api v0.27.2 // indirect
75+
k8s.io/client-go v0.27.2 // indirect
76+
k8s.io/klog/v2 v2.100.1 // indirect
77+
k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
78+
k8s.io/utils v0.0.0-20230505201702-9f6742963106 // indirect
79+
sigs.k8s.io/controller-runtime v0.15.0 // indirect
7780
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
7881
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
7982
sigs.k8s.io/yaml v1.3.0 // indirect

0 commit comments

Comments
 (0)