Skip to content

Commit 0093f4a

Browse files
brianseedersbrianseeders
authored
[buildkite] Add third-party tests to periodic pipeline (#99376) (#100094)
(cherry picked from commit f69dce0)
1 parent 2bb29c3 commit 0093f4a

File tree

8 files changed

+221
-10
lines changed

8 files changed

+221
-10
lines changed

‎.buildkite/hooks/pre-command‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ BUILDKITE_API_TOKEN=$(vault read -field=token secret/ci/elastic-elasticsearch/bu
4848
export BUILDKITE_API_TOKEN
4949

5050
if [[ "${USE_LUCENE_SNAPSHOT_CREDS:-}" == "true" ]]; then
51-
data=$(.buildkite/scripts/lucene-snapshot/get-credentials.sh)
51+
data=$(.buildkite/scripts/get-legacy-secret.sh aws-elastic/creds/lucene-snapshots)
5252

5353
AWS_ACCESS_KEY_ID=$(echo "$data" | jq -r .data.access_key)
5454
export AWS_ACCESS_KEY_ID
@@ -70,6 +70,8 @@ if [[ "${USE_DRA_CREDENTIALS:-}" == "true" ]]; then
7070
export DRA_VAULT_ADDR
7171
fi
7272

73+
source .buildkite/scripts/third-party-test-credentials.sh
74+
7375
if [[ "${USE_SNYK_CREDENTIALS:-}" == "true" ]]; then
7476
SNYK_TOKEN=$(vault read -field=token secret/ci/elastic-elasticsearch/migrated/snyk)
7577
export SNYK_TOKEN

‎.buildkite/pipelines/periodic.template.yml‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,73 @@ steps:
103103
image: family/elasticsearch-ubuntu-2004
104104
diskSizeGb: 350
105105
machineType: custom-32-98304
106+
- group: third-party tests
107+
steps:
108+
- label: third-party / azure-sas
109+
command: |
110+
export azure_storage_container=elasticsearch-ci-thirdparty-sas
111+
export azure_storage_base_path=$BUILDKITE_BRANCH
112+
113+
.ci/scripts/run-gradle.sh azureThirdPartyTest
114+
env:
115+
USE_3RD_PARTY_AZURE_SAS_CREDENTIALS: "true"
116+
timeout_in_minutes: 30
117+
agents:
118+
provider: gcp
119+
image: family/elasticsearch-ubuntu-2004
120+
machineType: n2-standard-8
121+
buildDirectory: /dev/shm/bk
122+
- label: third-party / azure
123+
command: |
124+
export azure_storage_container=elasticsearch-ci-thirdparty
125+
export azure_storage_base_path=$BUILDKITE_BRANCH
126+
127+
.ci/scripts/run-gradle.sh azureThirdPartyTest
128+
env:
129+
USE_3RD_PARTY_AZURE_CREDENTIALS: "true"
130+
timeout_in_minutes: 30
131+
agents:
132+
provider: gcp
133+
image: family/elasticsearch-ubuntu-2004
134+
machineType: n2-standard-8
135+
buildDirectory: /dev/shm/bk
136+
- label: third-party / gcs
137+
command: |
138+
export google_storage_bucket=elasticsearch-ci-thirdparty
139+
export google_storage_base_path=$BUILDKITE_BRANCH
140+
141+
.ci/scripts/run-gradle.sh gcsThirdPartyTest
142+
env:
143+
USE_3RD_PARTY_GCS_CREDENTIALS: "true"
144+
timeout_in_minutes: 30
145+
agents:
146+
provider: gcp
147+
image: family/elasticsearch-ubuntu-2004
148+
machineType: n2-standard-8
149+
buildDirectory: /dev/shm/bk
150+
- label: third-party / geoip
151+
command: |
152+
.ci/scripts/run-gradle.sh :modules:ingest-geoip:internalClusterTest -Dtests.jvm.argline="-Dgeoip_use_service=true"
153+
timeout_in_minutes: 30
154+
agents:
155+
provider: gcp
156+
image: family/elasticsearch-ubuntu-2004
157+
machineType: n2-standard-8
158+
buildDirectory: /dev/shm/bk
159+
- label: third-party / s3
160+
command: |
161+
export amazon_s3_bucket=elasticsearch-ci.us-west-2
162+
export amazon_s3_base_path=$BUILDKITE_BRANCH
163+
164+
.ci/scripts/run-gradle.sh s3ThirdPartyTest
165+
env:
166+
USE_3RD_PARTY_S3_CREDENTIALS: "true"
167+
timeout_in_minutes: 30
168+
agents:
169+
provider: gcp
170+
image: family/elasticsearch-ubuntu-2004
171+
machineType: n2-standard-8
172+
buildDirectory: /dev/shm/bk
106173
- label: Upload Snyk Dependency Graph
107174
command: .ci/scripts/run-gradle.sh uploadSnykDependencyGraph -PsnykTargetReference=$BUILDKITE_BRANCH
108175
env:

‎.buildkite/pipelines/periodic.yml‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1114,6 +1114,72 @@ steps:
11141114
image: family/elasticsearch-ubuntu-2004
11151115
diskSizeGb: 350
11161116
machineType: custom-32-98304
1117+
- group: third-party tests
1118+
steps:
1119+
- label: third-party / azure-sas
1120+
command: |
1121+
export azure_storage_container=elasticsearch-ci-thirdparty-sas
1122+
export azure_storage_base_path=$BUILDKITE_BRANCH
1123+
1124+
.ci/scripts/run-gradle.sh azureThirdPartyTest
1125+
env:
1126+
USE_3RD_PARTY_AZURE_SAS_CREDENTIALS: "true"
1127+
timeout_in_minutes: 30
1128+
agents:
1129+
provider: gcp
1130+
image: family/elasticsearch-ubuntu-2004
1131+
machineType: n2-standard-8
1132+
buildDirectory: /dev/shm/bk
1133+
- label: third-party / azure
1134+
command: |
1135+
export azure_storage_container=elasticsearch-ci-thirdparty
1136+
export azure_storage_base_path=$BUILDKITE_BRANCH
1137+
1138+
.ci/scripts/run-gradle.sh azureThirdPartyTest
1139+
env:
1140+
USE_3RD_PARTY_AZURE_CREDENTIALS: "true"
1141+
timeout_in_minutes: 30
1142+
agents:
1143+
provider: gcp
1144+
image: family/elasticsearch-ubuntu-2004
1145+
machineType: n2-standard-8
1146+
buildDirectory: /dev/shm/bk
1147+
- label: third-party / gcs
1148+
command: |
1149+
export google_storage_bucket=elasticsearch-ci-thirdparty
1150+
export google_storage_base_path=$BUILDKITE_BRANCH
1151+
1152+
.ci/scripts/run-gradle.sh gcsThirdPartyTest
1153+
env:
1154+
USE_3RD_PARTY_GCS_CREDENTIALS: "true"
1155+
timeout_in_minutes: 30
1156+
agents:
1157+
provider: gcp
1158+
image: family/elasticsearch-ubuntu-2004
1159+
machineType: n2-standard-8
1160+
buildDirectory: /dev/shm/bk
1161+
- label: third-party / geoip
1162+
command: |
1163+
.ci/scripts/run-gradle.sh :modules:ingest-geoip:internalClusterTest -Dtests.jvm.argline="-Dgeoip_use_service=true"
1164+
timeout_in_minutes: 30
1165+
agents:
1166+
provider: gcp
1167+
image: family/elasticsearch-ubuntu-2004
1168+
machineType: n2-standard-8
1169+
- label: third-party / s3
1170+
command: |
1171+
export amazon_s3_bucket=elasticsearch-ci.us-west-2
1172+
export amazon_s3_base_path=$BUILDKITE_BRANCH
1173+
1174+
.ci/scripts/run-gradle.sh s3ThirdPartyTest
1175+
env:
1176+
USE_3RD_PARTY_S3_CREDENTIALS: "true"
1177+
timeout_in_minutes: 30
1178+
agents:
1179+
provider: gcp
1180+
image: family/elasticsearch-ubuntu-2004
1181+
machineType: n2-standard-8
1182+
buildDirectory: /dev/shm/bk
11171183
- label: Upload Snyk Dependency Graph
11181184
command: .ci/scripts/run-gradle.sh uploadSnykDependencyGraph -PsnykTargetReference=$BUILDKITE_BRANCH
11191185
env:

‎.buildkite/scripts/get-legacy-secret.sh‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
#!/bin/bash
2+
3+
set -euo pipefail
4+
5+
# WARNING: this script will echo the credentials to the console. It is meant to be called from another script and captured in a variable.
6+
# It should really only be used inside .buildkite/hooks/pre-command
7+
8+
source .buildkite/scripts/setup-legacy-vault.sh
9+
10+
vault read -format=json "$1"

‎.buildkite/scripts/lucene-snapshot/get-credentials.sh‎ renamed to ‎.buildkite/scripts/setup-legacy-vault.sh‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,10 @@
22

33
set -euo pipefail
44

5-
# WARNING: this script will echo the credentials to the console. It is meant to be called from another script and captured in a variable.
6-
# It should really only be used inside .buildkite/hooks/pre-command
7-
85
VAULT_ROLE_ID=$(vault read -field=role-id secret/ci/elastic-elasticsearch/legacy-vault-credentials)
96
VAULT_SECRET_ID=$(vault read -field=secret-id secret/ci/elastic-elasticsearch/legacy-vault-credentials)
107
VAULT_ADDR=https://secrets.elastic.co:8200
118

129
unset VAULT_TOKEN
1310
VAULT_TOKEN=$(vault write -field=token auth/approle/login role_id=$VAULT_ROLE_ID secret_id=$VAULT_SECRET_ID)
1411
export VAULT_TOKEN
15-
16-
vault read -format=json aws-elastic/creds/lucene-snapshots

‎.buildkite/scripts/third-party-test-credentials.gcs.sh‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/bin/bash
2+
3+
set -euo pipefail
4+
5+
# Usage: .buildkite/scripts/third-party-test-credentials.gcs.sh <path/to/write/credentials.json>
6+
7+
source .buildkite/scripts/setup-legacy-vault.sh
8+
9+
vault read -field=private_key_data gcp-elastic-ci-prod/key/elasticsearch-ci-thirdparty-gcs | base64 --decode > "$1"

‎.buildkite/scripts/third-party-test-credentials.sh‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
#!/bin/bash
2+
3+
set -euo pipefail
4+
5+
# You'll notice that most of the variables are exported twice with different names here
6+
# The first/uppercase export is to ensure that Buildkite masks the values in the logs should they accidentally be output
7+
# The second/lowercase export is what the tests expect/require
8+
9+
if [[ "${USE_3RD_PARTY_AZURE_CREDENTIALS:-}" == "true" ]]; then
10+
json=$(vault read -format=json secret/ci/elastic-elasticsearch/migrated/azure_thirdparty_test_creds)
11+
12+
AZURE_STORAGE_ACCOUNT_SECRET=$(echo "$json" | jq -r .data.account_id)
13+
export AZURE_STORAGE_ACCOUNT_SECRET
14+
export azure_storage_account="$AZURE_STORAGE_ACCOUNT_SECRET"
15+
16+
AZURE_STORAGE_KEY=$(echo "$json" | jq -r .data.account_key)
17+
export AZURE_STORAGE_KEY
18+
export azure_storage_key="$AZURE_STORAGE_KEY"
19+
fi
20+
21+
if [[ "${USE_3RD_PARTY_AZURE_SAS_CREDENTIALS:-}" == "true" ]]; then
22+
json=$(vault read -format=json secret/ci/elastic-elasticsearch/migrated/azure_thirdparty_sas_test_creds)
23+
24+
AZURE_STORAGE_ACCOUNT_SECRET=$(echo "$json" | jq -r .data.account_id)
25+
export AZURE_STORAGE_ACCOUNT_SECRET
26+
export azure_storage_account="$AZURE_STORAGE_ACCOUNT_SECRET"
27+
28+
AZURE_STORAGE_SAS_TOKEN=$(echo "$json" | jq -r .data.account_sas_token)
29+
export AZURE_STORAGE_SAS_TOKEN
30+
export azure_storage_sas_token="$AZURE_STORAGE_SAS_TOKEN"
31+
fi
32+
33+
if [[ "${USE_3RD_PARTY_S3_CREDENTIALS:-}" == "true" ]]; then
34+
json=$(.buildkite/scripts/get-legacy-secret.sh aws-test/creds/elasticsearch-ci-s3)
35+
AMAZON_S3_ACCESS_KEY=$(echo "$json" | jq -r .data.access_key)
36+
export AMAZON_S3_ACCESS_KEY
37+
export amazon_s3_access_key="$AMAZON_S3_ACCESS_KEY"
38+
39+
AMAZON_S3_SECRET_KEY=$(echo "$json" | jq -r .data.secret_key)
40+
export AMAZON_S3_SECRET_KEY
41+
export amazon_s3_secret_key="$AMAZON_S3_SECRET_KEY"
42+
fi
43+
44+
if [[ "${USE_3RD_PARTY_GCS_CREDENTIALS:-}" == "true" ]]; then
45+
export google_storage_service_account=$(mktemp)
46+
.buildkite/scripts/third-party-test-credentials.gcs.sh "$google_storage_service_account"
47+
fi
48+
49+
50+
51+
unset json

‎test/fixtures/geoip-fixture/src/main/java/fixture/geoip/GeoIpHttpFixture.java‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import java.nio.charset.StandardCharsets;
2626
import java.nio.file.Files;
2727
import java.nio.file.Path;
28+
import java.nio.file.StandardCopyOption;
2829

2930
public class GeoIpHttpFixture extends ExternalResource {
3031

@@ -106,15 +107,25 @@ protected void after() {
106107
}
107108

108109
private void copyFiles() throws Exception {
109-
Files.copy(GeoIpHttpFixture.class.getResourceAsStream("/geoip-fixture/GeoLite2-ASN.tgz"), source.resolve("GeoLite2-ASN.tgz"));
110-
Files.copy(GeoIpHttpFixture.class.getResourceAsStream("/geoip-fixture/GeoLite2-City.mmdb"), source.resolve("GeoLite2-City.mmdb"));
110+
Files.copy(
111+
GeoIpHttpFixture.class.getResourceAsStream("/geoip-fixture/GeoLite2-ASN.tgz"),
112+
source.resolve("GeoLite2-ASN.tgz"),
113+
StandardCopyOption.REPLACE_EXISTING
114+
);
115+
Files.copy(
116+
GeoIpHttpFixture.class.getResourceAsStream("/geoip-fixture/GeoLite2-City.mmdb"),
117+
source.resolve("GeoLite2-City.mmdb"),
118+
StandardCopyOption.REPLACE_EXISTING
119+
);
111120
Files.copy(
112121
GeoIpHttpFixture.class.getResourceAsStream("/geoip-fixture/GeoLite2-Country.mmdb"),
113-
source.resolve("GeoLite2-Country.mmdb")
122+
source.resolve("GeoLite2-Country.mmdb"),
123+
StandardCopyOption.REPLACE_EXISTING
114124
);
115125
Files.copy(
116126
GeoIpHttpFixture.class.getResourceAsStream("/geoip-fixture/MyCustomGeoLite2-City.mmdb"),
117-
source.resolve("MyCustomGeoLite2-City.mmdb")
127+
source.resolve("MyCustomGeoLite2-City.mmdb"),
128+
StandardCopyOption.REPLACE_EXISTING
118129
);
119130

120131
new GeoIpCli().main(

0 commit comments

Comments
 (0)