Skip to content

Commit f586bb9

Browse files
AnaethelionAnaethelion
committed
CI: Backport artifact based CI to 7.x (#255)
* CI: Backport artifact based CI to 7.x
1 parent abb073a commit f586bb9

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+3304
-213
lines changed

‎.ci/run-elasticsearch.sh‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,16 @@
77
# Export the TEST_SUITE variable, eg. 'free' or 'platinum' defaults to 'free'.
88
# Export the NUMBER_OF_NODES variable to start more than 1 node
99

10-
# Version 1.2.0
10+
# Version 1.3.0
1111
# - Initial version of the run-elasticsearch.sh script
1212
# - Deleting the volume should not dependent on the container still running
1313
# - Fixed `ES_JAVA_OPTS` config
1414
# - Moved to STACK_VERSION and TEST_VERSION
1515
# - Refactored into functions and imports
1616
# - Support NUMBER_OF_NODES
1717
# - Added 5 retries on docker pull for fixing transient network errors
18+
# - Added flags to make local CCR configurations work
19+
# - Added action.destructive_requires_name=false as the default will be true in v8
1820

1921
script_path=$(dirname $(realpath -s $0))
2022
source $script_path/functions/imports.sh
@@ -37,6 +39,7 @@ environment=($(cat <<-END
3739
--env node.attr.testattr=test
3840
--env path.repo=/tmp
3941
--env repositories.url.allowed_urls=http://snapshot.test*
42+
--env action.destructive_requires_name=false
4043
END
4144
))
4245
if [[ "$TEST_SUITE" == "platinum" ]]; then
@@ -45,11 +48,11 @@ if [[ "$TEST_SUITE" == "platinum" ]]; then
4548
--env xpack.license.self_generated.type=trial
4649
--env xpack.security.enabled=true
4750
--env xpack.security.http.ssl.enabled=true
48-
--env xpack.security.http.ssl.verification_mode=certificate
4951
--env xpack.security.http.ssl.key=certs/testnode.key
5052
--env xpack.security.http.ssl.certificate=certs/testnode.crt
5153
--env xpack.security.http.ssl.certificate_authorities=certs/ca.crt
5254
--env xpack.security.transport.ssl.enabled=true
55+
--env xpack.security.transport.ssl.verification_mode=certificate
5356
--env xpack.security.transport.ssl.key=certs/testnode.key
5457
--env xpack.security.transport.ssl.certificate=certs/testnode.crt
5558
--env xpack.security.transport.ssl.certificate_authorities=certs/ca.crt
@@ -104,7 +107,7 @@ END
104107
docker run \
105108
--name "$node_name" \
106109
--network "$network_name" \
107-
--env "ES_JAVA_OPTS=-Xms1g -Xmx1g" \
110+
--env "ES_JAVA_OPTS=-Xms1g -Xmx1g -da:org.elasticsearch.xpack.ccr.index.engine.FollowingEngineAssertions" \
108111
"${environment[@]}" \
109112
"${volumes[@]}" \
110113
--publish "$http_port":9200 \
@@ -124,4 +127,3 @@ END
124127
fi
125128

126129
done
127-

‎.ci/run-repository.sh‎

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -40,16 +40,10 @@ echo -e "\033[34;1mINFO:\033[0m Retrieving Elasticsearch Version & Hash from con
4040
ELASTICSEARCH_BUILD_VERSION=$(curl -sSk $external_elasticsearch_url | jq -r '.version.number')
4141
ELASTICSEARCH_BUILD_HASH=$(curl -sSk $external_elasticsearch_url | jq -r '.version.build_hash')
4242

43-
echo -e "\033[34;1mINFO:\033[0m Downloading Elasticsearch repository @ $ELASTICSEARCH_BUILD_HASH... \033[0m"
44-
mkdir -p $WORKSPACE/tmp
45-
curl --retry 10 -sSL "https://github.com/elastic/elasticsearch/archive/$ELASTICSEARCH_BUILD_HASH.zip" > "$WORKSPACE/tmp/elasticsearch-$ELASTICSEARCH_BUILD_HASH.zip"
46-
47-
echo -e "\033[34;1mINFO:\033[0m Extracting the Elasticsearch source... \033[0m"
48-
docker run --volume=$WORKSPACE/tmp:/tmp --workdir=/tmp --rm elastic/go-elasticsearch unzip -q -o "elasticsearch-$ELASTICSEARCH_BUILD_HASH.zip" '*.properties' '*.json' '*.y*ml'
49-
docker run --volume=$WORKSPACE/tmp:/tmp --workdir=/tmp --rm elastic/go-elasticsearch /bin/sh -c "
50-
rm -rf /tmp/elasticsearch-$ELASTICSEARCH_BUILD_HASH.zip
51-
rm -rf /tmp/elasticsearch/
52-
mv /tmp/elasticsearch-$ELASTICSEARCH_BUILD_HASH* /tmp/elasticsearch/
43+
echo -e "\033[34;1mINFO:\033[0m Download Elasticsearch specs... \033[0m"
44+
docker run --volume=$WORKSPACE/tmp:/tmp --workdir=/go-elasticsearch/internal/build --rm elastic/go-elasticsearch /bin/sh -c "
45+
go mod download
46+
go run main.go download-spec -o /tmp -d
5347
"
5448

5549
echo -e "\033[34;1mINFO:\033[0m Execute [$TEST_SUITE] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"

‎.ci/scripts/tests-free.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,13 @@ rm -rf esapi/test/xpack
1111

1212
echo -e "\033[34;1mINFO:\033[0m Generating the API registry\033[0m"
1313

14-
cd /go-elasticsearch/internal/cmd/generate || exit
14+
cd /go-elasticsearch/internal/build || exit
1515
go get -u golang.org/x/tools/cmd/goimports
1616
PACKAGE_PATH=/go-elasticsearch/esapi go generate ./...
1717

1818
echo -e "\033[34;1mINFO:\033[0m Generating the test files\033[0m"
1919

20-
go run main.go apitests --output '/go-elasticsearch/esapi/test' --input '/tmp/elasticsearch/rest-api-spec/src/main/resources/rest-api-spec/test/**/*.y*ml'
20+
go run main.go apitests --output '/go-elasticsearch/esapi/test' --input "/tmp/rest-api-spec/test/free/**/*.y*ml"
2121

2222
echo -e "\033[34;1mINFO:\033[0m Download tests deps >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
2323

‎.ci/scripts/tests-platinum.sh‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,13 @@ rm -rf rm -rf esapi/test/ml*
1111

1212
echo -e "\033[34;1mINFO:\033[0m Generating the API registry\033[0m"
1313

14-
cd /go-elasticsearch/internal/cmd/generate || exit
14+
cd /go-elasticsearch/internal/build || exit
1515
go get -u golang.org/x/tools/cmd/goimports
1616
PACKAGE_PATH=/go-elasticsearch/esapi go generate ./...
1717

1818
echo -e "\033[34;1mINFO:\033[0m Generating the test files\033[0m"
1919

20-
go run main.go apitests --output '/go-elasticsearch/esapi/test/xpack' --input '/tmp/elasticsearch/x-pack/plugin/src/test/resources/rest-api-spec/test/**/*.yml'
21-
go run main.go apitests --output '/go-elasticsearch/esapi/test/xpack' --input '/tmp/elasticsearch/x-pack/plugin/src/test/resources/rest-api-spec/test/**/**/*.yml'
20+
go run main.go apitests --output '/go-elasticsearch/esapi/test/xpack' --input '/tmp/rest-api-spec/test/platinum/**/*.yml'
2221

2322
cd /go-elasticsearch || exit
2423

‎Makefile‎

Lines changed: 41 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
SHELL := /bin/bash
22

3+
ELASTICSEARCH_DEFAULT_BUILD_VERSION = "7.13.0-SNAPSHOT"
4+
35
##@ Test
46
test-unit: ## Run unit tests
57
@printf "\033[2m→ Running unit tests...\033[0m\n"
@@ -149,8 +151,8 @@ lint: ## Run lint on the package
149151
@{ \
150152
set -e ; \
151153
trap "test -d ../../../.git && git checkout --quiet go.mod" INT TERM EXIT; \
152-
echo "cd internal/cmd/generate/ && go vet ./..."; \
153-
cd "internal/cmd/generate/" && go mod download && go vet ./...; \
154+
echo "cd internal/build/ && go vet ./..."; \
155+
cd "internal/build/" && go mod download && go vet ./...; \
154156
}
155157

156158

@@ -269,6 +271,12 @@ cluster: ## Launch an Elasticsearch cluster with Docker
269271
$(eval flavor ?= "core")
270272
$(eval elasticsearch_url = "http://es1:9200")
271273

274+
ifdef ELASTICSEARCH_BUILD_VERSION
275+
$(eval version ?= "elasticsearch:"${ELASTICSEARCH_BUILD_VERSION})
276+
else
277+
$(eval version ?= "elasticsearch:"${ELASTICSEARCH_DEFAULT_BUILD_VERSION})
278+
endif
279+
272280
ifeq ($(origin nodes), undefined)
273281
$(eval nodes = 1)
274282
endif
@@ -285,16 +293,14 @@ ifeq ($(flavor), platinum)
285293
$(eval xpack_env += --env "xpack.license.self_generated.type=trial")
286294
$(eval xpack_env += --env "xpack.security.enabled=true")
287295
$(eval xpack_env += --env "xpack.security.http.ssl.enabled=true")
288-
$(eval xpack_env += --env "xpack.security.http.ssl.verification_mode=certificate")
289296
$(eval xpack_env += --env "xpack.security.http.ssl.key=certs/testnode.key")
290297
$(eval xpack_env += --env "xpack.security.http.ssl.certificate=certs/testnode.crt")
291298
$(eval xpack_env += --env "xpack.security.http.ssl.certificate_authorities=certs/ca.crt")
292-
$(eval xpack_env += --env "xpack.security.http.ssl.verification_mode=none")
293299
$(eval xpack_env += --env "xpack.security.transport.ssl.enabled=true")
300+
$(eval xpack_env += --env "xpack.security.transport.ssl.verification_mode=certificate")
294301
$(eval xpack_env += --env "xpack.security.transport.ssl.key=certs/testnode.key")
295302
$(eval xpack_env += --env "xpack.security.transport.ssl.certificate=certs/testnode.crt")
296303
$(eval xpack_env += --env "xpack.security.transport.ssl.certificate_authorities=certs/ca.crt")
297-
$(eval xpack_env += --env "xpack.security.transport.ssl.verification_mode=none")
298304
$(eval xpack_volumes += --volume "$(PWD)/.ci/certs/testnode.crt:/usr/share/elasticsearch/config/certs/testnode.crt")
299305
$(eval xpack_volumes += --volume "$(PWD)/.ci/certs/testnode.key:/usr/share/elasticsearch/config/certs/testnode.key")
300306
$(eval xpack_volumes += --volume "$(PWD)/.ci/certs/ca.crt:/usr/share/elasticsearch/config/certs/ca.crt")
@@ -345,7 +351,11 @@ ifdef detach
345351
endif
346352

347353
cluster-update: ## Update the Docker image
348-
$(eval version ?= "elasticsearch:7.13.0-SNAPSHOT")
354+
ifdef ELASTICSEARCH_BUILD_VERSION
355+
$(eval version ?= "elasticsearch:"${ELASTICSEARCH_BUILD_VERSION})
356+
else
357+
$(eval version ?= "elasticsearch:"${ELASTICSEARCH_DEFAULT_BUILD_VERSION})
358+
endif
349359
@printf "\033[2m→ Updating the Docker image...\033[0m\n"
350360
@docker pull docker.elastic.co/elasticsearch/$(version);
351361

@@ -360,48 +370,47 @@ docker: ## Build the Docker image and run it
360370

361371
##@ Generator
362372
gen-api: ## Generate the API package from the JSON specification
363-
$(eval input ?= tmp/elasticsearch)
373+
$(eval input ?= tmp/rest-api-spec)
364374
$(eval output ?= esapi)
365375
ifdef debug
366376
$(eval args += --debug)
367377
endif
368378
ifdef ELASTICSEARCH_BUILD_VERSION
369379
$(eval version = $(ELASTICSEARCH_BUILD_VERSION))
370380
else
371-
$(eval version = $(shell cat "$(input)/buildSrc/version.properties" | grep 'elasticsearch' | cut -d '=' -f 2 | tr -d ' '))
381+
$(eval version = $(ELASTICSEARCH_DEFAULT_BUILD_VERSION))
372382
endif
373383
ifdef ELASTICSEARCH_BUILD_HASH
374384
$(eval build_hash = $(ELASTICSEARCH_BUILD_HASH))
375385
else
376-
$(eval build_hash = $(shell git --git-dir='$(input)/.git' rev-parse --short HEAD))
386+
$(eval build_hash = $(shell cat tmp/elasticsearch.json | jq ".projects.elasticsearch.commit_hash"))
377387
endif
378388
@printf "\033[2m→ Generating API package from specification ($(version):$(build_hash))...\033[0m\n"
379389
@{ \
380390
set -e; \
381-
trap "test -d .git && git checkout --quiet $(PWD)/internal/cmd/generate/go.mod" INT TERM EXIT; \
391+
trap "test -d .git && git checkout --quiet $(PWD)/internal/build/go.mod" INT TERM EXIT; \
382392
export ELASTICSEARCH_BUILD_VERSION=$(version) && \
383393
export ELASTICSEARCH_BUILD_HASH=$(build_hash) && \
384-
cd internal/cmd/generate && \
385-
go run main.go apisource --input '$(PWD)/$(input)/rest-api-spec/src/main/resources/rest-api-spec/api/*.json' --output '$(PWD)/$(output)' $(args) && \
386-
go run main.go apisource --input '$(PWD)/$(input)/x-pack/plugin/src/test/resources/rest-api-spec/api/*.json' --output '$(PWD)/$(output)' $(args) && \
394+
cd internal/build && \
395+
go run main.go apisource --input '$(PWD)/$(input)/api/*.json' --output '$(PWD)/$(output)' $(args) && \
387396
go run main.go apistruct --output '$(PWD)/$(output)'; \
388397
}
389398

390399
gen-tests: ## Generate the API tests from the YAML specification
391-
$(eval input ?= tmp/elasticsearch)
400+
$(eval input ?= tmp/rest-api-spec)
392401
$(eval output ?= esapi/test)
393402
ifdef debug
394403
$(eval args += --debug)
395404
endif
396405
ifdef ELASTICSEARCH_BUILD_VERSION
397406
$(eval version = $(ELASTICSEARCH_BUILD_VERSION))
398407
else
399-
$(eval version = $(shell cat "$(input)/buildSrc/version.properties" | grep 'elasticsearch' | cut -d '=' -f 2 | tr -d ' '))
408+
$(eval version = $(ELASTICSEARCH_DEFAULT_BUILD_VERSION))
400409
endif
401410
ifdef ELASTICSEARCH_BUILD_HASH
402411
$(eval build_hash = $(ELASTICSEARCH_BUILD_HASH))
403412
else
404-
$(eval build_hash = $(shell git --git-dir='$(input)/.git' rev-parse --short HEAD))
413+
$(eval build_hash = $(shell cat tmp/elasticsearch.json | jq ".projects.elasticsearch.commit_hash"))
405414
endif
406415
@printf "\033[2m→ Generating API tests from specification ($(version):$(build_hash))...\033[0m\n"
407416
@{ \
@@ -411,12 +420,11 @@ endif
411420
export ELASTICSEARCH_BUILD_HASH=$(build_hash) && \
412421
rm -rf $(output)/*_test.go && \
413422
rm -rf $(output)/xpack && \
414-
cd internal/cmd/generate && \
423+
cd internal/build && \
415424
go get golang.org/x/tools/cmd/goimports && \
416425
go generate ./... && \
417-
go run main.go apitests --input '$(PWD)/$(input)/rest-api-spec/src/main/resources/rest-api-spec/test/**/*.y*ml' --output '$(PWD)/$(output)' $(args) && \
418-
go run main.go apitests --input '$(PWD)/$(input)/x-pack/plugin/src/test/resources/rest-api-spec/test/**/*.yml' --output '$(PWD)/$(output)/xpack' $(args) && \
419-
go run main.go apitests --input '$(PWD)/$(input)/x-pack/plugin/src/test/resources/rest-api-spec/test/**/**/*.yml' --output '$(PWD)/$(output)/xpack' $(args) && \
426+
go run main.go apitests --input '$(PWD)/$(input)/test/free/**/*.y*ml' --output '$(PWD)/$(output)' $(args) && \
427+
go run main.go apitests --input '$(PWD)/$(input)/test/platinum/**/*.yml' --output '$(PWD)/$(output)/xpack' $(args) && \
420428
mkdir -p '$(PWD)/esapi/test/xpack/ml' && \
421429
mkdir -p '$(PWD)/esapi/test/xpack/ml-crud' && \
422430
mv $(PWD)/esapi/test/xpack/xpack_ml* $(PWD)/esapi/test/xpack/ml/ && \
@@ -445,11 +453,23 @@ gen-docs: ## Generate the skeleton of documentation examples
445453
fi; \
446454
) && \
447455
printf "\n\033[2m→ Generating ASCIIDoc files from Go source\033[0m\n" && \
448-
( cd '$(PWD)/internal/cmd/generate' && \
456+
( cd '$(PWD)/internal/build' && \
449457
go run main.go examples doc --debug --input='$(PWD)/.doc/examples/src/' --output='$(PWD)/.doc/examples/' \
450458
) \
451459
}
452460

461+
download-specs: ## Download the latest specs for the specified Elasticsearch version
462+
$(eval output ?= tmp)
463+
@mkdir -p tmp
464+
@{ \
465+
set -e; \
466+
printf "\n\033[2m→ Downloading latest Elasticsearch specs for version [$(ELASTICSEARCH_DEFAULT_BUILD_VERSION)]\033[0m\n" && \
467+
rm -rf $(output)/rest-api-spec && \
468+
rm -rf $(output)/elasticsearch.json && \
469+
cd internal/build && \
470+
go run main.go download-spec --output '$(PWD)/$(output)'; \
471+
}
472+
453473
##@ Other
454474
#------------------------------------------------------------------------------
455475
help: ## Display help

‎_examples/xkcdsearch/go.sum‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
12
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
23
github.com/rs/zerolog v1.11.0 h1:DRuq/S+4k52uJzBQciUcofXx45GrMC6yrEbb/CoK6+M=
34
github.com/rs/zerolog v1.11.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=

‎go.sum‎

Whitespace-only changes.

‎internal/cmd/generate/commands/commands.go‎ renamed to ‎internal/build/cmd/commands.go‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,20 +2,19 @@
22
// Elasticsearch B.V. licenses this file to you under the Apache 2.0 License.
33
// See the LICENSE file in the project root for more information.
44

5-
package commands
5+
package cmd
66

77
import (
88
"os"
99

1010
"github.com/spf13/cobra"
1111

12-
"github.com/elastic/go-elasticsearch/v7/internal/cmd/generate/utils"
12+
"github.com/elastic/go-elasticsearch/v7/internal/build/utils"
1313
)
1414

1515
var rootCmd = &cobra.Command{
16-
Use: "generate",
17-
Short: "generate allows you to generate APIs and tests",
18-
// Long: "TODO",
16+
Use: "build",
17+
Short: "Build tools for Elasticsearch client, helpers for CI, etc...",
1918
}
2019

2120
// Execute launches the CLI application.

0 commit comments

Comments
 (0)