Skip to content

Commit 8d1a0f1

Browse files
kamil-holubickikamil-holubicki
committed
DISTMYSQL-140: Create dev Jenkins pipeline for Orchestrator
https://jira.percona.com/browse/DISTMYSQL-140 Adjusted Orchestrator's files to support parametrized Jenkins build. 1. MySql tarball passed as the parameter via env variable TARBALL_URL. This allows testing the Orchestrator against the requested version of MySql. If env variable not specified, fallback to the original, hard coded 5.7.26 version. 2. Allow tests continuation even if the test fails. (controlled via env variable ALLOW_TESTS_FAILURES). Originally the test suite was immediately finished on the 1st error. 3. Added convenience scripts test-system.sh and test-integration.sh 4. Use dbdeployer-1.64.0. This can be changed on Dockerfile level 5. Allow automatic start of system tests after container start (controlled via env variable RUN_TESTS) 6. Allow use of custom Orchestrator's ci-env (controlled via env variables CI_ENV_REPO and CI_ENV_BRANCH)
1 parent f3ed383 commit 8d1a0f1

File tree

9 files changed

+157
-14
lines changed

9 files changed

+157
-14
lines changed

‎docker/Dockerfile.system‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,33 @@
11
FROM golang:1.16.6-stretch
22
LABEL maintainer="openark@github.com"
33

4+
ARG ci_env_repo="https://github.com/percona/orchestrator-ci-env.git"
5+
ARG ci_env_branch=master
6+
7+
RUN echo "ci_env_repo: $ci_env_repo"
8+
RUN echo "ci_env_branch: $ci_env_branch"
9+
410
RUN apt-get update -q -y
511
RUN apt-get install -y sudo haproxy python git jq rsync libaio1 libnuma1 mysql-client bsdmainutils less vim
612

713
RUN mkdir /orchestrator
814
WORKDIR /orchestrator
915

10-
RUN git clone https://github.com/openark/orchestrator-ci-env.git # cache
16+
RUN git clone -b $ci_env_branch $ci_env_repo # cache
17+
# For dev purposes only, just to avoid cloning over and over
18+
# COPY docker/orchestrator-ci-env /orchestrator/orchestrator-ci-env
19+
20+
# Setup dbdeployer
21+
RUN mkdir /dbdeployer
22+
RUN (cd /dbdeployer && wget https://github.com/datacharmer/dbdeployer/releases/download/v1.64.0/dbdeployer-1.64.0.linux.tar.gz)
23+
RUN (cd /dbdeployer && tar -xf dbdeployer-1.64.0.linux.tar.gz)
24+
RUN (cd /dbdeployer && ln -s dbdeployer-1.64.0.linux dbdeployer)
25+
RUN (cd /dbdeployer && ./dbdeployer defaults update reserved-ports '0')
26+
RUN (cd /orchestrator/orchestrator-ci-env/bin/linux && ln -s /dbdeployer/dbdeployer)
27+
28+
# For dev purposes only, just to avoid downloading over and over via download-mysql script
29+
# RUN (mkdir /orchestrator/orchestrator-ci-env/mysql-tarballs-downloaded)
30+
# COPY docker/Percona-Server-8.0.26-16-Linux.x86_64.glibc2.12-minimal.tar.gz /orchestrator/orchestrator-ci-env/mysql-tarballs-downloaded/
1131

1232
RUN (cd /orchestrator/orchestrator-ci-env && cp bin/linux/systemctl.py /usr/bin/systemctl)
1333
RUN (cd /orchestrator/orchestrator-ci-env && script/deploy-haproxy)
@@ -19,4 +39,4 @@ WORKDIR /orchestrator
1939
COPY . .
2040
RUN (cd /orchestrator && script/build)
2141

22-
CMD (cd /orchestrator/orchestrator-ci-env && script/docker-entry && cd /orchestrator && docker/docker-entry-system && /bin/bash)
42+
CMD (cd /orchestrator/orchestrator-ci-env && script/docker-entry && cd /orchestrator && docker/docker-entry-system && docker/docker-entry-system-tests)

‎docker/Dockerfile.test‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,19 @@ RUN rm -rf /var/lib/apt/lists/*
88
ENV WORKPATH /go/src/github.com/openark/orchestrator
99
WORKDIR $WORKPATH
1010

11-
RUN curl -O "https://raw.githubusercontent.com/openark/orchestrator-ci-env/master/bin/linux/dbdeployer.gz"
12-
RUN curl -O "https://raw.githubusercontent.com/openark/orchestrator-ci-env/master/mysql-tarballs/mysql-5.7.26.tar.xz"
13-
RUN gunzip ./dbdeployer.gz
14-
RUN chmod +x ./dbdeployer
15-
RUN mkdir -p ./sandbox/binary
11+
# Setup dbdeployer
12+
RUN mkdir /dbdeployer
13+
RUN (cd /dbdeployer && wget https://github.com/datacharmer/dbdeployer/releases/download/v1.64.0/dbdeployer-1.64.0.linux.tar.gz)
14+
RUN (cd /dbdeployer && tar -xf dbdeployer-1.64.0.linux.tar.gz)
15+
RUN (cd /dbdeployer && ln -s dbdeployer-1.64.0.linux dbdeployer)
16+
RUN (cd /dbdeployer && ./dbdeployer defaults update reserved-ports '0')
1617

17-
RUN ./dbdeployer unpack mysql-5.7.26.tar.xz --sandbox-binary $WORKPATH/sandbox/binary
18+
# For dev purposes only, just to avoid downloading over and over via download-mysql script
19+
# RUN (mkdir /mysql-tarballs-downloaded)
20+
# COPY docker/Percona-Server-8.0.26-16-Linux.x86_64.glibc2.12-minimal.tar.gz /mysql-tarballs-downloaded/
1821

1922
COPY . .
23+
RUN (cd /dbdeployer && ./dbdeployer defaults update reserved-ports '0')
2024

21-
CMD ["script/test-all"]
25+
ENV TARBAL_URL_DEFAULT https://raw.githubusercontent.com/kamil-holubicki/orchestrator-ci-env/master/mysql-tarballs/mysql-5.7.26.tar.xz
26+
CMD (cd $WORKPATH && "script/download-mysql" && "script/test-all")

‎docker/docker-entry-system-tests‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/bin/bash
2+
3+
# Make sure that we use binaries from the bundle we installed, not the system ones
4+
mysql_bin_dir=(~/opt/mysql/*/bin)
5+
PATH=$mysql_bin_dir:$PATH
6+
echo PATH: $PATH
7+
8+
if [ "$RUN_TESTS" == "YES" ] ; then
9+
echo "Automatic tests start requested"
10+
script/test-system
11+
else
12+
echo "Automatic tests start skipped"
13+
/bin/bash
14+
fi

‎run/test-integration.sh‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/bin/bash
2+
3+
export TARBALL_URL=
4+
export RUN_TESTS=
5+
export ALLOW_TESTS_FAILURES=
6+
export CI_ENV_REPO=
7+
export CI_ENV_BRANCH=
8+
9+
# Configure test run parameters
10+
export TARBALL_URL=https://downloads.percona.com/downloads/Percona-Server-8.0/Percona-Server-8.0.26-16/binary/tarball/Percona-Server-8.0.26-16-Linux.x86_64.glibc2.12-minimal.tar.gz
11+
export RUN_TESTS=YES
12+
export ALLOW_TESTS_FAILURES=YES
13+
14+
script/dock test

‎run/test-system.sh‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/bin/bash
2+
3+
export TARBALL_URL=
4+
export RUN_TESTS=
5+
export ALLOW_TESTS_FAILURES=
6+
export CI_ENV_REPO=
7+
export CI_ENV_BRANCH=
8+
9+
# Configure test run parameters
10+
export TARBALL_URL=https://downloads.percona.com/downloads/Percona-Server-8.0/Percona-Server-8.0.26-16/binary/tarball/Percona-Server-8.0.26-16-Linux.x86_64.glibc2.12-minimal.tar.gz
11+
export RUN_TESTS=YES
12+
export ALLOW_TESTS_FAILURES=YES
13+
14+
# It is also possible to specify custom ci-env
15+
#export CI_ENV_REPO=https://github.com/kamil-holubicki/orchestrator-ci-env.git
16+
#export CI_ENV_BRANCH=DISTMYSQL-140
17+
18+
script/dock system

‎script/dock‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,30 @@
99
# dock system: create and run a system test environment
1010
# dock raft: build and run orchestrator in a 3-node raft setup
1111

12+
13+
if [[ -z ${CI_ENV_REPO} ]] ; then
14+
CI_ENV_REPO_ENV=https://github.com/percona/orchestrator-ci-env.git
15+
else
16+
CI_ENV_REPO_ENV=${CI_ENV_REPO}
17+
fi
18+
19+
if [[ -z ${CI_ENV_BRANCH} ]] ; then
20+
CI_ENV_BRANCH_ENV=master
21+
else
22+
CI_ENV_BRANCH_ENV=${CI_ENV_BRANCH}
23+
fi
24+
25+
1226
command="$1"
1327

1428
case "$command" in
1529
"test")
1630
docker_target="orchestrator-test"
17-
docker build . -f docker/Dockerfile.test -t "${docker_target}" && docker run --rm -it "${docker_target}:latest"
31+
docker build . -f docker/Dockerfile.test -t "${docker_target}" && docker run --rm -it --env TARBALL_URL=${TARBALL_URL} --env RUN_TESTS=${RUN_TESTS} --env ALLOW_TESTS_FAILURES=${ALLOW_TESTS_FAILURES} "${docker_target}:latest"
32+
;;
33+
"test-no-it")
34+
docker_target="orchestrator-test"
35+
docker build . -f docker/Dockerfile.test -t "${docker_target}" && docker run --rm --env TARBALL_URL=${TARBALL_URL} --env RUN_TESTS=${RUN_TESTS} --env ALLOW_TESTS_FAILURES=${ALLOW_TESTS_FAILURES} "${docker_target}:latest"
1836
;;
1937
"alpine")
2038
docker_target="orchestrator-alpine"
@@ -37,7 +55,11 @@ case "$command" in
3755
;;
3856
"system")
3957
docker_target="orchestrator-system"
40-
docker build . -f docker/Dockerfile.system -t "${docker_target}" && docker run --rm -it -p 3000:3000 "${docker_target}:latest"
58+
docker build . -f docker/Dockerfile.system -t "${docker_target}" --build-arg ci_env_repo=${CI_ENV_REPO_ENV} --build-arg ci_env_branch=${CI_ENV_BRANCH_ENV} && docker run --rm -it -p 3000:3000 --env TARBALL_URL=${TARBALL_URL} --env RUN_TESTS=${RUN_TESTS} --env ALLOW_TESTS_FAILURES=${ALLOW_TESTS_FAILURES} "${docker_target}:latest"
59+
;;
60+
"system-no-it")
61+
docker_target="orchestrator-system"
62+
docker build . -f docker/Dockerfile.system -t "${docker_target}" --build-arg ci_env_repo=${CI_ENV_REPO_ENV} --build-arg ci_env_branch=${CI_ENV_BRANCH_ENV} && docker run --rm --env TARBALL_URL=${TARBALL_URL} --env RUN_TESTS=${RUN_TESTS} --env ALLOW_TESTS_FAILURES=${ALLOW_TESTS_FAILURES} "${docker_target}:latest"
4163
;;
4264
"raft")
4365
docker_target="orchestrator-raft"

‎script/download-mysql‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/bin/bash
2+
3+
# Download PS tarball. Comment it out if direct copy is done in Dockerfile
4+
5+
if [[ -z "${TARBALL_URL}" ]] ; then
6+
echo Downloading custom tarball skipped. Will use default one.
7+
if [[ -n "${TARBAL_URL_DEFAULT}" ]] ; then
8+
echo Downloading default tarball.
9+
mkdir /mysql-tarballs-downloaded
10+
pushd /mysql-tarballs-downloaded
11+
wget ${TARBAL_URL_DEFAULT}
12+
popd
13+
fi
14+
else
15+
mkdir /mysql-tarballs-downloaded
16+
pushd /mysql-tarballs-downloaded
17+
wget ${TARBALL_URL}
18+
popd
19+
fi

‎script/test-integration‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,30 @@ export GOPATH="$PWD/.gopath"
44
cd .gopath/src/github.com/openark/orchestrator
55

66
setup_mysql() {
7-
if [ ! -f "dbdeployer" ] ; then
7+
if [ ! -f "/dbdeployer/dbdeployer" ] ; then
88
return
99
fi
10-
./dbdeployer deploy single "5.7.21" --sandbox-binary $PWD/sandbox/binary --sandbox-home $PWD/sandboxes --sandbox-directory orc-sandbox --port=3306
10+
11+
if [ -d "/mysql-tarballs-downloaded" ] ; then
12+
# if we downloaded MySQL, use it for tests
13+
tar_file=(/mysql-tarballs-downloaded/*)
14+
else
15+
# use bundled version
16+
tar_file=(mysql-tarballs/*)
17+
fi
18+
19+
mkdir -p ~/opt/mysql
20+
/dbdeployer/dbdeployer unpack $tar_file
21+
22+
pushd ~/opt/mysql
23+
version=(*)
24+
popd
25+
26+
echo tar_file: $tar_file
27+
echo version: $version
28+
29+
/dbdeployer/dbdeployer deploy single $version --sandbox-home $PWD/sandboxes --sandbox-directory orc-sandbox --port=3306
30+
1131
mkdir -p bin
1232
ln -s /go/src/github.com/openark/orchestrator/sandboxes/orc-sandbox/use bin/mysql
1333
chmod +x bin/mysql

‎tests/system/test.sh‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,12 @@ test_step() {
165165
diff_result=$?
166166
if [ $diff_result -ne 0 ] ; then
167167
echo "ERROR $test_name/$test_step_name output does not match expect_output"
168+
echo "TEST OUTPUT"
169+
echo "---"
170+
cat $test_outfile
171+
echo "---"
172+
173+
echo "DIFF"
168174
echo "---"
169175
cat $test_diff_file
170176
echo "---"
@@ -311,7 +317,12 @@ test_all() {
311317
echo "$test_name" >> $tests_successful_file
312318
else
313319
echo "$test_name" >> $tests_failed_file
314-
exit 1
320+
if [ "$ALLOW_TESTS_FAILURES" != "YES" ] ; then
321+
echo "Tests failures not allowed. Exiting."
322+
exit 1
323+
else
324+
echo "Tests failures allowed. Continuing."
325+
fi
315326
fi
316327
else
317328
: # echo "# should not attempt $test_name"

0 commit comments

Comments
 (0)