Skip to content

Commit 0d032dd

Browse files
kamil-holubickikamil-holubicki
authored
Merge pull request #1 from kamil-holubicki/DISTMYSQL-140
DISTMYSQL-140: Create dev Jenkins pipeline for Orchestrator
2 parents f3ed383 + 3c3dea1 commit 0d032dd

File tree

9 files changed

+157
-14
lines changed

9 files changed

+157
-14
lines changed

‎docker/Dockerfile.system‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,33 @@
11
FROM golang:1.16.6-stretch
22
LABEL maintainer="[email protected]"
33

4+
ARG ci_env_repo="https://github.com/percona/orchestrator-ci-env.git"
5+
ARG ci_env_branch=master
6+
7+
RUN echo "ci_env_repo: $ci_env_repo"
8+
RUN echo "ci_env_branch: $ci_env_branch"
9+
410
RUN apt-get update -q -y
511
RUN apt-get install -y sudo haproxy python git jq rsync libaio1 libnuma1 mysql-client bsdmainutils less vim
612

713
RUN mkdir /orchestrator
814
WORKDIR /orchestrator
915

10-
RUN git clone https://github.com/openark/orchestrator-ci-env.git # cache
16+
RUN git clone -b $ci_env_branch $ci_env_repo # cache
17+
# For dev purposes only, just to avoid cloning over and over
18+
# COPY docker/orchestrator-ci-env /orchestrator/orchestrator-ci-env
19+
20+
# Setup dbdeployer
21+
RUN mkdir /dbdeployer
22+
RUN (cd /dbdeployer && wget https://github.com/datacharmer/dbdeployer/releases/download/v1.64.0/dbdeployer-1.64.0.linux.tar.gz)
23+
RUN (cd /dbdeployer && tar -xf dbdeployer-1.64.0.linux.tar.gz)
24+
RUN (cd /dbdeployer && ln -s dbdeployer-1.64.0.linux dbdeployer)
25+
RUN (cd /dbdeployer && ./dbdeployer defaults update reserved-ports '0')
26+
RUN (cd /orchestrator/orchestrator-ci-env/bin/linux && ln -s /dbdeployer/dbdeployer)
27+
28+
# For dev purposes only, just to avoid downloading over and over via download-mysql script
29+
# RUN (mkdir /orchestrator/orchestrator-ci-env/mysql-tarballs-downloaded)
30+
# COPY docker/Percona-Server-8.0.26-16-Linux.x86_64.glibc2.12-minimal.tar.gz /orchestrator/orchestrator-ci-env/mysql-tarballs-downloaded/
1131

1232
RUN (cd /orchestrator/orchestrator-ci-env && cp bin/linux/systemctl.py /usr/bin/systemctl)
1333
RUN (cd /orchestrator/orchestrator-ci-env && script/deploy-haproxy)
@@ -19,4 +39,4 @@ WORKDIR /orchestrator
1939
COPY . .
2040
RUN (cd /orchestrator && script/build)
2141

22-
CMD (cd /orchestrator/orchestrator-ci-env && script/docker-entry && cd /orchestrator && docker/docker-entry-system && /bin/bash)
42+
CMD (cd /orchestrator/orchestrator-ci-env && script/docker-entry && cd /orchestrator && docker/docker-entry-system && docker/docker-entry-system-tests)

‎docker/Dockerfile.test‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,19 @@ RUN rm -rf /var/lib/apt/lists/*
88
ENV WORKPATH /go/src/github.com/openark/orchestrator
99
WORKDIR $WORKPATH
1010

11-
RUN curl -O "https://raw.githubusercontent.com/openark/orchestrator-ci-env/master/bin/linux/dbdeployer.gz"
12-
RUN curl -O "https://raw.githubusercontent.com/openark/orchestrator-ci-env/master/mysql-tarballs/mysql-5.7.26.tar.xz"
13-
RUN gunzip ./dbdeployer.gz
14-
RUN chmod +x ./dbdeployer
15-
RUN mkdir -p ./sandbox/binary
11+
# Setup dbdeployer
12+
RUN mkdir /dbdeployer
13+
RUN (cd /dbdeployer && wget https://github.com/datacharmer/dbdeployer/releases/download/v1.64.0/dbdeployer-1.64.0.linux.tar.gz)
14+
RUN (cd /dbdeployer && tar -xf dbdeployer-1.64.0.linux.tar.gz)
15+
RUN (cd /dbdeployer && ln -s dbdeployer-1.64.0.linux dbdeployer)
16+
RUN (cd /dbdeployer && ./dbdeployer defaults update reserved-ports '0')
1617

17-
RUN ./dbdeployer unpack mysql-5.7.26.tar.xz --sandbox-binary $WORKPATH/sandbox/binary
18+
# For dev purposes only, just to avoid downloading over and over via download-mysql script
19+
# RUN (mkdir /mysql-tarballs-downloaded)
20+
# COPY docker/Percona-Server-8.0.26-16-Linux.x86_64.glibc2.12-minimal.tar.gz /mysql-tarballs-downloaded/
1821

1922
COPY . .
23+
RUN (cd /dbdeployer && ./dbdeployer defaults update reserved-ports '0')
2024

21-
CMD ["script/test-all"]
25+
ENV TARBAL_URL_DEFAULT https://raw.githubusercontent.com/percona/orchestrator-ci-env/master/mysql-tarballs/mysql-5.7.26.tar.xz
26+
CMD (cd $WORKPATH && "script/download-mysql" && "script/test-all")

‎docker/docker-entry-system-tests‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/bin/bash
2+
3+
# Make sure that we use binaries from the bundle we installed, not the system ones
4+
mysql_bin_dir=(~/opt/mysql/*/bin)
5+
PATH=$mysql_bin_dir:$PATH
6+
echo PATH: $PATH
7+
8+
if [ "$RUN_TESTS" == "YES" ] ; then
9+
echo "Automatic tests start requested"
10+
script/test-system
11+
else
12+
echo "Automatic tests start skipped"
13+
/bin/bash
14+
fi

‎run/test-integration.sh‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/bin/bash
2+
3+
export TARBALL_URL=
4+
export RUN_TESTS=
5+
export ALLOW_TESTS_FAILURES=
6+
export CI_ENV_REPO=
7+
export CI_ENV_BRANCH=
8+
9+
# Configure test run parameters
10+
export TARBALL_URL=https://downloads.percona.com/downloads/Percona-Server-8.0/Percona-Server-8.0.26-16/binary/tarball/Percona-Server-8.0.26-16-Linux.x86_64.glibc2.12-minimal.tar.gz
11+
export RUN_TESTS=YES
12+
export ALLOW_TESTS_FAILURES=YES
13+
14+
script/dock test

‎run/test-system.sh‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/bin/bash
2+
3+
export TARBALL_URL=
4+
export RUN_TESTS=
5+
export ALLOW_TESTS_FAILURES=
6+
export CI_ENV_REPO=
7+
export CI_ENV_BRANCH=
8+
9+
# Configure test run parameters
10+
export TARBALL_URL=https://downloads.percona.com/downloads/Percona-Server-8.0/Percona-Server-8.0.26-16/binary/tarball/Percona-Server-8.0.26-16-Linux.x86_64.glibc2.12-minimal.tar.gz
11+
export RUN_TESTS=YES
12+
export ALLOW_TESTS_FAILURES=YES
13+
14+
# It is also possible to specify custom ci-env
15+
#export CI_ENV_REPO=https://github.com/kamil-holubicki/orchestrator-ci-env.git
16+
#export CI_ENV_BRANCH=DISTMYSQL-140
17+
18+
script/dock system

‎script/dock‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,30 @@
99
# dock system: create and run a system test environment
1010
# dock raft: build and run orchestrator in a 3-node raft setup
1111

12+
13+
if [[ -z ${CI_ENV_REPO} ]] ; then
14+
CI_ENV_REPO_ENV=https://github.com/percona/orchestrator-ci-env.git
15+
else
16+
CI_ENV_REPO_ENV=${CI_ENV_REPO}
17+
fi
18+
19+
if [[ -z ${CI_ENV_BRANCH} ]] ; then
20+
CI_ENV_BRANCH_ENV=master
21+
else
22+
CI_ENV_BRANCH_ENV=${CI_ENV_BRANCH}
23+
fi
24+
25+
1226
command="$1"
1327

1428
case "$command" in
1529
"test")
1630
docker_target="orchestrator-test"
17-
docker build . -f docker/Dockerfile.test -t "${docker_target}" && docker run --rm -it "${docker_target}:latest"
31+
docker build . -f docker/Dockerfile.test -t "${docker_target}" && docker run --rm -it --env TARBALL_URL=${TARBALL_URL} --env RUN_TESTS=${RUN_TESTS} --env ALLOW_TESTS_FAILURES=${ALLOW_TESTS_FAILURES} "${docker_target}:latest"
32+
;;
33+
"test-no-it")
34+
docker_target="orchestrator-test"
35+
docker build . -f docker/Dockerfile.test -t "${docker_target}" && docker run --rm --env TARBALL_URL=${TARBALL_URL} --env RUN_TESTS=${RUN_TESTS} --env ALLOW_TESTS_FAILURES=${ALLOW_TESTS_FAILURES} "${docker_target}:latest"
1836
;;
1937
"alpine")
2038
docker_target="orchestrator-alpine"
@@ -37,7 +55,11 @@ case "$command" in
3755
;;
3856
"system")
3957
docker_target="orchestrator-system"
40-
docker build . -f docker/Dockerfile.system -t "${docker_target}" && docker run --rm -it -p 3000:3000 "${docker_target}:latest"
58+
docker build . -f docker/Dockerfile.system -t "${docker_target}" --build-arg ci_env_repo=${CI_ENV_REPO_ENV} --build-arg ci_env_branch=${CI_ENV_BRANCH_ENV} && docker run --rm -it -p 3000:3000 --env TARBALL_URL=${TARBALL_URL} --env RUN_TESTS=${RUN_TESTS} --env ALLOW_TESTS_FAILURES=${ALLOW_TESTS_FAILURES} "${docker_target}:latest"
59+
;;
60+
"system-no-it")
61+
docker_target="orchestrator-system"
62+
docker build . -f docker/Dockerfile.system -t "${docker_target}" --build-arg ci_env_repo=${CI_ENV_REPO_ENV} --build-arg ci_env_branch=${CI_ENV_BRANCH_ENV} && docker run --rm --env TARBALL_URL=${TARBALL_URL} --env RUN_TESTS=${RUN_TESTS} --env ALLOW_TESTS_FAILURES=${ALLOW_TESTS_FAILURES} "${docker_target}:latest"
4163
;;
4264
"raft")
4365
docker_target="orchestrator-raft"

‎script/download-mysql‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/bin/bash
2+
3+
# Download PS tarball. Comment it out if direct copy is done in Dockerfile
4+
5+
if [[ -z "${TARBALL_URL}" ]] ; then
6+
echo Downloading custom tarball skipped. Will use default one.
7+
if [[ -n "${TARBAL_URL_DEFAULT}" ]] ; then
8+
echo Downloading default tarball.
9+
mkdir /mysql-tarballs-downloaded
10+
pushd /mysql-tarballs-downloaded
11+
wget ${TARBAL_URL_DEFAULT}
12+
popd
13+
fi
14+
else
15+
mkdir /mysql-tarballs-downloaded
16+
pushd /mysql-tarballs-downloaded
17+
wget ${TARBALL_URL}
18+
popd
19+
fi

‎script/test-integration‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,30 @@ export GOPATH="$PWD/.gopath"
44
cd .gopath/src/github.com/openark/orchestrator
55

66
setup_mysql() {
7-
if [ ! -f "dbdeployer" ] ; then
7+
if [ ! -f "/dbdeployer/dbdeployer" ] ; then
88
return
99
fi
10-
./dbdeployer deploy single "5.7.21" --sandbox-binary $PWD/sandbox/binary --sandbox-home $PWD/sandboxes --sandbox-directory orc-sandbox --port=3306
10+
11+
if [ -d "/mysql-tarballs-downloaded" ] ; then
12+
# if we downloaded MySQL, use it for tests
13+
tar_file=(/mysql-tarballs-downloaded/*)
14+
else
15+
# use bundled version
16+
tar_file=(mysql-tarballs/*)
17+
fi
18+
19+
mkdir -p ~/opt/mysql
20+
/dbdeployer/dbdeployer unpack $tar_file
21+
22+
pushd ~/opt/mysql
23+
version=(*)
24+
popd
25+
26+
echo tar_file: $tar_file
27+
echo version: $version
28+
29+
/dbdeployer/dbdeployer deploy single $version --sandbox-home $PWD/sandboxes --sandbox-directory orc-sandbox --port=3306
30+
1131
mkdir -p bin
1232
ln -s /go/src/github.com/openark/orchestrator/sandboxes/orc-sandbox/use bin/mysql
1333
chmod +x bin/mysql

‎tests/system/test.sh‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,12 @@ test_step() {
165165
diff_result=$?
166166
if [ $diff_result -ne 0 ] ; then
167167
echo "ERROR $test_name/$test_step_name output does not match expect_output"
168+
echo "TEST OUTPUT"
169+
echo "---"
170+
cat $test_outfile
171+
echo "---"
172+
173+
echo "DIFF"
168174
echo "---"
169175
cat $test_diff_file
170176
echo "---"
@@ -311,7 +317,12 @@ test_all() {
311317
echo "$test_name" >> $tests_successful_file
312318
else
313319
echo "$test_name" >> $tests_failed_file
314-
exit 1
320+
if [ "$ALLOW_TESTS_FAILURES" != "YES" ] ; then
321+
echo "Tests failures not allowed. Exiting."
322+
exit 1
323+
else
324+
echo "Tests failures allowed. Continuing."
325+
fi
315326
fi
316327
else
317328
: # echo "# should not attempt $test_name"

0 commit comments

Comments
 (0)