Skip to content

Commit 17ffc47

Browse files
kenneth-jiakenneth-jia
committed
Add kafka cluster setup scripts and enable integration ci test
1 parent c84d935 commit 17ffc47

File tree

6 files changed

+307
-47
lines changed

6 files changed

+307
-47
lines changed

‎.github/workflows/on_pull_request.yml‎

Lines changed: 136 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
1-
name: On Pull Request
1+
name: pull_request
22

3-
on: [pull_request]
3+
on: [push, pull_request]
44

55
env:
66
CPU_CORE_NUM: 2
77

88
jobs:
99
build_gcc_debug:
10-
name: Build on Linux with GCC (Debug)
10+
name: Build on Linux (GCC, Debug)
1111
runs-on: ubuntu-latest
1212
env:
1313
BUILD_SUB_DIR: build/gcc-debug
@@ -44,6 +44,14 @@ jobs:
4444
./configure
4545
make -j$CPU_CORE_NUM && sudo make install
4646
cd ../
47+
# 6. Install zookeeper
48+
wget https://mirrors.bfsu.edu.cn/apache/zookeeper/zookeeper-3.6.2/apache-zookeeper-3.6.2-bin.tar.gz
49+
tar xvzf apache-zookeeper-3.6.2-bin.tar.gz
50+
# 7. Install kafka
51+
wget https://mirrors.bfsu.edu.cn/apache/kafka/2.6.0/kafka_2.13-2.6.0.tgz
52+
tar xvzf kafka_2.13-2.6.0.tgz
53+
# 8. Install kafkacat
54+
sudo apt install kafkacat
4755
- name: Config
4856
run: |
4957
cd $BUILD_SUB_DIR
@@ -55,10 +63,19 @@ jobs:
5563
- name: Test
5664
run: |
5765
cd $BUILD_SUB_DIR
58-
ctest -VV
66+
export PATH=`pwd`/kafka_2.13-2.6.0/bin:`pwd`/apache-zookeeper-3.6.2-bin/bin:$PATH
67+
# start kafka cluster
68+
../../scripts/start-local-kafka-cluster.py --zookeeper-port 42181 --broker-ports 40091 40092 40093 --log-dir ./tmp/log --output-dir ./tmp/output &
69+
sleep 10
70+
kafkacat -L -b 127.0.0.1:40091
71+
# run tests
72+
export KAFKA_BROKER_LIST="127.0.0.1:40091,127.0.0.1:40092,127.0.0.1:40093"; ctest -VV
73+
# stop kafka cluster
74+
kafka-server-stop.sh
75+
zookeeper-server-stop.sh
5976
6077
build_gcc_release:
61-
name: Build on Linux with GCC (Release)
78+
name: Build on Linux (GCC, Release)
6279
runs-on: ubuntu-latest
6380
env:
6481
BUILD_SUB_DIR: build/gcc-release
@@ -95,6 +112,14 @@ jobs:
95112
./configure
96113
make -j$CPU_CORE_NUM && sudo make install
97114
cd ../
115+
# 6. Install zookeeper
116+
wget https://mirrors.bfsu.edu.cn/apache/zookeeper/zookeeper-3.6.2/apache-zookeeper-3.6.2-bin.tar.gz
117+
tar xvzf apache-zookeeper-3.6.2-bin.tar.gz
118+
# 7. Install kafka
119+
wget https://mirrors.bfsu.edu.cn/apache/kafka/2.6.0/kafka_2.13-2.6.0.tgz
120+
tar xvzf kafka_2.13-2.6.0.tgz
121+
# 8. Install kafkacat
122+
sudo apt install kafkacat
98123
- name: Config
99124
run: |
100125
cd $BUILD_SUB_DIR
@@ -106,11 +131,19 @@ jobs:
106131
- name: Test
107132
run: |
108133
cd $BUILD_SUB_DIR
109-
ctest -VV
110-
134+
export PATH=`pwd`/kafka_2.13-2.6.0/bin:`pwd`/apache-zookeeper-3.6.2-bin/bin:$PATH
135+
# start kafka cluster
136+
../../scripts/start-local-kafka-cluster.py --zookeeper-port 42181 --broker-ports 40091 40092 40093 --log-dir ./tmp/log --output-dir ./tmp/output &
137+
sleep 10
138+
kafkacat -L -b 127.0.0.1:40091
139+
# run tests
140+
export KAFKA_BROKER_LIST="127.0.0.1:40091,127.0.0.1:40092,127.0.0.1:40093"; ctest -VV
141+
# stop kafka cluster
142+
kafka-server-stop.sh
143+
zookeeper-server-stop.sh
111144
112145
build_gcc_cxx14:
113-
name: Build on Linux with GCC (C++14)
146+
name: Build on Linux (GCC, C++14)
114147
runs-on: ubuntu-latest
115148
env:
116149
BUILD_SUB_DIR: build/gcc
@@ -147,6 +180,14 @@ jobs:
147180
./configure
148181
make && sudo make install
149182
cd ../
183+
# 6. Install zookeeper
184+
wget https://mirrors.bfsu.edu.cn/apache/zookeeper/zookeeper-3.6.2/apache-zookeeper-3.6.2-bin.tar.gz
185+
tar xvzf apache-zookeeper-3.6.2-bin.tar.gz
186+
# 7. Install kafka
187+
wget https://mirrors.bfsu.edu.cn/apache/kafka/2.6.0/kafka_2.13-2.6.0.tgz
188+
tar xvzf kafka_2.13-2.6.0.tgz
189+
# 8. Install kafkacat
190+
sudo apt install kafkacat
150191
- name: Config
151192
run: |
152193
cd $BUILD_SUB_DIR
@@ -158,10 +199,19 @@ jobs:
158199
- name: Test
159200
run: |
160201
cd $BUILD_SUB_DIR
161-
ctest -VV
202+
export PATH=`pwd`/kafka_2.13-2.6.0/bin:`pwd`/apache-zookeeper-3.6.2-bin/bin:$PATH
203+
# start kafka cluster
204+
../../scripts/start-local-kafka-cluster.py --zookeeper-port 42181 --broker-ports 40091 40092 40093 --log-dir ./tmp/log --output-dir ./tmp/output &
205+
sleep 10
206+
kafkacat -L -b 127.0.0.1:40091
207+
# run tests
208+
export KAFKA_BROKER_LIST="127.0.0.1:40091,127.0.0.1:40092,127.0.0.1:40093"; ctest -VV
209+
# stop kafka cluster
210+
kafka-server-stop.sh
211+
zookeeper-server-stop.sh
162212
163213
build_gcc_asan:
164-
name: Build on Linux with GCC (ASAN)
214+
name: Build on Linux (GCC, ASAN)
165215
runs-on: ubuntu-latest
166216
env:
167217
BUILD_SUB_DIR: build/gcc-asan
@@ -198,6 +248,14 @@ jobs:
198248
./configure
199249
make -j$CPU_CORE_NUM && sudo make install
200250
cd ../
251+
# 6. Install zookeeper
252+
wget https://mirrors.bfsu.edu.cn/apache/zookeeper/zookeeper-3.6.2/apache-zookeeper-3.6.2-bin.tar.gz
253+
tar xvzf apache-zookeeper-3.6.2-bin.tar.gz
254+
# 7. Install kafka
255+
wget https://mirrors.bfsu.edu.cn/apache/kafka/2.6.0/kafka_2.13-2.6.0.tgz
256+
tar xvzf kafka_2.13-2.6.0.tgz
257+
# 8. Install kafkacat
258+
sudo apt install kafkacat
201259
- name: Config
202260
run: |
203261
cd $BUILD_SUB_DIR
@@ -209,10 +267,19 @@ jobs:
209267
- name: Test
210268
run: |
211269
cd $BUILD_SUB_DIR
212-
ctest -VV
270+
export PATH=`pwd`/kafka_2.13-2.6.0/bin:`pwd`/apache-zookeeper-3.6.2-bin/bin:$PATH
271+
# start kafka cluster
272+
../../scripts/start-local-kafka-cluster.py --zookeeper-port 42181 --broker-ports 40091 40092 40093 --log-dir ./tmp/log --output-dir ./tmp/output &
273+
sleep 10
274+
kafkacat -L -b 127.0.0.1:40091
275+
# run tests
276+
export KAFKA_BROKER_LIST="127.0.0.1:40091,127.0.0.1:40092,127.0.0.1:40093"; ctest -VV
277+
# stop kafka cluster
278+
kafka-server-stop.sh
279+
zookeeper-server-stop.sh
213280
214281
build_gcc_tsan:
215-
name: Build on Linux with GCC (TSAN)
282+
name: Build on Linux (GCC, TSAN)
216283
runs-on: ubuntu-latest
217284
env:
218285
BUILD_SUB_DIR: build/gcc-tsan
@@ -248,6 +315,14 @@ jobs:
248315
./configure
249316
make -j$CPU_CORE_NUM && sudo make install
250317
cd ../
318+
# 6. Install zookeeper
319+
wget https://mirrors.bfsu.edu.cn/apache/zookeeper/zookeeper-3.6.2/apache-zookeeper-3.6.2-bin.tar.gz
320+
tar xvzf apache-zookeeper-3.6.2-bin.tar.gz
321+
# 7. Install kafka
322+
wget https://mirrors.bfsu.edu.cn/apache/kafka/2.6.0/kafka_2.13-2.6.0.tgz
323+
tar xvzf kafka_2.13-2.6.0.tgz
324+
# 8. Install kafkacat
325+
sudo apt install kafkacat
251326
- name: Config
252327
run: |
253328
cd $BUILD_SUB_DIR
@@ -259,11 +334,19 @@ jobs:
259334
- name: Test
260335
run: |
261336
cd $BUILD_SUB_DIR
262-
ctest -VV
263-
337+
export PATH=`pwd`/kafka_2.13-2.6.0/bin:`pwd`/apache-zookeeper-3.6.2-bin/bin:$PATH
338+
# start kafka cluster
339+
../../scripts/start-local-kafka-cluster.py --zookeeper-port 42181 --broker-ports 40091 40092 40093 --log-dir ./tmp/log --output-dir ./tmp/output &
340+
sleep 10
341+
kafkacat -L -b 127.0.0.1:40091
342+
# run tests
343+
export KAFKA_BROKER_LIST="127.0.0.1:40091,127.0.0.1:40092,127.0.0.1:40093"; ctest -VV
344+
# stop kafka cluster
345+
kafka-server-stop.sh
346+
zookeeper-server-stop.sh
264347
265348
build_gcc_ubsan:
266-
name: Build on Linux with GCC (UBSAN)
349+
name: Build on Linux (GCC, UBSAN)
267350
runs-on: ubuntu-latest
268351
env:
269352
BUILD_SUB_DIR: build/gcc-ubsan
@@ -299,6 +382,14 @@ jobs:
299382
./configure
300383
make -j$CPU_CORE_NUM && sudo make install
301384
cd ../
385+
# 6. Install zookeeper
386+
wget https://mirrors.bfsu.edu.cn/apache/zookeeper/zookeeper-3.6.2/apache-zookeeper-3.6.2-bin.tar.gz
387+
tar xvzf apache-zookeeper-3.6.2-bin.tar.gz
388+
# 7. Install kafka
389+
wget https://mirrors.bfsu.edu.cn/apache/kafka/2.6.0/kafka_2.13-2.6.0.tgz
390+
tar xvzf kafka_2.13-2.6.0.tgz
391+
# 8. Install kafkacat
392+
sudo apt install kafkacat
302393
- name: Config
303394
run: |
304395
cd $BUILD_SUB_DIR
@@ -310,10 +401,19 @@ jobs:
310401
- name: Test
311402
run: |
312403
cd $BUILD_SUB_DIR
313-
ctest -VV
404+
export PATH=`pwd`/kafka_2.13-2.6.0/bin:`pwd`/apache-zookeeper-3.6.2-bin/bin:$PATH
405+
# start kafka cluster
406+
../../scripts/start-local-kafka-cluster.py --zookeeper-port 42181 --broker-ports 40091 40092 40093 --log-dir ./tmp/log --output-dir ./tmp/output &
407+
sleep 10
408+
kafkacat -L -b 127.0.0.1:40091
409+
# run tests
410+
export KAFKA_BROKER_LIST="127.0.0.1:40091,127.0.0.1:40092,127.0.0.1:40093"; ctest -VV
411+
# stop kafka cluster
412+
kafka-server-stop.sh
413+
zookeeper-server-stop.sh
314414
315415
build_clang:
316-
name: Build on Linux with Clang
416+
name: Build on Linux (Clang)
317417
runs-on: ubuntu-latest
318418
env:
319419
BUILD_SUB_DIR: build/clang
@@ -350,6 +450,14 @@ jobs:
350450
./configure
351451
make -j$CPU_CORE_NUM && sudo make install
352452
cd ../
453+
# 6. Install zookeeper
454+
wget https://mirrors.bfsu.edu.cn/apache/zookeeper/zookeeper-3.6.2/apache-zookeeper-3.6.2-bin.tar.gz
455+
tar xvzf apache-zookeeper-3.6.2-bin.tar.gz
456+
# 7. Install kafka
457+
wget https://mirrors.bfsu.edu.cn/apache/kafka/2.6.0/kafka_2.13-2.6.0.tgz
458+
tar xvzf kafka_2.13-2.6.0.tgz
459+
# 8. Install kafkacat
460+
sudo apt install kafkacat
353461
- name: Config
354462
run: |
355463
cd $BUILD_SUB_DIR
@@ -361,10 +469,19 @@ jobs:
361469
- name: Test
362470
run: |
363471
cd $BUILD_SUB_DIR
364-
ctest -VV
472+
export PATH=`pwd`/kafka_2.13-2.6.0/bin:`pwd`/apache-zookeeper-3.6.2-bin/bin:$PATH
473+
# start kafka cluster
474+
../../scripts/start-local-kafka-cluster.py --zookeeper-port 42181 --broker-ports 40091 40092 40093 --log-dir ./tmp/log --output-dir ./tmp/output &
475+
sleep 10
476+
kafkacat -L -b 127.0.0.1:40091
477+
# run tests
478+
export KAFKA_BROKER_LIST="127.0.0.1:40091,127.0.0.1:40092,127.0.0.1:40093"; ctest -VV
479+
# stop kafka cluster
480+
kafka-server-stop.sh
481+
zookeeper-server-stop.sh
365482
366483
build_clang_tidy:
367-
name: Build on Linux with Clang (Tidy)
484+
name: Build on Linux (ClangTidy)
368485
runs-on: ubuntu-latest
369486
env:
370487
BUILD_SUB_DIR: build/clang-tidy
@@ -411,10 +528,6 @@ jobs:
411528
run: |
412529
cd $BUILD_SUB_DIR
413530
make -j$CPU_CORE_NUM VERBOSE=1
414-
- name: Test
415-
run: |
416-
cd $BUILD_SUB_DIR
417-
ctest -VV
418531
419532
build_doc:
420533
name: Generate Documents

0 commit comments

Comments
 (0)