Skip to content

Commit abe4c2d

Browse files
wuhuizuowuhuizuo
committed
refactor(pipelines/tidb): use OCI artifacts and per-stage agents in br
pipelines add OCI_TAG_* variables for pd/tikv/tiflash and other deps (ycsb, fake-gcs-server, kes, minio) and handle ticdc special-case; replace FILE_SERVER_URL with OCI_ARTIFACT_HOST. change top-level agent to none and run a kubernetes agent per "Checkout & Prepare" stage, merge debug, checkout and prepare steps into a single stage, increase checkout timeout to 15m, use REFS.repo for checkout dir and update caching. fix tiflash symlink typo across releases.
1 parent 35c9212 commit abe4c2d

File tree

9 files changed

+294
-291
lines changed

9 files changed

+294
-291
lines changed

‎pipelines/pingcap/tidb/release-6.1/pull_br_integration_test.groovy‎

Lines changed: 64 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -8,70 +8,43 @@ final GIT_FULL_REPO_NAME = 'pingcap/tidb'
88
final GIT_CREDENTIALS_ID = 'github-sre-bot-ssh'
99
final POD_TEMPLATE_FILE = 'pipelines/pingcap/tidb/release-6.1/pod-pull_br_integration_test.yaml'
1010
final REFS = readJSON(text: params.JOB_SPEC).refs
11+
final OCI_TAG_PD = component.computeArtifactOciTagFromPR('pd', (REFS.base_ref ==~ /^release-fts-[0-9]+$/ ? 'master' : REFS.base_ref), REFS.pulls[0].title, 'master')
12+
final OCI_TAG_TIKV = component.computeArtifactOciTagFromPR('tikv', REFS.base_ref, REFS.pulls[0].title, 'master')
13+
final OCI_TAG_TIFLASH = component.computeArtifactOciTagFromPR('tiflash', REFS.base_ref, REFS.pulls[0].title, 'master')
14+
final OCI_TAG_YCSB = 'v1.0.3'
15+
final OCI_TAG_FAKE_GCS_SERVER = 'v1.54.0'
16+
final OCI_TAG_KES = 'v0.14.0'
17+
final OCI_TAG_MINIO = 'RELEASE.2020-02-27T00-23-05Z'
1118

1219
prow.setPRDescription(REFS)
13-
1420
pipeline {
15-
agent {
16-
kubernetes {
17-
namespace K8S_NAMESPACE
18-
yamlFile POD_TEMPLATE_FILE
19-
defaultContainer 'golang'
20-
}
21-
}
21+
agent none
2222
environment {
23-
FILE_SERVER_URL = 'http://fileserver.pingcap.net'
23+
OCI_ARTIFACT_HOST = 'hub-zot.pingcap.net/mirrors/hub'
2424
}
2525
options {
2626
timeout(time: 60, unit: 'MINUTES')
27-
// parallelsAlwaysFailFast()
27+
parallelsAlwaysFailFast()
2828
}
2929
stages {
30-
stage('Debug info') {
31-
steps {
32-
sh label: 'Debug info', script: """
33-
printenv
34-
echo "-------------------------"
35-
go env
36-
echo "-------------------------"
37-
echo "debug command: kubectl -n ${K8S_NAMESPACE} exec -ti ${NODE_NAME} bash"
38-
"""
39-
container(name: 'net-tool') {
40-
sh 'dig github.com'
30+
stage('Checkout & Prepare') {
31+
agent {
32+
kubernetes {
33+
namespace K8S_NAMESPACE
34+
yamlFile POD_TEMPLATE_FILE
35+
defaultContainer 'golang'
4136
}
4237
}
43-
}
44-
stage('Checkout') {
45-
options { timeout(time: 5, unit: 'MINUTES') }
38+
options { timeout(time: 15, unit: 'MINUTES') }
4639
steps {
47-
dir("tidb") {
40+
dir(REFS.repo) {
4841
cache(path: "./", includes: '**/*', key: prow.getCacheKey('git', REFS), restoreKeys: prow.getRestoreKeys('git', REFS)) {
4942
retry(2) {
5043
script {
5144
prow.checkoutRefs(REFS)
5245
}
5346
}
5447
}
55-
}
56-
}
57-
}
58-
stage('Prepare') {
59-
steps {
60-
dir("third_party_download") {
61-
retry(2) {
62-
sh label: "download third_party", script: """
63-
chmod +x ../tidb/br/tests/*.sh
64-
chmod +x ${WORKSPACE}/scripts/pingcap/tidb/br_integration_test_download_dependency.sh
65-
${WORKSPACE}/scripts/pingcap/tidb/br_integration_test_download_dependency.sh release-6.1
66-
mkdir -p bin && mv third_bin/* bin/
67-
ls -alh bin/
68-
./bin/pd-server -V
69-
./bin/tikv-server -V
70-
./bin/tiflash --version
71-
"""
72-
}
73-
}
74-
dir('tidb') {
7548
cache(path: "./bin", includes: '**/*', key: prow.getCacheKey('binary', REFS, 'br-integration-test')) {
7649
// build br.test for integration test
7750
// only build binarys if not exist, use the cached binarys if exist
@@ -82,10 +55,40 @@ pipeline {
8255
./bin/tidb-server -V
8356
"""
8457
}
85-
cache(path: "./", includes: '**/*', key: "ws/${BUILD_TAG}/br-lightning") {
86-
sh label: "prepare", script: """
87-
cp -r ../third_party_download/bin/* ./bin/
88-
ls -alh ./bin
58+
dir("bin") {
59+
container("utils") {
60+
script {
61+
retry(2) {
62+
sh label: "download tidb components", script: """
63+
${WORKSPACE}/scripts/artifacts/download_pingcap_oci_artifact.sh \
64+
--pd=${OCI_TAG_PD} \
65+
--pd-ctl=${OCI_TAG_PD} \
66+
--tikv=${OCI_TAG_TIKV} \
67+
--tikv-ctl=${OCI_TAG_TIKV} \
68+
--tiflash=${OCI_TAG_TIFLASH} \
69+
--ycsb=${OCI_TAG_YCSB} \
70+
--fake-gcs-server=${OCI_TAG_FAKE_GCS_SERVER} \
71+
--kes=${OCI_TAG_KES} \
72+
--minio=${OCI_TAG_MINIO} \
73+
--brv408
74+
"""
75+
}
76+
}
77+
}
78+
sh """
79+
mv tiflash tiflash_dir
80+
ln -s tiflash_dir/tiflash tiflash
81+
82+
ls -alh .
83+
./pd-server -V
84+
./tikv-server -V
85+
./tiflash --version
86+
"""
87+
}
88+
// cache workspace for matrix pods
89+
cache(path: "./", includes: '**/*', key: "ws/${BUILD_TAG}") {
90+
sh """
91+
touch rev-${REFS.pulls[0].sha}
8992
"""
9093
}
9194
}
@@ -111,24 +114,25 @@ pipeline {
111114
environment { CODECOV_TOKEN = credentials('codecov-token-tidb') }
112115
options { timeout(time: 45, unit: 'MINUTES') }
113116
steps {
114-
dir('tidb') {
115-
cache(path: "./", includes: '**/*', key: "ws/${BUILD_TAG}/br-lightning") {
116-
sh label: "TEST_GROUP ${TEST_GROUP}", script: """
117-
#!/usr/bin/env bash
118-
cp ${WORKSPACE}/scripts/pingcap/tidb/br-lightning_run_group_v2.sh br/tests/run_group.sh
119-
chmod +x br/tests/*.sh
120-
ln -s ${WORKSPACE}/tidb/bin ${WORKSPACE}/tidb/br/bin
121-
cd br/
122-
./tests/run_group.sh ${TEST_GROUP}
123-
"""
117+
dir(REFS.repo) {
118+
cache(path: "./", includes: '**/*', key: "ws/${BUILD_TAG}") {
119+
sh "ls rev-${REFS.pulls[0].sha}"
124120
}
121+
sh label: "TEST_GROUP ${TEST_GROUP}", script: """
122+
#!/usr/bin/env bash
123+
cp ${WORKSPACE}/scripts/pingcap/tidb/br-lightning_run_group_v2.sh br/tests/run_group.sh
124+
chmod +x br/tests/*.sh
125+
ln -s ${WORKSPACE}/tidb/bin ${WORKSPACE}/tidb/br/bin
126+
cd br/
127+
./tests/run_group.sh ${TEST_GROUP}
128+
"""
125129
}
126130
}
127131
post{
128132
failure {
129133
sh label: "collect logs", script: """
130134
ls /tmp/backup_restore_test
131-
tar -cvzf log-${TEST_GROUP}.tar.gz \$(find /tmp/backup_restore_test/ -type f -name "*.log")
135+
tar --warning=no-file-changed -cvzf log-${TEST_GROUP}.tar.gz \$(find /tmp/backup_restore_test/ -type f -name "*.log")
132136
ls -alh log-${TEST_GROUP}.tar.gz
133137
"""
134138
archiveArtifacts artifacts: "log-${TEST_GROUP}.tar.gz", fingerprint: true

‎pipelines/pingcap/tidb/release-6.5-20241101-v6.5.7/pull_br_integration_test.groovy‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ pipeline {
7777
}
7878
sh """
7979
mv tiflash tiflash_dir
80-
ln -s tiflash_dir/tiflash tifflash
80+
ln -s tiflash_dir/tiflash tiflash
8181
8282
ls -alh .
8383
./pd-server -V

‎pipelines/pingcap/tidb/release-6.5-fips/pull_br_integration_test.groovy‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ pipeline {
7777
}
7878
sh """
7979
mv tiflash tiflash_dir
80-
ln -s tiflash_dir/tiflash tifflash
80+
ln -s tiflash_dir/tiflash tiflash
8181
8282
ls -alh .
8383
./pd-server -V

‎pipelines/pingcap/tidb/release-6.5/pull_br_integration_test.groovy‎

Lines changed: 64 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -8,70 +8,43 @@ final GIT_FULL_REPO_NAME = 'pingcap/tidb'
88
final GIT_CREDENTIALS_ID = 'github-sre-bot-ssh'
99
final POD_TEMPLATE_FILE = 'pipelines/pingcap/tidb/release-6.5/pod-pull_br_integration_test.yaml'
1010
final REFS = readJSON(text: params.JOB_SPEC).refs
11+
final OCI_TAG_PD = component.computeArtifactOciTagFromPR('pd', (REFS.base_ref ==~ /^release-fts-[0-9]+$/ ? 'master' : REFS.base_ref), REFS.pulls[0].title, 'master')
12+
final OCI_TAG_TIKV = component.computeArtifactOciTagFromPR('tikv', REFS.base_ref, REFS.pulls[0].title, 'master')
13+
final OCI_TAG_TIFLASH = component.computeArtifactOciTagFromPR('tiflash', REFS.base_ref, REFS.pulls[0].title, 'master')
14+
final OCI_TAG_YCSB = 'v1.0.3'
15+
final OCI_TAG_FAKE_GCS_SERVER = 'v1.54.0'
16+
final OCI_TAG_KES = 'v0.14.0'
17+
final OCI_TAG_MINIO = 'RELEASE.2020-02-27T00-23-05Z'
1118

1219
prow.setPRDescription(REFS)
13-
1420
pipeline {
15-
agent {
16-
kubernetes {
17-
namespace K8S_NAMESPACE
18-
yamlFile POD_TEMPLATE_FILE
19-
defaultContainer 'golang'
20-
}
21-
}
21+
agent none
2222
environment {
23-
FILE_SERVER_URL = 'http://fileserver.pingcap.net'
23+
OCI_ARTIFACT_HOST = 'hub-zot.pingcap.net/mirrors/hub'
2424
}
2525
options {
2626
timeout(time: 60, unit: 'MINUTES')
27-
// parallelsAlwaysFailFast()
27+
parallelsAlwaysFailFast()
2828
}
2929
stages {
30-
stage('Debug info') {
31-
steps {
32-
sh label: 'Debug info', script: """
33-
printenv
34-
echo "-------------------------"
35-
go env
36-
echo "-------------------------"
37-
echo "debug command: kubectl -n ${K8S_NAMESPACE} exec -ti ${NODE_NAME} bash"
38-
"""
39-
container(name: 'net-tool') {
40-
sh 'dig github.com'
30+
stage('Checkout & Prepare') {
31+
agent {
32+
kubernetes {
33+
namespace K8S_NAMESPACE
34+
yamlFile POD_TEMPLATE_FILE
35+
defaultContainer 'golang'
4136
}
4237
}
43-
}
44-
stage('Checkout') {
45-
options { timeout(time: 5, unit: 'MINUTES') }
38+
options { timeout(time: 15, unit: 'MINUTES') }
4639
steps {
47-
dir("tidb") {
40+
dir(REFS.repo) {
4841
cache(path: "./", includes: '**/*', key: prow.getCacheKey('git', REFS), restoreKeys: prow.getRestoreKeys('git', REFS)) {
4942
retry(2) {
5043
script {
5144
prow.checkoutRefs(REFS)
5245
}
5346
}
5447
}
55-
}
56-
}
57-
}
58-
stage('Prepare') {
59-
steps {
60-
dir("third_party_download") {
61-
retry(2) {
62-
sh label: "download third_party", script: """
63-
chmod +x ../tidb/br/tests/*.sh
64-
chmod +x ${WORKSPACE}/scripts/pingcap/tidb/br_integration_test_download_dependency.sh
65-
${WORKSPACE}/scripts/pingcap/tidb/br_integration_test_download_dependency.sh release-6.5
66-
mkdir -p bin && mv third_bin/* bin/
67-
ls -alh bin/
68-
./bin/pd-server -V
69-
./bin/tikv-server -V
70-
./bin/tiflash --version
71-
"""
72-
}
73-
}
74-
dir('tidb') {
7548
cache(path: "./bin", includes: '**/*', key: prow.getCacheKey('binary', REFS, 'br-integration-test')) {
7649
// build br.test for integration test
7750
// only build binarys if not exist, use the cached binarys if exist
@@ -82,10 +55,40 @@ pipeline {
8255
./bin/tidb-server -V
8356
"""
8457
}
85-
cache(path: "./", includes: '**/*', key: "ws/${BUILD_TAG}/br-lightning") {
86-
sh label: "prepare", script: """
87-
cp -r ../third_party_download/bin/* ./bin/
88-
ls -alh ./bin
58+
dir("bin") {
59+
container("utils") {
60+
script {
61+
retry(2) {
62+
sh label: "download tidb components", script: """
63+
${WORKSPACE}/scripts/artifacts/download_pingcap_oci_artifact.sh \
64+
--pd=${OCI_TAG_PD} \
65+
--pd-ctl=${OCI_TAG_PD} \
66+
--tikv=${OCI_TAG_TIKV} \
67+
--tikv-ctl=${OCI_TAG_TIKV} \
68+
--tiflash=${OCI_TAG_TIFLASH} \
69+
--ycsb=${OCI_TAG_YCSB} \
70+
--fake-gcs-server=${OCI_TAG_FAKE_GCS_SERVER} \
71+
--kes=${OCI_TAG_KES} \
72+
--minio=${OCI_TAG_MINIO} \
73+
--brv408
74+
"""
75+
}
76+
}
77+
}
78+
sh """
79+
mv tiflash tiflash_dir
80+
ln -s tiflash_dir/tiflash tiflash
81+
82+
ls -alh .
83+
./pd-server -V
84+
./tikv-server -V
85+
./tiflash --version
86+
"""
87+
}
88+
// cache workspace for matrix pods
89+
cache(path: "./", includes: '**/*', key: "ws/${BUILD_TAG}") {
90+
sh """
91+
touch rev-${REFS.pulls[0].sha}
8992
"""
9093
}
9194
}
@@ -111,24 +114,25 @@ pipeline {
111114
environment { CODECOV_TOKEN = credentials('codecov-token-tidb') }
112115
options { timeout(time: 45, unit: 'MINUTES') }
113116
steps {
114-
dir('tidb') {
115-
cache(path: "./", includes: '**/*', key: "ws/${BUILD_TAG}/br-lightning") {
116-
sh label: "TEST_GROUP ${TEST_GROUP}", script: """
117-
#!/usr/bin/env bash
118-
cp ${WORKSPACE}/scripts/pingcap/tidb/br-lightning_run_group_v2.sh br/tests/run_group.sh
119-
chmod +x br/tests/*.sh
120-
ln -s ${WORKSPACE}/tidb/bin ${WORKSPACE}/tidb/br/bin
121-
cd br/
122-
./tests/run_group.sh ${TEST_GROUP}
123-
"""
117+
dir(REFS.repo) {
118+
cache(path: "./", includes: '**/*', key: "ws/${BUILD_TAG}") {
119+
sh "ls rev-${REFS.pulls[0].sha}"
124120
}
121+
sh label: "TEST_GROUP ${TEST_GROUP}", script: """
122+
#!/usr/bin/env bash
123+
cp ${WORKSPACE}/scripts/pingcap/tidb/br-lightning_run_group_v2.sh br/tests/run_group.sh
124+
chmod +x br/tests/*.sh
125+
ln -s ${WORKSPACE}/tidb/bin ${WORKSPACE}/tidb/br/bin
126+
cd br/
127+
./tests/run_group.sh ${TEST_GROUP}
128+
"""
125129
}
126130
}
127131
post{
128132
failure {
129133
sh label: "collect logs", script: """
130134
ls /tmp/backup_restore_test
131-
tar -cvzf log-${TEST_GROUP}.tar.gz \$(find /tmp/backup_restore_test/ -type f -name "*.log")
135+
tar --warning=no-file-changed -cvzf log-${TEST_GROUP}.tar.gz \$(find /tmp/backup_restore_test/ -type f -name "*.log")
132136
ls -alh log-${TEST_GROUP}.tar.gz
133137
"""
134138
archiveArtifacts artifacts: "log-${TEST_GROUP}.tar.gz", fingerprint: true

0 commit comments

Comments
 (0)