Skip to content

Commit 35c9212

Browse files
wuhuizuoCopilot
wuhuizuo
and
Copilot
authored
refactor(pipelines): switch downloading binary from pingcap oci artifacts - part2 (#4138)
- **refactor(pipelines): add utils container to pipeline pod templates** - **refactor: add utils container to 9.0-beta pod yamls and switch to OCI artifact tool in pipelines** - **refactor(oci): extend utils container to remaining 9.0-beta pod manifests (lightning, br, e2e, jdbc, mysql, nodejs, python_orm, sqllogic, unit, check2)** --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
1 parent e34d3b8 commit 35c9212

File tree

103 files changed

+1823
-1266
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

103 files changed

+1823
-1266
lines changed

‎libraries/tipipeline/vars/component.groovy‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -50,12 +50,8 @@ def computeBranchFromPR(String component, String prTargetBranch, String prTitle,
5050
} else if (prTargetBranch =~ wipReleaseFeatureBranchReg ) {
5151
// Special handling for feature/materialized_view branch，use the same feature branch for all components
5252
// If the feature/materialized_view is no longer in use, clean up this logic
53-
if (prTargetBranch == 'feature/release-8.5-materialized-view') {
54-
if (component == "ticdc") {
55-
componentBranch = "release-8.5"
56-
} else {
57-
componentBranch = prTargetBranch
58-
}
53+
if (prTargetBranch == 'feature/release-8.5-materialized-view' && component !== "ticdc") {
54+
componentBranch = prTargetBranch
5955
} else {
6056
componentBranch = String.format('release-%s', (prTargetBranch =~ wipReleaseFeatureBranchReg)[0][1]) // => release-X.Y
6157
}

‎pipelines/pingcap-qe/tidb-test/release-6.0/ghpr_integration_mysql_test.groovy‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ final GIT_CREDENTIALS_ID = 'github-sre-bot-ssh'
77
final GIT_FULL_REPO_NAME = 'PingCAP-QE/tidb-test'
88
final POD_TEMPLATE_FILE = 'pipelines/pingcap-qe/tidb-test/release-6.0/pod-ghpr_integration_mysql_test.yaml'
99
final REFS = readJSON(text: params.JOB_SPEC).refs
10+
final OCI_TAG_PD = component.computeArtifactOciTagFromPR('pd', REFS.base_ref, REFS.pulls[0].title, REFS.base_ref)
11+
final OCI_TAG_TIKV = component.computeArtifactOciTagFromPR('tikv', REFS.base_ref, REFS.pulls[0].title, REFS.base_ref)
1012

1113
pipeline {
1214
agent {
@@ -17,7 +19,7 @@ pipeline {
1719
}
1820
}
1921
environment {
20-
FILE_SERVER_URL = 'http://fileserver.pingcap.net'
22+
OCI_ARTIFACT_HOST = 'hub-zot.pingcap.net/mirrors/hub'
2123
}
2224
options {
2325
timeout(time: 70, unit: 'MINUTES')
@@ -66,12 +68,14 @@ pipeline {
6668
dir('tidb') {
6769
cache(path: "./bin", includes: '**/*', key: "ws/${BUILD_TAG}/dependencies") {
6870
sh label: 'tidb-server', script: 'ls bin/tidb-server || make'
69-
sh label: 'download binary', script: """
70-
chmod +x ${WORKSPACE}/scripts/artifacts/*.sh
71-
${WORKSPACE}/scripts/artifacts/download_pingcap_artifact.sh --pd=${REFS.base_ref} --tikv=${REFS.base_ref}
72-
mv third_bin/* bin/
73-
ls -alh bin/
74-
"""
71+
container("utils") {
72+
dir("bin") {
73+
sh label: 'download binary', script: """
74+
${WORKSPACE}/scripts/artifacts/download_pingcap_oci_artifact.sh --pd=${OCI_TAG_PD} --tikv=${OCI_TAG_TIKV}
75+
ls -alh .
76+
"""
77+
}
78+
}
7579
}
7680
}
7781
dir('tidb-test') {

‎pipelines/pingcap-qe/tidb-test/release-6.2/ghpr_integration_mysql_test.groovy‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ final GIT_CREDENTIALS_ID = 'github-sre-bot-ssh'
77
final GIT_FULL_REPO_NAME = 'PingCAP-QE/tidb-test'
88
final POD_TEMPLATE_FILE = 'pipelines/pingcap-qe/tidb-test/release-6.2/pod-ghpr_integration_mysql_test.yaml'
99
final REFS = readJSON(text: params.JOB_SPEC).refs
10+
final OCI_TAG_PD = component.computeArtifactOciTagFromPR('pd', REFS.base_ref, REFS.pulls[0].title, REFS.base_ref)
11+
final OCI_TAG_TIKV = component.computeArtifactOciTagFromPR('tikv', REFS.base_ref, REFS.pulls[0].title, REFS.base_ref)
1012

1113
pipeline {
1214
agent {
@@ -17,7 +19,7 @@ pipeline {
1719
}
1820
}
1921
environment {
20-
FILE_SERVER_URL = 'http://fileserver.pingcap.net'
22+
OCI_ARTIFACT_HOST = 'hub-zot.pingcap.net/mirrors/hub'
2123
}
2224
options {
2325
timeout(time: 45, unit: 'MINUTES')
@@ -66,12 +68,14 @@ pipeline {
6668
dir('tidb') {
6769
cache(path: "./bin", includes: '**/*', key: "ws/${BUILD_TAG}/dependencies") {
6870
sh label: 'tidb-server', script: 'ls bin/tidb-server || make'
69-
sh label: 'download binary', script: """
70-
chmod +x ${WORKSPACE}/scripts/artifacts/*.sh
71-
${WORKSPACE}/scripts/artifacts/download_pingcap_artifact.sh --pd=${REFS.base_ref} --tikv=${REFS.base_ref}
72-
mv third_bin/* bin/
73-
ls -alh bin/
74-
"""
71+
container("utils") {
72+
dir("bin") {
73+
sh label: 'download binary', script: """
74+
${WORKSPACE}/scripts/artifacts/download_pingcap_oci_artifact.sh --pd=${OCI_TAG_PD} --tikv=${OCI_TAG_TIKV}
75+
ls -alh .
76+
"""
77+
}
78+
}
7579
}
7680
}
7781
dir('tidb-test') {

‎pipelines/pingcap-qe/tidb-test/release-6.2/pod-ghpr_integration_mysql_test.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,16 @@ spec:
3030
limits:
3131
memory: 256Mi
3232
cpu: 100m
33+
- name: utils
34+
image: ghcr.io/pingcap-qe/cd/utils/release:v2025.10.12-7-gfdd779c
35+
tty: true
36+
resources:
37+
requests:
38+
memory: 256Mi
39+
cpu: 100m
40+
limits:
41+
cpu: "1"
42+
memory: 4Gi
3343
affinity:
3444
nodeAffinity:
3545
requiredDuringSchedulingIgnoredDuringExecution:

‎pipelines/pingcap/tidb/latest/pull_br_integration_test.groovy‎

Lines changed: 62 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -8,41 +8,44 @@ final GIT_FULL_REPO_NAME = 'pingcap/tidb'
88
final GIT_CREDENTIALS_ID = 'github-sre-bot-ssh'
99
final POD_TEMPLATE_FILE = 'pipelines/pingcap/tidb/latest/pod-pull_br_integration_test.yaml'
1010
final REFS = readJSON(text: params.JOB_SPEC).refs
11+
final OCI_TAG_PD = component.computeArtifactOciTagFromPR('pd', (REFS.base_ref ==~ /^release-fts-[0-9]+$/ ? 'master' : REFS.base_ref), REFS.pulls[0].title, 'master')
12+
final OCI_TAG_TIKV = component.computeArtifactOciTagFromPR('tikv', REFS.base_ref, REFS.pulls[0].title, 'master')
13+
final OCI_TAG_TIFLASH = component.computeArtifactOciTagFromPR('tiflash', REFS.base_ref, REFS.pulls[0].title, 'master')
14+
final OCI_TAG_YCSB = 'v1.0.3'
15+
final OCI_TAG_FAKE_GCS_SERVER = 'v1.54.0'
16+
final OCI_TAG_KES = 'v0.14.0'
17+
final OCI_TAG_MINIO = 'RELEASE.2020-02-27T00-23-05Z'
1118

1219
prow.setPRDescription(REFS)
1320
pipeline {
14-
agent {
15-
kubernetes {
16-
namespace K8S_NAMESPACE
17-
yamlFile POD_TEMPLATE_FILE
18-
defaultContainer 'golang'
19-
}
20-
}
21+
agent none
2122
environment {
22-
FILE_SERVER_URL = 'http://fileserver.pingcap.net'
23+
OCI_ARTIFACT_HOST = 'hub-zot.pingcap.net/mirrors/hub'
2324
}
2425
options {
2526
timeout(time: 90, unit: 'MINUTES')
2627
parallelsAlwaysFailFast()
2728
}
2829
stages {
29-
stage('Checkout') {
30-
options { timeout(time: 5, unit: 'MINUTES') }
30+
stage('Checkout & Prepare') {
31+
agent {
32+
kubernetes {
33+
namespace K8S_NAMESPACE
34+
yamlFile POD_TEMPLATE_FILE
35+
defaultContainer 'golang'
36+
}
37+
}
38+
options { timeout(time: 15, unit: 'MINUTES') }
3139
steps {
32-
dir("tidb") {
40+
dir(REFS.repo) {
3341
cache(path: "./", includes: '**/*', key: prow.getCacheKey('git', REFS), restoreKeys: prow.getRestoreKeys('git', REFS)) {
3442
retry(2) {
3543
script {
3644
prow.checkoutRefs(REFS)
3745
}
3846
}
3947
}
40-
}
41-
}
42-
}
43-
stage('Prepare') {
44-
steps {
45-
dir('tidb') {
48+
4649
cache(path: "./bin", includes: '**/*', key: prow.getCacheKey('binary', REFS, 'br-integration-test')) {
4750
// build br.test for integration test
4851
// only build binarys if not exist, use the cached binarys if exist
@@ -53,6 +56,43 @@ pipeline {
5356
./bin/tidb-server -V
5457
"""
5558
}
59+
dir("bin") {
60+
container("utils") {
61+
script {
62+
retry(2) {
63+
sh label: "download tidb components", script: """
64+
${WORKSPACE}/scripts/artifacts/download_pingcap_oci_artifact.sh \
65+
--pd=${OCI_TAG_PD} \
66+
--pd-ctl=${OCI_TAG_PD} \
67+
--tikv=${OCI_TAG_TIKV} \
68+
--tikv-ctl=${OCI_TAG_TIKV} \
69+
--tiflash=${OCI_TAG_TIFLASH} \
70+
--ycsb=${OCI_TAG_YCSB} \
71+
--fake-gcs-server=${OCI_TAG_FAKE_GCS_SERVER} \
72+
--kes=${OCI_TAG_KES} \
73+
--minio=${OCI_TAG_MINIO} \
74+
--brv408
75+
"""
76+
}
77+
}
78+
}
79+
sh """
80+
mv tiflash tiflash_dir
81+
ln -s tiflash_dir/tiflash tiflash
82+
83+
ls -alh .
84+
./pd-server -V
85+
./tikv-server -V
86+
./tiflash --version
87+
"""
88+
}
89+
90+
// cache workspace for matrix pods
91+
cache(path: "./", includes: '**/*', key: "ws/${BUILD_TAG}") {
92+
sh """
93+
touch rev-${REFS.pulls[0].sha}
94+
"""
95+
}
5696
}
5797
}
5898
}
@@ -72,57 +112,14 @@ pipeline {
72112
}
73113
}
74114
stages {
75-
stage('Prepare') {
76-
steps {
77-
dir("tidb") {
78-
cache(path: "./", includes: '**/*', key: prow.getCacheKey('git', REFS)) {
79-
sh 'ls -lh'
80-
}
81-
}
82-
dir("third_party_download") {
83-
script {
84-
def otherComponentBranch = component.computeBranchFromPR('other', REFS.base_ref, REFS.pulls[0].title, 'master')
85-
retry(2) {
86-
sh label: "download third_party", script: """
87-
chmod +x ${WORKSPACE}/tidb/br/tests/*.sh
88-
${WORKSPACE}/tidb/br/tests/download_integration_test_binaries.sh ${otherComponentBranch}
89-
rm -rf bin/ && mkdir -p bin
90-
mv third_bin/* bin/
91-
ls -alh bin/
92-
./bin/pd-server -V
93-
./bin/tikv-server -V
94-
./bin/tiflash --version
95-
"""
96-
}
97-
}
98-
}
99-
dir('tidb') {
100-
sh label: "check all tests added to group", script: """#!/usr/bin/env bash
101-
chmod +x br/tests/*.sh
102-
./br/tests/run_group_br_tests.sh others
103-
"""
104-
// build br.test for integration test
105-
// only build binarys if not exist, use the cached binarys if exist
106-
cache(path: "./bin", includes: '**/*', key: prow.getCacheKey('binary', REFS, 'br-integration-test')) {
107-
sh label: "prepare", script: """
108-
[ -f ./bin/tidb-server ] || (make failpoint-enable && make && make failpoint-disable)
109-
[ -f ./bin/br.test ] || make build_for_br_integration_test
110-
ls -alh ./bin
111-
./bin/tidb-server -V
112-
"""
113-
}
114-
sh label: "prepare", script: """
115-
cp -r ../third_party_download/bin/* ./bin/
116-
ls -alh ./bin
117-
"""
118-
}
119-
}
120-
}
121115
stage("Test") {
122116
environment { CODECOV_TOKEN = credentials('codecov-token-tidb') }
123117
options { timeout(time: 60, unit: 'MINUTES') }
124118
steps {
125-
dir('tidb') {
119+
dir(REFS.repo) {
120+
cache(path: "./", includes: '**/*', key: "ws/${BUILD_TAG}") {
121+
sh "ls rev-${REFS.pulls[0].sha}"
122+
}
126123
sh label: "TEST_GROUP ${TEST_GROUP}", script: """#!/usr/bin/env bash
127124
chmod +x br/tests/*.sh
128125
./br/tests/run_group_br_tests.sh ${TEST_GROUP}
@@ -139,7 +136,7 @@ pipeline {
139136
archiveArtifacts artifacts: "log-${TEST_GROUP}.tar.gz", fingerprint: true
140137
}
141138
success {
142-
dir('tidb'){
139+
dir(REFS.repo) {
143140
sh label: "upload coverage", script: """
144141
ls -alh /tmp/group_cover
145142
gocovmerge /tmp/group_cover/cov.* > coverage.txt

‎pipelines/pingcap/tidb/latest/pull_integration_e2e_test.groovy‎

Lines changed: 32 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,10 @@
66
final K8S_NAMESPACE = "jenkins-tidb"
77
final POD_TEMPLATE_FILE = 'pipelines/pingcap/tidb/latest/pod-pull_integration_e2e_test.yaml'
88
final REFS = readJSON(text: params.JOB_SPEC).refs
9+
final OCI_TAG_PD = component.computeArtifactOciTagFromPR('pd', (REFS.base_ref ==~ /^release-fts-[0-9]+$/ ? 'master' : REFS.base_ref), REFS.pulls[0].title, 'master')
10+
final OCI_TAG_TIKV = component.computeArtifactOciTagFromPR('tikv', REFS.base_ref, REFS.pulls[0].title, 'master')
11+
final OCI_TAG_TIFLASH = component.computeArtifactOciTagFromPR('tiflash', REFS.base_ref, REFS.pulls[0].title, 'master')
12+
final OCI_TAG_TICDC_NEW = (REFS.base_ref == "feature/materialized_view" ? "release-8.5" : component.computeArtifactOciTagFromPR('ticdc', REFS.base_ref, REFS.pulls[0].title, 'master'))
913

1014
prow.setPRDescription(REFS)
1115
pipeline {
@@ -17,15 +21,15 @@ pipeline {
1721
}
1822
}
1923
environment {
20-
FILE_SERVER_URL = 'http://fileserver.pingcap.net'
24+
OCI_ARTIFACT_HOST = 'hub-zot.pingcap.net/mirrors/hub' // cache mirror for us-docker.pkg.dev/pingcap-testing-account/hub
2125
}
2226
options {
2327
timeout(time: 60, unit: 'MINUTES')
2428
}
2529
stages {
2630
stage('Checkout') {
2731
steps {
28-
dir('tidb') {
32+
dir(REFS.repo) {
2933
cache(path: "./", includes: '**/*', key: prow.getCacheKey('git', REFS), restoreKeys: prow.getRestoreKeys('git', REFS)) {
3034
retry(2) {
3135
script {
@@ -36,40 +40,39 @@ pipeline {
3640
}
3741
}
3842
}
39-
stage('Prepare') {
40-
steps {
41-
dir('tidb') {
42-
script {
43-
// Computes the branch name for downloading binaries based on the PR target branch and title.
44-
def otherComponentBranch = component.computeBranchFromPR('other', REFS.base_ref, REFS.pulls[0].title, 'master')
45-
retry(3) {
46-
sh label: 'download binary', script: """
47-
cd tests/integrationtest2 && ./download_integration_test_binaries.sh ${otherComponentBranch}
48-
ls -alh third_bin/
49-
./third_bin/tikv-server -V
50-
./third_bin/pd-server -V
51-
./third_bin/tiflash --version
52-
./third_bin/cdc version
53-
"""
54-
}
55-
}
56-
}
57-
}
58-
}
5943
stage('Tests') {
6044
options { timeout(time: 45, unit: 'MINUTES') }
6145
steps {
62-
dir('tidb') {
63-
sh label: 'test', script: """
64-
cd tests/integrationtest2 && ./run-tests.sh
65-
"""
46+
dir("${REFS.repo}/tests/integrationtest2") {
47+
dir("third_bin") {
48+
container("utils") {
49+
script {
50+
retry(2) {
51+
sh label: "download tidb components", script: """
52+
${WORKSPACE}/scripts/artifacts/download_pingcap_oci_artifact.sh \
53+
--pd=${OCI_TAG_PD} \
54+
--tikv=${OCI_TAG_TIKV} \
55+
--tiflash=${OCI_TAG_TIFLASH} \
56+
--ticdc-new=${OCI_TAG_TICDC_NEW}
57+
"""
58+
}
59+
}
60+
}
61+
sh '''
62+
mv tiflash tiflash_dir
63+
ln -s `pwd`/tiflash_dir/tiflash tiflash
64+
65+
./tikv-server -V
66+
./pd-server -V
67+
./tiflash --version
68+
'''
69+
}
70+
sh './run-tests.sh'
6671
}
6772
}
6873
post{
6974
failure {
70-
script {
71-
archiveArtifacts(artifacts: 'tidb/tests/integrationtest2/logs/*.log', allowEmptyArchive: true)
72-
}
75+
archiveArtifacts(artifacts: 'tidb/tests/integrationtest2/logs/*.log', allowEmptyArchive: true)
7376
}
7477
}
7578
}

0 commit comments

Comments
 (0)