Skip to content

Commit 5c1c2eb

Browse files
wankai123wankai123
authored
BanyanDB stages E2E (#13249)
1 parent ad61a58 commit 5c1c2eb

File tree

28 files changed

+1270
-22
lines changed

28 files changed

+1270
-22
lines changed

‎.github/workflows/skywalking.yaml‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -716,6 +716,10 @@ jobs:
716716
with:
717717
submodules: true
718718
persist-credentials: false
719+
- run: grep -v '^#' test/e2e-v2/script/env >> "$GITHUB_ENV"
720+
- uses: apache/skywalking-cli/actions/setup@master
721+
with:
722+
version: ${{ env.SW_CTL_COMMIT }}
719723
- uses: actions/download-artifact@v4
720724
name: Download docker images
721725
with:
@@ -796,6 +800,10 @@ jobs:
796800
with:
797801
submodules: true
798802
persist-credentials: false
803+
- run: grep -v '^#' test/e2e-v2/script/env >> "$GITHUB_ENV"
804+
- uses: apache/skywalking-cli/actions/setup@master
805+
with:
806+
version: ${{ env.SW_CTL_COMMIT }}
799807
- uses: actions/download-artifact@v4
800808
name: Download docker images
801809
with:
@@ -848,6 +856,10 @@ jobs:
848856
with:
849857
submodules: true
850858
persist-credentials: false
859+
- run: grep -v '^#' test/e2e-v2/script/env >> "$GITHUB_ENV"
860+
- uses: apache/skywalking-cli/actions/setup@master
861+
with:
862+
version: ${{ env.SW_CTL_COMMIT }}
851863
- uses: actions/download-artifact@v4
852864
name: Download docker images
853865
with:
@@ -877,6 +889,111 @@ jobs:
877889
with:
878890
e2e-file: $GITHUB_WORKSPACE/test/e2e-v2/cases/simple/jdk/e2e.yaml
879891

892+
e2e-test-banyandb-stages:
893+
if: |
894+
( always() && ! cancelled() ) &&
895+
((github.event_name == 'schedule' && github.repository == 'apache/skywalking') || needs.changes.outputs.oap == 'true')
896+
name: E2E test
897+
needs: [docker, dist-tar]
898+
runs-on: ${{ matrix.test.runs-on || 'ubuntu-latest' }}
899+
timeout-minutes: 60
900+
env:
901+
BANYANDB_DATA_GENERATE_ROOT: test/e2e-v2/cases/storage/banyandb/data-generate
902+
strategy:
903+
fail-fast: false
904+
matrix:
905+
test:
906+
- name: BanyanDB Stages
907+
config: test/e2e-v2/cases/storage/banyandb/stages/e2e.yaml
908+
steps:
909+
- uses: actions/checkout@v4
910+
with:
911+
submodules: true
912+
persist-credentials: false
913+
- run: grep -v '^#' test/e2e-v2/script/env >> "$GITHUB_ENV"
914+
- uses: apache/skywalking-cli/actions/setup@master
915+
with:
916+
version: ${{ env.SW_CTL_COMMIT }}
917+
- uses: actions/download-artifact@v4
918+
name: Download docker images
919+
with:
920+
name: docker-images-11
921+
path: docker-images
922+
- name: Load docker images
923+
run: |
924+
find docker-images -name "*.tar" -exec docker load -i {} \;
925+
find docker-images -name "*.tar" -exec rm {} \;
926+
- uses: actions/download-artifact@v4
927+
name: Download distribution tar
928+
with:
929+
name: dist
930+
path: dist
931+
- name: Login to ghcr
932+
uses: docker/login-action@v3
933+
with:
934+
registry: ghcr.io
935+
username: ${{ github.repository_owner }}
936+
password: ${{ secrets.GITHUB_TOKEN }}
937+
- name: Cache maven repository
938+
uses: actions/cache@v4
939+
with:
940+
path: ~/.m2/repository
941+
key: ${{ runner.os }}-maven-${{ hashFiles('test/e2e-v2/java-test-service/**/pom.xml') }}
942+
restore-keys: ${{ runner.os }}-maven-
943+
- name: Prepare test services
944+
shell: bash
945+
run: ./mvnw -B -q -f test/e2e-v2/java-test-service/pom.xml clean flatten:flatten package
946+
- name: Set env var
947+
run: |
948+
echo "${{ matrix.test.env }}" >> $GITHUB_ENV
949+
- name: Build test image
950+
if: matrix.test.docker != null
951+
run: docker build -t ${{ matrix.test.docker.name }} -f ${{ matrix.test.docker.base }}/${{ matrix.test.docker.file }} ${{ matrix.test.docker.base }}
952+
- name: Generate BanyanDB cold data
953+
run: |
954+
export $(grep -v '^#' test/e2e-v2/script/env | xargs)
955+
docker compose -f ${BANYANDB_DATA_GENERATE_ROOT}/docker-compose.yml up -d
956+
CONTAINER_ID=$(docker compose -f ${BANYANDB_DATA_GENERATE_ROOT}/docker-compose.yml ps -q banyandb)
957+
echo "⌛ monitoring segment files..."
958+
found=false
959+
for i in {1..60}; do
960+
# check if segment files exist
961+
if docker exec $CONTAINER_ID sh -c '[ -n "$(ls /tmp/measure-data/measure/data/day/seg* 2>/dev/null)" ]'; then
962+
echo "✅ found segment files"
963+
sleep 30
964+
# create and copy files
965+
docker cp $CONTAINER_ID:/tmp ${BANYANDB_DATA_GENERATE_ROOT}
966+
docker cp $CONTAINER_ID:/tmp/measure-data/measure/data/index ${BANYANDB_DATA_GENERATE_ROOT}
967+
found=true
968+
break
969+
else
970+
echo "⏳ didn't find segment files (retry $i/60)"
971+
sleep 10
972+
fi
973+
done
974+
if $found; then
975+
echo "✅ segment files copied to ${BANYANDB_DATA_GENERATE_ROOT}"
976+
else
977+
echo "❌ segment files not found"
978+
exit 1
979+
fi
980+
docker compose -f ${BANYANDB_DATA_GENERATE_ROOT}/docker-compose.yml down -v
981+
- name: ${{ matrix.test.name }}
982+
uses: apache/skywalking-infra-e2e@cf589b4a0b9f8e6f436f78e9cfd94a1ee5494180
983+
with:
984+
e2e-file: $GITHUB_WORKSPACE/${{ matrix.test.config }}
985+
- if: ${{ failure() }}
986+
run: |
987+
df -h
988+
du -sh .
989+
docker images
990+
- uses: actions/upload-artifact@v4
991+
if: ${{ failure() }}
992+
name: Upload Logs
993+
with:
994+
name: test-logs-${{ matrix.test.name }}
995+
path: "${{ env.SW_INFRA_E2E_LOG_DIR }}"
996+
880997
required:
881998
if: always()
882999
name: Required

‎test/e2e-v2/cases/apisix/otel-collector/e2e.yaml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,8 @@ trigger:
3737

3838
verify:
3939
retry:
40-
count: 10
41-
interval: 3s
40+
count: 20
41+
interval: 10s
4242
cases:
4343
- includes:
44-
- ../apisix-cases.yaml
44+
- ../apisix-cases.yaml

‎test/e2e-v2/cases/aws/api-gateway/e2e.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,8 @@ trigger:
3737

3838
verify:
3939
retry:
40-
count: 10
41-
interval: 3s
40+
count: 20
41+
interval: 10s
4242
cases:
4343
- includes:
4444
- ./gateway-cases.yaml

‎test/e2e-v2/cases/aws/dynamodb/e2e.yaml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,8 @@ trigger:
3737

3838
verify:
3939
retry:
40-
count: 10
41-
interval: 3s
40+
count: 20
41+
interval: 10s
4242
cases:
4343
- includes:
44-
- ./dynamodb-cases.yaml
44+
- ./dynamodb-cases.yaml

‎test/e2e-v2/cases/aws/eks/e2e.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,8 @@ trigger:
3737

3838
verify:
3939
retry:
40-
count: 10
41-
interval: 3s
40+
count: 20
41+
interval: 10s
4242
cases:
4343
- includes:
4444
- ./eks-cases.yaml

‎test/e2e-v2/cases/aws/s3/e2e.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,8 @@ trigger:
3737

3838
verify:
3939
retry:
40-
count: 10
41-
interval: 3s
40+
count: 20
41+
interval: 10s
4242
cases:
4343
- includes:
4444
- ./s3-cases.yaml

‎test/e2e-v2/cases/php/e2e.yaml‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,10 +38,8 @@ trigger:
3838
verify:
3939
# verify with retry strategy
4040
retry:
41-
# max retry count
4241
count: 20
43-
# the interval between two retries, in millisecond.
44-
interval: 3s
42+
interval: 10s
4543
cases:
4644
# layer list
4745
- query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql layer ls

‎test/e2e-v2/cases/storage/banyandb/data-generate/docker-compose.yml‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
version: '2.1'
17+
18+
services:
19+
banyandb:
20+
image: "ghcr.io/apache/skywalking-banyandb:${SW_BANYANDB_COMMIT}"
21+
ports:
22+
- 17912:17912
23+
- 17913:17913
24+
command: standalone --stream-root-path /tmp/stream-data --measure-root-path /tmp/measure-data
25+
networks:
26+
- e2e
27+
healthcheck:
28+
test: [ "CMD", "sh", "-c", "nc -nz 127.0.0.1 17912" ]
29+
interval: 5s
30+
timeout: 60s
31+
retries: 120
32+
33+
oap:
34+
extends:
35+
file: ../../../../script/docker-compose/base-compose.yml
36+
service: oap
37+
environment:
38+
SW_STORAGE: banyandb
39+
SW_STORAGE_BANYANDB_GR_NORMAL_TTL_DAYS: 5
40+
SW_STORAGE_BANYANDB_GR_SUPER_TTL_DAYS: 5
41+
ports:
42+
- 12800
43+
depends_on:
44+
banyandb:
45+
condition: service_healthy
46+
47+
trace-mocker:
48+
image: "eclipse-temurin:17-jre"
49+
volumes:
50+
- ../../../../java-test-service/trace-mocker/target/trace-mocker-2.0.0.jar:/trace-mocker-2.0.0.jar
51+
command: [ "java", "-jar", "/trace-mocker-2.0.0.jar" ]
52+
environment:
53+
OAP_HOST: oap
54+
MINUS_DAYS: 3
55+
networks:
56+
- e2e
57+
depends_on:
58+
oap:
59+
condition: service_healthy
60+
61+
networks:
62+
e2e:

‎test/e2e-v2/cases/storage/banyandb/stages/docker-compose.yml‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
version: '2.1'
17+
18+
services:
19+
etcd:
20+
extends:
21+
file: ../../../../script/docker-compose/base-compose.yml
22+
service: etcd
23+
networks:
24+
- e2e
25+
26+
data-hot1:
27+
extends:
28+
file: ../../../../script/docker-compose/base-compose.yml
29+
service: banyandb-data
30+
hostname: data-hot1
31+
command: data --etcd-endpoints=http://etcd:2379 --node-labels type=hot
32+
volumes:
33+
- ../data-generate/index:/tmp/measure/data/index
34+
networks:
35+
- e2e
36+
37+
data-warm1:
38+
extends:
39+
file: ../../../../script/docker-compose/base-compose.yml
40+
service: banyandb-data
41+
hostname: data-warm1
42+
command: data --etcd-endpoints=http://etcd:2379 --node-labels type=warm
43+
networks:
44+
- e2e
45+
46+
data-cold1:
47+
extends:
48+
file: ../../../../script/docker-compose/base-compose.yml
49+
service: banyandb-data
50+
hostname: data-cold1
51+
command: data --etcd-endpoints=http://etcd:2379 --node-labels type=cold
52+
volumes:
53+
- ../data-generate/tmp/measure-data/measure:/tmp/measure
54+
- ../data-generate/tmp/stream-data/stream:/tmp/stream
55+
- ../data-generate/tmp/property/data:/tmp/property
56+
networks:
57+
- e2e
58+
59+
liaison:
60+
extends:
61+
file: ../../../../script/docker-compose/base-compose.yml
62+
service: liaison
63+
command: liaison --etcd-endpoints=http://etcd:2379 --data-node-selector type=hot
64+
networks:
65+
- e2e
66+
67+
oap:
68+
extends:
69+
file: ../../../../script/docker-compose/base-compose.yml
70+
service: oap
71+
environment:
72+
SW_STORAGE: banyandb
73+
SW_STORAGE_BANYANDB_TARGETS: "liaison:17912"
74+
SW_STORAGE_BANYANDB_GM_MINUTE_ENABLE_WARM_STAGE: "true"
75+
SW_STORAGE_BANYANDB_GM_MINUTE_ENABLE_COLD_STAGE: "true"
76+
SW_STORAGE_BANYANDB_GM_HOUR_ENABLE_WARM_STAGE: "true"
77+
SW_STORAGE_BANYANDB_GM_HOUR_ENABLE_COLD_STAGE: "true"
78+
SW_STORAGE_BANYANDB_GM_DAY_ENABLE_WARM_STAGE: "true"
79+
SW_STORAGE_BANYANDB_GM_DAY_ENABLE_COLD_STAGE: "true"
80+
SW_STORAGE_BANYANDB_GR_NORMAL_ENABLE_WARM_STAGE: "true"
81+
SW_STORAGE_BANYANDB_GR_NORMAL_ENABLE_COLD_STAGE: "true"
82+
SW_STORAGE_BANYANDB_GR_SUPER_ENABLE_WARM_STAGE: "true"
83+
SW_STORAGE_BANYANDB_GR_SUPER_ENABLE_COLD_STAGE: "true"
84+
ports:
85+
- 12800:12800
86+
- 11800:11800
87+
depends_on:
88+
liaison:
89+
condition: service_healthy
90+
91+
provider:
92+
extends:
93+
file: ../../../../script/docker-compose/base-compose.yml
94+
service: provider
95+
ports:
96+
- 9090
97+
networks:
98+
- e2e
99+
depends_on:
100+
oap:
101+
condition: service_healthy
102+
103+
consumer:
104+
extends:
105+
file: ../../../../script/docker-compose/base-compose.yml
106+
service: consumer
107+
ports:
108+
- 9092
109+
depends_on:
110+
oap:
111+
condition: service_healthy
112+
provider:
113+
condition: service_healthy
114+
115+
networks:
116+
e2e:
117+

0 commit comments

Comments
 (0)