Skip to content

Commit 218a6b8

Browse files
zhanglei1949zhanglei1949
committed
merge
Committed-by: xiaolei.zl@alibaba-inc.com from Dev container
2 parents d9303ef + 3f0a2bd commit 218a6b8

File tree

170 files changed

+7706
-705
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

170 files changed

+7706
-705
lines changed

‎.github/workflows/flex-interactive.yml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ jobs:
5656
# install gsctl
5757
python3 -m pip install ${GITHUB_WORKSPACE}/python/dist/*.whl
5858
# launch service: 8080 for coordinator http port; 7687 for cypher port;
59-
gsctl instance deploy --type interactive --image-registry graphscope --image-tag latest --interactive-config ${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml
59+
gsctl instance deploy --type interactive --image-registry graphscope --image-tag latest --cypher-port 7688 --interactive-config ${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml
6060
sleep 20
6161
# test
6262
python3 -m pip install --no-cache-dir pytest pytest-cov pytest-timeout pytest-xdist
@@ -66,6 +66,10 @@ jobs:
6666
--exitfirst \
6767
$(dirname $(python3 -c "import graphscope.gsctl as gsctl; print(gsctl.__file__)"))/tests/test_interactive.py
6868
69+
# test coordinator
70+
res=`curl http://127.0.0.1:8080/api/v1/service`
71+
echo $res | grep 7688 || exit 1
72+
6973
# destroy instance
7074
gsctl instance destroy --type interactive -y
7175

‎.github/workflows/interactive.yml‎

Lines changed: 76 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,52 @@ concurrency:
2424
cancel-in-progress: true
2525

2626
jobs:
27+
build-interactive:
28+
runs-on: ubuntu-20.04
29+
if: ${{ github.repository == 'alibaba/GraphScope' }}
30+
container:
31+
image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.24.2-amd64
32+
steps:
33+
- uses: actions/checkout@v4
34+
- name: Install latest libgrape-lite
35+
if: false
36+
run: |
37+
git clone --single-branch https://github.com/alibaba/libgrape-lite.git /tmp/libgrape-lite
38+
cd /tmp/libgrape-lite
39+
mkdir -p build && cd build
40+
cmake ..
41+
make -j$(nproc)
42+
make install
43+
44+
- name: Setup tmate session
45+
if: false
46+
uses: mxschmitt/action-tmate@v3
47+
48+
- name: Build Interactive
49+
env:
50+
GIE_HOME: ${{ github.workspace }}/interactive_engine/
51+
HOME: /home/graphscope/
52+
run: |
53+
. /home/graphscope/.graphscope_env
54+
cd ${GITHUB_WORKSPACE}/
55+
git submodule update --init
56+
cd ${GITHUB_WORKSPACE}/flex
57+
mkdir build && cd build
58+
cmake .. -DCMAKE_INSTALL_PREFIX=/opt/graphscope && sudo make -j$(nproc)
59+
# package the build artifacts
60+
cd .. && tar -zcf build.tar.gz build
61+
62+
- name: Upload Artifacts
63+
uses: actions/upload-artifact@v4
64+
with:
65+
name: interactive_build-${{ github.sha }}
66+
path: |
67+
${{ github.workspace }}/flex/build.tar.gz
68+
69+
2770
test-hqps-engine:
2871
runs-on: ubuntu-20.04
72+
needs: build-interactive
2973
if: ${{ github.repository == 'alibaba/GraphScope' }}
3074
container:
3175
image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.24.2-amd64
@@ -58,23 +102,28 @@ jobs:
58102
cmake ..
59103
make -j$(nproc)
60104
make install
105+
106+
- name: Download Artifacts
107+
uses: actions/download-artifact@v4
108+
with:
109+
name: interactive_build-${{ github.sha }}
61110

62111
- name: Setup tmate session
63112
if: false
64113
uses: mxschmitt/action-tmate@v3
65114

115+
- name: Extract build artifacts
116+
run: |
117+
cd ${GITHUB_WORKSPACE}
118+
tar zxf build.tar.gz -C flex && rm build.tar.gz
119+
66120
- name: Build
67121
env:
68122
GIE_HOME: ${{ github.workspace }}/interactive_engine/
69123
HOME: /home/graphscope/
70124
run: |
71125
. /home/graphscope/.graphscope_env
72126
cd ${GITHUB_WORKSPACE}/
73-
git submodule update --init
74-
cd ${GITHUB_WORKSPACE}/flex
75-
mkdir build && cd build
76-
cmake .. && sudo make -j$(nproc)
77-
sudo make install
78127
79128
# cargo
80129
. /home/graphscope/.cargo/env
@@ -175,7 +224,7 @@ jobs:
175224
INTERACTIVE_WORKSPACE: /tmp/interactive_workspace
176225
run: |
177226
cd ${GITHUB_WORKSPACE}/flex/tests/hqps
178-
bash hqps_robust_test.sh ${INTERACTIVE_WORKSPACE} ./interactive_config_test.yaml
227+
bash hqps_robust_test.sh ${INTERACTIVE_WORKSPACE} ./interactive_config_test.yaml ./interactive_config_test_cbo.yaml
179228
180229
- name: Sample Query test
181230
env:
@@ -400,6 +449,7 @@ jobs:
400449
401450
test-flex:
402451
runs-on: ubuntu-20.04
452+
needs: build-interactive
403453
if: ${{ github.repository == 'alibaba/GraphScope' }}
404454
container:
405455
image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.24.2-amd64
@@ -416,14 +466,15 @@ jobs:
416466
make -j$(nproc)
417467
make install
418468
419-
- name: Build
420-
env:
421-
HOME: /home/graphscope/
469+
- name: Download Artifacts
470+
uses: actions/download-artifact@v4
471+
with:
472+
name: interactive_build-${{ github.sha }}
473+
474+
- name: Extract build artifacts
422475
run: |
423-
cd ${GITHUB_WORKSPACE}/flex
424-
git submodule update --init
425-
mkdir build && cd build
426-
cmake .. && sudo make -j$(nproc)
476+
cd ${GITHUB_WORKSPACE}
477+
tar zxf build.tar.gz -C flex && rm build.tar.gz
427478
428479
- name: Test GRIN on mutable csr
429480
run: |
@@ -554,3 +605,15 @@ jobs:
554605
SCHEMA_FILE=${GITHUB_WORKSPACE}/flex/tests/rt_mutable_graph/movie_schema_test.yaml
555606
BULK_LOAD_FILE=${GITHUB_WORKSPACE}/flex/tests/rt_mutable_graph/movie_import_test.yaml
556607
GLOG_v=10 ./bin/bulk_loader -g ${SCHEMA_FILE} -l ${BULK_LOAD_FILE} -d /tmp/csr-data-dir/
608+
609+
- name: Test graph loading with different delimiter
610+
env:
611+
GS_TEST_DIR: ${{ github.workspace }}/gstest/
612+
FLEX_DATA_DIR: ${{ github.workspace }}/gstest/flex/modern_graph_tab_delimiter/
613+
run: |
614+
rm -rf /tmp/csr-data-dir/
615+
cd ${GITHUB_WORKSPACE}/flex/build/
616+
SCHEMA_FILE=${GITHUB_WORKSPACE}/flex/interactive/examples/modern_graph/graph.yaml
617+
BULK_LOAD_FILE=${GITHUB_WORKSPACE}/flex/interactive/examples/modern_graph/bulk_load.yaml
618+
sed -i 's/|/\\t/g' ${BULK_LOAD_FILE}
619+
GLOG_v=10 ./bin/bulk_loader -g ${SCHEMA_FILE} -l ${BULK_LOAD_FILE} -d /tmp/csr-data-dir/

‎charts/graphscope-store/templates/configmap.yaml‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,7 @@ data:
116116
117117
sudo chown -R graphscope:graphscope {{ .Values.storeDataPath }} || true
118118
sudo chown -R graphscope:graphscope /etc/groot || true
119+
sudo chown -R graphscope:graphscope /var/log/graphscope || true
119120
120121
[[ `hostname` =~ -([0-9]+)$ ]] || exit 1
121122
ordinal=${BASH_REMATCH[1]}
@@ -180,7 +181,12 @@ data:
180181
export GRPC_PORT=${GROOT_GRPC_PORT}
181182
export GREMLIN_PORT=${GROOT_GREMLIN_PORT}
182183
echo "${HOST} ${GRPC_PORT} ${GREMLIN_PORT}"
183-
python3 -c 'import base64;import os;from gremlin_python.driver.client import Client;ip=os.getenv("HOST");gremlin_port=os.getenv("GREMLIN_PORT");graph_url=f"ws://{ip}:{gremlin_port}/gremlin";username=os.getenv("GROOT_USERNAME");password=base64.b64decode(os.getenv("GROOT_PASSWORD")).decode("utf-8");client = Client(graph_url, "g", username=username, password=password); ret = client.submit("g.V().limit(1)").all().result(); client.close();' && break
184+
cmd="nc -zv ${HOST} ${GREMLIN_PORT}"
185+
res=$(eval $cmd 2>&1)
186+
if [[ $res == *succeeded* ]]; then
187+
# Expected Output is <Connection to localhost (::1) 8182 port [tcp/*] succeeded!>
188+
break
189+
fi
184190
sleep 3
185191
done
186192

‎charts/graphscope-store/templates/coordinator/statefulset.yaml‎

Lines changed: 38 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -135,20 +135,31 @@ spec:
135135
- name: config
136136
mountPath: /etc/groot/setup.sh
137137
subPath: setup.sh
138+
- name: log
139+
mountPath: {{ .Values.logPersistence.mountPath }}
138140
volumes:
139141
- name: config
140142
configMap:
141143
name: {{ include "graphscope-store.configmapName" . }}
142144
defaultMode: 0755
143-
{{- if and .Values.coordinator.persistence.enabled .Values.coordinator.persistence.existingClaim }}
145+
{{- if and .Values.coordinator.persistence.enabled .Values.coordinator.persistence.existingClaim }}
144146
- name: meta
145147
persistentVolumeClaim:
146148
claimName: {{ printf "%s" (tpl .Values.coordinator.persistence.existingClaim .) }}
147-
{{- else if not .Values.coordinator.persistence.enabled }}
149+
{{- else if not .Values.coordinator.persistence.enabled }}
148150
- name: meta
149151
emptyDir: {}
150-
{{- else if and .Values.coordinator.persistence.enabled (not .Values.coordinator.persistence.existingClaim) }}
152+
{{- end }}
153+
{{- if not .Values.logPersistence.enabled }}
154+
- name: log
155+
emptyDir: {}
156+
{{- else if .Values.logPersistence.existingClaim }}
157+
- name: log
158+
persistentVolumeClaim:
159+
claimName: {{ printf "%s" (tpl .Values.logPersistence.existingClaim .) }}
160+
{{- end }}
151161
volumeClaimTemplates:
162+
{{- if and .Values.coordinator.persistence.enabled (not .Values.coordinator.persistence.existingClaim) }}
152163
- metadata:
153164
name: meta
154165
{{- if .Values.persistence.annotations }}
@@ -169,5 +180,27 @@ spec:
169180
{{- if .Values.coordinator.persistence.selector }}
170181
selector: {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.coordinator.persistence.selector "context" $) | nindent 10 }}
171182
{{- end -}}
172-
{{- end }}
173-
{{- end -}}
183+
{{- end }}
184+
{{- if and .Values.logPersistence.enabled (not .Values.logPersistence.existingClaim) }}
185+
- metadata:
186+
name: log
187+
{{- if .Values.persistence.annotations }}
188+
annotations: {{- include "common.tplvalues.render" (dict "value" .Values.persistence.annotations "context" $) | nindent 10 }}
189+
{{- end }}
190+
{{- if .Values.persistence.labels }}
191+
labels: {{- include "common.tplvalues.render" (dict "value" .Values.persistence.labels "context" $) | nindent 10 }}
192+
{{- end }}
193+
spec:
194+
accessModes:
195+
{{- range .Values.logPersistence.accessModes }}
196+
- {{ . | quote }}
197+
{{- end }}
198+
resources:
199+
requests:
200+
storage: {{ .Values.logPersistence.size | quote }}
201+
{{ include "graphscope-store.storageClass" . | nindent 8 }}
202+
{{- if .Values.logPersistence.selector }}
203+
selector: {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.logPersistence.selector "context" $) | nindent 10 }}
204+
{{- end -}}
205+
{{- end }}
206+
{{- end -}}

‎charts/graphscope-store/templates/frontend/statefulset.yaml‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,9 +137,42 @@ spec:
137137
- name: config
138138
mountPath: /etc/groot/setup.sh
139139
subPath: setup.sh
140+
- name: log
141+
mountPath: {{ .Values.logPersistence.mountPath }}
140142
volumes:
141143
- name: config
142144
configMap:
143145
name: {{ include "graphscope-store.configmapName" . }}
144146
defaultMode: 0755
147+
{{- if not .Values.logPersistence.enabled }}
148+
- name: log
149+
emptyDir: {}
150+
{{- else if .Values.logPersistence.existingClaim }}
151+
- name: log
152+
persistentVolumeClaim:
153+
claimName: {{ printf "%s" (tpl .Values.logPersistence.existingClaim .) }}
154+
{{- end }}
155+
volumeClaimTemplates:
156+
{{- if and .Values.logPersistence.enabled (not .Values.logPersistence.existingClaim) }}
157+
- metadata:
158+
name: log
159+
{{- if .Values.persistence.annotations }}
160+
annotations: {{- include "common.tplvalues.render" (dict "value" .Values.persistence.annotations "context" $) | nindent 10 }}
161+
{{- end }}
162+
{{- if .Values.persistence.labels }}
163+
labels: {{- include "common.tplvalues.render" (dict "value" .Values.persistence.labels "context" $) | nindent 10 }}
164+
{{- end }}
165+
spec:
166+
accessModes:
167+
{{- range .Values.logPersistence.accessModes }}
168+
- {{ . | quote }}
169+
{{- end }}
170+
resources:
171+
requests:
172+
storage: {{ .Values.logPersistence.size | quote }}
173+
{{ include "graphscope-store.storageClass" . | nindent 8 }}
174+
{{- if .Values.logPersistence.selector }}
175+
selector: {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.logPersistence.selector "context" $) | nindent 10 }}
176+
{{- end -}}
177+
{{- end }}
145178
{{- end }}

‎charts/graphscope-store/templates/onepod/statefulset.yaml‎

Lines changed: 37 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,8 @@ spec:
137137
- name: config
138138
mountPath: /etc/groot/setup.sh
139139
subPath: setup.sh
140+
- name: log
141+
mountPath: {{ .Values.logPersistence.mountPath }}
140142
{{- if .Values.store.extraVolumeMounts }}
141143
{{- include "common.tplvalues.render" (dict "value" .Values.store.extraVolumeMounts "context" $) | nindent 12 }}
142144
{{- end }}
@@ -148,15 +150,24 @@ spec:
148150
{{- if .Values.store.extraVolumes }}
149151
{{- include "common.tplvalues.render" (dict "value" .Values.store.extraVolumes "context" $) | nindent 8 }}
150152
{{- end }}
151-
{{- if and .Values.store.persistence.enabled .Values.store.persistence.existingClaim }}
153+
{{- if and .Values.store.persistence.enabled .Values.store.persistence.existingClaim }}
152154
- name: data
153155
persistentVolumeClaim:
154156
claimName: {{ printf "%s" (tpl .Values.store.persistence.existingClaim .) }}
155-
{{- else if not .Values.store.persistence.enabled }}
157+
{{- else if not .Values.store.persistence.enabled }}
156158
- name: data
157159
emptyDir: {}
158-
{{- else if and .Values.store.persistence.enabled (not .Values.store.persistence.existingClaim) }}
160+
{{- end }}
161+
{{- if not .Values.logPersistence.enabled }}
162+
- name: log
163+
emptyDir: {}
164+
{{- else if .Values.logPersistence.existingClaim }}
165+
- name: log
166+
persistentVolumeClaim:
167+
claimName: {{ printf "%s" (tpl .Values.logPersistence.existingClaim .) }}
168+
{{- end }}
159169
volumeClaimTemplates:
170+
{{- if and .Values.store.persistence.enabled (not .Values.store.persistence.existingClaim) }}
160171
- metadata:
161172
name: data
162173
{{- if .Values.persistence.annotations }}
@@ -177,5 +188,27 @@ spec:
177188
{{- if .Values.store.persistence.selector }}
178189
selector: {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.store.persistence.selector "context" $) | nindent 10 }}
179190
{{- end -}}
180-
{{- end }}
191+
{{- end }}
192+
{{- if and .Values.logPersistence.enabled (not .Values.logPersistence.existingClaim) }}
193+
- metadata:
194+
name: log
195+
{{- if .Values.persistence.annotations }}
196+
annotations: {{- include "common.tplvalues.render" (dict "value" .Values.persistence.annotations "context" $) | nindent 10 }}
197+
{{- end }}
198+
{{- if .Values.persistence.labels }}
199+
labels: {{- include "common.tplvalues.render" (dict "value" .Values.persistence.labels "context" $) | nindent 10 }}
200+
{{- end }}
201+
spec:
202+
accessModes:
203+
{{- range .Values.logPersistence.accessModes }}
204+
- {{ . | quote }}
205+
{{- end }}
206+
resources:
207+
requests:
208+
storage: {{ .Values.logPersistence.size | quote }}
209+
{{ include "graphscope-store.storageClass" . | nindent 8 }}
210+
{{- if .Values.logPersistence.selector }}
211+
selector: {{- include "graphscope-store.tplvalues.render" (dict "value" .Values.logPersistence.selector "context" $) | nindent 10 }}
212+
{{- end -}}
213+
{{- end }}
181214
{{- end }}

‎charts/graphscope-store/templates/portal/statefulset.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,8 @@ spec:
115115
value: {{ .Values.portal.runtimePath | quote }}
116116
- name: STUDIO_WRAPPER_ENDPOINT
117117
value: {{ .Values.portal.studioWrapperEndpoint | quote }}
118+
- name: BASEID
119+
value: {{ .Values.portal.baseId | quote }}
118120
- name: SOLUTION
119121
value: "GRAPHSCOPE_INSIGHT"
120122
{{- range $key, $value := .Values.env }}

0 commit comments

Comments
 (0)