Skip to content

Commit 5bcefbd

Browse files
zhanglei1949liulx20shirly121
authored
ci(interactive): Refactor Interactive CI for adhoc queries (#4462)
We now support all LDBC SNB IC1 to IC14 queries. To maintain proper functionality during continuous integration, we are incorporating these queries into the ad hoc query test. Fix #4461 TODOs - [x] Remove the temporarily added .cypher and .json files @zhanglei1949 - [x] Make sure all plans could be supported by the compiler @shirly121 - [x] Make sure the test cases could pass @zhanglei1949 --------- Co-authored-by: liulx20 <[email protected]> Co-authored-by: shirly121 <[email protected]>
1 parent f4f261a commit 5bcefbd

File tree

78 files changed

+3462
-247
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

78 files changed

+3462
-247
lines changed

‎.github/workflows/interactive.yml‎

Lines changed: 7 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -141,8 +141,8 @@ jobs:
141141
# download dataset
142142
git clone -b master --single-branch --depth=1 https://github.com/GraphScope/gstest.git ${GS_TEST_DIR}
143143
mkdir -p ${INTERACTIVE_WORKSPACE}/data/ldbc
144-
GRAPH_SCHEMA_YAML=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_graph_schema.yaml
145-
BUILD_LOAD_FILE=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_bulk_load.yaml
144+
GRAPH_SCHEMA_YAML=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_graph_schema_creationDate.yaml
145+
BUILD_LOAD_FILE=${GS_TEST_DIR}/flex/ldbc-sf01-long-date/audit_bulk_load_creationDate.yaml
146146
cp ${GRAPH_SCHEMA_YAML} ${INTERACTIVE_WORKSPACE}/data/ldbc/graph.yaml
147147
cp ${BUILD_LOAD_FILE} ${INTERACTIVE_WORKSPACE}/data/ldbc/import.yaml
148148
mkdir -p ${INTERACTIVE_WORKSPACE}/data/movies
@@ -295,14 +295,12 @@ jobs:
295295
# plus_one: (num: int64) -> (num: int64), CppEncoder
296296
# sample_app: (num: int64) -> (num: int64), kCypherJson
297297

298-
sed -i 's/default_graph: ldbc/default_graph: modern_graph/g' ${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml
299298
sed -i 's/interactive_workspace/temp_workspace/g' ${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml
300299
cd ${GITHUB_WORKSPACE}/flex/tests/interactive/
301300
bash test_plugin_loading.sh ${TMP_INTERACTIVE_WORKSPACE} modern_graph \
302301
${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml \
303302
./modern_graph_schema_v0_0.yaml ./modern_graph_schema_v0_1.yaml
304303
sed -i 's/temp_workspace/interactive_workspace/g' ${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml
305-
sed -i 's/default_graph: modern_graph/default_graph: movies/g' ${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml
306304

307305
- name: Let compiler use latest interactive java sdk
308306
env:
@@ -346,10 +344,8 @@ jobs:
346344
run: |
347345
cd ${GITHUB_WORKSPACE}/flex/tests/hqps/
348346
export ENGINE_TYPE=hiactor
349-
# change the default_graph config in ./interactive_config_test.yaml to ldbc
350-
sed -i 's/default_graph: movies/default_graph: ldbc/g' ./interactive_config_test.yaml
351-
bash hqps_adhoc_test.sh ${INTERACTIVE_WORKSPACE} ldbc \
352-
${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml
347+
bash hqps_adhoc_test.sh ${INTERACTIVE_WORKSPACE} ldbc CBO
348+
bash hqps_adhoc_test.sh ${INTERACTIVE_WORKSPACE} ldbc RBO
353349
354350
- name: Run End-to-End cypher adhoc movie query test
355351
env:
@@ -359,10 +355,7 @@ jobs:
359355
run: |
360356
cd ${GITHUB_WORKSPACE}/flex/tests/hqps/
361357
export ENGINE_TYPE=hiactor
362-
# change the default_graph config in ./interactive_config_test.yaml to movies
363-
sed -i 's/default_graph: ldbc/default_graph: movies/g' ./interactive_config_test.yaml
364-
bash hqps_adhoc_test.sh ${INTERACTIVE_WORKSPACE} movies \
365-
${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml
358+
bash hqps_adhoc_test.sh ${INTERACTIVE_WORKSPACE} movies RBO
366359
367360
- name: Run End-to-End cypher adhoc graph_algo query test
368361
env:
@@ -372,10 +365,7 @@ jobs:
372365
run: |
373366
cd ${GITHUB_WORKSPACE}/flex/tests/hqps/
374367
export ENGINE_TYPE=hiactor
375-
# change the default_graph config in ${GS_TEST_DIR}/flex/ldbc-sf01-long-date/interactive_config.yaml to graph_algo
376-
sed -i 's/default_graph: movies/default_graph: graph_algo/g' ./interactive_config_test.yaml
377-
bash hqps_adhoc_test.sh ${INTERACTIVE_WORKSPACE} graph_algo \
378-
${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml
368+
bash hqps_adhoc_test.sh ${INTERACTIVE_WORKSPACE} graph_algo RBO
379369
380370
- name: Run Gremlin test on modern graph
381371
env:
@@ -385,9 +375,7 @@ jobs:
385375
run: |
386376
cd ${GITHUB_WORKSPACE}/flex/tests/hqps/
387377
export ENGINE_TYPE=hiactor
388-
sed -i 's/default_graph: graph_algo/default_graph: modern_graph/g' ./interactive_config_test.yaml
389-
bash hqps_adhoc_test.sh ${INTERACTIVE_WORKSPACE} modern_graph \
390-
${GITHUB_WORKSPACE}/flex/tests/hqps/interactive_config_test.yaml gremlin
378+
bash hqps_adhoc_test.sh ${INTERACTIVE_WORKSPACE} modern_graph RBO gremlin
391379
392380
test-build-flex:
393381
runs-on: ubuntu-22.04

‎flex/engines/graph_db/runtime/common/columns/vertex_columns.cc‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -131,8 +131,15 @@ std::shared_ptr<IContextColumn> SLVertexColumn::union_col(
131131
return builder.finish(nullptr);
132132
}
133133
}
134-
LOG(FATAL) << "not support...";
135-
return nullptr;
134+
auto builder = MLVertexColumnBuilder::builder();
135+
for (auto v : vertices_) {
136+
builder.push_back_vertex({label_, v});
137+
}
138+
auto col = dynamic_cast<const IVertexColumn*>(other.get());
139+
for (size_t i = 0; i < col->size(); ++i) {
140+
builder.push_back_vertex(col->get_vertex(i));
141+
}
142+
return builder.finish(nullptr);
136143
}
137144

138145
std::shared_ptr<IContextColumn> SLVertexColumnBuilder::finish(

‎flex/engines/graph_db/runtime/common/operators/retrieve/intersect.cc‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,8 @@ static Context left_outer_intersect(Context&& ctx, Context&& ctx0,
6666
ctx.reshuffle(ctx0.get_offsets().data());
6767
for (size_t i = 0; i < ctx0.col_num() || i < ctx1.col_num(); ++i) {
6868
if (i < ctx0.col_num()) {
69-
if (ctx0.get(i) != nullptr) {
69+
if ((i >= ctx.col_num() || ctx.get(i) == nullptr) &&
70+
ctx0.get(i) != nullptr) {
7071
ctx.set(i, ctx0.get(i));
7172
}
7273
}

‎flex/engines/graph_db/runtime/execute/ops/retrieve/edge.cc‎

Lines changed: 20 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -552,25 +552,27 @@ bl::result<ReadOpBuildResultT> EdgeExpandOprBuilder::Build(
552552
eep.is_optional = is_optional;
553553
if (opr.expand_opt() == physical::EdgeExpand_ExpandOpt_VERTEX) {
554554
if (query_params.has_predicate()) {
555-
if (parse_sp_pred(query_params.predicate()) ==
556-
SPPredicateType::kPropertyGT) {
557-
std::string param_name =
558-
query_params.predicate().operators(2).param().name();
559-
return std::make_pair(std::make_unique<EdgeExpandVWithEPGTOpr>(
560-
eep, param_name, query_params.predicate()),
561-
meta);
562-
} else if (parse_sp_pred(query_params.predicate()) ==
563-
SPPredicateType::kPropertyLT) {
564-
std::string param_name =
565-
query_params.predicate().operators(2).param().name();
566-
return std::make_pair(std::make_unique<EdgeExpandVWithEPLTOpr>(
567-
eep, param_name, query_params.predicate()),
568-
meta);
569-
} else {
570-
return std::make_pair(std::make_unique<EdgeExpandVWithEdgePredOpr>(
571-
eep, query_params.predicate()),
572-
meta);
555+
auto tp = parse_sp_pred(query_params.predicate());
556+
const auto& op2 = query_params.predicate().operators(2);
557+
if (op2.has_param()) {
558+
if (tp == SPPredicateType::kPropertyGT) {
559+
std::string param_name =
560+
query_params.predicate().operators(2).param().name();
561+
return std::make_pair(std::make_unique<EdgeExpandVWithEPGTOpr>(
562+
eep, param_name, query_params.predicate()),
563+
meta);
564+
} else if (tp == SPPredicateType::kPropertyLT) {
565+
std::string param_name =
566+
query_params.predicate().operators(2).param().name();
567+
return std::make_pair(std::make_unique<EdgeExpandVWithEPLTOpr>(
568+
eep, param_name, query_params.predicate()),
569+
meta);
570+
}
573571
}
572+
return std::make_pair(std::make_unique<EdgeExpandVWithEdgePredOpr>(
573+
eep, query_params.predicate()),
574+
meta);
575+
574576
} else {
575577
return std::make_pair(std::make_unique<EdgeExpandVWithoutPredOpr>(eep),
576578
meta);

‎flex/engines/graph_db/runtime/execute/ops/retrieve/select.cc‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,8 @@ bl::result<ReadOpBuildResultT> SelectOprBuilder::Build(
133133
const physical::PhysicalPlan& plan, int op_idx) {
134134
auto opr = plan.plan(op_idx).opr().select();
135135
auto type = parse_sp_pred(opr.predicate());
136-
if (type == SPPredicateType::kPropertyNE) {
136+
const auto& op2 = opr.predicate().operators(2);
137+
if (type == SPPredicateType::kPropertyNE && op2.has_param()) {
137138
auto var = opr.predicate().operators(0).var();
138139
if (var.has_property()) {
139140
auto name = var.property().key().name();

‎flex/resources/queries/ic/runtime/ic1.cypher‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
MATCH shortestPath(p: PERSON {id : 30786325579101L}) -[k:KNOWS*1..4]-(f:PERSON {firstName : "Ian"})
2+
where f <> p
3+
WITH f, length(k) as distance
4+
ORDER BY distance ASC, f.lastName ASC, f.id ASC
5+
LIMIT 20
6+
OPTIONAL MATCH (f: PERSON)-[workAt:WORKAT]->(company:ORGANISATION)-[:ISLOCATEDIN]->(country:PLACE)
7+
WITH
8+
f, distance,
9+
CASE
10+
WHEN company is null Then null
11+
ELSE [company.name, workAt.workFrom, country.name]
12+
END as companies
13+
WITH f, collect(companies) as company_info, distance
14+
15+
OPTIONAL MATCH (f: PERSON)-[studyAt:STUDYAT]->(university)-[:ISLOCATEDIN]->(universityCity:PLACE)
16+
WITH
17+
f, company_info, distance,
18+
CASE
19+
WHEN university is null Then null
20+
ELSE [university.name, studyAt.classYear, universityCity.name]
21+
END as universities
22+
WITH f, collect(universities) as university_info , company_info, distance
23+
24+
MATCH (f:PERSON)-[:ISLOCATEDIN]->(locationCity:PLACE)
25+
26+
return f.id AS friendId,
27+
distance AS distanceFromPerson,
28+
f.lastName AS friendLastName,
29+
f.birthday AS friendBirthday,
30+
f.creationDate AS friendCreationDate,
31+
f.gender AS friendGender,
32+
f.browserUsed AS friendBrowserUsed,
33+
f.locationIP AS friendLocationIp,
34+
locationCity.name AS friendCityName,
35+
f.email AS friendEmail,
36+
f.language AS friendLanguage,
37+
university_info AS friendUniversities,
38+
company_info AS friendCompanies;

‎flex/resources/queries/ic/runtime/ic1.json‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
{
2+
"operation": {
3+
"personIdQ1": 30786325579101,
4+
"firstName": "Ian",
5+
"limit": 20
6+
},
7+
"result": [
8+
{
9+
"friendId": 10995116279328,
10+
"distanceFromPerson": 2,
11+
"friendLastName": "Smith",
12+
"friendBirthday": 513388800000,
13+
"friendCreationDate": 1291220622426,
14+
"friendGender": "male",
15+
"friendBrowserUsed": "Internet Explorer",
16+
"friendLocationIp": "199.103.115.201",
17+
"friendCityName": "Bristol",
18+
"friendEmail": "[email protected];[email protected]",
19+
"friendLanguage": "en",
20+
"friendUniversities": [
21+
[
22+
"Faculty_of_Health",
23+
2005,
24+
"Birmingham"
25+
]
26+
],
27+
"friendCompanies": [
28+
[
29+
"Flybe",
30+
2007,
31+
"United_Kingdom"
32+
],
33+
[
34+
"British_Mediterranean_Airways",
35+
2007,
36+
"United_Kingdom"
37+
],
38+
[
39+
"Airtours_International_Airways",
40+
2006,
41+
"United_Kingdom"
42+
]
43+
]
44+
}
45+
]
46+
}

‎flex/resources/queries/ic/runtime/ic10.cypher‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
MATCH (person:PERSON {id: 2199023256097L})-[:KNOWS*2..3]-(friend: PERSON)
2+
WHERE
3+
NOT friend=person
4+
AND NOT (friend:PERSON)-[:KNOWS]-(person :PERSON {id: 2199023256097L})
5+
WITH distinct
6+
friend
7+
WITH friend, friend.birthday as birthday
8+
WHERE (birthday.month=10 AND birthday.day>=21) OR (birthday.month=(10%12) + 1 AND birthday.day<22)
9+
10+
OPTIONAL MATCH (friend : PERSON)<-[:HASCREATOR]-(post:POST)
11+
WITH friend, count(post) as postCount
12+
13+
CALL {
14+
WITH friend, postCount
15+
MATCH (friend: PERSON) <- [:HASCREATOR]- (post: POST) - [:HASTAG] -> (tag:TAG)
16+
WITH friend, postCount, post, tag
17+
MATCH (tag:TAG)<-[:HASINTEREST]-(person: PERSON {id: 2199023256097L})
18+
WITH friend, postCount, count(distinct post) as commonPostCount
19+
return friend, commonPostCount - (postCount - commonPostCount) AS commonInterestScore, friend.id as friendId
20+
ORDER BY commonInterestScore DESC, friendId ASC
21+
LIMIT 10
22+
}
23+
UNION
24+
CALL {
25+
return friend, (0L-postCount) AS commonInterestScore, friend.id as friendId
26+
ORDER BY commonInterestScore DESC, friendId ASC
27+
LIMIT 10
28+
}
29+
WITH friend, max(commonInterestScore) AS score
30+
ORDER BY score DESC, friend.id ASC
31+
LIMIT 10
32+
MATCH (friend: PERSON)-[:ISLOCATEDIN]->(city:PLACE)
33+
RETURN friend.id AS personId,
34+
friend.firstName AS personFirstName,
35+
friend.lastName AS personLastName,
36+
score AS commonInterestScore,
37+
friend.gender AS personGender,
38+
city.name AS personCityName

‎flex/resources/queries/ic/runtime/ic10.json‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
{
2+
"operation": {
3+
"personIdQ10": 2199023256097,
4+
"month": 10,
5+
"limit": 10
6+
},
7+
"result": [
8+
{
9+
"personId": 555,
10+
"personFirstName": "Chen",
11+
"personLastName": "Yang",
12+
"commonInterestScore": 7,
13+
"personGender": "female",
14+
"personCityName": "Dali"
15+
},
16+
{
17+
"personId": 17592186045047,
18+
"personFirstName": "Hans",
19+
"personLastName": "Berg",
20+
"commonInterestScore": 6,
21+
"personGender": "female",
22+
"personCityName": "Umeå"
23+
},
24+
{
25+
"personId": 2199023256351,
26+
"personFirstName": "Ai",
27+
"personLastName": "Liu",
28+
"commonInterestScore": 4,
29+
"personGender": "male",
30+
"personCityName": "Chongqing"
31+
},
32+
{
33+
"personId": 30786325577905,
34+
"personFirstName": "Carlos",
35+
"personLastName": "Alvarez",
36+
"commonInterestScore": 0,
37+
"personGender": "male",
38+
"personCityName": "Guadalajara"
39+
},
40+
{
41+
"personId": 24189255811103,
42+
"personFirstName": "Otto",
43+
"personLastName": "Richter",
44+
"commonInterestScore": -1,
45+
"personGender": "male",
46+
"personCityName": "Ludwigsburg"
47+
},
48+
{
49+
"personId": 17592186045803,
50+
"personFirstName": "Radu",
51+
"personLastName": "Antonescu",
52+
"commonInterestScore": -3,
53+
"personGender": "female",
54+
"personCityName": "Constanța"
55+
},
56+
{
57+
"personId": 13194139534185,
58+
"personFirstName": "Tomi",
59+
"personLastName": "Blomqvist",
60+
"commonInterestScore": -7,
61+
"personGender": "female",
62+
"personCityName": "Helsinki"
63+
},
64+
{
65+
"personId": 4398046511592,
66+
"personFirstName": "Antonio",
67+
"personLastName": "Garcia",
68+
"commonInterestScore": -10,
69+
"personGender": "female",
70+
"personCityName": "Colima"
71+
},
72+
{
73+
"personId": 17592186044731,
74+
"personFirstName": "Andre",
75+
"personLastName": "Chihab",
76+
"commonInterestScore": -14,
77+
"personGender": "female",
78+
"personCityName": "Marrakech"
79+
},
80+
{
81+
"personId": 30786325578585,
82+
"personFirstName": "Ayesha",
83+
"personLastName": "Butt",
84+
"commonInterestScore": -20,
85+
"personGender": "male",
86+
"personCityName": "Tando_Jam"
87+
}
88+
]
89+
}

‎flex/resources/queries/ic/runtime/ic11.cypher‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
MATCH (p:PERSON {id: 24189255811707L})-[:KNOWS*1..3]-(friend:PERSON)
2+
WITH distinct friend
3+
WHERE
4+
friend.id <> 24189255811707L
5+
6+
MATCH (friend:PERSON)-[wa:WORKAT]->(com:ORGANISATION)-[:ISLOCATEDIN]->(:PLACE {name: "Switzerland"})
7+
WHERE wa.workFrom < 2006
8+
WITH
9+
friend as friend,
10+
com AS com,
11+
wa.workFrom as organizationWorkFromYear
12+
ORDER BY
13+
organizationWorkFromYear ASC,
14+
friend.id ASC, com.name DESC
15+
LIMIT 10
16+
return
17+
friend.id AS personId,
18+
friend.firstName AS personFirstName,
19+
friend.lastName AS personLastName,
20+
com.name as organizationName,
21+
organizationWorkFromYear as organizationWorkFromYear;

0 commit comments

Comments
 (0)