Skip to content

Commit 4b4b3b9

Browse files
morningmanmorningman
authored
[feat](sql-convertor) support enable sql convertor's feature by session variable (apache#50707)
### What problem does this PR solve? Problem Summary: Support new session variable `enable_sql_convertor_features`. Some features of sql convertor is disable by default, User can enable certain features of sql convertor by this variable. eg: ``` // enable ctas statement conversion. set enable_sql_convertor_features="ctas" // enable ctas statement conversion, and will delete all comment in statement set enable_sql_convertor_features="ctas,delete_all_comment" ```
1 parent 4aa3b49 commit 4b4b3b9

File tree

5 files changed

+74
-9
lines changed

5 files changed

+74
-9
lines changed

‎fe/fe-core/src/main/java/org/apache/doris/plugin/dialect/HttpDialectConverterPlugin.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,8 @@ public ImmutableSet<Dialect> acceptDialects() {
103103
if (Strings.isNullOrEmpty(targetURL)) {
104104
return null;
105105
}
106-
return HttpDialectUtils.convertSql(targetURL, originSql, sessionVariable.getSqlDialect());
106+
return HttpDialectUtils.convertSql(targetURL, originSql, sessionVariable.getSqlDialect(),
107+
sessionVariable.getSqlConvertorFeatures());
107108
}
108109

109110
// no need to override parseSqlWithDialect, just return null

‎fe/fe-core/src/main/java/org/apache/doris/plugin/dialect/HttpDialectUtils.java‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,9 @@
3838
public class HttpDialectUtils {
3939
private static final Logger LOG = LogManager.getLogger(HttpDialectUtils.class);
4040

41-
public static String convertSql(String targetURL, String originStmt, String dialect) {
42-
ConvertRequest convertRequest = new ConvertRequest(originStmt, dialect);
41+
public static String convertSql(String targetURL, String originStmt, String dialect,
42+
String[] features) {
43+
ConvertRequest convertRequest = new ConvertRequest(originStmt, dialect, features);
4344

4445
HttpURLConnection connection = null;
4546
try {
@@ -110,14 +111,16 @@ private static class ConvertRequest {
110111
private String to; // CHECKSTYLE IGNORE THIS LINE
111112
private String source; // CHECKSTYLE IGNORE THIS LINE
112113
private String case_sensitive; // CHECKSTYLE IGNORE THIS LINE
114+
private String[] enable_sql_convertor_features; // CHECKSTYLE IGNORE THIS LINE
113115

114-
public ConvertRequest(String originStmt, String dialect) {
116+
public ConvertRequest(String originStmt, String dialect, String[] features) {
115117
this.version = "v1";
116118
this.sql_query = originStmt;
117119
this.from = dialect;
118120
this.to = "doris";
119121
this.source = "text";
120122
this.case_sensitive = "0";
123+
this.enable_sql_convertor_features = features;
121124
}
122125

123126
public String toJson() {

‎fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -732,6 +732,8 @@ public class SessionVariable implements Serializable, Writable {
732732

733733
public static final String ENABLE_TEXT_VALIDATE_UTF8 = "enable_text_validate_utf8";
734734

735+
public static final String ENABLE_SQL_CONVERTOR_FEATURES = "enable_sql_convertor_features";
736+
735737
/**
736738
* If set false, user couldn't submit analyze SQL and FE won't allocate any related resources.
737739
*/
@@ -2575,6 +2577,14 @@ public void setDetailShapePlanNodes(String detailShapePlanNodes) {
25752577
})
25762578
public boolean skipCheckingAcidVersionFile = false;
25772579

2580+
@VariableMgr.VarAttr(name = ENABLE_SQL_CONVERTOR_FEATURES, needForward = true,
2581+
checker = "checkSqlConvertorFeatures",
2582+
description = {
2583+
"开启 SQL 转换器的指定功能。多个功能使用逗号分隔",
2584+
"enable SQL convertor features. Multiple features are separated by commas"
2585+
})
2586+
public String enableSqlConvertorFeatures = "";
2587+
25782588
public void setEnableEsParallelScroll(boolean enableESParallelScroll) {
25792589
this.enableESParallelScroll = enableESParallelScroll;
25802590
}
@@ -3520,6 +3530,10 @@ public String getSqlDialect() {
35203530
return sqlDialect;
35213531
}
35223532

3533+
public String[] getSqlConvertorFeatures() {
3534+
return enableSqlConvertorFeatures.split(",");
3535+
}
3536+
35233537
public Dialect getSqlParseDialect() {
35243538
return Dialect.getByName(sqlDialect);
35253539
}
@@ -4794,6 +4808,20 @@ public void checkProfileLevel(String profileLevel) {
47944808
}
47954809
}
47964810

4811+
public void checkSqlConvertorFeatures(String features) {
4812+
if (Strings.isNullOrEmpty(features)) {
4813+
return;
4814+
}
4815+
String[] featureArray = features.split(",");
4816+
for (String feature : featureArray) {
4817+
if (!feature.equalsIgnoreCase("ctas")
4818+
&& !feature.equalsIgnoreCase("delete_all_comment")) {
4819+
throw new UnsupportedOperationException("Unknown sql convertor feature: " + feature
4820+
+ ", current support: ctas, delete_all_comment");
4821+
}
4822+
}
4823+
}
4824+
47974825
public boolean getEnableLocalMergeSort() {
47984826
return enableLocalMergeSort;
47994827
}
@@ -4805,4 +4833,5 @@ public boolean getEnableExternalTableBatchMode() {
48054833
public boolean showSplitProfileInfo() {
48064834
return enableProfile() && getProfileLevel() > 1;
48074835
}
4836+
48084837
}

‎fe/fe-core/src/test/java/org/apache/doris/plugin/HttpDialectUtilsTest.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -52,22 +52,22 @@ public void tearDown() {
5252
public void testSqlConvert() {
5353
String originSql = "select * from t1 where \"k1\" = 1";
5454
String expectedSql = "select * from t1 where `k1` = 1";
55-
55+
String[] features = new String[] {"ctas"};
5656
String targetURL = "http://127.0.0.1:" + port + "/api/v1/convert";
57-
String res = HttpDialectUtils.convertSql(targetURL, originSql, "presto");
57+
String res = HttpDialectUtils.convertSql(targetURL, originSql, "presto", features);
5858
Assert.assertEquals(originSql, res);
5959
// test presto
6060
server.setResponse("{\"version\": \"v1\", \"data\": \"" + expectedSql + "\", \"code\": 0, \"message\": \"\"}");
61-
res = HttpDialectUtils.convertSql(targetURL, originSql, "presto");
61+
res = HttpDialectUtils.convertSql(targetURL, originSql, "presto", features);
6262
Assert.assertEquals(expectedSql, res);
6363
// test response version error
6464
server.setResponse("{\"version\": \"v2\", \"data\": \"" + expectedSql + "\", \"code\": 0, \"message\": \"\"}");
65-
res = HttpDialectUtils.convertSql(targetURL, originSql, "presto");
65+
res = HttpDialectUtils.convertSql(targetURL, originSql, "presto", features);
6666
Assert.assertEquals(originSql, res);
6767
// 7. test response code error
6868
server.setResponse(
6969
"{\"version\": \"v1\", \"data\": \"" + expectedSql + "\", \"code\": 400, \"message\": \"\"}");
70-
res = HttpDialectUtils.convertSql(targetURL, originSql, "presto");
70+
res = HttpDialectUtils.convertSql(targetURL, originSql, "presto", features);
7171
Assert.assertEquals(originSql, res);
7272
}
7373

‎fe/fe-core/src/test/java/org/apache/doris/qe/VariableMgrTest.java‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -292,4 +292,36 @@ public void testAutoCommitType() throws AnalysisException {
292292
Literal l = VariableMgr.getLiteral(sv, name, setType);
293293
Assert.assertEquals(BigIntType.INSTANCE, l.getDataType());
294294
}
295+
296+
@Test
297+
public void testCheckSqlConvertorFeatures() throws DdlException {
298+
// set wrong var
299+
SetVar setVar = new SetVar(SetType.SESSION, SessionVariable.ENABLE_SQL_CONVERTOR_FEATURES,
300+
new StringLiteral("wrong"));
301+
SessionVariable var = new SessionVariable();
302+
try {
303+
VariableMgr.setVar(var, setVar);
304+
} catch (DdlException e) {
305+
Assert.assertTrue(e.getMessage().contains("Unknown sql convertor feature: wrong"));
306+
}
307+
308+
// set one var
309+
Assert.assertEquals(new String[] {""}, var.getSqlConvertorFeatures());
310+
setVar = new SetVar(SetType.SESSION, SessionVariable.ENABLE_SQL_CONVERTOR_FEATURES,
311+
new StringLiteral("ctas"));
312+
VariableMgr.setVar(var, setVar);
313+
Assert.assertEquals(new String[] {"ctas"}, var.getSqlConvertorFeatures());
314+
315+
// set multiple var
316+
setVar = new SetVar(SetType.SESSION, SessionVariable.ENABLE_SQL_CONVERTOR_FEATURES,
317+
new StringLiteral("ctas,delete_all_comment"));
318+
VariableMgr.setVar(var, setVar);
319+
Assert.assertEquals(new String[] {"ctas", "delete_all_comment"}, var.getSqlConvertorFeatures());
320+
321+
// set to empty
322+
setVar = new SetVar(SetType.SESSION, SessionVariable.ENABLE_SQL_CONVERTOR_FEATURES,
323+
new StringLiteral(""));
324+
VariableMgr.setVar(var, setVar);
325+
Assert.assertEquals(new String[] {""}, var.getSqlConvertorFeatures());
326+
}
295327
}

0 commit comments

Comments
 (0)