Skip to content

Commit fdfc35b

Browse files
chernserchernser
committed
Fixed javacc to support single line comments and some new statements
1 parent 505272d commit fdfc35b

File tree

9 files changed

+40
-24
lines changed

9 files changed

+40
-24
lines changed

‎jdbc-v2/pom.xml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
<url>https://github.com/ClickHouse/clickhouse-java/tree/main/jdbc-v2</url>
1818

1919
<properties>
20-
<javacc-plugin.version>4.1.4</javacc-plugin.version>
20+
<javacc-plugin.version>5.0.0</javacc-plugin.version>
2121
<spec.title>JDBC</spec.title>
2222
<spec.version>4.2</spec.version>
2323
<jackson.version>2.17.2</jackson.version>
@@ -197,7 +197,6 @@
197197
<javadocFriendlyComments>true</javadocFriendlyComments>
198198
<packageName>com.clickhouse.jdbc.internal.parser.javacc</packageName>
199199
<sourceDirectory>src/main/javacc</sourceDirectory>
200-
<outputDirectory>src/main/java</outputDirectory>
201200
</configuration>
202201
</execution>
203202
</executions>

‎jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/ParsedPreparedStatement.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ public final class ParsedPreparedStatement {
3434

3535
private int assignValuesListStopPosition = -1;
3636

37-
private int assignValuesGroups = -1;
37+
private int assignValuesGroups = 0;
3838

3939
public void setHasResultSet(boolean hasResultSet) {
4040
this.hasResultSet = hasResultSet;

‎jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/SqlParserFacade.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,10 +68,10 @@ public ParsedPreparedStatement parsePreparedStatement(String sql) {
6868
stmt.setHasResultSet(isStmtWithResultSet(parsedStmt));
6969
stmt.setTable(parsedStmt.getTable());
7070
stmt.setInsertWithSelect(parsedStmt.containsKeyword("SELECT") && (parsedStmt.getStatementType() == StatementType.INSERT));
71+
stmt.setAssignValuesGroups(parsedStmt.getValueGroups());
7172

7273
Integer startIndex = parsedStmt.getPositions().get(ClickHouseSqlStatement.KEYWORD_VALUES_START);
7374
if (startIndex != null) {
74-
stmt.setAssignValuesGroups(1);
7575
int endIndex = parsedStmt.getPositions().get(ClickHouseSqlStatement.KEYWORD_VALUES_END);
7676
stmt.setAssignValuesListStartPosition(startIndex);
7777
stmt.setAssignValuesListStopPosition(endIndex);

‎jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/ClickHouseSqlStatement.java‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,19 +41,20 @@ public class ClickHouseSqlStatement {
4141
private final Map<String, Integer> positions;
4242
private final Map<String, String> settings;
4343
private final Set<String> tempTables;
44+
private final int valueGroups;
4445

4546
public ClickHouseSqlStatement(String sql) {
46-
this(sql, StatementType.UNKNOWN, null, null, null, null, null, null, null, null, null, null, null, null);
47+
this(sql, StatementType.UNKNOWN, null, null, null, null, null, null, null, null, null, null, null, null, 0);
4748
}
4849

4950
public ClickHouseSqlStatement(String sql, StatementType stmtType) {
50-
this(sql, stmtType, null, null, null, null, null, null, null, null, null, null, null, null);
51+
this(sql, stmtType, null, null, null, null, null, null, null, null, null, null, null, null, 0);
5152
}
5253

5354
public ClickHouseSqlStatement(String sql, StatementType stmtType, String cluster, String database, String table,
5455
String input, String compressAlgorithm, String compressLevel, String format, String file,
5556
List<Integer> parameters, Map<String, Integer> positions, Map<String, String> settings,
56-
Set<String> tempTables) {
57+
Set<String> tempTables, int valueGroups) {
5758
this.sql = sql;
5859
this.stmtType = stmtType;
5960

@@ -65,6 +66,7 @@ public ClickHouseSqlStatement(String sql, StatementType stmtType, String cluster
6566
this.compressLevel = compressLevel;
6667
this.format = format;
6768
this.file = file;
69+
this.valueGroups = valueGroups;
6870

6971
if (parameters != null && !parameters.isEmpty()) {
7072
this.parameters = Collections.unmodifiableList(parameters);
@@ -297,6 +299,10 @@ public Map<String, Integer> getPositions() {
297299
return this.positions;
298300
}
299301

302+
public int getValueGroups() {
303+
return valueGroups;
304+
}
305+
300306
public Map<String, String> getSettings() {
301307
return this.settings;
302308
}

‎jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/JdbcParseHandler.java‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ private ClickHouseSqlStatement handleDelete(String sql, StatementType stmtType,
7171
builder.append("TRUNCATE TABLE").append(sql.substring(index + 4));
7272
}
7373
return new ClickHouseSqlStatement(builder.toString(), stmtType, cluster, database, table, input,
74-
compressAlgorithm, compressLevel, format, file, parameters, null, settings, null);
74+
compressAlgorithm, compressLevel, format, file, parameters, null, settings, null, 0);
7575
}
7676

7777
private ClickHouseSqlStatement handleUpdate(String sql, StatementType stmtType, String cluster, String database,
@@ -91,13 +91,13 @@ private ClickHouseSqlStatement handleUpdate(String sql, StatementType stmtType,
9191
builder.append('`').append(table).append('`').append(" UPDATE"); // .append(sql.substring(index + 3));
9292
addMutationSetting(sql, builder, positions, settings, index + 3);
9393
return new ClickHouseSqlStatement(builder.toString(), stmtType, cluster, database, table, input,
94-
compressAlgorithm, compressLevel, format, file, parameters, null, settings, null);
94+
compressAlgorithm, compressLevel, format, file, parameters, null, settings, null, 0);
9595
}
9696

9797
private ClickHouseSqlStatement handleInFileForInsertQuery(String sql, StatementType stmtType, String cluster,
98-
String database, String table, String input, String compressAlgorithm, String compressLevel, String format,
99-
String file, List<Integer> parameters, Map<String, Integer> positions, Map<String, String> settings,
100-
Set<String> tempTables) {
98+
String database, String table, String input, String compressAlgorithm, String compressLevel, String format,
99+
String file, List<Integer> parameters, Map<String, Integer> positions, Map<String, String> settings,
100+
Set<String> tempTables, int valueGroups) {
101101
StringBuilder builder = new StringBuilder(sql.length());
102102
builder.append(sql.substring(0, positions.get("FROM")));
103103
Integer index = positions.get("SETTINGS");
@@ -115,7 +115,7 @@ private ClickHouseSqlStatement handleInFileForInsertQuery(String sql, StatementT
115115
builder.append("FORMAT ").append(format);
116116
}
117117
return new ClickHouseSqlStatement(builder.toString(), stmtType, cluster, database, table, input,
118-
compressAlgorithm, compressLevel, format, file, parameters, null, settings, null);
118+
compressAlgorithm, compressLevel, format, file, parameters, null, settings, null, valueGroups);
119119
}
120120

121121
private ClickHouseSqlStatement handleOutFileForSelectQuery(String sql, StatementType stmtType, String cluster,
@@ -129,14 +129,14 @@ private ClickHouseSqlStatement handleOutFileForSelectQuery(String sql, Statement
129129
builder.append(sql.substring(index));
130130
}
131131
return new ClickHouseSqlStatement(builder.toString(), stmtType, cluster, database, table, input,
132-
compressAlgorithm, compressLevel, format, file, parameters, null, settings, null);
132+
compressAlgorithm, compressLevel, format, file, parameters, null, settings, null, 0);
133133
}
134134

135135
@Override
136136
public ClickHouseSqlStatement handleStatement(String sql, StatementType stmtType, String cluster, String database,
137137
String table, String input, String compressAlgorithm, String compressLevel, String format, String file,
138138
List<Integer> parameters, Map<String, Integer> positions, Map<String, String> settings,
139-
Set<String> tempTables) {
139+
Set<String> tempTables, int valueGroups) {
140140
boolean hasFile = allowLocalFile && !ClickHouseChecker.isNullOrEmpty(file) && file.charAt(0) == '\'';
141141
ClickHouseSqlStatement s = null;
142142
if (stmtType == StatementType.DELETE) {
@@ -149,7 +149,7 @@ public ClickHouseSqlStatement handleStatement(String sql, StatementType stmtType
149149
format, file, parameters, positions, settings, tempTables);
150150
} else if (stmtType == StatementType.INSERT && hasFile) {
151151
s = handleInFileForInsertQuery(sql, stmtType, cluster, database, table, input, compressAlgorithm,
152-
compressLevel, format, file, parameters, positions, settings, tempTables);
152+
compressLevel, format, file, parameters, positions, settings, tempTables, valueGroups);
153153
} else if (stmtType == StatementType.SELECT && hasFile) {
154154
s = handleOutFileForSelectQuery(sql, stmtType, cluster, database, table, input, compressAlgorithm,
155155
compressLevel, format, file, parameters, positions, settings, tempTables);

‎jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/parser/javacc/ParseHandler.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ public String handleParameter(String cluster, String database, String table, int
5252
public ClickHouseSqlStatement handleStatement(String sql, StatementType stmtType, String cluster, String database,
5353
String table, String input, String compressAlgorithm, String compressLevel, String format, String file,
5454
List<Integer> parameters, Map<String, Integer> positions, Map<String, String> settings,
55-
Set<String> tempTables) {
55+
Set<String> tempTables, int valueGroup) {
5656
return null;
5757
}
5858
}

‎jdbc-v2/src/main/javacc/ClickHouseSqlParser.jj‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,7 @@ TOKEN_MGR_DECLS: {
174174
String compressLevel = null;
175175
String format = null;
176176
String file = null;
177+
int valueGroups = 0;
177178

178179
final List<Integer> parameters = new ArrayList<>();
179180
final Map<String, Integer> positions = new HashMap<>();
@@ -276,12 +277,12 @@ TOKEN_MGR_DECLS: {
276277

277278
if (handler != null) {
278279
s = handler.handleStatement(
279-
sqlStmt, stmtType, cluster, database, table, input, compressAlgorithm, compressLevel, format, file, parameters, positions, settings, tempTables);
280+
sqlStmt, stmtType, cluster, database, table, input, compressAlgorithm, compressLevel, format, file, parameters, positions, settings, tempTables, valueGroups);
280281
}
281282

282283
if (s == null) {
283284
s = new ClickHouseSqlStatement(
284-
sqlStmt, stmtType, cluster, database, table, input, compressAlgorithm, compressLevel, format, file, parameters, positions, settings, tempTables);
285+
sqlStmt, stmtType, cluster, database, table, input, compressAlgorithm, compressLevel, format, file, parameters, positions, settings, tempTables, valueGroups);
285286
}
286287

287288
// reset variables
@@ -323,6 +324,9 @@ TOKEN_MGR_DECLS: {
323324

324325
this.settings.put(key.toLowerCase(Locale.ROOT), value);
325326
}
327+
void incValueGroup() {
328+
this.valueGroups++;
329+
}
326330
}
327331

328332
SKIP: {
@@ -366,7 +370,7 @@ SKIP: {
366370
}
367371
}
368372
}
369-
| <SINGLE_LINE_COMMENT: ("--" | "//") (~["\r", "\n"])*> { append(image); }
373+
| <SINGLE_LINE_COMMENT: ("--" | "//" | "#" | "#!") (~["\r", "\n"])*> { append(image); }
370374
| "/*" { commentNestingDepth = 1; append(image); }: MULTI_LINE_COMMENT
371375
}
372376

@@ -418,6 +422,7 @@ void stmt(): {} {
418422
| optimizeStmt() { token_source.stmtType = StatementType.OPTIMIZE; }
419423
| renameStmt() { token_source.stmtType = StatementType.RENAME; }
420424
| revokeStmt() { token_source.stmtType = StatementType.REVOKE; }
425+
| <LPAREN> selectStmt() <RPAREN> { token_source.stmtType = StatementType.SELECT; }
421426
| selectStmt() { token_source.stmtType = StatementType.SELECT; }
422427
| setStmt() { token_source.stmtType = StatementType.SET; }
423428
| showStmt() { token_source.stmtType = StatementType.SHOW; }
@@ -569,12 +574,12 @@ void dataClause(): {} {
569574
try {
570575
LOOKAHEAD(2) <VALUES> { token_source.addPosition(token); }
571576
<LPAREN> { token_source.addCustomKeywordPosition(ClickHouseSqlStatement.KEYWORD_VALUES_START, token); }
572-
columnExprList()
577+
columnExprList() { token_source.incValueGroup(); }
573578
<RPAREN> { token_source.addCustomKeywordPosition(ClickHouseSqlStatement.KEYWORD_VALUES_END, token); }
574579
(
575580
LOOKAHEAD(2)
576581
(<COMMA>)?
577-
<LPAREN> { token_source.removePosition(ClickHouseSqlStatement.KEYWORD_VALUES_START); }
582+
<LPAREN> { token_source.removePosition(ClickHouseSqlStatement.KEYWORD_VALUES_START); token_source.incValueGroup(); }
578583
columnExprList()
579584
<RPAREN> { token_source.removePosition(ClickHouseSqlStatement.KEYWORD_VALUES_END); }
580585
)*
@@ -656,6 +661,7 @@ void showStmt(): {} {
656661
LOOKAHEAD(2)
657662
<CREATE>
658663
(<DATABASE> databaseIdentifier(true))
664+
| LOOKAHEAD(2) (LOOKAHEAD(1) <CREATE>)? (LOOKAHEAD(1) <SETTINGS>)? <PROFILE> anyIdentifier()
659665
| LOOKAHEAD(2) (<DICTIONARY> tableIdentifier(true))
660666
| LOOKAHEAD(2) ((LOOKAHEAD(2) <TEMPORARY>)? (LOOKAHEAD(2) <TABLE>)? tableIdentifier(true))
661667
)

‎jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/BaseSqlParserFacadeTest.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,9 +125,14 @@ public static Object[][] testPreparedStatementInsertSQLDP() {
125125
+ "#! line comment3 ?\n"
126126
+ "/* block comment ? \n */"
127127
+ "INSERT INTO `with_complex_id`(`v?``1`, \"v?\"\"2\",`v?\\`3`, \"v?\\\"4\") VALUES (?, ?, ?, ?);", 1, false, 4},
128+
{"-- line comment1 ?\n"
129+
// + "# line comment2 ?\n"
130+
// + "#! line comment3 ?\n"
131+
+ "/* block comment ? \n */"
132+
+ "INSERT INTO `with_complex_id`(`v?``1`, \"v?\"\"2\",`v?\\`3`, \"v?\\\"4\") VALUES (?, ?, ?, ?);", 1, false, 4},
128133
{ "INSERT INTO `test_stmt_split2` VALUES (1, 'abc'), (2, '?'), (3, '?')", 3, false, 0 },
129134
{ "INSERT INTO `with_complex_id`(`v?``1`, \"v?\"\"2\",`v?\\`3`, \"v?\\\"4\") VALUES (?, ?, ?, ?);", 1, false, 4},
130-
{ "INSERT INTO tt SELECT now(), 10, 20.0, 30", -1, true, 0 },
135+
{ "INSERT INTO tt SELECT now(), 10, 20.0, 30", 0, true, 0 },
131136
{ "INSERT INTO `users` (`name`, `last_login`, `password`, `id`) VALUES\n" +
132137
" (?, `parseDateTimeBestEffort`(?, ?), ?, 1)\n", 1, false, 4 },
133138
};

‎jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/JavaCCParserTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33

44
import org.testng.annotations.Ignore;
55

6-
@Ignore
6+
//@Ignore
77
public class JavaCCParserTest extends BaseSqlParserFacadeTest {
88
public JavaCCParserTest() throws Exception {
99
super(SqlParserFacade.SQLParser.JAVACC.name());

0 commit comments

Comments
 (0)