Skip to content

Commit 27f5249

Browse files
idodeclareVladimir Kotal
idodeclare
authored and
Vladimir Kotal
committed
Extract JointSQLProductions.lexh. Add SQL tokenizers.
1 parent d4c862c commit 27f5249

20 files changed

+3059
-318
lines changed

‎opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/JFlexStackingLexer.java

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
/*
2121
* Copyright (c) 2009, 2018, Oracle and/or its affiliates. All rights reserved.
2222
* Portions Copyright 2011 Jens Elkner.
23-
* Portions Copyright (c) 2017, Chris Fraire <[email protected]>.
23+
* Portions Copyright (c) 2017, 2019, Chris Fraire <[email protected]>.
2424
*/
2525

2626
package org.opengrok.indexer.analysis;
@@ -47,15 +47,18 @@ public interface JFlexStackingLexer extends JFlexLexer {
4747
*/
4848
void yypop() throws IOException;
4949

50+
/**
51+
* Gets the yychar value.
52+
*/
53+
int getYYCHAR();
54+
5055
/**
5156
* Gets the YYEOF value.
52-
* @return YYEOF
5357
*/
5458
int getYYEOF();
5559

5660
/**
5761
* Gets the yyline value.
58-
* @return yyline
5962
*/
6063
int getLineNumber();
6164

‎opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/sql/JointSQLLexer.java

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
/*
2+
* CDDL HEADER START
3+
*
4+
* The contents of this file are subject to the terms of the
5+
* Common Development and Distribution License (the "License").
6+
* You may not use this file except in compliance with the License.
7+
*
8+
* See LICENSE.txt included in this distribution for the specific
9+
* language governing permissions and limitations under the License.
10+
*
11+
* When distributing Covered Code, include this CDDL HEADER in each
12+
* file and include the License file at LICENSE.txt.
13+
* If applicable, add the following below this CDDL HEADER, with the
14+
* fields enclosed by brackets "[]" replaced with your own identifying
15+
* information: Portions Copyright [yyyy] [name of copyright owner]
16+
*
17+
* CDDL HEADER END
18+
*/
19+
20+
/*
21+
* Copyright (c) 2019, Chris Fraire <[email protected]>.
22+
*/
23+
24+
package org.opengrok.indexer.analysis.sql;
25+
26+
import org.opengrok.indexer.analysis.JFlexJointLexer;
27+
import org.opengrok.indexer.analysis.JFlexSymbolMatcher;
28+
import org.opengrok.indexer.analysis.Resettable;
29+
30+
import java.io.IOException;
31+
import java.util.Set;
32+
33+
/**
34+
* Represents an abstract base class for SQL lexers of various dialects.
35+
*/
36+
abstract class JointSQLLexer extends JFlexSymbolMatcher implements JFlexJointLexer, Resettable {
37+
38+
protected int commentLevel;
39+
40+
@Override
41+
public void reset() {
42+
super.reset();
43+
commentLevel = 0;
44+
}
45+
46+
@Override
47+
public void yypop() throws IOException {
48+
onDisjointSpanChanged(null, getYYCHAR());
49+
super.yypop();
50+
}
51+
52+
/**
53+
* Calls {@link #phLOC()} if the yystate is not BRACKETED_COMMENT or
54+
* SINGLE_LINE_COMMENT.
55+
*/
56+
public void chkLOC() {
57+
if (yystate() != BRACKETED_COMMENT() && yystate() != SINGLE_LINE_COMMENT()) {
58+
phLOC();
59+
}
60+
}
61+
62+
/**
63+
* Subclasses must override to get the dialect's keywords set.
64+
*/
65+
abstract Set<String> getDialectKeywords();
66+
67+
/**
68+
* Subclasses must override to get the constant value created by JFlex to
69+
* represent BRACKETED_COMMENT.
70+
*/
71+
abstract int BRACKETED_COMMENT();
72+
73+
/**
74+
* Subclasses must override to get the constant value created by JFlex to
75+
* represent SINGLE_LINE_COMMENT.
76+
*/
77+
abstract int SINGLE_LINE_COMMENT();
78+
}

‎opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/sql/JointSQLSymbolTokenizer.java

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
/*
2+
* CDDL HEADER START
3+
*
4+
* The contents of this file are subject to the terms of the
5+
* Common Development and Distribution License (the "License").
6+
* You may not use this file except in compliance with the License.
7+
*
8+
* See LICENSE.txt included in this distribution for the specific
9+
* language governing permissions and limitations under the License.
10+
*
11+
* When distributing Covered Code, include this CDDL HEADER in each
12+
* file and include the License file at LICENSE.txt.
13+
* If applicable, add the following below this CDDL HEADER, with the
14+
* fields enclosed by brackets "[]" replaced with your own identifying
15+
* information: Portions Copyright [yyyy] [name of copyright owner]
16+
*
17+
* CDDL HEADER END
18+
*/
19+
20+
/*
21+
* Copyright (c) 2019, Chris Fraire <[email protected]>.
22+
*/
23+
24+
package org.opengrok.indexer.analysis.sql;
25+
26+
import java.util.Locale;
27+
28+
/**
29+
* Represents an abstract base class for SQL symbol tokenizers of various
30+
* dialects.
31+
*/
32+
abstract class JointSQLSymbolTokenizer extends JointSQLLexer {
33+
34+
private String lastSymbol;
35+
36+
/**
37+
* Resets the SQL tracked state. {@inheritDoc}
38+
*/
39+
@Override
40+
public void reset() {
41+
super.reset();
42+
lastSymbol = null;
43+
}
44+
45+
/** noop. */
46+
@Override
47+
public void offer(String value) {
48+
}
49+
50+
@Override
51+
public boolean offerSymbol(String value, int captureOffset, boolean ignoreKwd) {
52+
if (ignoreKwd || !getDialectKeywords().contains(value.toLowerCase(Locale.ROOT))) {
53+
lastSymbol = value;
54+
onSymbolMatched(value, getYYCHAR() + captureOffset);
55+
return true;
56+
} else {
57+
lastSymbol = null;
58+
}
59+
return false;
60+
}
61+
62+
@Override
63+
public void skipSymbol() {
64+
lastSymbol = null;
65+
}
66+
67+
@Override
68+
public void offerKeyword(String value) {
69+
lastSymbol = null;
70+
}
71+
72+
/** noop. */
73+
@Override
74+
public void startNewLine() {
75+
}
76+
77+
/** noop. */
78+
@Override
79+
public void disjointSpan(String className) {
80+
}
81+
82+
/** noop. */
83+
@Override
84+
public void phLOC() {
85+
}
86+
87+
/** Gets the value {@code false}. */
88+
protected boolean takeAllContent() {
89+
return false;
90+
}
91+
92+
/** Gets the value indicating if {@link #lastSymbol} is defined. */
93+
protected boolean returnOnSymbol() {
94+
return lastSymbol != null;
95+
}
96+
}

‎opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/sql/JointSQLXref.java

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
/*
2+
* CDDL HEADER START
3+
*
4+
* The contents of this file are subject to the terms of the
5+
* Common Development and Distribution License (the "License").
6+
* You may not use this file except in compliance with the License.
7+
*
8+
* See LICENSE.txt included in this distribution for the specific
9+
* language governing permissions and limitations under the License.
10+
*
11+
* When distributing Covered Code, include this CDDL HEADER in each
12+
* file and include the License file at LICENSE.txt.
13+
* If applicable, add the following below this CDDL HEADER, with the
14+
* fields enclosed by brackets "[]" replaced with your own identifying
15+
* information: Portions Copyright [yyyy] [name of copyright owner]
16+
*
17+
* CDDL HEADER END
18+
*/
19+
20+
/*
21+
* Copyright (c) 2019, Chris Fraire <[email protected]>.
22+
*/
23+
24+
package org.opengrok.indexer.analysis.sql;
25+
26+
import java.util.Set;
27+
28+
/**
29+
* Represents an abstract base class for SQL xrefers of various dialects.
30+
*/
31+
abstract class JointSQLXref extends JointSQLLexer {
32+
33+
@Override
34+
public void offer(String value) {
35+
onNonSymbolMatched(value, getYYCHAR());
36+
}
37+
38+
@Override
39+
public boolean offerSymbol(String value, int captureOffset, boolean ignoreKwd) {
40+
Set<String> keywords = ignoreKwd ? null : getDialectKeywords();
41+
return onFilteredSymbolMatched(value, getYYCHAR(), keywords, false);
42+
}
43+
44+
/** noop. */
45+
@Override
46+
public void skipSymbol() {
47+
}
48+
49+
@Override
50+
public void offerKeyword(String value) {
51+
onKeywordMatched(value, getYYCHAR());
52+
}
53+
54+
@Override
55+
public void startNewLine() {
56+
onEndOfLineMatched("\n", getYYCHAR());
57+
}
58+
59+
@Override
60+
public void disjointSpan(String className) {
61+
onDisjointSpanChanged(className, getYYCHAR());
62+
}
63+
64+
/** Gets the value {@code true}. */
65+
protected boolean takeAllContent() {
66+
return true;
67+
}
68+
69+
/** Gets the value {@code false}. */
70+
protected boolean returnOnSymbol() {
71+
return false;
72+
}
73+
}

‎opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/sql/PLSQLAnalyzer.java

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,14 +24,18 @@
2424
package org.opengrok.indexer.analysis.sql;
2525

2626
import java.io.Reader;
27+
28+
import org.opengrok.indexer.analysis.AbstractAnalyzer;
2729
import org.opengrok.indexer.analysis.AnalyzerFactory;
30+
import org.opengrok.indexer.analysis.JFlexTokenizer;
2831
import org.opengrok.indexer.analysis.JFlexXref;
2932
import org.opengrok.indexer.analysis.plain.PlainAnalyzer;
3033

3134
public class PLSQLAnalyzer extends PlainAnalyzer {
3235

3336
public PLSQLAnalyzer(AnalyzerFactory factory) {
34-
super(factory);
37+
super(factory, new JFlexTokenizer(new PLSQLSymbolTokenizer(
38+
AbstractAnalyzer.DUMMY_READER)));
3539
}
3640

3741
/**

‎opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/sql/SQLAnalyzer.java

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,14 +24,18 @@
2424
package org.opengrok.indexer.analysis.sql;
2525

2626
import java.io.Reader;
27+
28+
import org.opengrok.indexer.analysis.AbstractAnalyzer;
2729
import org.opengrok.indexer.analysis.AnalyzerFactory;
30+
import org.opengrok.indexer.analysis.JFlexTokenizer;
2831
import org.opengrok.indexer.analysis.JFlexXref;
2932
import org.opengrok.indexer.analysis.plain.PlainAnalyzer;
3033

3134
public class SQLAnalyzer extends PlainAnalyzer {
3235

3336
public SQLAnalyzer(AnalyzerFactory factory) {
34-
super(factory);
37+
super(factory, new JFlexTokenizer(new SQLSymbolTokenizer(
38+
AbstractAnalyzer.DUMMY_READER)));
3539
}
3640

3741
/**

‎opengrok-indexer/src/main/resources/analysis/CommonLexer.lexh

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,17 @@
1818
*/
1919

2020
/*
21-
* Copyright (c) 2017, Chris Fraire <[email protected]>.
21+
* Copyright (c) 2017, 2019, Chris Fraire <[email protected]>.
2222
*/
2323

2424
%{
25+
/**
26+
* Gets the yychar value.
27+
* @return yychar
28+
*/
29+
@Override
30+
public int getYYCHAR() { return yychar; }
31+
2532
/**
2633
* Gets the YYEOF value.
2734
* @return YYEOF

‎opengrok-indexer/src/main/resources/analysis/sql/JointSQL.lexh

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
/*
2+
* CDDL HEADER START
3+
*
4+
* The contents of this file are subject to the terms of the
5+
* Common Development and Distribution License (the "License").
6+
* You may not use this file except in compliance with the License.
7+
*
8+
* See LICENSE.txt included in this distribution for the specific
9+
* language governing permissions and limitations under the License.
10+
*
11+
* When distributing Covered Code, include this CDDL HEADER in each
12+
* file and include the License file at LICENSE.txt.
13+
* If applicable, add the following below this CDDL HEADER, with the
14+
* fields enclosed by brackets "[]" replaced with your own identifying
15+
* information: Portions Copyright [yyyy] [name of copyright owner]
16+
*
17+
* CDDL HEADER END
18+
*/
19+
20+
/*
21+
* Copyright (c) 2007, 2018, Oracle and/or its affiliates. All rights reserved.
22+
* Portions Copyright (c) 2017, 2019, Chris Fraire <[email protected]>.
23+
*/
24+
25+
Sign = "+" | "-"
26+
SimpleNumber = [0-9]+ | [0-9]+ "." [0-9]* | [0-9]* "." [0-9]+
27+
ScientificNumber = ({SimpleNumber} [eE] {Sign}? [0-9]+)

0 commit comments

Comments
 (0)