Skip to content
This repository was archived by the owner on Oct 15, 2020. It is now read-only.

Commit df53469

Browse files
GlassAndOneHalfGlassAndOneHalf
authored
Fix issues with template parsing. (#148)
* Fix various parsing errors. - Support parameter packs in parameter lists. - Support move parameters e.g. `Foo&& foo`. - Support `const` & `volatile` template arguments - Support multiple template definitions e.g. `template<class A> template<class B> void foo(...)` * Fix anonymous parameter parsing. * Revert rethrow of parser exception.
1 parent 0ab7022 commit df53469

File tree

11 files changed

+316
-72
lines changed

11 files changed

+316
-72
lines changed

‎src/main/antlr4/io/shiftleft/fuzzyc2cpg/Common.g4‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -110,11 +110,11 @@ template_decl_param_list: template_template template_decl_keyword template_name
110110
template_decl_param |
111111
template_decl_param_list ',' template_decl_param;
112112
template_template: TEMPLATE '<' (template_decl_keyword ','?)+ '>';
113-
template_decl_param: template_decl_keyword template_name?;
113+
template_decl_param: (template_decl_keyword | identifier) template_name?;
114114
template_decl_keyword: 'typename' | 'class';
115115
template_name: ALPHA_NUMERIC+ ELLIPSIS? ;
116116

117-
template_args: ('<' template_args '>' | '(' template_args ')' | base_type ELLIPSIS? | ',')+;
117+
template_args: ('<' template_args '>' | '(' template_args ')' | CV_QUALIFIER? base_type ELLIPSIS? | ',')+;
118118

119119
// water
120120

@@ -140,13 +140,14 @@ number: HEX_LITERAL | DECIMAL_LITERAL | OCTAL_LITERAL;
140140

141141
ptrs: (CV_QUALIFIER? ptr_operator 'restrict'?)+;
142142
func_ptrs: ptrs;
143+
rvalue_ref: '&&';
143144

144145
class_key: 'struct' | 'class' | 'union' | 'enum';
145146

146-
class_def: template_decl? class_key gcc_attribute? class_name? template_args? base_classes? OPENING_CURLY {skipToEndOfObject(); } ;
147+
class_def: template_decl* class_key gcc_attribute? class_name? template_args? base_classes? OPENING_CURLY {skipToEndOfObject(); } ;
147148
class_name: identifier;
148149
base_classes: ':' base_class (',' base_class)*;
149-
base_class: VIRTUAL? access_specifier? identifier;
150+
base_class: VIRTUAL? access_specifier? identifier template_args?;
150151

151152

152153
type_name : (CV_QUALIFIER* (class_key | UNSIGNED | SIGNED)?
@@ -155,7 +156,7 @@ type_name : (CV_QUALIFIER* (class_key | UNSIGNED | SIGNED)?
155156
| SIGNED
156157
;
157158

158-
base_type: (ALPHA_NUMERIC | AUTO | VOID | LONG | LONG)+;
159+
base_type: ((ALPHA_NUMERIC | AUTO | VOID | LONG | LONG) ELLIPSIS?)+;
159160

160161
gcc_attribute: GCC_ATTRIBUTE '(' '(' identifier ')' ')';
161162

‎src/main/antlr4/io/shiftleft/fuzzyc2cpg/Function.g4‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,4 +100,4 @@ param_type_list: '(' VOID ')'
100100
| '(' (param_type (',' param_type)*)? ')';
101101

102102
param_type: param_decl_specifiers param_type_id;
103-
param_type_id: ptrs? ('(' param_type_id ')' | parameter_name?) type_suffix?;
103+
param_type_id: (ptrs | rvalue_ref)? ('(' param_type_id ')' | parameter_name?) type_suffix?;

‎src/main/antlr4/io/shiftleft/fuzzyc2cpg/Module.g4‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,18 +22,20 @@ code : (function_decl | function_def | simple_decl | using_directive | water)*;
2222

2323
using_directive: USING NAMESPACE identifier ';';
2424

25-
function_decl: ('extern' | template_decl)? return_type? function_name function_param_list ctor_list? ';';
25+
function_decl: ('extern'? | template_decl*) return_type? function_name function_param_list ctor_list? ';';
2626

27-
function_def: template_decl? return_type? function_name function_param_list ctor_list? compound_statement;
27+
function_def: template_decl* return_type? function_name function_param_list ctor_list? compound_statement;
2828

2929
return_type : (function_decl_specifiers* type_name) ptr_operator*;
3030

3131
function_param_list : '(' parameter_decl_clause? ')' CV_QUALIFIER* exception_specification?;
3232

3333
parameter_decl_clause: (parameter_decl (',' parameter_decl)*) (',' '...')?
3434
| VOID;
35-
parameter_decl : param_decl_specifiers parameter_id;
36-
parameter_id: ptrs? ('(' parameter_id ')' | parameter_name) type_suffix?;
35+
parameter_ptrs: ptrs | rvalue_ref;
36+
parameter_decl: param_decl_specifiers parameter_id |
37+
param_decl_specifiers parameter_ptrs?;
38+
parameter_id: parameter_ptrs? ('(' parameter_id ')' | parameter_name) type_suffix?;
3739

3840
compound_statement: OPENING_CURLY { skipToEndOfObject(); };
3941

@@ -73,7 +75,7 @@ simple_decl : storage_class_specifier* var_decl;
7375
storage_class_specifier: (EXTERN | TYPEDEF);
7476

7577
var_decl : class_def init_declarator_list? #declByClass
76-
| template_decl? type_name init_declarator_list #declByType
78+
| template_decl* type_name init_declarator_list #declByType
7779
;
7880

7981
init_declarator_list: init_declarator (',' init_declarator)* ';';

‎src/main/java/io/shiftleft/fuzzyc2cpg/parser/AstNodeFactory.java‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -88,12 +88,17 @@ public static Parameter create(Parameter_declContext ctx) {
8888
Parameter param = new Parameter();
8989

9090
Parameter_declContext paramCtx = ctx;
91-
Parameter_nameContext paramName = getNameOfParameter(paramCtx);
92-
9391
Identifier name = new Identifier();
92+
93+
if (ctx.parameter_id() != null) {
94+
Parameter_nameContext paramName = getNameOfParameter(paramCtx);
95+
initializeFromContext(name, paramName);
96+
} else {
97+
name.setCodeStr("<anonymous>");
98+
}
99+
94100
ParameterType type = new ParameterType();
95101
initializeFromContext(type, ctx);
96-
initializeFromContext(name, paramName);
97102
initializeFromContext(param, ctx);
98103

99104
param.addChild(type);

‎src/main/java/io/shiftleft/fuzzyc2cpg/parser/ModuleFunctionParserInterface.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,7 @@ public static CompoundStatement parseFunctionContents(
2828
try {
2929
driver.parseAndWalkString(text);
3030
} catch (RuntimeException ex) {
31-
logger.info(ctx.function_name().getText() + " was skipped.");
32-
//ex.printStackTrace();
31+
logger.error(ctx.function_name().getText() + " was skipped."/*, ex*/);
3332
}
3433
CompoundStatement result = driver.getResult();
3534
Compound_statementContext statementContext = ctx.compound_statement();

‎src/main/java/io/shiftleft/fuzzyc2cpg/parser/functions/builder/ParameterListBuilder.java‎

Lines changed: 36 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313

1414
public class ParameterListBuilder extends AstNodeBuilder {
1515

16-
ParameterList thisItem;
16+
private ParameterList thisItem;
1717

1818
@Override
1919
public void createNew(ParserRuleContext ctx) {
@@ -25,59 +25,69 @@ public void createNew(ParserRuleContext ctx) {
2525
public void addParameter(Parameter_declContext aCtx,
2626
Stack<AstNodeBuilder> itemStack) {
2727
Parameter_declContext ctx = aCtx;
28-
Parameter_idContext parameter_id = ctx.parameter_id();
2928
ParameterBase param = AstNodeFactory.create(ctx);
3029

31-
String baseType = ParseTreeUtils
32-
.childTokenString(ctx.param_decl_specifiers());
33-
String completeType = determineCompleteType(parameter_id, baseType);
30+
String baseType = ParseTreeUtils.childTokenString(ctx.param_decl_specifiers());
31+
String completeType = ctx.parameter_id() != null ?
32+
determineCompleteType(ctx.parameter_id(), baseType) :
33+
determineCompleteAnonymousType(ctx, baseType);
3434

3535
((ParameterType) param.getType()).setBaseType(baseType);
3636
((ParameterType) param.getType()).setCompleteType(completeType);
3737

3838
thisItem.addChild(param);
3939
}
4040

41-
public String determineCompleteType(Parameter_idContext parameter_id,
42-
String baseType) {
43-
String retType = baseType;
41+
private String determineCompleteAnonymousType(Parameter_declContext ctx,
42+
String baseType) {
43+
StringBuilder retType = new StringBuilder(baseType);
4444

45-
// TODO: use a string-builder here and clean this up.
45+
if (ctx.parameter_ptrs() != null) {
46+
retType.append(" ");
47+
retType.append(ParseTreeUtils.childTokenString(ctx.parameter_ptrs()));
48+
}
49+
50+
return retType.toString();
51+
}
52+
53+
private String determineCompleteType(Parameter_idContext parameter_id,
54+
String baseType) {
55+
56+
StringBuilder retType = new StringBuilder(baseType);
4657

4758
// iterate until nesting level is reached
4859
// where type is given.
49-
5060
while (parameter_id.parameter_name() == null) {
61+
final StringBuilder newCompleteType = new StringBuilder();
5162

52-
String newCompleteType = "";
63+
newCompleteType.append("(");
5364

54-
newCompleteType += "(";
55-
56-
if (parameter_id.ptrs() != null) {
57-
newCompleteType += ParseTreeUtils
58-
.childTokenString(parameter_id.ptrs()) + " ";
65+
if (parameter_id.parameter_ptrs() != null) {
66+
newCompleteType.append(ParseTreeUtils.childTokenString(parameter_id.parameter_ptrs()));
67+
newCompleteType.append(" ");
5968
}
6069
if (parameter_id.type_suffix() != null) {
61-
newCompleteType += ParseTreeUtils
62-
.childTokenString(parameter_id.type_suffix()) + " ";
70+
newCompleteType.append(ParseTreeUtils.childTokenString(parameter_id.type_suffix()));
71+
newCompleteType.append(" ");
6372
}
6473

65-
newCompleteType += retType;
66-
newCompleteType += ")";
74+
newCompleteType.append(retType);
75+
newCompleteType.append(")");
6776
retType = newCompleteType;
6877
parameter_id = parameter_id.parameter_id();
6978
}
7079

71-
if (parameter_id.ptrs() != null) {
72-
retType += " "
73-
+ ParseTreeUtils.childTokenString(parameter_id.ptrs());
80+
if (parameter_id.parameter_ptrs() != null) {
81+
retType.append(" ");
82+
retType.append(ParseTreeUtils.childTokenString(parameter_id.parameter_ptrs()));
7483
}
84+
7585
if (parameter_id.type_suffix() != null) {
76-
retType += " " + ParseTreeUtils
77-
.childTokenString(parameter_id.type_suffix());
86+
retType.append(" ");
87+
retType.append(ParseTreeUtils.childTokenString(parameter_id.type_suffix()));
7888
}
7989

80-
return retType;
90+
return retType.toString();
8191
}
8292

8393
}

‎src/main/java/io/shiftleft/fuzzyc2cpg/parser/modules/CModuleParserTreeListener.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -112,11 +112,10 @@ public void enterParameter_decl(ModuleParser.Parameter_declContext ctx) {
112112

113113
@Override
114114
public void enterTemplate_decl(ModuleParser.Template_declContext ctx) {
115-
TemplateAstBuilder<?> builder = (TemplateAstBuilder<?>) p.builderStack.peek();
115+
TemplateAstBuilder<?> builder = (TemplateAstBuilder) p.builderStack.peek();
116116
builder.setTemplateList(ctx);
117117
}
118118

119-
120119
@Override
121120
public void enterTemplate_name(ModuleParser.Template_nameContext ctx) {
122121
TemplateAstBuilder<?> builder = (TemplateAstBuilder) p.builderStack.peek();

‎src/test/java/io/shiftleft/fuzzyc2cpg/antlrparsers/moduleparser/FunctionParameterTests.java‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,15 @@
11
package io.shiftleft.fuzzyc2cpg.antlrparsers.moduleparser;
22

3+
import static org.junit.Assert.assertEquals;
34
import static org.junit.Assert.assertTrue;
45

56
import io.shiftleft.fuzzyc2cpg.ModuleParser;
7+
8+
import org.antlr.v4.runtime.tree.ParseTree;
69
import org.junit.Test;
710

811

9-
public class FunctionParameterTests extends ModuleParserTest
10-
{
12+
public class FunctionParameterTests extends ModuleParserTest {
1113

1214
@Test
1315
public void testFunctionPtrParam()
@@ -17,8 +19,8 @@ public void testFunctionPtrParam()
1719
ModuleParser parser = createParser(input);
1820
String output = parser.function_def().toStringTree(parser);
1921

20-
assertTrue(output.startsWith(
21-
"(function_def (return_type (type_name (base_type int))) (function_name (identifier foo)) (function_param_list ( (parameter_decl_clause (parameter_decl (param_decl_specifiers (type_name (base_type char))) (parameter_id (ptrs (ptr_operator *)) ( (parameter_id (ptrs (ptr_operator *)) (parameter_name (identifier param))) ) (type_suffix (param_type_list ( void )))))) )) (compound_statement { }))"));
22+
assertEquals("(function_def (return_type (type_name (base_type int))) (function_name (identifier foo)) (function_param_list ( (parameter_decl_clause (parameter_decl (param_decl_specifiers (type_name (base_type char))) (parameter_id (parameter_ptrs (ptrs (ptr_operator *))) ( (parameter_id (parameter_ptrs (ptrs (ptr_operator *))) (parameter_name (identifier param))) ) (type_suffix (param_type_list ( void )))))) )) (compound_statement { }))",
23+
output);
2224
}
2325

2426
@Test
@@ -71,4 +73,12 @@ public void testConstConstPtr()
7173
assertTrue(output.startsWith("(function_def"));
7274
}
7375

76+
@Test
77+
public void testMoveParameters() {
78+
String input = "void foo(std::string&& s) {}";
79+
ModuleParser parser = createParser(input);
80+
String output = parser.function_def().toStringTree(parser);
81+
assertEquals("(function_def (return_type (type_name (base_type void))) (function_name (identifier foo)) (function_param_list ( (parameter_decl_clause (parameter_decl (param_decl_specifiers (type_name (base_type std) :: (base_type string))) (parameter_id (parameter_ptrs (rvalue_ref &&)) (parameter_name (identifier s))))) )) (compound_statement { }))",
82+
output);
83+
}
7484
}

0 commit comments

Comments
 (0)