Skip to content

Commit 20681a3

Browse files
firewavefirewave
authored
testrunner: some SimpleTokenizer refactoring (danmar#7467)
1 parent 3b293b4 commit 20681a3

24 files changed

+178
-168
lines changed

‎test/helpers.h‎

Lines changed: 15 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -44,81 +44,55 @@ namespace tinyxml2 {
4444
// TODO: make Tokenizer private
4545
class SimpleTokenizer : public Tokenizer {
4646
public:
47-
template<size_t size>
48-
SimpleTokenizer(ErrorLogger& errorlogger, const char (&code)[size], bool cpp = true)
47+
explicit SimpleTokenizer(ErrorLogger& errorlogger, bool cpp = true)
4948
: Tokenizer{s_settings, errorlogger}
5049
{
51-
if (!tokenize(code, cpp))
52-
throw std::runtime_error("creating tokens failed");
50+
list.setLang(cpp ? Standards::Language::CPP : Standards::Language::C, true);
5351
}
5452

55-
SimpleTokenizer(const Settings& settings, ErrorLogger& errorlogger)
53+
SimpleTokenizer(const Settings& settings, ErrorLogger& errorlogger, bool cpp = true)
5654
: Tokenizer{settings, errorlogger}
57-
{}
58-
59-
/*
60-
Token* tokens() {
61-
return Tokenizer::tokens();
62-
}
63-
64-
const Token* tokens() const {
65-
return Tokenizer::tokens();
66-
}
67-
*/
68-
69-
template<size_t size>
70-
bool tokenize(const char (&code)[size],
71-
const std::string& filename,
72-
const std::string &configuration = "")
7355
{
74-
std::istringstream istr(code);
75-
return tokenize(istr, filename, configuration);
56+
list.setLang(cpp ? Standards::Language::CPP : Standards::Language::C, true);
7657
}
7758

78-
template<size_t size>
79-
bool tokenize(const char (&code)[size],
80-
bool cpp = true,
81-
const std::string &configuration = "")
59+
SimpleTokenizer(const Settings& settings, ErrorLogger& errorlogger, const std::string& filename)
60+
: Tokenizer{settings, errorlogger}
8261
{
83-
std::istringstream istr(code);
84-
return tokenize(istr, std::string(cpp ? "test.cpp" : "test.c"), configuration);
62+
list.setLang(Path::identify(filename, false));
63+
list.appendFileIfNew(filename);
8564
}
8665

87-
bool tokenize(const std::string& code,
88-
const std::string& filename,
89-
const std::string &configuration = "")
66+
template<size_t size>
67+
bool tokenize(const char (&code)[size])
9068
{
9169
std::istringstream istr(code);
92-
return tokenize(istr, filename, configuration);
70+
return tokenize(istr, std::string(list.isCPP() ? "test.cpp" : "test.c"));
9371
}
9472

95-
bool tokenize(const std::string& code,
96-
bool cpp = true,
97-
const std::string &configuration = "")
73+
bool tokenize(const std::string& code)
9874
{
9975
std::istringstream istr(code);
100-
return tokenize(istr, std::string(cpp ? "test.cpp" : "test.c"), configuration);
76+
return tokenize(istr, std::string(list.isCPP() ? "test.cpp" : "test.c"));
10177
}
10278

10379
private:
10480
/**
10581
* Tokenize code
10682
* @param istr The code as stream
10783
* @param filename Indicates if the code is C++
108-
* @param configuration E.g. "A" for code where "#ifdef A" is true
10984
* @return false if source code contains syntax errors
11085
*/
11186
bool tokenize(std::istream& istr,
112-
const std::string& filename,
113-
const std::string &configuration = "")
87+
const std::string& filename)
11488
{
11589
if (list.front())
11690
throw std::runtime_error("token list is not empty");
11791
list.appendFileIfNew(filename);
11892
if (!list.createTokens(istr, Path::identify(filename, false)))
11993
return false;
12094

121-
return simplifyTokens1(configuration);
95+
return simplifyTokens1("");
12296
}
12397

12498
// TODO: find a better solution

‎test/test64bit.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,8 @@ class Test64BitPortability : public TestFixture {
4646
template<size_t size>
4747
void check_(const char* file, int line, const char (&code)[size], bool cpp = true) {
4848
// Tokenize..
49-
SimpleTokenizer tokenizer(settings, *this);
50-
ASSERT_LOC(tokenizer.tokenize(code, cpp), file, line);
49+
SimpleTokenizer tokenizer(settings, *this, cpp);
50+
ASSERT_LOC(tokenizer.tokenize(code), file, line);
5151

5252
// Check char variable usage..
5353
Check64BitPortability check64BitPortability(&tokenizer, &settings, this);

‎test/testastutils.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -174,8 +174,8 @@ class TestAstUtils : public TestFixture {
174174
#define isSameExpression(...) isSameExpression_(__FILE__, __LINE__, __VA_ARGS__)
175175
template<size_t size>
176176
bool isSameExpression_(const char* file, int line, const char (&code)[size], const char tokStr1[], const char tokStr2[], bool cpp) {
177-
SimpleTokenizer tokenizer(settingsDefault, *this);
178-
ASSERT_LOC(tokenizer.tokenize(code, cpp), file, line);
177+
SimpleTokenizer tokenizer(settingsDefault, *this, cpp);
178+
ASSERT_LOC(tokenizer.tokenize(code), file, line);
179179
const Token * const tok1 = Token::findsimplematch(tokenizer.tokens(), tokStr1, strlen(tokStr1));
180180
const Token * const tok2 = Token::findsimplematch(tok1->next(), tokStr2, strlen(tokStr2));
181181
return (isSameExpression)(false, tok1, tok2, settingsDefault, false, true);

‎test/testautovariables.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,8 +44,8 @@ class TestAutoVariables : public TestFixture {
4444
const Settings settings1 = settingsBuilder(settings).certainty(Certainty::inconclusive, options.inconclusive).build();
4545

4646
// Tokenize..
47-
SimpleTokenizer tokenizer(settings1, *this);
48-
ASSERT_LOC(tokenizer.tokenize(code, options.cpp), file, line);
47+
SimpleTokenizer tokenizer(settings1, *this, options.cpp);
48+
ASSERT_LOC(tokenizer.tokenize(code), file, line);
4949

5050
runChecks<CheckAutoVariables>(tokenizer, this);
5151
}

‎test/testbool.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -86,8 +86,8 @@ class TestBool : public TestFixture {
8686
template<size_t size>
8787
void check_(const char* file, int line, const char (&code)[size], const CheckOptions& options = make_default_obj()) {
8888
// Tokenize..
89-
SimpleTokenizer tokenizer(settings, *this);
90-
ASSERT_LOC(tokenizer.tokenize(code, options.cpp), file, line);
89+
SimpleTokenizer tokenizer(settings, *this, options.cpp);
90+
ASSERT_LOC(tokenizer.tokenize(code), file, line);
9191

9292
// Check...
9393
runChecks<CheckBool>(tokenizer, this);

‎test/testbufferoverrun.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,8 @@ class TestBufferOverrun : public TestFixture {
5151
const Settings settings = options.s ? *options.s : settingsBuilder(settings0).certainty(Certainty::inconclusive).build();
5252

5353
// Tokenize..
54-
SimpleTokenizer tokenizer(settings, *this);
55-
ASSERT_LOC(tokenizer.tokenize(code, options.cpp), file, line);
54+
SimpleTokenizer tokenizer(settings, *this, options.cpp);
55+
ASSERT_LOC(tokenizer.tokenize(code), file, line);
5656

5757
// Check for buffer overruns..
5858
runChecks<CheckBufferOverrun>(tokenizer, this);

‎test/testclass.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9081,9 +9081,9 @@ class TestClass : public TestFixture {
90819081
// getFileInfo
90829082
std::list<Check::FileInfo*> fileInfo;
90839083
for (const std::string& c: code) {
9084-
SimpleTokenizer tokenizer{settingsDefault, *this};
90859084
const std::string filename = std::to_string(fileInfo.size()) + ".cpp";
9086-
ASSERT(tokenizer.tokenize(c, filename));
9085+
SimpleTokenizer tokenizer{settingsDefault, *this, filename};
9086+
ASSERT(tokenizer.tokenize(c));
90879087
fileInfo.push_back(check.getFileInfo(tokenizer, settingsDefault));
90889088
}
90899089

‎test/testfunctions.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -126,8 +126,8 @@ class TestFunctions : public TestFixture {
126126
const Settings& s = options.s ? *options.s : settings;
127127

128128
// Tokenize..
129-
SimpleTokenizer tokenizer(s, *this);
130-
ASSERT_LOC(tokenizer.tokenize(code, options.cpp), file, line);
129+
SimpleTokenizer tokenizer(s, *this, options.cpp);
130+
ASSERT_LOC(tokenizer.tokenize(code), file, line);
131131

132132
runChecks<CheckFunctions>(tokenizer, this);
133133
}

‎test/testgarbage.cpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -288,8 +288,8 @@ class TestGarbage : public TestFixture {
288288
template<size_t size>
289289
std::string checkCodeInternal_(const char* file, int line, const char (&code)[size], bool cpp) {
290290
// tokenize..
291-
SimpleTokenizer tokenizer(settings, *this);
292-
ASSERT_LOC(tokenizer.tokenize(code, cpp), file, line);
291+
SimpleTokenizer tokenizer(settings, *this, cpp);
292+
ASSERT_LOC(tokenizer.tokenize(code), file, line);
293293

294294
// call all "runChecks" in all registered Check classes
295295
for (auto it = Check::instances().cbegin(); it != Check::instances().cend(); ++it) {
@@ -396,8 +396,8 @@ class TestGarbage : public TestFixture {
396396
const char code[] = "class x y { };";
397397

398398
{
399-
SimpleTokenizer tokenizer(settings, *this);
400-
ASSERT(tokenizer.tokenize(code, false));
399+
SimpleTokenizer tokenizer(settings, *this, false);
400+
ASSERT(tokenizer.tokenize(code));
401401
ASSERT_EQUALS("", errout_str());
402402
}
403403
{

‎test/testio.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -102,8 +102,8 @@ class TestIO : public TestFixture {
102102
settings1.platform.defaultSign = options.defaultSign;
103103

104104
// Tokenize..
105-
SimpleTokenizer tokenizer(settings1, *this);
106-
ASSERT_LOC(tokenizer.tokenize(code, options.cpp), file, line);
105+
SimpleTokenizer tokenizer(settings1, *this, options.cpp);
106+
ASSERT_LOC(tokenizer.tokenize(code), file, line);
107107

108108
// Check..
109109
if (options.onlyFormatStr) {

0 commit comments

Comments
 (0)