Skip to content

Commit 5ba1c9d

Browse files
firewavefirewave
authored
testrunner: more Tokenizer related refactoring (danmar#7440)
1 parent 9ae8547 commit 5ba1c9d

File tree

10 files changed

+94
-94
lines changed

10 files changed

+94
-94
lines changed

‎Makefile‎

Lines changed: 45 additions & 45 deletions
Large diffs are not rendered by default.

‎lib/tokenlist.cpp‎

Lines changed: 3 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,8 @@ void TokenList::determineCppC()
114114

115115
int TokenList::appendFileIfNew(std::string fileName)
116116
{
117+
ASSERT_LANG(!fileName.empty());
118+
117119
// Has this file been tokenized already?
118120
auto it = std::find_if(mFiles.cbegin(), mFiles.cend(), [&](const std::string& f) {
119121
return Path::sameFileName(f, fileName);
@@ -340,19 +342,6 @@ void TokenList::insertTokens(Token *dest, const Token *src, nonneg int n)
340342
}
341343
}
342344

343-
//---------------------------------------------------------------------------
344-
// Tokenize - tokenizes a given file.
345-
//---------------------------------------------------------------------------
346-
347-
bool TokenList::createTokens(std::istream &code, const std::string& file0)
348-
{
349-
ASSERT_LANG(!file0.empty());
350-
351-
appendFileIfNew(file0);
352-
353-
return createTokensInternal(code, file0);
354-
}
355-
356345
//---------------------------------------------------------------------------
357346

358347
bool TokenList::createTokens(std::istream &code, Standards::Language lang)
@@ -364,7 +353,7 @@ bool TokenList::createTokens(std::istream &code, Standards::Language lang)
364353
ASSERT_LANG(lang == mLang);
365354
}
366355

367-
return createTokensInternal(code, "");
356+
return createTokensInternal(code, mFiles.empty() ? "" : *mFiles.cbegin());
368357
}
369358

370359
//---------------------------------------------------------------------------

‎lib/tokenlist.h‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -103,9 +103,8 @@ class CPPCHECKLIB TokenList {
103103
* - UTF in the code are not handled.
104104
* - comments are not handled.
105105
* @param code input stream for code
106-
* @param file0 source file name
106+
* @param lang the language of the code
107107
*/
108-
bool createTokens(std::istream &code, const std::string& file0);
109108
bool createTokens(std::istream &code, Standards::Language lang);
110109

111110
void createTokens(simplecpp::TokenList&& tokenList);

‎test/helpers.h‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#define helpersH
2121

2222
#include "library.h"
23+
#include "path.h"
2324
#include "settings.h"
2425
#include "standards.h"
2526
#include "tokenize.h"
@@ -111,7 +112,10 @@ class SimpleTokenizer : public Tokenizer {
111112
const std::string& filename,
112113
const std::string &configuration = "")
113114
{
114-
if (!list.createTokens(istr, filename))
115+
if (list.front())
116+
throw std::runtime_error("token list is not empty");
117+
list.appendFileIfNew(filename);
118+
if (!list.createTokens(istr, Path::identify(filename, false)))
115119
return false;
116120

117121
return simplifyTokens1(configuration);
@@ -278,4 +282,15 @@ class SimpleTokenizer2 : public Tokenizer {
278282
std::vector<std::string> mFiles;
279283
};
280284

285+
struct TokenListHelper
286+
{
287+
static bool createTokens(TokenList& tokenlist, std::istream& istr, const std::string& file)
288+
{
289+
if (tokenlist.front())
290+
throw std::runtime_error("token list is not empty");
291+
tokenlist.appendFileIfNew(file);
292+
return tokenlist.createTokens(istr, Path::identify(file, false));
293+
}
294+
};
295+
281296
#endif // helpersH

‎test/testsimplifytemplate.cpp‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
#include "errortypes.h"
2020
#include "fixture.h"
2121
#include "helpers.h"
22+
#include "path.h"
2223
#include "settings.h"
2324
#include "templatesimplifier.h"
2425
#include "token.h"
@@ -5429,7 +5430,8 @@ class TestSimplifyTemplate : public TestFixture {
54295430
Tokenizer tokenizer(settings, *this);
54305431

54315432
std::istringstream istr(code);
5432-
if (!tokenizer.list.createTokens(istr, "test.cpp"))
5433+
tokenizer.list.appendFileIfNew("test.cpp");
5434+
if (!tokenizer.list.createTokens(istr, Path::identify("test.cpp", false)))
54335435
return false;
54345436
tokenizer.createLinks();
54355437
tokenizer.splitTemplateRightAngleBrackets(false);
@@ -5497,7 +5499,8 @@ class TestSimplifyTemplate : public TestFixture {
54975499
Tokenizer tokenizer(settings, *this);
54985500

54995501
std::istringstream istr(code);
5500-
if (!tokenizer.list.createTokens(istr, "test.cpp"))
5502+
tokenizer.list.appendFileIfNew("test.cpp");
5503+
if (!tokenizer.list.createTokens(istr, Path::identify("test.cpp", false)))
55015504
return false;
55025505
tokenizer.createLinks();
55035506
tokenizer.splitTemplateRightAngleBrackets(false);
@@ -5568,7 +5571,7 @@ class TestSimplifyTemplate : public TestFixture {
55685571
Tokenizer tokenizer(settings, *this);
55695572

55705573
std::istringstream istr(code);
5571-
if (!tokenizer.list.createTokens(istr, "test.cpp"))
5574+
if (!TokenListHelper::createTokens(tokenizer.list, istr, "test.cpp"))
55725575
return false;
55735576
tokenizer.createLinks();
55745577
tokenizer.splitTemplateRightAngleBrackets(false);
@@ -5598,7 +5601,7 @@ class TestSimplifyTemplate : public TestFixture {
55985601
Tokenizer tokenizer(settings, *this);
55995602

56005603
std::istringstream istr(code);
5601-
if (!tokenizer.list.createTokens(istr, "test.cpp"))
5604+
if (!TokenListHelper::createTokens(tokenizer.list, istr, "test.cpp"))
56025605
return false;
56035606
tokenizer.createLinks();
56045607
tokenizer.splitTemplateRightAngleBrackets(false);

‎test/testsimplifytypedef.cpp‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@
2828
#include <cstddef>
2929
#include <sstream>
3030
#include <string>
31-
#include <vector>
3231

3332
class TestSimplifyTypedef : public TestFixture {
3433
public:
@@ -310,7 +309,7 @@ class TestSimplifyTypedef : public TestFixture {
310309
Tokenizer tokenizer(settings1, *this);
311310

312311
std::istringstream istr(code);
313-
if (!tokenizer.list.createTokens(istr, "file.c"))
312+
if (!TokenListHelper::createTokens(tokenizer.list, istr, "file.c"))
314313
return "";
315314
tokenizer.createLinks();
316315
tokenizer.simplifyTypedef();
@@ -326,7 +325,7 @@ class TestSimplifyTypedef : public TestFixture {
326325
Tokenizer tokenizer(settings1, *this);
327326

328327
std::istringstream istr(code);
329-
if (!tokenizer.list.createTokens(istr, "file.c"))
328+
if (!TokenListHelper::createTokens(tokenizer.list, istr, "file.c"))
330329
return {};
331330
tokenizer.createLinks();
332331
tokenizer.simplifyTypedef();
@@ -4454,7 +4453,7 @@ class TestSimplifyTypedef : public TestFixture {
44544453

44554454
Tokenizer tokenizer(settings1, *this);
44564455
std::istringstream istr(code);
4457-
ASSERT(tokenizer.list.createTokens(istr, "file.c"));
4456+
ASSERT(TokenListHelper::createTokens(tokenizer.list, istr, "file.c"));
44584457
tokenizer.createLinks();
44594458
tokenizer.simplifyTypedef();
44604459

@@ -4496,7 +4495,7 @@ class TestSimplifyTypedef : public TestFixture {
44964495

44974496
Tokenizer tokenizer(settings1, *this);
44984497
std::istringstream istr(code);
4499-
ASSERT(tokenizer.list.createTokens(istr, "file.c"));
4498+
ASSERT(TokenListHelper::createTokens(tokenizer.list, istr, "file.c"));
45004499
tokenizer.createLinks();
45014500
tokenizer.simplifyTypedef();
45024501

@@ -4514,7 +4513,7 @@ class TestSimplifyTypedef : public TestFixture {
45144513

45154514
Tokenizer tokenizer(settings1, *this);
45164515
std::istringstream istr(code);
4517-
ASSERT(tokenizer.list.createTokens(istr, "file.c"));
4516+
ASSERT(TokenListHelper::createTokens(tokenizer.list, istr, "file.c"));
45184517
tokenizer.createLinks();
45194518
tokenizer.simplifyTypedef();
45204519

‎test/testsimplifyusing.cpp‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,10 @@
2222
#include "platform.h"
2323
#include "settings.h"
2424
#include "token.h"
25-
#include "tokenlist.h"
2625
#include "utils.h"
2726

2827
#include <cstddef>
29-
#include <sstream>
3028
#include <string>
31-
#include <vector>
3229

3330
class TestSimplifyUsing : public TestFixture {
3431
public:
@@ -115,8 +112,6 @@ class TestSimplifyUsing : public TestFixture {
115112
if (options.preprocess) {
116113
SimpleTokenizer2 tokenizer(settings, *this, code, "test.cpp");
117114

118-
std::istringstream istr(code);
119-
ASSERT_LOC(tokenizer.list.createTokens(istr, "test.cpp"), file, line); // TODO: this creates the tokens a second time
120115
ASSERT_LOC(tokenizer.simplifyTokens1(""), file, line);
121116
return tokenizer.tokens()->stringifyList(nullptr);
122117
}
@@ -1588,7 +1583,7 @@ class TestSimplifyUsing : public TestFixture {
15881583
"STAMP(B, A);\n"
15891584
"STAMP(C, B);\n";
15901585
(void)tok(code, dinit(TokOptions, $.preprocess = true));
1591-
ASSERT(startsWith(errout_str(), "[test.cpp:6]: (debug) Failed to parse 'using C = S < S < S < int"));
1586+
TODO_ASSERT(startsWith(errout_str(), "[test.cpp:6]: (debug) Failed to parse 'using C = S < S < S < int"));
15921587
}
15931588

15941589
void scopeInfo1() {

‎test/testtokenize.cpp‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
#include "errortypes.h"
2020
#include "fixture.h"
2121
#include "helpers.h"
22+
#include "path.h"
2223
#include "platform.h"
2324
#include "preprocessor.h" // usually tests here should not use preprocessor...
2425
#include "settings.h"
@@ -867,7 +868,8 @@ class TestTokenizer : public TestFixture {
867868
Tokenizer tokenizer(settings1, *this);
868869
const char code[] = "void foo(int i) { reinterpret_cast<char>(i) };";
869870
std::istringstream istr(code);
870-
ASSERT(tokenizer.list.createTokens(istr, "test.h"));
871+
tokenizer.list.appendFileIfNew("test.h");
872+
ASSERT(tokenizer.list.createTokens(istr, Path::identify("test.h", false)));
871873
ASSERT_THROW_INTERNAL(tokenizer.simplifyTokens1(""), SYNTAX);
872874
}
873875
}
@@ -6135,7 +6137,8 @@ class TestTokenizer : public TestFixture {
61356137
// tokenize given code..
61366138
Tokenizer tokenizer(settings0, *this);
61376139
std::istringstream istr(code);
6138-
if (!tokenizer.list.createTokens(istr,"test.cpp"))
6140+
tokenizer.list.appendFileIfNew("test.cpp");
6141+
if (!tokenizer.list.createTokens(istr,Path::identify("test.cpp", false)))
61396142
return "ERROR";
61406143

61416144
tokenizer.combineStringAndCharLiterals();

‎test/testtokenlist.cpp‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
#include "settings.h"
2020
#include "fixture.h"
2121
#include "helpers.h"
22+
#include "path.h"
2223
#include "platform.h"
2324
#include "preprocessor.h"
2425
#include "standards.h"
@@ -129,7 +130,8 @@ class TestTokenList : public TestFixture {
129130
const Settings s = settingsBuilder().c(Standards::C89).build();
130131
TokenList tokenlist(&s);
131132
std::istringstream istr(code2);
132-
ASSERT(tokenlist.createTokens(istr, "a.c"));
133+
tokenlist.appendFileIfNew("a.c");
134+
ASSERT(tokenlist.createTokens(istr, Path::identify("a.c", false)));
133135
ASSERT_EQUALS(false, tokenlist.front()->isKeyword());
134136
}
135137

@@ -150,7 +152,8 @@ class TestTokenList : public TestFixture {
150152
const Settings s = settingsBuilder().cpp(Standards::CPP03).build();
151153
TokenList tokenlist(&s);
152154
std::istringstream istr(code2);
153-
ASSERT(tokenlist.createTokens(istr, "a.cpp"));
155+
tokenlist.appendFileIfNew("a.cpp");
156+
ASSERT(tokenlist.createTokens(istr, Path::identify("a.cpp", false)));
154157
ASSERT_EQUALS(false, tokenlist.front()->isKeyword());
155158
}
156159
}

‎test/testunusedvar.cpp‎

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,9 @@
2222
#include "helpers.h"
2323
#include "preprocessor.h"
2424
#include "settings.h"
25-
#include "standards.h"
2625

2726
#include <list>
2827
#include <string>
29-
#include <vector>
3028

3129
class TestUnusedVar : public TestFixture {
3230
public:
@@ -281,21 +279,19 @@ class TestUnusedVar : public TestFixture {
281279
{
282280
CheckStructMemberUsageOptions() = default;
283281
const std::list<Directive>* directives = nullptr;
284-
const Settings *s = nullptr;
282+
bool cpp = true;
285283
};
286284

287285
#define checkStructMemberUsage(...) checkStructMemberUsage_(__FILE__, __LINE__, __VA_ARGS__)
288286
void checkStructMemberUsage_(const char* file, int line, const char code[], const CheckStructMemberUsageOptions& options = make_default_obj()) {
289-
const Settings *settings1 = options.s ? options.s : &settings;
290-
291287
// Tokenize..
292-
SimpleTokenizer tokenizer(*settings1, *this);
288+
SimpleTokenizer tokenizer(settings, *this);
293289
if (options.directives)
294290
tokenizer.setDirectives(*options.directives);
295-
ASSERT_LOC(tokenizer.tokenize(code), file, line);
291+
ASSERT_LOC(tokenizer.tokenize(code, options.cpp), file, line);
296292

297293
// Check for unused variables..
298-
CheckUnusedVar checkUnusedVar(&tokenizer, settings1, this);
294+
CheckUnusedVar checkUnusedVar(&tokenizer, &settings, this);
299295
(checkUnusedVar.checkStructMemberUsage)();
300296
}
301297

@@ -1831,8 +1827,6 @@ class TestUnusedVar : public TestFixture {
18311827
ASSERT_EQUALS("[test.cpp:1]: (style) struct member 'A::i' is never used.\n",
18321828
errout_str());
18331829

1834-
/*const*/ Settings s = settings;
1835-
s.enforcedLang = Standards::Language::C;
18361830
checkStructMemberUsage("struct A {\n" // #10852
18371831
" struct B {\n"
18381832
" int x;\n"
@@ -1841,7 +1835,7 @@ class TestUnusedVar : public TestFixture {
18411835
"void f() {\n"
18421836
" struct B* pb = &a.b;\n"
18431837
" pb->x = 1;\n"
1844-
"}\n", dinit(CheckStructMemberUsageOptions, $.s = &s));
1838+
"}\n", dinit(CheckStructMemberUsageOptions, $.cpp = false));
18451839
ASSERT_EQUALS("", errout_str());
18461840

18471841
checkStructMemberUsage("union U {\n"
@@ -1859,7 +1853,7 @@ class TestUnusedVar : public TestFixture {
18591853
" pb->x = 1;\n"
18601854
" struct C* pc = &u.c;\n"
18611855
" pc->s[0] = 1;\n"
1862-
"}\n", dinit(CheckStructMemberUsageOptions, $.s = &s));
1856+
"}\n", dinit(CheckStructMemberUsageOptions, $.cpp = false));
18631857
ASSERT_EQUALS("", errout_str());
18641858
}
18651859

0 commit comments

Comments
 (0)