Skip to content

Commit 5ec627f

Browse files
firewavefirewave
authored
store reference of associated tokenlist in Preprocessor (danmar#7903)
1 parent 5fff5da commit 5ec627f

File tree

9 files changed

+118
-115
lines changed

9 files changed

+118
-115
lines changed

‎lib/cppcheck.cpp‎

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -847,7 +847,7 @@ unsigned int CppCheck::check(const FileSettings &fs)
847847
return returnValue;
848848
}
849849

850-
std::size_t CppCheck::calculateHash(const Preprocessor& preprocessor, const simplecpp::TokenList& tokens, const std::string& filePath) const
850+
std::size_t CppCheck::calculateHash(const Preprocessor& preprocessor, const std::string& filePath) const
851851
{
852852
std::ostringstream toolinfo;
853853
toolinfo << (mSettings.cppcheckCfgProductName.empty() ? CPPCHECK_VERSION_STRING : mSettings.cppcheckCfgProductName);
@@ -865,7 +865,7 @@ std::size_t CppCheck::calculateHash(const Preprocessor& preprocessor, const simp
865865
toolinfo << mSettings.premiumArgs;
866866
// TODO: do we need to add more options?
867867
mSuppressions.nomsg.dump(toolinfo, filePath);
868-
return preprocessor.calculateHash(tokens, toolinfo.str());
868+
return preprocessor.calculateHash(toolinfo.str());
869869
}
870870

871871
unsigned int CppCheck::checkBuffer(const FileWithDetails &file, const std::string &cfgname, int fileIndex, const uint8_t* data, std::size_t size)
@@ -938,8 +938,8 @@ unsigned int CppCheck::checkInternal(const FileWithDetails& file, const std::str
938938
std::vector<std::string> files;
939939
simplecpp::TokenList tokens = createTokenList(files, nullptr);
940940
if (analyzerInformation) {
941-
const Preprocessor preprocessor(mSettings, mErrorLogger, file.lang());
942-
hash = calculateHash(preprocessor, tokens);
941+
const Preprocessor preprocessor(tokens, mSettings, mErrorLogger, file.lang());
942+
hash = calculateHash(preprocessor);
943943
}
944944
tokenlist.createTokens(std::move(tokens));
945945
// this is not a real source file - we just want to tokenize it. treat it as C anyways as the language needs to be determined.
@@ -984,9 +984,9 @@ unsigned int CppCheck::checkInternal(const FileWithDetails& file, const std::str
984984
return mLogger->exitcode();
985985
}
986986

987-
Preprocessor preprocessor(mSettings, mErrorLogger, file.lang());
987+
Preprocessor preprocessor(tokens1, mSettings, mErrorLogger, file.lang());
988988

989-
if (!preprocessor.loadFiles(tokens1, files))
989+
if (!preprocessor.loadFiles(files))
990990
return mLogger->exitcode();
991991

992992
if (!mSettings.plistOutput.empty()) {
@@ -1006,14 +1006,14 @@ unsigned int CppCheck::checkInternal(const FileWithDetails& file, const std::str
10061006
}
10071007

10081008
// Parse comments and then remove them
1009-
mLogger->setRemarkComments(preprocessor.getRemarkComments(tokens1));
1010-
preprocessor.inlineSuppressions(tokens1, mSuppressions.nomsg);
1009+
mLogger->setRemarkComments(preprocessor.getRemarkComments());
1010+
preprocessor.inlineSuppressions(mSuppressions.nomsg);
10111011
if (mSettings.dump || !mSettings.addons.empty()) {
10121012
std::ostringstream oss;
10131013
mSuppressions.nomsg.dump(oss);
10141014
dumpProlog += oss.str();
10151015
}
1016-
preprocessor.removeComments(tokens1);
1016+
preprocessor.removeComments();
10171017

10181018
if (!mSettings.buildDir.empty()) {
10191019
analyzerInformation.reset(new AnalyzerInformation);
@@ -1022,7 +1022,7 @@ unsigned int CppCheck::checkInternal(const FileWithDetails& file, const std::str
10221022

10231023
if (analyzerInformation) {
10241024
// Calculate hash so it can be compared with old hash / future hashes
1025-
const std::size_t hash = calculateHash(preprocessor, tokens1, file.spath());
1025+
const std::size_t hash = calculateHash(preprocessor, file.spath());
10261026
std::list<ErrorMessage> errors;
10271027
if (!analyzerInformation->analyzeFile(mSettings.buildDir, file.spath(), cfgname, fileIndex, hash, errors)) {
10281028
while (!errors.empty()) {
@@ -1035,24 +1035,24 @@ unsigned int CppCheck::checkInternal(const FileWithDetails& file, const std::str
10351035
}
10361036

10371037
// Get directives
1038-
std::list<Directive> directives = preprocessor.createDirectives(tokens1);
1039-
preprocessor.simplifyPragmaAsm(tokens1);
1038+
std::list<Directive> directives = preprocessor.createDirectives();
1039+
preprocessor.simplifyPragmaAsm();
10401040

1041-
Preprocessor::setPlatformInfo(tokens1, mSettings);
1041+
preprocessor.setPlatformInfo();
10421042

10431043
// Get configurations..
10441044
std::set<std::string> configurations;
10451045
if ((mSettings.checkAllConfigurations && mSettings.userDefines.empty()) || mSettings.force) {
10461046
Timer::run("Preprocessor::getConfigs", mSettings.showtime, &s_timerResults, [&]() {
1047-
configurations = preprocessor.getConfigs(tokens1);
1047+
configurations = preprocessor.getConfigs();
10481048
});
10491049
} else {
10501050
configurations.insert(mSettings.userDefines);
10511051
}
10521052

10531053
if (mSettings.checkConfiguration) {
10541054
for (const std::string &config : configurations)
1055-
(void)preprocessor.getcode(tokens1, config, files, false);
1055+
(void)preprocessor.getcode(config, files, false);
10561056

10571057
if (analyzerInformation)
10581058
mLogger->setAnalyzerInfo(nullptr);
@@ -1125,7 +1125,7 @@ unsigned int CppCheck::checkInternal(const FileWithDetails& file, const std::str
11251125
if (mSettings.preprocessOnly) {
11261126
std::string codeWithoutCfg;
11271127
Timer::run("Preprocessor::getcode", mSettings.showtime, &s_timerResults, [&]() {
1128-
codeWithoutCfg = preprocessor.getcode(tokens1, currentConfig, files, true);
1128+
codeWithoutCfg = preprocessor.getcode(currentConfig, files, true);
11291129
});
11301130

11311131
if (startsWith(codeWithoutCfg,"#file"))
@@ -1148,7 +1148,7 @@ unsigned int CppCheck::checkInternal(const FileWithDetails& file, const std::str
11481148

11491149
// Create tokens, skip rest of iteration if failed
11501150
Timer::run("Tokenizer::createTokens", mSettings.showtime, &s_timerResults, [&]() {
1151-
simplecpp::TokenList tokensP = preprocessor.preprocess(tokens1, currentConfig, files, true);
1151+
simplecpp::TokenList tokensP = preprocessor.preprocess(currentConfig, files, true);
11521152
tokenlist.createTokens(std::move(tokensP));
11531153
});
11541154
hasValidConfig = true;

‎lib/cppcheck.h‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -172,10 +172,9 @@ class CPPCHECKLIB CppCheck {
172172
* @brief Calculate hash used to detect when a file needs to be reanalyzed.
173173
*
174174
* @param preprocessor Preprocessor used to calculate the hash.
175-
* @param tokens Token list from preprocessed file.
176175
* @return hash
177176
*/
178-
std::size_t calculateHash(const Preprocessor &preprocessor, const simplecpp::TokenList &tokens, const std::string& filePath = {}) const;
177+
std::size_t calculateHash(const Preprocessor &preprocessor, const std::string& filePath = {}) const;
179178

180179
/**
181180
* @brief Check a file

‎lib/preprocessor.cpp‎

Lines changed: 44 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,9 @@ Directive::DirectiveToken::DirectiveToken(const simplecpp::Token & _tok) :
6565

6666
char Preprocessor::macroChar = char(1);
6767

68-
Preprocessor::Preprocessor(const Settings& settings, ErrorLogger &errorLogger, Standards::Language lang)
69-
: mSettings(settings)
68+
Preprocessor::Preprocessor(simplecpp::TokenList& tokens, const Settings& settings, ErrorLogger &errorLogger, Standards::Language lang)
69+
: mTokens(tokens)
70+
, mSettings(settings)
7071
, mErrorLogger(errorLogger)
7172
, mLang(lang)
7273
{
@@ -301,12 +302,12 @@ static void addInlineSuppressions(const simplecpp::TokenList &tokens, const Sett
301302
bad.emplace_back(suppr.fileName, suppr.lineNumber, "Suppress Begin: No matching end");
302303
}
303304

304-
void Preprocessor::inlineSuppressions(const simplecpp::TokenList &tokens, SuppressionList &suppressions)
305+
void Preprocessor::inlineSuppressions(SuppressionList &suppressions)
305306
{
306307
if (!mSettings.inlineSuppressions)
307308
return;
308309
std::list<BadInlineSuppression> err;
309-
::addInlineSuppressions(tokens, mSettings, suppressions, err);
310+
::addInlineSuppressions(mTokens, mSettings, suppressions, err);
310311
for (const auto &filedata : mFileCache) {
311312
::addInlineSuppressions(filedata->tokens, mSettings, suppressions, err);
312313
}
@@ -315,24 +316,24 @@ void Preprocessor::inlineSuppressions(const simplecpp::TokenList &tokens, Suppre
315316
}
316317
}
317318

318-
std::vector<RemarkComment> Preprocessor::getRemarkComments(const simplecpp::TokenList &tokens) const
319+
std::vector<RemarkComment> Preprocessor::getRemarkComments() const
319320
{
320321
std::vector<RemarkComment> ret;
321-
addRemarkComments(tokens, ret);
322+
addRemarkComments(mTokens, ret);
322323
for (const auto &filedata : mFileCache) {
323324
addRemarkComments(filedata->tokens, ret);
324325
}
325326
return ret;
326327
}
327328

328-
std::list<Directive> Preprocessor::createDirectives(const simplecpp::TokenList &tokens) const
329+
std::list<Directive> Preprocessor::createDirectives() const
329330
{
330331
// directive list..
331332
std::list<Directive> directives;
332333

333334
std::vector<const simplecpp::TokenList *> list;
334335
list.reserve(1U + mFileCache.size());
335-
list.push_back(&tokens);
336+
list.push_back(&mTokens);
336337
std::transform(mFileCache.cbegin(), mFileCache.cend(), std::back_inserter(list),
337338
[](const std::unique_ptr<simplecpp::FileData> &filedata) {
338339
return &filedata->tokens;
@@ -656,15 +657,15 @@ static void getConfigs(const simplecpp::TokenList &tokens, std::set<std::string>
656657
}
657658

658659

659-
std::set<std::string> Preprocessor::getConfigs(const simplecpp::TokenList &tokens) const
660+
std::set<std::string> Preprocessor::getConfigs() const
660661
{
661662
std::set<std::string> ret = { "" };
662-
if (!tokens.cfront())
663+
if (!mTokens.cfront())
663664
return ret;
664665

665666
std::set<std::string> defined = { "__cplusplus" };
666667

667-
::getConfigs(tokens, defined, mSettings.userDefines, mSettings.userUndefs, ret);
668+
::getConfigs(mTokens, defined, mSettings.userDefines, mSettings.userUndefs, ret);
668669

669670
for (const auto &filedata : mFileCache) {
670671
if (!mSettings.configurationExcluded(filedata->filename))
@@ -774,53 +775,53 @@ void Preprocessor::handleErrors(const simplecpp::OutputList& outputList, bool th
774775
}
775776
}
776777

777-
bool Preprocessor::loadFiles(const simplecpp::TokenList &rawtokens, std::vector<std::string> &files)
778+
bool Preprocessor::loadFiles(std::vector<std::string> &files)
778779
{
779780
const simplecpp::DUI dui = createDUI(mSettings, "", mLang);
780781

781782
simplecpp::OutputList outputList;
782-
mFileCache = simplecpp::load(rawtokens, files, dui, &outputList);
783+
mFileCache = simplecpp::load(mTokens, files, dui, &outputList);
783784
handleErrors(outputList, false);
784785
return !hasErrors(outputList);
785786
}
786787

787-
void Preprocessor::removeComments(simplecpp::TokenList &tokens) const
788+
void Preprocessor::removeComments()
788789
{
789-
tokens.removeComments();
790+
mTokens.removeComments();
790791
for (const auto &filedata : mFileCache) {
791792
filedata->tokens.removeComments();
792793
}
793794
}
794795

795-
void Preprocessor::setPlatformInfo(simplecpp::TokenList &tokens, const Settings& settings)
796+
void Preprocessor::setPlatformInfo()
796797
{
797-
tokens.sizeOfType["bool"] = settings.platform.sizeof_bool;
798-
tokens.sizeOfType["short"] = settings.platform.sizeof_short;
799-
tokens.sizeOfType["int"] = settings.platform.sizeof_int;
800-
tokens.sizeOfType["long"] = settings.platform.sizeof_long;
801-
tokens.sizeOfType["long long"] = settings.platform.sizeof_long_long;
802-
tokens.sizeOfType["float"] = settings.platform.sizeof_float;
803-
tokens.sizeOfType["double"] = settings.platform.sizeof_double;
804-
tokens.sizeOfType["long double"] = settings.platform.sizeof_long_double;
805-
tokens.sizeOfType["bool *"] = settings.platform.sizeof_pointer;
806-
tokens.sizeOfType["short *"] = settings.platform.sizeof_pointer;
807-
tokens.sizeOfType["int *"] = settings.platform.sizeof_pointer;
808-
tokens.sizeOfType["long *"] = settings.platform.sizeof_pointer;
809-
tokens.sizeOfType["long long *"] = settings.platform.sizeof_pointer;
810-
tokens.sizeOfType["float *"] = settings.platform.sizeof_pointer;
811-
tokens.sizeOfType["double *"] = settings.platform.sizeof_pointer;
812-
tokens.sizeOfType["long double *"] = settings.platform.sizeof_pointer;
798+
mTokens.sizeOfType["bool"] = mSettings.platform.sizeof_bool;
799+
mTokens.sizeOfType["short"] = mSettings.platform.sizeof_short;
800+
mTokens.sizeOfType["int"] = mSettings.platform.sizeof_int;
801+
mTokens.sizeOfType["long"] = mSettings.platform.sizeof_long;
802+
mTokens.sizeOfType["long long"] = mSettings.platform.sizeof_long_long;
803+
mTokens.sizeOfType["float"] = mSettings.platform.sizeof_float;
804+
mTokens.sizeOfType["double"] = mSettings.platform.sizeof_double;
805+
mTokens.sizeOfType["long double"] = mSettings.platform.sizeof_long_double;
806+
mTokens.sizeOfType["bool *"] = mSettings.platform.sizeof_pointer;
807+
mTokens.sizeOfType["short *"] = mSettings.platform.sizeof_pointer;
808+
mTokens.sizeOfType["int *"] = mSettings.platform.sizeof_pointer;
809+
mTokens.sizeOfType["long *"] = mSettings.platform.sizeof_pointer;
810+
mTokens.sizeOfType["long long *"] = mSettings.platform.sizeof_pointer;
811+
mTokens.sizeOfType["float *"] = mSettings.platform.sizeof_pointer;
812+
mTokens.sizeOfType["double *"] = mSettings.platform.sizeof_pointer;
813+
mTokens.sizeOfType["long double *"] = mSettings.platform.sizeof_pointer;
813814
}
814815

815-
simplecpp::TokenList Preprocessor::preprocess(const simplecpp::TokenList &tokens1, const std::string &cfg, std::vector<std::string> &files, bool throwError)
816+
simplecpp::TokenList Preprocessor::preprocess(const std::string &cfg, std::vector<std::string> &files, bool throwError)
816817
{
817818
const simplecpp::DUI dui = createDUI(mSettings, cfg, mLang);
818819

819820
simplecpp::OutputList outputList;
820821
std::list<simplecpp::MacroUsage> macroUsage;
821822
std::list<simplecpp::IfCond> ifCond;
822823
simplecpp::TokenList tokens2(files);
823-
simplecpp::preprocess(tokens2, tokens1, files, mFileCache, dui, &outputList, &macroUsage, &ifCond);
824+
simplecpp::preprocess(tokens2, mTokens, files, mFileCache, dui, &outputList, &macroUsage, &ifCond);
824825
mMacroUsage = std::move(macroUsage);
825826
mIfCond = std::move(ifCond);
826827

@@ -831,9 +832,9 @@ simplecpp::TokenList Preprocessor::preprocess(const simplecpp::TokenList &tokens
831832
return tokens2;
832833
}
833834

834-
std::string Preprocessor::getcode(const simplecpp::TokenList &tokens1, const std::string &cfg, std::vector<std::string> &files, const bool writeLocations)
835+
std::string Preprocessor::getcode(const std::string &cfg, std::vector<std::string> &files, const bool writeLocations)
835836
{
836-
simplecpp::TokenList tokens2 = preprocess(tokens1, cfg, files, false);
837+
simplecpp::TokenList tokens2 = preprocess(cfg, files, false);
837838
unsigned int prevfile = 0;
838839
unsigned int line = 1;
839840
std::ostringstream ret;
@@ -929,7 +930,9 @@ void Preprocessor::missingInclude(const std::string &filename, unsigned int line
929930

930931
void Preprocessor::getErrorMessages(ErrorLogger &errorLogger, const Settings &settings)
931932
{
932-
Preprocessor preprocessor(settings, errorLogger, Standards::Language::CPP);
933+
std::vector<std::string> files;
934+
simplecpp::TokenList tokens(files);
935+
Preprocessor preprocessor(tokens, settings, errorLogger, Standards::Language::CPP);
933936
preprocessor.missingInclude("", 1, "", UserHeader);
934937
preprocessor.missingInclude("", 1, "", SystemHeader);
935938
preprocessor.error("", 1, "#error message"); // #error ..
@@ -971,10 +974,10 @@ void Preprocessor::dump(std::ostream &out) const
971974
}
972975
}
973976

974-
std::size_t Preprocessor::calculateHash(const simplecpp::TokenList &tokens1, const std::string &toolinfo) const
977+
std::size_t Preprocessor::calculateHash(const std::string &toolinfo) const
975978
{
976979
std::string hashData = toolinfo;
977-
for (const simplecpp::Token *tok = tokens1.cfront(); tok; tok = tok->next) {
980+
for (const simplecpp::Token *tok = mTokens.cfront(); tok; tok = tok->next) {
978981
if (!tok->comment) {
979982
hashData += tok->str();
980983
hashData += static_cast<char>(tok->location.line);
@@ -993,9 +996,9 @@ std::size_t Preprocessor::calculateHash(const simplecpp::TokenList &tokens1, con
993996
return (std::hash<std::string>{})(hashData);
994997
}
995998

996-
void Preprocessor::simplifyPragmaAsm(simplecpp::TokenList &tokenList) const
999+
void Preprocessor::simplifyPragmaAsm()
9971000
{
998-
Preprocessor::simplifyPragmaAsmPrivate(tokenList);
1001+
Preprocessor::simplifyPragmaAsmPrivate(mTokens);
9991002
for (const auto &filedata : mFileCache) {
10001003
Preprocessor::simplifyPragmaAsmPrivate(filedata->tokens);
10011004
}

‎lib/preprocessor.h‎

Lines changed: 14 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -100,45 +100,42 @@ class CPPCHECKLIB RemarkComment {
100100
*/
101101
class CPPCHECKLIB WARN_UNUSED Preprocessor {
102102
// TODO: get rid of this
103-
friend class PreprocessorHelper;
104103
friend class TestPreprocessor;
105-
friend class TestUnusedVar;
106104

107105
public:
108106
/** character that is inserted in expanded macros */
109107
static char macroChar;
110108

111-
explicit Preprocessor(const Settings& settings, ErrorLogger &errorLogger, Standards::Language lang);
109+
explicit Preprocessor(simplecpp::TokenList& tokens, const Settings& settings, ErrorLogger &errorLogger, Standards::Language lang);
112110
virtual ~Preprocessor() = default;
113111

114-
void inlineSuppressions(const simplecpp::TokenList &tokens, SuppressionList &suppressions);
112+
void inlineSuppressions(SuppressionList &suppressions);
115113

116-
std::list<Directive> createDirectives(const simplecpp::TokenList &tokens) const;
114+
std::list<Directive> createDirectives() const;
117115

118-
std::set<std::string> getConfigs(const simplecpp::TokenList &tokens) const;
116+
std::set<std::string> getConfigs() const;
119117

120-
std::vector<RemarkComment> getRemarkComments(const simplecpp::TokenList &tokens) const;
118+
std::vector<RemarkComment> getRemarkComments() const;
121119

122-
bool loadFiles(const simplecpp::TokenList &rawtokens, std::vector<std::string> &files);
120+
bool loadFiles(std::vector<std::string> &files);
123121

124-
void removeComments(simplecpp::TokenList &tokens) const;
122+
void removeComments();
125123

126-
static void setPlatformInfo(simplecpp::TokenList &tokens, const Settings& settings);
124+
void setPlatformInfo();
127125

128-
simplecpp::TokenList preprocess(const simplecpp::TokenList &tokens1, const std::string &cfg, std::vector<std::string> &files, bool throwError = false);
126+
simplecpp::TokenList preprocess(const std::string &cfg, std::vector<std::string> &files, bool throwError = false);
129127

130-
std::string getcode(const simplecpp::TokenList &tokens1, const std::string &cfg, std::vector<std::string> &files, bool writeLocations);
128+
std::string getcode(const std::string &cfg, std::vector<std::string> &files, bool writeLocations);
131129

132130
/**
133131
* Calculate HASH. Using toolinfo, tokens1, filedata.
134132
*
135-
* @param tokens1 Sourcefile tokens
136133
* @param toolinfo Arbitrary extra toolinfo
137134
* @return HASH
138135
*/
139-
std::size_t calculateHash(const simplecpp::TokenList &tokens1, const std::string &toolinfo) const;
136+
std::size_t calculateHash(const std::string &toolinfo) const;
140137

141-
void simplifyPragmaAsm(simplecpp::TokenList &tokenList) const;
138+
void simplifyPragmaAsm();
142139

143140
static void getErrorMessages(ErrorLogger &errorLogger, const Settings &settings);
144141

@@ -171,6 +168,8 @@ class CPPCHECKLIB WARN_UNUSED Preprocessor {
171168

172169
void addRemarkComments(const simplecpp::TokenList &tokens, std::vector<RemarkComment> &remarkComments) const;
173170

171+
simplecpp::TokenList& mTokens;
172+
174173
const Settings& mSettings;
175174
ErrorLogger &mErrorLogger;
176175

0 commit comments

Comments
 (0)