Skip to content

store reference of associated tokenlist in Preprocessor #7903

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

store reference of associated tokenlist in Preprocessor #7903

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 17 additions & 17 deletions lib/cppcheck.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -847,7 +847,7 @@ unsigned int CppCheck::check(const FileSettings &fs)
return returnValue;
}

std::size_t CppCheck::calculateHash(const Preprocessor& preprocessor, const simplecpp::TokenList& tokens, const std::string& filePath) const
std::size_t CppCheck::calculateHash(const Preprocessor& preprocessor, const std::string& filePath) const
{
std::ostringstream toolinfo;
toolinfo << (mSettings.cppcheckCfgProductName.empty() ? CPPCHECK_VERSION_STRING : mSettings.cppcheckCfgProductName);
Expand All @@ -865,7 +865,7 @@ std::size_t CppCheck::calculateHash(const Preprocessor& preprocessor, const simp
toolinfo << mSettings.premiumArgs;
// TODO: do we need to add more options?
mSuppressions.nomsg.dump(toolinfo, filePath);
return preprocessor.calculateHash(tokens, toolinfo.str());
return preprocessor.calculateHash(toolinfo.str());
}

unsigned int CppCheck::checkBuffer(const FileWithDetails &file, const std::string &cfgname, int fileIndex, const uint8_t* data, std::size_t size)
Expand Down Expand Up @@ -938,8 +938,8 @@ unsigned int CppCheck::checkInternal(const FileWithDetails& file, const std::str
std::vector<std::string> files;
simplecpp::TokenList tokens = createTokenList(files, nullptr);
if (analyzerInformation) {
const Preprocessor preprocessor(mSettings, mErrorLogger, file.lang());
hash = calculateHash(preprocessor, tokens);
const Preprocessor preprocessor(tokens, mSettings, mErrorLogger, file.lang());
hash = calculateHash(preprocessor);
}
tokenlist.createTokens(std::move(tokens));
// this is not a real source file - we just want to tokenize it. treat it as C anyways as the language needs to be determined.
Expand Down Expand Up @@ -984,9 +984,9 @@ unsigned int CppCheck::checkInternal(const FileWithDetails& file, const std::str
return mLogger->exitcode();
}

Preprocessor preprocessor(mSettings, mErrorLogger, file.lang());
Preprocessor preprocessor(tokens1, mSettings, mErrorLogger, file.lang());

if (!preprocessor.loadFiles(tokens1, files))
if (!preprocessor.loadFiles(files))
return mLogger->exitcode();

if (!mSettings.plistOutput.empty()) {
Expand All @@ -1006,14 +1006,14 @@ unsigned int CppCheck::checkInternal(const FileWithDetails& file, const std::str
}

// Parse comments and then remove them
mLogger->setRemarkComments(preprocessor.getRemarkComments(tokens1));
preprocessor.inlineSuppressions(tokens1, mSuppressions.nomsg);
mLogger->setRemarkComments(preprocessor.getRemarkComments());
preprocessor.inlineSuppressions(mSuppressions.nomsg);
if (mSettings.dump || !mSettings.addons.empty()) {
std::ostringstream oss;
mSuppressions.nomsg.dump(oss);
dumpProlog += oss.str();
}
preprocessor.removeComments(tokens1);
preprocessor.removeComments();

if (!mSettings.buildDir.empty()) {
analyzerInformation.reset(new AnalyzerInformation);
Expand All @@ -1022,7 +1022,7 @@ unsigned int CppCheck::checkInternal(const FileWithDetails& file, const std::str

if (analyzerInformation) {
// Calculate hash so it can be compared with old hash / future hashes
const std::size_t hash = calculateHash(preprocessor, tokens1, file.spath());
const std::size_t hash = calculateHash(preprocessor, file.spath());
std::list<ErrorMessage> errors;
if (!analyzerInformation->analyzeFile(mSettings.buildDir, file.spath(), cfgname, fileIndex, hash, errors)) {
while (!errors.empty()) {
Expand All @@ -1035,24 +1035,24 @@ unsigned int CppCheck::checkInternal(const FileWithDetails& file, const std::str
}

// Get directives
std::list<Directive> directives = preprocessor.createDirectives(tokens1);
preprocessor.simplifyPragmaAsm(tokens1);
std::list<Directive> directives = preprocessor.createDirectives();
preprocessor.simplifyPragmaAsm();

Preprocessor::setPlatformInfo(tokens1, mSettings);
preprocessor.setPlatformInfo();

// Get configurations..
std::set<std::string> configurations;
if ((mSettings.checkAllConfigurations && mSettings.userDefines.empty()) || mSettings.force) {
Timer::run("Preprocessor::getConfigs", mSettings.showtime, &s_timerResults, [&]() {
configurations = preprocessor.getConfigs(tokens1);
configurations = preprocessor.getConfigs();
});
} else {
configurations.insert(mSettings.userDefines);
}

if (mSettings.checkConfiguration) {
for (const std::string &config : configurations)
(void)preprocessor.getcode(tokens1, config, files, false);
(void)preprocessor.getcode(config, files, false);

if (analyzerInformation)
mLogger->setAnalyzerInfo(nullptr);
Expand Down Expand Up @@ -1125,7 +1125,7 @@ unsigned int CppCheck::checkInternal(const FileWithDetails& file, const std::str
if (mSettings.preprocessOnly) {
std::string codeWithoutCfg;
Timer::run("Preprocessor::getcode", mSettings.showtime, &s_timerResults, [&]() {
codeWithoutCfg = preprocessor.getcode(tokens1, currentConfig, files, true);
codeWithoutCfg = preprocessor.getcode(currentConfig, files, true);
});

if (startsWith(codeWithoutCfg,"#file"))
Expand All @@ -1148,7 +1148,7 @@ unsigned int CppCheck::checkInternal(const FileWithDetails& file, const std::str

// Create tokens, skip rest of iteration if failed
Timer::run("Tokenizer::createTokens", mSettings.showtime, &s_timerResults, [&]() {
simplecpp::TokenList tokensP = preprocessor.preprocess(tokens1, currentConfig, files, true);
simplecpp::TokenList tokensP = preprocessor.preprocess(currentConfig, files, true);
tokenlist.createTokens(std::move(tokensP));
});
hasValidConfig = true;
Expand Down
3 changes: 1 addition & 2 deletions lib/cppcheck.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,10 +172,9 @@ class CPPCHECKLIB CppCheck {
* @brief Calculate hash used to detect when a file needs to be reanalyzed.
*
* @param preprocessor Preprocessor used to calculate the hash.
* @param tokens Token list from preprocessed file.
* @return hash
*/
std::size_t calculateHash(const Preprocessor &preprocessor, const simplecpp::TokenList &tokens, const std::string& filePath = {}) const;
std::size_t calculateHash(const Preprocessor &preprocessor, const std::string& filePath = {}) const;

/**
* @brief Check a file
Expand Down
85 changes: 44 additions & 41 deletions lib/preprocessor.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,9 @@ Directive::DirectiveToken::DirectiveToken(const simplecpp::Token & _tok) :

char Preprocessor::macroChar = char(1);

Preprocessor::Preprocessor(const Settings& settings, ErrorLogger &errorLogger, Standards::Language lang)
: mSettings(settings)
Preprocessor::Preprocessor(simplecpp::TokenList& tokens, const Settings& settings, ErrorLogger &errorLogger, Standards::Language lang)
: mTokens(tokens)
, mSettings(settings)
, mErrorLogger(errorLogger)
, mLang(lang)
{
Expand Down Expand Up @@ -301,12 +302,12 @@ static void addInlineSuppressions(const simplecpp::TokenList &tokens, const Sett
bad.emplace_back(suppr.fileName, suppr.lineNumber, "Suppress Begin: No matching end");
}

void Preprocessor::inlineSuppressions(const simplecpp::TokenList &tokens, SuppressionList &suppressions)
void Preprocessor::inlineSuppressions(SuppressionList &suppressions)
{
if (!mSettings.inlineSuppressions)
return;
std::list<BadInlineSuppression> err;
::addInlineSuppressions(tokens, mSettings, suppressions, err);
::addInlineSuppressions(mTokens, mSettings, suppressions, err);
for (const auto &filedata : mFileCache) {
::addInlineSuppressions(filedata->tokens, mSettings, suppressions, err);
}
Expand All @@ -315,24 +316,24 @@ void Preprocessor::inlineSuppressions(const simplecpp::TokenList &tokens, Suppre
}
}

std::vector<RemarkComment> Preprocessor::getRemarkComments(const simplecpp::TokenList &tokens) const
std::vector<RemarkComment> Preprocessor::getRemarkComments() const
{
std::vector<RemarkComment> ret;
addRemarkComments(tokens, ret);
addRemarkComments(mTokens, ret);
for (const auto &filedata : mFileCache) {
addRemarkComments(filedata->tokens, ret);
}
return ret;
}

std::list<Directive> Preprocessor::createDirectives(const simplecpp::TokenList &tokens) const
std::list<Directive> Preprocessor::createDirectives() const
{
// directive list..
std::list<Directive> directives;

std::vector<const simplecpp::TokenList *> list;
list.reserve(1U + mFileCache.size());
list.push_back(&tokens);
list.push_back(&mTokens);
std::transform(mFileCache.cbegin(), mFileCache.cend(), std::back_inserter(list),
[](const std::unique_ptr<simplecpp::FileData> &filedata) {
return &filedata->tokens;
Expand Down Expand Up @@ -656,15 +657,15 @@ static void getConfigs(const simplecpp::TokenList &tokens, std::set<std::string>
}


std::set<std::string> Preprocessor::getConfigs(const simplecpp::TokenList &tokens) const
std::set<std::string> Preprocessor::getConfigs() const
{
std::set<std::string> ret = { "" };
if (!tokens.cfront())
if (!mTokens.cfront())
return ret;

std::set<std::string> defined = { "__cplusplus" };

::getConfigs(tokens, defined, mSettings.userDefines, mSettings.userUndefs, ret);
::getConfigs(mTokens, defined, mSettings.userDefines, mSettings.userUndefs, ret);

for (const auto &filedata : mFileCache) {
if (!mSettings.configurationExcluded(filedata->filename))
Expand Down Expand Up @@ -774,53 +775,53 @@ void Preprocessor::handleErrors(const simplecpp::OutputList& outputList, bool th
}
}

bool Preprocessor::loadFiles(const simplecpp::TokenList &rawtokens, std::vector<std::string> &files)
bool Preprocessor::loadFiles(std::vector<std::string> &files)
{
const simplecpp::DUI dui = createDUI(mSettings, "", mLang);

simplecpp::OutputList outputList;
mFileCache = simplecpp::load(rawtokens, files, dui, &outputList);
mFileCache = simplecpp::load(mTokens, files, dui, &outputList);
handleErrors(outputList, false);
return !hasErrors(outputList);
}

void Preprocessor::removeComments(simplecpp::TokenList &tokens) const
void Preprocessor::removeComments()
{
tokens.removeComments();
mTokens.removeComments();
for (const auto &filedata : mFileCache) {
filedata->tokens.removeComments();
}
}

void Preprocessor::setPlatformInfo(simplecpp::TokenList &tokens, const Settings& settings)
void Preprocessor::setPlatformInfo()
{
tokens.sizeOfType["bool"] = settings.platform.sizeof_bool;
tokens.sizeOfType["short"] = settings.platform.sizeof_short;
tokens.sizeOfType["int"] = settings.platform.sizeof_int;
tokens.sizeOfType["long"] = settings.platform.sizeof_long;
tokens.sizeOfType["long long"] = settings.platform.sizeof_long_long;
tokens.sizeOfType["float"] = settings.platform.sizeof_float;
tokens.sizeOfType["double"] = settings.platform.sizeof_double;
tokens.sizeOfType["long double"] = settings.platform.sizeof_long_double;
tokens.sizeOfType["bool *"] = settings.platform.sizeof_pointer;
tokens.sizeOfType["short *"] = settings.platform.sizeof_pointer;
tokens.sizeOfType["int *"] = settings.platform.sizeof_pointer;
tokens.sizeOfType["long *"] = settings.platform.sizeof_pointer;
tokens.sizeOfType["long long *"] = settings.platform.sizeof_pointer;
tokens.sizeOfType["float *"] = settings.platform.sizeof_pointer;
tokens.sizeOfType["double *"] = settings.platform.sizeof_pointer;
tokens.sizeOfType["long double *"] = settings.platform.sizeof_pointer;
mTokens.sizeOfType["bool"] = mSettings.platform.sizeof_bool;
mTokens.sizeOfType["short"] = mSettings.platform.sizeof_short;
mTokens.sizeOfType["int"] = mSettings.platform.sizeof_int;
mTokens.sizeOfType["long"] = mSettings.platform.sizeof_long;
mTokens.sizeOfType["long long"] = mSettings.platform.sizeof_long_long;
mTokens.sizeOfType["float"] = mSettings.platform.sizeof_float;
mTokens.sizeOfType["double"] = mSettings.platform.sizeof_double;
mTokens.sizeOfType["long double"] = mSettings.platform.sizeof_long_double;
mTokens.sizeOfType["bool *"] = mSettings.platform.sizeof_pointer;
mTokens.sizeOfType["short *"] = mSettings.platform.sizeof_pointer;
mTokens.sizeOfType["int *"] = mSettings.platform.sizeof_pointer;
mTokens.sizeOfType["long *"] = mSettings.platform.sizeof_pointer;
mTokens.sizeOfType["long long *"] = mSettings.platform.sizeof_pointer;
mTokens.sizeOfType["float *"] = mSettings.platform.sizeof_pointer;
mTokens.sizeOfType["double *"] = mSettings.platform.sizeof_pointer;
mTokens.sizeOfType["long double *"] = mSettings.platform.sizeof_pointer;
}

simplecpp::TokenList Preprocessor::preprocess(const simplecpp::TokenList &tokens1, const std::string &cfg, std::vector<std::string> &files, bool throwError)
simplecpp::TokenList Preprocessor::preprocess(const std::string &cfg, std::vector<std::string> &files, bool throwError)
{
const simplecpp::DUI dui = createDUI(mSettings, cfg, mLang);

simplecpp::OutputList outputList;
std::list<simplecpp::MacroUsage> macroUsage;
std::list<simplecpp::IfCond> ifCond;
simplecpp::TokenList tokens2(files);
simplecpp::preprocess(tokens2, tokens1, files, mFileCache, dui, &outputList, &macroUsage, &ifCond);
simplecpp::preprocess(tokens2, mTokens, files, mFileCache, dui, &outputList, &macroUsage, &ifCond);
mMacroUsage = std::move(macroUsage);
mIfCond = std::move(ifCond);

Expand All @@ -831,9 +832,9 @@ simplecpp::TokenList Preprocessor::preprocess(const simplecpp::TokenList &tokens
return tokens2;
}

std::string Preprocessor::getcode(const simplecpp::TokenList &tokens1, const std::string &cfg, std::vector<std::string> &files, const bool writeLocations)
std::string Preprocessor::getcode(const std::string &cfg, std::vector<std::string> &files, const bool writeLocations)
{
simplecpp::TokenList tokens2 = preprocess(tokens1, cfg, files, false);
simplecpp::TokenList tokens2 = preprocess(cfg, files, false);
unsigned int prevfile = 0;
unsigned int line = 1;
std::ostringstream ret;
Expand Down Expand Up @@ -929,7 +930,9 @@ void Preprocessor::missingInclude(const std::string &filename, unsigned int line

void Preprocessor::getErrorMessages(ErrorLogger &errorLogger, const Settings &settings)
{
Preprocessor preprocessor(settings, errorLogger, Standards::Language::CPP);
std::vector<std::string> files;
simplecpp::TokenList tokens(files);
Preprocessor preprocessor(tokens, settings, errorLogger, Standards::Language::CPP);
preprocessor.missingInclude("", 1, "", UserHeader);
preprocessor.missingInclude("", 1, "", SystemHeader);
preprocessor.error("", 1, "#error message"); // #error ..
Expand Down Expand Up @@ -971,10 +974,10 @@ void Preprocessor::dump(std::ostream &out) const
}
}

std::size_t Preprocessor::calculateHash(const simplecpp::TokenList &tokens1, const std::string &toolinfo) const
std::size_t Preprocessor::calculateHash(const std::string &toolinfo) const
{
std::string hashData = toolinfo;
for (const simplecpp::Token *tok = tokens1.cfront(); tok; tok = tok->next) {
for (const simplecpp::Token *tok = mTokens.cfront(); tok; tok = tok->next) {
if (!tok->comment) {
hashData += tok->str();
hashData += static_cast<char>(tok->location.line);
Expand All @@ -993,9 +996,9 @@ std::size_t Preprocessor::calculateHash(const simplecpp::TokenList &tokens1, con
return (std::hash<std::string>{})(hashData);
}

void Preprocessor::simplifyPragmaAsm(simplecpp::TokenList &tokenList) const
void Preprocessor::simplifyPragmaAsm()
{
Preprocessor::simplifyPragmaAsmPrivate(tokenList);
Preprocessor::simplifyPragmaAsmPrivate(mTokens);
for (const auto &filedata : mFileCache) {
Preprocessor::simplifyPragmaAsmPrivate(filedata->tokens);
}
Expand Down
29 changes: 14 additions & 15 deletions lib/preprocessor.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,45 +100,42 @@ class CPPCHECKLIB RemarkComment {
*/
class CPPCHECKLIB WARN_UNUSED Preprocessor {
// TODO: get rid of this
friend class PreprocessorHelper;
friend class TestPreprocessor;
friend class TestUnusedVar;

public:
/** character that is inserted in expanded macros */
static char macroChar;

explicit Preprocessor(const Settings& settings, ErrorLogger &errorLogger, Standards::Language lang);
explicit Preprocessor(simplecpp::TokenList& tokens, const Settings& settings, ErrorLogger &errorLogger, Standards::Language lang);
virtual ~Preprocessor() = default;

void inlineSuppressions(const simplecpp::TokenList &tokens, SuppressionList &suppressions);
void inlineSuppressions(SuppressionList &suppressions);

std::list<Directive> createDirectives(const simplecpp::TokenList &tokens) const;
std::list<Directive> createDirectives() const;

std::set<std::string> getConfigs(const simplecpp::TokenList &tokens) const;
std::set<std::string> getConfigs() const;

std::vector<RemarkComment> getRemarkComments(const simplecpp::TokenList &tokens) const;
std::vector<RemarkComment> getRemarkComments() const;

bool loadFiles(const simplecpp::TokenList &rawtokens, std::vector<std::string> &files);
bool loadFiles(std::vector<std::string> &files);

void removeComments(simplecpp::TokenList &tokens) const;
void removeComments();

static void setPlatformInfo(simplecpp::TokenList &tokens, const Settings& settings);
void setPlatformInfo();

simplecpp::TokenList preprocess(const simplecpp::TokenList &tokens1, const std::string &cfg, std::vector<std::string> &files, bool throwError = false);
simplecpp::TokenList preprocess(const std::string &cfg, std::vector<std::string> &files, bool throwError = false);

std::string getcode(const simplecpp::TokenList &tokens1, const std::string &cfg, std::vector<std::string> &files, bool writeLocations);
std::string getcode(const std::string &cfg, std::vector<std::string> &files, bool writeLocations);

/**
* Calculate HASH. Using toolinfo, tokens1, filedata.
*
* @param tokens1 Sourcefile tokens
* @param toolinfo Arbitrary extra toolinfo
* @return HASH
*/
std::size_t calculateHash(const simplecpp::TokenList &tokens1, const std::string &toolinfo) const;
std::size_t calculateHash(const std::string &toolinfo) const;

void simplifyPragmaAsm(simplecpp::TokenList &tokenList) const;
void simplifyPragmaAsm();

static void getErrorMessages(ErrorLogger &errorLogger, const Settings &settings);

Expand Down Expand Up @@ -171,6 +168,8 @@ class CPPCHECKLIB WARN_UNUSED Preprocessor {

void addRemarkComments(const simplecpp::TokenList &tokens, std::vector<RemarkComment> &remarkComments) const;

simplecpp::TokenList& mTokens;

const Settings& mSettings;
ErrorLogger &mErrorLogger;

Expand Down
Loading