Skip to content

some TokenList cleanups#5848

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
firewave merged 5 commits into from
Jan 6, 2024
Merged

some TokenList cleanups #5848

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
2014086
TokenList: added missing standards checks
firewave Jan 5, 2024
52d6c89
TokenList: reduced dependency on settings
firewave Jan 5, 2024
3b2a5c6
TokenList: store `Settings::Language` instead of individual booleans
firewave Jan 5, 2024
bf2dd0c
TokenList: added missing settings checks
firewave Jan 6, 2024
19bf024
TokenList: apply enforced language early
firewave Jan 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions Makefile
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -491,7 +491,7 @@ $(libcppdir)/checkautovariables.o: lib/checkautovariables.cpp lib/addoninfo.h li
$(libcppdir)/checkbool.o: lib/checkbool.cpp lib/addoninfo.h lib/astutils.h lib/check.h lib/checkbool.h lib/config.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/smallvector.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h
$(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/checkbool.cpp

$(libcppdir)/checkboost.o: lib/checkboost.cpp lib/check.h lib/checkboost.h lib/config.h lib/errortypes.h lib/library.h lib/mathlib.h lib/sourcelocation.h lib/standards.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h
$(libcppdir)/checkboost.o: lib/checkboost.cpp lib/addoninfo.h lib/check.h lib/checkboost.h lib/config.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h
$(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/checkboost.cpp

$(libcppdir)/checkbufferoverrun.o: lib/checkbufferoverrun.cpp externals/tinyxml2/tinyxml2.h lib/addoninfo.h lib/astutils.h lib/check.h lib/checkbufferoverrun.h lib/color.h lib/config.h lib/ctu.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/smallvector.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/valueflow.h lib/vfvalue.h lib/xml.h
Expand DownExpand Up@@ -593,7 +593,7 @@ $(libcppdir)/infer.o: lib/infer.cpp lib/calculate.h lib/config.h lib/errortypes.
$(libcppdir)/keywords.o: lib/keywords.cpp lib/config.h lib/keywords.h lib/standards.h lib/utils.h
$(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/keywords.cpp

$(libcppdir)/library.o: lib/library.cpp externals/tinyxml2/tinyxml2.h lib/astutils.h lib/config.h lib/errortypes.h lib/library.h lib/mathlib.h lib/path.h lib/smallvector.h lib/sourcelocation.h lib/standards.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenlist.h lib/utils.h lib/valueflow.h lib/vfvalue.h lib/xml.h
$(libcppdir)/library.o: lib/library.cpp externals/tinyxml2/tinyxml2.h lib/addoninfo.h lib/astutils.h lib/config.h lib/errortypes.h lib/library.h lib/mathlib.h lib/path.h lib/platform.h lib/settings.h lib/smallvector.h lib/sourcelocation.h lib/standards.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenlist.h lib/utils.h lib/valueflow.h lib/vfvalue.h lib/xml.h
$(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/library.cpp

$(libcppdir)/mathlib.o: lib/mathlib.cpp externals/simplecpp/simplecpp.h lib/config.h lib/errortypes.h lib/mathlib.h lib/utils.h
Expand DownExpand Up@@ -626,7 +626,7 @@ $(libcppdir)/settings.o: lib/settings.cpp externals/picojson/picojson.h lib/addo
$(libcppdir)/summaries.o: lib/summaries.cpp lib/addoninfo.h lib/analyzerinfo.h lib/config.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/sourcelocation.h lib/standards.h lib/summaries.h lib/suppressions.h lib/symboldatabase.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h
$(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/summaries.cpp

$(libcppdir)/suppressions.o: lib/suppressions.cpp externals/tinyxml2/tinyxml2.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/mathlib.h lib/path.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h lib/xml.h
$(libcppdir)/suppressions.o: lib/suppressions.cpp externals/tinyxml2/tinyxml2.h lib/addoninfo.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/path.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h lib/xml.h
$(CXX) ${INCLUDE_FOR_LIB} $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $(libcppdir)/suppressions.cpp

$(libcppdir)/templatesimplifier.o: lib/templatesimplifier.cpp lib/addoninfo.h lib/color.h lib/config.h lib/errorlogger.h lib/errortypes.h lib/library.h lib/mathlib.h lib/platform.h lib/settings.h lib/standards.h lib/suppressions.h lib/templatesimplifier.h lib/token.h lib/tokenize.h lib/tokenlist.h lib/utils.h lib/vfvalue.h
Expand Down
4 changes: 2 additions & 2 deletions lib/tokenize.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -3361,10 +3361,10 @@ bool Tokenizer::simplifyTokens1(const std::string &configuration)
if (mTimerResults) {
Timer t("Tokenizer::simplifyTokens1::createAst", mSettings->showtime, mTimerResults);
list.createAst();
list.validateAst();
list.validateAst(mSettings->debugnormal);
} else {
list.createAst();
list.validateAst();
list.validateAst(mSettings->debugnormal);
}

if (mTimerResults) {
Expand Down
27 changes: 15 additions & 12 deletions lib/tokenlist.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -50,6 +50,9 @@ TokenList::TokenList(const Settings* settings) :
mSettings(settings)
{
mTokensFrontBack.list = this;
if (mSettings && (mSettings->enforcedLang != Settings::Language::None)) {
mLang = mSettings->enforcedLang;
}
}

TokenList::~TokenList()
Expand DownExpand Up@@ -80,12 +83,12 @@ void TokenList::deallocateTokens()

void TokenList::determineCppC()
{
if (!mSettings) {
mIsC = Path::isC(getSourceFilePath());
mIsCpp = Path::isCPP(getSourceFilePath());
} else {
mIsC = mSettings->enforcedLang == Settings::Language::C || (mSettings->enforcedLang == Settings::Language::None && Path::isC(getSourceFilePath()));
mIsCpp = mSettings->enforcedLang == Settings::Language::CPP || (mSettings->enforcedLang == Settings::Language::None && Path::isCPP(getSourceFilePath()));
// only try to determine it if it wasn't enforced
if (mLang == Settings::Language::None) {
if (Path::isC(getSourceFilePath()))
mLang = Settings::Language::C;
else if (Path::isCPP(getSourceFilePath()))
mLang = Settings::Language::CPP;
}
}

Expand DownExpand Up@@ -1722,10 +1725,10 @@ namespace {
};
}

void TokenList::validateAst() const
void TokenList::validateAst(bool print) const
{
OnException oe{[&] {
if (mSettings && mSettings->debugnormal)
if (print)
mTokensFrontBack.front->printOut();
}};
// Check for some known issues in AST to avoid crash/hang later on
Expand DownExpand Up@@ -1854,7 +1857,7 @@ void TokenList::simplifyPlatformTypes()
if (!mSettings)
return;

const bool isCPP11 = mSettings->standards.cpp >= Standards::CPP11;
const bool isCPP11 = isCPP() && (mSettings->standards.cpp >= Standards::CPP11);

enum { isLongLong, isLong, isInt } type;

Expand DownExpand Up@@ -1996,7 +1999,7 @@ void TokenList::simplifyStdType()
continue;
}

if (Token::Match(tok, "char|short|int|long|unsigned|signed|double|float") || (mSettings->standards.c >= Standards::C99 && Token::Match(tok, "complex|_Complex"))) {
if (Token::Match(tok, "char|short|int|long|unsigned|signed|double|float") || (isC() && (!mSettings || (mSettings->standards.c >= Standards::C99)) && Token::Match(tok, "complex|_Complex"))) {
bool isFloat= false;
bool isSigned = false;
bool isUnsigned = false;
Expand All@@ -2019,7 +2022,7 @@ void TokenList::simplifyStdType()
else if (Token::Match(tok2, "float|double")) {
isFloat = true;
typeSpec = tok2;
} else if (mSettings->standards.c >= Standards::C99 && Token::Match(tok2, "complex|_Complex"))
} else if (isC() && (!mSettings || (mSettings->standards.c >= Standards::C99)) && Token::Match(tok2, "complex|_Complex"))
isComplex = !isFloat || tok2->str() == "_Complex" || Token::Match(tok2->next(), "*|&|%name%"); // Ensure that "complex" is not the variables name
else if (Token::Match(tok2, "char|int")) {
if (!typeSpec)
Expand DownExpand Up@@ -2057,7 +2060,7 @@ void TokenList::simplifyStdType()

bool TokenList::isKeyword(const std::string &str) const
{
if (mIsCpp) {
if (isCPP()) {
// TODO: integrate into keywords?
// types and literals are not handled as keywords
static const std::unordered_set<std::string> cpp_types = {"bool", "false", "true"};
Expand Down
11 changes: 5 additions & 6 deletions lib/tokenlist.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -22,13 +22,13 @@
//---------------------------------------------------------------------------

#include "config.h"
#include "settings.h"

#include <cstddef>
#include <iosfwd>
#include <string>
#include <vector>

class Settings;
class Token;
class TokenList;

Expand DownExpand Up@@ -61,12 +61,12 @@ class CPPCHECKLIB TokenList {

/** Is the code C. Used for bailouts */
bool isC() const {
return mIsC;
return mLang == Settings::Language::C;
}

/** Is the code CPP. Used for bailouts */
bool isCPP() const {
return mIsCpp;
return mLang == Settings::Language::CPP;
}

/**
Expand DownExpand Up@@ -171,7 +171,7 @@ class CPPCHECKLIB TokenList {
* Check abstract syntax tree.
* Throws InternalError on failure
*/
void validateAst() const;
void validateAst(bool print) const;

/**
* Verify that the given token is an element of the tokenlist.
Expand DownExpand Up@@ -214,8 +214,7 @@ class CPPCHECKLIB TokenList {
const Settings* const mSettings{};

/** File is known to be C/C++ code */
bool mIsC{};
bool mIsCpp{};
Settings::Language mLang{Settings::Language::None};
};

/// @}
Expand Down
17 changes: 11 additions & 6 deletions test/testtokenize.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -2873,28 +2873,33 @@ class TestTokenizer : public TestFixture {
}
{
const char code[] = "float complex x;";
const char expected[] = "_Complex float x ;";
const char expected[] = "float complex x ;";
ASSERT_EQUALS(expected, tokenizeAndStringify(code));
}
{
const char code[] = "float complex x;";
const char expected[] = "_Complex float x ;";
ASSERT_EQUALS(expected, tokenizeAndStringify(code, true, Platform::Native, "test.c"));
}
{
const char code[] = "complex float x;";
const char expected[] = "_Complex float x ;";
ASSERT_EQUALS(expected, tokenizeAndStringify(code));
ASSERT_EQUALS(expected, tokenizeAndStringify(code, true, Platform::Native, "test.c"));
}
{
const char code[] = "complex long double x;";
const char expected[] = "_Complex long double x ;";
ASSERT_EQUALS(expected, tokenizeAndStringify(code));
ASSERT_EQUALS(expected, tokenizeAndStringify(code, true, Platform::Native, "test.c"));
}
{
const char code[] = "long double complex x;";
const char expected[] = "_Complex long double x ;";
ASSERT_EQUALS(expected, tokenizeAndStringify(code));
ASSERT_EQUALS(expected, tokenizeAndStringify(code, true, Platform::Native, "test.c"));
}
{
const char code[] = "double complex;";
const char expected[] = "double complex ;";
ASSERT_EQUALS(expected, tokenizeAndStringify(code));
ASSERT_EQUALS(expected, tokenizeAndStringify(code, true, Platform::Native, "test.c"));
}
}

Expand DownExpand Up@@ -5975,7 +5980,7 @@ class TestTokenizer : public TestFixture {
tokenList.prepareTernaryOpForAST();
tokenList.list.createAst();

tokenList.list.validateAst();
tokenList.list.validateAst(false);

// Basic AST validation
for (const Token *tok = tokenList.list.front(); tok; tok = tok->next()) {
Expand Down