Skip to content

Fix #684: Keep line directives in tokenized output#685

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
glankk wants to merge 7 commits into from
Closed

Fix #684: Keep line directives in tokenized output #685

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
eebfbad
Keep line directives in tokenized output
glankk Jul 20, 2026
2448c36
Fix gotoNextLine
glankk Jul 20, 2026
31bf2b8
Fix lastLineTok
glankk Jul 20, 2026
1821aad
Fix clang-tidy issues
glankk Jul 20, 2026
4c6ba36
Add tests
glankk Jul 21, 2026
8a226f3
Fix clang-tidy issues
glankk Jul 21, 2026
6cbce17
Fix formatting
glankk Jul 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 17 additions & 26 deletions simplecpp.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -640,23 +640,6 @@ static bool isStringLiteralPrefix(const std::string &str)
str == "R" || str == "uR" || str == "UR" || str == "LR" || str == "u8R";
}

void simplecpp::TokenList::lineDirective(unsigned int fileIndex_, unsigned int line, Location &location)
{
if (fileIndex_ != location.fileIndex || line >= location.line) {
location.fileIndex = fileIndex_;
location.line = line;
return;
}

if (line + 2 >= location.line) {
location.line = line;
while (cback()->op != '#')
deleteToken(back());
deleteToken(back());
return;
}
}

static const std::string COMMENT_END("*/");

void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename, OutputList *outputList)
Expand All@@ -665,6 +648,8 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename,

const Token *oldLastToken = nullptr;

bool locationchange = false;

Location location(fileIndex(filename), 1, 1);
while (stream.good()) {
unsigned char ch = stream.readChar();
Expand DownExpand Up@@ -726,17 +711,14 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename,
if (!ppTok || !ppTok->number)
continue;

const unsigned int line = std::atol(ppTok->str().c_str());
ppTok = advanceAndSkipComments(ppTok);
location.line = std::atol(ppTok->str().c_str());

unsigned int fileindex;
ppTok = advanceAndSkipComments(ppTok);

if (ppTok && ppTok->str()[0] == '\"')
fileindex = fileIndex(replaceAll(ppTok->str().substr(1U, ppTok->str().size() - 2U),"\\\\","\\"));
else
fileindex = location.fileIndex;
location.fileIndex = fileIndex(replaceAll(ppTok->str().substr(1U, ppTok->str().size() - 2U),"\\\\","\\"));

lineDirective(fileindex, line, location);
locationchange = true;
}

continue;
Expand DownExpand Up@@ -955,6 +937,11 @@ void simplecpp::TokenList::readfile(Stream &stream, const std::string &filename,

push_back(new Token(currentToken, location, !!std::isspace(stream.peekChar())));

if (locationchange) {
back()->locationchange = true;
locationchange = false;
}

if (multiline)
location.col += currentToken.size();
else
Expand DownExpand Up@@ -1458,6 +1445,8 @@ const simplecpp::Token* simplecpp::TokenList::lastLineTok(int maxsize) const
for (const Token *tok = cback(); ; tok = tok->previous) {
if (!sameline(tok, cback()))
break;
if (tok->locationchange)
return tok;
if (tok->comment)
continue;
if (++count > maxsize)
Expand DownExpand Up@@ -3000,7 +2989,9 @@ static const simplecpp::Token *gotoNextLine(const simplecpp::Token *tok)
{
const unsigned int line = tok->location.line;
const unsigned int file = tok->location.fileIndex;
while (tok && tok->location.line == line && tok->location.fileIndex == file)
if (tok)
tok = tok->next;
while (tok && tok->location.line == line && tok->location.fileIndex == file && !tok->locationchange)
tok = tok->next;
return tok;
}
Expand DownExpand Up@@ -3541,7 +3532,7 @@ void simplecpp::preprocess(simplecpp::TokenList &output, const simplecpp::TokenL
continue;
}

if (rawtok->op == '#' && !sameline(rawtok->previousSkipComments(), rawtok)) {
if (rawtok->op == '#' && (!sameline(rawtok->previousSkipComments(), rawtok) || rawtok->locationchange)) {
if (!sameline(rawtok, rawtok->next)) {
rawtok = rawtok->next;
continue;
Expand Down
5 changes: 3 additions & 2 deletions simplecpp.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -156,7 +156,7 @@ namespace simplecpp {
}

Token(const Token &tok) :
macro(tok.macro), op(tok.op), comment(tok.comment), name(tok.name), number(tok.number), whitespaceahead(tok.whitespaceahead), location(tok.location), string(tok.string), mExpandedFrom(tok.mExpandedFrom) {}
macro(tok.macro), op(tok.op), comment(tok.comment), name(tok.name), number(tok.number), whitespaceahead(tok.whitespaceahead), locationchange(tok.locationchange), location(tok.location), string(tok.string), mExpandedFrom(tok.mExpandedFrom) {}

Token &operator=(const Token &tok) = delete;

Expand All@@ -182,6 +182,8 @@ namespace simplecpp {
bool name;
bool number;
bool whitespaceahead;
/** whether token location is at a discontinuity from a #line directive */
bool locationchange {false};
Location location;
Token *previous{};
Token *next{};
Expand DownExpand Up@@ -396,7 +398,6 @@ namespace simplecpp {
void constFoldQuestionOp(Token *&tok1);

std::string readUntil(Stream &stream, const Location &location, char start, char end, OutputList *outputList);
void lineDirective(unsigned int fileIndex_, unsigned int line, Location &location);

const Token* lastLineTok(int maxsize=1000) const;
const Token* isLastLinePreprocessor(int maxsize=1000) const;
Expand Down
59 changes: 59 additions & 0 deletions test.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -2749,6 +2749,62 @@ static void nullDirective3()
ASSERT_EQUALS("\n\n\n\nx = 1 ;", preprocess(code));
}

static void lineDirective1()
{
const char code[] = "#line 1\n"
"#line 2 \"header1.h\"\n"
"# 1\n"
"# 2 \"header2.h\"\n"
"# 3 \"header2.h\" 0\n";

std::vector<std::string> files;
const simplecpp::TokenList rawtokens(code, files);

ASSERT_EQUALS("1: # line 1 # line 2 \"header1.h\"\n"
"#line 2 \"header1.h\"\n"
"2: # 1\n"
"#line 1 \"header1.h\"\n"
"1: # 2 \"header2.h\"\n"
"#line 2 \"header2.h\"\n"
"2: # 3 \"header2.h\" 0",
rawtokens.stringify(true));

simplecpp::FileDataCache cache;
simplecpp::TokenList out(files);
const simplecpp::DUI dui;
simplecpp::preprocess(out, rawtokens, files, cache, dui);

ASSERT_EQUALS("", out.stringify(true));
}

static void lineDirective2()
{
const char code[] = "#\n"
"#line 1\n"
"include\n"
"#line 1\n"
"\"header1.h\"\n";

std::vector<std::string> files;
const simplecpp::TokenList rawtokens(code, files);

ASSERT_EQUALS("1: #\n"
"2: # line 1\n"
"#line 1 \"\"\n"
"1: include\n"
"2: # line 1\n"
"#line 1 \"\"\n"
"1: \"header1.h\"",
rawtokens.stringify(true));

simplecpp::FileDataCache cache;
simplecpp::TokenList out(files);
const simplecpp::DUI dui;
simplecpp::preprocess(out, rawtokens, files, cache, dui);

ASSERT_EQUALS("1: include \"header1.h\"", out.stringify(true));
}

static void include1()
{
const char code[] = "#include \"A.h\"\n";
Expand DownExpand Up@@ -4167,6 +4223,9 @@ static void runTests(int argc, char **argv, Input input)
TEST_CASE(nullDirective2);
TEST_CASE(nullDirective3);

TEST_CASE(lineDirective1);
TEST_CASE(lineDirective2);

TEST_CASE(include1);
TEST_CASE(include2);
TEST_CASE(include3);
Expand Down
Loading