Skip to content

made it possible to create a TokenList from a buffer#347

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
firewave merged 3 commits into from
May 24, 2024
Merged

made it possible to create a TokenList from a buffer #347

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1696577
added `StdCharBufStream` which reads from a buffer
firewave Mar 29, 2024
1cb98c8
use `StdCharBufStream` in `Macro`
firewave Mar 29, 2024
1895980
added `TokenList` constructors which take a buffer
firewave Mar 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 51 additions & 2 deletions simplecpp.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -377,6 +377,42 @@ class StdIStream : public simplecpp::TokenList::Stream {
std::istream &istr;
};

class StdCharBufStream : public simplecpp::TokenList::Stream {
Comment thread
danmar marked this conversation as resolved.
public:
// cppcheck-suppress uninitDerivedMemberVar - we call Stream::init() to initialize the private members
StdCharBufStream(const unsigned char* str, std::size_t size)
: str(str)
, size(size)
, pos(0)
, lastStatus(0)
{
init();
}

virtual int get() OVERRIDE {
if (pos >= size)

@firewavefirewaveMay 3, 2024

Copy link
Copy Markdown
CollaboratorAuthor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These checks are still a hot spot. No idea if there is a way to implement this in a more performant way.

return lastStatus = EOF;
return str[pos++];
}
virtual int peek() OVERRIDE {
if (pos >= size)
return lastStatus = EOF;
return str[pos];
}
virtual void unget() OVERRIDE {
--pos;
Comment thread
danmar marked this conversation as resolved.
}
virtual bool good() OVERRIDE {
return lastStatus != EOF;
}

private:
const unsigned char *str;
Comment thread
danmar marked this conversation as resolved.
const std::size_t size;
std::size_t pos;
int lastStatus;
};

class FileStream : public simplecpp::TokenList::Stream {
public:
// cppcheck-suppress uninitDerivedMemberVar - we call Stream::init() to initialize the private members
Expand DownExpand Up@@ -442,6 +478,20 @@ simplecpp::TokenList::TokenList(std::istream &istr, std::vector<std::string> &fi
readfile(stream,filename,outputList);
}

simplecpp::TokenList::TokenList(const unsigned char* data, std::size_t size, std::vector<std::string> &filenames, const std::string &filename, OutputList *outputList)
: frontToken(nullptr), backToken(nullptr), files(filenames)
{
StdCharBufStream stream(data, size);
readfile(stream,filename,outputList);
}

simplecpp::TokenList::TokenList(const char* data, std::size_t size, std::vector<std::string> &filenames, const std::string &filename, OutputList *outputList)
: frontToken(nullptr), backToken(nullptr), files(filenames)
{
StdCharBufStream stream(reinterpret_cast<const unsigned char*>(data), size);
readfile(stream,filename,outputList);
}

simplecpp::TokenList::TokenList(const std::string &filename, std::vector<std::string> &filenames, OutputList *outputList)
: frontToken(nullptr), backToken(nullptr), files(filenames)
{
Expand DownExpand Up@@ -1447,8 +1497,7 @@ namespace simplecpp {

Macro(const std::string &name, const std::string &value, std::vector<std::string> &f) : nameTokDef(nullptr), files(f), tokenListDefine(f), valueDefinedInCode_(false) {
const std::string def(name + ' ' + value);
std::istringstream istr(def);
StdIStream stream(istr);
StdCharBufStream stream(reinterpret_cast<const unsigned char*>(def.data()), def.size());
tokenListDefine.readfile(stream);
if (!parseDefine(tokenListDefine.cfront()))
throw std::runtime_error("bad macro syntax. macroname=" + name + " value=" + value);
Expand Down
4 changes: 4 additions & 0 deletions simplecpp.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -198,6 +198,10 @@ namespace simplecpp {
explicit TokenList(std::vector<std::string> &filenames);
/** generates a token list from the given std::istream parameter */
TokenList(std::istream &istr, std::vector<std::string> &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr);
/** generates a token list from the given buffer */
TokenList(const unsigned char* data, std::size_t size, std::vector<std::string> &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr);
/** generates a token list from the given buffer */
TokenList(const char* data, std::size_t size, std::vector<std::string> &filenames, const std::string &filename=std::string(), OutputList *outputList = nullptr);
/** generates a token list from the given filename parameter */
TokenList(const std::string &filename, std::vector<std::string> &filenames, OutputList *outputList = nullptr);
TokenList(const TokenList &other);
Expand Down