gbdev/rgbds
1
0
Fork 0
mirror of https://github.com/gbdev/rgbds.git synced 2025-11-20 10:12:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor to keep lexerState and lexerStateEOL static

Browse Source 
Also run `clang-format` on everything
This commit is contained in:
Rangi42
2024-03-23 16:42:28 -04:00
parent c075ffb570
commit e9e8915725
9 changed files with 179 additions and 198 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
include/asm/fstack.hpp
View File

@@ -24,8 +24,9 @@ struct FileStackNode {
data;
std::shared_ptr<FileStackNode> parent; // Pointer to parent node, for error reporting
// Line at which the parent context was exited; meaningless for the root level
uint32_t lineNo;
// Line at which the parent context was exited
// Meaningless at the root level, but gets written to the object file anyway, so init it
uint32_t lineNo = 0;
// Set only if referenced: ID within the object file, -1 if not output yet
uint32_t ID = -1;
@@ -61,7 +62,7 @@ void fstk_SetPreIncludeFile(std::string const &path);
std::optional<std::string> fstk_FindFile(std::string const &path);
bool yywrap();
void fstk_RunInclude(std::string const &path);
void fstk_RunInclude(std::string const &path, bool updateStateNow);
void fstk_RunMacro(std::string const &macroName, std::shared_ptr<MacroArgs> macroArgs);
void fstk_RunRept(uint32_t count, int32_t reptLineNo, char const *body, size_t size);
void fstk_RunFor(

22
include/asm/lexer.hpp
View File

@@ -94,19 +94,14 @@ struct LexerState {
LexerState &operator=(LexerState &&) = default;
LexerState &operator=(LexerState const &) = delete;
void setAsCurrentState();
bool setFileAsNextState(std::string const &filePath, bool updateStateNow);
void setViewAsNextState(char const *filePath, char const *buf, size_t size, uint32_t lineNo_);
void clear(uint32_t lineNo_);
};
extern LexerState *lexerState;
extern LexerState *lexerStateEOL;
static inline void lexer_SetState(LexerState *state) {
lexerState = state;
}
static inline void lexer_SetStateAtEOL(LexerState *state) {
lexerStateEOL = state;
}
extern char binDigits[2];
extern char gfxDigits[4];
@@ -122,11 +117,6 @@ static inline void lexer_SetGfxDigits(char const digits[4]) {
gfxDigits[3] = digits[3];
}
// `path` is referenced, but not held onto..!
bool lexer_OpenFile(LexerState &state, std::string const &path);
void lexer_OpenFileView(
LexerState &state, char const *path, char const *buf, size_t size, uint32_t lineNo
);
void lexer_RestartRept(uint32_t lineNo);
void lexer_Init();
void lexer_SetMode(LexerMode mode);

1
include/asm/macro.hpp
View File

@@ -20,5 +20,4 @@ struct MacroArgs {
void shiftArgs(int32_t count);
};
#endif // RGBDS_MACRO_H

241
src/asm/fstack.cpp
View File

@@ -34,7 +34,7 @@ struct Context {
// for this context yet, and it should be generated.
// Note that several contexts can share the same unique ID (since `INCLUDE` preserves its
// parent's, and likewise "back-propagates" a unique ID if requested), hence using `shared_ptr`.
std::shared_ptr<std::string> uniqueIDStr;
std::shared_ptr<std::string> uniqueIDStr = nullptr;
std::shared_ptr<MacroArgs> macroArgs = nullptr; // Macro args are *saved* here
uint32_t nbReptIters = 0;
bool isForLoop = false;
@@ -130,7 +130,6 @@ void fstk_AddIncludePath(std::string const &path) {
return;
std::string &includePath = includePaths.emplace_back(path);
if (includePath.back() != '/')
includePath += '/';
}
@@ -151,20 +150,14 @@ static void printDep(std::string const &path) {
}
}
static bool isPathValid(std::string const &path) {
struct stat statbuf;
if (stat(path.c_str(), &statbuf) != 0)
return false;
// Reject directories
return !S_ISDIR(statbuf.st_mode);
static bool isValidFilePath(std::string const &path) {
struct stat statBuf;
return stat(path.c_str(), &statBuf) == 0 && !S_ISDIR(statBuf.st_mode); // Reject directories
}
std::optional<std::string> fstk_FindFile(std::string const &path) {
for (std::string &str : includePaths) {
std::string fullPath = str + path;
if (isPathValid(fullPath)) {
for (std::string &incPath : includePaths) {
if (std::string fullPath = incPath + path; isValidFilePath(fullPath)) {
printDep(fullPath);
return fullPath;
}
@@ -221,34 +214,114 @@ bool yywrap() {
}
contextStack.pop();
lexer_SetState(&contextStack.top().lexerState);
contextStack.top().lexerState.setAsCurrentState();
return false;
}
// Make sure not to switch the lexer state before calling this, so the saved line no is correct.
// BE CAREFUL! This modifies the file stack directly, you should have set up the file info first.
// Callers should set `contextStack.top().lexerState` after this so it is not `nullptr`.
static Context &newContext(std::shared_ptr<FileStackNode> fileInfo) {
static void checkRecursionDepth() {
if (contextStack.size() > maxRecursionDepth)
fatalerror("Recursion limit (%zu) exceeded\n", maxRecursionDepth);
fileInfo->parent = contextStack.top().fileInfo;
fileInfo->lineNo = lexer_GetLineNo();
return contextStack.emplace(Context{
.fileInfo = fileInfo,
.uniqueIDStr = fileInfo->generatesUniqueID()
? std::make_shared<std::string>() // Create a new, not-yet-generated ID.
: contextStack.top().uniqueIDStr, // Make a copy.
.macroArgs = contextStack.top().macroArgs,
});
}
void fstk_RunInclude(std::string const &path) {
static bool newFileContext(std::string const &filePath, bool updateStateNow) {
checkRecursionDepth();
std::shared_ptr<std::string> uniqueIDStr = nullptr;
std::shared_ptr<MacroArgs> macroArgs = nullptr;
auto fileInfo =
std::make_shared<FileStackNode>(NODE_MACRO, filePath == "-" ? "<stdin>" : filePath);
if (!contextStack.empty()) {
Context &oldContext = contextStack.top();
fileInfo->parent = oldContext.fileInfo;
fileInfo->lineNo = lexer_GetLineNo(); // Called before setting the lexer state
uniqueIDStr = oldContext.uniqueIDStr; // Make a copy of the ID
macroArgs = oldContext.macroArgs;
}
Context &context = contextStack.emplace(Context{
.fileInfo = fileInfo,
.uniqueIDStr = uniqueIDStr,
.macroArgs = macroArgs,
});
return context.lexerState.setFileAsNextState(filePath, updateStateNow);
}
static void newMacroContext(Symbol const &macro, std::shared_ptr<MacroArgs> macroArgs) {
checkRecursionDepth();
Context &oldContext = contextStack.top();
std::string fileInfoName;
for (FileStackNode const *node = macro.src.get(); node; node = node->parent.get()) {
if (node->type != NODE_REPT) {
fileInfoName.append(node->name());
break;
}
}
if (macro.src->type == NODE_REPT) {
std::vector<uint32_t> const &srcIters = macro.src->iters();
for (uint32_t i = srcIters.size(); i--;) {
fileInfoName.append("::REPT~");
fileInfoName.append(std::to_string(srcIters[i]));
}
}
fileInfoName.append("::");
fileInfoName.append(macro.name);
auto fileInfo = std::make_shared<FileStackNode>(NODE_MACRO, fileInfoName);
assert(!contextStack.empty()); // The top level context cannot be a MACRO
fileInfo->parent = oldContext.fileInfo;
fileInfo->lineNo = lexer_GetLineNo();
Context &context = contextStack.emplace(Context{
.fileInfo = fileInfo,
.uniqueIDStr = std::make_shared<std::string>(), // Create a new, not-yet-generated ID
.macroArgs = macroArgs,
});
std::string_view *macroView = macro.getMacro();
context.lexerState.setViewAsNextState(
"MACRO", macroView->data(), macroView->size(), macro.fileLine
);
}
static Context &newReptContext(int32_t reptLineNo, char const *body, size_t size, uint32_t count) {
checkRecursionDepth();
Context &oldContext = contextStack.top();
std::vector<uint32_t> fileInfoIters{1};
if (oldContext.fileInfo->type == NODE_REPT && !oldContext.fileInfo->iters().empty()) {
// Append all parent iter counts
fileInfoIters.insert(fileInfoIters.end(), RANGE(oldContext.fileInfo->iters()));
}
auto fileInfo = std::make_shared<FileStackNode>(NODE_REPT, fileInfoIters);
assert(!contextStack.empty()); // The top level context cannot be a REPT
fileInfo->parent = oldContext.fileInfo;
fileInfo->lineNo = reptLineNo;
Context &context = contextStack.emplace(Context{
.fileInfo = fileInfo,
.uniqueIDStr = std::make_shared<std::string>(), // Create a new, not-yet-generated ID
.macroArgs = oldContext.macroArgs,
});
context.lexerState.setViewAsNextState("REPT", body, size, reptLineNo);
context.nbReptIters = count;
return context;
}
void fstk_RunInclude(std::string const &path, bool preInclude) {
std::optional<std::string> fullPath = fstk_FindFile(path);
if (!fullPath) {
if (generatedMissingIncludes) {
if (generatedMissingIncludes && !preInclude) {
if (verbose)
printf("Aborting (-MG) on INCLUDE file '%s' (%s)\n", path.c_str(), strerror(errno));
failedOnMissingInclude = true;
@@ -258,30 +331,8 @@ void fstk_RunInclude(std::string const &path) {
return;
}
auto fileInfo = std::make_shared<FileStackNode>(NODE_FILE, *fullPath);
Context &context = newContext(fileInfo);
if (!lexer_OpenFile(context.lexerState, fileInfo->name()))
if (!newFileContext(*fullPath, false))
fatalerror("Failed to set up lexer for file include\n");
lexer_SetStateAtEOL(&context.lexerState);
}
// Similar to `fstk_RunInclude`, but not subject to `-MG`, and
// calling `lexer_SetState` instead of `lexer_SetStateAtEOL`.
static void runPreIncludeFile() {
if (preIncludeName.empty())
return;
std::optional<std::string> fullPath = fstk_FindFile(preIncludeName);
if (!fullPath) {
error("Unable to open included file '%s': %s\n", preIncludeName.c_str(), strerror(errno));
return;
}
auto fileInfo = std::make_shared<FileStackNode>(NODE_FILE, *fullPath);
Context &context = newContext(fileInfo);
if (!lexer_OpenFile(context.lexerState, fileInfo->name()))
fatalerror("Failed to set up lexer for file include\n");
lexer_SetState(&context.lexerState);
}
void fstk_RunMacro(std::string const &macroName, std::shared_ptr<MacroArgs> macroArgs) {
@@ -296,63 +347,14 @@ void fstk_RunMacro(std::string const &macroName, std::shared_ptr<MacroArgs> macr
return;
}
auto fileInfo = std::make_shared<FileStackNode>(NODE_MACRO, "");
// Print the name...
std::string &fileInfoName = fileInfo->name();
for (FileStackNode const *node = macro->src.get(); node; node = node->parent.get()) {
if (node->type != NODE_REPT) {
fileInfoName.append(node->name());
break;
}
}
if (macro->src->type == NODE_REPT) {
std::vector<uint32_t> const &srcIters = macro->src->iters();
for (uint32_t i = srcIters.size(); i--;) {
fileInfoName.append("::REPT~");
fileInfoName.append(std::to_string(srcIters[i]));
}
}
fileInfoName.append("::");
fileInfoName.append(macro->name);
Context &context = newContext(fileInfo);
std::string_view *macroView = macro->getMacro();
lexer_OpenFileView(
context.lexerState, "MACRO", macroView->data(), macroView->size(), macro->fileLine
);
lexer_SetStateAtEOL(&context.lexerState);
context.macroArgs = macroArgs;
}
static bool newReptContext(int32_t reptLineNo, char const *body, size_t size) {
auto fileInfo = std::make_shared<FileStackNode>(NODE_REPT, std::vector<uint32_t>{1});
if (contextStack.top().fileInfo->type == NODE_REPT
&& !contextStack.top().fileInfo->iters().empty()) {
// Append all parent iter counts
fileInfo->iters().insert(
fileInfo->iters().end(), RANGE(contextStack.top().fileInfo->iters())
);
}
Context &context = newContext(fileInfo);
// Correct our line number, which currently points to the `ENDR` line
context.fileInfo->lineNo = reptLineNo;
lexer_OpenFileView(context.lexerState, "REPT", body, size, reptLineNo);
lexer_SetStateAtEOL(&context.lexerState);
return true;
newMacroContext(*macro, macroArgs);
}
void fstk_RunRept(uint32_t count, int32_t reptLineNo, char const *body, size_t size) {
if (count == 0)
return;
if (!newReptContext(reptLineNo, body, size))
return;
contextStack.top().nbReptIters = count;
newReptContext(reptLineNo, body, size, count);
}
void fstk_RunFor(
@@ -364,13 +366,10 @@ void fstk_RunFor(
char const *body,
size_t size
) {
Symbol *sym = sym_AddVar(symName, start);
if (sym->type != SYM_VAR)
if (Symbol *sym = sym_AddVar(symName, start); sym->type != SYM_VAR)
return;
uint32_t count = 0;
if (step > 0 && start < stop)
count = ((int64_t)stop - start - 1) / step + 1;
else if (step < 0 && stop < start)
@@ -385,12 +384,8 @@ void fstk_RunFor(
if (count == 0)
return;
if (!newReptContext(reptLineNo, body, size))
return;
Context &context = contextStack.top();
context.nbReptIters = count;
Context &context = newReptContext(reptLineNo, body, size, count);
context.isForLoop = true;
context.forValue = start;
context.forStep = step;
@@ -398,8 +393,7 @@ void fstk_RunFor(
}
void fstk_StopRept() {
// Prevent more iterations
contextStack.top().nbReptIters = 0;
contextStack.top().nbReptIters = 0; // Prevent more iterations
}
bool fstk_Break() {
@@ -419,20 +413,11 @@ void fstk_NewRecursionDepth(size_t newDepth) {
}
void fstk_Init(std::string const &mainPath, size_t maxDepth) {
Context &context = contextStack.emplace(Context{
.fileInfo = nullptr, // We're going to init it just below.
.uniqueIDStr = nullptr, // `\@` is not allowed at top level.
});
if (!lexer_OpenFile(context.lexerState, mainPath))
if (!newFileContext(mainPath, true))
fatalerror("Failed to open main file\n");
lexer_SetState(&context.lexerState);
context.fileInfo = std::make_shared<FileStackNode>(NODE_FILE, context.lexerState.path);
// lineNo and nbReptIters are unused on the top-level context
context.fileInfo->parent = nullptr;
context.fileInfo->lineNo = 0; // This still gets written to the object file, so init it
maxRecursionDepth = maxDepth;
runPreIncludeFile();
if (!preIncludeName.empty())
fstk_RunInclude(preIncludeName, true);
}

87
src/asm/lexer.cpp
View File

@@ -322,25 +322,27 @@ static bool isWhitespace(int c) {
return c == ' ' || c == '\t';
}
LexerState *lexerState = nullptr;
LexerState *lexerStateEOL = nullptr;
static LexerState *lexerState = nullptr;
static LexerState *lexerStateEOL = nullptr;
static void initState(LexerState &state) {
state.mode = LEXER_NORMAL;
state.atLineStart = true; // yylex() will init colNo due to this
state.lastToken = T_(YYEOF);
void LexerState::clear(uint32_t lineNo_) {
mode = LEXER_NORMAL;
atLineStart = true; // yylex() will init colNo due to this
lastToken = T_(YYEOF);
state.ifStack.clear();
ifStack.clear();
state.capturing = false;
state.captureBuf = nullptr;
capturing = false;
captureBuf = nullptr;
state.disableMacroArgs = false;
state.disableInterpolation = false;
state.macroArgScanDistance = 0;
state.expandStrings = true;
disableMacroArgs = false;
disableInterpolation = false;
macroArgScanDistance = 0;
expandStrings = true;
state.expansions.clear();
expansions.clear();
lineNo = lineNo_; // Will be incremented at next line start
}
static void nextLine() {
@@ -379,19 +381,23 @@ void lexer_ReachELSEBlock() {
lexerState->ifStack.front().reachedElseBlock = true;
}
bool lexer_OpenFile(LexerState &state, std::string const &path) {
if (path == "-") {
state.path = "<stdin>";
state.content = BufferedLexerState{.fd = STDIN_FILENO, .index = 0, .buf = {}, .nbChars = 0};
void LexerState::setAsCurrentState() {
lexerState = this;
}
bool LexerState::setFileAsNextState(std::string const &filePath, bool updateStateNow) {
if (filePath == "-") {
path = "<stdin>";
content = BufferedLexerState{.fd = STDIN_FILENO, .index = 0, .buf = {}, .nbChars = 0};
if (verbose)
printf("Opening stdin\n");
} else {
struct stat fileInfo;
if (stat(path.c_str(), &fileInfo) != 0) {
error("Failed to stat file \"%s\": %s\n", path.c_str(), strerror(errno));
struct stat statBuf;
if (stat(filePath.c_str(), &statBuf) != 0) {
error("Failed to stat file \"%s\": %s\n", filePath.c_str(), strerror(errno));
return false;
}
state.path = path;
path = filePath;
int fd = open(path.c_str(), O_RDONLY);
if (fd < 0) {
@@ -401,16 +407,16 @@ bool lexer_OpenFile(LexerState &state, std::string const &path) {
bool isMmapped = false;
if (fileInfo.st_size > 0) {
if (statBuf.st_size > 0) {
// Try using `mmap` for better performance
void *mappingAddr;
mapFile(mappingAddr, fd, path, fileInfo.st_size);
mapFile(mappingAddr, fd, path, statBuf.st_size);
if (mappingAddr != MAP_FAILED) {
close(fd);
state.content = MmappedLexerState{
content = MmappedLexerState{
.ptr = (char *)mappingAddr,
.size = (size_t)fileInfo.st_size,
.size = (size_t)statBuf.st_size,
.offset = 0,
.isReferenced = false,
};
@@ -422,9 +428,9 @@ bool lexer_OpenFile(LexerState &state, std::string const &path) {
if (!isMmapped) {
// Sometimes mmap() fails or isn't available, so have a fallback
state.content = BufferedLexerState{.fd = fd, .index = 0, .buf = {}, .nbChars = 0};
content = BufferedLexerState{.fd = fd, .index = 0, .buf = {}, .nbChars = 0};
if (verbose) {
if (fileInfo.st_size == 0) {
if (statBuf.st_size == 0) {
printf("File \"%s\" is empty\n", path.c_str());
} else {
printf(
@@ -435,18 +441,21 @@ bool lexer_OpenFile(LexerState &state, std::string const &path) {
}
}
initState(state);
state.lineNo = 0; // Will be incremented at first line start
clear(0);
if (updateStateNow)
lexerState = this;
else
lexerStateEOL = this;
return true;
}
void lexer_OpenFileView(
LexerState &state, char const *path, char const *buf, size_t size, uint32_t lineNo
void LexerState::setViewAsNextState(
char const *filePath, char const *buf, size_t size, uint32_t lineNo_
) {
state.path = path; // Used to report read errors in `peekInternal`
state.content = ViewedLexerState{.ptr = buf, .size = size, .offset = 0};
initState(state);
state.lineNo = lineNo; // Will be incremented at first line start
path = filePath; // Used to report read errors in `peekInternal`
content = ViewedLexerState{.ptr = buf, .size = size, .offset = 0};
clear(lineNo_);
lexerStateEOL = this;
}
void lexer_RestartRept(uint32_t lineNo) {
@@ -455,8 +464,7 @@ void lexer_RestartRept(uint32_t lineNo) {
} else if (auto *view = std::get_if<ViewedLexerState>(&lexerState->content); view) {
view->offset = 0;
}
initState(*lexerState);
lexerState->lineNo = lineNo;
lexerState->clear(lineNo);
}
LexerState::~LexerState() {
@@ -2151,10 +2159,9 @@ finish:
yy::parser::symbol_type yylex() {
if (lexerState->atLineStart && lexerStateEOL) {
lexer_SetState(lexerStateEOL);
lexerState = lexerStateEOL;
lexerStateEOL = nullptr;
}
// `lexer_SetState` updates `lexerState`, so check for EOF after it
if (lexerState->lastToken == T_(EOB) && yywrap())
return yy::parser::make_YYEOF();
// Newlines read within an expansion should not increase the line count

3
src/asm/macro.cpp
View File

@@ -54,7 +54,8 @@ void MacroArgs::appendArg(std::shared_ptr<std::string> arg) {
}
void MacroArgs::shiftArgs(int32_t count) {
if (size_t nbArgs = args.size(); count > 0 && ((uint32_t)count > nbArgs || shift > nbArgs - count)) {
if (size_t nbArgs = args.size();
count > 0 && ((uint32_t)count > nbArgs || shift > nbArgs - count)) {
warning(WARNING_MACRO_SHIFT, "Cannot shift macro arguments past their end\n");
shift = nbArgs;
} else if (count < 0 && shift < (uint32_t)-count) {

2
src/asm/parser.y
View File

@@ -1128,7 +1128,7 @@ export_list_entry:
include:
label POP_INCLUDE string endofline {
fstk_RunInclude($3);
fstk_RunInclude($3, false);
if (failedOnMissingInclude)
YYACCEPT;
}

12
src/asm/symbol.cpp
View File

@@ -594,13 +594,11 @@ void sym_Init(time_t now) {
sym_AddString("__TIME__"s, std::make_shared<std::string>(savedTIME))->isBuiltin = true;
sym_AddString("__DATE__"s, std::make_shared<std::string>(savedDATE))->isBuiltin = true;
sym_AddString(
"__ISO_8601_LOCAL__"s,
std::make_shared<std::string>(savedTIMESTAMP_ISO8601_LOCAL)
)->isBuiltin = true;
sym_AddString(
"__ISO_8601_UTC__"s,
std::make_shared<std::string>(savedTIMESTAMP_ISO8601_UTC)
)->isBuiltin = true;
"__ISO_8601_LOCAL__"s, std::make_shared<std::string>(savedTIMESTAMP_ISO8601_LOCAL)
)
->isBuiltin = true;
sym_AddString("__ISO_8601_UTC__"s, std::make_shared<std::string>(savedTIMESTAMP_ISO8601_UTC))
->isBuiltin = true;
sym_AddEqu("__UTC_YEAR__"s, time_utc->tm_year + 1900)->isBuiltin = true;
sym_AddEqu("__UTC_MONTH__"s, time_utc->tm_mon + 1)->isBuiltin = true;