rgbds/src/asm/lexer.c

/*
 * This file is part of RGBDS.
 *
 * Copyright (c) 2020, Eldred Habert and RGBDS contributors.
 *
 * SPDX-License-Identifier: MIT
 */

#include <sys/types.h>
#include <sys/stat.h>
#include <assert.h>
#include <ctype.h>
#include <errno.h>
#include <fcntl.h>
#include <inttypes.h>
#include <math.h>
#include <limits.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#ifndef _MSC_VER
#include <unistd.h>
#endif

#include "extern/utf8decoder.h"
#include "platform.h" /* For `ssize_t` */

#include "asm/lexer.h"
#include "asm/format.h"
#include "asm/fstack.h"
#include "asm/macro.h"
#include "asm/main.h"
#include "asm/rpn.h"
#include "asm/symbol.h"
#include "asm/util.h"
#include "asm/warning.h"
/* Include this last so it gets all type & constant definitions */
#include "parser.h" /* For token definitions, generated from parser.y */

#ifdef LEXER_DEBUG
  #define dbgPrint(...) fprintf(stderr, "[lexer] " __VA_ARGS__)
#else
  #define dbgPrint(...)
#endif

/* Neither MSVC nor MinGW provide `mmap` */
#if defined(_MSC_VER) || defined(__MINGW32__)
# define WIN32_LEAN_AND_MEAN // include less from windows.h
# include <windows.h> // target architecture
# include <fileapi.h> // CreateFileA
# include <winbase.h> // CreateFileMappingA
# include <memoryapi.h> // MapViewOfFile
# include <handleapi.h> // CloseHandle
# define MAP_FAILED NULL
# define mapFile(ptr, fd, path, size) do { \
	(ptr) = MAP_FAILED; \
	HANDLE file = CreateFileA(path, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, \
				  FILE_FLAG_POSIX_SEMANTICS | FILE_FLAG_RANDOM_ACCESS, NULL); \
	HANDLE mappingObj; \
	\
	if (file == INVALID_HANDLE_VALUE) \
		break; \
	mappingObj  = CreateFileMappingA(file, NULL, PAGE_READONLY, 0, 0, NULL); \
	if (mappingObj != INVALID_HANDLE_VALUE) \
		(ptr) = MapViewOfFile(mappingObj, FILE_MAP_READ, 0, 0, 0); \
	CloseHandle(mappingObj); \
	CloseHandle(file); \
} while (0)
# define munmap(ptr, size)  UnmapViewOfFile((ptr))

#else /* defined(_MSC_VER) || defined(__MINGW32__) */

# include <sys/mman.h>
# define mapFile(ptr, fd, path, size) do { \
	(ptr) = mmap(NULL, (size), PROT_READ, MAP_PRIVATE, (fd), 0); \
	\
	if ((ptr) == MAP_FAILED && errno == ENOTSUP) { \
		/*
		 * The implementation may not support MAP_PRIVATE; try again with MAP_SHARED
		 * instead, offering, I believe, weaker guarantees about external modifications to
		 * the file while reading it. That's still better than not opening it at all, though
		 */ \
		if (verbose) \
			printf("mmap(%s, MAP_PRIVATE) failed, retrying with MAP_SHARED\n", path); \
		(ptr) = mmap(NULL, (size), PROT_READ, MAP_SHARED, (fd), 0); \
	} \
} while (0)
#endif /* !( defined(_MSC_VER) || defined(__MINGW32__) ) */

/*
 * Identifiers that are also keywords are listed here. This ONLY applies to ones
 * that would normally be matched as identifiers! Check out `yylex_NORMAL` to
 * see how this is used.
 * Tokens / keywords not handled here are handled in `yylex_NORMAL`'s switch.
 */
static struct KeywordMapping {
	char const *name;
	int token;
} const keywords[] = {
	/*
	 * CAUTION when editing this: adding keywords will probably require extra nodes in the
	 * `keywordDict` array. If you forget to, you will probably trip up an assertion, anyways.
	 * Also, all entries in this array must be in uppercase for the dict to build correctly.
	 */
	{"ADC", T_Z80_ADC},
	{"ADD", T_Z80_ADD},
	{"AND", T_Z80_AND},
	{"BIT", T_Z80_BIT},
	{"CALL", T_Z80_CALL},
	{"CCF", T_Z80_CCF},
	{"CPL", T_Z80_CPL},
	{"CP", T_Z80_CP},
	{"DAA", T_Z80_DAA},
	{"DEC", T_Z80_DEC},
	{"DI", T_Z80_DI},
	{"EI", T_Z80_EI},
	{"HALT", T_Z80_HALT},
	{"INC", T_Z80_INC},
	{"JP", T_Z80_JP},
	{"JR", T_Z80_JR},
	{"LD", T_Z80_LD},
	{"LDI", T_Z80_LDI},
	{"LDD", T_Z80_LDD},
	{"LDIO", T_Z80_LDH},
	{"LDH", T_Z80_LDH},
	{"NOP", T_Z80_NOP},
	{"OR", T_Z80_OR},
	{"POP", T_Z80_POP},
	{"PUSH", T_Z80_PUSH},
	{"RES", T_Z80_RES},
	{"RETI", T_Z80_RETI},
	{"RET", T_Z80_RET},
	{"RLCA", T_Z80_RLCA},
	{"RLC", T_Z80_RLC},
	{"RLA", T_Z80_RLA},
	{"RL", T_Z80_RL},
	{"RRC", T_Z80_RRC},
	{"RRCA", T_Z80_RRCA},
	{"RRA", T_Z80_RRA},
	{"RR", T_Z80_RR},
	{"RST", T_Z80_RST},
	{"SBC", T_Z80_SBC},
	{"SCF", T_Z80_SCF},
	{"SET", T_POP_SET},
	{"SLA", T_Z80_SLA},
	{"SRA", T_Z80_SRA},
	{"SRL", T_Z80_SRL},
	{"STOP", T_Z80_STOP},
	{"SUB", T_Z80_SUB},
	{"SWAP", T_Z80_SWAP},
	{"XOR", T_Z80_XOR},

	{"NZ", T_CC_NZ},
	{"Z", T_CC_Z},
	{"NC", T_CC_NC},
	/* Handled after as T_TOKEN_C */
	/* { "C", T_CC_C }, */

	{"AF", T_MODE_AF},
	{"BC", T_MODE_BC},
	{"DE", T_MODE_DE},
	{"HL", T_MODE_HL},
	{"SP", T_MODE_SP},
	{"HLD", T_MODE_HL_DEC},
	{"HLI", T_MODE_HL_INC},

	{"A", T_TOKEN_A},
	{"B", T_TOKEN_B},
	{"C", T_TOKEN_C},
	{"D", T_TOKEN_D},
	{"E", T_TOKEN_E},
	{"H", T_TOKEN_H},
	{"L", T_TOKEN_L},

	{"DEF", T_OP_DEF},

	{"FRAGMENT", T_POP_FRAGMENT},
	{"BANK", T_OP_BANK},
	{"ALIGN", T_OP_ALIGN},

	{"SIZEOF", T_OP_SIZEOF},
	{"STARTOF", T_OP_STARTOF},

	{"ROUND", T_OP_ROUND},
	{"CEIL", T_OP_CEIL},
	{"FLOOR", T_OP_FLOOR},
	{"DIV", T_OP_FDIV},
	{"MUL", T_OP_FMUL},
	{"POW", T_OP_POW},
	{"LOG", T_OP_LOG},
	{"SIN", T_OP_SIN},
	{"COS", T_OP_COS},
	{"TAN", T_OP_TAN},
	{"ASIN", T_OP_ASIN},
	{"ACOS", T_OP_ACOS},
	{"ATAN", T_OP_ATAN},
	{"ATAN2", T_OP_ATAN2},

	{"HIGH", T_OP_HIGH},
	{"LOW", T_OP_LOW},
	{"ISCONST", T_OP_ISCONST},

	{"STRCMP", T_OP_STRCMP},
	{"STRIN", T_OP_STRIN},
	{"STRRIN", T_OP_STRRIN},
	{"STRSUB", T_OP_STRSUB},
	{"STRLEN", T_OP_STRLEN},
	{"STRCAT", T_OP_STRCAT},
	{"STRUPR", T_OP_STRUPR},
	{"STRLWR", T_OP_STRLWR},
	{"STRRPL", T_OP_STRRPL},
	{"STRFMT", T_OP_STRFMT},

	{"CHARLEN", T_OP_CHARLEN},
	{"CHARSUB", T_OP_CHARSUB},

	{"INCLUDE", T_POP_INCLUDE},
	{"PRINT", T_POP_PRINT},
	{"PRINTLN", T_POP_PRINTLN},
	{"PRINTT", T_POP_PRINTT},
	{"PRINTI", T_POP_PRINTI},
	{"PRINTV", T_POP_PRINTV},
	{"PRINTF", T_POP_PRINTF},
	{"EXPORT", T_POP_EXPORT},
	{"DS", T_POP_DS},
	{"DB", T_POP_DB},
	{"DW", T_POP_DW},
	{"DL", T_POP_DL},
	{"SECTION", T_POP_SECTION},
	{"PURGE", T_POP_PURGE},

	{"RSRESET", T_POP_RSRESET},
	{"RSSET", T_POP_RSSET},

	{"INCBIN", T_POP_INCBIN},
	{"CHARMAP", T_POP_CHARMAP},
	{"NEWCHARMAP", T_POP_NEWCHARMAP},
	{"SETCHARMAP", T_POP_SETCHARMAP},
	{"PUSHC", T_POP_PUSHC},
	{"POPC", T_POP_POPC},

	{"FAIL", T_POP_FAIL},
	{"WARN", T_POP_WARN},
	{"FATAL", T_POP_FATAL},
	{"ASSERT", T_POP_ASSERT},
	{"STATIC_ASSERT", T_POP_STATIC_ASSERT},

	{"MACRO", T_POP_MACRO},
	{"ENDM", T_POP_ENDM},
	{"SHIFT", T_POP_SHIFT},

	{"REPT", T_POP_REPT},
	{"FOR", T_POP_FOR},
	{"ENDR", T_POP_ENDR},
	{"BREAK", T_POP_BREAK},

	{"LOAD", T_POP_LOAD},
	{"ENDL", T_POP_ENDL},

	{"IF", T_POP_IF},
	{"ELSE", T_POP_ELSE},
	{"ELIF", T_POP_ELIF},
	{"ENDC", T_POP_ENDC},

	{"UNION", T_POP_UNION},
	{"NEXTU", T_POP_NEXTU},
	{"ENDU", T_POP_ENDU},

	{"WRAM0", T_SECT_WRAM0},
	{"VRAM", T_SECT_VRAM},
	{"ROMX", T_SECT_ROMX},
	{"ROM0", T_SECT_ROM0},
	{"HRAM", T_SECT_HRAM},
	{"WRAMX", T_SECT_WRAMX},
	{"SRAM", T_SECT_SRAM},
	{"OAM", T_SECT_OAM},

	{"RB", T_POP_RB},
	{"RW", T_POP_RW},
	/* Handled before as T_Z80_RL */
	/* {"RL", T_POP_RL}, */

	{"EQU", T_POP_EQU},
	{"EQUS", T_POP_EQUS},
	{"REDEF", T_POP_REDEF},
	/* Handled before as T_Z80_SET */
	/* {"SET", T_POP_SET}, */

	{"PUSHS", T_POP_PUSHS},
	{"POPS", T_POP_POPS},
	{"PUSHO", T_POP_PUSHO},
	{"POPO", T_POP_POPO},

	{"OPT", T_POP_OPT},

	{".", T_PERIOD},
};

static bool isWhitespace(int c)
{
	return c == ' ' || c == '\t';
}

#define LEXER_BUF_SIZE 42 /* TODO: determine a sane value for this */
/* The buffer needs to be large enough for the maximum `peekInternal` lookahead distance */
static_assert(LEXER_BUF_SIZE > 1, "Lexer buffer size is too small");
/* This caps the size of buffer reads, and according to POSIX, passing more than SSIZE_MAX is UB */
static_assert(LEXER_BUF_SIZE <= SSIZE_MAX, "Lexer buffer size is too large");

struct Expansion {
	struct Expansion *parent;
	char *name;
	union {
		char const *unowned;
		char *owned;
	} contents;
	size_t size; /* Length of the contents */
	size_t offset; /* Cursor into the contents */
	bool owned; /* Whether or not to free contents when this expansion is freed */
};

struct IfStack {
	struct IfStack *next;
	bool ranIfBlock; /* Whether an IF/ELIF/ELSE block ran already */
	bool reachedElseBlock; /* Whether an ELSE block ran already */
};

struct LexerState {
	char const *path;

	/* mmap()-dependent IO state */
	bool isMmapped;
	union {
		struct { /* If mmap()ed */
			char *ptr; /* Technically `const` during the lexer's execution */
			size_t size;
			size_t offset;
			bool isReferenced; /* If a macro in this file requires not unmapping it */
		};
		struct { /* Otherwise */
			int fd;
			size_t index; /* Read index into the buffer */
			char buf[LEXER_BUF_SIZE]; /* Circular buffer */
			size_t nbChars; /* Number of "fresh" chars in the buffer */
		};
	};

	/* Common state */
	bool isFile;

	enum LexerMode mode;
	bool atLineStart;
	uint32_t lineNo;
	uint32_t colNo;
	int lastToken;

	struct IfStack *ifStack;

	bool capturing; /* Whether the text being lexed should be captured */
	size_t captureSize; /* Amount of text captured */
	char *captureBuf; /* Buffer to send the captured text to if non-NULL */
	size_t captureCapacity; /* Size of the buffer above */

	bool disableMacroArgs;
	bool disableInterpolation;
	size_t macroArgScanDistance; /* Max distance already scanned for macro args */
	bool expandStrings;
	struct Expansion *expansions; /* Points to the innermost current expansion */
};

struct LexerState *lexerState = NULL;
struct LexerState *lexerStateEOL = NULL;

static void initState(struct LexerState *state)
{
	state->mode = LEXER_NORMAL;
	state->atLineStart = true; /* yylex() will init colNo due to this */
	state->lastToken = T_EOF;

	state->ifStack = NULL;

	state->capturing = false;
	state->captureBuf = NULL;

	state->disableMacroArgs = false;
	state->disableInterpolation = false;
	state->macroArgScanDistance = 0;
	state->expandStrings = true;
	state->expansions = NULL;
}

static void nextLine(void)
{
	lexerState->lineNo++;
	lexerState->colNo = 1;
}

uint32_t lexer_GetIFDepth(void)
{
	uint32_t depth = 0;

	for (struct IfStack *stack = lexerState->ifStack; stack != NULL; stack = stack->next)
		depth++;

	return depth;
}

void lexer_IncIFDepth(void)
{
	struct IfStack *new = malloc(sizeof(*new));

	if (!new)
		fatalerror("Unable to allocate new IF depth: %s\n", strerror(errno));

	new->ranIfBlock = false;
	new->reachedElseBlock = false;
	new->next = lexerState->ifStack;

	lexerState->ifStack = new;
}

void lexer_DecIFDepth(void)
{
	if (!lexerState->ifStack)
		fatalerror("Found ENDC outside an IF construct\n");

	struct IfStack *top = lexerState->ifStack->next;

	free(lexerState->ifStack);

	lexerState->ifStack = top;
}

bool lexer_RanIFBlock(void)
{
	return lexerState->ifStack->ranIfBlock;
}

bool lexer_ReachedELSEBlock(void)
{
	return lexerState->ifStack->reachedElseBlock;
}

void lexer_RunIFBlock(void)
{
	lexerState->ifStack->ranIfBlock = true;
}

void lexer_ReachELSEBlock(void)
{
	lexerState->ifStack->reachedElseBlock = true;
}

struct LexerState *lexer_OpenFile(char const *path)
{
	dbgPrint("Opening file \"%s\"\n", path);

	bool isStdin = !strcmp(path, "-");
	struct LexerState *state = malloc(sizeof(*state));
	struct stat fileInfo;

	/* Give stdin a nicer file name */
	if (isStdin)
		path = "<stdin>";
	if (!state) {
		error("Failed to allocate memory for lexer state: %s\n", strerror(errno));
		return NULL;
	}
	if (!isStdin && stat(path, &fileInfo) != 0) {
		error("Failed to stat file \"%s\": %s\n", path, strerror(errno));
		free(state);
		return NULL;
	}
	state->path = path;
	state->isFile = true;
	state->fd = isStdin ? STDIN_FILENO : open(path, O_RDONLY);
	if (state->fd < 0) {
		error("Failed to open file \"%s\": %s\n", path, strerror(errno));
		free(state);
		return NULL;
	}
	state->isMmapped = false; /* By default, assume it won't be mmap()ed */
	if (!isStdin && fileInfo.st_size > 0) {
		/* Try using `mmap` for better performance */

		/*
		 * Important: do NOT assign to `state->ptr` directly, to avoid a cast that may
		 * alter an eventual `MAP_FAILED` value. It would also invalidate `state->fd`,
		 * being on the other side of the union.
		 */
		void *mappingAddr;

		mapFile(mappingAddr, state->fd, state->path, fileInfo.st_size);
		if (mappingAddr == MAP_FAILED) {
			/* If mmap()ing failed, try again using another method (below) */
			state->isMmapped = false;
		} else {
			/* IMPORTANT: the `union` mandates this is accessed before other members! */
			close(state->fd);

			state->isMmapped = true;
			state->isReferenced = false; // By default, a state isn't referenced
			state->ptr = mappingAddr;
			assert(fileInfo.st_size >= 0);
			state->size = (size_t)fileInfo.st_size;
			state->offset = 0;

			if (verbose)
				printf("File %s successfully mmap()ped\n", path);
		}
	}
	if (!state->isMmapped) {
		/* Sometimes mmap() fails or isn't available, so have a fallback */
		if (verbose) {
			if (isStdin)
				printf("Opening stdin\n");
			else if (fileInfo.st_size == 0)
				printf("File %s is empty\n", path);
			else
				printf("File %s opened as regular, errno reports \"%s\"\n",
				       path, strerror(errno));
		}
		state->index = 0;
		state->nbChars = 0;
	}

	initState(state);
	state->lineNo = 0; /* Will be incremented at first line start */
	return state;
}

struct LexerState *lexer_OpenFileView(char const *path, char *buf, size_t size, uint32_t lineNo)
{
	dbgPrint("Opening view on buffer \"%.*s\"[...]\n", size < 16 ? (int)size : 16, buf);

	struct LexerState *state = malloc(sizeof(*state));

	if (!state) {
		error("Failed to allocate memory for lexer state: %s\n", strerror(errno));
		return NULL;
	}

	state->path = path; /* Used to report read errors in `peekInternal` */
	state->isFile = false;
	state->isMmapped = true; /* It's not *really* mmap()ed, but it behaves the same */
	state->ptr = buf;
	state->size = size;
	state->offset = 0;

	initState(state);
	state->lineNo = lineNo; /* Will be incremented at first line start */
	return state;
}

void lexer_RestartRept(uint32_t lineNo)
{
	dbgPrint("Restarting REPT/FOR\n");
	lexerState->offset = 0;
	initState(lexerState);
	lexerState->lineNo = lineNo;
}

void lexer_DeleteState(struct LexerState *state)
{
	// A big chunk of the lexer state soundness is the file stack ("fstack").
	// Each context in the fstack has its own *unique* lexer state; thus, we always guarantee
	// that lexer states lifetimes are always properly managed, since they're handled solely
	// by the fstack... with *one* exception.
	// Assume a context is pushed on top of the fstack, and the corresponding lexer state gets
	// scheduled at EOF; `lexerStateEOL` thus becomes a (weak) ref to that lexer state...
	// It has been possible, due to a bug, that the corresponding fstack context gets popped
	// before EOL, deleting the associated state... but it would still be switched to at EOL.
	// This assertion checks that this doesn't happen again.
	// It could be argued that deleting a state that's scheduled for EOF could simply clear
	// `lexerStateEOL`, but there's currently no situation in which this should happen.
	assert(state != lexerStateEOL);

	if (!state->isMmapped)
		close(state->fd);
	else if (state->isFile && !state->isReferenced)
		munmap(state->ptr, state->size);
	free(state);
}

struct KeywordDictNode {
	/*
	 * The identifier charset is (currently) 44 characters big. By storing entries for the
	 * entire printable ASCII charset, minus lower-case due to case-insensitivity,
	 * we only waste (0x60 - 0x20) - 70 = 20 entries per node, which should be acceptable.
	 * In turn, this allows greatly simplifying checking an index into this array,
	 * which should help speed up the lexer.
	 */
	uint16_t children[0x60 - ' '];
	struct KeywordMapping const *keyword;
/* Since the keyword structure is invariant, the min number of nodes is known at compile time */
} keywordDict[365] = {0}; /* Make sure to keep this correct when adding keywords! */

/* Convert a char into its index into the dict */
static uint8_t dictIndex(char c)
{
	/* Translate uppercase to lowercase (roughly) */
	if (c > 0x60)
		c = c - ('a' - 'A');
	return c - ' ';
}

void lexer_Init(void)
{
	/*
	 * Build the dictionary of keywords. This could be done at compile time instead, however:
	 *  - Doing so manually is a task nobody wants to undertake
	 *  - It would be massively hard to read
	 *  - Doing it within CC or CPP would be quite non-trivial
	 *  - Doing it externally would require some extra work to use only POSIX tools
	 *  - The startup overhead isn't much compared to the program's
	 */
	uint16_t usedNodes = 1;

	for (size_t i = 0; i < ARRAY_SIZE(keywords); i++) {
		uint16_t nodeID = 0;

		/* Walk the dictionary, creating intermediate nodes for the keyword */
		for (char const *ptr = keywords[i].name; *ptr; ptr++) {
			/* We should be able to assume all entries are well-formed */
			if (keywordDict[nodeID].children[*ptr - ' '] == 0) {
				/*
				 * If this gets tripped up, set the size of keywordDict to
				 * something high, compile with `-DPRINT_NODE_COUNT` (see below),
				 * and set the size to that.
				 */
				assert(usedNodes < sizeof(keywordDict) / sizeof(*keywordDict));

				/* There is no node at that location, grab one from the pool */
				keywordDict[nodeID].children[*ptr - ' '] = usedNodes;
				usedNodes++;
			}
			nodeID = keywordDict[nodeID].children[*ptr - ' '];
		}

		/* This assumes that no two keywords have the same name */
		keywordDict[nodeID].keyword = &keywords[i];
	}

#ifdef PRINT_NODE_COUNT /* For the maintainer to check how many nodes are needed */
	printf("Lexer keyword dictionary: %zu keywords in %u nodes (pool size %zu)\n",
	       ARRAY_SIZE(keywords), usedNodes, ARRAY_SIZE(keywordDict));
#endif
}

void lexer_SetMode(enum LexerMode mode)
{
	lexerState->mode = mode;
}

void lexer_ToggleStringExpansion(bool enable)
{
	lexerState->expandStrings = enable;
}

/* Functions for the actual lexer to obtain characters */

static void reallocCaptureBuf(void)
{
	if (lexerState->captureCapacity == SIZE_MAX)
		fatalerror("Cannot grow capture buffer past %zu bytes\n", SIZE_MAX);
	else if (lexerState->captureCapacity > SIZE_MAX / 2)
		lexerState->captureCapacity = SIZE_MAX;
	else
		lexerState->captureCapacity *= 2;
	lexerState->captureBuf = realloc(lexerState->captureBuf, lexerState->captureCapacity);
	if (!lexerState->captureBuf)
		fatalerror("realloc error while resizing capture buffer: %s\n", strerror(errno));
}

static void beginExpansion(char const *str, bool owned, char const *name)
{
	size_t size = strlen(str);

	/* Do not expand empty strings */
	if (!size)
		return;

	if (name) {
		size_t depth = 0;

		for (struct Expansion *exp = lexerState->expansions; exp; exp = exp->parent) {
			if (depth++ >= maxRecursionDepth)
				fatalerror("Recursion limit (%zu) exceeded\n", maxRecursionDepth);
		}
	}

	struct Expansion *new = malloc(sizeof(*new));

	if (!new)
		fatalerror("Unable to allocate new expansion: %s\n", strerror(errno));

	new->parent = lexerState->expansions;
	new->name = name ? strdup(name) : NULL;
	new->contents.unowned = str;
	new->size = size;
	new->offset = 0;
	new->owned = owned;

	lexerState->expansions = new;
}

static void freeExpansion(struct Expansion *expansion)
{
	free(expansion->name);
	if (expansion->owned)
		free(expansion->contents.owned);
	free(expansion);
}

static bool isMacroChar(char c)
{
	return c == '@' || c == '#' || c == '<' || (c >= '0' && c <= '9');
}

/* forward declarations for readBracketedMacroArgNum */
static int peek(void);
static void shiftChar(void);
static uint32_t readNumber(int radix, uint32_t baseValue);
static bool startsIdentifier(int c);
static bool continuesIdentifier(int c);

static uint32_t readBracketedMacroArgNum(void)
{
	dbgPrint("Reading bracketed macro arg\n");
	bool disableMacroArgs = lexerState->disableMacroArgs;
	bool disableInterpolation = lexerState->disableInterpolation;

	lexerState->disableMacroArgs = false;
	lexerState->disableInterpolation = false;

	uint32_t num = 0;
	int c = peek();
	bool empty = false;
	bool symbolError = false;

	if (c >= '0' && c <= '9') {
		num = readNumber(10, 0);
	} else if (startsIdentifier(c)) {
		char symName[MAXSYMLEN + 1];
		size_t i = 0;

		for (; continuesIdentifier(c); c = peek()) {
			if (i < sizeof(symName))
				symName[i++] = c;
			shiftChar();
		}

		if (i == sizeof(symName)) {
			warning(WARNING_LONG_STR, "Bracketed symbol name too long\n");
			i--;
		}
		symName[i] = '\0';

		struct Symbol const *sym = sym_FindScopedSymbol(symName);

		if (!sym) {
			error("Bracketed symbol \"%s\" does not exist\n", symName);
			num = 0;
			symbolError = true;
		} else if (!sym_IsNumeric(sym)) {
			error("Bracketed symbol \"%s\" is not numeric\n", symName);
			num = 0;
			symbolError = true;
		} else {
			num = sym_GetConstantSymValue(sym);
		}
	} else {
		empty = true;
	}

	c = peek();
	shiftChar();
	if (c != '>') {
		error("Invalid character in bracketed macro argument %s\n", printChar(c));
		return 0;
	} else if (empty) {
		error("Empty bracketed macro argument\n");
		return 0;
	} else if (num == 0 && !symbolError) {
		error("Invalid bracketed macro argument '\\<0>'\n");
		return 0;
	}

	lexerState->disableMacroArgs = disableMacroArgs;
	lexerState->disableInterpolation = disableInterpolation;
	return num;
}

static char const *readMacroArg(char name)
{
	char const *str = NULL;

	if (name == '@') {
		str = macro_GetUniqueIDStr();
	} else if (name == '#') {
		str = macro_GetAllArgs();
	} else if (name == '<') {
		uint32_t num = readBracketedMacroArgNum();

		if (num == 0)
			return NULL;
		str = macro_GetArg(num);
		if (!str)
			error("Macro argument '\\<%" PRIu32 ">' not defined\n", num);
		return str;
	} else if (name == '0') {
		error("Invalid macro argument '\\0'\n");
		return NULL;
	} else {
		assert(name > '0' && name <= '9');
		str = macro_GetArg(name - '0');
	}

	if (!str)
		error("Macro argument '\\%c' not defined\n", name);
	return str;
}

/* We only need one character of lookahead, for macro arguments */
static int peekInternal(uint8_t distance)
{
	for (struct Expansion *exp = lexerState->expansions; exp; exp = exp->parent) {
		/*
		 * An expansion that has reached its end will have `exp->offset` == `exp->size`,
		 * and `peekInternal` will continue with its parent
		 */
		assert(exp->offset <= exp->size);
		if (distance < exp->size - exp->offset)
			return exp->contents.unowned[exp->offset + distance];
		distance -= exp->size - exp->offset;
	}

	if (distance >= LEXER_BUF_SIZE)
		fatalerror("Internal lexer error: buffer has insufficient size for peeking (%"
			   PRIu8 " >= %u)\n", distance, LEXER_BUF_SIZE);

	if (lexerState->isMmapped) {
		if (lexerState->offset + distance >= lexerState->size)
			return EOF;

		return (unsigned char)lexerState->ptr[lexerState->offset + distance];
	}

	if (lexerState->nbChars <= distance) {
		/* Buffer isn't full enough, read some chars in */
		size_t target = LEXER_BUF_SIZE - lexerState->nbChars; /* Aim: making the buf full */

		/* Compute the index we'll start writing to */
		size_t writeIndex = (lexerState->index + lexerState->nbChars) % LEXER_BUF_SIZE;
		ssize_t nbCharsRead = 0, totalCharsRead = 0;

#define readChars(size) do { \
	/* This buffer overflow made me lose WEEKS of my life. Never again. */ \
	assert(writeIndex + (size) <= LEXER_BUF_SIZE); \
	nbCharsRead = read(lexerState->fd, &lexerState->buf[writeIndex], (size)); \
	if (nbCharsRead == -1) \
		fatalerror("Error while reading \"%s\": %s\n", lexerState->path, strerror(errno)); \
	totalCharsRead += nbCharsRead; \
	writeIndex += nbCharsRead; \
	if (writeIndex == LEXER_BUF_SIZE) \
		writeIndex = 0; \
	target -= nbCharsRead; \
} while (0)

		/* If the range to fill passes over the buffer wrapping point, we need two reads */
		if (writeIndex + target > LEXER_BUF_SIZE) {
			size_t nbExpectedChars = LEXER_BUF_SIZE - writeIndex;

			readChars(nbExpectedChars);
			// If the read was incomplete, don't perform a second read
			// `nbCharsRead` cannot be negative, so it's fine to cast to `size_t`
			if ((size_t)nbCharsRead < nbExpectedChars)
				target = 0;
		}
		if (target != 0)
			readChars(target);

#undef readChars

		lexerState->nbChars += totalCharsRead;

		/* If there aren't enough chars even after refilling, give up */
		if (lexerState->nbChars <= distance)
			return EOF;
	}
	return (unsigned char)lexerState->buf[(lexerState->index + distance) % LEXER_BUF_SIZE];
}

/* forward declarations for peek */
static void shiftChar(void);
static char const *readInterpolation(size_t depth);

static int peek(void)
{
	int c = peekInternal(0);

	if (lexerState->macroArgScanDistance > 0)
		return c;

	lexerState->macroArgScanDistance++; /* Do not consider again */

	if (c == '\\' && !lexerState->disableMacroArgs) {
		/* If character is a backslash, check for a macro arg */
		lexerState->macroArgScanDistance++;
		c = peekInternal(1);
		if (isMacroChar(c)) {
			shiftChar();
			shiftChar();
			char const *str = readMacroArg(c);

			/*
			 * If the macro arg is invalid or an empty string, it cannot be
			 * expanded, so skip it and keep peeking.
			 */
			if (!str || !str[0])
				return peek();

			beginExpansion(str, c == '#', NULL);

			/*
			 * Assuming macro args can't be recursive (I'll be damned if a way
			 * is found...), then we mark the entire macro arg as scanned.
			 */
			lexerState->macroArgScanDistance += strlen(str);

			c = str[0];
		} else {
			c = '\\';
		}
	} else if (c == '{' && !lexerState->disableInterpolation) {
		/* If character is an open brace, do symbol interpolation */
		shiftChar();
		char const *str = readInterpolation(0);

		if (str && str[0])
			beginExpansion(str, false, str);
		return peek();
	}

	return c;
}

static void shiftChar(void)
{
	if (lexerState->capturing) {
		if (lexerState->captureBuf) {
			if (lexerState->captureSize + 1 >= lexerState->captureCapacity)
				reallocCaptureBuf();
			/* TODO: improve this? */
			lexerState->captureBuf[lexerState->captureSize] = peek();
		}
		lexerState->captureSize++;
	}

	lexerState->macroArgScanDistance--;

restart:
	if (lexerState->expansions) {
		/* Advance within the current expansion */
		assert(lexerState->expansions->offset <= lexerState->expansions->size);
		lexerState->expansions->offset++;
		if (lexerState->expansions->offset > lexerState->expansions->size) {
			/*
			 * When advancing would go past an expansion's end, free it,
			 * move up to its parent, and try again to advance
			 */
			struct Expansion *exp = lexerState->expansions;

			lexerState->expansions = lexerState->expansions->parent;
			freeExpansion(exp);
			goto restart;
		}
	} else {
		/* Advance within the file contents */
		lexerState->colNo++;
		if (lexerState->isMmapped) {
			lexerState->offset++;
		} else {
			assert(lexerState->index < LEXER_BUF_SIZE);
			lexerState->index++;
			if (lexerState->index == LEXER_BUF_SIZE)
				lexerState->index = 0; /* Wrap around if necessary */
			assert(lexerState->nbChars > 0);
			lexerState->nbChars--;
		}
	}
}

static int nextChar(void)
{
	int c = peek();

	/* If not at EOF, advance read position */
	if (c != EOF)
		shiftChar();
	return c;
}

static void handleCRLF(int c)
{
	if (c == '\r' && peek() == '\n')
		shiftChar();
}

/* "Services" provided by the lexer to the rest of the program */

char const *lexer_GetFileName(void)
{
	return lexerState ? lexerState->path : NULL;
}

uint32_t lexer_GetLineNo(void)
{
	return lexerState->lineNo;
}

uint32_t lexer_GetColNo(void)
{
	return lexerState->colNo;
}

void lexer_DumpStringExpansions(void)
{
	if (!lexerState)
		return;

	for (struct Expansion *exp = lexerState->expansions; exp; exp = exp->parent) {
		/* Only register EQUS expansions, not string args */
		if (exp->name)
			fprintf(stderr, "while expanding symbol \"%s\"\n", exp->name);
	}
}

/* Discards a block comment */
static void discardBlockComment(void)
{
	dbgPrint("Discarding block comment\n");
	lexerState->disableMacroArgs = true;
	lexerState->disableInterpolation = true;
	for (;;) {
		int c = nextChar();

		switch (c) {
		case EOF:
			error("Unterminated block comment\n");
			goto finish;
		case '\r':
			/* Handle CRLF before nextLine() since shiftChar updates colNo */
			handleCRLF(c);
			/* fallthrough */
		case '\n':
			if (!lexerState->expansions)
				nextLine();
			continue;
		case '/':
			if (peek() == '*') {
				warning(WARNING_NESTED_COMMENT,
					"/* in block comment\n");
			}
			continue;
		case '*':
			if (peek() == '/') {
				shiftChar();
				goto finish;
			}
			/* fallthrough */
		default:
			continue;
		}
	}
finish:
	lexerState->disableMacroArgs = false;
	lexerState->disableInterpolation = false;
}

/* Function to discard all of a line's comments */

static void discardComment(void)
{
	dbgPrint("Discarding comment\n");
	lexerState->disableMacroArgs = true;
	lexerState->disableInterpolation = true;
	for (;;) {
		int c = peek();

		if (c == EOF || c == '\r' || c == '\n')
			break;
		shiftChar();
	}
	lexerState->disableMacroArgs = false;
	lexerState->disableInterpolation = false;
}

/* Function to read a line continuation */

static void readLineContinuation(void)
{
	dbgPrint("Beginning line continuation\n");
	for (;;) {
		int c = peek();

		if (isWhitespace(c)) {
			shiftChar();
		} else if (c == '\r' || c == '\n') {
			shiftChar();
			/* Handle CRLF before nextLine() since shiftChar updates colNo */
			handleCRLF(c);
			if (!lexerState->expansions)
				nextLine();
			return;
		} else if (c == ';') {
			discardComment();
		} else {
			error("Begun line continuation, but encountered character %s\n",
			      printChar(c));
			return;
		}
	}
}

/* Function to read an anonymous label ref */

static void readAnonLabelRef(char c)
{
	uint32_t n = 0;

	// We come here having already peeked at one char, so no need to do it again
	do {
		shiftChar();
		n++;
	} while (peek() == c);

	sym_WriteAnonLabelName(yylval.symName, n, c == '-');
}

/* Functions to lex numbers of various radixes */

static uint32_t readNumber(int radix, uint32_t baseValue)
{
	uint32_t value = baseValue;

	for (;; shiftChar()) {
		int c = peek();

		if (c == '_')
			continue;
		else if (c < '0' || c > '0' + radix - 1)
			break;
		if (value > (UINT32_MAX - (c - '0')) / radix)
			warning(WARNING_LARGE_CONSTANT, "Integer constant is too large\n");
		value = value * radix + (c - '0');
	}

	return value;
}

static uint32_t readFractionalPart(int32_t integer)
{
	uint32_t value = 0, divisor = 1;

	dbgPrint("Reading fractional part\n");
	for (;; shiftChar()) {
		int c = peek();

		if (c == '_')
			continue;
		else if (c < '0' || c > '9')
			break;
		if (divisor > (UINT32_MAX - (c - '0')) / 10) {
			warning(WARNING_LARGE_CONSTANT,
				"Precision of fixed-point constant is too large\n");
			/* Discard any additional digits */
			shiftChar();
			while (c = peek(), (c >= '0' && c <= '9') || c == '_')
				shiftChar();
			break;
		}
		value = value * 10 + (c - '0');
		divisor *= 10;
	}

	if (integer > INT16_MAX || integer < INT16_MIN)
		warning(WARNING_LARGE_CONSTANT, "Magnitude of fixed-point constant is too large\n");

	/* Cast to unsigned avoids UB if shifting discards bits */
	integer = (uint32_t)integer << 16;
	/* Cast to unsigned avoids undefined overflow behavior */
	uint16_t fractional = (uint16_t)round(value * 65536.0 / divisor);

	return (uint32_t)integer | (fractional * (integer >= 0 ? 1 : -1));
}

char binDigits[2];

static uint32_t readBinaryNumber(void)
{
	uint32_t value = 0;

	dbgPrint("Reading binary number with digits [%c,%c]\n", binDigits[0], binDigits[1]);
	for (;; shiftChar()) {
		int c = peek();
		int bit;

		if (c == binDigits[0])
			bit = 0;
		else if (c == binDigits[1])
			bit = 1;
		else if (c == '_')
			continue;
		else
			break;
		if (value > (UINT32_MAX - bit) / 2)
			warning(WARNING_LARGE_CONSTANT, "Integer constant is too large\n");
		value = value * 2 + bit;
	}

	return value;
}

static uint32_t readHexNumber(void)
{
	uint32_t value = 0;
	bool empty = true;

	dbgPrint("Reading hex number\n");
	for (;; shiftChar()) {
		int c = peek();

		if (c >= 'a' && c <= 'f') /* Convert letters to right after digits */
			c = c - 'a' + 10;
		else if (c >= 'A' && c <= 'F')
			c = c - 'A' + 10;
		else if (c >= '0' && c <= '9')
			c = c - '0';
		else if (c == '_' && !empty)
			continue;
		else
			break;

		if (value > (UINT32_MAX - c) / 16)
			warning(WARNING_LARGE_CONSTANT, "Integer constant is too large\n");
		value = value * 16 + c;

		empty = false;
	}

	if (empty)
		error("Invalid integer constant, no digits after '$'\n");

	return value;
}

char gfxDigits[4];

static uint32_t readGfxConstant(void)
{
	uint32_t bp0 = 0, bp1 = 0;
	uint8_t width = 0;

	dbgPrint("Reading gfx constant with digits [%c,%c,%c,%c]\n",
		 gfxDigits[0], gfxDigits[1], gfxDigits[2], gfxDigits[3]);
	for (;;) {
		int c = peek();
		uint32_t pixel;

		if (c == gfxDigits[0])
			pixel = 0;
		else if (c == gfxDigits[1])
			pixel = 1;
		else if (c == gfxDigits[2])
			pixel = 2;
		else if (c == gfxDigits[3])
			pixel = 3;
		else
			break;

		if (width < 8) {
			bp0 = bp0 << 1 | (pixel & 1);
			bp1 = bp1 << 1 | (pixel >> 1);
		}
		if (width < 9)
			width++;
		shiftChar();
	}

	if (width == 0)
		error("Invalid graphics constant, no digits after '`'\n");
	else if (width == 9)
		warning(WARNING_LARGE_CONSTANT,
			"Graphics constant is too long, only 8 first pixels considered\n");

	return bp1 << 8 | bp0;
}

/* Functions to read identifiers & keywords */

static bool startsIdentifier(int c)
{
	// Anonymous labels internally start with '!'
	return (c <= 'Z' && c >= 'A') || (c <= 'z' && c >= 'a') || c == '.' || c == '_';
}

static bool continuesIdentifier(int c)
{
	return startsIdentifier(c) || (c <= '9' && c >= '0') || c == '#' || c == '@';
}

static int readIdentifier(char firstChar)
{
	dbgPrint("Reading identifier or keyword\n");
	/* Lex while checking for a keyword */
	yylval.symName[0] = firstChar;
	uint16_t nodeID = keywordDict[0].children[dictIndex(firstChar)];
	int tokenType = firstChar == '.' ? T_LOCAL_ID : T_ID;
	size_t i = 1;

	/* Continue reading while the char is in the symbol charset */
	for (int c = peek(); continuesIdentifier(c); i++, c = peek()) {
		shiftChar();

		if (i < sizeof(yylval.symName) - 1) {
			/* Write the char to the identifier's name */
			yylval.symName[i] = c;

			/* If the char was a dot, mark the identifier as local */
			if (c == '.')
				tokenType = T_LOCAL_ID;

			/* Attempt to traverse the tree to check for a keyword */
			if (nodeID) /* Do nothing if matching already failed */
				nodeID = keywordDict[nodeID].children[dictIndex(c)];
		}
	}

	if (i > sizeof(yylval.symName) - 1) {
		warning(WARNING_LONG_STR, "Symbol name too long, got truncated\n");
		i = sizeof(yylval.symName) - 1;
	}
	yylval.symName[i] = '\0'; /* Terminate the string */
	dbgPrint("Ident/keyword = \"%s\"\n", yylval.symName);

	if (keywordDict[nodeID].keyword)
		return keywordDict[nodeID].keyword->token;

	return tokenType;
}

/* Functions to read strings */

static char const *readInterpolation(size_t depth)
{
	if (depth >= maxRecursionDepth)
		fatalerror("Recursion limit (%zu) exceeded\n", maxRecursionDepth);

	char symName[MAXSYMLEN + 1];
	size_t i = 0;
	struct FormatSpec fmt = fmt_NewSpec();
	bool disableInterpolation = lexerState->disableInterpolation;

	/*
	 * In a context where `lexerState->disableInterpolation` is true, `peek` will expand
	 * nested interpolations itself, which can lead to stack overflow. This lets
	 * `readInterpolation` handle its own nested expansions, increasing `depth` each time.
	 */
	lexerState->disableInterpolation = true;

	for (;;) {
		int c = peek();

		if (c == '{') { /* Nested interpolation */
			shiftChar();
			char const *str = readInterpolation(depth + 1);

			if (str && str[0])
				beginExpansion(str, false, str);
			continue; /* Restart, reading from the new buffer */
		} else if (c == EOF || c == '\r' || c == '\n' || c == '"') {
			error("Missing }\n");
			break;
		} else if (c == '}') {
			shiftChar();
			break;
		} else if (c == ':' && !fmt_IsFinished(&fmt)) { /* Format spec, only once */
			shiftChar();
			for (size_t j = 0; j < i; j++)
				fmt_UseCharacter(&fmt, symName[j]);
			fmt_FinishCharacters(&fmt);
			symName[i] = '\0';
			if (!fmt_IsValid(&fmt))
				error("Invalid format spec '%s'\n", symName);
			i = 0; /* Now that format has been set, restart at beginning of string */
		} else {
			shiftChar();
			if (i < sizeof(symName)) /* Allow writing an extra char to flag overflow */
				symName[i++] = c;
		}
	}

	if (i == sizeof(symName)) {
		warning(WARNING_LONG_STR, "Interpolated symbol name too long\n");
		i--;
	}
	symName[i] = '\0';

	/* Don't return before `lexerState->disableInterpolation` is reset! */ 
	lexerState->disableInterpolation = disableInterpolation;

	static char buf[MAXSTRLEN + 1];

	struct Symbol const *sym = sym_FindScopedSymbol(symName);

	if (!sym) {
		error("Interpolated symbol \"%s\" does not exist\n", symName);
	} else if (sym->type == SYM_EQUS) {
		if (fmt_IsEmpty(&fmt))
			/* No format was specified */
			fmt.type = 's';
		fmt_PrintString(buf, sizeof(buf), &fmt, sym_GetStringValue(sym));
		return buf;
	} else if (sym_IsNumeric(sym)) {
		if (fmt_IsEmpty(&fmt)) {
			/* No format was specified; default to uppercase $hex */
			fmt.type = 'X';
			fmt.prefix = true;
		}
		fmt_PrintNumber(buf, sizeof(buf), &fmt, sym_GetConstantSymValue(sym));
		return buf;
	} else {
		error("Only numerical and string symbols can be interpolated\n");
	}
	return NULL;
}

#define append_yylval_string(c) do { \
	char v = (c); /* Evaluate c exactly once in case it has side effects. */ \
	if (i < sizeof(yylval.string)) \
		yylval.string[i++] = v; \
} while (0)

static size_t appendEscapedSubstring(char const *str, size_t i)
{
	/* Copy one extra to flag overflow */
	while (*str) {
		char c = *str++;

		/* Escape characters that need escaping */
		switch (c) {
		case '\\':
		case '"':
		case '{':
			append_yylval_string('\\');
			break;
		case '\n':
			append_yylval_string('\\');
			c = 'n';
			break;
		case '\r':
			append_yylval_string('\\');
			c = 'r';
			break;
		case '\t':
			append_yylval_string('\\');
			c = 't';
			break;
		}

		append_yylval_string(c);
	}

	return i;
}

static void readString(void)
{
	dbgPrint("Reading string\n");
	lexerState->disableMacroArgs = true;
	lexerState->disableInterpolation = true;

	size_t i = 0;
	bool multiline = false;
	char const *str;

	// We reach this function after reading a single quote, but we also support triple quotes
	if (peek() == '"') {
		shiftChar();
		if (peek() == '"') {
			// """ begins a multi-line string
			shiftChar();
			multiline = true;
		} else {
			// "" is an empty string, skip the loop
			goto finish;
		}
	}

	for (;;) {
		int c = peek();

		// '\r', '\n' or EOF ends a single-line string early
		if (c == EOF || (!multiline && (c == '\r' || c == '\n'))) {
			error("Unterminated string\n");
			break;
		}

		// We'll be staying in the string, so we can safely consume the char
		shiftChar();

		// Handle '\r' or '\n' (in multiline strings only, already handled above otherwise)
		if (c == '\r' || c == '\n') {
			/* Handle CRLF before nextLine() since shiftChar updates colNo */
			handleCRLF(c);
			nextLine();
			c = '\n';
		}

		switch (c) {
		case '"':
			if (multiline) {
				// Only """ ends a multi-line string
				if (peek() != '"')
					break;
				shiftChar();
				if (peek() != '"') {
					append_yylval_string('"');
					break;
				}
				shiftChar();
			}
			goto finish;

		case '\\': // Character escape or macro arg
			c = peek();
			switch (c) {
			case '\\':
			case '"':
			case '{':
			case '}':
				// Return that character unchanged
				shiftChar();
				break;
			case 'n':
				c = '\n';
				shiftChar();
				break;
			case 'r':
				c = '\r';
				shiftChar();
				break;
			case 't':
				c = '\t';
				shiftChar();
				break;

			// Line continuation
			case ' ':
			case '\r':
			case '\n':
				readLineContinuation();
				continue;

			// Macro arg
			case '@':
			case '#':
			case '0':
			case '1':
			case '2':
			case '3':
			case '4':
			case '5':
			case '6':
			case '7':
			case '8':
			case '9':
			case '<':
				shiftChar();
				str = readMacroArg(c);
				if (str) {
					while (*str)
						append_yylval_string(*str++);
				}
				continue; // Do not copy an additional character

			case EOF: // Can't really print that one
				error("Illegal character escape at end of input\n");
				c = '\\';
				break;

			default:
				error("Illegal character escape %s\n", printChar(c));
				shiftChar();
				break;
			}
			break;

		case '{': // Symbol interpolation
			// We'll be exiting the string scope, so re-enable expansions
			// (Not interpolations, since they're handled by the function itself...)
			lexerState->disableMacroArgs = false;
			str = readInterpolation(0);
			if (str) {
				while (*str)
					append_yylval_string(*str++);
			}
			lexerState->disableMacroArgs = true;
			continue; // Do not copy an additional character

		// Regular characters will just get copied
		}

		append_yylval_string(c);
	}

finish:
	if (i == sizeof(yylval.string)) {
		i--;
		warning(WARNING_LONG_STR, "String constant too long\n");
	}
	yylval.string[i] = '\0';

	dbgPrint("Read string \"%s\"\n", yylval.string);
	lexerState->disableMacroArgs = false;
	lexerState->disableInterpolation = false;
}

static size_t appendStringLiteral(size_t i)
{
	dbgPrint("Reading string\n");
	lexerState->disableMacroArgs = true;
	lexerState->disableInterpolation = true;

	bool multiline = false;
	char const *str;

	// We reach this function after reading a single quote, but we also support triple quotes
	append_yylval_string('"');
	if (peek() == '"') {
		append_yylval_string('"');
		shiftChar();
		if (peek() == '"') {
			// """ begins a multi-line string
			append_yylval_string('"');
			shiftChar();
			multiline = true;
		} else {
			// "" is an empty string, skip the loop
			goto finish;
		}
	}

	for (;;) {
		int c = peek();

		// '\r', '\n' or EOF ends a single-line string early
		if (c == EOF || (!multiline && (c == '\r' || c == '\n'))) {
			error("Unterminated string\n");
			break;
		}

		// We'll be staying in the string, so we can safely consume the char
		shiftChar();

		// Handle '\r' or '\n' (in multiline strings only, already handled above otherwise)
		if (c == '\r' || c == '\n') {
			/* Handle CRLF before nextLine() since shiftChar updates colNo */
			handleCRLF(c);
			nextLine();
			c = '\n';
		}

		switch (c) {
		case '"':
			if (multiline) {
				// Only """ ends a multi-line string
				if (peek() != '"')
					break;
				append_yylval_string('"');
				shiftChar();
				if (peek() != '"')
					break;
				append_yylval_string('"');
				shiftChar();
			}
			append_yylval_string('"');
			goto finish;

		case '\\': // Character escape or macro arg
			c = peek();
			switch (c) {
			// Character escape
			case '\\':
			case '"':
			case '{':
			case '}':
			case 'n':
			case 'r':
			case 't':
				// Return that character unchanged
				append_yylval_string('\\');
				shiftChar();
				break;

			// Line continuation
			case ' ':
			case '\r':
			case '\n':
				readLineContinuation();
				continue;

			// Macro arg
			case '@':
			case '#':
			case '0':
			case '1':
			case '2':
			case '3':
			case '4':
			case '5':
			case '6':
			case '7':
			case '8':
			case '9':
			case '<':
				shiftChar();
				str = readMacroArg(c);
				if (str && str[0])
					i = appendEscapedSubstring(str, i);
				continue; // Do not copy an additional character

			case EOF: // Can't really print that one
				error("Illegal character escape at end of input\n");
				c = '\\';
				break;

			case ',': /* `\,` inside a macro arg string literal */
				warning(WARNING_OBSOLETE,
					"`\\,` is deprecated inside strings\n");
				shiftChar();
				break;

			default:
				error("Illegal character escape %s\n", printChar(c));
				shiftChar();
				break;
			}
			break;

		case '{': // Symbol interpolation
			// We'll be exiting the string scope, so re-enable expansions
			// (Not interpolations, since they're handled by the function itself...)
			lexerState->disableMacroArgs = false;
			str = readInterpolation(0);
			if (str && str[0])
				i = appendEscapedSubstring(str, i);
			lexerState->disableMacroArgs = true;
			continue; // Do not copy an additional character

		// Regular characters will just get copied
		}

		append_yylval_string(c);
	}

finish:
	if (i == sizeof(yylval.string)) {
		i--;
		warning(WARNING_LONG_STR, "String constant too long\n");
	}
	yylval.string[i] = '\0';

	dbgPrint("Read string \"%s\"\n", yylval.string);
	lexerState->disableMacroArgs = false;
	lexerState->disableInterpolation = false;

	return i;
}

/* Lexer core */

static int yylex_SKIP_TO_ENDC(void); // forward declaration for yylex_NORMAL

static int yylex_NORMAL(void)
{
	dbgPrint("Lexing in normal mode, line=%" PRIu32 ", col=%" PRIu32 "\n",
		 lexer_GetLineNo(), lexer_GetColNo());

	for (;;) {
		int c = nextChar();
		char secondChar;

		switch (c) {
		/* Ignore whitespace and comments */

		case ';':
			discardComment();
			/* fallthrough */
		case ' ':
		case '\t':
			break;

		/* Handle unambiguous single-char tokens */

		case '^':
			return T_OP_XOR;
		case '+':
			return T_OP_ADD;
		case '-':
			return T_OP_SUB;
		case '~':
			return T_OP_NOT;

		case '@':
			yylval.symName[0] = '@';
			yylval.symName[1] = '\0';
			return T_ID;

		case '[':
			return T_LBRACK;
		case ']':
			return T_RBRACK;
		case '(':
			return T_LPAREN;
		case ')':
			return T_RPAREN;
		case ',':
			return T_COMMA;

		/* Handle ambiguous 1- or 2-char tokens */

		case '*': /* Either MUL or EXP */
			if (peek() == '*') {
				shiftChar();
				return T_OP_EXP;
			}
			return T_OP_MUL;

		case '/': /* Either division or a block comment */
			if (peek() == '*') {
				shiftChar();
				discardBlockComment();
				break;
			}
			return T_OP_DIV;

		case '|': /* Either binary or logical OR */
			if (peek() == '|') {
				shiftChar();
				return T_OP_LOGICOR;
			}
			return T_OP_OR;

		case '=': /* Either SET alias, or EQ */
			if (peek() == '=') {
				shiftChar();
				return T_OP_LOGICEQU;
			}
			return T_POP_EQUAL;

		case '<': /* Either a LT, LTE, or left shift */
			switch (peek()) {
			case '=':
				shiftChar();
				return T_OP_LOGICLE;
			case '<':
				shiftChar();
				return T_OP_SHL;
			default:
				return T_OP_LOGICLT;
			}

		case '>': /* Either a GT, GTE, or right shift */
			switch (peek()) {
			case '=':
				shiftChar();
				return T_OP_LOGICGE;
			case '>':
				shiftChar();
				return T_OP_SHR;
			default:
				return T_OP_LOGICGT;
			}

		case '!': /* Either a NEQ, or negation */
			if (peek() == '=') {
				shiftChar();
				return T_OP_LOGICNE;
			}
			return T_OP_LOGICNOT;

		/* Handle colon, which may begin an anonymous label ref */

		case ':':
			c = peek();
			if (c != '+' && c != '-')
				return T_COLON;

			readAnonLabelRef(c);
			return T_ANON;

		/* Handle numbers */

		case '$':
			yylval.constValue = readHexNumber();
			return T_NUMBER;

		case '0': /* Decimal number */
		case '1':
		case '2':
		case '3':
		case '4':
		case '5':
		case '6':
		case '7':
		case '8':
		case '9':
			yylval.constValue = readNumber(10, c - '0');
			if (peek() == '.') {
				shiftChar();
				yylval.constValue = readFractionalPart(yylval.constValue);
			}
			return T_NUMBER;

		case '&':
			secondChar = peek();
			if (secondChar == '&') {
				shiftChar();
				return T_OP_LOGICAND;
			} else if (secondChar >= '0' && secondChar <= '7') {
				yylval.constValue = readNumber(8, 0);
				return T_NUMBER;
			}
			return T_OP_AND;

		case '%': /* Either a modulo, or a binary constant */
			secondChar = peek();
			if (secondChar != binDigits[0] && secondChar != binDigits[1])
				return T_OP_MOD;

			yylval.constValue = readBinaryNumber();
			return T_NUMBER;

		case '`': /* Gfx constant */
			yylval.constValue = readGfxConstant();
			return T_NUMBER;

		/* Handle strings */

		case '"':
			readString();
			return T_STRING;

		/* Handle newlines and EOF */

		case '\r':
			handleCRLF(c);
			/* fallthrough */
		case '\n':
			return T_NEWLINE;

		case EOF:
			return T_EOF;

		/* Handle line continuations */

		case '\\':
			// Macro args were handled by `peek`, and character escapes do not exist
			// outside of string literals, so this must be a line continuation.
			readLineContinuation();
			break;

		/* Handle identifiers... or report garbage characters */

		default:
			if (startsIdentifier(c)) {
				int tokenType = readIdentifier(c);

				/* An ELIF after a taken IF needs to not evaluate its condition */
				if (tokenType == T_POP_ELIF && lexerState->lastToken == T_NEWLINE
				 && lexer_GetIFDepth() > 0 && lexer_RanIFBlock()
				 && !lexer_ReachedELSEBlock())
					return yylex_SKIP_TO_ENDC();

				/* If a keyword, don't try to expand */
				if (tokenType != T_ID && tokenType != T_LOCAL_ID)
					return tokenType;

				/* Local symbols cannot be string expansions */
				if (tokenType == T_ID && lexerState->expandStrings) {
					/* Attempt string expansion */
					struct Symbol const *sym = sym_FindExactSymbol(yylval.symName);

					if (sym && sym->type == SYM_EQUS) {
						char const *s = sym_GetStringValue(sym);

						assert(s);
						if (s[0])
							beginExpansion(s, false, sym->name);
						continue; /* Restart, reading from the new buffer */
					}
				}

				if (tokenType == T_ID && (lexerState->atLineStart || peek() == ':'))
					return T_LABEL;

				return tokenType;
			}

			/* Do not report weird characters when capturing, it'll be done later */
			if (!lexerState->capturing) {
				/* TODO: try to group reportings */
				error("Unknown character %s\n", printChar(c));
			}
		}
		lexerState->atLineStart = false;
	}
}

static int yylex_RAW(void)
{
	dbgPrint("Lexing in raw mode, line=%" PRIu32 ", col=%" PRIu32 "\n",
		 lexer_GetLineNo(), lexer_GetColNo());

	/* This is essentially a modified `appendStringLiteral` */
	size_t parenDepth = 0;
	size_t i = 0;
	int c;

	/* Trim left whitespace (stops at a block comment or line continuation) */
	while (isWhitespace(peek()))
		shiftChar();

	for (;;) {
		c = peek();

		switch (c) {
		case '"': /* String literals inside macro args */
			shiftChar();
			i = appendStringLiteral(i);
			break;

		case ';': /* Comments inside macro args */
			discardComment();
			c = peek();
			/* fallthrough */
		case '\r': /* End of line */
		case '\n':
		case EOF:
			goto finish;

		case '/': /* Block comments inside macro args */
			shiftChar();
			if (peek() == '*') {
				shiftChar();
				discardBlockComment();
				continue;
			}
			append_yylval_string(c); /* Append the slash */
			break;

		case ',': /* End of macro arg */
			if (parenDepth == 0)
				goto finish;
			goto append;

		case '(': /* Open parentheses inside macro args */
			if (parenDepth < UINT_MAX)
				parenDepth++;
			goto append;

		case ')': /* Close parentheses inside macro args */
			if (parenDepth > 0)
				parenDepth--;
			goto append;

		case '\\': /* Character escape */
			shiftChar();
			c = peek();

			switch (c) {
			case ',': /* Escapes only valid inside a macro arg */
			case '(':
			case ')':
			case '\\': /* Escapes shared with string literals */
			case '"':
			case '{':
			case '}':
				break;

			case 'n':
				c = '\n';
				break;
			case 'r':
				c = '\r';
				break;
			case 't':
				c = '\t';
				break;

			case ' ':
			case '\r':
			case '\n':
				readLineContinuation();
				continue;

			case EOF: /* Can't really print that one */
				error("Illegal character escape at end of input\n");
				c = '\\';
				break;

			/*
			 * Macro args were already handled by peek, so '\@',
			 * '\#', and '\0'-'\9' should not occur here.
			 */

			default:
				error("Illegal character escape %s\n", printChar(c));
				break;
			}
			/* fallthrough */

		default: /* Regular characters will just get copied */
append:
			append_yylval_string(c);
			shiftChar();
			break;
		}
	}

finish:
	if (i == sizeof(yylval.string)) {
		i--;
		warning(WARNING_LONG_STR, "Macro argument too long\n");
	}
	/* Trim right whitespace */
	while (i && isWhitespace(yylval.string[i - 1]))
		i--;
	yylval.string[i] = '\0';

	dbgPrint("Read raw string \"%s\"\n", yylval.string);

	// Returning T_COMMAs to the parser would mean that two consecutive commas
	// (i.e. an empty argument) need to return two different tokens (T_STRING
	// then T_COMMA) without advancing the read. To avoid this, commas in raw
	// mode end the current macro argument but are not tokenized themselves.
	if (c == ',') {
		shiftChar();
		return T_STRING;
	}

	// The last argument may end in a trailing comma, newline, or EOF.
	// To allow trailing commas, raw mode will continue after the last
	// argument, immediately lexing the newline or EOF again (i.e. with
	// an empty raw string before it). This will not be treated as a
	// macro argument. To pass an empty last argument, use a second
	// trailing comma.
	if (i > 0)
		return T_STRING;
	lexer_SetMode(LEXER_NORMAL);

	if (c == '\r' || c == '\n') {
		shiftChar();
		handleCRLF(c);
		return T_NEWLINE;
	}

	return T_EOF;
}

#undef append_yylval_string

/*
 * This function uses the fact that `if`, etc. constructs are only valid when
 * there's nothing before them on their lines. This enables filtering
 * "meaningful" (= at line start) vs. "meaningless" (everything else) tokens.
 * It's especially important due to macro args not being handled in this
 * state, and lexing them in "normal" mode potentially producing such tokens.
 */
static int skipIfBlock(bool toEndc)
{
	dbgPrint("Skipping IF block (toEndc = %s)\n", toEndc ? "true" : "false");
	lexer_SetMode(LEXER_NORMAL);
	uint32_t startingDepth = lexer_GetIFDepth();
	int token;
	bool atLineStart = lexerState->atLineStart;

	/* Prevent expanding macro args and symbol interpolation in this state */
	lexerState->disableMacroArgs = true;
	lexerState->disableInterpolation = true;

	for (;;) {
		if (atLineStart) {
			int c;

			for (;;) {
				c = peek();
				if (!isWhitespace(c))
					break;
				shiftChar();
			}

			if (startsIdentifier(c)) {
				shiftChar();
				token = readIdentifier(c);
				switch (token) {
				case T_POP_IF:
					lexer_IncIFDepth();
					break;

				case T_POP_ELIF:
					if (lexer_ReachedELSEBlock())
						fatalerror("Found ELIF after an ELSE block\n");
					goto maybeFinish;

				case T_POP_ELSE:
					if (lexer_ReachedELSEBlock())
						fatalerror("Found ELSE after an ELSE block\n");
					lexer_ReachELSEBlock();
					/* fallthrough */
				maybeFinish:
					if (toEndc) /* Ignore ELIF and ELSE, go to ENDC */
						break;
					/* fallthrough */
				case T_POP_ENDC:
					if (lexer_GetIFDepth() == startingDepth)
						goto finish;
					if (token == T_POP_ENDC)
						lexer_DecIFDepth();
				}
			}
			atLineStart = false;
		}

		/* Read chars until EOL */
		do {
			int c = nextChar();

			if (c == EOF) {
				token = T_EOF;
				goto finish;
			} else if (c == '\\') {
				/* Unconditionally skip the next char, including line conts */
				c = nextChar();
			} else if (c == '\r' || c == '\n') {
				atLineStart = true;
			}

			if (c == '\r' || c == '\n') {
				/* Handle CRLF before nextLine() since shiftChar updates colNo */
				handleCRLF(c);
				/* Do this both on line continuations and plain EOLs */
				nextLine();
			}
		} while (!atLineStart);
	}
finish:

	lexerState->disableMacroArgs = false;
	lexerState->disableInterpolation = false;
	lexerState->atLineStart = false;

	return token;
}

static int yylex_SKIP_TO_ELIF(void)
{
	return skipIfBlock(false);
}

static int yylex_SKIP_TO_ENDC(void)
{
	return skipIfBlock(true);
}

static int yylex_SKIP_TO_ENDR(void)
{
	dbgPrint("Skipping remainder of REPT/FOR block\n");
	lexer_SetMode(LEXER_NORMAL);
	int depth = 1;
	bool atLineStart = lexerState->atLineStart;

	/* Prevent expanding macro args and symbol interpolation in this state */
	lexerState->disableMacroArgs = true;
	lexerState->disableInterpolation = true;

	for (;;) {
		if (atLineStart) {
			int c;

			for (;;) {
				c = peek();
				if (!isWhitespace(c))
					break;
				shiftChar();
			}

			if (startsIdentifier(c)) {
				shiftChar();
				switch (readIdentifier(c)) {
				case T_POP_FOR:
				case T_POP_REPT:
					depth++;
					break;

				case T_POP_ENDR:
					depth--;
					if (!depth)
						goto finish;
					break;

				case T_POP_IF:
					lexer_IncIFDepth();
					break;

				case T_POP_ENDC:
					lexer_DecIFDepth();
				}
			}
			atLineStart = false;
		}

		/* Read chars until EOL */
		do {
			int c = nextChar();

			if (c == EOF) {
				goto finish;
			} else if (c == '\\') {
				/* Unconditionally skip the next char, including line conts */
				c = nextChar();
			} else if (c == '\r' || c == '\n') {
				atLineStart = true;
			}

			if (c == '\r' || c == '\n') {
				/* Handle CRLF before nextLine() since shiftChar updates colNo */
				handleCRLF(c);
				/* Do this both on line continuations and plain EOLs */
				nextLine();
			}
		} while (!atLineStart);
	}
finish:

	lexerState->disableMacroArgs = false;
	lexerState->disableInterpolation = false;
	lexerState->atLineStart = false;

	/* yywrap() will finish the REPT/FOR loop */
	return T_EOF;
}

int yylex(void)
{
	if (lexerState->atLineStart && lexerStateEOL) {
		lexer_SetState(lexerStateEOL);
		lexerStateEOL = NULL;
	}
	/* `lexer_SetState` updates `lexerState`, so check for EOF after it */
	if (lexerState->lastToken == T_EOB) {
		if (yywrap()) {
			dbgPrint("Reached end of input.\n");
			return T_EOF;
		}
	}
	if (lexerState->atLineStart) {
		/* Newlines read within an expansion should not increase the line count */
		if (!lexerState->expansions)
			nextLine();
	}

	static int (* const lexerModeFuncs[])(void) = {
		[LEXER_NORMAL]       = yylex_NORMAL,
		[LEXER_RAW]          = yylex_RAW,
		[LEXER_SKIP_TO_ELIF] = yylex_SKIP_TO_ELIF,
		[LEXER_SKIP_TO_ENDC] = yylex_SKIP_TO_ENDC,
		[LEXER_SKIP_TO_ENDR] = yylex_SKIP_TO_ENDR,
	};
	int token = lexerModeFuncs[lexerState->mode]();

	if (token == T_EOF) {
		dbgPrint("Reached EOB!\n");
		/* Captures end at their buffer's boundary no matter what */
		if (!lexerState->capturing)
			token = T_EOB;
	}
	lexerState->lastToken = token;
	lexerState->atLineStart = token == T_NEWLINE || token == T_EOB;

	return token;
}

static void startCapture(struct CaptureBody *capture)
{
	assert(!lexerState->capturing);
	lexerState->capturing = true;
	lexerState->captureSize = 0;
	lexerState->disableMacroArgs = true;
	lexerState->disableInterpolation = true;

	capture->lineNo = lexer_GetLineNo();

	if (lexerState->isMmapped && !lexerState->expansions) {
		capture->body = &lexerState->ptr[lexerState->offset];
	} else {
		lexerState->captureCapacity = 128; /* The initial size will be twice that */
		assert(lexerState->captureBuf == NULL);
		reallocCaptureBuf();
		capture->body = NULL; // Indicate to retrieve the capture buffer when done capturing
	}
}

static void endCapture(struct CaptureBody *capture)
{
	// This being NULL means we're capturing from the capture buf, which is `realloc`'d during
	// the whole capture process, and so MUST be retrieved at the end
	if (!capture->body)
		capture->body = lexerState->captureBuf;
	capture->size = lexerState->captureSize;

	lexerState->capturing = false;
	lexerState->captureBuf = NULL;
	lexerState->disableMacroArgs = false;
	lexerState->disableInterpolation = false;
}

bool lexer_CaptureRept(struct CaptureBody *capture)
{
	startCapture(capture);

	size_t depth = 0;
	int c = EOF;

	/*
	 * Due to parser internals, it reads the EOL after the expression before calling this.
	 * Thus, we don't need to keep one in the buffer afterwards.
	 * The following assertion checks that.
	 */
	assert(lexerState->atLineStart);
	for (;;) {
		nextLine();
		/* We're at line start, so attempt to match a `REPT` or `ENDR` token */
		do { /* Discard initial whitespace */
			c = nextChar();
		} while (isWhitespace(c));
		/* Now, try to match `REPT`, `FOR` or `ENDR` as a **whole** identifier */
		if (startsIdentifier(c)) {
			switch (readIdentifier(c)) {
			case T_POP_REPT:
			case T_POP_FOR:
				depth++;
				/* Ignore the rest of that line */
				break;

			case T_POP_ENDR:
				if (!depth) {
					/*
					 * The final ENDR has been captured, but we don't want it!
					 * We know we have read exactly "ENDR", not e.g. an EQUS
					 */
					lexerState->captureSize -= strlen("ENDR");
					goto finish;
				}
				depth--;
			}
		}

		/* Just consume characters until EOL or EOF */
		for (;;) {
			if (c == EOF) {
				error("Unterminated REPT/FOR block\n");
				goto finish;
			} else if (c == '\n' || c == '\r') {
				handleCRLF(c);
				break;
			}
			c = nextChar();
		}
	}

finish:
	endCapture(capture);
	/* ENDR or EOF puts us past the start of the line */
	lexerState->atLineStart = false;

	/* Returns true if an ENDR terminated the block, false if it reached EOF first */
	return c != EOF;
}

bool lexer_CaptureMacroBody(struct CaptureBody *capture)
{
	startCapture(capture);

	/* If the file is `mmap`ed, we need not to unmap it to keep access to the macro */
	if (lexerState->isMmapped)
		lexerState->isReferenced = true;

	int c = EOF;

	/*
	 * Due to parser internals, it reads the EOL after the expression before calling this.
	 * Thus, we don't need to keep one in the buffer afterwards.
	 * The following assertion checks that.
	 */
	assert(lexerState->atLineStart);
	for (;;) {
		nextLine();
		/* We're at line start, so attempt to match an `ENDM` token */
		do { /* Discard initial whitespace */
			c = nextChar();
		} while (isWhitespace(c));
		/* Now, try to match `ENDM` as a **whole** identifier */
		if (startsIdentifier(c)) {
			switch (readIdentifier(c)) {
			case T_POP_ENDM:
				/*
				 * The ENDM has been captured, but we don't want it!
				 * We know we have read exactly "ENDM", not e.g. an EQUS
				 */
				lexerState->captureSize -= strlen("ENDM");
				goto finish;
			}
		}

		/* Just consume characters until EOL or EOF */
		for (;;) {
			if (c == EOF) {
				error("Unterminated macro definition\n");
				goto finish;
			} else if (c == '\n' || c == '\r') {
				handleCRLF(c);
				break;
			}
			c = nextChar();
		}
	}

finish:
	endCapture(capture);
	/* ENDM or EOF puts us past the start of the line */
	lexerState->atLineStart = false;

	/* Returns true if an ENDM terminated the block, false if it reached EOF first */
	return c != EOF;
}