laentropia/Calculator
1
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

refactor: adapted lexer to work with new tokens

Browse Source 
Now its fine, the code i find it clear if one just sits down to read it
for a moment, next is the parser that REALLY needs reworking and a few
helper structs.
This commit is contained in:
Alonso Adrian Martinez Hernandez
2026-05-13 09:49:28 -06:00
parent 2a73f5f9d6
commit f3373123e1
2 changed files with 18 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
include/lexer.h
View File

@@ -61,7 +61,7 @@ typedef struct {
bool is_valid; bool is_valid;
union { union {
LexerErr err; LexerErr err;
int64_t number; int64_t num;
}; };
} LexerI64Result; } LexerI64Result;

36
src/lexer.c
View File

@@ -15,7 +15,7 @@ typedef enum {
TokenizeResult tokenize(const char *input) { TokenizeResult tokenize(const char *input) {
ArrayList *arr = arraylist_init(64, sizeof(ASTNode)); ArrayList *arr = arraylist_init(64, sizeof(Token));
size_t offset = 0; size_t offset = 0;
while ( while (
@@ -24,19 +24,17 @@ TokenizeResult tokenize(const char *input) {
input[offset] != '\0') { input[offset] != '\0') {
if (isdigit(input[offset])) { if (isdigit(input[offset])) {
ASTNodeResult result = tokenize_number(input, &offset); TokenResult result = tokenize_number(input, &offset);
if (!result.is_valid) { if (!result.is_valid) {
return (TokenizeResult) {.is_valid = false, .err = result.err}; return (TokenizeResult) {.is_valid = false, .err = result.err};
} }
arraylist_push_back(arr, &result.node); arraylist_push_back(arr, &result.token);
} else if (isoperator(input[offset])) { } else if (isoperator(input[offset])) {
ASTNode op_node = { Token op_node = {
.type = NODE_BINARY_OP, .type = TOKEN_OPERATOR,
.data.binary.op = char_to_operator(input[offset]), .op = char_to_operator(input[offset]),
.data.binary.left = NULL,
.data.binary.right = NULL,
}; };
arraylist_push_back(arr, &op_node); arraylist_push_back(arr, &op_node);
@@ -60,7 +58,7 @@ TokenizeResult tokenize(const char *input) {
// CURRENTLY, it only supports ints, not clear how floating // CURRENTLY, it only supports ints, not clear how floating
// point is implemented but i'll figure it out // point is implemented but i'll figure it out
ASTNodeResult tokenize_number(const char *input, size_t *offset) { TokenResult tokenize_number(const char *input, size_t *offset) {
char buf[64] = { '\0' }; char buf[64] = { '\0' };
size_t buf_pos = 0; size_t buf_pos = 0;
bool is_integer = true; // Will later be used to differentiate fractions bool is_integer = true; // Will later be used to differentiate fractions
@@ -71,7 +69,7 @@ ASTNodeResult tokenize_number(const char *input, size_t *offset) {
buf[buf_pos] = input[current]; buf[buf_pos] = input[current];
if (buf_pos >= sizeof(buf)) { if (buf_pos >= sizeof(buf)) {
return (ASTNodeResult) { return (TokenResult) {
.is_valid = false, .is_valid = false,
.err = LEXER_BUF_OVERFLOW}; .err = LEXER_BUF_OVERFLOW};
} }
@@ -80,23 +78,23 @@ ASTNodeResult tokenize_number(const char *input, size_t *offset) {
buf_pos++; buf_pos++;
} }
ASTNode new_node; Token new_token;
if (is_integer) { if (is_integer) {
new_node.type = NODE_INTEGER; new_token.type = TOKEN_INTEGER;
LexerI64Result status = string_to_integer(buf); LexerI64Result result = string_to_integer(buf);
if (!status.is_valid) { if (!result.is_valid) {
return (ASTNodeResult) {.is_valid = false, .err = status.err}; return (TokenResult) {.is_valid = false, .err = result.err};
} }
new_node.data.integer = status.number; new_token.num = result.num;
*offset = current; *offset = current;
return (ASTNodeResult) {.is_valid = true, .node = new_node}; return (TokenResult) {.is_valid = true, .token = new_token};
} }
return (ASTNodeResult) { return (TokenResult) {
.is_valid = false, .is_valid = false,
.err = LEXER_FAILED_NUMBER_CONVERSION}; .err = LEXER_FAILED_NUMBER_CONVERSION};
} }
@@ -122,7 +120,7 @@ LexerI64Result string_to_integer(const char *buf) {
c++; c++;
} }
return (LexerI64Result) {.is_valid = true, .number = count}; return (LexerI64Result) {.is_valid = true, .num = count};
} }
bool isoperator(int c) { bool isoperator(int c) {