refactor: delete ASTNode, add Token to lexer

So, total refactor, now we serious. I feel ASTNode was feeling very
bloated so we need to rewrite and adapt everything, by now lets get the
lexer working again, is already well written for me at least.

include/lexer.h

@@ -8,12 +8,9 @@
 
 // For identifing
 typedef enum {
-    NODE_INTEGER,
+    TOKEN_INTEGER,
-    NODE_BINARY_OP,
+    TOKEN_OPERATOR,
-    NODE_UNARY_OP,
+} TokenType;
-    NODE_PARENTHESIS,
-} ASTNodeType;
-
 // For classify operators
 typedef enum {
     OP_ADD,
@@ -36,25 +33,13 @@ typedef enum {
 } LexerErr;
 
 // Can be thought as tokens, they will be used by the parser.
-typedef struct ASTNode {
+typedef struct {
-    ASTNodeType type;
+    TokenType type;
     union {
-        int64_t integer;
+        int64_t num;
-        struct {
+        Operator op;
-            struct ASTNode *left;
+    };
-            struct ASTNode *right;
+} Token;
-            Operator op;
-        } binary;
-        struct {
-            struct ASTNode *val;
-            Operator op;
-        } unary;
-        struct {
-            struct ASTNode *val;
-            Operator op;
-        } parenthesis;
-    } data;
-} ASTNode;
 
 typedef struct {
     bool is_valid;
@@ -68,9 +53,9 @@ typedef struct {
     bool is_valid;
     union {
         LexerErr err;
-        ASTNode node;
+        Token token;
     };
-} ASTNodeResult;
+} TokenResult;
 
 typedef struct {
     bool is_valid;
@@ -82,7 +67,7 @@ typedef struct {
 
 // Lexer funtions as well as few functionality
 TokenizeResult tokenize(const char* input);
-ASTNodeResult tokenize_number(const char* input, size_t *offset);
+TokenResult tokenize_number(const char* input, size_t *offset);
 LexerI64Result string_to_integer(const char buf[]);
 bool isoperator(int c);
 Operator char_to_operator(int c);