Add ability to import other files

This still requires a lot of work to avoid duplicate imports, and
handle cyclic imports, but it is a good enough for small examples
which just want to include some common definitions from std/

4 files changed, 62 insertions, 3 deletions

diff --git a/src/lexer.c b/src/lexer.c
index 524e87f..af5a14c 100644
--- a/src/lexer.c
+++ b/src/lexer.c
@@ -14,6 +14,25 @@ Lexer *Lexer_new(char *filename, char *src, i64 len)
     return self;
 }
 
+Lexer *Lexer_new_open_file(char *filename)
+{
+    FILE *fp = fopen(filename, "r");
+    fseek(fp, 0, SEEK_END);
+    i64 source_len = ftell(fp);
+    fseek(fp, 0, SEEK_SET);
+    char *source = malloc(source_len + 1);
+    fread(source, source_len, 1, fp);
+    source[source_len] = 0;
+    fclose(fp);
+
+    Lexer *self = calloc(1, sizeof(Lexer));
+    self->filename = filename;
+    self->src = source;
+    self->len = source_len;
+
+    return self;
+}
+
 static Location Lexer_loc(Lexer *lexer)
 {
     Location loc = {0};
@@ -201,7 +220,7 @@ Token Lexer_next(Lexer *lexer)
                 // Careful with indexing here, because we want to skip opening and closing quotes
                 char *str = calloc(pos - lexer->pos, 1);
                 strncpy(str, lexer->src + lexer->pos + 1, pos - lexer->pos - 1);
-                Token token = Token_from_identifier(str, Lexer_loc(lexer));
+                Token token = Token_from_string(str, Lexer_loc(lexer));
                 advance(lexer, pos - lexer->pos + 1);
                 return token;
             }
diff --git a/src/lexer.h b/src/lexer.h
index b7f3db2..96bed26 100644
--- a/src/lexer.h
+++ b/src/lexer.h
@@ -14,6 +14,7 @@ typedef struct {
 } Lexer;
 
 Lexer *Lexer_new(char *filename, char *src, i64 len);
+Lexer *Lexer_new_open_file(char *filename);
 
 Token Lexer_next(Lexer *lexer);
 Token Lexer_peek(Lexer *lexer);
\ No newline at end of file
diff --git a/src/parser.c b/src/parser.c
index 128f18a..4dc891d 100644
--- a/src/parser.c
+++ b/src/parser.c
@@ -15,6 +15,10 @@ static Node *block_stack[BLOCK_STACK_SIZE];
 static i64 block_stack_count = 0;
 static i64 cur_stack_offset = 0;
 
+#define LEXER_STACK_SIZE 64
+static Lexer *lexer_stack[LEXER_STACK_SIZE];
+static i64 lexer_stack_count = 0;
+
 
 Token do_assert_token(Token token, TokenType type, char *filename, int line)
 {
@@ -576,20 +580,54 @@ Node *parse_func(Lexer *lexer)
     return func;
 }
 
+void push_new_lexer(Lexer *lexer)
+{
+    assert(lexer_stack_count < LEXER_STACK_SIZE);
+    lexer_stack[lexer_stack_count++] = lexer;
+}
+
+Lexer *remove_lexer()
+{
+    assert(lexer_stack_count > 0);
+    free(lexer_stack[--lexer_stack_count]);
+    if (lexer_stack_count == 0)
+        return NULL;
+    return lexer_stack[lexer_stack_count - 1];
+}
+
 Node *parse_program(Lexer *lexer)
 {
     initialize_builtins();
     Node *program = Node_new(AST_PROGRAM);
-    Token token;
-    while ((token = Lexer_peek(lexer)).type != TOKEN_EOF) {
+    
+    push_new_lexer(lexer);
+    
+    Token token = Lexer_peek(lexer);
+    while (token.type != TOKEN_EOF) {
         if (token.type == TOKEN_FN) {
             Node *func = parse_func(lexer);
             Node_add_child(program, func);
+        } else if (token.type == TOKEN_IMPORT) {
+            // TODO: Handle circular imports
+            // TODO: Handle complex import graphs (#pragma once)
+            // TODO: Validation of imports
+            // TODO: Have default directories to search for imports
+            Lexer_next(lexer);
+            token = assert_token(Lexer_next(lexer), TOKEN_STRINGLIT);
+            char *filename = token.value.as_string;
+            lexer = Lexer_new_open_file(filename);
+            push_new_lexer(lexer);
         } else {
             die_location(token.loc, "Unexpected token in parse_program: `%s`\n", token_type_to_str(token.type));
             exit(1);
             break;
         }
+
+        token = Lexer_peek(lexer);
+        while (token.type == TOKEN_EOF && lexer_stack_count > 1) {
+            lexer = remove_lexer();
+            token = Lexer_peek(lexer);
+        }
     }
     return program;
 }
\ No newline at end of file
diff --git a/src/tokens.h b/src/tokens.h
index 3917f0f..7af4780 100644
--- a/src/tokens.h
+++ b/src/tokens.h
@@ -51,6 +51,7 @@
   F(TOKEN_LET, "let")                                                          \
   F(TOKEN_RETURN, "return")                                                    \
   F(TOKEN_WHILE, "while")                                                      \
+  F(TOKEN_IMPORT, "import")                                                    \
 
 
 typedef enum {