Commit 49c5c117 authored by Guillaume Charifi's avatar Guillaume Charifi

compiler/lexer: Allocate token array by chunks.

parent c31b03ca
......@@ -27,6 +27,8 @@
#include "lexer.h"
#include "utils.h"
#define TOKEN_CHUNK_SIZE_INV_FACTOR 10
typedef int (*token_handler_callback)(struct bm_token *token, const wchar_t *line, const wchar_t **cursor, const wchar_t *keyword, enum bm_token_type type);
struct token_handler
......@@ -134,14 +136,21 @@ int token_deinit(struct bm_token *token)
return 0;
}
static int output_append_token(struct lexer_output *out, struct bm_token *token)
static int output_append_token(struct lexer_output *out, struct bm_token *token, size_t in_file_size)
{
struct bm_token *tokens;
tokens = realloc(out->tokens, (out->nb_tokens + 1) * sizeof(out->tokens[0]));
if (!tokens)
return -1;
out->tokens = tokens;
if (out->nb_tokens + 1 > out->nb_tokens_allocated)
{
size_t incr = in_file_size / TOKEN_CHUNK_SIZE_INV_FACTOR;
tokens = realloc(out->tokens, (out->nb_tokens_allocated + incr) * sizeof(out->tokens[0]));
if (!tokens)
return -1;
out->nb_tokens_allocated += incr;
out->tokens = tokens;
}
out->tokens[out->nb_tokens++] = *token;
return 0;
......@@ -533,6 +542,7 @@ int lexer(const wchar_t *in, struct lexer_output *out)
unsigned long count;
out->nb_tokens = 0;
out->nb_tokens_allocated = 0;
out->tokens = NULL;
buffer_init_from_split_str(&buf, in, L'\n');
......@@ -556,11 +566,11 @@ int lexer(const wchar_t *in, struct lexer_output *out)
goto leave;
}
output_append_token(out, &token);
output_append_token(out, &token, buf.total_size);
}
token_init(&token, BM_TOK_NEWLINE);
output_append_token(out, &token);
output_append_token(out, &token, buf.total_size);
wprintf(L"\n");
}
......@@ -578,6 +588,7 @@ int lexer_destroy(struct lexer_output *out)
out->tokens = NULL;
out->nb_tokens = 0;
out->nb_tokens_allocated = 0;
return 0;
}
......
......@@ -26,6 +26,7 @@
struct lexer_output
{
unsigned long nb_tokens;
unsigned long nb_tokens_allocated;
struct bm_token *tokens;
};
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment