4coder/4ed_file_lex.cpp

208 lines
7.6 KiB
C++

/*
* Mr. 4th Dimention - Allen Webster
*
* 19.08.2018
*
* Ways of calling lexer for the text in Editing_File
*
*/
// TOP
internal void
file_kill_tokens(System_Functions *system, Heap *heap, Editing_File *file){
file->settings.tokens_exist = false;
if (file->state.token_array.tokens != 0){
heap_free(heap, file->state.token_array.tokens);
}
block_zero_struct(&file->state.token_array);
}
internal void
file_first_lex_serial(System_Functions *system, Models *models, Editing_File *file){
Mem_Options *mem = &models->mem;
Arena *scratch = &mem->arena;
Heap *heap = &mem->heap;
file->settings.tokens_exist = true;
if (file->is_loading == 0){
Assert(file->state.token_array.tokens == 0);
Temp_Memory temp = begin_temp(scratch);
Parse_Context parse_context = parse_context_get(scratch, &models->parse_context_memory, file->settings.parse_context_id);
Assert(parse_context.valid);
Gap_Buffer *buffer = &file->state.buffer;
i32 text_size = buffer_size(buffer);
// TODO(allen): REWRITE REWRITE REWRITE
Cpp_Token_Array new_tokens = {};
new_tokens.max_count = Million(1);
new_tokens.count = 0;
new_tokens.tokens = push_array(scratch, Cpp_Token, new_tokens.max_count);
b32 still_lexing = true;
Cpp_Lex_Data lex = cpp_lex_data_init(file->settings.tokens_without_strings, parse_context.kw_table, parse_context.pp_table);
String_Const_u8_Array chunk_space[3];
Cursor chunk_cursor = make_cursor(chunk_space, sizeof(chunk_space));
String_Const_u8_Array chunks = buffer_get_chunks(&chunk_cursor, buffer, BufferGetChunk_ZeroTerminated);
i32 chunk_index = 0;
Cpp_Token_Array new_array = {};
do{
u8 *chunk = chunks.vals[chunk_index].str;
umem chunk_size = chunks.vals[chunk_index].size;
i32 result = cpp_lex_step(&lex, (char*)chunk, (i32)chunk_size, text_size, &new_tokens, NO_OUT_LIMIT);
switch (result){
case LexResult_NeedChunk:
{
++chunk_index;
Assert(chunk_index < chunks.count);
}break;
case LexResult_Finished:
case LexResult_NeedTokenMemory:
{
u32 new_max = round_up_u32(new_array.count + new_tokens.count + 1, KB(1));
if (new_array.tokens == 0){
new_array.tokens = heap_array(heap, Cpp_Token, new_max);
}
else{
u32 old_count = new_array.count;
Cpp_Token *new_token_mem = heap_array(heap, Cpp_Token, new_max);
memcpy(new_token_mem, new_array.tokens, sizeof(*new_token_mem)*old_count);
heap_free(heap, new_array.tokens);
new_array.tokens = new_token_mem;
}
new_array.max_count = new_max;
Assert(new_array.count + new_tokens.count <= new_array.max_count);
memcpy(new_array.tokens + new_array.count, new_tokens.tokens, new_tokens.count*sizeof(Cpp_Token));
new_array.count += new_tokens.count;
new_tokens.count = 0;
if (result == LexResult_Finished){
still_lexing = false;
}
}break;
case LexResult_HitTokenLimit: InvalidPath;
}
} while (still_lexing);
Cpp_Token_Array *token_array = &file->state.token_array;
token_array->count = new_array.count;
token_array->max_count = new_array.max_count;
if (token_array->tokens != 0){
heap_free(heap, token_array->tokens);
}
token_array->tokens = new_array.tokens;
new_array.tokens = 0;
new_array.count = 0;
new_array.max_count = 0;
end_temp(temp);
file_mark_edit_finished(&models->working_set, file);
}
}
internal b32
file_relex_serial(System_Functions *system, Models *models, Editing_File *file, i32 start_i, i32 end_i, i32 shift_amount){
Mem_Options *mem = &models->mem;
Arena *scratch = &mem->arena;
Heap *heap = &mem->heap;
if (file->state.token_array.tokens == 0){
file_first_lex_serial(system, models, file);
}
else{
Gap_Buffer *buffer = &file->state.buffer;
Cpp_Token_Array *array = &file->state.token_array;
Temp_Memory temp = begin_temp(scratch);
Parse_Context parse_context = parse_context_get(scratch, &models->parse_context_memory, file->settings.parse_context_id);
Assert(parse_context.valid);
Cpp_Token_Array relex_array = {};
relex_array.count = 0;
relex_array.max_count = Million(1);
relex_array.tokens = push_array(scratch, Cpp_Token, relex_array.max_count);
i32 size = buffer_size(buffer);
Cpp_Relex_Data state = cpp_relex_init(array, start_i, end_i, shift_amount, file->settings.tokens_without_strings, parse_context.kw_table, parse_context.pp_table);
String_Const_u8_Array chunk_space[3];
Cursor chunk_cursor = make_cursor(chunk_space, sizeof(chunk_space));
String_Const_u8_Array chunks = buffer_get_chunks(&chunk_cursor, buffer, BufferGetChunk_ZeroTerminated);
i32 chunk_index = 0;
u8 *chunk = chunks.vals[chunk_index].str;
umem chunk_size = chunks.vals[chunk_index].size;
for (;!cpp_relex_is_start_chunk(&state, (char*)chunk, (i32)chunk_size);){
++chunk_index;
Assert(chunk_index < chunks.count);
chunk = chunks.vals[chunk_index].str;
chunk_size = chunks.vals[chunk_index].size;
}
for(;;){
Cpp_Lex_Result lex_result = cpp_relex_step(&state, (char*)chunk, (i32)chunk_size, size, array, &relex_array);
switch (lex_result){
case LexResult_NeedChunk:
{
++chunk_index;
Assert(chunk_index < chunks.count);
chunk = chunks.vals[chunk_index].str;
chunk_size = chunks.vals[chunk_index].size;
}break;
case LexResult_NeedTokenMemory: InvalidPath;
case LexResult_Finished: goto doublebreak;
}
}
doublebreak:;
i32 new_count = cpp_relex_get_new_count(&state, array->count, &relex_array);
if (new_count > array->max_count){
i32 new_max = round_up_i32(new_count, KB(1));
Cpp_Token *new_tokens = heap_array(heap, Cpp_Token, new_max);
memcpy(new_tokens, array->tokens, array->count*sizeof(Cpp_Token));
heap_free(heap, array->tokens);
array->tokens = new_tokens;
array->max_count = new_max;
}
cpp_relex_complete(&state, array, &relex_array);
end_temp(temp);
}
return(true);
}
internal void
file_first_lex(System_Functions *system, Models *models, Editing_File *file){
file_first_lex_serial(system, models, file);
}
internal void
file_relex(System_Functions *system, Models *models, Editing_File *file, i32 start, i32 end, i32 shift_amount){
file_relex_serial(system, models, file, start, end, shift_amount);
}
// BOTTOM