208 lines
7.6 KiB
C++
208 lines
7.6 KiB
C++
/*
|
|
* Mr. 4th Dimention - Allen Webster
|
|
*
|
|
* 19.08.2018
|
|
*
|
|
* Ways of calling lexer for the text in Editing_File
|
|
*
|
|
*/
|
|
|
|
// TOP
|
|
|
|
internal void
|
|
file_kill_tokens(System_Functions *system, Heap *heap, Editing_File *file){
|
|
file->settings.tokens_exist = false;
|
|
if (file->state.token_array.tokens != 0){
|
|
heap_free(heap, file->state.token_array.tokens);
|
|
}
|
|
block_zero_struct(&file->state.token_array);
|
|
}
|
|
|
|
internal void
|
|
file_first_lex_serial(System_Functions *system, Models *models, Editing_File *file){
|
|
Mem_Options *mem = &models->mem;
|
|
Arena *scratch = &mem->arena;
|
|
Heap *heap = &mem->heap;
|
|
file->settings.tokens_exist = true;
|
|
|
|
if (file->is_loading == 0){
|
|
Assert(file->state.token_array.tokens == 0);
|
|
|
|
Temp_Memory temp = begin_temp(scratch);
|
|
|
|
Parse_Context parse_context = parse_context_get(scratch, &models->parse_context_memory, file->settings.parse_context_id);
|
|
Assert(parse_context.valid);
|
|
|
|
Gap_Buffer *buffer = &file->state.buffer;
|
|
i32 text_size = buffer_size(buffer);
|
|
|
|
// TODO(allen): REWRITE REWRITE REWRITE
|
|
Cpp_Token_Array new_tokens = {};
|
|
new_tokens.max_count = Million(1);
|
|
new_tokens.count = 0;
|
|
new_tokens.tokens = push_array(scratch, Cpp_Token, new_tokens.max_count);
|
|
|
|
b32 still_lexing = true;
|
|
|
|
Cpp_Lex_Data lex = cpp_lex_data_init(file->settings.tokens_without_strings, parse_context.kw_table, parse_context.pp_table);
|
|
|
|
String_Const_u8_Array chunk_space[3];
|
|
Cursor chunk_cursor = make_cursor(chunk_space, sizeof(chunk_space));
|
|
String_Const_u8_Array chunks = buffer_get_chunks(&chunk_cursor, buffer, BufferGetChunk_ZeroTerminated);
|
|
|
|
i32 chunk_index = 0;
|
|
|
|
Cpp_Token_Array new_array = {};
|
|
|
|
do{
|
|
u8 *chunk = chunks.vals[chunk_index].str;
|
|
umem chunk_size = chunks.vals[chunk_index].size;
|
|
|
|
i32 result = cpp_lex_step(&lex, (char*)chunk, (i32)chunk_size, text_size, &new_tokens, NO_OUT_LIMIT);
|
|
|
|
switch (result){
|
|
case LexResult_NeedChunk:
|
|
{
|
|
++chunk_index;
|
|
Assert(chunk_index < chunks.count);
|
|
}break;
|
|
|
|
case LexResult_Finished:
|
|
case LexResult_NeedTokenMemory:
|
|
{
|
|
u32 new_max = round_up_u32(new_array.count + new_tokens.count + 1, KB(1));
|
|
if (new_array.tokens == 0){
|
|
new_array.tokens = heap_array(heap, Cpp_Token, new_max);
|
|
}
|
|
else{
|
|
u32 old_count = new_array.count;
|
|
Cpp_Token *new_token_mem = heap_array(heap, Cpp_Token, new_max);
|
|
memcpy(new_token_mem, new_array.tokens, sizeof(*new_token_mem)*old_count);
|
|
heap_free(heap, new_array.tokens);
|
|
new_array.tokens = new_token_mem;
|
|
}
|
|
new_array.max_count = new_max;
|
|
|
|
Assert(new_array.count + new_tokens.count <= new_array.max_count);
|
|
memcpy(new_array.tokens + new_array.count, new_tokens.tokens, new_tokens.count*sizeof(Cpp_Token));
|
|
new_array.count += new_tokens.count;
|
|
new_tokens.count = 0;
|
|
|
|
if (result == LexResult_Finished){
|
|
still_lexing = false;
|
|
}
|
|
}break;
|
|
|
|
case LexResult_HitTokenLimit: InvalidPath;
|
|
}
|
|
} while (still_lexing);
|
|
|
|
Cpp_Token_Array *token_array = &file->state.token_array;
|
|
token_array->count = new_array.count;
|
|
token_array->max_count = new_array.max_count;
|
|
if (token_array->tokens != 0){
|
|
heap_free(heap, token_array->tokens);
|
|
}
|
|
token_array->tokens = new_array.tokens;
|
|
|
|
new_array.tokens = 0;
|
|
new_array.count = 0;
|
|
new_array.max_count = 0;
|
|
|
|
end_temp(temp);
|
|
|
|
file_mark_edit_finished(&models->working_set, file);
|
|
}
|
|
}
|
|
|
|
internal b32
|
|
file_relex_serial(System_Functions *system, Models *models, Editing_File *file, i32 start_i, i32 end_i, i32 shift_amount){
|
|
Mem_Options *mem = &models->mem;
|
|
Arena *scratch = &mem->arena;
|
|
Heap *heap = &mem->heap;
|
|
|
|
if (file->state.token_array.tokens == 0){
|
|
file_first_lex_serial(system, models, file);
|
|
}
|
|
else{
|
|
Gap_Buffer *buffer = &file->state.buffer;
|
|
Cpp_Token_Array *array = &file->state.token_array;
|
|
|
|
Temp_Memory temp = begin_temp(scratch);
|
|
Parse_Context parse_context = parse_context_get(scratch, &models->parse_context_memory, file->settings.parse_context_id);
|
|
Assert(parse_context.valid);
|
|
|
|
Cpp_Token_Array relex_array = {};
|
|
relex_array.count = 0;
|
|
relex_array.max_count = Million(1);
|
|
relex_array.tokens = push_array(scratch, Cpp_Token, relex_array.max_count);
|
|
|
|
i32 size = buffer_size(buffer);
|
|
|
|
Cpp_Relex_Data state = cpp_relex_init(array, start_i, end_i, shift_amount, file->settings.tokens_without_strings, parse_context.kw_table, parse_context.pp_table);
|
|
|
|
String_Const_u8_Array chunk_space[3];
|
|
Cursor chunk_cursor = make_cursor(chunk_space, sizeof(chunk_space));
|
|
String_Const_u8_Array chunks = buffer_get_chunks(&chunk_cursor, buffer, BufferGetChunk_ZeroTerminated);
|
|
|
|
i32 chunk_index = 0;
|
|
u8 *chunk = chunks.vals[chunk_index].str;
|
|
umem chunk_size = chunks.vals[chunk_index].size;
|
|
|
|
for (;!cpp_relex_is_start_chunk(&state, (char*)chunk, (i32)chunk_size);){
|
|
++chunk_index;
|
|
Assert(chunk_index < chunks.count);
|
|
chunk = chunks.vals[chunk_index].str;
|
|
chunk_size = chunks.vals[chunk_index].size;
|
|
}
|
|
|
|
for(;;){
|
|
Cpp_Lex_Result lex_result = cpp_relex_step(&state, (char*)chunk, (i32)chunk_size, size, array, &relex_array);
|
|
|
|
switch (lex_result){
|
|
case LexResult_NeedChunk:
|
|
{
|
|
++chunk_index;
|
|
Assert(chunk_index < chunks.count);
|
|
chunk = chunks.vals[chunk_index].str;
|
|
chunk_size = chunks.vals[chunk_index].size;
|
|
}break;
|
|
|
|
case LexResult_NeedTokenMemory: InvalidPath;
|
|
|
|
case LexResult_Finished: goto doublebreak;
|
|
}
|
|
}
|
|
doublebreak:;
|
|
|
|
i32 new_count = cpp_relex_get_new_count(&state, array->count, &relex_array);
|
|
if (new_count > array->max_count){
|
|
i32 new_max = round_up_i32(new_count, KB(1));
|
|
Cpp_Token *new_tokens = heap_array(heap, Cpp_Token, new_max);
|
|
memcpy(new_tokens, array->tokens, array->count*sizeof(Cpp_Token));
|
|
heap_free(heap, array->tokens);
|
|
array->tokens = new_tokens;
|
|
array->max_count = new_max;
|
|
}
|
|
|
|
cpp_relex_complete(&state, array, &relex_array);
|
|
|
|
end_temp(temp);
|
|
}
|
|
|
|
return(true);
|
|
}
|
|
|
|
internal void
|
|
file_first_lex(System_Functions *system, Models *models, Editing_File *file){
|
|
file_first_lex_serial(system, models, file);
|
|
}
|
|
|
|
internal void
|
|
file_relex(System_Functions *system, Models *models, Editing_File *file, i32 start, i32 end, i32 shift_amount){
|
|
file_relex_serial(system, models, file, start, end, shift_amount);
|
|
}
|
|
|
|
// BOTTOM
|
|
|