/* * Mr. 4th Dimention - Allen Webster * * 19.08.2018 * * Ways of calling lexer for the text in Editing_File * */ // TOP internal void file_kill_tokens(System_Functions *system, Heap *heap, Editing_File *file){ file->settings.tokens_exist = false; if (file->state.token_array.tokens != 0){ heap_free(heap, file->state.token_array.tokens); } block_zero_struct(&file->state.token_array); } internal void file_first_lex_serial(System_Functions *system, Models *models, Editing_File *file){ Mem_Options *mem = &models->mem; Arena *scratch = &mem->arena; Heap *heap = &mem->heap; file->settings.tokens_exist = true; if (file->is_loading == 0){ Assert(file->state.token_array.tokens == 0); Temp_Memory temp = begin_temp(scratch); Parse_Context parse_context = parse_context_get(scratch, &models->parse_context_memory, file->settings.parse_context_id); Assert(parse_context.valid); Gap_Buffer *buffer = &file->state.buffer; i32 text_size = buffer_size(buffer); // TODO(allen): REWRITE REWRITE REWRITE Cpp_Token_Array new_tokens = {}; new_tokens.max_count = Million(1); new_tokens.count = 0; new_tokens.tokens = push_array(scratch, Cpp_Token, new_tokens.max_count); b32 still_lexing = true; Cpp_Lex_Data lex = cpp_lex_data_init(file->settings.tokens_without_strings, parse_context.kw_table, parse_context.pp_table); String_Const_u8_Array chunk_space[3]; Cursor chunk_cursor = make_cursor(chunk_space, sizeof(chunk_space)); String_Const_u8_Array chunks = buffer_get_chunks(&chunk_cursor, buffer, BufferGetChunk_ZeroTerminated); i32 chunk_index = 0; Cpp_Token_Array new_array = {}; do{ u8 *chunk = chunks.vals[chunk_index].str; umem chunk_size = chunks.vals[chunk_index].size; i32 result = cpp_lex_step(&lex, (char*)chunk, (i32)chunk_size, text_size, &new_tokens, NO_OUT_LIMIT); switch (result){ case LexResult_NeedChunk: { ++chunk_index; Assert(chunk_index < chunks.count); }break; case LexResult_Finished: case LexResult_NeedTokenMemory: { u32 new_max = round_up_u32(new_array.count + new_tokens.count + 1, KB(1)); if (new_array.tokens == 0){ new_array.tokens = heap_array(heap, Cpp_Token, new_max); } else{ u32 old_count = new_array.count; Cpp_Token *new_token_mem = heap_array(heap, Cpp_Token, new_max); memcpy(new_token_mem, new_array.tokens, sizeof(*new_token_mem)*old_count); heap_free(heap, new_array.tokens); new_array.tokens = new_token_mem; } new_array.max_count = new_max; Assert(new_array.count + new_tokens.count <= new_array.max_count); memcpy(new_array.tokens + new_array.count, new_tokens.tokens, new_tokens.count*sizeof(Cpp_Token)); new_array.count += new_tokens.count; new_tokens.count = 0; if (result == LexResult_Finished){ still_lexing = false; } }break; case LexResult_HitTokenLimit: InvalidPath; } } while (still_lexing); Cpp_Token_Array *token_array = &file->state.token_array; token_array->count = new_array.count; token_array->max_count = new_array.max_count; if (token_array->tokens != 0){ heap_free(heap, token_array->tokens); } token_array->tokens = new_array.tokens; new_array.tokens = 0; new_array.count = 0; new_array.max_count = 0; end_temp(temp); file_mark_edit_finished(&models->working_set, file); } } internal b32 file_relex_serial(System_Functions *system, Models *models, Editing_File *file, i32 start_i, i32 end_i, i32 shift_amount){ Mem_Options *mem = &models->mem; Arena *scratch = &mem->arena; Heap *heap = &mem->heap; if (file->state.token_array.tokens == 0){ file_first_lex_serial(system, models, file); } else{ Gap_Buffer *buffer = &file->state.buffer; Cpp_Token_Array *array = &file->state.token_array; Temp_Memory temp = begin_temp(scratch); Parse_Context parse_context = parse_context_get(scratch, &models->parse_context_memory, file->settings.parse_context_id); Assert(parse_context.valid); Cpp_Token_Array relex_array = {}; relex_array.count = 0; relex_array.max_count = Million(1); relex_array.tokens = push_array(scratch, Cpp_Token, relex_array.max_count); i32 size = buffer_size(buffer); Cpp_Relex_Data state = cpp_relex_init(array, start_i, end_i, shift_amount, file->settings.tokens_without_strings, parse_context.kw_table, parse_context.pp_table); String_Const_u8_Array chunk_space[3]; Cursor chunk_cursor = make_cursor(chunk_space, sizeof(chunk_space)); String_Const_u8_Array chunks = buffer_get_chunks(&chunk_cursor, buffer, BufferGetChunk_ZeroTerminated); i32 chunk_index = 0; u8 *chunk = chunks.vals[chunk_index].str; umem chunk_size = chunks.vals[chunk_index].size; for (;!cpp_relex_is_start_chunk(&state, (char*)chunk, (i32)chunk_size);){ ++chunk_index; Assert(chunk_index < chunks.count); chunk = chunks.vals[chunk_index].str; chunk_size = chunks.vals[chunk_index].size; } for(;;){ Cpp_Lex_Result lex_result = cpp_relex_step(&state, (char*)chunk, (i32)chunk_size, size, array, &relex_array); switch (lex_result){ case LexResult_NeedChunk: { ++chunk_index; Assert(chunk_index < chunks.count); chunk = chunks.vals[chunk_index].str; chunk_size = chunks.vals[chunk_index].size; }break; case LexResult_NeedTokenMemory: InvalidPath; case LexResult_Finished: goto doublebreak; } } doublebreak:; i32 new_count = cpp_relex_get_new_count(&state, array->count, &relex_array); if (new_count > array->max_count){ i32 new_max = round_up_i32(new_count, KB(1)); Cpp_Token *new_tokens = heap_array(heap, Cpp_Token, new_max); memcpy(new_tokens, array->tokens, array->count*sizeof(Cpp_Token)); heap_free(heap, array->tokens); array->tokens = new_tokens; array->max_count = new_max; } cpp_relex_complete(&state, array, &relex_array); end_temp(temp); } return(true); } internal void file_first_lex(System_Functions *system, Models *models, Editing_File *file){ file_first_lex_serial(system, models, file); } internal void file_relex(System_Functions *system, Models *models, Editing_File *file, i32 start, i32 end, i32 shift_amount){ file_relex_serial(system, models, file, start, end, shift_amount); } // BOTTOM