diff --git a/4coder_mem.h b/4coder_mem.h index 242f7584..e960893c 100644 --- a/4coder_mem.h +++ b/4coder_mem.h @@ -55,7 +55,9 @@ inline Partition partition_sub_part(Partition *data, int32_t size){ Partition result = {}; void *d = partition_allocate(data, size); - if (d) result = make_part(d, size); + if (d){ + result = make_part(d, size); + } return result; } diff --git a/4ed.cpp b/4ed.cpp index 65f1899b..045fe379 100644 --- a/4ed.cpp +++ b/4ed.cpp @@ -478,7 +478,6 @@ COMMAND_DECL(toggle_line_wrap){ } COMMAND_DECL(toggle_tokens){ -#if BUFFER_EXPERIMENT_SCALPEL <= 0 USE_MODELS(models); REQ_OPEN_VIEW(view); REQ_FILE(file, view); @@ -487,14 +486,12 @@ COMMAND_DECL(toggle_tokens){ file_kill_tokens(system, &models->mem.general, file); } else{ - file_first_lex_parallel(system, &models->mem.general, file); + file_first_lex_parallel(system, &models->mem, file); } -#endif } internal void case_change_range(System_Functions *system, Mem_Options *mem, View *view, Editing_File *file, u8 a, u8 z, u8 char_delta){ -#if BUFFER_EXPERIMENT_SCALPEL <= 0 Range range = make_range(view->edit_pos->cursor.pos, view->edit_pos->mark); if (range.start < range.end){ Edit_Step step = {}; @@ -519,7 +516,6 @@ case_change_range(System_Functions *system, Mem_Options *mem, View *view, Editin if (file->state.token_array.tokens) file_relex_parallel(system, mem, file, range.start, range.end, 0); } -#endif } COMMAND_DECL(open_panel_vsplit){ diff --git a/4ed_api_implementation.cpp b/4ed_api_implementation.cpp index ce7b46a0..71c8ef9d 100644 --- a/4ed_api_implementation.cpp +++ b/4ed_api_implementation.cpp @@ -743,12 +743,12 @@ DOC_SEE(Buffer_Setting_ID) Models *models = cmd->models; Editing_File *file = imp_get_file(cmd, buffer); - bool32 result = false; + bool32 result = 0; i32 new_mapid = 0; if (file){ - result = true; + result = 11; switch (setting){ case BufferSetting_Lex: { @@ -759,7 +759,12 @@ DOC_SEE(Buffer_Setting_ID) } else{ if (value){ - file_first_lex_parallel(system, &models->mem.general, file); + if (!file->settings.virtual_white){ + file_first_lex_parallel(system, &models->mem, file); + } + else{ + file_first_lex_serial(system, &models->mem, file); + } } } }break; @@ -837,8 +842,13 @@ DOC_SEE(Buffer_Setting_ID) b32 full_remeasure = 0; if (value){ if (!file->settings.virtual_white){ + if (file->settings.tokens_exist && !file->state.still_lexing){ file->settings.virtual_white = 1; full_remeasure = 1; + } + else{ + result = 0; + } } } else{ @@ -858,6 +868,8 @@ DOC_SEE(Buffer_Setting_ID) file_update_cursor_positions(models, file); } }break; + + default: result = 0; break; } fill_buffer_summary(buffer, file, cmd); } diff --git a/4ed_defines.h b/4ed_defines.h index 0af387f2..758bed8b 100644 --- a/4ed_defines.h +++ b/4ed_defines.h @@ -14,14 +14,14 @@ #include typedef uint8_t u8; -typedef uint64_t u64; -typedef uint32_t u32; typedef uint16_t u16; +typedef uint32_t u32; +typedef uint64_t u64; typedef int8_t i8; -typedef int64_t i64; -typedef int32_t i32; typedef int16_t i16; +typedef int32_t i32; +typedef int64_t i64; typedef i32 bool32; typedef i8 bool8; diff --git a/4ed_file_view.cpp b/4ed_file_view.cpp index c3edf868..a015679c 100644 --- a/4ed_file_view.cpp +++ b/4ed_file_view.cpp @@ -1955,9 +1955,9 @@ Job_Callback_Sig(job_full_lex){ Buffer_Type *buffer = &file->state.buffer; i32 text_size = buffer_size(buffer); - i32 buffer_size = (text_size + 3)&(~3); + i32 aligned_buffer_size = (text_size + 3)&(~3); - while (memory->size < buffer_size){ + while (memory->size < aligned_buffer_size){ system->grow_thread_memory(memory); } @@ -2019,8 +2019,7 @@ Job_Callback_Sig(job_full_lex){ system->acquire_lock(FRAME_LOCK); { Assert(file->state.swap_array.tokens == 0); - file->state.swap_array.tokens = (Cpp_Token*) - general_memory_allocate(system, general, new_max*sizeof(Cpp_Token)); + file->state.swap_array.tokens = (Cpp_Token*) general_memory_allocate(system, general, new_max*sizeof(Cpp_Token)); } system->release_lock(FRAME_LOCK); @@ -2049,10 +2048,8 @@ Job_Callback_Sig(job_full_lex){ file->state.still_lexing = 0; } - internal void -file_kill_tokens(System_Functions *system, - General_Memory *general, Editing_File *file){ +file_kill_tokens(System_Functions *system, General_Memory *general, Editing_File *file){ file->settings.tokens_exist = 0; if (file->state.still_lexing){ system->cancel_job(BACKGROUND_THREADS, file->state.lex_job); @@ -2069,8 +2066,8 @@ file_kill_tokens(System_Functions *system, } internal void -file_first_lex_parallel(System_Functions *system, - General_Memory *general, Editing_File *file){ +file_first_lex_parallel(System_Functions *system, Mem_Options *mem, Editing_File *file){ + General_Memory *general = &mem->general; file->settings.tokens_exist = 1; if (file->is_loading == 0 && file->state.still_lexing == 0){ @@ -2087,15 +2084,106 @@ file_first_lex_parallel(System_Functions *system, } } +internal void +file_first_lex_serial(System_Functions *system, Mem_Options *mem, Editing_File *file){ + Partition *part = &mem->part; + General_Memory *general = &mem->general; + file->settings.tokens_exist = 1; + + Assert(file->state.still_lexing == 0); + + if (file->is_loading == 0){ + Assert(file->state.token_array.tokens == 0); + + { + Temp_Memory temp = begin_temp_memory(part); + + Buffer_Type *buffer = &file->state.buffer; + i32 text_size = buffer_size(buffer); + + i32 mem_size = partition_remaining(part); + + Cpp_Token_Array tokens; + tokens.max_count = mem_size / sizeof(Cpp_Token); + tokens.count = 0; + tokens.tokens = push_array(part, Cpp_Token, tokens.max_count); + + b32 still_lexing = 1; + + Cpp_Lex_Data lex = cpp_lex_data_init(); + + // TODO(allen): deduplicate this against relex + char *chunks[3]; + i32 chunk_sizes[3]; + + chunks[0] = buffer->data; + chunk_sizes[0] = buffer->size1; + + chunks[1] = buffer->data + buffer->size1 + buffer->gap_size; + chunk_sizes[1] = buffer->size2; + + chunks[2] = 0; + chunk_sizes[2] = 0; + + i32 chunk_index = 0; + + do{ + char *chunk = chunks[chunk_index]; + i32 chunk_size = chunk_sizes[chunk_index]; + + i32 result = cpp_lex_step(&lex, chunk, chunk_size, text_size, &tokens, NO_OUT_LIMIT); + + switch (result){ + case LexResult_NeedChunk: ++chunk_index; break; + case LexResult_NeedTokenMemory: InvalidCodePath; + case LexResult_HitTokenLimit: InvalidCodePath; + case LexResult_Finished: still_lexing = 0; break; + } + } while (still_lexing); + + i32 new_max = LargeRoundUp(tokens.count+1, Kbytes(1)); + + { + Assert(file->state.swap_array.tokens == 0); + file->state.swap_array.tokens = (Cpp_Token*) general_memory_allocate(system, general, new_max*sizeof(Cpp_Token)); + } + + u8 *dest = (u8*)file->state.swap_array.tokens; + u8 *src = (u8*)tokens.tokens; + + memcpy(dest, src, tokens.count*sizeof(Cpp_Token)); + + { + Cpp_Token_Array *file_token_array = &file->state.token_array; + file_token_array->count = tokens.count; + file_token_array->max_count = new_max; + if (file_token_array->tokens){ + general_memory_free(system, general, file_token_array->tokens); + } + file_token_array->tokens = file->state.swap_array.tokens; + file->state.swap_array.tokens = 0; + } + + // NOTE(allen): These are outside the locked section because I don't + // think getting these out of order will cause critical bugs, and I + // want to minimize what's done in locked sections. + file->state.tokens_complete = 1; + file->state.still_lexing = 0; + + end_temp_memory(temp); + } + + file->state.tokens_complete = 1; + } +} + internal b32 -file_relex_parallel(System_Functions *system, - Mem_Options *mem, Editing_File *file, - i32 start_i, i32 end_i, i32 shift_amount){ +file_relex_parallel(System_Functions *system, Mem_Options *mem, Editing_File *file, i32 start_i, i32 end_i, i32 shift_amount){ General_Memory *general = &mem->general; Partition *part = &mem->part; if (file->state.token_array.tokens == 0){ - file_first_lex_parallel(system, general, file); + file_first_lex_parallel(system, mem, file); return(0); } @@ -2106,8 +2194,7 @@ file_relex_parallel(System_Functions *system, i32 extra_tolerance = 100; Cpp_Token_Array *array = &file->state.token_array; - Cpp_Relex_Range relex_range = - cpp_get_relex_range(array, start_i, end_i); + Cpp_Relex_Range relex_range = cpp_get_relex_range(array, start_i, end_i); i32 relex_space_size = relex_range.end_token_index - relex_range.start_token_index + extra_tolerance; @@ -2135,7 +2222,6 @@ file_relex_parallel(System_Functions *system, chunk_sizes[2] = 0; i32 chunk_index = 0; - char *chunk = chunks[chunk_index]; i32 chunk_size = chunk_sizes[chunk_index]; @@ -2218,6 +2304,85 @@ file_relex_parallel(System_Functions *system, return(result); } +internal b32 +file_relex_serial(System_Functions *system, Mem_Options *mem, Editing_File *file, i32 start_i, i32 end_i, i32 shift_amount){ + General_Memory *general = &mem->general; + Partition *part = &mem->part; + + if (file->state.token_array.tokens == 0){ + file_first_lex_serial(system, mem, file); + return(1); + } + + Assert(!file->state.still_lexing); + + Buffer_Type *buffer = &file->state.buffer; + Cpp_Token_Array *array = &file->state.token_array; + + Temp_Memory temp = begin_temp_memory(part); + Cpp_Token_Array relex_array; + relex_array.count = 0; + relex_array.max_count = partition_remaining(part) / sizeof(Cpp_Token); + relex_array.tokens = push_array(part, Cpp_Token, relex_array.max_count); + + i32 size = buffer_size(buffer); + + Cpp_Relex_Data state = cpp_relex_init(array, start_i, end_i, shift_amount); + + char *chunks[3]; + i32 chunk_sizes[3]; + + chunks[0] = buffer->data; + chunk_sizes[0] = buffer->size1; + + chunks[1] = buffer->data + buffer->size1 + buffer->gap_size; + chunk_sizes[1] = buffer->size2; + + chunks[2] = 0; + chunk_sizes[2] = 0; + + i32 chunk_index = 0; + char *chunk = chunks[chunk_index]; + i32 chunk_size = chunk_sizes[chunk_index]; + + while (!cpp_relex_is_start_chunk(&state, chunk, chunk_size)){ + ++chunk_index; + chunk = chunks[chunk_index]; + chunk_size = chunk_sizes[chunk_index]; + } + + for(;;){ + Cpp_Lex_Result lex_result = cpp_relex_step(&state, chunk, chunk_size, size, array, &relex_array); + + switch (lex_result){ + case LexResult_NeedChunk: + ++chunk_index; + chunk = chunks[chunk_index]; + chunk_size = chunk_sizes[chunk_index]; + break; + + case LexResult_NeedTokenMemory: InvalidCodePath; + + case LexResult_Finished: goto doublebreak; + } + } + doublebreak:; + + i32 new_count = cpp_relex_get_new_count(&state, array->count, &relex_array); + if (new_count > array->max_count){ + i32 new_max = LargeRoundUp(new_count, Kbytes(1)); + array->tokens = (Cpp_Token*) + general_memory_reallocate(system, general, array->tokens, array->count*sizeof(Cpp_Token), new_max*sizeof(Cpp_Token)); + array->max_count = new_max; + } + + cpp_relex_complete(&state, array, &relex_array); + + end_temp_memory(temp); + + return(1); +} + internal void undo_stack_grow_string(System_Functions *system, General_Memory *general, Edit_Stack *stack, i32 extra_size){ i32 old_max = stack->max; @@ -2903,7 +3068,12 @@ file_do_single_edit(System_Functions *system, Models *models, Editing_File *file // NOTE(allen): token fixing if (file->settings.tokens_exist){ + if (!file->settings.virtual_white){ file_relex_parallel(system, mem, file, start, end, shift_amount); + } + else{ + file_relex_serial(system, mem, file, start, end, shift_amount); + } } // NOTE(allen): meta data @@ -2989,7 +3159,13 @@ file_do_batch_edit(System_Functions *system, Models *models, Editing_File *file, // TODO(allen): Write a smart fast one here someday. Buffer_Edit *first_edit = batch; Buffer_Edit *last_edit = batch + batch_size - 1; + + if (!file->settings.virtual_white){ file_relex_parallel(system, mem, file, first_edit->start, last_edit->end, shift_total); + } + else{ + file_relex_serial(system, mem, file, first_edit->start, last_edit->end, shift_total); + } } }break; @@ -3500,27 +3676,34 @@ make_string_terminated(Partition *part, char *str, i32 len){ } internal void -init_normal_file(System_Functions *system, Models *models, Editing_File *file, - char *buffer, i32 size){ - General_Memory *general = &models->mem.general; - - String val = make_string(buffer, size); +init_normal_file(System_Functions *system, Models *models, Editing_File *file, char *buffer, i32 size){ + String val = make_string(buffer, size); file_create_from_string(system, models, file, val); + Mem_Options *mem = &models->mem; if (file->settings.tokens_exist && file->state.token_array.tokens == 0){ - file_first_lex_parallel(system, general, file); + if (!file->settings.virtual_white){ + file_first_lex_parallel(system, mem, file); + } + else{ + file_first_lex_serial(system, mem, file); + } } } internal void init_read_only_file(System_Functions *system, Models *models, Editing_File *file){ - General_Memory *general = &models->mem.general; - String val = null_string; file_create_from_string(system, models, file, val, 1); + Mem_Options *mem = &models->mem; if (file->settings.tokens_exist && file->state.token_array.tokens == 0){ - file_first_lex_parallel(system, general, file); + if (!file->settings.virtual_white){ + file_first_lex_parallel(system, mem, file); + } + else{ + file_first_lex_serial(system, mem, file); + } } }