From 46241c4450b4b768af327e6c5c2d5f9b676d0b97 Mon Sep 17 00:00:00 2001 From: Allen Webster Date: Thu, 31 Oct 2019 14:42:11 -0700 Subject: [PATCH] Initial code indexer up and running; more time with global frame mutex free; bugs with layout function selection fixed --- 4ed_api_implementation.cpp | 30 ++- custom/4coder_async_tasks.cpp | 30 +-- custom/4coder_async_tasks.h | 1 - custom/4coder_code_index.cpp | 266 ++++++++++++++++++---- custom/4coder_code_index.h | 26 +++ custom/4coder_default_bindings.cpp | 1 + custom/4coder_default_hooks.cpp | 139 +++++------ custom/generated/custom_api.cpp | 2 + custom/generated/custom_api.h | 17 +- custom/generated/custom_api_master_list.h | 5 +- platform_win32/win32_4ed.cpp | 6 + 11 files changed, 363 insertions(+), 160 deletions(-) diff --git a/4ed_api_implementation.cpp b/4ed_api_implementation.cpp index 3e10d68c..131eb79b 100644 --- a/4ed_api_implementation.cpp +++ b/4ed_api_implementation.cpp @@ -365,7 +365,7 @@ buffer_line_y_difference(Application_Links *app, Buffer_ID buffer_id, if (api_check_buffer(file)){ Face *face = font_set_face_from_id(&models->font_set, face_id); if (face != 0){ - Layout_Function *layout_func = models->layout_func; + Layout_Function *layout_func = file_get_layout_func(file); result = file_line_y_difference(app->tctx, models, file, layout_func, width, face, line_a, line_b); @@ -384,7 +384,7 @@ buffer_line_shift_y(Application_Links *app, Buffer_ID buffer_id, if (api_check_buffer(file)){ Face *face = font_set_face_from_id(&models->font_set, face_id); if (face != 0){ - Layout_Function *layout_func = models->layout_func; + Layout_Function *layout_func = file_get_layout_func(file); result = file_line_shift_y(app->tctx, models, file, layout_func, width, face, line, y_shift); @@ -403,7 +403,7 @@ buffer_pos_at_relative_xy(Application_Links *app, Buffer_ID buffer_id, if (api_check_buffer(file)){ Face *face = font_set_face_from_id(&models->font_set, face_id); if (face != 0){ - Layout_Function *layout_func = models->layout_func; + Layout_Function *layout_func = file_get_layout_func(file); result = file_pos_at_relative_xy(app->tctx, models, file, layout_func, width, face, base_line, relative_xy); @@ -422,7 +422,7 @@ buffer_relative_box_of_pos(Application_Links *app, Buffer_ID buffer_id, if (api_check_buffer(file)){ Face *face = font_set_face_from_id(&models->font_set, face_id); if (face != 0){ - Layout_Function *layout_func = models->layout_func; + Layout_Function *layout_func = file_get_layout_func(file); result = file_relative_box_of_pos(app->tctx, models, file, layout_func, width, face, base_line, pos); @@ -441,7 +441,7 @@ buffer_relative_character_from_pos(Application_Links *app, Buffer_ID buffer_id, if (api_check_buffer(file)){ Face *face = font_set_face_from_id(&models->font_set, face_id); if (face != 0){ - Layout_Function *layout_func = models->layout_func; + Layout_Function *layout_func = file_get_layout_func(file); result = file_relative_character_from_pos(app->tctx, models, file, layout_func, width, face, base_line, pos); @@ -459,7 +459,7 @@ buffer_pos_from_relative_character(Application_Links *app, Buffer_ID buffer_id, if (api_check_buffer(file)){ Face *face = font_set_face_from_id(&models->font_set, face_id); if (face != 0){ - Layout_Function *layout_func = models->layout_func; + Layout_Function *layout_func = file_get_layout_func(file); result = file_pos_from_relative_character(app->tctx, models, file, layout_func, width, face, base_line, relative_character); @@ -643,14 +643,26 @@ buffer_set_dirty_state(Application_Links *app, Buffer_ID buffer_id, Dirty_State } api(custom) function b32 -buffer_set_layout(Application_Links *app, Buffer_ID buffer_id, - Layout_Function *layout_func){ +buffer_set_layout(Application_Links *app, Buffer_ID buffer_id, Layout_Function *layout_func){ Models *models = (Models*)app->cmd_context; Editing_File *file = imp_get_file(models, buffer_id); b32 result = false; if (api_check_buffer(file)){ result = true; file->settings.layout_func = layout_func; + file_clear_layout_cache(file); + } + return(result); +} + +api(custom) function b32 +file_clear_layout_cache(Application_Links *app, Buffer_ID buffer_id){ + Models *models = (Models*)app->cmd_context; + Editing_File *file = imp_get_file(models, buffer_id); + b32 result = false; + if (api_check_buffer(file)){ + result = true; + file_clear_layout_cache(file); } return(result); } @@ -2774,7 +2786,7 @@ text_layout_create(Application_Links *app, Buffer_ID buffer_id, Rect_f32 rect, B Gap_Buffer *buffer = &file->state.buffer; - Layout_Function *layout_func = models->layout_func; + Layout_Function *layout_func = file_get_layout_func(file); Vec2_f32 dim = rect_dim(rect); diff --git a/custom/4coder_async_tasks.cpp b/custom/4coder_async_tasks.cpp index 1415231a..4706bd88 100644 --- a/custom/4coder_async_tasks.cpp +++ b/custom/4coder_async_tasks.cpp @@ -43,7 +43,6 @@ async_push_node__inner(Async_System *async_system, Async_Task_Function_Type *fun node->data.data = (u8*)heap_allocate(&async_system->node_heap, data.size); block_copy(node->data.data, data.data, data.size); node->data.size = data.size; - node->following_task = 0; dll_insert_back(&async_system->task_sent, &node->node); async_system->task_count += 1; system_condition_variable_signal(async_system->cv); @@ -107,17 +106,7 @@ async_task_thread(void *thread_ptr){ thread->task = 0; thread->cancel_signal = false; thread->join_signal = false; - { - Async_Node *next = node->following_task; - async_free_node(async_system, node); - for (node = next; node != 0; node = next){ - next = node->following_task; - Assert(&node->node != &async_system->task_sent); - dll_remove(&node->node); - async_system->task_count -= 1; - async_free_node(async_system, node); - } - } + async_free_node(async_system, node); system_mutex_release(async_system->mutex); } } @@ -173,22 +162,6 @@ async_task_no_dep(Async_System *async_system, Async_Task_Function_Type *func, return(result); } -function Async_Task -async_task_single_dep(Async_System *async_system, Async_Task_Function_Type *func, - Data data, Async_Task dependency){ - system_mutex_acquire(async_system->mutex); - Async_Node *dep_node = async_get_pending_node(async_system, dependency); - if (dep_node == 0){ - dep_node = async_get_running_node(async_system, dependency); - } - Async_Node *node = async_push_node__inner(async_system, func, data); - if (dep_node != 0){ - dep_node->following_task = node; - } - system_mutex_release(async_system->mutex); - return(node->task); -} - function b32 async_task_is_pending(Async_System *async_system, Async_Task task){ system_mutex_acquire(async_system->mutex); @@ -216,6 +189,7 @@ async_task_is_running_or_pending(Async_System *async_system, Async_Task task){ return(node != 0); } +// TODO(allen): ensure that the job is canceled before this returns. function void async_task_cancel(Async_System *async_system, Async_Task task){ system_mutex_acquire(async_system->mutex); diff --git a/custom/4coder_async_tasks.h b/custom/4coder_async_tasks.h index 5226d975..2e1af6e4 100644 --- a/custom/4coder_async_tasks.h +++ b/custom/4coder_async_tasks.h @@ -28,7 +28,6 @@ struct Async_Node{ Async_Thread *thread; Async_Task_Function_Type *func; Data data; - Async_Node *following_task; }; struct Async_System{ diff --git a/custom/4coder_code_index.cpp b/custom/4coder_code_index.cpp index 82edd803..51743628 100644 --- a/custom/4coder_code_index.cpp +++ b/custom/4coder_code_index.cpp @@ -4,6 +4,40 @@ // TOP +global Code_Index global_code_index = {}; + +function void +code_index_init(void){ + global_code_index.mutex = system_mutex_make(); + global_code_index.node_arena = make_arena_system(KB(4)); + global_code_index.buffer_to_index_file = + make_table_u64_u64(global_code_index.node_arena.base_allocator, 500); +} + +function Code_Index_File_Storage* +code_index__alloc_storage(void){ + Code_Index_File_Storage *result = global_code_index.free_storage; + if (result == 0){ + result = push_array_zero(&global_code_index.node_arena, + Code_Index_File_Storage, 1); + } + else{ + sll_stack_pop(global_code_index.free_storage); + } + zdll_push_back(global_code_index.storage_first, global_code_index.storage_last, + result); + global_code_index.storage_count += 1; + return(result); +} + +function void +code_index__free_storage(Code_Index_File_Storage *storage){ + zdll_remove(global_code_index.storage_first, global_code_index.storage_last, + storage); + global_code_index.storage_count -= 1; + sll_stack_push(global_code_index.free_storage, storage); +} + function void code_index_push_nest(Code_Index_Nest_List *list, Code_Index_Nest *nest){ sll_queue_push(list->first, list->last, nest); @@ -26,12 +60,133 @@ code_index_nest_ptr_array_from_list(Arena *arena, Code_Index_Nest_List *list){ } function void -code_index_set_file(Application_Links *app, Buffer_ID buffer, Arena arena, Code_Index_File *index){ - NotImplemented; +code_index_lock(void){ + system_mutex_acquire(global_code_index.mutex); +} + +function void +code_index_unlock(void){ + system_mutex_release(global_code_index.mutex); +} + +function void +code_index_set_file(Buffer_ID buffer, Arena arena, Code_Index_File *index){ + Code_Index_File_Storage *storage = 0; + + Table_Lookup lookup = table_lookup(&global_code_index.buffer_to_index_file, + buffer); + if (lookup.found_match){ + u64 val = 0; + table_read(&global_code_index.buffer_to_index_file, lookup, &val); + storage = (Code_Index_File_Storage*)IntAsPtr(val); + linalloc_clear(&storage->arena); + } + else{ + storage = code_index__alloc_storage(); + table_insert(&global_code_index.buffer_to_index_file, buffer, + (u64)PtrAsInt(storage)); + } + storage->arena = arena; + storage->file = index; +} + +function void +code_index_erase_file(Buffer_ID buffer){ + Table_Lookup lookup = table_lookup(&global_code_index.buffer_to_index_file, + buffer); + if (lookup.found_match){ + u64 val = 0; + table_read(&global_code_index.buffer_to_index_file, lookup, &val); + Code_Index_File_Storage *storage = (Code_Index_File_Storage*)IntAsPtr(val); + linalloc_clear(&storage->arena); + table_erase(&global_code_index.buffer_to_index_file, lookup); + code_index__free_storage(storage); + } +} + +function Code_Index_File* +code_index_get_file(Buffer_ID buffer){ + Code_Index_File *result = 0; + Table_Lookup lookup = table_lookup(&global_code_index.buffer_to_index_file, + buffer); + if (lookup.found_match){ + u64 val = 0; + table_read(&global_code_index.buffer_to_index_file, lookup, &val); + result = (Code_Index_File*)IntAsPtr(val); + } + return(result); +} + +function Code_Index_Nest* +code_index_get_nest(Code_Index_Nest_Ptr_Array *array, i64 pos){ + Code_Index_Nest *result = 0; + i32 count = array->count; + Code_Index_Nest **nest_ptrs = array->ptrs; + for (i32 i = 0; i < count; i += 1){ + Code_Index_Nest *nest = nest_ptrs[i]; + if (nest->open.max <= pos && pos < nest->close.min){ + Code_Index_Nest *sub_nest = + code_index_get_nest(&nest->nest_array, pos); + if (sub_nest != 0){ + result = sub_nest; + } + else{ + result = nest; + } + break; + } + } + return(result); +} + +function Code_Index_Nest* +code_index_get_nest(Code_Index_Nest *nest, i64 pos){ + return(code_index_get_nest(&nest->nest_array, pos)); +} + +function Code_Index_Nest* +code_index_get_nest(Code_Index_File *file, i64 pos){ + return(code_index_get_nest(&file->nest_array, pos)); } //////////////////////////////// +function void +generic_parse_inc(Generic_Parse_State *state){ + if (!token_it_inc_all(&state->it)){ + state->finished = true; + } +} + +function void +generic_parse_skip_soft_tokens(Code_Index_File *index, Generic_Parse_State *state){ + Token *token = token_it_read(&state->it); + for (;token != 0 && !state->finished;){ + if (token->kind == TokenBaseKind_Comment){ + state->handle_comment(state->app, state->arena, index, token, state->contents); + token_it_inc_non_whitespace(&state->it); + token = token_it_read(&state->it); + } + else if (token->kind == TokenBaseKind_Whitespace){ + Range_i64 range = Ii64(token); + u8 *ptr = state->contents.str + range.one_past_last - 1; + for (i64 i = range.one_past_last - 1; + i >= range.first; + i -= 1, ptr -= 1){ + if (*ptr == '\n'){ + state->prev_line_start = ptr + 1; + break; + } + } + } + else{ + break; + } + generic_parse_inc(state); + token = token_it_read(&state->it); + } +} + function void generic_parse_init(Application_Links *app, Arena *arena, String_Const_u8 contents, Token_Array *tokens, Generic_Parse_Comment_Function *handle_comment, Generic_Parse_State *state){ @@ -40,93 +195,121 @@ generic_parse_init(Application_Links *app, Arena *arena, String_Const_u8 content state->contents = contents; state->it = token_iterator(0, tokens); state->handle_comment = handle_comment; - - Token *token = token_it_read(&state->it); - if (token != 0 && token->kind == TokenBaseKind_Whitespace){ - token_it_inc_non_whitespace(&state->it); - } -} - -function Token* -generic_parse_read_token(Code_Index_File *index, Generic_Parse_State *state){ - Token *token = token_it_read(&state->it); - for (;token != 0 && token->kind == TokenBaseKind_Comment;){ - state->handle_comment(state->app, state->arena, index, token, state->contents); - token_it_inc_non_whitespace(&state->it); - token = token_it_read(&state->it); - } - return(token); + state->prev_line_start = contents.str; } function Code_Index_Nest* -generic_parse_parenthical(Code_Index_File *index, Generic_Parse_State *state); +generic_parse_parenthical(Code_Index_File *index, Generic_Parse_State *state, + i64 indentation); function Code_Index_Nest* -generic_parse_scope(Code_Index_File *index, Generic_Parse_State *state){ +generic_parse_scope(Code_Index_File *index, Generic_Parse_State *state, + i64 indentation){ Token *token = token_it_read(&state->it); Code_Index_Nest *result = push_array_zero(state->arena, Code_Index_Nest, 1); result->kind = CodeIndexNest_Scope; result->open = Ii64(token); + result->file = index; + result->interior_indentation = indentation + 4; + result->close_indentation = indentation; + indentation = result->interior_indentation; + + generic_parse_inc(state); for (;;){ - token = generic_parse_read_token(index, state); - if (token == 0){ + generic_parse_skip_soft_tokens(index, state); + token = token_it_read(&state->it); + if (token == 0 || state->finished){ break; } if (token->kind == TokenBaseKind_ScopeOpen){ - Code_Index_Nest *nest = generic_parse_scope(index, state); + Code_Index_Nest *nest = generic_parse_scope(index, state, indentation); + nest->parent = result; code_index_push_nest(&result->nest_list, nest); } else if (token->kind == TokenBaseKind_ParentheticalOpen){ - Code_Index_Nest *nest = generic_parse_parenthical(index, state); + Code_Index_Nest *nest = generic_parse_parenthical(index, state, + indentation); + nest->parent = result; code_index_push_nest(&result->nest_list, nest); } else if (token->kind == TokenBaseKind_ScopeClose){ + result->is_closed = true; result->close = Ii64(token); + generic_parse_inc(state); break; } else{ - token_it_inc_non_whitespace(&state->it); + generic_parse_inc(state); } } - result->nest_array = code_index_nest_ptr_array_from_list(state->arena, &result->nest_list); + result->nest_array = + code_index_nest_ptr_array_from_list(state->arena, &result->nest_list); return(result); } function Code_Index_Nest* -generic_parse_parenthical(Code_Index_File *index, Generic_Parse_State *state){ +generic_parse_parenthical(Code_Index_File *index, Generic_Parse_State *state, + i64 indentation){ Token *token = token_it_read(&state->it); Code_Index_Nest *result = push_array_zero(state->arena, Code_Index_Nest, 1); result->kind = CodeIndexNest_Paren; result->open = Ii64(token); + result->file = index; + i64 manifested_characters_on_line = 0; + { + u8 *ptr = state->prev_line_start; + u8 *end_ptr = state->contents.str + token->pos; + // NOTE(allen): Initial whitespace + for (;ptr < end_ptr; ptr += 1){ + if (!character_is_whitespace(*ptr)){ + break; + } + } + // NOTE(allen): Manifested characters + manifested_characters_on_line = (i64)(end_ptr - ptr) + token->size; + } + + indentation += manifested_characters_on_line; + result->interior_indentation = indentation; + result->close_indentation = indentation; + + generic_parse_inc(state); for (;;){ - token = generic_parse_read_token(index, state); - if (token == 0){ + generic_parse_skip_soft_tokens(index, state); + token = token_it_read(&state->it); + if (token == 0 || state->finished){ break; } if (token->kind == TokenBaseKind_ScopeOpen){ - Code_Index_Nest *nest = generic_parse_scope(index, state); + Code_Index_Nest *nest = generic_parse_scope(index, state, indentation); + nest->parent = result; code_index_push_nest(&result->nest_list, nest); } else if (token->kind == TokenBaseKind_ParentheticalOpen){ - Code_Index_Nest *nest = generic_parse_parenthical(index, state); + Code_Index_Nest *nest = generic_parse_parenthical(index, state, + indentation); + nest->parent = result; code_index_push_nest(&result->nest_list, nest); } else if (token->kind == TokenBaseKind_ParentheticalClose){ + result->is_closed = true; result->close = Ii64(token); + generic_parse_inc(state); break; } else{ - token_it_inc_non_whitespace(&state->it); + generic_parse_inc(state); } } - result->nest_array = code_index_nest_ptr_array_from_list(state->arena, &result->nest_list); + result->nest_array = + code_index_nest_ptr_array_from_list(state->arena, &result->nest_list); return(result); } @@ -135,26 +318,28 @@ function b32 generic_parse_full_input_breaks(Code_Index_File *index, Generic_Parse_State *state, i32 limit){ b32 result = false; + i64 indentation = 0; i64 first_index = token_it_index(&state->it); i64 one_past_last_index = first_index + limit; for (;;){ - Token *token = generic_parse_read_token(index, state); + generic_parse_skip_soft_tokens(index, state); + Token *token = token_it_read(&state->it); - if (token == 0){ + if (token == 0 || state->finished){ result = true; break; } if (token->kind == TokenBaseKind_ScopeOpen){ - Code_Index_Nest *nest = generic_parse_scope(index, state); + Code_Index_Nest *nest = generic_parse_scope(index, state, indentation); code_index_push_nest(&index->nest_list, nest); } else if (token->kind == TokenBaseKind_ParentheticalOpen){ - Code_Index_Nest *nest = generic_parse_parenthical(index, state); + Code_Index_Nest *nest = generic_parse_parenthical(index, state, indentation); code_index_push_nest(&index->nest_list, nest); } else{ - token_it_inc_non_whitespace(&state->it); + generic_parse_inc(state); } i64 index = token_it_index(&state->it); @@ -167,6 +352,11 @@ generic_parse_full_input_breaks(Code_Index_File *index, Generic_Parse_State *sta } } + if (result){ + index->nest_array = + code_index_nest_ptr_array_from_list(state->arena, &index->nest_list); + } + return(result); } diff --git a/custom/4coder_code_index.h b/custom/4coder_code_index.h index de68c9b7..35899de2 100644 --- a/custom/4coder_code_index.h +++ b/custom/4coder_code_index.h @@ -32,6 +32,12 @@ struct Code_Index_Nest{ Range_i64 open; Range_i64 close; + i64 interior_indentation; + i64 close_indentation; + + struct Code_Index_File *file; + Code_Index_Nest *parent; + Code_Index_Nest_List nest_list; Code_Index_Nest_Ptr_Array nest_array; }; @@ -39,6 +45,24 @@ struct Code_Index_Nest{ struct Code_Index_File{ Code_Index_Nest_List nest_list; Code_Index_Nest_Ptr_Array nest_array; + Buffer_ID buffer; +}; + +struct Code_Index_File_Storage{ + Code_Index_File_Storage *next; + Code_Index_File_Storage *prev; + Arena arena; + Code_Index_File *file; +}; + +struct Code_Index{ + System_Mutex mutex; + Arena node_arena; + Table_u64_u64 buffer_to_index_file; + Code_Index_File_Storage *free_storage; + Code_Index_File_Storage *storage_first; + Code_Index_File_Storage *storage_last; + i32 storage_count; }; //////////////////////////////// @@ -52,6 +76,8 @@ struct Generic_Parse_State{ String_Const_u8 contents; Token_Iterator_Array it; Generic_Parse_Comment_Function *handle_comment; + u8 *prev_line_start; + b32 finished; }; #endif diff --git a/custom/4coder_default_bindings.cpp b/custom/4coder_default_bindings.cpp index 4779d72f..cca8259d 100644 --- a/custom/4coder_default_bindings.cpp +++ b/custom/4coder_default_bindings.cpp @@ -19,6 +19,7 @@ custom_layer_init(Application_Links *app){ mapping_init(tctx, &framework_mapping); setup_default_mapping(&framework_mapping); async_task_handler_init(app, &global_async_system); + code_index_init(); Profile_Global_List *list = get_core_profile_list(app); ProfileThreadName(tctx, list, string_u8_litexpr("main")); diff --git a/custom/4coder_default_hooks.cpp b/custom/4coder_default_hooks.cpp index 159398ad..78383e8f 100644 --- a/custom/4coder_default_hooks.cpp +++ b/custom/4coder_default_hooks.cpp @@ -582,7 +582,7 @@ BUFFER_NAME_RESOLVER_SIG(default_buffer_name_resolution){ } function void -do_full_lex_async__inner(Async_Context *actx, Buffer_ID buffer_id){ +do_full_lex_and_parse_async__inner(Async_Context *actx, Buffer_ID buffer_id){ Application_Links *app = actx->app; ProfileScope(app, "async lex"); Thread_Context *tctx = get_thread_context(app); @@ -593,26 +593,43 @@ do_full_lex_async__inner(Async_Context *actx, Buffer_ID buffer_id){ ProfileBlock(app, "async lex contents (before mutex)"); system_acquire_global_frame_mutex(tctx); ProfileBlock(app, "async lex contents (after mutex)"); + + Managed_Scope scope = buffer_get_managed_scope(app, buffer_id); + if (scope != 0){ + Base_Allocator *allocator = managed_scope_allocator(app, scope); + Token_Array *tokens_ptr = scope_attachment(app, scope, attachment_tokens, + Token_Array); + base_free(allocator, tokens_ptr->tokens); + } + code_index_lock(); + code_index_erase_file(buffer_id); + code_index_unlock(); + contents = push_whole_buffer(app, scratch, buffer_id); system_release_global_frame_mutex(tctx); } - Lex_State_Cpp state = {}; - lex_full_input_cpp_init(&state, contents); + i32 limit_factor = 10000; Token_List list = {}; b32 canceled = false; - for (;;){ - ProfileBlock(app, "async lex block"); - if (lex_full_input_cpp_breaks(scratch, &list, &state, 10000)){ - break; - } - if (async_check_canceled(actx)){ - canceled = true; - break; + + { + Lex_State_Cpp state = {}; + lex_full_input_cpp_init(&state, contents); + for (;;){ + ProfileBlock(app, "async lex block"); + if (lex_full_input_cpp_breaks(scratch, &list, &state, limit_factor)){ + break; + } + if (async_check_canceled(actx)){ + canceled = true; + break; + } } } + Token_Array tokens = {}; if (!canceled){ ProfileBlock(app, "async lex save results (before mutex)"); system_acquire_global_frame_mutex(tctx); @@ -623,8 +640,6 @@ do_full_lex_async__inner(Async_Context *actx, Buffer_ID buffer_id){ Token_Array *tokens_ptr = scope_attachment(app, scope, attachment_tokens, Token_Array); base_free(allocator, tokens_ptr->tokens); - - Token_Array tokens = {}; tokens.tokens = base_array(allocator, Token, list.total_count); tokens.count = list.total_count; tokens.max = list.total_count; @@ -633,67 +648,42 @@ do_full_lex_async__inner(Async_Context *actx, Buffer_ID buffer_id){ } system_release_global_frame_mutex(tctx); } -} - -function void -do_full_lex_async(Async_Context *actx, Data data){ - if (data.size == sizeof(Buffer_ID)){ - Buffer_ID buffer = *(Buffer_ID*)data.data; - do_full_lex_async__inner(actx, buffer); - } -} - -function void -do_full_parse_async__inner(Async_Context *actx, Buffer_ID buffer){ - Application_Links *app = actx->app; - ProfileScope(app, "async parse"); - Thread_Context *tctx = get_thread_context(app); - Scratch_Block scratch(tctx); - - String_Const_u8 contents = {}; - Token_Array tokens = {}; - { - ProfileBlock(app, "async parse contents (before mutex)"); - system_acquire_global_frame_mutex(tctx); - ProfileBlock(app, "async parse contents (after mutex)"); - contents = push_whole_buffer(app, scratch, buffer); - Managed_Scope scope = buffer_get_managed_scope(app, buffer); - Token_Array *tokens_ptr = scope_attachment(app, scope, attachment_tokens, Token_Array); - tokens.count = tokens_ptr->count; - tokens.tokens = push_array_write(scratch, Token, tokens.count, tokens_ptr->tokens); - system_release_global_frame_mutex(tctx); - } - - Arena arena = make_arena_system(KB(16)); - - Generic_Parse_State state = {}; - generic_parse_init(app, &arena, contents, &tokens, &state); - - Code_Index_File index = {}; - b32 canceled = false; - for (;;){ - ProfileBlock(app, "async parse block"); - if (generic_parse_full_input_breaks(&index, &state, 10000)){ - break; + if (tokens.count > 0){ + ProfileBlock(app, "async parse"); + Arena arena = make_arena_system(KB(16)); + Code_Index_File *index = push_array_zero(&arena, Code_Index_File, 1); + index->buffer = buffer_id; + + Generic_Parse_State state = {}; + generic_parse_init(app, &arena, contents, &tokens, &state); + + for (;;){ + if (generic_parse_full_input_breaks(index, &state, limit_factor)){ + break; + } + if (async_check_canceled(actx)){ + canceled = true; + break; + } } - if (async_check_canceled(actx)){ - canceled = true; - break; + + if (!canceled){ + code_index_lock(); + code_index_set_file(buffer_id, arena, index); + code_index_unlock(); + } + else{ + linalloc_clear(&arena); } } - - if (!canceled){ - ProfileBlock(app, "async parse save results"); - code_index_set_file(app, buffer, arena, &index); - } } function void -do_full_parse_async(Async_Context *actx, Data data){ +do_full_lex_and_parse_async(Async_Context *actx, Data data){ if (data.size == sizeof(Buffer_ID)){ Buffer_ID buffer = *(Buffer_ID*)data.data; - do_full_parse_async__inner(actx, buffer); + do_full_lex_and_parse_async__inner(actx, buffer); } } @@ -815,9 +805,7 @@ BUFFER_HOOK_SIG(default_begin_buffer){ if (use_lexer){ ProfileBlock(app, "begin buffer kick off lexer"); Async_Task *lex_task_ptr = scope_attachment(app, scope, buffer_lex_task, Async_Task); - *lex_task_ptr = async_task_no_dep(&global_async_system, do_full_lex_async, make_data_struct(&buffer_id)); - async_task_single_dep(&global_async_system, do_full_parse_async, make_data_struct(&buffer_id), - *lex_task_ptr); + *lex_task_ptr = async_task_no_dep(&global_async_system, do_full_lex_and_parse_async, make_data_struct(&buffer_id)); } if (wrap_lines){ @@ -887,9 +875,9 @@ BUFFER_EDIT_RANGE_SIG(default_buffer_edit_range){ Async_Task *lex_task_ptr = scope_attachment(app, scope, buffer_lex_task, Async_Task); if (async_task_is_running_or_pending(&global_async_system, *lex_task_ptr)){ async_task_cancel(&global_async_system, *lex_task_ptr); - *lex_task_ptr = async_task_no_dep(&global_async_system, do_full_lex_async, make_data_struct(&buffer_id)); - async_task_single_dep(&global_async_system, do_full_parse_async, make_data_struct(&buffer_id), - *lex_task_ptr); + *lex_task_ptr = async_task_no_dep(&global_async_system, + do_full_lex_and_parse_async, + make_data_struct(&buffer_id)); } else{ Token_Array *ptr = scope_attachment(app, scope, attachment_tokens, Token_Array); @@ -965,13 +953,9 @@ BUFFER_EDIT_RANGE_SIG(default_buffer_edit_range){ } if (do_full_relex){ - base_free(allocator, ptr->tokens); - block_zero_struct(ptr); *lex_task_ptr = async_task_no_dep(&global_async_system, - do_full_lex_async, + do_full_lex_and_parse_async, make_data_struct(&buffer_id)); - async_task_single_dep(&global_async_system, do_full_parse_async, make_data_struct(&buffer_id), - *lex_task_ptr); } } } @@ -986,6 +970,9 @@ BUFFER_HOOK_SIG(default_end_buffer){ if (lex_task_ptr != 0){ async_task_cancel(&global_async_system, *lex_task_ptr); } + code_index_lock(); + code_index_erase_file(buffer_id); + code_index_unlock(); // no meaning for return return(0); } diff --git a/custom/generated/custom_api.cpp b/custom/generated/custom_api.cpp index 3a57ea1c..4ef0f243 100644 --- a/custom/generated/custom_api.cpp +++ b/custom/generated/custom_api.cpp @@ -43,6 +43,7 @@ vtable->push_buffer_file_name = push_buffer_file_name; vtable->buffer_get_dirty_state = buffer_get_dirty_state; vtable->buffer_set_dirty_state = buffer_set_dirty_state; vtable->buffer_set_layout = buffer_set_layout; +vtable->file_clear_layout_cache = file_clear_layout_cache; vtable->buffer_get_layout = buffer_get_layout; vtable->buffer_get_setting = buffer_get_setting; vtable->buffer_set_setting = buffer_set_setting; @@ -222,6 +223,7 @@ push_buffer_file_name = vtable->push_buffer_file_name; buffer_get_dirty_state = vtable->buffer_get_dirty_state; buffer_set_dirty_state = vtable->buffer_set_dirty_state; buffer_set_layout = vtable->buffer_set_layout; +file_clear_layout_cache = vtable->file_clear_layout_cache; buffer_get_layout = vtable->buffer_get_layout; buffer_get_setting = vtable->buffer_get_setting; buffer_set_setting = vtable->buffer_set_setting; diff --git a/custom/generated/custom_api.h b/custom/generated/custom_api.h index fc88cbb5..42903f74 100644 --- a/custom/generated/custom_api.h +++ b/custom/generated/custom_api.h @@ -40,8 +40,9 @@ #define custom_push_buffer_file_name_sig() String_Const_u8 custom_push_buffer_file_name(Application_Links* app, Arena* arena, Buffer_ID buffer_id) #define custom_buffer_get_dirty_state_sig() Dirty_State custom_buffer_get_dirty_state(Application_Links* app, Buffer_ID buffer_id) #define custom_buffer_set_dirty_state_sig() b32 custom_buffer_set_dirty_state(Application_Links* app, Buffer_ID buffer_id, Dirty_State dirty_state) -#define custom_buffer_set_layout_sig() b32 custom_buffer_set_layout(Application_Links* app, Buffer_ID buffer, Layout_Function* layout_func) -#define custom_buffer_get_layout_sig() Layout_Function* custom_buffer_get_layout(Application_Links* app, Buffer_ID buffer) +#define custom_buffer_set_layout_sig() b32 custom_buffer_set_layout(Application_Links* app, Buffer_ID buffer_id, Layout_Function* layout_func) +#define custom_file_clear_layout_cache_sig() b32 custom_file_clear_layout_cache(Application_Links* app, Buffer_ID buffer) +#define custom_buffer_get_layout_sig() Layout_Function* custom_buffer_get_layout(Application_Links* app, Buffer_ID buffer_id) #define custom_buffer_get_setting_sig() b32 custom_buffer_get_setting(Application_Links* app, Buffer_ID buffer_id, Buffer_Setting_ID setting, i64* value_out) #define custom_buffer_set_setting_sig() b32 custom_buffer_set_setting(Application_Links* app, Buffer_ID buffer_id, Buffer_Setting_ID setting, i64 value) #define custom_buffer_get_managed_scope_sig() Managed_Scope custom_buffer_get_managed_scope(Application_Links* app, Buffer_ID buffer_id) @@ -215,8 +216,9 @@ typedef String_Const_u8 custom_push_buffer_unique_name_type(Application_Links* a typedef String_Const_u8 custom_push_buffer_file_name_type(Application_Links* app, Arena* arena, Buffer_ID buffer_id); typedef Dirty_State custom_buffer_get_dirty_state_type(Application_Links* app, Buffer_ID buffer_id); typedef b32 custom_buffer_set_dirty_state_type(Application_Links* app, Buffer_ID buffer_id, Dirty_State dirty_state); -typedef b32 custom_buffer_set_layout_type(Application_Links* app, Buffer_ID buffer, Layout_Function* layout_func); -typedef Layout_Function* custom_buffer_get_layout_type(Application_Links* app, Buffer_ID buffer); +typedef b32 custom_buffer_set_layout_type(Application_Links* app, Buffer_ID buffer_id, Layout_Function* layout_func); +typedef b32 custom_file_clear_layout_cache_type(Application_Links* app, Buffer_ID buffer); +typedef Layout_Function* custom_buffer_get_layout_type(Application_Links* app, Buffer_ID buffer_id); typedef b32 custom_buffer_get_setting_type(Application_Links* app, Buffer_ID buffer_id, Buffer_Setting_ID setting, i64* value_out); typedef b32 custom_buffer_set_setting_type(Application_Links* app, Buffer_ID buffer_id, Buffer_Setting_ID setting, i64 value); typedef Managed_Scope custom_buffer_get_managed_scope_type(Application_Links* app, Buffer_ID buffer_id); @@ -392,6 +394,7 @@ custom_push_buffer_file_name_type *push_buffer_file_name; custom_buffer_get_dirty_state_type *buffer_get_dirty_state; custom_buffer_set_dirty_state_type *buffer_set_dirty_state; custom_buffer_set_layout_type *buffer_set_layout; +custom_file_clear_layout_cache_type *file_clear_layout_cache; custom_buffer_get_layout_type *buffer_get_layout; custom_buffer_get_setting_type *buffer_get_setting; custom_buffer_set_setting_type *buffer_set_setting; @@ -568,8 +571,9 @@ internal String_Const_u8 push_buffer_unique_name(Application_Links* app, Arena* internal String_Const_u8 push_buffer_file_name(Application_Links* app, Arena* arena, Buffer_ID buffer_id); internal Dirty_State buffer_get_dirty_state(Application_Links* app, Buffer_ID buffer_id); internal b32 buffer_set_dirty_state(Application_Links* app, Buffer_ID buffer_id, Dirty_State dirty_state); -internal b32 buffer_set_layout(Application_Links* app, Buffer_ID buffer, Layout_Function* layout_func); -internal Layout_Function* buffer_get_layout(Application_Links* app, Buffer_ID buffer); +internal b32 buffer_set_layout(Application_Links* app, Buffer_ID buffer_id, Layout_Function* layout_func); +internal b32 file_clear_layout_cache(Application_Links* app, Buffer_ID buffer); +internal Layout_Function* buffer_get_layout(Application_Links* app, Buffer_ID buffer_id); internal b32 buffer_get_setting(Application_Links* app, Buffer_ID buffer_id, Buffer_Setting_ID setting, i64* value_out); internal b32 buffer_set_setting(Application_Links* app, Buffer_ID buffer_id, Buffer_Setting_ID setting, i64 value); internal Managed_Scope buffer_get_managed_scope(Application_Links* app, Buffer_ID buffer_id); @@ -746,6 +750,7 @@ global custom_push_buffer_file_name_type *push_buffer_file_name = 0; global custom_buffer_get_dirty_state_type *buffer_get_dirty_state = 0; global custom_buffer_set_dirty_state_type *buffer_set_dirty_state = 0; global custom_buffer_set_layout_type *buffer_set_layout = 0; +global custom_file_clear_layout_cache_type *file_clear_layout_cache = 0; global custom_buffer_get_layout_type *buffer_get_layout = 0; global custom_buffer_get_setting_type *buffer_get_setting = 0; global custom_buffer_set_setting_type *buffer_set_setting = 0; diff --git a/custom/generated/custom_api_master_list.h b/custom/generated/custom_api_master_list.h index 53194849..f89271e1 100644 --- a/custom/generated/custom_api_master_list.h +++ b/custom/generated/custom_api_master_list.h @@ -40,8 +40,9 @@ api(custom) function String_Const_u8 push_buffer_unique_name(Application_Links* api(custom) function String_Const_u8 push_buffer_file_name(Application_Links* app, Arena* arena, Buffer_ID buffer_id); api(custom) function Dirty_State buffer_get_dirty_state(Application_Links* app, Buffer_ID buffer_id); api(custom) function b32 buffer_set_dirty_state(Application_Links* app, Buffer_ID buffer_id, Dirty_State dirty_state); -api(custom) function b32 buffer_set_layout(Application_Links* app, Buffer_ID buffer, Layout_Function* layout_func); -api(custom) function Layout_Function* buffer_get_layout(Application_Links* app, Buffer_ID buffer); +api(custom) function b32 buffer_set_layout(Application_Links* app, Buffer_ID buffer_id, Layout_Function* layout_func); +api(custom) function b32 file_clear_layout_cache(Application_Links* app, Buffer_ID buffer); +api(custom) function Layout_Function* buffer_get_layout(Application_Links* app, Buffer_ID buffer_id); api(custom) function b32 buffer_get_setting(Application_Links* app, Buffer_ID buffer_id, Buffer_Setting_ID setting, i64* value_out); api(custom) function b32 buffer_set_setting(Application_Links* app, Buffer_ID buffer_id, Buffer_Setting_ID setting, i64 value); api(custom) function Managed_Scope buffer_get_managed_scope(Application_Links* app, Buffer_ID buffer_id); diff --git a/platform_win32/win32_4ed.cpp b/platform_win32/win32_4ed.cpp index 723340d1..e2fc3e4a 100644 --- a/platform_win32/win32_4ed.cpp +++ b/platform_win32/win32_4ed.cpp @@ -1723,6 +1723,10 @@ WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPSTR lpCmdLine, int nCmdS win32vars.got_useful_event = false; } + // NOTE(allen): while we're doing this (and possibly sleeping) + // we can let async processes get there time in. + system_mutex_release(win32vars.global_frame_mutex); + b32 get_more_messages = true; do{ if (win32vars.got_useful_event == 0){ @@ -1797,6 +1801,8 @@ WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPSTR lpCmdLine, int nCmdS } } }while (get_more_messages); + + system_mutex_acquire(win32vars.global_frame_mutex); } // NOTE(allen): Mouse Out of Window Detection