now officially running through the gap buffer with 100 of features working!!!!
Allen Webster
parent
3a7f4db69e
commit
c551c081a7
122
4cpp_lexer.h
122
4cpp_lexer.h
|
|
@ -1125,42 +1125,6 @@ cpp_index_array(Cpp_Token_Array *array, int32_t file_size, int32_t index){
|
|||
return(result);
|
||||
}
|
||||
|
||||
#if 0
|
||||
FCPP_INTERNAL Cpp_Relex_State
|
||||
cpp_relex_nonalloc_start(Cpp_Token_Array *array, int32_t start, int32_t end, int32_t tolerance){
|
||||
Cpp_Relex_State state;
|
||||
state.array = array;
|
||||
state.start = start;
|
||||
state.end = end;
|
||||
|
||||
Cpp_Get_Token_Result result = cpp_get_token(array, start);
|
||||
|
||||
state.start_token_i = result.token_index-1;
|
||||
if (state.start_token_i < 0){
|
||||
state.start_token_i = 0;
|
||||
}
|
||||
|
||||
result = cpp_get_token(array, end);
|
||||
|
||||
state.end_token_i = result.token_index;
|
||||
if (end > array->tokens[state.end_token_i].start){
|
||||
++state.end_token_i;
|
||||
}
|
||||
if (state.end_token_i < 0){
|
||||
state.end_token_i = 0;
|
||||
}
|
||||
|
||||
state.relex_start = array->tokens[state.start_token_i].start;
|
||||
if (start < state.relex_start){
|
||||
state.relex_start = start;
|
||||
}
|
||||
|
||||
state.space_request = state.end_token_i - state.start_token_i + tolerance + 1;
|
||||
|
||||
return(state);
|
||||
}
|
||||
#endif
|
||||
|
||||
FCPP_INTERNAL Cpp_Relex_Range
|
||||
cpp_get_relex_range(Cpp_Token_Array *array, int32_t start_pos, int32_t end_pos){
|
||||
Cpp_Relex_Range range = {0};
|
||||
|
|
@ -1301,7 +1265,6 @@ cpp_relex_step(Cpp_Relex_Data *S_ptr, char *chunk, int32_t chunk_size, int32_t f
|
|||
}
|
||||
|
||||
double_break:;
|
||||
|
||||
DrReturn(LexResult_Finished);
|
||||
}
|
||||
|
||||
|
|
@ -1371,91 +1334,6 @@ cpp_relex_abort(Cpp_Relex_Data *S_ptr, Cpp_Token_Array *array){
|
|||
cpp_shift_token_starts(array, S_ptr->original_end_token_index, -S_ptr->character_shift_amount);
|
||||
}
|
||||
|
||||
#if 0
|
||||
// TODO(allen): rename shift_amount to character_shift_amount
|
||||
FCPP_INTERNAL Cpp_Lex_Result
|
||||
cpp_relex_nonalloc_main(Cpp_Relex_Data *S_ptr,
|
||||
|
||||
Cpp_Relex_Range range, int32_t shift_amount,
|
||||
|
||||
char *chunk, int32_t chunk_size, int32_t size,
|
||||
|
||||
Cpp_Token_Array *array,
|
||||
|
||||
Cpp_Token_Array *relex_array,
|
||||
int32_t *relex_end_out,
|
||||
|
||||
char *spare){
|
||||
|
||||
cpp_shift_token_starts(array, range.end_token_index, shift_amount);
|
||||
Cpp_Token match_token = cpp_index_array(array, size, range.end_token_index);
|
||||
Cpp_Token end_token = match_token;
|
||||
|
||||
Cpp_Relex_State result = LexResult_Finished;
|
||||
|
||||
Cpp_Token *tokens = array->tokens;
|
||||
int32_t relex_end_index = range.end_token_index;
|
||||
|
||||
if (state->relex_start < size){
|
||||
Cpp_Lex_Data lex = cpp_lex_data_init(spare);
|
||||
lex.pp_state = cpp_token_get_pp_state(tokens[state->start_token_i].state_flags);
|
||||
lex.pos = state->relex_start;
|
||||
|
||||
// TODO(allen): This can be better I suspect.
|
||||
for (;;){
|
||||
int32_t step_result =
|
||||
cpp_lex_nonalloc_no_null_out_limit(&lex, data, size, size,
|
||||
relex_array, 1);
|
||||
|
||||
switch (step_result){
|
||||
case LexResult_HitTokenLimit:
|
||||
{
|
||||
Cpp_Token token = relex_array->tokens[relex_array->count-1];
|
||||
if (token.start == end_token.start &&
|
||||
token.size == end_token.size &&
|
||||
token.flags == end_token.flags &&
|
||||
token.state_flags == end_token.state_flags){
|
||||
--relex_array->count;
|
||||
goto double_break;
|
||||
}
|
||||
|
||||
while (lex.pos > end_token.start && relex_end_i < array->count){
|
||||
++relex_end_i;
|
||||
end_token = cpp_index_array(array, size, relex_end_i);
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
case LexResult_NeedChunk: Assert(!"Invalid path"); break;
|
||||
|
||||
case LexResult_NeedTokenMemory:
|
||||
result = LexResult_NeedTokenMemory;
|
||||
goto double_break;
|
||||
|
||||
case LexResult_Finished:
|
||||
goto double_break;
|
||||
}
|
||||
}
|
||||
double_break:;
|
||||
}
|
||||
|
||||
switch (result){
|
||||
case LexResult_Finished:
|
||||
{
|
||||
*relex_end_out = relex_end_i;
|
||||
}break;
|
||||
|
||||
case LexResult_NeedTokenMemory:
|
||||
{
|
||||
cpp_shift_token_starts(array, state->end_token_i, -shift_amount);
|
||||
}break;
|
||||
|
||||
default: Assert(!"Invalid path");
|
||||
}
|
||||
|
||||
return(result);
|
||||
}
|
||||
#endif
|
||||
|
||||
#if !defined(FCPP_FORBID_MALLOC)
|
||||
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ fill_buffer_summary(Buffer_Summary *buffer, Editing_File *file, Working_Set *wor
|
|||
buffer->ready = file_is_ready(file);
|
||||
|
||||
buffer->buffer_id = file->id.id;
|
||||
buffer->size = file->state.buffer.size;
|
||||
buffer->size = buffer_size(&file->state.buffer);
|
||||
buffer->line_count = file->state.buffer.line_count;
|
||||
|
||||
buffer->file_name_len = file->name.source_path.size;
|
||||
|
|
@ -724,7 +724,6 @@ DOC_SEE(Buffer_Setting_ID)
|
|||
switch (setting){
|
||||
case BufferSetting_Lex:
|
||||
{
|
||||
#if BUFFER_EXPERIMENT_SCALPEL <= 0
|
||||
if (file->settings.tokens_exist){
|
||||
if (!value){
|
||||
file_kill_tokens(system, &models->mem.general, file);
|
||||
|
|
@ -735,7 +734,6 @@ DOC_SEE(Buffer_Setting_ID)
|
|||
file_first_lex_parallel(system, &models->mem.general, file);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}break;
|
||||
|
||||
case BufferSetting_WrapLine:
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@
|
|||
|
||||
#include "4coder_custom.h"
|
||||
|
||||
#define BUFFER_EXPERIMENT_SCALPEL 0
|
||||
#define BUFFER_EXPERIMENT_SCALPEL 1
|
||||
|
||||
#include "4ed_math.h"
|
||||
|
||||
|
|
|
|||
|
|
@ -1123,10 +1123,9 @@ Job_Callback_Sig(job_full_lex){
|
|||
General_Memory *general = (General_Memory*)data[1];
|
||||
|
||||
char *text_data = file->state.buffer.data;
|
||||
i32 text_size = file->state.buffer.size;
|
||||
i32 text_size = buffer_size(&file->state.buffer);
|
||||
|
||||
i32 buffer_size = file->state.buffer.size;
|
||||
buffer_size = (buffer_size + 3)&(~3);
|
||||
i32 buffer_size = (text_size + 3)&(~3);
|
||||
|
||||
while (memory->size < buffer_size*2){
|
||||
system->grow_thread_memory(memory);
|
||||
|
|
@ -1224,7 +1223,6 @@ file_kill_tokens(System_Functions *system,
|
|||
file->state.token_array = null_cpp_token_array;
|
||||
}
|
||||
|
||||
#if BUFFER_EXPERIMENT_SCALPEL <= 0
|
||||
internal void
|
||||
file_first_lex_parallel(System_Functions *system,
|
||||
General_Memory *general, Editing_File *file){
|
||||
|
|
@ -1243,7 +1241,6 @@ file_first_lex_parallel(System_Functions *system,
|
|||
file->state.lex_job = system->post_job(BACKGROUND_THREADS, job);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
internal b32
|
||||
file_relex_parallel(System_Functions *system,
|
||||
|
|
@ -1260,6 +1257,7 @@ file_relex_parallel(System_Functions *system,
|
|||
b32 result = true;
|
||||
b32 inline_lex = !file->state.still_lexing;
|
||||
if (inline_lex){
|
||||
Buffer_Type *buffer = &file->state.buffer;
|
||||
i32 extra_tolerance = 100;
|
||||
|
||||
Cpp_Token_Array *array = &file->state.token_array;
|
||||
|
|
@ -1275,33 +1273,32 @@ file_relex_parallel(System_Functions *system,
|
|||
relex_array.max_count = relex_space_size;
|
||||
relex_array.tokens = push_array(part, Cpp_Token, relex_array.max_count);
|
||||
|
||||
i32 size = file->state.buffer.size;
|
||||
i32 size = buffer_size(buffer);
|
||||
char *spare = push_array(part, char, size+1);
|
||||
|
||||
Cpp_Relex_Data state = cpp_relex_init(array, start_i, end_i, shift_amount, spare);
|
||||
|
||||
char *chunk = file->state.buffer.data;
|
||||
i32 chunk_size = 1024;
|
||||
i32 chunk_index = 0;
|
||||
char *chunks[3];
|
||||
i32 chunk_sizes[3];
|
||||
|
||||
int32_t start_position = cpp_relex_start_position(&state);
|
||||
chunks[0] = buffer->data;
|
||||
chunk_sizes[0] = buffer->size1;
|
||||
|
||||
if (start_position == size){
|
||||
chunk = 0;
|
||||
chunk_size = 0;
|
||||
cpp_relex_declare_first_chunk_position(&state, size);
|
||||
}
|
||||
else{
|
||||
chunk_index = start_position / chunk_size;
|
||||
|
||||
int32_t chunk_start_position = chunk_index*1024;
|
||||
if (chunk_start_position + chunk_size > size){
|
||||
chunk_size = size - chunk_start_position;
|
||||
}
|
||||
|
||||
cpp_relex_declare_first_chunk_position(&state, chunk_start_position);
|
||||
|
||||
chunk += chunk_start_position;
|
||||
chunks[1] = buffer->data + buffer->size1 + buffer->gap_size;
|
||||
chunk_sizes[1] = buffer->size2;
|
||||
|
||||
chunks[2] = 0;
|
||||
chunk_sizes[2] = 0;
|
||||
|
||||
int32_t chunk_index = 0;
|
||||
|
||||
char *chunk = chunks[chunk_index];
|
||||
int32_t chunk_size = chunk_sizes[chunk_index];
|
||||
|
||||
while (!cpp_relex_is_start_chunk(&state, chunk, chunk_size)){
|
||||
++chunk_index;
|
||||
chunk = chunks[chunk_index];
|
||||
chunk_size = chunk_sizes[chunk_index];
|
||||
}
|
||||
|
||||
for(;;){
|
||||
|
|
@ -1310,15 +1307,10 @@ file_relex_parallel(System_Functions *system,
|
|||
|
||||
switch (lex_result){
|
||||
case LexResult_NeedChunk:
|
||||
{
|
||||
++chunk_index;
|
||||
chunk += chunk_size;
|
||||
|
||||
int32_t chunk_start_position = chunk_index*1024;
|
||||
if (chunk_start_position + chunk_size > size){
|
||||
chunk_size = size - chunk_start_position;
|
||||
}
|
||||
}break;
|
||||
++chunk_index;
|
||||
chunk = chunks[chunk_index];
|
||||
chunk_size = chunk_sizes[chunk_index];
|
||||
break;
|
||||
|
||||
case LexResult_NeedTokenMemory:
|
||||
inline_lex = 0;
|
||||
|
|
@ -1342,35 +1334,6 @@ file_relex_parallel(System_Functions *system,
|
|||
}
|
||||
|
||||
cpp_relex_complete(&state, array, &relex_array);
|
||||
|
||||
#if 0
|
||||
i32 delete_amount = relex_end - state.start_token_i;
|
||||
i32 shift_amount = relex_space.count - delete_amount;
|
||||
|
||||
if (shift_amount != 0){
|
||||
i32 new_count = array->count + shift_amount;
|
||||
if (new_count > array->max_count){
|
||||
i32 new_max = LargeRoundUp(new_count, Kbytes(1));
|
||||
array->tokens = (Cpp_Token*)
|
||||
general_memory_reallocate(general, array->tokens,
|
||||
array->count*sizeof(Cpp_Token),
|
||||
new_max*sizeof(Cpp_Token));
|
||||
array->max_count = new_max;
|
||||
}
|
||||
|
||||
i32 shift_size = array->count - relex_end;
|
||||
if (shift_size > 0){
|
||||
Cpp_Token *old_base = array->tokens + relex_end;
|
||||
memmove(old_base + shift_amount, old_base,
|
||||
sizeof(Cpp_Token)*shift_size);
|
||||
}
|
||||
|
||||
array->count += shift_amount;
|
||||
}
|
||||
|
||||
memcpy(state.array->tokens + state.start_token_i, relex_space.tokens,
|
||||
sizeof(Cpp_Token)*relex_space.count);
|
||||
#endif
|
||||
}
|
||||
else{
|
||||
cpp_relex_abort(&state, array);
|
||||
|
|
@ -2164,12 +2127,10 @@ file_do_single_edit(System_Functions *system,
|
|||
|
||||
file_edit_cursor_fix(system, part, general, file, layout, desc, 0);
|
||||
|
||||
#if BUFFER_EXPERIMENT_SCALPEL <= 0
|
||||
// NOTE(allen): token fixing
|
||||
if (file->settings.tokens_exist){
|
||||
file_relex_parallel(system, mem, file, start, end, shift_amount);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
internal void
|
||||
|
|
|
|||
|
|
@ -599,8 +599,13 @@ internal_4tech Partial_Cursor
|
|||
buffer_partial_from_pos(Buffer_Type *buffer, int pos){
|
||||
Partial_Cursor result = {0};
|
||||
|
||||
if (pos > buffer->size) pos = buffer->size;
|
||||
if (pos < 0) pos = 0;
|
||||
int32_t size = buffer_size(buffer);
|
||||
if (pos > size){
|
||||
pos = size;
|
||||
}
|
||||
if (pos < 0){
|
||||
pos = 0;
|
||||
}
|
||||
|
||||
int line_index = buffer_get_line_index_range(buffer, pos, 0, buffer->line_count);
|
||||
result.pos = pos;
|
||||
|
|
@ -616,8 +621,13 @@ buffer_cursor_from_pos(Buffer_Type *buffer, int pos, float *wraps,
|
|||
Full_Cursor result;
|
||||
int line_index;
|
||||
|
||||
if (pos > buffer->size) pos = buffer->size;
|
||||
if (pos < 0) pos = 0;
|
||||
int32_t size = buffer_size(buffer);
|
||||
if (pos > size){
|
||||
pos = size;
|
||||
}
|
||||
if (pos < 0){
|
||||
pos = 0;
|
||||
}
|
||||
|
||||
line_index = buffer_get_line_index_range(buffer, pos, 0, buffer->line_count);
|
||||
result = make_cursor_hint(line_index, buffer->line_starts, wraps, font_height);
|
||||
|
|
@ -635,8 +645,9 @@ buffer_partial_from_line_character(Buffer_Type *buffer, int line, int character)
|
|||
if (line_index >= buffer->line_count) line_index = buffer->line_count - 1;
|
||||
if (line_index < 0) line_index = 0;
|
||||
|
||||
int32_t size = buffer_size(buffer);
|
||||
int this_start = buffer->line_starts[line_index];
|
||||
int max_character = (buffer->size-this_start) + 1;
|
||||
int max_character = (size-this_start) + 1;
|
||||
if (line_index+1 < buffer->line_count){
|
||||
int next_start = buffer->line_starts[line_index+1];
|
||||
max_character = (next_start-this_start);
|
||||
|
|
|
|||
Loading…
Reference in New Issue