mr4th-public
/
4coder
1
Fork 0
Code Issues Releases Activity

begining rewrite of token iteration

master
Allen Webster 2019-02-01 14:50:33 -08:00
parent 93fe0e1472
commit 21a7a58871
7 changed files with 270 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

126
4coder_function_list.cpp
View File

@ -47,6 +47,131 @@ buffered_write_stream_write(Application_Links *app, Buffered_Write_Stream *strea
}
}
static Get_Positions_Results
get_function_positions(Application_Links *app, Buffer_Summary *buffer, int32_t token_index, Function_Positions *positions_array, int32_t positions_max){
Get_Positions_Results result = {};
Token_Range token_range = buffer_get_token_range(app, buffer->buffer_id);
if (token_range.first != 0){
Token_Iterator token_it = make_token_iterator(token_range, token_index);
int32_t nest_level = 0;
int32_t paren_nest_level = 0;
int32_t first_paren_index = 0;
int32_t first_paren_position = 0;
int32_t last_paren_index = 0;
// Look for the next token at global scope that might need to be printed.
mode1:
Assert(nest_level == 0);
Assert(paren_nest_level == 0);
first_paren_index = 0;
first_paren_position = 0;
last_paren_index = 0;
for (Cpp_Token *token = token_iterator_current(&token_it);
token != 0;
token = token_iterator_goto_next(&token_it)){
if (!(token->flags & CPP_TFLAG_PP_BODY)){
switch (token->type){
case CPP_TOKEN_BRACE_OPEN:
{
++nest_level;
}break;
case CPP_TOKEN_BRACE_CLOSE:
{
if (nest_level > 0){
--nest_level;
}
}break;
case CPP_TOKEN_PARENTHESE_OPEN:
{
if (nest_level == 0){
first_paren_index = token_index;
first_paren_position = token->start;
goto paren_mode1;
}
}break;
}
}
}
goto end;
// Look for a closing parenthese to mark the end of a function signature.
paren_mode1:
paren_nest_level = 0;
for (Cpp_Token *token = token_iterator_current(&token_it);
token != 0;
token = token_iterator_goto_next(&token_it)){
if (!(token->flags & CPP_TFLAG_PP_BODY)){
switch (token->type){
case CPP_TOKEN_PARENTHESE_OPEN:
{
++paren_nest_level;
}break;
case CPP_TOKEN_PARENTHESE_CLOSE:
{
--paren_nest_level;
if (paren_nest_level == 0){
last_paren_index = token_index;
goto paren_mode2;
}
}break;
}
}
}
goto end;
// Look backwards from an open parenthese to find the start of a function signature.
paren_mode2:
{
Cpp_Token *restore_point = token_iterator_current(&token_it);
int32_t local_index = first_paren_index;
int32_t signature_start_index = 0;
for (Cpp_Token *token = token_iterator_current(&token_it);
token != 0;
token = token_iterator_goto_prev(&token_it)){
if ((token->flags & CPP_TFLAG_PP_BODY) || (token->flags & CPP_TFLAG_PP_DIRECTIVE) || token->type == CPP_TOKEN_BRACE_CLOSE || token->type == CPP_TOKEN_SEMICOLON || token->type == CPP_TOKEN_PARENTHESE_CLOSE){
++local_index;
signature_start_index = local_index;
goto paren_mode2_done;
}
}
// When this loop ends by going all the way back to the beginning set the signature start to 0 and fall through to the printing phase.
signature_start_index = 0;
paren_mode2_done:;
{
Function_Positions positions;
positions.sig_start_index = signature_start_index;
positions.sig_end_index = last_paren_index;
positions.open_paren_pos = first_paren_position;
positions_array[result.positions_count++] = positions;
}
token_iterator_set(&token_it, restore_point);
if (result.positions_count >= positions_max){
result.next_token_index = token_index;
result.still_looping = true;
goto end;
}
goto mode1;
}
end:;
}
return(result);
}
#if 0
static Get_Positions_Results
get_function_positions(Application_Links *app, Buffer_Summary *buffer, int32_t token_index, Function_Positions *positions_array, int32_t positions_max){
Get_Positions_Results result = {};
@ -178,6 +303,7 @@ get_function_positions(Application_Links *app, Buffer_Summary *buffer, int32_t t
return(result);
}
#endif
static void
print_positions_buffered(Application_Links *app, Buffer_Summary *buffer, Function_Positions *positions_array, int32_t positions_count, Buffered_Write_Stream *stream){

7
4coder_generated/app_functions.h
View File

@ -22,6 +22,7 @@ struct Application_Links;
#define BUFFER_GET_MANAGED_SCOPE_SIG(n) Managed_Scope n(Application_Links *app, Buffer_ID buffer_id)
#define BUFFER_TOKEN_COUNT_SIG(n) int32_t n(Application_Links *app, Buffer_Summary *buffer)
#define BUFFER_READ_TOKENS_SIG(n) bool32 n(Application_Links *app, Buffer_Summary *buffer, int32_t start_token, int32_t end_token, Cpp_Token *tokens_out)
#define BUFFER_GET_TOKEN_RANGE_SIG(n) bool32 n(Application_Links *app, Buffer_Summary *buffer, Cpp_Token **first_token_out, Cpp_Token **one_past_last_token_out)
#define BUFFER_GET_TOKEN_INDEX_SIG(n) bool32 n(Application_Links *app, Buffer_Summary *buffer, int32_t pos, Cpp_Get_Token_Result *get_result)
#define BUFFER_SEND_END_SIGNAL_SIG(n) bool32 n(Application_Links *app, Buffer_Summary *buffer)
#define CREATE_BUFFER_SIG(n) Buffer_Summary n(Application_Links *app, char *filename, int32_t filename_len, Buffer_Create_Flag flags)
@ -149,6 +150,7 @@ typedef BUFFER_SET_SETTING_SIG(Buffer_Set_Setting_Function);
typedef BUFFER_GET_MANAGED_SCOPE_SIG(Buffer_Get_Managed_Scope_Function);
typedef BUFFER_TOKEN_COUNT_SIG(Buffer_Token_Count_Function);
typedef BUFFER_READ_TOKENS_SIG(Buffer_Read_Tokens_Function);
typedef BUFFER_GET_TOKEN_RANGE_SIG(Buffer_Get_Token_Range_Function);
typedef BUFFER_GET_TOKEN_INDEX_SIG(Buffer_Get_Token_Index_Function);
typedef BUFFER_SEND_END_SIGNAL_SIG(Buffer_Send_End_Signal_Function);
typedef CREATE_BUFFER_SIG(Create_Buffer_Function);
@ -278,6 +280,7 @@ Buffer_Set_Setting_Function *buffer_set_setting;
Buffer_Get_Managed_Scope_Function *buffer_get_managed_scope;
Buffer_Token_Count_Function *buffer_token_count;
Buffer_Read_Tokens_Function *buffer_read_tokens;
Buffer_Get_Token_Range_Function *buffer_get_token_range;
Buffer_Get_Token_Index_Function *buffer_get_token_index;
Buffer_Send_End_Signal_Function *buffer_send_end_signal;
Create_Buffer_Function *create_buffer;
@ -406,6 +409,7 @@ Buffer_Set_Setting_Function *buffer_set_setting_;
Buffer_Get_Managed_Scope_Function *buffer_get_managed_scope_;
Buffer_Token_Count_Function *buffer_token_count_;
Buffer_Read_Tokens_Function *buffer_read_tokens_;
Buffer_Get_Token_Range_Function *buffer_get_token_range_;
Buffer_Get_Token_Index_Function *buffer_get_token_index_;
Buffer_Send_End_Signal_Function *buffer_send_end_signal_;
Create_Buffer_Function *create_buffer_;
@ -542,6 +546,7 @@ app_links->buffer_set_setting_ = Buffer_Set_Setting;\
app_links->buffer_get_managed_scope_ = Buffer_Get_Managed_Scope;\
app_links->buffer_token_count_ = Buffer_Token_Count;\
app_links->buffer_read_tokens_ = Buffer_Read_Tokens;\
app_links->buffer_get_token_range_ = Buffer_Get_Token_Range;\
app_links->buffer_get_token_index_ = Buffer_Get_Token_Index;\
app_links->buffer_send_end_signal_ = Buffer_Send_End_Signal;\
app_links->create_buffer_ = Create_Buffer;\
@ -670,6 +675,7 @@ static bool32 buffer_set_setting(Application_Links *app, Buffer_Summary *buffer,
static Managed_Scope buffer_get_managed_scope(Application_Links *app, Buffer_ID buffer_id){return(app->buffer_get_managed_scope(app, buffer_id));}
static int32_t buffer_token_count(Application_Links *app, Buffer_Summary *buffer){return(app->buffer_token_count(app, buffer));}
static bool32 buffer_read_tokens(Application_Links *app, Buffer_Summary *buffer, int32_t start_token, int32_t end_token, Cpp_Token *tokens_out){return(app->buffer_read_tokens(app, buffer, start_token, end_token, tokens_out));}
static bool32 buffer_get_token_range(Application_Links *app, Buffer_Summary *buffer, Cpp_Token **first_token_out, Cpp_Token **one_past_last_token_out){return(app->buffer_get_token_range(app, buffer, first_token_out, one_past_last_token_out));}
static bool32 buffer_get_token_index(Application_Links *app, Buffer_Summary *buffer, int32_t pos, Cpp_Get_Token_Result *get_result){return(app->buffer_get_token_index(app, buffer, pos, get_result));}
static bool32 buffer_send_end_signal(Application_Links *app, Buffer_Summary *buffer){return(app->buffer_send_end_signal(app, buffer));}
static Buffer_Summary create_buffer(Application_Links *app, char *filename, int32_t filename_len, Buffer_Create_Flag flags){return(app->create_buffer(app, filename, filename_len, flags));}
@ -798,6 +804,7 @@ static bool32 buffer_set_setting(Application_Links *app, Buffer_Summary *buffer,
static Managed_Scope buffer_get_managed_scope(Application_Links *app, Buffer_ID buffer_id){return(app->buffer_get_managed_scope_(app, buffer_id));}
static int32_t buffer_token_count(Application_Links *app, Buffer_Summary *buffer){return(app->buffer_token_count_(app, buffer));}
static bool32 buffer_read_tokens(Application_Links *app, Buffer_Summary *buffer, int32_t start_token, int32_t end_token, Cpp_Token *tokens_out){return(app->buffer_read_tokens_(app, buffer, start_token, end_token, tokens_out));}
static bool32 buffer_get_token_range(Application_Links *app, Buffer_Summary *buffer, Cpp_Token **first_token_out, Cpp_Token **one_past_last_token_out){return(app->buffer_get_token_range_(app, buffer, first_token_out, one_past_last_token_out));}
static bool32 buffer_get_token_index(Application_Links *app, Buffer_Summary *buffer, int32_t pos, Cpp_Get_Token_Result *get_result){return(app->buffer_get_token_index_(app, buffer, pos, get_result));}
static bool32 buffer_send_end_signal(Application_Links *app, Buffer_Summary *buffer){return(app->buffer_send_end_signal_(app, buffer));}
static Buffer_Summary create_buffer(Application_Links *app, char *filename, int32_t filename_len, Buffer_Create_Flag flags){return(app->create_buffer_(app, filename, filename_len, flags));}

8
4coder_generated/command_metadata.h
View File

@ -312,10 +312,10 @@ static Command_Metadata fcoder_metacmd_table[220] = {
{ PROC_LINKS(interactive_switch_buffer, 0), "interactive_switch_buffer", 25, "Interactively switch to an open buffer.", 39, "w:\\4ed\\code\\4coder_lists.cpp", 28, 765 },
{ PROC_LINKS(kill_buffer, 0), "kill_buffer", 11, "Kills the current buffer.", 25, "w:\\4ed\\code\\4coder_base_commands.cpp", 36, 1557 },
{ PROC_LINKS(left_adjust_view, 0), "left_adjust_view", 16, "Sets the left size of the view near the x position of the cursor.", 65, "w:\\4ed\\code\\4coder_base_commands.cpp", 36, 133 },
{ PROC_LINKS(list_all_functions_all_buffers, 0), "list_all_functions_all_buffers", 30, "Creates a jump list of lines from all buffers that appear to define or declare functions.", 89, "w:\\4ed\\code\\4coder_function_list.cpp", 36, 343 },
{ PROC_LINKS(list_all_functions_all_buffers_lister, 0), "list_all_functions_all_buffers_lister", 37, "Creates a lister of locations that look like function definitions and declarations all buffers.", 95, "w:\\4ed\\code\\4coder_function_list.cpp", 36, 349 },
{ PROC_LINKS(list_all_functions_current_buffer, 0), "list_all_functions_current_buffer", 33, "Creates a jump list of lines of the current buffer that appear to define or declare functions.", 94, "w:\\4ed\\code\\4coder_function_list.cpp", 36, 320 },
{ PROC_LINKS(list_all_functions_current_buffer_lister, 0), "list_all_functions_current_buffer_lister", 40, "Creates a lister of locations that look like function definitions and declarations in the buffer.", 97, "w:\\4ed\\code\\4coder_function_list.cpp", 36, 330 },
{ PROC_LINKS(list_all_functions_all_buffers, 0), "list_all_functions_all_buffers", 30, "Creates a jump list of lines from all buffers that appear to define or declare functions.", 89, "w:\\4ed\\code\\4coder_function_list.cpp", 36, 469 },
{ PROC_LINKS(list_all_functions_all_buffers_lister, 0), "list_all_functions_all_buffers_lister", 37, "Creates a lister of locations that look like function definitions and declarations all buffers.", 95, "w:\\4ed\\code\\4coder_function_list.cpp", 36, 475 },
{ PROC_LINKS(list_all_functions_current_buffer, 0), "list_all_functions_current_buffer", 33, "Creates a jump list of lines of the current buffer that appear to define or declare functions.", 94, "w:\\4ed\\code\\4coder_function_list.cpp", 36, 446 },
{ PROC_LINKS(list_all_functions_current_buffer_lister, 0), "list_all_functions_current_buffer_lister", 40, "Creates a lister of locations that look like function definitions and declarations in the buffer.", 97, "w:\\4ed\\code\\4coder_function_list.cpp", 36, 456 },
{ PROC_LINKS(list_all_locations, 0), "list_all_locations", 18, "Queries the user for a string and lists all exact case-sensitive matches found in all open buffers.", 99, "w:\\4ed\\code\\4coder_search.cpp", 29, 769 },
{ PROC_LINKS(list_all_locations_case_insensitive, 0), "list_all_locations_case_insensitive", 35, "Queries the user for a string and lists all exact case-insensitive matches found in all open buffers.", 101, "w:\\4ed\\code\\4coder_search.cpp", 29, 783 },
{ PROC_LINKS(list_all_locations_of_identifier, 0), "list_all_locations_of_identifier", 32, "Reads a token or word under the cursor and lists all exact case-sensitive mathces in all open buffers.", 102, "w:\\4ed\\code\\4coder_search.cpp", 29, 797 },

107
4coder_helper.cpp
View File

@ -924,7 +924,7 @@ backward_stream_chunk(Stream_Chunk *chunk){
////////////////////////////////
static bool32
init_stream_tokens(Stream_Tokens *stream, Application_Links *app, Buffer_Summary *buffer,
init_stream_tokens(Stream_Tokens_DEP *stream, Application_Links *app, Buffer_Summary *buffer,
int32_t pos, Cpp_Token *data, int32_t count){
bool32 result = false;
@ -955,13 +955,13 @@ init_stream_tokens(Stream_Tokens *stream, Application_Links *app, Buffer_Summary
return(result);
}
static Stream_Tokens
begin_temp_stream_token(Stream_Tokens *stream){
static Stream_Tokens_DEP
begin_temp_stream_token(Stream_Tokens_DEP *stream){
return(*stream);
}
static void
end_temp_stream_token(Stream_Tokens *stream, Stream_Tokens temp){
end_temp_stream_token(Stream_Tokens_DEP *stream, Stream_Tokens_DEP temp){
if (stream->start != temp.start || stream->end != temp.end){
Application_Links *app = stream->app;
buffer_read_tokens(app, temp.buffer, temp.start, temp.end, temp.base_tokens);
@ -972,7 +972,7 @@ end_temp_stream_token(Stream_Tokens *stream, Stream_Tokens temp){
}
static bool32
forward_stream_tokens(Stream_Tokens *stream){
forward_stream_tokens(Stream_Tokens_DEP *stream){
Application_Links *app = stream->app;
Buffer_Summary *buffer = stream->buffer;
bool32 result = false;
@ -997,7 +997,7 @@ forward_stream_tokens(Stream_Tokens *stream){
}
static bool32
backward_stream_tokens(Stream_Tokens *stream){
backward_stream_tokens(Stream_Tokens_DEP *stream){
Application_Links *app = stream->app;
Buffer_Summary *buffer = stream->buffer;
bool32 result = false;
@ -1021,6 +1021,101 @@ backward_stream_tokens(Stream_Tokens *stream){
return(result);
}
////////////////////////////////
static Token_Range
buffer_get_token_range(Application_Links *app, Buffer_ID buffer_id){
Token_Range range = {};
Buffer_Summary buffer = get_buffer(app, buffer_id, AccessAll);
buffer_get_token_range(app, &buffer, &range.first, &range.one_past_last);
return(range);
}
static Token_Iterator
make_token_iterator(Token_Range range, Cpp_Token *token){
Token_Iterator iterator = {};
if (range.first != 0 && range.one_past_last != 0){
if (token == 0 || token < range.first){
token = range.first;
}
if (token > range.one_past_last){
token = range.one_past_last;
}
iterator.token = token;
iterator.range = range;
}
return(iterator);
}
static Token_Iterator
make_token_iterator(Token_Range range, int32_t index){
return(make_token_iterator(range, range.first + index));
}
static void
token_iterator_set(Token_Iterator *iterator, Cpp_Token *token){
*iterator = make_token_iterator(iterator->range, token);
}
static Cpp_Token*
token_iterator_current(Token_Iterator *iterator){
Cpp_Token *token = iterator->token;
if (token < iterator->range.first || iterator->range.one_past_last <= token){
token = 0;
}
return(token);
}
static int32_t
token_iterator_current_index(Token_Iterator *iterator){
int32_t index = -1;
Cpp_Token *token = token_iterator_current(iterator);
if (token != 0 && iterator->range.first <= token && token <= iterator->range.one_past_last){
index = (int32_t)(token - iterator->range.first);
}
return(index);
}
static Cpp_Token*
token_iterator_goto_next(Token_Iterator *iterator){
Cpp_Token *token = iterator->token;
Cpp_Token *one_past_last = iterator->range.one_past_last;
for (token += 1; token < one_past_last; token += 1){
if (token->type != CPP_TOKEN_COMMENT){
break;
}
}
*iterator = make_token_iterator(iterator->range, token);
return(token_iterator_current(iterator));
}
static Cpp_Token*
token_iterator_goto_next_raw(Token_Iterator *iterator){
*iterator = make_token_iterator(iterator->range, iterator->token + 1);
return(token_iterator_current(iterator));
}
static Cpp_Token*
token_iterator_goto_prev(Token_Iterator *iterator){
Cpp_Token *token = iterator->token;
Cpp_Token *first = iterator->range.first;
for (token -= 1; token > first; token -= 1){
if (token->type != CPP_TOKEN_COMMENT){
break;
}
}
*iterator = make_token_iterator(iterator->range, token);
return(token_iterator_current(iterator));
}
static Cpp_Token*
token_iterator_goto_prev_raw(Token_Iterator *iterator){
*iterator = make_token_iterator(iterator->range, iterator->token - 1);
return(token_iterator_current(iterator));
}
////////////////////////////////
static String
get_query_string(Application_Links *app, char *query_str, char *string_space, int32_t space_size){
Query_Bar bar;

18
4coder_helper.h
View File

@ -115,14 +115,26 @@ struct Stream_Chunk{
char *data;
};
struct Stream_Tokens{
struct Stream_Tokens_DEP{
Application_Links *app;
Buffer_Summary *buffer;
Cpp_Token *base_tokens;
Cpp_Token *tokens;
int32_t start, end;
int32_t count, token_count;
int32_t start;
int32_t end;
int32_t count;
int32_t token_count;
};
struct Token_Range{
Cpp_Token *first;
Cpp_Token *one_past_last;
};
struct Token_Iterator{
Cpp_Token *token;
Token_Range range;
};
////////////////////////////////

3
4coder_scope_commands.h
View File

@ -16,8 +16,7 @@ enum{
};
struct Statement_Parser{
Stream_Tokens stream;
int32_t token_index;
Token_Iterator token_iterator;
Buffer_Summary *buffer;
};

17
4ed_api_implementation.cpp
View File

@ -1063,7 +1063,7 @@ The number of output tokens will be end_token - start_token.)
Editing_File *file = imp_get_file(models, buffer);
Cpp_Token_Array token_array = file->state.token_array;
bool32 result = false;
if (file && token_array.tokens && file->state.tokens_complete){
if (file != 0 && token_array.tokens != 0 && file->state.tokens_complete){
if (0 <= start_token && start_token <= end_token && end_token <= token_array.count){
result = true;
memcpy(tokens_out, token_array.tokens + start_token, sizeof(Cpp_Token)*(end_token - start_token));
@ -1072,6 +1072,21 @@ The number of output tokens will be end_token - start_token.)
return(result);
}
API_EXPORT bool32
Buffer_Get_Token_Range(Application_Links *app, Buffer_Summary *buffer, Cpp_Token **first_token_out, Cpp_Token **one_past_last_token_out)
{
Models *models = (Models*)app->cmd_context;
Editing_File *file = imp_get_file(models, buffer);
Cpp_Token_Array token_array = file->state.token_array;
bool32 result = false;
if (file != 0 && token_array.tokens != 0 && file->state.tokens_complete){
result = true;
*first_token_out = token_array.tokens;
*one_past_last_token_out = token_array.tokens + token_array.count;
}
return(result);
}
API_EXPORT bool32
Buffer_Get_Token_Index(Application_Links *app, Buffer_Summary *buffer, int32_t pos, Cpp_Get_Token_Result *get_result)
/*