begining rewrite of token iteration

4coder_function_list.cpp

@@ -47,6 +47,131 @@ buffered_write_stream_write(Application_Links *app, Buffered_Write_Stream *strea
     }
 }
 
+static Get_Positions_Results
+get_function_positions(Application_Links *app, Buffer_Summary *buffer, int32_t token_index, Function_Positions *positions_array, int32_t positions_max){
+    Get_Positions_Results result = {};
+    
+    Token_Range token_range = buffer_get_token_range(app, buffer->buffer_id);
+    if (token_range.first != 0){
+        Token_Iterator token_it = make_token_iterator(token_range, token_index);
+        
+        
+        int32_t nest_level = 0;
+        int32_t paren_nest_level = 0;
+        
+        int32_t first_paren_index = 0;
+        int32_t first_paren_position = 0;
+        int32_t last_paren_index = 0;
+        
+        // Look for the next token at global scope that might need to be printed.
+        mode1:
+        Assert(nest_level == 0);
+        Assert(paren_nest_level == 0);
+        first_paren_index = 0;
+        first_paren_position = 0;
+        last_paren_index = 0;
+        for (Cpp_Token *token = token_iterator_current(&token_it);
+             token != 0;
+             token = token_iterator_goto_next(&token_it)){
+            if (!(token->flags & CPP_TFLAG_PP_BODY)){
+                switch (token->type){
+                    case CPP_TOKEN_BRACE_OPEN:
+                    {
+                        ++nest_level;
+                    }break;
+                    
+                    case CPP_TOKEN_BRACE_CLOSE:
+                    {
+                        if (nest_level > 0){
+                            --nest_level;
+                        }
+                    }break;
+                    
+                    case CPP_TOKEN_PARENTHESE_OPEN:
+                    {
+                        if (nest_level == 0){
+                            first_paren_index = token_index;
+                            first_paren_position = token->start;
+                            goto paren_mode1;
+                        }
+                    }break;
+                }
+            }
+        }
+        goto end;
+        
+        // Look for a closing parenthese to mark the end of a function signature.
+        paren_mode1:
+        paren_nest_level = 0;
+        for (Cpp_Token *token = token_iterator_current(&token_it);
+             token != 0;
+             token = token_iterator_goto_next(&token_it)){
+            if (!(token->flags & CPP_TFLAG_PP_BODY)){
+                switch (token->type){
+                    case CPP_TOKEN_PARENTHESE_OPEN:
+                    {
+                        ++paren_nest_level;
+                    }break;
+                    
+                    case CPP_TOKEN_PARENTHESE_CLOSE:
+                    {
+                        --paren_nest_level;
+                        if (paren_nest_level == 0){
+                            last_paren_index = token_index;
+                            goto paren_mode2;
+                        }
+                    }break;
+                }
+            }
+        }
+        goto end;
+        
+        // Look backwards from an open parenthese to find the start of a function signature.
+        paren_mode2:
+        {
+            Cpp_Token *restore_point = token_iterator_current(&token_it);
+            int32_t local_index = first_paren_index;
+            int32_t signature_start_index = 0;
+            
+            for (Cpp_Token *token = token_iterator_current(&token_it);
+                 token != 0;
+                 token = token_iterator_goto_prev(&token_it)){
+                if ((token->flags & CPP_TFLAG_PP_BODY) || (token->flags & CPP_TFLAG_PP_DIRECTIVE) || token->type == CPP_TOKEN_BRACE_CLOSE || token->type == CPP_TOKEN_SEMICOLON || token->type == CPP_TOKEN_PARENTHESE_CLOSE){
+                    ++local_index;
+                    signature_start_index = local_index;
+                    goto paren_mode2_done;
+                }
+            }
+            
+            // When this loop ends by going all the way back to the beginning set the signature start to 0 and fall through to the printing phase.
+            signature_start_index = 0;
+            
+            paren_mode2_done:;
+            {
+                Function_Positions positions;
+                positions.sig_start_index = signature_start_index;
+                positions.sig_end_index = last_paren_index;
+                positions.open_paren_pos = first_paren_position;
+                positions_array[result.positions_count++] = positions;
+            }
+            
+            token_iterator_set(&token_it, restore_point);
+            if (result.positions_count >= positions_max){
+                result.next_token_index = token_index;
+                result.still_looping = true;
+                goto end;
+            }
+            
+            goto mode1;
+        }
+        end:;
+        
+    }
+    
+    return(result);
+}
+
+#if 0
 static Get_Positions_Results
 get_function_positions(Application_Links *app, Buffer_Summary *buffer, int32_t token_index, Function_Positions *positions_array, int32_t positions_max){
     Get_Positions_Results result = {};
@@ -178,6 +303,7 @@ get_function_positions(Application_Links *app, Buffer_Summary *buffer, int32_t t
     
     return(result);
 }
+#endif
 
 static void
 print_positions_buffered(Application_Links *app, Buffer_Summary *buffer, Function_Positions *positions_array, int32_t positions_count, Buffered_Write_Stream *stream){

4coder_generated/app_functions.h

@@ -22,6 +22,7 @@ struct Application_Links;
 #define BUFFER_GET_MANAGED_SCOPE_SIG(n) Managed_Scope n(Application_Links *app, Buffer_ID buffer_id)
 #define BUFFER_TOKEN_COUNT_SIG(n) int32_t n(Application_Links *app, Buffer_Summary *buffer)
 #define BUFFER_READ_TOKENS_SIG(n) bool32 n(Application_Links *app, Buffer_Summary *buffer, int32_t start_token, int32_t end_token, Cpp_Token *tokens_out)
+#define BUFFER_GET_TOKEN_RANGE_SIG(n) bool32 n(Application_Links *app, Buffer_Summary *buffer, Cpp_Token **first_token_out, Cpp_Token **one_past_last_token_out)
 #define BUFFER_GET_TOKEN_INDEX_SIG(n) bool32 n(Application_Links *app, Buffer_Summary *buffer, int32_t pos, Cpp_Get_Token_Result *get_result)
 #define BUFFER_SEND_END_SIGNAL_SIG(n) bool32 n(Application_Links *app, Buffer_Summary *buffer)
 #define CREATE_BUFFER_SIG(n) Buffer_Summary n(Application_Links *app, char *filename, int32_t filename_len, Buffer_Create_Flag flags)
@@ -149,6 +150,7 @@ typedef BUFFER_SET_SETTING_SIG(Buffer_Set_Setting_Function);
 typedef BUFFER_GET_MANAGED_SCOPE_SIG(Buffer_Get_Managed_Scope_Function);
 typedef BUFFER_TOKEN_COUNT_SIG(Buffer_Token_Count_Function);
 typedef BUFFER_READ_TOKENS_SIG(Buffer_Read_Tokens_Function);
+typedef BUFFER_GET_TOKEN_RANGE_SIG(Buffer_Get_Token_Range_Function);
 typedef BUFFER_GET_TOKEN_INDEX_SIG(Buffer_Get_Token_Index_Function);
 typedef BUFFER_SEND_END_SIGNAL_SIG(Buffer_Send_End_Signal_Function);
 typedef CREATE_BUFFER_SIG(Create_Buffer_Function);
@@ -278,6 +280,7 @@ Buffer_Set_Setting_Function *buffer_set_setting;
 Buffer_Get_Managed_Scope_Function *buffer_get_managed_scope;
 Buffer_Token_Count_Function *buffer_token_count;
 Buffer_Read_Tokens_Function *buffer_read_tokens;
+Buffer_Get_Token_Range_Function *buffer_get_token_range;
 Buffer_Get_Token_Index_Function *buffer_get_token_index;
 Buffer_Send_End_Signal_Function *buffer_send_end_signal;
 Create_Buffer_Function *create_buffer;
@@ -406,6 +409,7 @@ Buffer_Set_Setting_Function *buffer_set_setting_;
 Buffer_Get_Managed_Scope_Function *buffer_get_managed_scope_;
 Buffer_Token_Count_Function *buffer_token_count_;
 Buffer_Read_Tokens_Function *buffer_read_tokens_;
+Buffer_Get_Token_Range_Function *buffer_get_token_range_;
 Buffer_Get_Token_Index_Function *buffer_get_token_index_;
 Buffer_Send_End_Signal_Function *buffer_send_end_signal_;
 Create_Buffer_Function *create_buffer_;
@@ -542,6 +546,7 @@ app_links->buffer_set_setting_ = Buffer_Set_Setting;\
 app_links->buffer_get_managed_scope_ = Buffer_Get_Managed_Scope;\
 app_links->buffer_token_count_ = Buffer_Token_Count;\
 app_links->buffer_read_tokens_ = Buffer_Read_Tokens;\
+app_links->buffer_get_token_range_ = Buffer_Get_Token_Range;\
 app_links->buffer_get_token_index_ = Buffer_Get_Token_Index;\
 app_links->buffer_send_end_signal_ = Buffer_Send_End_Signal;\
 app_links->create_buffer_ = Create_Buffer;\
@@ -670,6 +675,7 @@ static bool32 buffer_set_setting(Application_Links *app, Buffer_Summary *buffer,
 static Managed_Scope buffer_get_managed_scope(Application_Links *app, Buffer_ID buffer_id){return(app->buffer_get_managed_scope(app, buffer_id));}
 static int32_t buffer_token_count(Application_Links *app, Buffer_Summary *buffer){return(app->buffer_token_count(app, buffer));}
 static bool32 buffer_read_tokens(Application_Links *app, Buffer_Summary *buffer, int32_t start_token, int32_t end_token, Cpp_Token *tokens_out){return(app->buffer_read_tokens(app, buffer, start_token, end_token, tokens_out));}
+static bool32 buffer_get_token_range(Application_Links *app, Buffer_Summary *buffer, Cpp_Token **first_token_out, Cpp_Token **one_past_last_token_out){return(app->buffer_get_token_range(app, buffer, first_token_out, one_past_last_token_out));}
 static bool32 buffer_get_token_index(Application_Links *app, Buffer_Summary *buffer, int32_t pos, Cpp_Get_Token_Result *get_result){return(app->buffer_get_token_index(app, buffer, pos, get_result));}
 static bool32 buffer_send_end_signal(Application_Links *app, Buffer_Summary *buffer){return(app->buffer_send_end_signal(app, buffer));}
 static Buffer_Summary create_buffer(Application_Links *app, char *filename, int32_t filename_len, Buffer_Create_Flag flags){return(app->create_buffer(app, filename, filename_len, flags));}
@@ -798,6 +804,7 @@ static bool32 buffer_set_setting(Application_Links *app, Buffer_Summary *buffer,
 static Managed_Scope buffer_get_managed_scope(Application_Links *app, Buffer_ID buffer_id){return(app->buffer_get_managed_scope_(app, buffer_id));}
 static int32_t buffer_token_count(Application_Links *app, Buffer_Summary *buffer){return(app->buffer_token_count_(app, buffer));}
 static bool32 buffer_read_tokens(Application_Links *app, Buffer_Summary *buffer, int32_t start_token, int32_t end_token, Cpp_Token *tokens_out){return(app->buffer_read_tokens_(app, buffer, start_token, end_token, tokens_out));}
+static bool32 buffer_get_token_range(Application_Links *app, Buffer_Summary *buffer, Cpp_Token **first_token_out, Cpp_Token **one_past_last_token_out){return(app->buffer_get_token_range_(app, buffer, first_token_out, one_past_last_token_out));}
 static bool32 buffer_get_token_index(Application_Links *app, Buffer_Summary *buffer, int32_t pos, Cpp_Get_Token_Result *get_result){return(app->buffer_get_token_index_(app, buffer, pos, get_result));}
 static bool32 buffer_send_end_signal(Application_Links *app, Buffer_Summary *buffer){return(app->buffer_send_end_signal_(app, buffer));}
 static Buffer_Summary create_buffer(Application_Links *app, char *filename, int32_t filename_len, Buffer_Create_Flag flags){return(app->create_buffer_(app, filename, filename_len, flags));}

4coder_generated/command_metadata.h

@@ -312,10 +312,10 @@ static Command_Metadata fcoder_metacmd_table[220] = {
 { PROC_LINKS(interactive_switch_buffer, 0), "interactive_switch_buffer", 25,  "Interactively switch to an open buffer.", 39, "w:\\4ed\\code\\4coder_lists.cpp", 28, 765 },
 { PROC_LINKS(kill_buffer, 0), "kill_buffer", 11,  "Kills the current buffer.", 25, "w:\\4ed\\code\\4coder_base_commands.cpp", 36, 1557 },
 { PROC_LINKS(left_adjust_view, 0), "left_adjust_view", 16,  "Sets the left size of the view near the x position of the cursor.", 65, "w:\\4ed\\code\\4coder_base_commands.cpp", 36, 133 },
-{ PROC_LINKS(list_all_functions_all_buffers, 0), "list_all_functions_all_buffers", 30,  "Creates a jump list of lines from all buffers that appear to define or declare functions.", 89, "w:\\4ed\\code\\4coder_function_list.cpp", 36, 343 },
+{ PROC_LINKS(list_all_functions_all_buffers, 0), "list_all_functions_all_buffers", 30,  "Creates a jump list of lines from all buffers that appear to define or declare functions.", 89, "w:\\4ed\\code\\4coder_function_list.cpp", 36, 469 },
-{ PROC_LINKS(list_all_functions_all_buffers_lister, 0), "list_all_functions_all_buffers_lister", 37,  "Creates a lister of locations that look like function definitions and declarations all buffers.", 95, "w:\\4ed\\code\\4coder_function_list.cpp", 36, 349 },
+{ PROC_LINKS(list_all_functions_all_buffers_lister, 0), "list_all_functions_all_buffers_lister", 37,  "Creates a lister of locations that look like function definitions and declarations all buffers.", 95, "w:\\4ed\\code\\4coder_function_list.cpp", 36, 475 },
-{ PROC_LINKS(list_all_functions_current_buffer, 0), "list_all_functions_current_buffer", 33,  "Creates a jump list of lines of the current buffer that appear to define or declare functions.", 94, "w:\\4ed\\code\\4coder_function_list.cpp", 36, 320 },
+{ PROC_LINKS(list_all_functions_current_buffer, 0), "list_all_functions_current_buffer", 33,  "Creates a jump list of lines of the current buffer that appear to define or declare functions.", 94, "w:\\4ed\\code\\4coder_function_list.cpp", 36, 446 },
-{ PROC_LINKS(list_all_functions_current_buffer_lister, 0), "list_all_functions_current_buffer_lister", 40,  "Creates a lister of locations that look like function definitions and declarations in the buffer.", 97, "w:\\4ed\\code\\4coder_function_list.cpp", 36, 330 },
+{ PROC_LINKS(list_all_functions_current_buffer_lister, 0), "list_all_functions_current_buffer_lister", 40,  "Creates a lister of locations that look like function definitions and declarations in the buffer.", 97, "w:\\4ed\\code\\4coder_function_list.cpp", 36, 456 },
 { PROC_LINKS(list_all_locations, 0), "list_all_locations", 18,  "Queries the user for a string and lists all exact case-sensitive matches found in all open buffers.", 99, "w:\\4ed\\code\\4coder_search.cpp", 29, 769 },
 { PROC_LINKS(list_all_locations_case_insensitive, 0), "list_all_locations_case_insensitive", 35,  "Queries the user for a string and lists all exact case-insensitive matches found in all open buffers.", 101, "w:\\4ed\\code\\4coder_search.cpp", 29, 783 },
 { PROC_LINKS(list_all_locations_of_identifier, 0), "list_all_locations_of_identifier", 32,  "Reads a token or word under the cursor and lists all exact case-sensitive mathces in all open buffers.", 102, "w:\\4ed\\code\\4coder_search.cpp", 29, 797 },

4coder_helper.cpp

@@ -924,7 +924,7 @@ backward_stream_chunk(Stream_Chunk *chunk){
 ////////////////////////////////
 
 static bool32
-init_stream_tokens(Stream_Tokens *stream, Application_Links *app, Buffer_Summary *buffer,
+init_stream_tokens(Stream_Tokens_DEP *stream, Application_Links *app, Buffer_Summary *buffer,
                    int32_t pos, Cpp_Token *data, int32_t count){
     bool32 result = false;
     
@@ -955,13 +955,13 @@ init_stream_tokens(Stream_Tokens *stream, Application_Links *app, Buffer_Summary
     return(result);
 }
 
-static Stream_Tokens
+static Stream_Tokens_DEP
-begin_temp_stream_token(Stream_Tokens *stream){
+begin_temp_stream_token(Stream_Tokens_DEP *stream){
     return(*stream);
 }
 
 static void
-end_temp_stream_token(Stream_Tokens *stream, Stream_Tokens temp){
+end_temp_stream_token(Stream_Tokens_DEP *stream, Stream_Tokens_DEP temp){
     if (stream->start != temp.start || stream->end != temp.end){
         Application_Links *app = stream->app;
         buffer_read_tokens(app, temp.buffer, temp.start, temp.end, temp.base_tokens);
@@ -972,7 +972,7 @@ end_temp_stream_token(Stream_Tokens *stream, Stream_Tokens temp){
 }
 
 static bool32
-forward_stream_tokens(Stream_Tokens *stream){
+forward_stream_tokens(Stream_Tokens_DEP *stream){
     Application_Links *app = stream->app;
     Buffer_Summary *buffer = stream->buffer;
     bool32 result = false;
@@ -997,7 +997,7 @@ forward_stream_tokens(Stream_Tokens *stream){
 }
 
 static bool32
-backward_stream_tokens(Stream_Tokens *stream){
+backward_stream_tokens(Stream_Tokens_DEP *stream){
     Application_Links *app = stream->app;
     Buffer_Summary *buffer = stream->buffer;
     bool32 result = false;
@@ -1021,6 +1021,101 @@ backward_stream_tokens(Stream_Tokens *stream){
     return(result);
 }
 
+////////////////////////////////
+
+static Token_Range
+buffer_get_token_range(Application_Links *app, Buffer_ID buffer_id){
+    Token_Range range = {};
+    Buffer_Summary buffer = get_buffer(app, buffer_id, AccessAll);
+    buffer_get_token_range(app, &buffer, &range.first, &range.one_past_last);
+    return(range);
+}
+
+static Token_Iterator
+make_token_iterator(Token_Range range, Cpp_Token *token){
+    Token_Iterator iterator = {};
+    if (range.first != 0 && range.one_past_last != 0){
+        if (token == 0 || token < range.first){
+            token = range.first;
+        }
+        if (token > range.one_past_last){
+            token = range.one_past_last;
+        }
+        iterator.token = token;
+        iterator.range = range;
+    }
+    return(iterator);
+}
+
+static Token_Iterator
+make_token_iterator(Token_Range range, int32_t index){
+    return(make_token_iterator(range, range.first + index));
+}
+
+static void
+token_iterator_set(Token_Iterator *iterator, Cpp_Token *token){
+    *iterator = make_token_iterator(iterator->range, token);
+}
+
+static Cpp_Token*
+token_iterator_current(Token_Iterator *iterator){
+    Cpp_Token *token = iterator->token;
+    if (token < iterator->range.first || iterator->range.one_past_last <= token){
+        token = 0;
+    }
+    return(token);
+}
+
+static int32_t
+token_iterator_current_index(Token_Iterator *iterator){
+    int32_t index = -1;
+    Cpp_Token *token = token_iterator_current(iterator);
+    if (token != 0 && iterator->range.first <= token && token <= iterator->range.one_past_last){
+        index = (int32_t)(token - iterator->range.first);
+    }
+    return(index);
+}
+
+static Cpp_Token*
+token_iterator_goto_next(Token_Iterator *iterator){
+    Cpp_Token *token = iterator->token;
+    Cpp_Token *one_past_last = iterator->range.one_past_last;
+    for (token += 1; token < one_past_last; token += 1){
+        if (token->type != CPP_TOKEN_COMMENT){
+            break;
+        }
+    }
+    *iterator = make_token_iterator(iterator->range, token);
+    return(token_iterator_current(iterator));
+}
+
+static Cpp_Token*
+token_iterator_goto_next_raw(Token_Iterator *iterator){
+    *iterator = make_token_iterator(iterator->range, iterator->token + 1);
+    return(token_iterator_current(iterator));
+}
+
+static Cpp_Token*
+token_iterator_goto_prev(Token_Iterator *iterator){
+    Cpp_Token *token = iterator->token;
+    Cpp_Token *first = iterator->range.first;
+    for (token -= 1; token > first; token -= 1){
+        if (token->type != CPP_TOKEN_COMMENT){
+            break;
+        }
+    }
+    *iterator = make_token_iterator(iterator->range, token);
+    return(token_iterator_current(iterator));
+}
+
+static Cpp_Token*
+token_iterator_goto_prev_raw(Token_Iterator *iterator){
+    *iterator = make_token_iterator(iterator->range, iterator->token - 1);
+    return(token_iterator_current(iterator));
+}
+
+////////////////////////////////
+
 static String
 get_query_string(Application_Links *app, char *query_str, char *string_space, int32_t space_size){
     Query_Bar bar;

4coder_helper.h

@@ -115,14 +115,26 @@ struct Stream_Chunk{
     char *data;
 };
 
-struct Stream_Tokens{
+struct Stream_Tokens_DEP{
     Application_Links *app;
     Buffer_Summary *buffer;
     
     Cpp_Token *base_tokens;
     Cpp_Token *tokens;
-    int32_t start, end;
+    int32_t start;
-    int32_t count, token_count;
+    int32_t end;
+    int32_t count;
+    int32_t token_count;
+};
+
+struct Token_Range{
+    Cpp_Token *first;
+    Cpp_Token *one_past_last;
+};
+
+struct Token_Iterator{
+    Cpp_Token *token;
+    Token_Range range;
 };
 
 ////////////////////////////////

4coder_scope_commands.h

@@ -16,8 +16,7 @@ enum{
 };
 
 struct Statement_Parser{
-    Stream_Tokens stream;
+    Token_Iterator token_iterator;
-    int32_t token_index;
     Buffer_Summary *buffer;
 };
 

4ed_api_implementation.cpp

@@ -1063,7 +1063,7 @@ The number of output tokens will be end_token - start_token.)
     Editing_File *file = imp_get_file(models, buffer);
     Cpp_Token_Array token_array = file->state.token_array;
     bool32 result = false;
-    if (file && token_array.tokens && file->state.tokens_complete){
+    if (file != 0 && token_array.tokens != 0 && file->state.tokens_complete){
         if (0 <= start_token && start_token <= end_token && end_token <= token_array.count){
             result = true;
             memcpy(tokens_out, token_array.tokens + start_token, sizeof(Cpp_Token)*(end_token - start_token));
@@ -1072,6 +1072,21 @@ The number of output tokens will be end_token - start_token.)
     return(result);
 }
 
+API_EXPORT bool32
+Buffer_Get_Token_Range(Application_Links *app, Buffer_Summary *buffer, Cpp_Token **first_token_out, Cpp_Token **one_past_last_token_out)
+{
+    Models *models = (Models*)app->cmd_context;
+    Editing_File *file = imp_get_file(models, buffer);
+    Cpp_Token_Array token_array = file->state.token_array;
+    bool32 result = false;
+    if (file != 0 && token_array.tokens != 0 && file->state.tokens_complete){
+        result = true;
+        *first_token_out = token_array.tokens;
+        *one_past_last_token_out = token_array.tokens + token_array.count;
+    }
+    return(result);
+}
+
 API_EXPORT bool32
 Buffer_Get_Token_Index(Application_Links *app, Buffer_Summary *buffer, int32_t pos, Cpp_Get_Token_Result *get_result)
 /*