introduced token streams

4coder_custom_api.h

@@ -15,6 +15,7 @@
 #define BUFFER_SET_SETTING_SIG(n) bool32 n(Application_Links *app, Buffer_Summary *buffer, Buffer_Setting_ID setting, int32_t value)
 #define BUFFER_TOKEN_COUNT_SIG(n) int32_t n(Application_Links *app, Buffer_Summary *buffer)
 #define BUFFER_READ_TOKENS_SIG(n) bool32 n(Application_Links *app, Buffer_Summary *buffer, int32_t start_token, int32_t end_token, Cpp_Token *tokens_out)
+#define BUFFER_GET_TOKEN_INDEX_SIG(n) bool32 n(Application_Links *app, Buffer_Summary *buffer, int32_t pos, Cpp_Get_Token_Result *get_result)
 #define CREATE_BUFFER_SIG(n) Buffer_Summary n(Application_Links *app, char *filename, int32_t filename_len, Buffer_Create_Flag flags)
 #define SAVE_BUFFER_SIG(n) bool32 n(Application_Links *app, Buffer_Summary *buffer, char *filename, int32_t filename_len, uint32_t flags)
 #define KILL_BUFFER_SIG(n) bool32 n(Application_Links *app, Buffer_Identifier buffer, View_ID view_id, Buffer_Kill_Flag flags)
@@ -75,6 +76,7 @@ typedef BUFFER_BATCH_EDIT_SIG(Buffer_Batch_Edit_Function);
 typedef BUFFER_SET_SETTING_SIG(Buffer_Set_Setting_Function);
 typedef BUFFER_TOKEN_COUNT_SIG(Buffer_Token_Count_Function);
 typedef BUFFER_READ_TOKENS_SIG(Buffer_Read_Tokens_Function);
+typedef BUFFER_GET_TOKEN_INDEX_SIG(Buffer_Get_Token_Index_Function);
 typedef CREATE_BUFFER_SIG(Create_Buffer_Function);
 typedef SAVE_BUFFER_SIG(Save_Buffer_Function);
 typedef KILL_BUFFER_SIG(Kill_Buffer_Function);
@@ -136,6 +138,7 @@ Buffer_Batch_Edit_Function *buffer_batch_edit;
 Buffer_Set_Setting_Function *buffer_set_setting;
 Buffer_Token_Count_Function *buffer_token_count;
 Buffer_Read_Tokens_Function *buffer_read_tokens;
+Buffer_Get_Token_Index_Function *buffer_get_token_index;
 Create_Buffer_Function *create_buffer;
 Save_Buffer_Function *save_buffer;
 Kill_Buffer_Function *kill_buffer;
@@ -204,6 +207,7 @@ app_links->buffer_batch_edit = Buffer_Batch_Edit;\
 app_links->buffer_set_setting = Buffer_Set_Setting;\
 app_links->buffer_token_count = Buffer_Token_Count;\
 app_links->buffer_read_tokens = Buffer_Read_Tokens;\
+app_links->buffer_get_token_index = Buffer_Get_Token_Index;\
 app_links->create_buffer = Create_Buffer;\
 app_links->save_buffer = Save_Buffer;\
 app_links->kill_buffer = Kill_Buffer;\

4coder_default_include.cpp

@@ -131,6 +131,20 @@ init_memory(Application_Links *app){
     general_memory_open(&global_general, general_mem, general_size);
 }
 
+//
+// Helpers
+//
+
+static void
+refresh_buffer(Application_Links *app, Buffer_Summary *buffer){
+    *buffer = app->get_buffer(app, buffer->buffer_id, AccessAll);
+}
+
+static void
+refresh_view(Application_Links *app, View_Summary *view){
+    *view = app->get_view(app, view->view_id, AccessAll);
+}
+
 //
 // Buffer Streaming
 //
@@ -148,7 +162,7 @@ struct Stream_Chunk{
     char *data;
 };
 
-int32_t
+static int32_t
 round_down(int32_t x, int32_t b){
     int32_t r = 0;
     if (x >= 0){
@@ -157,7 +171,7 @@ round_down(int32_t x, int32_t b){
     return(r);
 }
 
-int32_t
+static int32_t
 round_up(int32_t x, int32_t b){
     int32_t r = 0;
     if (x >= 0){
@@ -166,21 +180,10 @@ round_up(int32_t x, int32_t b){
     return(r);
 }
 
-void
-refresh_buffer(Application_Links *app, Buffer_Summary *buffer){
-    *buffer = app->get_buffer(app, buffer->buffer_id, AccessAll);
-}
-
-void
-refresh_view(Application_Links *app, View_Summary *view){
-    *view = app->get_view(app, view->view_id, AccessAll);
-}
-
-bool32
-init_stream_chunk(Stream_Chunk *chunk,
-                  Application_Links *app, Buffer_Summary *buffer,
+static bool32
+init_stream_chunk(Stream_Chunk *chunk, Application_Links *app, Buffer_Summary *buffer,
                   int32_t pos, char *data, int32_t size){
-    bool32 result = false;
+    bool32 result = 0;
     
     refresh_buffer(app, buffer);
     if (pos >= 0 && pos < buffer->size && size > 0){
@@ -205,17 +208,18 @@ init_stream_chunk(Stream_Chunk *chunk,
         if (chunk->start < chunk->end){
             app->buffer_read_range(app, buffer, chunk->start, chunk->end, chunk->base_data);
             chunk->data = chunk->base_data - chunk->start;
-            result = true;
+            result = 1;
         }
     }
+    
     return(result);
 }
 
-int32_t
+static bool32
 forward_stream_chunk(Stream_Chunk *chunk){
     Application_Links *app = chunk->app;
     Buffer_Summary *buffer = chunk->buffer;
-    int32_t result = false;
+    bool32 result = 0;
     
     refresh_buffer(app, buffer);
     if (chunk->end < buffer->size){
@@ -232,7 +236,7 @@ forward_stream_chunk(Stream_Chunk *chunk){
         if (chunk->start < chunk->end){
             app->buffer_read_range(app, buffer, chunk->start, chunk->end, chunk->base_data);
             chunk->data = chunk->base_data - chunk->start;
-            result = true;
+            result = 1;
         }
     }
     
@@ -241,17 +245,17 @@ forward_stream_chunk(Stream_Chunk *chunk){
         chunk->end = buffer->size + 1;
         chunk->base_data[0] = 0;
         chunk->data = chunk->base_data - chunk->start;
-        result = true;
+        result = 1;
     }
     
     return(result);
 }
 
-int32_t
+static bool32
 backward_stream_chunk(Stream_Chunk *chunk){
     Application_Links *app = chunk->app;
     Buffer_Summary *buffer = chunk->buffer;
-    int32_t result = false;
+    bool32 result = 0;
     
     refresh_buffer(app, buffer);
     if (chunk->start > 0){
@@ -268,7 +272,7 @@ backward_stream_chunk(Stream_Chunk *chunk){
         if (chunk->start < chunk->end){
             app->buffer_read_range(app, buffer, chunk->start, chunk->end, chunk->base_data);
             chunk->data = chunk->base_data - chunk->start;
-            result = true;
+            result = 1;
         }
     }
     
@@ -277,6 +281,91 @@ backward_stream_chunk(Stream_Chunk *chunk){
         chunk->end = 0;
         chunk->base_data[0] = 0;
         chunk->data = chunk->base_data - chunk->start;
+        result = 1;
+    }
+    
+    return(result);
+}
+
+typedef struct Stream_Tokens{
+    Application_Links *app;
+    Buffer_Summary *buffer;
+    
+    Cpp_Token *base_tokens;
+    Cpp_Token *tokens;
+    int32_t start, end;
+    int32_t count, token_count;
+} Stream_Tokens;
+
+static bool32
+init_stream_tokens(Stream_Tokens *stream, Application_Links *app, Buffer_Summary *buffer,
+                   int32_t pos, Cpp_Token *data, int32_t count){
+    bool32 result = 0;
+    
+    refresh_buffer(app, buffer);
+    
+    int32_t token_count = app->buffer_token_count(app, buffer);
+    if (pos >= 0 && pos < token_count && count > 0){
+        stream->app = app;
+        stream->buffer = buffer;
+        stream->base_tokens = data;
+        stream->count = count;
+        stream->start = round_down(pos, count);
+        stream->end = round_up(pos, count);
+        
+        app->buffer_read_tokens(app, buffer, stream->start, stream->end, stream->base_tokens);
+        stream->tokens = stream->base_tokens - stream->start;
+        result = 1;
+    }
+    
+    return(result);
+}
+
+static bool32
+forward_stream_tokens(Stream_Tokens *stream){
+    Application_Links *app = stream->app;
+    Buffer_Summary *buffer = stream->buffer;
+    bool32 result = 0;
+    
+    refresh_buffer(app, buffer);
+    if (stream->end < stream->token_count){
+        stream->start = stream->end;
+        stream->end += stream->count;
+        
+        if (stream->token_count < stream->end){
+            stream->end = stream->token_count;
+        }
+        
+        if (stream->start < stream->end){
+            app->buffer_read_tokens(app, buffer, stream->start, stream->end, stream->base_tokens);
+            stream->tokens = stream->base_tokens - stream->start;
+            result = 1;
+        }
+    }
+    
+    return(result);
+}
+
+static bool32
+backward_stream_tokens(Stream_Tokens *stream){
+    Application_Links *app = stream->app;
+    Buffer_Summary *buffer = stream->buffer;
+    bool32 result = 0;
+    
+    refresh_buffer(app, buffer);
+    if (stream->start > 0){
+        stream->end = stream->start;
+        stream->start -= stream->count;
+        
+        if (0 > stream->start){
+            stream->start = 0;
+        }
+        
+        if (stream->start < stream->end){
+            app->buffer_read_tokens(app, buffer, stream->start, stream->end, stream->base_tokens);
+            stream->tokens = stream->base_tokens - stream->start;
+            result = 1;
+        }
     }
     
     return(result);
@@ -310,7 +399,7 @@ buffer_seek_delimiter_forward(Application_Links *app, Buffer_Summary *buffer,
     finished:;
 }
 
-void
+static void
 buffer_seek_delimiter_backward(Application_Links *app, Buffer_Summary *buffer,
                               int32_t pos, char delim, int32_t *result){
     if (buffer->exists){
@@ -344,7 +433,7 @@ buffer_seek_delimiter_backward(Application_Links *app, Buffer_Summary *buffer,
 // NOTE(allen): This is limitted to a string size of 512.
 // You can push it up or do something more clever by just
 // replacing char read_buffer[512]; with more memory.
-void
+static void
 buffer_seek_string_forward(Application_Links *app, Buffer_Summary *buffer,
                            int32_t pos, int32_t end, char *str, int32_t size, int32_t *result){
     char read_buffer[512];
@@ -399,7 +488,7 @@ buffer_seek_string_forward(Application_Links *app, Buffer_Summary *buffer,
 // NOTE(allen): This is limitted to a string size of 512.
 // You can push it up or do something more clever by just
 // replacing char read_buffer[512]; with more memory.
-void
+static void
 buffer_seek_string_backward(Application_Links *app, Buffer_Summary *buffer,
                             int32_t pos, int32_t min, char *str, int32_t size, int32_t *result){
     char read_buffer[512];
@@ -448,7 +537,7 @@ buffer_seek_string_backward(Application_Links *app, Buffer_Summary *buffer,
 // NOTE(allen): This is limitted to a string size of 512.
 // You can push it up or do something more clever by just
 // replacing char read_buffer[512]; with more memory.
-void
+static void
 buffer_seek_string_insensitive_forward(Application_Links *app, Buffer_Summary *buffer,
                                        int32_t pos, int32_t end, char *str, int32_t size, int32_t *result){
     char read_buffer[512];
@@ -498,7 +587,7 @@ buffer_seek_string_insensitive_forward(Application_Links *app, Buffer_Summary *b
 // NOTE(allen): This is limitted to a string size of 512.
 // You can push it up or do something more clever by just
 // replacing char read_buffer[512]; with more memory.
-void
+static void
 buffer_seek_string_insensitive_backward(Application_Links *app, Buffer_Summary *buffer,
                                         int32_t pos, int32_t min, char *str, int32_t size, int32_t *result){
     char read_buffer[512];

4ed_api_implementation.cpp

@@ -842,6 +842,23 @@ The number of output tokens will be end_token - start_token.)
     return(result);
 }
 
+API_EXPORT bool32
+Buffer_Get_Token_Index(Application_Links *app, Buffer_Summary *buffer, int32_t pos, Cpp_Get_Token_Result *get_result){
+    Command_Data *cmd = (Command_Data*)app->cmd_context;
+    Editing_File *file = imp_get_file(cmd, buffer);
+    Cpp_Token_Array token_array = file->state.token_array;
+    
+    bool32 result = 0;
+    if (file && token_array.tokens && file->state.tokens_complete){
+        result = 1;
+        Cpp_Get_Token_Result get = {0};
+        get = cpp_get_token(&token_array, pos);
+        *get_result = get;
+    }
+    
+    return(result);
+}
+
 API_EXPORT Buffer_Summary
 Create_Buffer(Application_Links *app, char *filename, int32_t filename_len, Buffer_Create_Flag flags)/*
 DOC_PARAM(filename, The filename parameter specifies the name of the file to be opened or created;