fixed subtle relex bug

4coder_custom.h

@@ -12,6 +12,7 @@
 #include "4coder_style.h"
 #include "4coder_rect.h"
 #include "4coder_mem.h"
+#include "4cpp_lexer_types.h"
 
 #ifndef FSTRING_STRUCT
 #define FSTRING_STRUCT

4coder_default_include.cpp

@@ -887,6 +887,8 @@ move_past_lead_whitespace(Application_Links *app, View_Summary *view, Buffer_Sum
     }
 }
 
+//#include "4coder_auto_indent.cpp"
+
 CUSTOM_COMMAND_SIG(auto_tab_line_at_cursor){
     uint32_t access = AccessOpen;
     View_Summary view = app->get_active_view(app, access);

4cpp_lexer.h

@@ -14,6 +14,9 @@
 
 #define FCPP_INTERNAL FCPP_LINK
 
+#include <stdint.h>
+#define FSTRING_IMPLEMENTATION
+#include "4coder_string.h"
 #include "4cpp_lexer_types.h"
 #include "4cpp_lexer_tables.c"
 
@@ -433,7 +436,7 @@ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, int32_t siz
                 
                 int32_t sub_match = -1;
                 string_set_match_table(keywords, sizeof(*keywords), ArrayCount(keywords),
-                                       make_string(S.tb, S.tb_pos), &sub_match);
+                                       make_string(S.tb, S.tb_pos-1), &sub_match);
                 
                 if (sub_match != -1){
                     String_And_Flag data = keywords[sub_match];
@@ -1165,6 +1168,8 @@ cpp_shift_token_starts(Cpp_Token_Array *array, int32_t from_token_i, int32_t shi
     }
 }
 
+// TODO(allen): This relex system is a little bit broken.  It doesn't allow for the
+// data chunks and it doesn't actually set up the state mid-data stream properly.
 FCPP_INTERNAL int32_t
 cpp_relex_nonalloc_main(Cpp_Relex_State *state,
                         Cpp_Token_Array *relex_array,
@@ -1175,52 +1180,54 @@ cpp_relex_nonalloc_main(Cpp_Relex_State *state,
     
     cpp_shift_token_starts(array, state->end_token_i, state->amount);
     
-    Cpp_Lex_Data lex = cpp_lex_data_init(spare);
-    lex.pp_state = cpp_token_get_pp_state(tokens[state->start_token_i].state_flags);
-    lex.pos = state->relex_start;
-    
     int32_t relex_end_i = state->end_token_i;
     Cpp_Token match_token = cpp_index_array(array, state->size, relex_end_i);
     Cpp_Token end_token = match_token;
     int32_t went_too_far = false;
     
-    // TODO(allen): This can be better I suspect.
+    if (state->relex_start < state->size){
-    for (;;){
+        Cpp_Lex_Data lex = cpp_lex_data_init(spare);
-        int32_t result = 
+        lex.pp_state = cpp_token_get_pp_state(tokens[state->start_token_i].state_flags);
-            cpp_lex_nonalloc_no_null_out_limit(&lex, state->data,
+        lex.pos = state->relex_start;
-                                               state->size, state->size,
-                                               relex_array, 1);
         
-        switch (result){
+        // TODO(allen): This can be better I suspect.
-            case LexResult_HitTokenLimit:
+        for (;;){
-            {
+            int32_t result = 
-                Cpp_Token token = relex_array->tokens[relex_array->count-1];
+                cpp_lex_nonalloc_no_null_out_limit(&lex, state->data,
-                if (token.start == end_token.start &&
+                                                   state->size, state->size,
-                    token.size == end_token.size &&
+                                                   relex_array, 1);
-                    token.flags == end_token.flags &&
+            
-                    token.state_flags == end_token.state_flags){
+            switch (result){
-                    --relex_array->count;
+                case LexResult_HitTokenLimit:
-                    goto double_break;
+                {
+                    Cpp_Token token = relex_array->tokens[relex_array->count-1];
+                    if (token.start == end_token.start &&
+                        token.size == end_token.size &&
+                        token.flags == end_token.flags &&
+                        token.state_flags == end_token.state_flags){
+                        --relex_array->count;
+                        goto double_break;
+                    }
+                    
+                    while (lex.pos > end_token.start && relex_end_i < array->count){
+                        ++relex_end_i;
+                        end_token = cpp_index_array(array, state->size, relex_end_i);
+                    }
                 }
+                break;
                 
-                while (lex.pos > end_token.start && relex_end_i < array->count){
+                case LexResult_NeedChunk: Assert(!"Invalid path"); break;
-                    ++relex_end_i;
+                
-                    end_token = cpp_index_array(array, state->size, relex_end_i);
+                case LexResult_NeedTokenMemory:
-                }
+                went_too_far = true;
+                goto double_break;
+                
+                case LexResult_Finished:
+                goto double_break;
             }
-            break;
-            
-            case LexResult_NeedChunk: Assert(!"Invalid path"); break;
-            
-            case LexResult_NeedTokenMemory:
-            went_too_far = true;
-            goto double_break;
-            
-            case LexResult_Finished:
-            goto double_break;
         }
+        double_break:;
     }
-    double_break:;
     
     if (!went_too_far){
         *relex_end = relex_end_i;
@@ -1312,6 +1319,8 @@ DOC_SEE(cpp_make_token_array)
     S.tb = (char*)malloc(size);
     int32_t quit = 0;
     
+    char empty = 0;
+    
     token_array_out->count = 0;
     for (;!quit;){
         int32_t result = cpp_lex_step(&S, data, size, HAS_NULL_TERM, token_array_out, NO_OUT_LIMIT);
@@ -1326,10 +1335,10 @@ DOC_SEE(cpp_make_token_array)
                 Assert(token_array_out->count < token_array_out->max_count);
                 
                 // NOTE(allen): We told the system we would provide the null
-                // terminator, but we didn't actually, so provide the null
+                // terminator, but as it turned out we didn't actually. So in
-                // terminator via this one byte chunk.
+                // the next iteration pass a 1 byte chunk with the null terminator.
-                char empty = 0;
+                data = &empty;
-                cpp_lex_step(&S, &empty, 1, HAS_NULL_TERM, token_array_out, NO_OUT_LIMIT);
+                size = 1;
             }break;
             
             case LexResult_NeedTokenMemory:

4ed_metagen.cpp

@@ -2545,10 +2545,8 @@ generate_custom_headers(){
         {make_lit_string("ENUM")    , Item_Enum    } ,
     };
     
-#if 0
     Meta_Unit unit = compile_meta_unit(part, type_files, ArrayCount(type_files),
                                        type_keys, ArrayCount(type_keys));
-#endif
     
     // NOTE(allen): Output
     String out = str_alloc(part, 10 << 20);
@@ -2922,9 +2920,9 @@ generate_custom_headers(){
         
         static Section sections[] = {
             {"introduction", "Introduction"},
-        //    {"4coder_systems", "4coder Systems"},
+            {"4coder_systems", "4coder Systems"},
-        //    {"types_and_functions", "Types and Functions"},
+            {"types_and_functions", "Types and Functions"},
-        //    {"string_library", "String Library"},
+            {"string_library", "String Library"},
             {"lexer_library", "Lexer Library"}
         };
         
@@ -2952,6 +2950,8 @@ generate_custom_headers(){
         append_sc(&out, sections[msection].display_string);
         append_sc(&out, "</h2>");
         
+#if 0
+        // NOTE(allen): doc intro for lexer standalone
         append_sc(&out,
                   "<div>"
                   "<p>This is the documentation for the 4cpp lexer version 1.0. "
@@ -2964,8 +2964,7 @@ generate_custom_headers(){
                   "4coder forums hosted on handmade.network at "
                   "<span style='"CODE_STYLE"'>4coder.handmade.network</span></p>"
                   "</div>");
-        
+#endif
-#if 0
         
         append_sc(&out,
                   "<div>"
@@ -2995,6 +2994,7 @@ generate_custom_headers(){
         
 #undef MAJOR_SECTION
 #define MAJOR_SECTION "3"
+        msection = 2;
         
         append_sc(&out, "\n<h2 id='section_");
         append_sc(&out, sections[msection].id_string);
@@ -3056,6 +3056,7 @@ generate_custom_headers(){
         
 #undef MAJOR_SECTION
 #define MAJOR_SECTION "4"
+        msection = 3;
         
         append_sc(&out, "\n<h2 id='section_");
         append_sc(&out, sections[msection].id_string);
@@ -3092,13 +3093,7 @@ generate_custom_headers(){
         
 #undef MAJOR_SECTION
 #define MAJOR_SECTION "5"
-        
+        msection = 4;
-#endif
-        
-#undef MAJOR_SECTION
-#define MAJOR_SECTION "2"
-        
-        msection = 1;
         
         append_sc(&out, "\n<h2 id='section_");
         append_sc(&out, sections[msection].id_string);

TODO.txt

@@ -75,7 +75,6 @@
 ;
 ; [] indication on failure to save
 ; [] history is broken, revist the entire system
-;
 ; [] 8.0\Include\um\dsound.h (not reproduced, get more info)
 ; [] paste external text from bug report (in email) (not reproduced, get more info)
 ;
@@ -89,6 +88,7 @@
 ; [] option to not open *messages* every startup
 ; [] commands for resizing panels
 ; [] make panel resizing not whacky with child panels
+; [] killing compilation panel changes active panel
 ; [] control over how mouse effects panel focus
 ; [] API docs as text file
 ; [] user file bar string