fixed crash bug in relexing

4ed.cpp

@@ -1689,6 +1689,9 @@ App_Step_Sig(app_step){
         Partition *part = &models->mem.part;
         Temp_Memory temp = begin_temp_memory(part);
         char *buffer = push_array(part, char, buffer_size);
+        i32 unmark_top = 0;
+        i32 unmark_max = (8 << 10);
+        Editing_File **unmark = (Editing_File**)push_array(part, Editing_File*, unmark_max);
         
         Working_Set *working_set = &models->working_set;
         
@@ -1702,13 +1705,22 @@ App_Step_Sig(app_step){
                     if (file->state.ignore_behind_os == 0){
                         file_mark_behind_os(file);
                     }
-                    else{
+                    else if (file->state.ignore_behind_os == 1){
-                        file->state.ignore_behind_os = 0;
+                        // TODO(allen): I need the ability to put a file back on the list
+                        if (unmark_top == unmark_max){
+                            break;
+                        }
+                        file->state.ignore_behind_os = 2;
+                        unmark[unmark_top++] = file;
                     }
                 }
             }
         }
         
+        for (i32 i = 0; i < unmark_top; ++i){
+            unmark[i]->state.ignore_behind_os = 0;
+        }
+        
         end_temp_memory(temp);
     }
     

4ed_file_view.cpp

@@ -1118,6 +1118,7 @@ struct Shift_Information{
     i32 start, end, amount;
 };
 
+// TODO(allen): I want this code audited soon
 internal
 Job_Callback_Sig(job_full_lex){
     Editing_File *file = (Editing_File*)data[0];
@@ -1129,7 +1130,6 @@ Job_Callback_Sig(job_full_lex){
     i32 buffer_size = file->state.buffer.size;
     buffer_size = (buffer_size + 3)&(~3);
     
-#if USE_NEW_LEXER
     while (memory->size < buffer_size*2){
         system->grow_thread_memory(memory);
     }
@@ -1174,48 +1174,6 @@ Job_Callback_Sig(job_full_lex){
         }
     } while (still_lexing);
     
-#else
-    
-    while (memory->size < buffer_size){
-        system->grow_thread_memory(memory);
-    }
-    
-    Cpp_Token_Stack tokens;
-    tokens.tokens = (Cpp_Token*)(char*)memory->data;
-    tokens.max_count = (memory->size) / sizeof(Cpp_Token);
-    tokens.count = 0;
-    
-    Cpp_Lex_Data status = {};
-    
-    do{
-        for (i32 r = 2048; r > 0 && status.pos < text_size; --r){
-            Cpp_Lex_Data prev_lex = status;
-            Cpp_Read_Result step_result = cpp_lex_step(text_data, text_size, &status);
-            
-            if (step_result.has_result){
-                if (!cpp_push_token_nonalloc(&tokens, step_result.token)){
-                    status = prev_lex;
-                    system->grow_thread_memory(memory);
-                    tokens.tokens = (Cpp_Token*)memory->data;
-                    tokens.max_count = memory->size / sizeof(Cpp_Token);
-                }
-            }
-        }
-        
-        if (status.pos >= text_size){
-            status.complete = 1;
-        }
-        else{
-            if (system->check_cancel(thread)){
-                return;
-            }
-        }
-    } while(!status.complete);
-    
-#endif
-    
-    
-    
     i32 new_max = LargeRoundUp(tokens.count+1, Kbytes(1));
     
     system->acquire_lock(FRAME_LOCK);
@@ -1322,12 +1280,8 @@ file_relex_parallel(System_Functions *system,
         relex_space.max_count = state.space_request;
         relex_space.tokens = push_array(part, Cpp_Token, relex_space.max_count);
         
-#if USE_NEW_LEXER
         char *spare = push_array(part, char, size+1);
         if (cpp_relex_nonalloc_main(&state, &relex_space, &relex_end, spare)){
-#else
-        if (cpp_relex_nonalloc_main(&state, &relex_space, &relex_end)){
-#endif
             inline_lex = 0;
         }
         else{
@@ -3318,7 +3272,9 @@ kill_file(System_Functions *system, Models *models, Editing_File *file){
     
     if (file && !file->settings.never_kill){
         buffer_unbind_name(working_set, file);
+        if (file->canon.name.size != 0){
             buffer_unbind_file(system, working_set, file);
+        }
         file_close(system, &models->mem.general, file);
         working_set_free_file(working_set, file);
         

test/4cpp_new_lexer.h

@@ -447,6 +447,7 @@ cpp_lex_nonalloc(Lex_Data *S_ptr,
         if (S.fsm.state >= LS_count) S.fsm.state -= LS_count;
         pos_update_rule = PUR_none;
         if (S.pp_state == LSPP_include){
+            if (c == 0) S.fsm.emit_token = 0;
             switch (S.fsm.state){
                 case LSINC_default:break;