relex API allows starting from any chunk, fixed null check bug

4cpp_lexer.h

@@ -800,7 +800,7 @@ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, int32_t siz
             break;
         }
         
-        if (chunk[S.pos-1] == 0){
+        if (S.pos > S.chunk_pos && chunk[S.pos-1] == 0){
             --S.pos;
         }
         
@@ -1207,6 +1207,40 @@ cpp_relex_init(Cpp_Token_Array *array, int32_t start_pos, int32_t end_pos, int32
     return(state);
 }
 
+FCPP_LINK int32_t
+cpp_relex_start_position(Cpp_Relex_Data *S_ptr){
+    int32_t result = S_ptr->relex_start_position;
+    return(result);
+}
+
+FCPP_LINK void
+cpp_relex_declare_first_chunk_position(Cpp_Relex_Data *S_ptr, int32_t position){
+    S_ptr->lex.chunk_pos = position;
+}
+
+FCPP_LINK int32_t
+cpp_relex_is_start_chunk(Cpp_Relex_Data *S_ptr, char *chunk, int32_t chunk_size){
+    int32_t pos = S_ptr->relex_start_position;
+    int32_t start = S_ptr->lex.chunk_pos;
+    int32_t end = start + chunk_size;
+    
+    int32_t good_chunk = 0;
+    if (start <= pos && pos < end){
+        good_chunk = 1;
+    }
+    else{
+        if (chunk == 0){
+            good_chunk = 1;
+            S_ptr->lex.chunk_pos = pos;
+        }
+        else{
+            S_ptr->lex.chunk_pos += chunk_size;
+        }
+    }
+    
+    return(good_chunk);
+}
+
 // duff-routine defines
 #define DrCase(PC) case PC: goto resumespot_##PC
 
@@ -1232,7 +1266,6 @@ cpp_relex_step(Cpp_Relex_Data *S_ptr, char *chunk, int32_t chunk_size, int32_t f
     cpp_shift_token_starts(array, S.end_token_index, S.character_shift_amount);
     S.end_token = cpp_index_array(array, full_size, S.end_token_index);
     
-    if (S.relex_start_position < full_size){
     // TODO(allen): This can be better I suspect.
     for (;;){
         Cpp_Lex_Result step_result = 
@@ -1266,7 +1299,7 @@ cpp_relex_step(Cpp_Relex_Data *S_ptr, char *chunk, int32_t chunk_size, int32_t f
             case LexResult_Finished: goto double_break;
         }
     }
-    }
+    
     double_break:;
     
     DrReturn(LexResult_Finished);

4ed_file_view.cpp

@@ -1276,20 +1276,49 @@ file_relex_parallel(System_Functions *system,
         relex_array.tokens = push_array(part, Cpp_Token, relex_array.max_count);
         
         i32 size = file->state.buffer.size;
-        char *spare = push_array(part, char, size);
+        char *spare = push_array(part, char, size+1);
         
         Cpp_Relex_Data state = cpp_relex_init(array, start_i, end_i, shift_amount, spare);
         
         char *chunk = file->state.buffer.data;
-        i32 chunk_size = size;
+        i32 chunk_size = 1024;
+        i32 chunk_index = 0;
+        
+        int32_t start_position = cpp_relex_start_position(&state);
+        
+        if (start_position == size){
+            chunk = 0;
+            chunk_size = 0;
+            cpp_relex_declare_first_chunk_position(&state, size);
+        }
+        else{
+            chunk_index = start_position / chunk_size;
+            
+            int32_t chunk_start_position = chunk_index*1024;
+            if (chunk_start_position + chunk_size > size){
+                chunk_size = size - chunk_start_position;
+            }
+            
+            cpp_relex_declare_first_chunk_position(&state, chunk_start_position);
+            
+            chunk += chunk_start_position;
+        }
+        
         for(;;){
             Cpp_Lex_Result lex_result =
                 cpp_relex_step(&state, chunk, chunk_size, size, array, &relex_array);
             
             switch (lex_result){
                 case LexResult_NeedChunk:
-                Assert(!"There is only one chunk in the current system.");
-                break;
+                {
+                    ++chunk_index;
+                    chunk += chunk_size;
+                    
+                    int32_t chunk_start_position = chunk_index*1024;
+                    if (chunk_start_position + chunk_size > size){
+                        chunk_size = size - chunk_start_position;
+                    }
+                }break;
                 
                 case LexResult_NeedTokenMemory:
                 inline_lex = 0;