Rust parsing fixed

4coder_default_hooks.cpp

@@ -116,6 +116,7 @@ OPEN_FILE_HOOK_SIG(default_file_settings){
     bool32 treat_as_code = false;
     bool32 treat_as_todo = false;
     bool32 wrap_lines = true;
+    bool32 lex_without_strings = false;
     
     int32_t extension_count = 0;
     char **extension_list = get_current_code_extensions(&extension_count);
@@ -148,6 +149,7 @@ OPEN_FILE_HOOK_SIG(default_file_settings){
                         init_language_rust(app);
                     }
                     parse_context_id = parse_context_language_rust;
+                    lex_without_strings = true;
                 }
                 
                 if (match(ext, "cpp") || match(ext, "h") || match(ext, "c") || match(ext, "hpp") || match(ext, "cc")){
@@ -211,6 +213,9 @@ OPEN_FILE_HOOK_SIG(default_file_settings){
         // Unfortunantely without tokens virtual whitespace doesn't really make sense.
         // So for now I have it automatically turning on lexing when virtual whitespace is turned on.
         // Cleaning some of that up is a goal for future versions.
+        if (lex_without_strings){
+            buffer_set_setting(app, &buffer, BufferSetting_LexWithoutStrings, true);
+        }
         buffer_set_setting(app, &buffer, BufferSetting_WrapLine, true);
         buffer_set_setting(app, &buffer, BufferSetting_VirtualWhitespace, true);
     }

4coder_lib/4cpp_lexer.h

@@ -460,36 +460,44 @@ cpp__pp_directive_to_state(Cpp_Token_Type type){
     Cpp_Lex_PP_State result = LSPP_default;
     switch (type){
         case CPP_PP_INCLUDE: case CPP_PP_IMPORT: case CPP_PP_USING:
-        result = LSPP_include;
+        {
-        break;
+            result = LSPP_include;
+        }break;
         
         case CPP_PP_DEFINE:
-        result = LSPP_macro_identifier;
+        {
-        break;
+            result = LSPP_macro_identifier;
+        }break;
         
         case CPP_PP_UNDEF: case CPP_PP_IFDEF: case CPP_PP_IFNDEF:
-        result = LSPP_identifier;
+        {
-        break;
+            result = LSPP_identifier;
+        }break;
         
         case CPP_PP_IF: case CPP_PP_ELIF:
-        result = LSPP_body_if;
+        {
-        break;
+            result = LSPP_body_if;
+        }break;
         
         case CPP_PP_PRAGMA:
-        result = LSPP_body;
+        {
-        break;
+            result = LSPP_body;
+        }break;
         
         case CPP_PP_VERSION: case CPP_PP_LINE:
-        result = LSPP_number;
+        {
-        break;
+            result = LSPP_number;
+        }break;
         
         case CPP_PP_ERROR:
-        result = LSPP_error;
+        {
-        break;
+            result = LSPP_error;
+        }break;
         
         case CPP_PP_UNKNOWN: case CPP_PP_ELSE: case CPP_PP_ENDIF:
-        result = LSPP_junk;
+        {
-        break;
+            result = LSPP_junk;
+        }break;
     }
     return(result);
 }

4coder_lib/4cpp_lexer_tables.c

@@ -660,10 +660,10 @@ u16_4tech no_string_fsm_eq_classes[] = {
  31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31,
 124,155,186,217,248,155,279,186,310,310,341,372,310,403,434,465,
 496,527,527,527,527,527,527,527,527,527,558,310,589,155,620,310,
-310,651,651,651,651,651,651,186,186,186,186,186,682,186,186,186,
+310,651,651,651,651,651,651,186,186,186,186,186,186,186,186,186,
-186,186,713,186,186,682,186,186,186,186,186,310,744,310,155,248,
+186,186,186,186,186,186,186,186,186,186,186,310,682,310,155,248,
- 31,651,651,651,651,775,651,186,186,186,186,186,186,186,186,186,
+ 31,651,651,651,651,713,651,186,186,186,186,186,186,186,186,186,
-186,186,186,186,186,806,186,186,837,186,186,310,868,310,310, 31,
+186,186,186,186,186,186,186,186,744,186,186,310,775,310,310, 31,
 651,651,651,651,651,651,651,651,651,651,651,651,651,651,651,651,
 651,651,651,651,651,651,651,651,651,651,651,651,651,651,651,651,
 651,651,651,651,651,651,651,651,651,651,651,651,651,651,651,651,
@@ -674,7 +674,7 @@ u16_4tech no_string_fsm_eq_classes[] = {
 651,651,651,651,651,651,651,651,651,651,651,651,651,651,651,651,
 };
 
-const i32_4tech num_no_string_fsm_eq_classes = 29;
+const i32_4tech num_no_string_fsm_eq_classes = 26;
 
 u8_4tech no_string_fsm_table[] = {
  31, 32, 33, 34, 35, 32, 32, 32, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61,
@@ -699,11 +699,8 @@ u8_4tech no_string_fsm_table[] = {
  21, 32, 33, 34, 35, 62, 62, 62, 39, 40, 41, 42, 43, 44, 45, 15, 15, 17, 17, 50, 51, 29, 53, 54, 55, 56, 57, 58, 59, 60, 30,
  22, 32, 33, 34, 35, 62, 62, 62, 39, 40, 41, 42, 43, 44, 45, 15, 15, 17, 17, 50, 51, 52, 29, 24, 55, 56, 57, 58, 59, 60, 30,
   1,  1, 33,  4,  4, 62, 62, 62, 39, 40, 41, 42, 43, 13, 45, 15, 15, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30,
-  6,  1, 33,  4,  4, 62, 62, 62, 39, 40, 41, 42, 43, 44, 45, 15, 15, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30,
-  5,  1, 33,  4,  4, 62, 62, 62, 39, 40, 41, 42, 43, 44, 45, 15, 15, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30,
  31, 32, 33, 34, 35, 62, 62, 62, 39, 40, 41, 42, 43, 44, 45, 16, 15, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30,
   1,  1, 33,  4,  4, 62, 62, 62, 39, 40, 11, 42, 43, 13, 45, 15, 15, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30,
-  7,  1, 33,  4,  4, 62, 62, 62, 39, 40, 41, 42, 43, 44, 45, 15, 15, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30,
   1,  1, 33,  4,  4, 62, 62, 62, 39, 13, 41, 42, 43, 44, 45, 15, 15, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30,
  26, 32, 33, 34, 35, 62, 62, 62, 39, 40, 41, 42, 43, 44, 45, 15, 15, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30,
 };

4coder_lib/4cpp_lexer_types.h

@@ -303,6 +303,7 @@ DOC_SEE(cpp_get_relex_range) */
 STRUCT Cpp_Relex_Range{
     /* DOC(The index of the first token in the unedited array that needs to be relexed.) */
     int32_t start_token_index;
+    
     /* DOC(The index of the first token in the unedited array after the edited range that may not need to be relexed.  Sometimes a relex operation has to lex past this position to find a token that is not effected by the edit.) */
     int32_t end_token_index;
 };
@@ -475,18 +476,18 @@ ENUM_INTERNAL(uint8_t, Cpp_Lex_Str_State){
 #define LSSTR_check_delim LSSTR_count
 
 ENUM_INTERNAL(uint8_t, Cpp_Lex_PP_State){
-    LSPP_default,
+    LSPP_default = 0,
-    LSPP_include,
+    LSPP_include = 1,
-    LSPP_macro_identifier,
+    LSPP_macro_identifier = 2,
-    LSPP_identifier,
+    LSPP_identifier = 3,
-    LSPP_body_if,
+    LSPP_body_if = 4,
-    LSPP_body,
+    LSPP_body = 5,
-    LSPP_number,
+    LSPP_number = 6,
-    LSPP_error,
+    LSPP_error = 7,
-    LSPP_junk,
+    LSPP_junk = 8,
-    LSPP_no_strings,
+    LSPP_no_strings = 9,
     //
-    LSPP_count
+    LSPP_count = 10
 };
 
 #endif

meta/4ed_fsm_table_generator.cpp

@@ -311,13 +311,13 @@ main_fsm(Cpp_Lex_FSM fsm, uint8_t pp_state, uint8_t c, bool32 ignore_string_deli
             default:
             switch (fsm.state){
                 case LS_default:
-                if (c == 'R'){
+                if (!ignore_string_delims && c == 'R'){
                     fsm.state = LS_string_R;
                 }
-                else if (c == 'U' || c == 'L'){
+                else if (!ignore_string_delims && (c == 'U' || c == 'L')){
                     fsm.state = LS_string_LUu8;
                 }
-                else if (c == 'u'){
+                else if (!ignore_string_delims && c == 'u'){
                     fsm.state = LS_string_u;
                 }
                 else if (is_identifier_char_non_numeric(c, ignore_string_delims)){