mr4th-public
/
4coder
1
Fork 0
Code Issues Releases Activity

Rust parsing fixed

master
Allen Webster 2017-11-27 16:13:17 -05:00
parent b0ec2a9422
commit efc595cbaa
5 changed files with 49 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
4coder_default_hooks.cpp
View File

@ -116,6 +116,7 @@ OPEN_FILE_HOOK_SIG(default_file_settings){
bool32 treat_as_code = false;
bool32 treat_as_todo = false;
bool32 wrap_lines = true;
bool32 lex_without_strings = false;
int32_t extension_count = 0;
char **extension_list = get_current_code_extensions(&extension_count);
@ -148,6 +149,7 @@ OPEN_FILE_HOOK_SIG(default_file_settings){
init_language_rust(app);
}
parse_context_id = parse_context_language_rust;
lex_without_strings = true;
}
if (match(ext, "cpp") || match(ext, "h") || match(ext, "c") || match(ext, "hpp") || match(ext, "cc")){
@ -211,6 +213,9 @@ OPEN_FILE_HOOK_SIG(default_file_settings){
// Unfortunantely without tokens virtual whitespace doesn't really make sense.
// So for now I have it automatically turning on lexing when virtual whitespace is turned on.
// Cleaning some of that up is a goal for future versions.
if (lex_without_strings){
buffer_set_setting(app, &buffer, BufferSetting_LexWithoutStrings, true);
}
buffer_set_setting(app, &buffer, BufferSetting_WrapLine, true);
buffer_set_setting(app, &buffer, BufferSetting_VirtualWhitespace, true);
}

40
4coder_lib/4cpp_lexer.h
View File

@ -460,36 +460,44 @@ cpp__pp_directive_to_state(Cpp_Token_Type type){
Cpp_Lex_PP_State result = LSPP_default;
switch (type){
case CPP_PP_INCLUDE: case CPP_PP_IMPORT: case CPP_PP_USING:
result = LSPP_include;
break;
{
result = LSPP_include;
}break;
case CPP_PP_DEFINE:
result = LSPP_macro_identifier;
break;
{
result = LSPP_macro_identifier;
}break;
case CPP_PP_UNDEF: case CPP_PP_IFDEF: case CPP_PP_IFNDEF:
result = LSPP_identifier;
break;
{
result = LSPP_identifier;
}break;
case CPP_PP_IF: case CPP_PP_ELIF:
result = LSPP_body_if;
break;
{
result = LSPP_body_if;
}break;
case CPP_PP_PRAGMA:
result = LSPP_body;
break;
{
result = LSPP_body;
}break;
case CPP_PP_VERSION: case CPP_PP_LINE:
result = LSPP_number;
break;
{
result = LSPP_number;
}break;
case CPP_PP_ERROR:
result = LSPP_error;
break;
{
result = LSPP_error;
}break;
case CPP_PP_UNKNOWN: case CPP_PP_ELSE: case CPP_PP_ENDIF:
result = LSPP_junk;
break;
{
result = LSPP_junk;
}break;
}
return(result);
}

13
4coder_lib/4cpp_lexer_tables.c
View File

@ -660,10 +660,10 @@ u16_4tech no_string_fsm_eq_classes[] = {
31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31,
124,155,186,217,248,155,279,186,310,310,341,372,310,403,434,465,
496,527,527,527,527,527,527,527,527,527,558,310,589,155,620,310,
310,651,651,651,651,651,651,186,186,186,186,186,682,186,186,186,
186,186,713,186,186,682,186,186,186,186,186,310,744,310,155,248,
31,651,651,651,651,775,651,186,186,186,186,186,186,186,186,186,
186,186,186,186,186,806,186,186,837,186,186,310,868,310,310, 31,
310,651,651,651,651,651,651,186,186,186,186,186,186,186,186,186,
186,186,186,186,186,186,186,186,186,186,186,310,682,310,155,248,
31,651,651,651,651,713,651,186,186,186,186,186,186,186,186,186,
186,186,186,186,186,186,186,186,744,186,186,310,775,310,310, 31,
651,651,651,651,651,651,651,651,651,651,651,651,651,651,651,651,
651,651,651,651,651,651,651,651,651,651,651,651,651,651,651,651,
651,651,651,651,651,651,651,651,651,651,651,651,651,651,651,651,
@ -674,7 +674,7 @@ u16_4tech no_string_fsm_eq_classes[] = {
651,651,651,651,651,651,651,651,651,651,651,651,651,651,651,651,
};
const i32_4tech num_no_string_fsm_eq_classes = 29;
const i32_4tech num_no_string_fsm_eq_classes = 26;
u8_4tech no_string_fsm_table[] = {
31, 32, 33, 34, 35, 32, 32, 32, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61,
@ -699,11 +699,8 @@ u8_4tech no_string_fsm_table[] = {
21, 32, 33, 34, 35, 62, 62, 62, 39, 40, 41, 42, 43, 44, 45, 15, 15, 17, 17, 50, 51, 29, 53, 54, 55, 56, 57, 58, 59, 60, 30,
22, 32, 33, 34, 35, 62, 62, 62, 39, 40, 41, 42, 43, 44, 45, 15, 15, 17, 17, 50, 51, 52, 29, 24, 55, 56, 57, 58, 59, 60, 30,
1, 1, 33, 4, 4, 62, 62, 62, 39, 40, 41, 42, 43, 13, 45, 15, 15, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30,
6, 1, 33, 4, 4, 62, 62, 62, 39, 40, 41, 42, 43, 44, 45, 15, 15, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30,
5, 1, 33, 4, 4, 62, 62, 62, 39, 40, 41, 42, 43, 44, 45, 15, 15, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30,
31, 32, 33, 34, 35, 62, 62, 62, 39, 40, 41, 42, 43, 44, 45, 16, 15, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30,
1, 1, 33, 4, 4, 62, 62, 62, 39, 40, 11, 42, 43, 13, 45, 15, 15, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30,
7, 1, 33, 4, 4, 62, 62, 62, 39, 40, 41, 42, 43, 44, 45, 15, 15, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30,
1, 1, 33, 4, 4, 62, 62, 62, 39, 13, 41, 42, 43, 44, 45, 15, 15, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30,
26, 32, 33, 34, 35, 62, 62, 62, 39, 40, 41, 42, 43, 44, 45, 15, 15, 17, 17, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 30,
};

23
4coder_lib/4cpp_lexer_types.h
View File

@ -303,6 +303,7 @@ DOC_SEE(cpp_get_relex_range) */
STRUCT Cpp_Relex_Range{
/* DOC(The index of the first token in the unedited array that needs to be relexed.) */
int32_t start_token_index;
/* DOC(The index of the first token in the unedited array after the edited range that may not need to be relexed. Sometimes a relex operation has to lex past this position to find a token that is not effected by the edit.) */
int32_t end_token_index;
};
@ -475,18 +476,18 @@ ENUM_INTERNAL(uint8_t, Cpp_Lex_Str_State){
#define LSSTR_check_delim LSSTR_count
ENUM_INTERNAL(uint8_t, Cpp_Lex_PP_State){
LSPP_default,
LSPP_include,
LSPP_macro_identifier,
LSPP_identifier,
LSPP_body_if,
LSPP_body,
LSPP_number,
LSPP_error,
LSPP_junk,
LSPP_no_strings,
LSPP_default = 0,
LSPP_include = 1,
LSPP_macro_identifier = 2,
LSPP_identifier = 3,
LSPP_body_if = 4,
LSPP_body = 5,
LSPP_number = 6,
LSPP_error = 7,
LSPP_junk = 8,
LSPP_no_strings = 9,
//
LSPP_count
LSPP_count = 10
};
#endif

6
meta/4ed_fsm_table_generator.cpp
View File

@ -311,13 +311,13 @@ main_fsm(Cpp_Lex_FSM fsm, uint8_t pp_state, uint8_t c, bool32 ignore_string_deli
default:
switch (fsm.state){
case LS_default:
if (c == 'R'){
if (!ignore_string_delims && c == 'R'){
fsm.state = LS_string_R;
}
else if (c == 'U' || c == 'L'){
else if (!ignore_string_delims && (c == 'U' || c == 'L')){
fsm.state = LS_string_LUu8;
}
else if (c == 'u'){
else if (!ignore_string_delims && c == 'u'){
fsm.state = LS_string_u;
}
else if (is_identifier_char_non_numeric(c, ignore_string_delims)){