mr4th-public
/
4coder
1
Fork 0
Code Issues Releases Activity

more lexer improvements

master
Allen Webster 2016-08-30 18:04:57 -04:00
parent c645b9a57b
commit c6b6203e53
1 changed files with 455 additions and 497 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
4cpp_lexer.h
View File

@ -251,13 +251,6 @@ cpp_shift_token_starts(Cpp_Token_Stack *stack, int32_t from_token_i, int32_t shi
} }
} }
#if 0
enum Pos_Update_Rule{
PUR_none,
PUR_back_one,
};
#endif
FCPP_LINK Lex_PP_State FCPP_LINK Lex_PP_State
cpp_pp_directive_to_state(Cpp_Token_Type type){ cpp_pp_directive_to_state(Cpp_Token_Type type){
Lex_PP_State result = LSPP_default; Lex_PP_State result = LSPP_default;
@ -478,31 +471,10 @@ cpp_lex_nonalloc(Lex_Data *S_ptr,
S.completed = 1; S.completed = 1;
} }
if (S.fsm.state >= LS_count) S.fsm.state -= LS_count; if (S.fsm.state >= LS_count){
//pos_update_rule = PUR_none; S.fsm.state -= LS_count;
#if 0
if (S.pp_state == LSPP_include){
if (c == 0) S.fsm.emit_token = 0;
switch (S.fsm.state){
case LSINC_default:break;
case LSINC_quotes:
case LSINC_pointy:
S.token.type = CPP_TOKEN_INCLUDE_FILE;
S.token.flags = 0;
break;
case LSINC_junk:
S.token.type = CPP_TOKEN_JUNK;
S.token.flags = 0;
break;
} }
}
else{}
#endif
{
switch (S.fsm.state){ switch (S.fsm.state){
case LS_default: case LS_default:
switch (c){ switch (c){
@ -941,25 +913,9 @@ cpp_lex_nonalloc(Lex_Data *S_ptr,
break; break;
} }
#if 0
switch (pos_update_rule){
case PUR_back_one:
--S.pos;
break;
default:
if (chunk[S.pos-1] == 0){ if (chunk[S.pos-1] == 0){
--S.pos; --S.pos;
} }
break;
}
#else
if (chunk[S.pos-1] == 0){
--S.pos;
}
#endif
if ((S.token.flags & CPP_TFLAG_PP_DIRECTIVE) == 0){ if ((S.token.flags & CPP_TFLAG_PP_DIRECTIVE) == 0){
switch (S.pp_state){ switch (S.pp_state){
@ -991,11 +947,13 @@ cpp_lex_nonalloc(Lex_Data *S_ptr,
break; break;
case LSPP_junk: case LSPP_junk:
if (S.token.type != CPP_TOKEN_COMMENT){
S.token.type = CPP_TOKEN_JUNK; S.token.type = CPP_TOKEN_JUNK;
}
break; break;
} }
} }
}
if (S.fsm.emit_token){ if (S.fsm.emit_token){
S.token.start = S.token_start; S.token.start = S.token_start;