mr4th-public
/
4coder
1
Fork 0
Code Issues Releases Activity

new handling for strings in lexer

master
Allen Webster 2017-04-17 19:34:54 -04:00
parent bc4c866cf0
commit 9d1756ad2b
5 changed files with 1666 additions and 992 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

358
4cpp/4cpp_lexer.h
View File

@ -340,6 +340,8 @@ cpp__table_match(String_And_Flag *table, i32_4tech count, char *s, i32_4tech len
return(result); return(result);
} }
#define LEXER_TB(n) ((n) & (sizeof(S.tb)-1))
FCPP_LINK Cpp_Lex_Result FCPP_LINK Cpp_Lex_Result
cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, i32_4tech size, Cpp_Token_Array *token_array_out){ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, i32_4tech size, Cpp_Token_Array *token_array_out){
Cpp_Lex_Data S = *S_ptr; Cpp_Lex_Data S = *S_ptr;
@ -359,7 +361,11 @@ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, i32_4tech s
DfrCase(3); DfrCase(3);
DfrCase(4); DfrCase(4);
DfrCase(5); DfrCase(5);
DfrCase(6);
DfrCase(7); DfrCase(7);
DfrCase(8);
DfrCase(9);
DfrCase(10);
} }
for (;;){ for (;;){
@ -401,11 +407,10 @@ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, i32_4tech s
for (; S.fsm.state < LS_count && S.pos < end_pos;){ for (; S.fsm.state < LS_count && S.pos < end_pos;){
c = chunk[S.pos++]; c = chunk[S.pos++];
S.tb[(S.tb_pos++) & (sizeof(S.tb)-1)] = c; S.tb[LEXER_TB(S.tb_pos++)] = c;
i32_4tech i = S.fsm.state + eq_classes[c]; i32_4tech i = S.fsm.state + eq_classes[c];
S.fsm.state = fsm_table[i]; S.fsm.state = fsm_table[i];
S.fsm.multi_line |= multiline_state_table[S.fsm.state];
} }
S.fsm.emit_token = (S.fsm.state >= LS_count); S.fsm.emit_token = (S.fsm.state >= LS_count);
} }
@ -432,6 +437,7 @@ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, i32_4tech s
switch (S.fsm.state){ switch (S.fsm.state){
case LS_default: case LS_default:
{
switch (c){ switch (c){
case 0: S.fsm.emit_token = 0; break; case 0: S.fsm.emit_token = 0; break;
@ -459,21 +465,23 @@ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, i32_4tech s
} }
else{ else{
S.pos_overide = S.pos; S.pos_overide = S.pos;
S.white_done = 0; S.white_done = false;
for (;;){ for (;;){
for (; S.white_done == 0 && S.pos < end_pos;){ for (; !S.white_done && S.pos < end_pos;){
c = chunk[S.pos++]; c = chunk[S.pos++];
if (!(c == ' ' || c == '\t' || c == '\r' || c == '\v' || c == '\f')){ if (!(c == ' ' || c == '\t' || c == '\r' || c == '\v' || c == '\f')){
S.white_done = 1; S.white_done = true;
} }
} }
if (S.white_done == 0){ if (!S.white_done){
S.chunk_pos += size; S.chunk_pos += size;
token_array_out->count = token_i; token_array_out->count = token_i;
DfrYield(1, LexResult_NeedChunk); DfrYield(1, LexResult_NeedChunk);
} }
else break; else{
break;
}
} }
if (c == '\n'){ if (c == '\n'){
@ -490,7 +498,7 @@ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, i32_4tech s
if (c != '@' && c != '\\'){ if (c != '@' && c != '\\'){
S.token.flags = CPP_TFLAG_IS_OPERATOR; S.token.flags = CPP_TFLAG_IS_OPERATOR;
} }
break; }break;
case LS_identifier: case LS_identifier:
{ {
@ -523,6 +531,7 @@ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, i32_4tech s
}break; }break;
case LS_pound: case LS_pound:
{
S.token.flags = 0; S.token.flags = 0;
switch (c){ switch (c){
case '#': S.token.type = CPP_PP_CONCAT; break; case '#': S.token.type = CPP_PP_CONCAT; break;
@ -531,7 +540,7 @@ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, i32_4tech s
--S.pos; --S.pos;
break; break;
} }
break; }break;
case LS_pp: case LS_pp:
{ {
@ -572,85 +581,221 @@ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, i32_4tech s
case LS_number: case LS_number:
case LS_number0: case LS_number0:
case LS_hex: case LS_hex:
S.fsm.int_state = LSINT_default; {
S.fsm.state = LSINT_default;
S.fsm.emit_token = 0; S.fsm.emit_token = 0;
--S.pos; --S.pos;
for (;;){ for (;;){
for (; S.fsm.int_state < LSINT_count && S.pos < end_pos;){ for (; S.fsm.state < LSINT_count && S.pos < end_pos;){
c = chunk[S.pos++]; c = chunk[S.pos++];
S.fsm.int_state = int_fsm_table[S.fsm.int_state + int_fsm_eq_classes[c]]; S.fsm.state = int_fsm_table[S.fsm.state + int_fsm_eq_classes[c]];
} }
S.fsm.emit_token = (S.fsm.int_state >= LSINT_count); S.fsm.emit_token = (S.fsm.state >= LSINT_count);
if (S.fsm.emit_token == 0){ if (S.fsm.emit_token == 0){
S.chunk_pos += size; S.chunk_pos += size;
token_array_out->count = token_i; token_array_out->count = token_i;
DfrYield(5, LexResult_NeedChunk); DfrYield(5, LexResult_NeedChunk);
} }
else break; else{
break;
}
} }
--S.pos; --S.pos;
S.token.type = CPP_TOKEN_INTEGER_CONSTANT; S.token.type = CPP_TOKEN_INTEGER_CONSTANT;
S.token.flags = 0; S.token.flags = 0;
break; }break;
case LS_float: case LS_float:
case LS_crazy_float0: case LS_crazy_float0:
case LS_crazy_float1: case LS_crazy_float1:
{
S.token.type = CPP_TOKEN_FLOATING_CONSTANT; S.token.type = CPP_TOKEN_FLOATING_CONSTANT;
S.token.flags = 0; S.token.flags = 0;
switch (c){ switch (c){
case 'f': case 'F': case 'f': case 'F': case 'l': case 'L': break;
case 'l': case 'L':break; default: --S.pos; break;
default:
--S.pos;
break;
} }
break; }break;
case LS_char: case LS_string_raw:
case LS_char_slashed: {
S.token.type = CPP_TOKEN_JUNK; S.tb_pos = 0;
if (c == '\''){ S.delim_length = 0;
S.token.type = CPP_TOKEN_CHARACTER_CONSTANT;
}
S.token.flags = 0;
break;
case LS_char_multiline: S.token.type = CPP_TOKEN_STRING_CONSTANT;
S.token.type = CPP_TOKEN_JUNK;
if (c == '\''){
S.token.type = CPP_TOKEN_CHARACTER_CONSTANT;
}
S.token.flags = CPP_TFLAG_MULTILINE;
break;
case LS_string: S.fsm.state = LSSTR_default;
case LS_string_slashed: S.fsm.flags = 0;
S.token.type = CPP_TOKEN_JUNK; for (;;){
if (S.pp_state == LSPP_include){ for (; S.fsm.state < LSSTR_count && S.pos < end_pos;){
if (c == '>' || c == '"'){ c = chunk[S.pos++];
S.token.type = CPP_PP_INCLUDE_FILE; S.tb[LEXER_TB(S.tb_pos++)] = c;
S.fsm.state = raw_str_table[S.fsm.state + raw_str_eq_classes[c]];
S.fsm.flags |= raw_str_flags[S.fsm.state];
} }
S.fsm.emit_token = (S.fsm.state >= LSSTR_count);
if (S.fsm.emit_token == 0){
S.chunk_pos += size;
token_array_out->count = token_i;
DfrYield(7, LexResult_NeedChunk);
} }
else{ else{
if (c == '"'){ u8_4tech emit_state = S.fsm.state - LSSTR_count;
S.token.type = CPP_TOKEN_STRING_CONSTANT; switch (emit_state){
case LSSTR_default:
{
S.token.type = CPP_TOKEN_JUNK;
goto doublebreak;
}break;
case LSSTR_get_delim:
{
if (S.tb_pos <= 17){
S.delim_length = S.tb_pos-1;
for (i32_4tech n = 0; n < S.delim_length; ++n){
S.raw_delim[n] = S.tb[n];
} }
S.tb_pos = 0;
}
else{
S.token.type = CPP_TOKEN_JUNK;
--S.pos;
goto doublebreak;
}
}break;
case LSSTR_check_delim:
{
if (c == 0){
goto doublebreak;
}
else if (S.tb_pos >= S.delim_length){
u32_4tech m = S.tb_pos - S.delim_length - 2;
if (S.tb[LEXER_TB(m)] == ')'){
b32_4tech is_match = true;
++m;
for (i32_4tech n = 0; n < S.delim_length; ++n, ++m){
if (S.tb[LEXER_TB(m)] != S.raw_delim[n]){
is_match = false;
break;
}
}
if (is_match){
goto doublebreak;
}
}
}
}break;
}
S.fsm.state = LSSTR_get_delim;
}
}
doublebreak:;
S.token.flags = (S.fsm.flags)?(CPP_TFLAG_MULTILINE):(0);
}break;
case LS_string_normal:
{
S.fsm.state = LSSTR_default;
S.fsm.flags = 0;
for (;;){
for (; S.fsm.state < LSSTR_count && S.pos < end_pos;){
c = chunk[S.pos++];
S.fsm.state = normal_str_table[S.fsm.state + normal_str_eq_classes[c]];
S.fsm.flags |= normal_str_flags[S.fsm.state];
}
S.fsm.emit_token = (S.fsm.state >= LSSTR_count);
if (S.fsm.emit_token == 0){
S.chunk_pos += size;
token_array_out->count = token_i;
DfrYield(8, LexResult_NeedChunk);
}
else{
break;
}
}
S.token.type = CPP_TOKEN_STRING_CONSTANT;
S.token.flags = (S.fsm.flags)?(CPP_TFLAG_MULTILINE):(0);
if (c == '\n'){
--S.pos;
}
}break;
case LS_string_include:
{
S.fsm.state = LSSTR_default;
for (;;){
for (; S.fsm.state < LSSTR_include_count && S.pos < end_pos;){
c = chunk[S.pos++];
S.fsm.state = include_str_table[S.fsm.state + include_str_eq_classes[c]];
}
S.fsm.emit_token = (S.fsm.state >= LSSTR_include_count);
if (S.fsm.emit_token == 0){
S.chunk_pos += size;
token_array_out->count = token_i;
DfrYield(6, LexResult_NeedChunk);
}
else{
break;
}
}
S.fsm.state -= LSSTR_include_count;
if (S.fsm.state == LSSTR_default){
S.token.type = CPP_PP_INCLUDE_FILE;
}
else{
S.token.type = CPP_TOKEN_JUNK;
} }
S.token.flags = 0; S.token.flags = 0;
break;
case LS_string_multiline: if (c == '\n'){
S.token.type = CPP_TOKEN_JUNK; --S.pos;
if (c == '"'){
S.token.type = CPP_TOKEN_STRING_CONSTANT;
} }
S.token.flags = CPP_TFLAG_MULTILINE; }break;
case LS_char:
{
S.fsm.state = LSSTR_default;
S.fsm.flags = 0;
for (;;){
for (; S.fsm.state < LSSTR_count && S.pos < end_pos;){
c = chunk[S.pos++];
S.fsm.state = normal_char_table[S.fsm.state + normal_char_eq_classes[c]];
S.fsm.flags |= normal_char_flags[S.fsm.state];
}
S.fsm.emit_token = (S.fsm.state >= LSSTR_count);
if (S.fsm.emit_token == 0){
S.chunk_pos += size;
token_array_out->count = token_i;
DfrYield(9, LexResult_NeedChunk);
}
else{
break; break;
}
}
S.token.type = CPP_TOKEN_CHARACTER_CONSTANT;
S.token.flags = (S.fsm.flags)?(CPP_TFLAG_MULTILINE):(0);
if (c == '\n'){
--S.pos;
}
}break;
case LS_comment_pre: case LS_comment_pre:
{
S.token.flags = CPP_TFLAG_IS_OPERATOR; S.token.flags = CPP_TFLAG_IS_OPERATOR;
switch (c){ switch (c){
case '=': S.token.type = CPP_TOKEN_DIVEQ; break; case '=': S.token.type = CPP_TOKEN_DIVEQ; break;
@ -659,28 +804,32 @@ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, i32_4tech s
--S.pos; --S.pos;
break; break;
} }
break; }break;
case LS_comment: case LS_comment:
case LS_comment_slashed: case LS_comment_slashed:
{
S.token.type = CPP_TOKEN_COMMENT; S.token.type = CPP_TOKEN_COMMENT;
S.token.flags = 0; S.token.flags = 0;
--S.pos; --S.pos;
break; }break;
case LS_comment_block: case LS_comment_block:
case LS_comment_block_ending: case LS_comment_block_ending:
{
S.token.type = CPP_TOKEN_COMMENT; S.token.type = CPP_TOKEN_COMMENT;
S.token.flags = 0; S.token.flags = 0;
break; }break;
case LS_error_message: case LS_error_message:
{
S.token.type = CPP_PP_ERROR_MESSAGE; S.token.type = CPP_PP_ERROR_MESSAGE;
S.token.flags = 0; S.token.flags = 0;
--S.pos; --S.pos;
break; }break;
case LS_dot: case LS_dot:
{
S.token.flags = CPP_TFLAG_IS_OPERATOR; S.token.flags = CPP_TFLAG_IS_OPERATOR;
switch (c){ switch (c){
case '*': S.token.type = CPP_TOKEN_PTRDOT; break; case '*': S.token.type = CPP_TOKEN_PTRDOT; break;
@ -689,9 +838,10 @@ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, i32_4tech s
--S.pos; --S.pos;
break; break;
} }
break; }break;
case LS_ellipsis: case LS_ellipsis:
{
switch (c){ switch (c){
case '.': case '.':
S.token.flags = CPP_TFLAG_IS_OPERATOR; S.token.flags = CPP_TFLAG_IS_OPERATOR;
@ -703,9 +853,10 @@ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, i32_4tech s
--S.pos; --S.pos;
break; break;
} }
break; }break;
case LS_less: case LS_less:
{
S.token.flags = CPP_TFLAG_IS_OPERATOR; S.token.flags = CPP_TFLAG_IS_OPERATOR;
switch (c){ switch (c){
case '=': S.token.type = CPP_TOKEN_LESSEQ; break; case '=': S.token.type = CPP_TOKEN_LESSEQ; break;
@ -714,9 +865,10 @@ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, i32_4tech s
--S.pos; --S.pos;
break; break;
} }
break; }break;
case LS_less_less: case LS_less_less:
{
S.token.flags = CPP_TFLAG_IS_OPERATOR; S.token.flags = CPP_TFLAG_IS_OPERATOR;
switch (c){ switch (c){
case '=': S.token.type = CPP_TOKEN_LSHIFTEQ; break; case '=': S.token.type = CPP_TOKEN_LSHIFTEQ; break;
@ -725,9 +877,10 @@ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, i32_4tech s
--S.pos; --S.pos;
break; break;
} }
break; }break;
case LS_more: case LS_more:
{
S.token.flags = CPP_TFLAG_IS_OPERATOR; S.token.flags = CPP_TFLAG_IS_OPERATOR;
switch (c){ switch (c){
case '=': S.token.type = CPP_TOKEN_GRTREQ; break; case '=': S.token.type = CPP_TOKEN_GRTREQ; break;
@ -736,9 +889,10 @@ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, i32_4tech s
--S.pos; --S.pos;
break; break;
} }
break; }break;
case LS_more_more: case LS_more_more:
{
S.token.flags = CPP_TFLAG_IS_OPERATOR; S.token.flags = CPP_TFLAG_IS_OPERATOR;
switch (c){ switch (c){
case '=': S.token.type = CPP_TOKEN_RSHIFTEQ; break; case '=': S.token.type = CPP_TOKEN_RSHIFTEQ; break;
@ -747,9 +901,10 @@ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, i32_4tech s
--S.pos; --S.pos;
break; break;
} }
break; }break;
case LS_minus: case LS_minus:
{
S.token.flags = CPP_TFLAG_IS_OPERATOR; S.token.flags = CPP_TFLAG_IS_OPERATOR;
switch (c){ switch (c){
case '-': S.token.type = CPP_TOKEN_DECREMENT; break; case '-': S.token.type = CPP_TOKEN_DECREMENT; break;
@ -759,9 +914,10 @@ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, i32_4tech s
--S.pos; --S.pos;
break; break;
} }
break; }break;
case LS_arrow: case LS_arrow:
{
S.token.flags = CPP_TFLAG_IS_OPERATOR; S.token.flags = CPP_TFLAG_IS_OPERATOR;
switch (c){ switch (c){
case '*': S.token.type = CPP_TOKEN_PTRARROW; break; case '*': S.token.type = CPP_TOKEN_PTRARROW; break;
@ -770,9 +926,10 @@ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, i32_4tech s
--S.pos; --S.pos;
break; break;
} }
break; }break;
case LS_and: case LS_and:
{
S.token.flags = CPP_TFLAG_IS_OPERATOR; S.token.flags = CPP_TFLAG_IS_OPERATOR;
switch (c){ switch (c){
case '&': S.token.type = CPP_TOKEN_AND; break; case '&': S.token.type = CPP_TOKEN_AND; break;
@ -782,9 +939,10 @@ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, i32_4tech s
--S.pos; --S.pos;
break; break;
} }
break; }break;
case LS_or: case LS_or:
{
S.token.flags = CPP_TFLAG_IS_OPERATOR; S.token.flags = CPP_TFLAG_IS_OPERATOR;
switch (c){ switch (c){
case '|': S.token.type = CPP_TOKEN_OR; break; case '|': S.token.type = CPP_TOKEN_OR; break;
@ -794,9 +952,10 @@ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, i32_4tech s
--S.pos; --S.pos;
break; break;
} }
break; }break;
case LS_plus: case LS_plus:
{
S.token.flags = CPP_TFLAG_IS_OPERATOR; S.token.flags = CPP_TFLAG_IS_OPERATOR;
switch (c){ switch (c){
case '+': S.token.type = CPP_TOKEN_INCREMENT; break; case '+': S.token.type = CPP_TOKEN_INCREMENT; break;
@ -806,9 +965,10 @@ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, i32_4tech s
--S.pos; --S.pos;
break; break;
} }
break; }break;
case LS_colon: case LS_colon:
{
S.token.flags = CPP_TFLAG_IS_OPERATOR; S.token.flags = CPP_TFLAG_IS_OPERATOR;
switch (c){ switch (c){
case ':': S.token.type = CPP_TOKEN_SCOPE; break; case ':': S.token.type = CPP_TOKEN_SCOPE; break;
@ -817,62 +977,27 @@ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, i32_4tech s
--S.pos; --S.pos;
break; break;
} }
break; }break;
case LS_star: case LS_single_op:
{
u32_4tech plain_version = 0;
u32_4tech eq_version = 0;
S.token.flags = CPP_TFLAG_IS_OPERATOR; S.token.flags = CPP_TFLAG_IS_OPERATOR;
switch (c){ switch (S.tb[0]){
case '=': S.token.type = CPP_TOKEN_MULEQ; break; case '*': plain_version = CPP_TOKEN_STAR; eq_version = CPP_TOKEN_MULEQ; break;
default: case '%': plain_version = CPP_TOKEN_MOD; eq_version = CPP_TOKEN_MODEQ; break;
S.token.type = CPP_TOKEN_STAR; case '^': plain_version = CPP_TOKEN_BIT_XOR; eq_version = CPP_TOKEN_XOREQ; break;
--S.pos; case '=': plain_version = CPP_TOKEN_EQ; eq_version = CPP_TOKEN_EQEQ; break;
break; case '!': plain_version = CPP_TOKEN_NOT; eq_version = CPP_TOKEN_NOTEQ; break;
} }
break;
case LS_modulo: S.token.type = eq_version;
S.token.flags = CPP_TFLAG_IS_OPERATOR; if (c != '='){
switch (c){ S.token.type = plain_version;
case '=': S.token.type = CPP_TOKEN_MODEQ; break;
default:
S.token.type = CPP_TOKEN_MOD;
--S.pos; --S.pos;
break;
} }
break; }break;
case LS_caret:
S.token.flags = CPP_TFLAG_IS_OPERATOR;
switch (c){
case '=': S.token.type = CPP_TOKEN_XOREQ; break;
default:
S.token.type = CPP_TOKEN_BIT_XOR;
--S.pos;
break;
}
break;
case LS_eq:
S.token.flags = CPP_TFLAG_IS_OPERATOR;
switch (c){
case '=': S.token.type = CPP_TOKEN_EQEQ; break;
default:
S.token.type = CPP_TOKEN_EQ;
--S.pos;
break;
}
break;
case LS_bang:
S.token.flags = CPP_TFLAG_IS_OPERATOR;
switch (c){
case '=': S.token.type = CPP_TOKEN_NOTEQ; break;
default:
S.token.type = CPP_TOKEN_NOT;
--S.pos;
break;
}
break;
} }
if (S.pos > S.chunk_pos && chunk[S.pos-1] == 0){ if (S.pos > S.chunk_pos && chunk[S.pos-1] == 0){
@ -940,7 +1065,7 @@ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, i32_4tech s
if (S.pos == end_pos){ if (S.pos == end_pos){
S.chunk_pos += size; S.chunk_pos += size;
token_array_out->count = token_i; token_array_out->count = token_i;
DfrYield(7, LexResult_NeedChunk); DfrYield(10, LexResult_NeedChunk);
} }
token_array_out->count = token_i; token_array_out->count = token_i;
DfrYield(2, LexResult_NeedTokenMemory); DfrYield(2, LexResult_NeedTokenMemory);
@ -1182,6 +1307,7 @@ The start and end points are based on the edited region of the file before the e
if (end_pos > array->tokens[range.end_token_index].start){ if (end_pos > array->tokens[range.end_token_index].start){
++range.end_token_index; ++range.end_token_index;
} }
++range.end_token_index;
if (range.end_token_index < 0){ if (range.end_token_index < 0){
range.end_token_index = 0; range.end_token_index = 0;
} }

988
4cpp/4cpp_lexer_tables.c
View File

File diff suppressed because it is too large Load Diff

134
4cpp/4cpp_lexer_types.h
View File

@ -75,32 +75,25 @@ ENUM(uint32_t, Cpp_Token_Type){
CPP_TOKEN_SEMICOLON = 46, CPP_TOKEN_SEMICOLON = 46,
CPP_TOKEN_ELLIPSIS = 47, CPP_TOKEN_ELLIPSIS = 47,
/* DOC(This is an 'ambiguous' token type because it requires /* DOC(This is an 'ambiguous' token type because it requires parsing to determine the full nature of the token.) */
parsing to determine the full nature of the token.) */
CPP_TOKEN_STAR = 48, CPP_TOKEN_STAR = 48,
/* DOC(This is an 'ambiguous' token type because it requires /* DOC(This is an 'ambiguous' token type because it requires parsing to determine the full nature of the token.) */
parsing to determine the full nature of the token.) */
CPP_TOKEN_AMPERSAND = 49, CPP_TOKEN_AMPERSAND = 49,
/* DOC(This is an 'ambiguous' token type because it requires /* DOC(This is an 'ambiguous' token type because it requires parsing to determine the full nature of the token.) */
parsing to determine the full nature of the token.) */
CPP_TOKEN_TILDE = 50, CPP_TOKEN_TILDE = 50,
/* DOC(This is an 'ambiguous' token type because it requires /* DOC(This is an 'ambiguous' token type because it requires parsing to determine the full nature of the token.) */
parsing to determine the full nature of the token.) */
CPP_TOKEN_PLUS = 51, CPP_TOKEN_PLUS = 51,
/* DOC(This is an 'ambiguous' token type because it requires /* DOC(This is an 'ambiguous' token type because it requires parsing to determine the full nature of the token.) */
parsing to determine the full nature of the token.) */
CPP_TOKEN_MINUS = 52, CPP_TOKEN_MINUS = 52,
/* DOC(This is an 'ambiguous' token type because it requires /* DOC(This is an 'ambiguous' token type because it requires parsing to determine the full nature of the token.) */
parsing to determine the full nature of the token.) */
CPP_TOKEN_INCREMENT = 53, CPP_TOKEN_INCREMENT = 53,
/* DOC(This is an 'ambiguous' token type because it requires /* DOC(This is an 'ambiguous' token type because it requires parsing to determine the full nature of the token.) */
parsing to determine the full nature of the token.) */
CPP_TOKEN_DECREMENT = 54, CPP_TOKEN_DECREMENT = 54,
// NOTE(allen): Precedence 1, LtoR // NOTE(allen): Precedence 1, LtoR
@ -309,9 +302,8 @@ STRUCT Cpp_Relex_Range{
struct Cpp_Lex_FSM{ struct Cpp_Lex_FSM{
uint8_t state; uint8_t state;
uint8_t int_state;
uint8_t emit_token; uint8_t emit_token;
uint8_t multi_line; uint8_t flags;
}; };
static Cpp_Lex_FSM null_lex_fsm = {0}; static Cpp_Lex_FSM null_lex_fsm = {0};
@ -322,23 +314,29 @@ DOC_SEE(cpp_lex_data_init)
HIDE_MEMBERS() */ HIDE_MEMBERS() */
STRUCT Cpp_Lex_Data{ STRUCT Cpp_Lex_Data{
char tb[32]; char tb[32];
int32_t tb_pos; i32_4tech tb_pos;
int32_t token_start; i32_4tech token_start;
int32_t pos; i32_4tech pos;
int32_t pos_overide; i32_4tech pos_overide;
int32_t chunk_pos; i32_4tech chunk_pos;
Cpp_Lex_FSM fsm; Cpp_Lex_FSM fsm;
uint8_t white_done; u8_4tech white_done;
uint8_t pp_state; u8_4tech pp_state;
uint8_t completed; u8_4tech completed;
Cpp_Token token; Cpp_Token token;
int32_t ignore_string_delims; char raw_delim[16];
i32_4tech delim_length;
int32_t __pc__; b8_4tech str_raw;
b8_4tech str_include;
i32_4tech ignore_string_delims;
i32_4tech __pc__;
}; };
/* DOC(Cpp_Lex_Result is returned from the lexing engine to indicate why it stopped lexing.) */ /* DOC(Cpp_Lex_Result is returned from the lexing engine to indicate why it stopped lexing.) */
@ -395,45 +393,44 @@ ENUM_INTERNAL(uint8_t, Cpp_Lex_State){
LS_pound = 2, LS_pound = 2,
LS_pp = 3, LS_pp = 3,
LS_ppdef = 4, LS_ppdef = 4,
LS_char = 5, LS_string_R = 5,
LS_char_multiline = 6, LS_string_LUu8 = 6,
LS_char_slashed = 7, LS_string_u = 7,
LS_string = 8, LS_number = 8,
LS_string_multiline = 9, LS_number0 = 9,
LS_string_slashed = 10, LS_float = 10,
LS_number = 11, LS_crazy_float0 = 11,
LS_number0 = 12, LS_crazy_float1 = 12,
LS_float = 13, LS_hex = 13,
LS_crazy_float0 = 14, LS_comment_pre = 14,
LS_crazy_float1 = 15, LS_comment = 15,
LS_hex = 16, LS_comment_slashed = 16,
LS_comment_pre = 17, LS_comment_block = 17,
LS_comment = 18, LS_comment_block_ending = 18,
LS_comment_slashed = 19, LS_dot = 19,
LS_comment_block = 20, LS_ellipsis = 20,
LS_comment_block_ending = 21, LS_less = 21,
LS_dot = 22, LS_less_less = 22,
LS_ellipsis = 23, LS_more = 23,
LS_less = 24, LS_more_more = 24,
LS_less_less = 25, LS_minus = 25,
LS_more = 26, LS_arrow = 26,
LS_more_more = 27, LS_and = 27,
LS_minus = 28, LS_or = 28,
LS_arrow = 29, LS_plus = 29,
LS_and = 30, LS_colon = 30,
LS_or = 31, LS_single_op = 31,
LS_plus = 32, LS_error_message = 32,
LS_colon = 33,
LS_star = 34,
LS_modulo = 35,
LS_caret = 36,
LS_eq = 37,
LS_bang = 38,
LS_error_message = 39,
// //
LS_count = 40 LS_count = 33,
LS_char = 34,
}; };
// NOTE(allen): These provide names that match the overloaded meanings of string states.
#define LS_string_raw LS_string_R
#define LS_string_normal LS_string_LUu8
#define LS_string_include LS_string_u
ENUM_INTERNAL(uint8_t, Cpp_Lex_Int_State){ ENUM_INTERNAL(uint8_t, Cpp_Lex_Int_State){
LSINT_default, LSINT_default,
LSINT_u, LSINT_u,
@ -447,6 +444,19 @@ ENUM_INTERNAL(uint8_t, Cpp_Lex_Int_State){
LSINT_count LSINT_count
}; };
ENUM_INTERNAL(uint8_t, Cpp_Lex_Str_State){
LSSTR_default,
LSSTR_escape,
LSSTR_multiline,
//
LSSTR_count
};
#define LSSTR_include_count 1
#define LSSTR_error LSSTR_escape
#define LSSTR_get_delim LSSTR_escape
#define LSSTR_check_delim LSSTR_count
ENUM_INTERNAL(uint8_t, Cpp_Lex_PP_State){ ENUM_INTERNAL(uint8_t, Cpp_Lex_PP_State){
LSPP_default, LSPP_default,
LSPP_include, LSPP_include,

BIN
4ed_site.ctm
View File

Binary file not shown.

662
meta/fsm_table_generator.cpp
View File

@ -7,13 +7,42 @@
// TOP // TOP
// 4tech_standard_preamble.h
#if !defined(FTECH_INTEGERS)
#define FTECH_INTEGERS
#include <stdint.h>
typedef int8_t i8_4tech;
typedef int16_t i16_4tech;
typedef int32_t i32_4tech;
typedef int64_t i64_4tech;
typedef uint8_t u8_4tech;
typedef uint16_t u16_4tech;
typedef uint32_t u32_4tech;
typedef uint64_t u64_4tech;
typedef float f32_4tech;
typedef double f64_4tech;
typedef int8_t b8_4tech;
typedef int32_t b32_4tech;
#endif
#if !defined(Assert)
# define Assert(n) do{ if (!(n)) *(int*)0 = 0xA11E; }while(0)
#endif
#if !defined(API_EXPORT)
# define API_EXPORT
#endif
// standard preamble end
#include <string.h>
#include <stdlib.h> #include <stdlib.h>
#include <stdio.h> #include <stdio.h>
#include <stdint.h>
typedef int32_t bool32; typedef int32_t bool32;
#define Assert(n) do{ if (!(n)) { *(int*)0 = 0xA11E; } }while(0)
#define ArrayCount(a) (sizeof(a)/sizeof(*a)) #define ArrayCount(a) (sizeof(a)/sizeof(*a))
#define LEXER_TABLE_FILE "4cpp/4cpp_lexer_tables.c" #define LEXER_TABLE_FILE "4cpp/4cpp_lexer_tables.c"
@ -37,86 +66,247 @@ whitespace_skip_fsm(Whitespace_FSM wfsm, char c){
return(wfsm); return(wfsm);
} }
Cpp_Lex_FSM #define FSM_SIG(n) Cpp_Lex_FSM n(Cpp_Lex_FSM fsm, char c, b32_4tech get_flags)
int_fsm(Cpp_Lex_FSM fsm, char c){ typedef FSM_SIG(FSM_Function);
switch (fsm.int_state){
FSM_SIG(int_fsm){
switch (fsm.state){
case LSINT_default: case LSINT_default:
{
switch (c){ switch (c){
case 'u': case 'U': fsm.int_state = LSINT_u; break; case 'u': case 'U': fsm.state = LSINT_u; break;
case 'l': fsm.int_state = LSINT_l; break; case 'l': fsm.state = LSINT_l; break;
case 'L': fsm.int_state = LSINT_L; break; case 'L': fsm.state = LSINT_L; break;
default: fsm.emit_token = 1; break; default: fsm.emit_token = true; break;
} }
break; }break;
case LSINT_u: case LSINT_u:
{
switch (c){ switch (c){
case 'l': fsm.int_state = LSINT_ul; break; case 'l': fsm.state = LSINT_ul; break;
case 'L': fsm.int_state = LSINT_uL; break; case 'L': fsm.state = LSINT_uL; break;
default: fsm.emit_token = 1; break; default: fsm.emit_token = true; break;
} }
break; }break;
case LSINT_l: case LSINT_l:
{
switch (c){ switch (c){
case 'l': fsm.int_state = LSINT_ll; break; case 'l': fsm.state = LSINT_ll; break;
case 'U': case 'u': fsm.int_state = LSINT_extra; break; case 'U': case 'u': fsm.state = LSINT_extra; break;
default: fsm.emit_token = 1; break; default: fsm.emit_token = true; break;
} }
break; }break;
case LSINT_L: case LSINT_L:
{
switch (c){ switch (c){
case 'L': fsm.int_state = LSINT_ll; break; case 'L': fsm.state = LSINT_ll; break;
case 'U': case 'u': fsm.int_state = LSINT_extra; break; case 'U': case 'u': fsm.state = LSINT_extra; break;
default: fsm.emit_token = 1; break; default: fsm.emit_token = true; break;
} }
break; }break;
case LSINT_ul: case LSINT_ul:
{
switch (c){ switch (c){
case 'l': fsm.int_state = LSINT_extra; break; case 'l': fsm.state = LSINT_extra; break;
default: fsm.emit_token = 1; break; default: fsm.emit_token = true; break;
} }
break; }break;
case LSINT_uL: case LSINT_uL:
{
switch (c){ switch (c){
case 'L': fsm.int_state = LSINT_extra; break; case 'L': fsm.state = LSINT_extra; break;
default: fsm.emit_token = 1; break; default: fsm.emit_token = true; break;
} }
break; }break;
case LSINT_ll: case LSINT_ll:
{
switch (c){ switch (c){
case 'u': case 'U': fsm.int_state = LSINT_extra; break; case 'u': case 'U': fsm.state = LSINT_extra; break;
default: fsm.emit_token = 1; break; default: fsm.emit_token = true; break;
} }
break; }break;
case LSINT_extra: case LSINT_extra:
fsm.emit_token = 1; fsm.emit_token = true;
break; break;
} }
return(fsm); return(fsm);
} }
FSM_SIG(normal_str_fsm){
if (!get_flags){
switch (fsm.state){
case LSSTR_default:
case LSSTR_multiline:
{
switch (c){
case '\n': case 0: fsm.state = LSSTR_error; fsm.emit_token = true; break;
case '\\': fsm.state = LSSTR_escape; break;
case '"': fsm.emit_token = true; break;
default: break;
}
}break;
case LSSTR_escape:
{
switch (c){
case '\n': fsm.state = LSSTR_multiline; break;
default: fsm.state = LSSTR_default; break;
}
}break;
}
}
else{
switch (fsm.state){
case LSSTR_multiline:
{
fsm.flags = 1;
}break;
}
}
return(fsm);
}
FSM_SIG(normal_char_fsm){
if (!get_flags){
switch (fsm.state){
case LSSTR_default:
case LSSTR_multiline:
{
switch (c){
case '\n': case 0: fsm.state = LSSTR_error; fsm.emit_token = true; break;
case '\\': fsm.state = LSSTR_escape; break;
case '\'': fsm.emit_token = true; break;
default: break;
}
}break;
case LSSTR_escape:
{
switch (c){
case '\n': fsm.state = LSSTR_multiline; break;
default: fsm.state = LSSTR_default; break;
}
}break;
}
}
else{
switch (fsm.state){
case LSSTR_multiline:
{
fsm.flags = 1;
}break;
}
}
return(fsm);
}
FSM_SIG(raw_str_fsm){
if (!get_flags){
switch (fsm.state){
case LSSTR_default:
{
switch (c){
case ')': case '\\': case ' ': case '\n': fsm.emit_token = true; break;
case '(': fsm.state = LSSTR_get_delim; fsm.emit_token = true; break;
default: break;
}
}break;
case LSSTR_get_delim:
case LSSTR_multiline:
{
switch (c){
case '\n': fsm.state = LSSTR_multiline; break;
case 0: case '"': fsm.state = LSSTR_check_delim; fsm.emit_token = true; break;
default: break;
}
}break;
}
}
else{
switch (fsm.state){
case LSSTR_multiline:
{
fsm.flags = 1;
}break;
}
}
return(fsm);
}
FSM_SIG(include_str_fsm){
switch (fsm.state){
case LSSTR_default:
{
switch (c){
case '\n': case 0: fsm.state = LSSTR_error; fsm.emit_token = true; break;
case '>': fsm.emit_token = true; break;
default: break;
}
}break;
}
return(fsm);
}
b32_4tech
is_identifier_char(u8_4tech c, b32_4tech ignore_string_delims){
b32_4tech result = (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || c == '$' || c >= 128 || (ignore_string_delims && (c == '\'' || c == '"'));
return(result);
}
b32_4tech
is_identifier_char_restricted(u8_4tech c, b32_4tech ignore_string_delims){
b32_4tech result = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c >= 128 || (ignore_string_delims && (c == '\'' || c == '"'));
return(result);
}
b32_4tech
is_identifier_char_non_numeric(u8_4tech c, b32_4tech ignore_string_delims){
b32_4tech result = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || c == '$' || c >= 128 || (ignore_string_delims && (c == '\'' || c == '"'));
return(result);
}
Cpp_Lex_FSM Cpp_Lex_FSM
main_fsm(Cpp_Lex_FSM fsm, uint8_t pp_state, uint8_t c, bool32 ignore_string_delims){ main_fsm(Cpp_Lex_FSM fsm, uint8_t pp_state, uint8_t c, bool32 ignore_string_delims){
if (c == 0){ if (c == 0){
fsm.emit_token = 1; fsm.emit_token = true;
} }
else{ else{
switch (pp_state){ switch (pp_state){
case LSPP_error: case LSPP_error:
{
fsm.state = LS_error_message; fsm.state = LS_error_message;
if (c == '\n') fsm.emit_token = 1; if (c == '\n'){
break; fsm.emit_token = true;
}
}break;
default: default:
switch (fsm.state){ switch (fsm.state){
case LS_default: case LS_default:
if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || c == '$' || c >= 128){ if (c == 'R'){
fsm.state = LS_string_R;
}
else if (c == 'U' || c == 'L'){
fsm.state = LS_string_LUu8;
}
else if (c == 'u'){
fsm.state = LS_string_u;
}
else if (is_identifier_char_non_numeric(c, ignore_string_delims)){
fsm.state = LS_identifier; fsm.state = LS_identifier;
} }
else if (c >= '1' && c <= '9'){ else if (c >= '1' && c <= '9'){
@ -134,6 +324,7 @@ main_fsm(Cpp_Lex_FSM fsm, uint8_t pp_state, uint8_t c, bool32 ignore_string_deli
} }
else{ else{
fsm.state = LS_char; fsm.state = LS_char;
fsm.emit_token = true;
} }
}break; }break;
@ -143,7 +334,8 @@ main_fsm(Cpp_Lex_FSM fsm, uint8_t pp_state, uint8_t c, bool32 ignore_string_deli
fsm.state = LS_identifier; fsm.state = LS_identifier;
} }
else{ else{
fsm.state = LS_string; fsm.state = LS_string_normal;
fsm.emit_token = true;
} }
}break; }break;
@ -153,7 +345,8 @@ main_fsm(Cpp_Lex_FSM fsm, uint8_t pp_state, uint8_t c, bool32 ignore_string_deli
case '<': case '<':
if (pp_state == LSPP_include && !ignore_string_delims){ if (pp_state == LSPP_include && !ignore_string_delims){
fsm.state = LS_string; fsm.state = LS_string_include;
fsm.emit_token = true;
} }
else{ else{
fsm.state = LS_less; fsm.state = LS_less;
@ -171,13 +364,8 @@ main_fsm(Cpp_Lex_FSM fsm, uint8_t pp_state, uint8_t c, bool32 ignore_string_deli
case ':': fsm.state = LS_colon; break; case ':': fsm.state = LS_colon; break;
case '*': fsm.state = LS_star; break; case '*': case '%': case '^': case '=':
case '!': fsm.state = LS_single_op; break;
case '%': fsm.state = LS_modulo; break;
case '^': fsm.state = LS_caret; break;
case '=': fsm.state = LS_eq; break;
case '!': fsm.state = LS_bang; break;
case '#': case '#':
if (pp_state == LSPP_default){ if (pp_state == LSPP_default){
@ -188,7 +376,7 @@ main_fsm(Cpp_Lex_FSM fsm, uint8_t pp_state, uint8_t c, bool32 ignore_string_deli
} }
break; break;
#define OperCase(op,type) case op: fsm.emit_token = 1; break; #define OperCase(op,type) case op: fsm.emit_token = true; break;
OperCase('{', CPP_TOKEN_BRACE_OPEN); OperCase('{', CPP_TOKEN_BRACE_OPEN);
OperCase('}', CPP_TOKEN_BRACE_CLOSE); OperCase('}', CPP_TOKEN_BRACE_CLOSE);
@ -212,17 +400,16 @@ main_fsm(Cpp_Lex_FSM fsm, uint8_t pp_state, uint8_t c, bool32 ignore_string_deli
case LS_identifier: case LS_identifier:
{ {
int is_ident = (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || c == '$' || c >= 128 || (ignore_string_delims && (c == '\'' || c == '"')); b32_4tech is_ident = is_identifier_char(c, ignore_string_delims);
if (!is_ident){ if (!is_ident){
fsm.emit_token = 1; fsm.emit_token = true;
} }
} }
break; break;
case LS_pound: case LS_pound:
{ {
fsm.emit_token = 1; fsm.emit_token = true;
}break; }break;
case LS_pp: case LS_pp:
@ -230,82 +417,86 @@ main_fsm(Cpp_Lex_FSM fsm, uint8_t pp_state, uint8_t c, bool32 ignore_string_deli
if (c == ' ' || c == '\r' || c == '\v' || c == '\f'){ if (c == ' ' || c == '\r' || c == '\v' || c == '\f'){
// NOTE(allen): do nothing // NOTE(allen): do nothing
} }
else if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c >= 128 || (ignore_string_delims && (c == '\'' || c == '"'))){ else if (is_identifier_char_restricted(c, ignore_string_delims)){
fsm.state = LS_ppdef; fsm.state = LS_ppdef;
} }
else{ else{
fsm.emit_token = 1; fsm.emit_token = true;
} }
}break; }break;
case LS_ppdef: case LS_ppdef:
{ {
int is_ident = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c >= 128 || (ignore_string_delims && (c == '\'' || c == '"')); int is_ident = is_identifier_char_restricted(c, ignore_string_delims);
if (!is_ident){ if (!is_ident){
fsm.emit_token = 1; fsm.emit_token = true;
} }
}break; }break;
case LS_char: case LS_char_multiline: case LS_string_R:
{ {
if (ignore_string_delims){ if (ignore_string_delims){
fsm.state = LS_string; fsm.state = LS_count;
fsm.emit_token = 1; fsm.emit_token = true;
}
else{
switch(c){
case '\n': case '\'': fsm.emit_token = 1; break;
case '\\': fsm.state = LS_char_slashed; break;
}
}
}break;
case LS_char_slashed:
{
if (ignore_string_delims){
fsm.state = LS_string;
fsm.emit_token = 1;
} }
else{ else{
switch (c){ switch (c){
case '\r': case '\f': case '\v': break; case '"': fsm.state = LS_string_raw; fsm.emit_token = true; break;
case '\n': fsm.state = LS_char_multiline; break; default:
default: fsm.state = LS_char; break; {
fsm.state = LS_identifier;
b32_4tech is_ident = is_identifier_char(c, ignore_string_delims);
if (!is_ident){
fsm.emit_token = true;
}
}break;
} }
} }
}break; }break;
case LS_string: case LS_string_LUu8:
case LS_string_multiline:
{ {
if (ignore_string_delims){ if (ignore_string_delims){
fsm.state = LS_string; fsm.state = LS_count;
fsm.emit_token = 1; fsm.emit_token = true;
}
else{
switch(c){
case '\n': case '"': fsm.emit_token = 1; break;
case '>':
if (pp_state == LSPP_include){
fsm.emit_token = 1;
}
break;
case '\\': fsm.state = LS_string_slashed; break;
}
}
}break;
case LS_string_slashed:
{
if (ignore_string_delims){
fsm.state = LS_string;
fsm.emit_token = 1;
} }
else{ else{
switch (c){ switch (c){
case '\r': case '\f': case '\v': break; case '"': fsm.state = LS_string_normal; fsm.emit_token = true; break;
case '\n': fsm.state = LS_string_multiline; break; case '\'': fsm.state = LS_char; fsm.emit_token = true; break;
default: fsm.state = LS_string; break; case 'R': fsm.state = LS_string_R; break;
default:
{
fsm.state = LS_identifier;
b32_4tech is_ident = is_identifier_char(c, ignore_string_delims);
if (!is_ident){
fsm.emit_token = true;
}
}break;
}
}
}break;
case LS_string_u:
{
if (ignore_string_delims){
fsm.state = LS_count;
fsm.emit_token = true;
}
else{
switch (c){
case '"': fsm.state = LS_string_normal; fsm.emit_token = true; break;
case '\'': fsm.state = LS_char; fsm.emit_token = true; break;
case '8': fsm.state = LS_string_LUu8; break;
case 'R': fsm.state = LS_string_R; break;
default:
{
fsm.state = LS_identifier;
b32_4tech is_ident = is_identifier_char(c, ignore_string_delims);
if (!is_ident){
fsm.emit_token = true;
}
}break;
} }
} }
}break; }break;
@ -317,7 +508,7 @@ main_fsm(Cpp_Lex_FSM fsm, uint8_t pp_state, uint8_t c, bool32 ignore_string_deli
else{ else{
switch (c){ switch (c){
case '.': fsm.state = LS_float; break; case '.': fsm.state = LS_float; break;
default: fsm.emit_token = 1; break; default: fsm.emit_token = true; break;
} }
} }
break; break;
@ -333,7 +524,7 @@ main_fsm(Cpp_Lex_FSM fsm, uint8_t pp_state, uint8_t c, bool32 ignore_string_deli
fsm.state = LS_float; fsm.state = LS_float;
} }
else{ else{
fsm.emit_token = 1; fsm.emit_token = true;
} }
break; break;
@ -341,7 +532,7 @@ main_fsm(Cpp_Lex_FSM fsm, uint8_t pp_state, uint8_t c, bool32 ignore_string_deli
if (!(c >= '0' && c <= '9')){ if (!(c >= '0' && c <= '9')){
switch (c){ switch (c){
case 'e': fsm.state = LS_crazy_float0; break; case 'e': fsm.state = LS_crazy_float0; break;
default: fsm.emit_token = 1; break; default: fsm.emit_token = true; break;
} }
} }
break; break;
@ -352,7 +543,7 @@ main_fsm(Cpp_Lex_FSM fsm, uint8_t pp_state, uint8_t c, bool32 ignore_string_deli
fsm.state = LS_crazy_float1; fsm.state = LS_crazy_float1;
} }
else{ else{
fsm.emit_token = 1; fsm.emit_token = true;
} }
} }
break; break;
@ -360,7 +551,7 @@ main_fsm(Cpp_Lex_FSM fsm, uint8_t pp_state, uint8_t c, bool32 ignore_string_deli
case LS_crazy_float1: case LS_crazy_float1:
{ {
if (!(c >= '0' && c <= '9')){ if (!(c >= '0' && c <= '9')){
fsm.emit_token = 1; fsm.emit_token = true;
} }
} }
break; break;
@ -369,7 +560,7 @@ main_fsm(Cpp_Lex_FSM fsm, uint8_t pp_state, uint8_t c, bool32 ignore_string_deli
{ {
int is_hex = c >= '0' && c <= '9' || c >= 'a' && c <= 'f' || c >= 'A' && c <= 'F' || c >= 128; int is_hex = c >= '0' && c <= '9' || c >= 'a' && c <= 'f' || c >= 'A' && c <= 'F' || c >= 128;
if (!is_hex){ if (!is_hex){
fsm.emit_token = 1; fsm.emit_token = true;
} }
}break; }break;
@ -381,162 +572,149 @@ main_fsm(Cpp_Lex_FSM fsm, uint8_t pp_state, uint8_t c, bool32 ignore_string_deli
else{ else{
switch (c){ switch (c){
case '.': fsm.state = LS_ellipsis; break; case '.': fsm.state = LS_ellipsis; break;
case '*': fsm.emit_token = 1; break; case '*': fsm.emit_token = true; break;
default: fsm.emit_token = 1; break; default: fsm.emit_token = true; break;
} }
} }
}break; }break;
case LS_ellipsis: fsm.emit_token = 1; break; case LS_ellipsis: fsm.emit_token = true; break;
case LS_less: case LS_less:
{
switch (c){ switch (c){
case '<': fsm.state = LS_less_less; break; case '<': fsm.state = LS_less_less; break;
case '=': fsm.emit_token = 1; break; case '=': fsm.emit_token = true; break;
default: fsm.emit_token = 1; break; default: fsm.emit_token = true; break;
} }
break; }break;
case LS_less_less: case LS_less_less:
{
switch (c){ switch (c){
case '=': fsm.emit_token = 1; break; case '=': fsm.emit_token = true; break;
default: fsm.emit_token = 1; break; default: fsm.emit_token = true; break;
} }
break; }break;
case LS_more: case LS_more:
{
switch (c){ switch (c){
case '>': fsm.state = LS_more_more; break; case '>': fsm.state = LS_more_more; break;
case '=': fsm.emit_token = 1; break; case '=': fsm.emit_token = true; break;
default: fsm.emit_token = 1; break; default: fsm.emit_token = true; break;
} }
break; }break;
case LS_more_more: case LS_more_more:
{
switch (c){ switch (c){
case '=': fsm.emit_token = 1; break; case '=': fsm.emit_token = true; break;
default: fsm.emit_token = 1; break; default: fsm.emit_token = true; break;
} }
break; }break;
case LS_comment_pre: case LS_comment_pre:
{
switch (c){ switch (c){
case '/': fsm.state = LS_comment; break; case '/': fsm.state = LS_comment; break;
case '*': fsm.state = LS_comment_block; break; case '*': fsm.state = LS_comment_block; break;
case '=': fsm.emit_token = 1; break; case '=': fsm.emit_token = true; break;
default: fsm.emit_token = 1; break; default: fsm.emit_token = true; break;
} }
break; }break;
case LS_comment: case LS_comment:
{
switch (c){ switch (c){
case '\\': fsm.state = LS_comment_slashed; break; case '\\': fsm.state = LS_comment_slashed; break;
case '\n': fsm.emit_token = 1; break; case '\n': fsm.emit_token = true; break;
} }
break; }break;
case LS_comment_slashed: case LS_comment_slashed:
{
switch (c){ switch (c){
case '\r': case '\f': case '\v': break; case '\r': case '\f': case '\v': break;
default: fsm.state = LS_comment; break; default: fsm.state = LS_comment; break;
} }
break; }break;
case LS_comment_block: case LS_comment_block:
{
switch (c){ switch (c){
case '*': fsm.state = LS_comment_block_ending; break; case '*': fsm.state = LS_comment_block_ending; break;
} }
break; }break;
case LS_comment_block_ending: case LS_comment_block_ending:
{
switch (c){ switch (c){
case '*': fsm.state = LS_comment_block_ending; break; case '*': fsm.state = LS_comment_block_ending; break;
case '/': fsm.emit_token = 1; break; case '/': fsm.emit_token = true; break;
default: fsm.state = LS_comment_block; break; default: fsm.state = LS_comment_block; break;
} }
break; }break;
case LS_minus: case LS_minus:
{
switch (c){ switch (c){
case '>': fsm.state = LS_arrow; break; case '>': fsm.state = LS_arrow; break;
case '-': fsm.emit_token = 1; break; case '-': fsm.emit_token = true; break;
case '=': fsm.emit_token = 1; break; case '=': fsm.emit_token = true; break;
default: fsm.emit_token = 1; break; default: fsm.emit_token = true; break;
} }
break; }break;
case LS_arrow: case LS_arrow:
{
switch (c){ switch (c){
case '*': fsm.emit_token = 1; break; case '*': fsm.emit_token = true; break;
default: fsm.emit_token = 1; break; default: fsm.emit_token = true; break;
} }
break; }break;
case LS_and: case LS_and:
switch (c){ switch (c){
case '&': fsm.emit_token = 1; break; case '&': fsm.emit_token = true; break;
case '=': fsm.emit_token = 1; break; case '=': fsm.emit_token = true; break;
default: fsm.emit_token = 1; break; default: fsm.emit_token = true; break;
} }
break; break;
case LS_or: case LS_or:
{
switch (c){ switch (c){
case '|': fsm.emit_token = 1; break; case '|': fsm.emit_token = true; break;
case '=': fsm.emit_token = 1; break; case '=': fsm.emit_token = true; break;
default: fsm.emit_token = 1; break; default: fsm.emit_token = true; break;
} }
break; }break;
case LS_plus: case LS_plus:
{
switch (c){ switch (c){
case '+': fsm.emit_token = 1; break; case '+': fsm.emit_token = true; break;
case '=': fsm.emit_token = 1; break; case '=': fsm.emit_token = true; break;
default: fsm.emit_token = 1; break; default: fsm.emit_token = true; break;
} }
break; }break;
case LS_colon: case LS_colon:
{
switch (c){ switch (c){
case ':': fsm.emit_token = 1; break; case ':': fsm.emit_token = true; break;
default: fsm.emit_token = 1; break; default: fsm.emit_token = true; break;
} }
break; }break;
case LS_star: case LS_single_op:
{
switch (c){ switch (c){
case '=': fsm.emit_token = 1; break; case '=': fsm.emit_token = true; break;
default: fsm.emit_token = 1; break; default: fsm.emit_token = true; break;
} }
break; }break;
case LS_modulo:
switch (c){
case '=': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
break;
case LS_caret:
switch (c){
case '=': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
break;
case LS_eq:
switch (c){
case '=': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
break;
case LS_bang:
switch (c){
case '=': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break;
}
break;
} }
break; break;
} }
@ -561,7 +739,17 @@ begin_ptr_table(FILE *file, char *type, char *table_name){
static void static void
do_table_item(FILE *file, uint16_t item){ do_table_item(FILE *file, uint16_t item){
fprintf(file, "%2d,", (int32_t)item); fprintf(file, "%3d,", (int32_t)item);
}
static void
do_table_item_binary(FILE *file, uint16_t item){
if (item == 0){
fprintf(file, "0x00,");
}
else{
fprintf(file, "%#04x,", item);
}
} }
static void static void
@ -580,14 +768,15 @@ end_table(FILE *file){
} }
typedef struct FSM_Tables{ typedef struct FSM_Tables{
uint8_t *full_transition_table; u8_4tech *full_transition_table;
uint8_t *marks; u8_4tech *marks;
uint8_t *eq_class; u8_4tech *eq_class;
uint8_t *eq_class_rep; u8_4tech *eq_class_rep;
uint8_t *reduced_transition_table; u8_4tech *reduced_transition_table;
u8_4tech *flags;
uint8_t eq_class_counter; u8_4tech eq_class_counter;
uint16_t state_count; u16_4tech state_count;
} FSM_Tables; } FSM_Tables;
static void static void
@ -637,7 +826,7 @@ do_table_reduction(FSM_Tables *table, uint16_t state_count){
static FSM_Tables static FSM_Tables
generate_whitespace_skip_table(){ generate_whitespace_skip_table(){
uint8_t state_count = LSPP_count; uint8_t state_count = LSPP_count;
FSM_Tables table; FSM_Tables table = {0};
allocate_full_tables(&table, state_count); allocate_full_tables(&table, state_count);
int32_t i = 0; int32_t i = 0;
@ -658,20 +847,33 @@ generate_whitespace_skip_table(){
} }
static FSM_Tables static FSM_Tables
generate_int_table(){ generate_table(u8_4tech state_count, FSM_Function *fsm_call){
uint8_t state_count = LSINT_count; FSM_Tables table = {0};
FSM_Tables table;
allocate_full_tables(&table, state_count); allocate_full_tables(&table, state_count);
int32_t i = 0; i32_4tech i = 0;
Cpp_Lex_FSM fsm = {0}; Cpp_Lex_FSM fsm = {0};
Cpp_Lex_FSM new_fsm = {0}; Cpp_Lex_FSM new_fsm = {0};
for (uint16_t c = 0; c < 256; ++c){ for (uint16_t c = 0; c < 256; ++c){
for (uint8_t state = 0; state < state_count; ++state){ for (u8_4tech state = 0; state < state_count; ++state){
fsm.int_state = state; fsm.state = state;
fsm.emit_token = 0; fsm.emit_token = false;
new_fsm = int_fsm(fsm, (uint8_t)c); new_fsm = fsm_call(fsm, (u8_4tech)c, false);
table.full_transition_table[i++] = new_fsm.int_state + state_count*new_fsm.emit_token; table.full_transition_table[i++] = new_fsm.state + state_count*new_fsm.emit_token;
}
}
for (u8_4tech state = 0; state < state_count; ++state){
fsm.state = state;
fsm.emit_token = false;
fsm.flags = 0;
new_fsm = fsm_call(fsm, 0, true);
if (new_fsm.flags != 0){
if (table.flags == 0){
table.flags = (u8_4tech*)malloc(state_count);
memset(table.flags, 0, state_count);
}
table.flags[state] = new_fsm.flags;
} }
} }
@ -683,7 +885,7 @@ generate_int_table(){
static FSM_Tables static FSM_Tables
generate_fsm_table(uint8_t pp_state, bool32 ignore_string_delims){ generate_fsm_table(uint8_t pp_state, bool32 ignore_string_delims){
uint8_t state_count = LS_count; uint8_t state_count = LS_count;
FSM_Tables table; FSM_Tables table = {0};
allocate_full_tables(&table, state_count); allocate_full_tables(&table, state_count);
int32_t i = 0; int32_t i = 0;
@ -692,7 +894,7 @@ generate_fsm_table(uint8_t pp_state, bool32 ignore_string_delims){
for (uint16_t c = 0; c < 256; ++c){ for (uint16_t c = 0; c < 256; ++c){
for (uint8_t state = 0; state < state_count; ++state){ for (uint8_t state = 0; state < state_count; ++state){
fsm.state = state; fsm.state = state;
fsm.emit_token = 0; fsm.emit_token = false;
new_fsm = main_fsm(fsm, pp_state, (uint8_t)c, ignore_string_delims); new_fsm = main_fsm(fsm, pp_state, (uint8_t)c, ignore_string_delims);
table.full_transition_table[i++] = new_fsm.state + state_count*new_fsm.emit_token; table.full_transition_table[i++] = new_fsm.state + state_count*new_fsm.emit_token;
} }
@ -705,24 +907,39 @@ generate_fsm_table(uint8_t pp_state, bool32 ignore_string_delims){
static void static void
render_fsm_table(FILE *file, FSM_Tables tables, char *group_name){ render_fsm_table(FILE *file, FSM_Tables tables, char *group_name){
begin_table(file, "uint16_t", group_name, "eq_classes"); begin_table(file, "u16_4tech", group_name, "eq_classes");
for (uint16_t c = 0; c < 256; ++c){ for (u16_4tech c = 0; c < 256; ++c){
if ((c % 16) == 0 && c > 0){
end_row(file);
}
do_table_item(file, tables.eq_class[c]*tables.state_count); do_table_item(file, tables.eq_class[c]*tables.state_count);
} }
end_row(file); end_row(file);
end_table(file); end_table(file);
fprintf(file, "const int32_t num_%s_eq_classes = %d;\n\n", group_name, tables.eq_class_counter); fprintf(file, "const i32_4tech num_%s_eq_classes = %d;\n\n", group_name, tables.eq_class_counter);
int32_t i = 0; i32_4tech i = 0;
begin_table(file, "uint8_t", group_name, "table"); begin_table(file, "u8_4tech", group_name, "table");
for (uint16_t c = 0; c < tables.eq_class_counter; ++c){ for (u16_4tech c = 0; c < tables.eq_class_counter; ++c){
for (uint8_t state = 0; state < tables.state_count; ++state){ for (u8_4tech state = 0; state < tables.state_count; ++state){
do_table_item(file, tables.reduced_transition_table[i++]); do_table_item(file, tables.reduced_transition_table[i++]);
} }
end_row(file); end_row(file);
} }
end_table(file); end_table(file);
if (tables.flags != 0){
begin_table(file, "u8_4tech", group_name, "flags");
for (u8_4tech state = 0; state < tables.state_count; ++state){
if ((state % 4) == 0 && state > 0){
end_row(file);
}
do_table_item_binary(file, tables.flags[state]);
}
end_row(file);
end_table(file);
}
} }
static void static void
@ -751,7 +968,7 @@ static PP_Names pp_names[] = {
{LSPP_number, "pp_number_fsm", false}, {LSPP_number, "pp_number_fsm", false},
{LSPP_error, "pp_error_fsm", false}, {LSPP_error, "pp_error_fsm", false},
{LSPP_junk, "pp_junk_fsm", false}, {LSPP_junk, "pp_junk_fsm", false},
{LSPP_default, "no_string_fsm", true}, {LSPP_default, "no_string_fsm", true },
}; };
int int
@ -761,15 +978,26 @@ main(){
FSM_Tables wtables = generate_whitespace_skip_table(); FSM_Tables wtables = generate_whitespace_skip_table();
render_fsm_table(file, wtables, "whitespace_fsm"); render_fsm_table(file, wtables, "whitespace_fsm");
FSM_Tables itables = generate_int_table(); FSM_Tables itables = generate_table(LSINT_count, int_fsm);
render_fsm_table(file, itables, "int_fsm"); render_fsm_table(file, itables, "int_fsm");
begin_table(file, "uint8_t", "multiline_state_table"); {
for (uint8_t state = 0; state < LS_count*2; ++state){ struct{
do_table_item(file, (state == LS_string_multiline || state == LS_char_multiline)); char *name;
uint8_t count;
FSM_Function *fsm_call;
} static tables[] = {
{"raw_str", LSSTR_count, raw_str_fsm },
{"normal_str", LSSTR_count, normal_str_fsm },
{"include_str", LSSTR_include_count, include_str_fsm },
{"normal_char", LSSTR_count, normal_char_fsm }
};
for (u32_4tech i = 0; i < ArrayCount(tables); ++i){
FSM_Tables str_tables = generate_table(tables[i].count, tables[i].fsm_call);
render_fsm_table(file, str_tables, tables[i].name);
}
} }
end_row(file);
end_table(file);
for (int32_t i = 0; i < ArrayCount(pp_names); ++i){ for (int32_t i = 0; i < ArrayCount(pp_names); ++i){
FSM_Tables tables = generate_fsm_table(pp_names[i].pp_state, pp_names[i].ignore_string_delims); FSM_Tables tables = generate_fsm_table(pp_names[i].pp_state, pp_names[i].ignore_string_delims);