mr4th-public
/
4coder
1
Fork 0
Code Issues Releases Activity

fix a few issues with string termination in the new lexer, switch over to single line strings

master
Allen Webster 2016-07-18 19:57:08 -04:00
parent 2076676156
commit 090e23d31e
5 changed files with 140 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
internal_4coder_tests.cpp Normal file
View File

@ -0,0 +1,19 @@
/*
A series of stress tests to more quickly verify that 4coder isn't totally
broken before I release in the future... "unit tests" perhaps, although
I guess I don't know what the "units" are... probably more like "tests".
Allen Webster
18.07.2016
*/
// TOP
#include "test_this.cpp"
// BOTTOM

23
test/4cpp_lexer_fsms.h
View File

@ -2,6 +2,29 @@
* FSMs for 4c++ lexer
*
* 23.03.2016 (dd.mm.yyyy)
*/
// TOP

16
test/4cpp_lexer_tables.c
View File

@ -36,7 +36,7 @@ const int num_main_fsm_eq_classes = 29;
unsigned char main_fsm_table[] = {
39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,
0,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41, 3, 4, 5, 5, 7, 8, 8,49,50,51,52,53,54,55,56,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41, 3,43,44, 5,46,47, 8,49,50,51,52,53,54,55,56,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41, 3, 4, 5, 6, 7, 8, 9,49,50,51,52,53,54,55,17,18,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
37,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
7,40,41, 3, 4, 5, 4,46,47, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
@ -74,7 +74,7 @@ const int num_pp_include_fsm_eq_classes = 6;
unsigned char pp_include_fsm_table[] = {
39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,
3, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,
3, 1, 2,42, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,
3,42,42,42, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,
1,40, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,
2, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,
3, 1,41, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,
@ -89,7 +89,7 @@ const int num_pp_macro_fsm_eq_classes = 29;
unsigned char pp_macro_fsm_table[] = {
39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,
0,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41, 3, 4, 5, 5, 7, 8, 8,49,50,51,52,53,54,55,56,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41, 3,43,44, 5,46,47, 8,49,50,51,52,53,54,55,56,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41, 3, 4, 5, 6, 7, 8, 9,49,50,51,52,53,54,55,17,18,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
37,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
7,40,41, 3, 4, 5, 4,46,47, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
@ -127,7 +127,7 @@ const int num_pp_identifier_fsm_eq_classes = 29;
unsigned char pp_identifier_fsm_table[] = {
39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,
0,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41, 3, 4, 5, 5, 7, 8, 8,49,50,51,52,53,54,55,56,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41, 3,43,44, 5,46,47, 8,49,50,51,52,53,54,55,56,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41, 3, 4, 5, 6, 7, 8, 9,49,50,51,52,53,54,55,17,18,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
37,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
7,40,41, 3, 4, 5, 4,46,47, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
@ -165,7 +165,7 @@ const int num_pp_body_if_fsm_eq_classes = 29;
unsigned char pp_body_if_fsm_table[] = {
39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,
0,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41, 3, 4, 5, 5, 7, 8, 8,49,50,51,52,53,54,55,56,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41, 3,43,44, 5,46,47, 8,49,50,51,52,53,54,55,56,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41, 3, 4, 5, 6, 7, 8, 9,49,50,51,52,53,54,55,17,18,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
37,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
7,40,41, 3, 4, 5, 4,46,47, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
@ -203,7 +203,7 @@ const int num_pp_body_fsm_eq_classes = 29;
unsigned char pp_body_fsm_table[] = {
39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,
0,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41, 3, 4, 5, 5, 7, 8, 8,49,50,51,52,53,54,55,56,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41, 3,43,44, 5,46,47, 8,49,50,51,52,53,54,55,56,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41, 3, 4, 5, 6, 7, 8, 9,49,50,51,52,53,54,55,17,18,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
37,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
7,40,41, 3, 4, 5, 4,46,47, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
@ -241,7 +241,7 @@ const int num_pp_number_fsm_eq_classes = 29;
unsigned char pp_number_fsm_table[] = {
39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,
0,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41, 3, 4, 5, 5, 7, 8, 8,49,50,51,52,53,54,55,56,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41, 3,43,44, 5,46,47, 8,49,50,51,52,53,54,55,56,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41, 3, 4, 5, 6, 7, 8, 9,49,50,51,52,53,54,55,17,18,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
37,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
7,40,41, 3, 4, 5, 4,46,47, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
@ -291,7 +291,7 @@ const int num_pp_junk_fsm_eq_classes = 29;
unsigned char pp_junk_fsm_table[] = {
39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,
0,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41, 3, 4, 5, 5, 7, 8, 8,49,50,51,52,53,54,55,56,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41, 3,43,44, 5,46,47, 8,49,50,51,52,53,54,55,56,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41, 3, 4, 5, 6, 7, 8, 9,49,50,51,52,53,54,55,17,18,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
37,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
7,40,41, 3, 4, 5, 4,46,47, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,

20
test/4cpp_new_lexer.h
View File

@ -640,23 +640,35 @@ cpp_lex_nonalloc(Lex_Data *S_ptr,
case LS_char:
case LS_char_slashed:
S.token.type = CPP_TOKEN_CHARACTER_CONSTANT;
S.token.type = CPP_TOKEN_JUNK;
if (c == '\''){
S.token.type = CPP_TOKEN_CHARACTER_CONSTANT;
}
S.token.flags = 0;
break;
case LS_char_multiline:
S.token.type = CPP_TOKEN_CHARACTER_CONSTANT;
S.token.type = CPP_TOKEN_JUNK;
if (c == '\''){
S.token.type = CPP_TOKEN_CHARACTER_CONSTANT;
}
S.token.flags = CPP_TFLAG_MULTILINE;
break;
case LS_string:
case LS_string_slashed:
S.token.type = CPP_TOKEN_STRING_CONSTANT;
S.token.type = CPP_TOKEN_JUNK;
if (c == '"'){
S.token.type = CPP_TOKEN_STRING_CONSTANT;
}
S.token.flags = 0;
break;
case LS_string_multiline:
S.token.type = CPP_TOKEN_STRING_CONSTANT;
S.token.type = CPP_TOKEN_JUNK;
if (c == '"'){
S.token.type = CPP_TOKEN_STRING_CONSTANT;
}
S.token.flags = CPP_TFLAG_MULTILINE;
break;

12
test/fsm_table_generator.cpp
View File

@ -673,10 +673,18 @@ main_fsm(Lex_FSM fsm, unsigned char pp_state, unsigned char c){
case LSINC_quotes:
if (c == '"') fsm.emit_token = 1;
else if (c == '\n'){
fsm.emit_token = 1;
fsm.state = LSINC_junk;
}
break;
case LSINC_pointy:
if (c == '>') fsm.emit_token = 1;
else if (c == '\n'){
fsm.emit_token = 1;
fsm.state = LSINC_junk;
}
break;
case LSINC_junk:
@ -775,7 +783,7 @@ main_fsm(Lex_FSM fsm, unsigned char pp_state, unsigned char c){
case LS_char:
case LS_char_multiline:
switch(c){
case '\'': fsm.emit_token = 1; break;
case '\n': case '\'': fsm.emit_token = 1; break;
case '\\': fsm.state = LS_char_slashed; break;
}
break;
@ -791,7 +799,7 @@ main_fsm(Lex_FSM fsm, unsigned char pp_state, unsigned char c){
case LS_string:
case LS_string_multiline:
switch(c){
case '\"': fsm.emit_token = 1; break;
case '\n': case '\"': fsm.emit_token = 1; break;
case '\\': fsm.state = LS_string_slashed; break;
}
break;