mr4th-public
/
4coder
1
Fork 0
Code Issues Releases Activity

fix a few issues with string termination in the new lexer, switch over to single line strings

master
Allen Webster 2016-07-18 19:57:08 -04:00
parent 2076676156
commit 090e23d31e
5 changed files with 140 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
internal_4coder_tests.cpp Normal file
View File

@ -0,0 +1,19 @@
/*
A series of stress tests to more quickly verify that 4coder isn't totally
broken before I release in the future... "unit tests" perhaps, although
I guess I don't know what the "units" are... probably more like "tests".
Allen Webster
18.07.2016
*/
// TOP
#include "test_this.cpp"
// BOTTOM

23
test/4cpp_lexer_fsms.h
View File

@ -2,6 +2,29 @@
* FSMs for 4c++ lexer * FSMs for 4c++ lexer
* *
* 23.03.2016 (dd.mm.yyyy) * 23.03.2016 (dd.mm.yyyy)
*/ */
// TOP // TOP

16
test/4cpp_lexer_tables.c
View File

@ -36,7 +36,7 @@ const int num_main_fsm_eq_classes = 29;
unsigned char main_fsm_table[] = { unsigned char main_fsm_table[] = {
39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77, 39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,
0,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38, 0,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41, 3, 4, 5, 5, 7, 8, 8,49,50,51,52,53,54,55,56,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38, 0,40,41, 3,43,44, 5,46,47, 8,49,50,51,52,53,54,55,56,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41, 3, 4, 5, 6, 7, 8, 9,49,50,51,52,53,54,55,17,18,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38, 0,40,41, 3, 4, 5, 6, 7, 8, 9,49,50,51,52,53,54,55,17,18,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
37,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38, 37,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
7,40,41, 3, 4, 5, 4,46,47, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38, 7,40,41, 3, 4, 5, 4,46,47, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
@ -74,7 +74,7 @@ const int num_pp_include_fsm_eq_classes = 6;
unsigned char pp_include_fsm_table[] = { unsigned char pp_include_fsm_table[] = {
39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77, 39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,
3, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38, 3, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,
3, 1, 2,42, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38, 3,42,42,42, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,
1,40, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38, 1,40, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,
2, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38, 2, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,
3, 1,41, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38, 3, 1,41, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,
@ -89,7 +89,7 @@ const int num_pp_macro_fsm_eq_classes = 29;
unsigned char pp_macro_fsm_table[] = { unsigned char pp_macro_fsm_table[] = {
39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77, 39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,
0,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38, 0,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41, 3, 4, 5, 5, 7, 8, 8,49,50,51,52,53,54,55,56,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38, 0,40,41, 3,43,44, 5,46,47, 8,49,50,51,52,53,54,55,56,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41, 3, 4, 5, 6, 7, 8, 9,49,50,51,52,53,54,55,17,18,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38, 0,40,41, 3, 4, 5, 6, 7, 8, 9,49,50,51,52,53,54,55,17,18,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
37,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38, 37,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
7,40,41, 3, 4, 5, 4,46,47, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38, 7,40,41, 3, 4, 5, 4,46,47, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
@ -127,7 +127,7 @@ const int num_pp_identifier_fsm_eq_classes = 29;
unsigned char pp_identifier_fsm_table[] = { unsigned char pp_identifier_fsm_table[] = {
39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77, 39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,
0,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38, 0,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41, 3, 4, 5, 5, 7, 8, 8,49,50,51,52,53,54,55,56,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38, 0,40,41, 3,43,44, 5,46,47, 8,49,50,51,52,53,54,55,56,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41, 3, 4, 5, 6, 7, 8, 9,49,50,51,52,53,54,55,17,18,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38, 0,40,41, 3, 4, 5, 6, 7, 8, 9,49,50,51,52,53,54,55,17,18,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
37,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38, 37,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
7,40,41, 3, 4, 5, 4,46,47, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38, 7,40,41, 3, 4, 5, 4,46,47, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
@ -165,7 +165,7 @@ const int num_pp_body_if_fsm_eq_classes = 29;
unsigned char pp_body_if_fsm_table[] = { unsigned char pp_body_if_fsm_table[] = {
39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77, 39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,
0,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38, 0,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41, 3, 4, 5, 5, 7, 8, 8,49,50,51,52,53,54,55,56,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38, 0,40,41, 3,43,44, 5,46,47, 8,49,50,51,52,53,54,55,56,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41, 3, 4, 5, 6, 7, 8, 9,49,50,51,52,53,54,55,17,18,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38, 0,40,41, 3, 4, 5, 6, 7, 8, 9,49,50,51,52,53,54,55,17,18,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
37,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38, 37,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
7,40,41, 3, 4, 5, 4,46,47, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38, 7,40,41, 3, 4, 5, 4,46,47, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
@ -203,7 +203,7 @@ const int num_pp_body_fsm_eq_classes = 29;
unsigned char pp_body_fsm_table[] = { unsigned char pp_body_fsm_table[] = {
39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77, 39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,
0,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38, 0,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41, 3, 4, 5, 5, 7, 8, 8,49,50,51,52,53,54,55,56,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38, 0,40,41, 3,43,44, 5,46,47, 8,49,50,51,52,53,54,55,56,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41, 3, 4, 5, 6, 7, 8, 9,49,50,51,52,53,54,55,17,18,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38, 0,40,41, 3, 4, 5, 6, 7, 8, 9,49,50,51,52,53,54,55,17,18,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
37,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38, 37,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
7,40,41, 3, 4, 5, 4,46,47, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38, 7,40,41, 3, 4, 5, 4,46,47, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
@ -241,7 +241,7 @@ const int num_pp_number_fsm_eq_classes = 29;
unsigned char pp_number_fsm_table[] = { unsigned char pp_number_fsm_table[] = {
39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77, 39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,
0,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38, 0,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41, 3, 4, 5, 5, 7, 8, 8,49,50,51,52,53,54,55,56,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38, 0,40,41, 3,43,44, 5,46,47, 8,49,50,51,52,53,54,55,56,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41, 3, 4, 5, 6, 7, 8, 9,49,50,51,52,53,54,55,17,18,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38, 0,40,41, 3, 4, 5, 6, 7, 8, 9,49,50,51,52,53,54,55,17,18,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
37,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38, 37,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
7,40,41, 3, 4, 5, 4,46,47, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38, 7,40,41, 3, 4, 5, 4,46,47, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
@ -291,7 +291,7 @@ const int num_pp_junk_fsm_eq_classes = 29;
unsigned char pp_junk_fsm_table[] = { unsigned char pp_junk_fsm_table[] = {
39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77, 39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,
0,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38, 0,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41, 3, 4, 5, 5, 7, 8, 8,49,50,51,52,53,54,55,56,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38, 0,40,41, 3,43,44, 5,46,47, 8,49,50,51,52,53,54,55,56,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
0,40,41, 3, 4, 5, 6, 7, 8, 9,49,50,51,52,53,54,55,17,18,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38, 0,40,41, 3, 4, 5, 6, 7, 8, 9,49,50,51,52,53,54,55,17,18,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
37,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38, 37,40,41, 3, 4, 5, 4, 7, 8, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,
7,40,41, 3, 4, 5, 4,46,47, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38, 7,40,41, 3, 4, 5, 4,46,47, 7,49,50,51,52,53,54,55,17,17,19,19,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,38,

20
test/4cpp_new_lexer.h
View File

@ -640,23 +640,35 @@ cpp_lex_nonalloc(Lex_Data *S_ptr,
case LS_char: case LS_char:
case LS_char_slashed: case LS_char_slashed:
S.token.type = CPP_TOKEN_CHARACTER_CONSTANT; S.token.type = CPP_TOKEN_JUNK;
if (c == '\''){
S.token.type = CPP_TOKEN_CHARACTER_CONSTANT;
}
S.token.flags = 0; S.token.flags = 0;
break; break;
case LS_char_multiline: case LS_char_multiline:
S.token.type = CPP_TOKEN_CHARACTER_CONSTANT; S.token.type = CPP_TOKEN_JUNK;
if (c == '\''){
S.token.type = CPP_TOKEN_CHARACTER_CONSTANT;
}
S.token.flags = CPP_TFLAG_MULTILINE; S.token.flags = CPP_TFLAG_MULTILINE;
break; break;
case LS_string: case LS_string:
case LS_string_slashed: case LS_string_slashed:
S.token.type = CPP_TOKEN_STRING_CONSTANT; S.token.type = CPP_TOKEN_JUNK;
if (c == '"'){
S.token.type = CPP_TOKEN_STRING_CONSTANT;
}
S.token.flags = 0; S.token.flags = 0;
break; break;
case LS_string_multiline: case LS_string_multiline:
S.token.type = CPP_TOKEN_STRING_CONSTANT; S.token.type = CPP_TOKEN_JUNK;
if (c == '"'){
S.token.type = CPP_TOKEN_STRING_CONSTANT;
}
S.token.flags = CPP_TFLAG_MULTILINE; S.token.flags = CPP_TFLAG_MULTILINE;
break; break;

140
test/fsm_table_generator.cpp
View File

@ -598,7 +598,7 @@ int_fsm(Lex_FSM fsm, char c){
default: fsm.emit_token = 1; break; default: fsm.emit_token = 1; break;
} }
break; break;
case LSINT_u: case LSINT_u:
switch (c){ switch (c){
case 'l': fsm.int_state = LSINT_ul; break; case 'l': fsm.int_state = LSINT_ul; break;
@ -606,7 +606,7 @@ int_fsm(Lex_FSM fsm, char c){
default: fsm.emit_token = 1; break; default: fsm.emit_token = 1; break;
} }
break; break;
case LSINT_l: case LSINT_l:
switch (c){ switch (c){
case 'l': fsm.int_state = LSINT_ll; break; case 'l': fsm.int_state = LSINT_ll; break;
@ -614,7 +614,7 @@ int_fsm(Lex_FSM fsm, char c){
default: fsm.emit_token = 1; break; default: fsm.emit_token = 1; break;
} }
break; break;
case LSINT_L: case LSINT_L:
switch (c){ switch (c){
case 'L': fsm.int_state = LSINT_ll; break; case 'L': fsm.int_state = LSINT_ll; break;
@ -622,28 +622,28 @@ int_fsm(Lex_FSM fsm, char c){
default: fsm.emit_token = 1; break; default: fsm.emit_token = 1; break;
} }
break; break;
case LSINT_ul: case LSINT_ul:
switch (c){ switch (c){
case 'l': fsm.int_state = LSINT_extra; break; case 'l': fsm.int_state = LSINT_extra; break;
default: fsm.emit_token = 1; break; default: fsm.emit_token = 1; break;
} }
break; break;
case LSINT_uL: case LSINT_uL:
switch (c){ switch (c){
case 'L': fsm.int_state = LSINT_extra; break; case 'L': fsm.int_state = LSINT_extra; break;
default: fsm.emit_token = 1; break; default: fsm.emit_token = 1; break;
} }
break; break;
case LSINT_ll: case LSINT_ll:
switch (c){ switch (c){
case 'u': case 'U': fsm.int_state = LSINT_extra; break; case 'u': case 'U': fsm.int_state = LSINT_extra; break;
default: fsm.emit_token = 1; break; default: fsm.emit_token = 1; break;
} }
break; break;
case LSINT_extra: case LSINT_extra:
fsm.emit_token = 1; fsm.emit_token = 1;
break; break;
@ -660,7 +660,7 @@ main_fsm(Lex_FSM fsm, unsigned char pp_state, unsigned char c){
fsm.state = LS_error_message; fsm.state = LS_error_message;
if (c == '\n') fsm.emit_token = 1; if (c == '\n') fsm.emit_token = 1;
break; break;
case LSPP_include: case LSPP_include:
switch (fsm.state){ switch (fsm.state){
case LSINC_default: case LSINC_default:
@ -670,21 +670,29 @@ main_fsm(Lex_FSM fsm, unsigned char pp_state, unsigned char c){
default: fsm.state = LSINC_junk; break; default: fsm.state = LSINC_junk; break;
} }
break; break;
case LSINC_quotes: case LSINC_quotes:
if (c == '"') fsm.emit_token = 1; if (c == '"') fsm.emit_token = 1;
else if (c == '\n'){
fsm.emit_token = 1;
fsm.state = LSINC_junk;
}
break; break;
case LSINC_pointy: case LSINC_pointy:
if (c == '>') fsm.emit_token = 1; if (c == '>') fsm.emit_token = 1;
else if (c == '\n'){
fsm.emit_token = 1;
fsm.state = LSINC_junk;
}
break; break;
case LSINC_junk: case LSINC_junk:
if (c == '\n') fsm.emit_token = 1; if (c == '\n') fsm.emit_token = 1;
break; break;
} }
break; break;
default: default:
switch (fsm.state){ switch (fsm.state){
case LS_default: case LS_default:
@ -700,31 +708,31 @@ main_fsm(Lex_FSM fsm, unsigned char pp_state, unsigned char c){
else switch (c){ else switch (c){
case '\'': fsm.state = LS_char; break; case '\'': fsm.state = LS_char; break;
case '"': fsm.state = LS_string; break; case '"': fsm.state = LS_string; break;
case '/': fsm.state = LS_comment_pre; break; case '/': fsm.state = LS_comment_pre; break;
case '.': fsm.state = LS_dot; break; case '.': fsm.state = LS_dot; break;
case '<': fsm.state = LS_less; break; case '<': fsm.state = LS_less; break;
case '>': fsm.state = LS_more; break; case '>': fsm.state = LS_more; break;
case '-': fsm.state = LS_minus; break; case '-': fsm.state = LS_minus; break;
case '&': fsm.state = LS_and; break; case '&': fsm.state = LS_and; break;
case '|': fsm.state = LS_or; break; case '|': fsm.state = LS_or; break;
case '+': fsm.state = LS_plus; break; case '+': fsm.state = LS_plus; break;
case ':': fsm.state = LS_colon; break; case ':': fsm.state = LS_colon; break;
case '*': fsm.state = LS_star; break; case '*': fsm.state = LS_star; break;
case '%': fsm.state = LS_modulo; break; case '%': fsm.state = LS_modulo; break;
case '^': fsm.state = LS_caret; break; case '^': fsm.state = LS_caret; break;
case '=': fsm.state = LS_eq; break; case '=': fsm.state = LS_eq; break;
case '!': fsm.state = LS_bang; break; case '!': fsm.state = LS_bang; break;
case '#': case '#':
if (pp_state == LSPP_default){ if (pp_state == LSPP_default){
fsm.state = LS_pp; fsm.state = LS_pp;
@ -734,52 +742,52 @@ main_fsm(Lex_FSM fsm, unsigned char pp_state, unsigned char c){
fsm.state = LS_pound; fsm.state = LS_pound;
} }
break; break;
#define OperCase(op,type) case op: fsm.emit_token = 1; break; #define OperCase(op,type) case op: fsm.emit_token = 1; break;
OperCase('{', CPP_TOKEN_BRACE_OPEN); OperCase('{', CPP_TOKEN_BRACE_OPEN);
OperCase('}', CPP_TOKEN_BRACE_CLOSE); OperCase('}', CPP_TOKEN_BRACE_CLOSE);
OperCase('[', CPP_TOKEN_BRACKET_OPEN); OperCase('[', CPP_TOKEN_BRACKET_OPEN);
OperCase(']', CPP_TOKEN_BRACKET_CLOSE); OperCase(']', CPP_TOKEN_BRACKET_CLOSE);
OperCase('(', CPP_TOKEN_PARENTHESE_OPEN); OperCase('(', CPP_TOKEN_PARENTHESE_OPEN);
OperCase(')', CPP_TOKEN_PARENTHESE_CLOSE); OperCase(')', CPP_TOKEN_PARENTHESE_CLOSE);
OperCase('~', CPP_TOKEN_TILDE); OperCase('~', CPP_TOKEN_TILDE);
OperCase(',', CPP_TOKEN_COMMA); OperCase(',', CPP_TOKEN_COMMA);
OperCase(';', CPP_TOKEN_SEMICOLON); OperCase(';', CPP_TOKEN_SEMICOLON);
OperCase('?', CPP_TOKEN_TERNARY_QMARK); OperCase('?', CPP_TOKEN_TERNARY_QMARK);
OperCase('@', CPP_TOKEN_JUNK); OperCase('@', CPP_TOKEN_JUNK);
OperCase('$', CPP_TOKEN_JUNK); OperCase('$', CPP_TOKEN_JUNK);
OperCase('\\', CPP_TOKEN_JUNK); OperCase('\\', CPP_TOKEN_JUNK);
#undef OperCase #undef OperCase
} }
break; break;
case LS_identifier: case LS_identifier:
if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')){ if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')){
fsm.emit_token = 1; fsm.emit_token = 1;
} }
break; break;
case LS_pound: case LS_pound:
switch (c){ switch (c){
case '#': fsm.emit_token = 1; break; case '#': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break; default: fsm.emit_token = 1; break;
} }
break; break;
case LS_pp:break; case LS_pp:break;
case LS_char: case LS_char:
case LS_char_multiline: case LS_char_multiline:
switch(c){ switch(c){
case '\'': fsm.emit_token = 1; break; case '\n': case '\'': fsm.emit_token = 1; break;
case '\\': fsm.state = LS_char_slashed; break; case '\\': fsm.state = LS_char_slashed; break;
} }
break; break;
case LS_char_slashed: case LS_char_slashed:
switch (c){ switch (c){
case '\r': case '\f': case '\v': break; case '\r': case '\f': case '\v': break;
@ -787,15 +795,15 @@ main_fsm(Lex_FSM fsm, unsigned char pp_state, unsigned char c){
default: fsm.state = LS_char; break; default: fsm.state = LS_char; break;
} }
break; break;
case LS_string: case LS_string:
case LS_string_multiline: case LS_string_multiline:
switch(c){ switch(c){
case '\"': fsm.emit_token = 1; break; case '\n': case '\"': fsm.emit_token = 1; break;
case '\\': fsm.state = LS_string_slashed; break; case '\\': fsm.state = LS_string_slashed; break;
} }
break; break;
case LS_string_slashed: case LS_string_slashed:
switch (c){ switch (c){
case '\r': case '\f': case '\v': break; case '\r': case '\f': case '\v': break;
@ -803,7 +811,7 @@ main_fsm(Lex_FSM fsm, unsigned char pp_state, unsigned char c){
default: fsm.state = LS_string; break; default: fsm.state = LS_string; break;
} }
break; break;
case LS_number: case LS_number:
if (c >= '0' && c <= '9'){ if (c >= '0' && c <= '9'){
fsm.state = LS_number; fsm.state = LS_number;
@ -815,7 +823,7 @@ main_fsm(Lex_FSM fsm, unsigned char pp_state, unsigned char c){
} }
} }
break; break;
case LS_number0: case LS_number0:
if (c >= '0' && c <= '9'){ if (c >= '0' && c <= '9'){
fsm.state = LS_number; fsm.state = LS_number;
@ -830,7 +838,7 @@ main_fsm(Lex_FSM fsm, unsigned char pp_state, unsigned char c){
fsm.emit_token = 1; fsm.emit_token = 1;
} }
break; break;
case LS_float: case LS_float:
if (!(c >= '0' && c <= '9')){ if (!(c >= '0' && c <= '9')){
switch (c){ switch (c){
@ -839,7 +847,7 @@ main_fsm(Lex_FSM fsm, unsigned char pp_state, unsigned char c){
} }
} }
break; break;
case LS_crazy_float0: case LS_crazy_float0:
{ {
if ((c >= '0' && c <= '9') || c == '-'){ if ((c >= '0' && c <= '9') || c == '-'){
@ -850,7 +858,7 @@ main_fsm(Lex_FSM fsm, unsigned char pp_state, unsigned char c){
} }
} }
break; break;
case LS_crazy_float1: case LS_crazy_float1:
{ {
if (!(c >= '0' && c <= '9')){ if (!(c >= '0' && c <= '9')){
@ -858,13 +866,13 @@ main_fsm(Lex_FSM fsm, unsigned char pp_state, unsigned char c){
} }
} }
break; break;
case LS_hex: case LS_hex:
if (!(c >= '0' && c <= '9' || c >= 'a' && c <= 'f' || c >= 'A' && c <= 'F')){ if (!(c >= '0' && c <= '9' || c >= 'a' && c <= 'f' || c >= 'A' && c <= 'F')){
fsm.emit_token = 1; fsm.emit_token = 1;
} }
break; break;
case LS_dot: case LS_dot:
if (c >= '0' && c <= '9'){ if (c >= '0' && c <= '9'){
fsm.state = LS_float; fsm.state = LS_float;
@ -876,9 +884,9 @@ main_fsm(Lex_FSM fsm, unsigned char pp_state, unsigned char c){
default: fsm.emit_token = 1; break; default: fsm.emit_token = 1; break;
} }
break; break;
case LS_ellipsis: fsm.emit_token = 1; break; case LS_ellipsis: fsm.emit_token = 1; break;
case LS_less: case LS_less:
switch (c){ switch (c){
case '<': fsm.state = LS_less_less; break; case '<': fsm.state = LS_less_less; break;
@ -886,14 +894,14 @@ main_fsm(Lex_FSM fsm, unsigned char pp_state, unsigned char c){
default: fsm.emit_token = 1; break; default: fsm.emit_token = 1; break;
} }
break; break;
case LS_less_less: case LS_less_less:
switch (c){ switch (c){
case '=': fsm.emit_token = 1; break; case '=': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break; default: fsm.emit_token = 1; break;
} }
break; break;
case LS_more: case LS_more:
switch (c){ switch (c){
case '>': fsm.state = LS_more_more; break; case '>': fsm.state = LS_more_more; break;
@ -901,14 +909,14 @@ main_fsm(Lex_FSM fsm, unsigned char pp_state, unsigned char c){
default: fsm.emit_token = 1; break; default: fsm.emit_token = 1; break;
} }
break; break;
case LS_more_more: case LS_more_more:
switch (c){ switch (c){
case '=': fsm.emit_token = 1; break; case '=': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break; default: fsm.emit_token = 1; break;
} }
break; break;
case LS_comment_pre: case LS_comment_pre:
switch (c){ switch (c){
case '/': fsm.state = LS_comment; break; case '/': fsm.state = LS_comment; break;
@ -917,27 +925,27 @@ main_fsm(Lex_FSM fsm, unsigned char pp_state, unsigned char c){
default: fsm.emit_token = 1; break; default: fsm.emit_token = 1; break;
} }
break; break;
case LS_comment: case LS_comment:
switch (c){ switch (c){
case '\\': fsm.state = LS_comment_slashed; break; case '\\': fsm.state = LS_comment_slashed; break;
case '\n': fsm.emit_token = 1; break; case '\n': fsm.emit_token = 1; break;
} }
break; break;
case LS_comment_slashed: case LS_comment_slashed:
switch (c){ switch (c){
case '\r': case '\f': case '\v': break; case '\r': case '\f': case '\v': break;
default: fsm.state = LS_comment; break; default: fsm.state = LS_comment; break;
} }
break; break;
case LS_comment_block: case LS_comment_block:
switch (c){ switch (c){
case '*': fsm.state = LS_comment_block_ending; break; case '*': fsm.state = LS_comment_block_ending; break;
} }
break; break;
case LS_comment_block_ending: case LS_comment_block_ending:
switch (c){ switch (c){
case '*': fsm.state = LS_comment_block_ending; break; case '*': fsm.state = LS_comment_block_ending; break;
@ -945,7 +953,7 @@ main_fsm(Lex_FSM fsm, unsigned char pp_state, unsigned char c){
default: fsm.state = LS_comment_block; break; default: fsm.state = LS_comment_block; break;
} }
break; break;
case LS_minus: case LS_minus:
switch (c){ switch (c){
case '>': fsm.state = LS_arrow; break; case '>': fsm.state = LS_arrow; break;
@ -954,14 +962,14 @@ main_fsm(Lex_FSM fsm, unsigned char pp_state, unsigned char c){
default: fsm.emit_token = 1; break; default: fsm.emit_token = 1; break;
} }
break; break;
case LS_arrow: case LS_arrow:
switch (c){ switch (c){
case '*': fsm.emit_token = 1; break; case '*': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break; default: fsm.emit_token = 1; break;
} }
break; break;
case LS_and: case LS_and:
switch (c){ switch (c){
case '&': fsm.emit_token = 1; break; case '&': fsm.emit_token = 1; break;
@ -969,7 +977,7 @@ main_fsm(Lex_FSM fsm, unsigned char pp_state, unsigned char c){
default: fsm.emit_token = 1; break; default: fsm.emit_token = 1; break;
} }
break; break;
case LS_or: case LS_or:
switch (c){ switch (c){
case '|': fsm.emit_token = 1; break; case '|': fsm.emit_token = 1; break;
@ -977,7 +985,7 @@ main_fsm(Lex_FSM fsm, unsigned char pp_state, unsigned char c){
default: fsm.emit_token = 1; break; default: fsm.emit_token = 1; break;
} }
break; break;
case LS_plus: case LS_plus:
switch (c){ switch (c){
case '+': fsm.emit_token = 1; break; case '+': fsm.emit_token = 1; break;
@ -985,42 +993,42 @@ main_fsm(Lex_FSM fsm, unsigned char pp_state, unsigned char c){
default: fsm.emit_token = 1; break; default: fsm.emit_token = 1; break;
} }
break; break;
case LS_colon: case LS_colon:
switch (c){ switch (c){
case ':': fsm.emit_token = 1; break; case ':': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break; default: fsm.emit_token = 1; break;
} }
break; break;
case LS_star: case LS_star:
switch (c){ switch (c){
case '=': fsm.emit_token = 1; break; case '=': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break; default: fsm.emit_token = 1; break;
} }
break; break;
case LS_modulo: case LS_modulo:
switch (c){ switch (c){
case '=': fsm.emit_token = 1; break; case '=': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break; default: fsm.emit_token = 1; break;
} }
break; break;
case LS_caret: case LS_caret:
switch (c){ switch (c){
case '=': fsm.emit_token = 1; break; case '=': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break; default: fsm.emit_token = 1; break;
} }
break; break;
case LS_eq: case LS_eq:
switch (c){ switch (c){
case '=': fsm.emit_token = 1; break; case '=': fsm.emit_token = 1; break;
default: fsm.emit_token = 1; break; default: fsm.emit_token = 1; break;
} }
break; break;
case LS_bang: case LS_bang:
switch (c){ switch (c){
case '=': fsm.emit_token = 1; break; case '=': fsm.emit_token = 1; break;