diff --git a/4ed_app_target.cpp b/4ed_app_target.cpp index 45738325..d3f4b609 100644 --- a/4ed_app_target.cpp +++ b/4ed_app_target.cpp @@ -29,8 +29,8 @@ #include "4tech_table.cpp" #define FCPP_LEXER_IMPLEMENTATION -#include "test/4cpp_new_lexer.h" -//#include "4cpp_lexer.h" +//#include "test/4cpp_new_lexer.h" +#include "4cpp_lexer.h" #include "4ed_template.cpp" diff --git a/4ed_file_view.cpp b/4ed_file_view.cpp index ebd3e94a..d0a0bb9a 100644 --- a/4ed_file_view.cpp +++ b/4ed_file_view.cpp @@ -807,6 +807,7 @@ Job_Callback_Sig(job_full_lex){ i32 buffer_size = file->state.buffer.size; buffer_size = (buffer_size + 3)&(~3); +#if 0 while (memory->size < buffer_size*2){ system->grow_thread_memory(memory); } @@ -818,8 +819,6 @@ Job_Callback_Sig(job_full_lex){ tokens.max_count = (memory->size - buffer_size) / sizeof(Cpp_Token); tokens.count = 0; -#if 1 - b32 still_lexing = 1; Lex_Data lex = lex_data_init(tb); @@ -855,7 +854,16 @@ Job_Callback_Sig(job_full_lex){ #else - Cpp_Lex_Data status = {0}; + while (memory->size < buffer_size){ + system->grow_thread_memory(memory); + } + + Cpp_Token_Stack tokens; + tokens.tokens = (Cpp_Token*)(char*)memory->data; + tokens.max_count = (memory->size) / sizeof(Cpp_Token); + tokens.count = 0; + + Cpp_Lex_Data status = {}; do{ for (i32 r = 2048; r > 0 && status.pos < cpp_file.size; --r){ @@ -991,9 +999,10 @@ file_relex_parallel(System_Functions *system, relex_space.max_count = state.space_request; relex_space.tokens = push_array(part, Cpp_Token, relex_space.max_count); - char *spare = push_array(part, char, cpp_file.size); +// char *spare = push_array(part, char, cpp_file.size); - if (cpp_relex_nonalloc_main(&state, &relex_space, &relex_end, spare)){ +// if (cpp_relex_nonalloc_main(&state, &relex_space, &relex_end, spare)){ + if (cpp_relex_nonalloc_main(&state, &relex_space, &relex_end)){ inline_lex = 0; } else{ diff --git a/test/4cpp_new_lexer.h b/test/4cpp_new_lexer.h index 57ed4e69..b3bc7fa1 100644 --- a/test/4cpp_new_lexer.h +++ b/test/4cpp_new_lexer.h @@ -632,6 +632,7 @@ cpp_lex_nonalloc(Lex_Data *S_ptr, break; case LS_char: + case LS_char_slashed: S.token.type = CPP_TOKEN_CHARACTER_CONSTANT; S.token.flags = 0; break; @@ -642,6 +643,7 @@ cpp_lex_nonalloc(Lex_Data *S_ptr, break; case LS_string: + case LS_string_slashed: S.token.type = CPP_TOKEN_STRING_CONSTANT; S.token.flags = 0; break; @@ -662,7 +664,10 @@ cpp_lex_nonalloc(Lex_Data *S_ptr, } break; - case LS_comment: case LS_comment_block_ending: + case LS_comment: + case LS_comment_slashed: + case LS_comment_block: + case LS_comment_block_ending: S.token.type = CPP_TOKEN_COMMENT; S.token.flags = 0; pos_update_rule = PUR_unget_whitespace; diff --git a/test/experiment.cpp b/test/experiment.cpp index 9f9e260a..730817c5 100644 --- a/test/experiment.cpp +++ b/test/experiment.cpp @@ -18,13 +18,14 @@ #include "../4cpp_types.h" #include "../4cpp_lexer_types.h" -#define FCPP_LEXER_IMPLEMENTATION -#include "../4cpp_lexer.h" namespace new_lex{ # include "4cpp_new_lexer.h" } +#define FCPP_LEXER_IMPLEMENTATION +#include "../4cpp_lexer.h" + #include #include @@ -403,12 +404,12 @@ show_time(Times t, int repeats, char *type){ int main(){ int repeats = 1; - int verbose_level = 0; - int chunk_start = 32; + int verbose_level = 1; + int chunk_start = 64; int chunk_end = 64; -#define TEST_FILE "parser_test1.cpp" -#define SINGLE_ITEM 0 - int token_limit = 2; +#define TEST_FILE "lexer_test2.cpp" +#define SINGLE_ITEM 1 + int token_limit = 1; int chunks = (chunk_start > 0 && chunk_start <= chunk_end); int c = 0;