mr4th-public
/
4coder
1
Fork 0
Code Issues Releases Activity

a little more work on the new lexer

master
Allen Webster 2016-06-06 12:01:57 -04:00
parent 682d99dacc
commit b62a044fec
4 changed files with 30 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
4ed_app_target.cpp
View File

@ -29,8 +29,8 @@
#include "4tech_table.cpp" #include "4tech_table.cpp"
#define FCPP_LEXER_IMPLEMENTATION #define FCPP_LEXER_IMPLEMENTATION
#include "test/4cpp_new_lexer.h" //#include "test/4cpp_new_lexer.h"
//#include "4cpp_lexer.h" #include "4cpp_lexer.h"
#include "4ed_template.cpp" #include "4ed_template.cpp"

19
4ed_file_view.cpp
View File

@ -807,6 +807,7 @@ Job_Callback_Sig(job_full_lex){
i32 buffer_size = file->state.buffer.size; i32 buffer_size = file->state.buffer.size;
buffer_size = (buffer_size + 3)&(~3); buffer_size = (buffer_size + 3)&(~3);
#if 0
while (memory->size < buffer_size*2){ while (memory->size < buffer_size*2){
system->grow_thread_memory(memory); system->grow_thread_memory(memory);
} }
@ -818,8 +819,6 @@ Job_Callback_Sig(job_full_lex){
tokens.max_count = (memory->size - buffer_size) / sizeof(Cpp_Token); tokens.max_count = (memory->size - buffer_size) / sizeof(Cpp_Token);
tokens.count = 0; tokens.count = 0;
#if 1
b32 still_lexing = 1; b32 still_lexing = 1;
Lex_Data lex = lex_data_init(tb); Lex_Data lex = lex_data_init(tb);
@ -855,7 +854,16 @@ Job_Callback_Sig(job_full_lex){
#else #else
Cpp_Lex_Data status = {0}; while (memory->size < buffer_size){
system->grow_thread_memory(memory);
}
Cpp_Token_Stack tokens;
tokens.tokens = (Cpp_Token*)(char*)memory->data;
tokens.max_count = (memory->size) / sizeof(Cpp_Token);
tokens.count = 0;
Cpp_Lex_Data status = {};
do{ do{
for (i32 r = 2048; r > 0 && status.pos < cpp_file.size; --r){ for (i32 r = 2048; r > 0 && status.pos < cpp_file.size; --r){
@ -991,9 +999,10 @@ file_relex_parallel(System_Functions *system,
relex_space.max_count = state.space_request; relex_space.max_count = state.space_request;
relex_space.tokens = push_array(part, Cpp_Token, relex_space.max_count); relex_space.tokens = push_array(part, Cpp_Token, relex_space.max_count);
char *spare = push_array(part, char, cpp_file.size); // char *spare = push_array(part, char, cpp_file.size);
if (cpp_relex_nonalloc_main(&state, &relex_space, &relex_end, spare)){ // if (cpp_relex_nonalloc_main(&state, &relex_space, &relex_end, spare)){
if (cpp_relex_nonalloc_main(&state, &relex_space, &relex_end)){
inline_lex = 0; inline_lex = 0;
} }
else{ else{

7
test/4cpp_new_lexer.h
View File

@ -632,6 +632,7 @@ cpp_lex_nonalloc(Lex_Data *S_ptr,
break; break;
case LS_char: case LS_char:
case LS_char_slashed:
S.token.type = CPP_TOKEN_CHARACTER_CONSTANT; S.token.type = CPP_TOKEN_CHARACTER_CONSTANT;
S.token.flags = 0; S.token.flags = 0;
break; break;
@ -642,6 +643,7 @@ cpp_lex_nonalloc(Lex_Data *S_ptr,
break; break;
case LS_string: case LS_string:
case LS_string_slashed:
S.token.type = CPP_TOKEN_STRING_CONSTANT; S.token.type = CPP_TOKEN_STRING_CONSTANT;
S.token.flags = 0; S.token.flags = 0;
break; break;
@ -662,7 +664,10 @@ cpp_lex_nonalloc(Lex_Data *S_ptr,
} }
break; break;
case LS_comment: case LS_comment_block_ending: case LS_comment:
case LS_comment_slashed:
case LS_comment_block:
case LS_comment_block_ending:
S.token.type = CPP_TOKEN_COMMENT; S.token.type = CPP_TOKEN_COMMENT;
S.token.flags = 0; S.token.flags = 0;
pos_update_rule = PUR_unget_whitespace; pos_update_rule = PUR_unget_whitespace;

15
test/experiment.cpp
View File

@ -18,13 +18,14 @@
#include "../4cpp_types.h" #include "../4cpp_types.h"
#include "../4cpp_lexer_types.h" #include "../4cpp_lexer_types.h"
#define FCPP_LEXER_IMPLEMENTATION
#include "../4cpp_lexer.h"
namespace new_lex{ namespace new_lex{
# include "4cpp_new_lexer.h" # include "4cpp_new_lexer.h"
} }
#define FCPP_LEXER_IMPLEMENTATION
#include "../4cpp_lexer.h"
#include <windows.h> #include <windows.h>
#include <intrin.h> #include <intrin.h>
@ -403,12 +404,12 @@ show_time(Times t, int repeats, char *type){
int main(){ int main(){
int repeats = 1; int repeats = 1;
int verbose_level = 0; int verbose_level = 1;
int chunk_start = 32; int chunk_start = 64;
int chunk_end = 64; int chunk_end = 64;
#define TEST_FILE "parser_test1.cpp" #define TEST_FILE "lexer_test2.cpp"
#define SINGLE_ITEM 0 #define SINGLE_ITEM 1
int token_limit = 2; int token_limit = 1;
int chunks = (chunk_start > 0 && chunk_start <= chunk_end); int chunks = (chunk_start > 0 && chunk_start <= chunk_end);
int c = 0; int c = 0;