mr4th-public
/
4coder
1
Fork 0
Code Issues Releases Activity

eliminate the Cpp_File that drove me crazy

master
Allen Webster 2016-07-16 14:40:26 -04:00
parent 89b6fe663d
commit 7ec039edfd
9 changed files with 330 additions and 357 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
4coder_default_include.cpp
View File

@ -1819,7 +1819,7 @@ CUSTOM_COMMAND_SIG(open_all_code){
// was originally, so that new appends overwrite old ones. // was originally, so that new appends overwrite old ones.
dir.size = dir_size; dir.size = dir_size;
append(&dir, info->filename); append(&dir, info->filename);
exec_command(app, cmdid_interactive_open); app->create_buffer(app, dir.str, dir.size, 0);
} }
} }
} }

342
4cpp_lexer.h
View File

@ -78,14 +78,6 @@ struct Cpp_Read_Result{
char has_result; char has_result;
}; };
Cpp_File
data_as_cpp_file(Data data){
Cpp_File result;
result.data = (char*)data.data;
result.size = data.size;
return(result);
}
// TODO(allen): revisit this keyword data declaration system // TODO(allen): revisit this keyword data declaration system
struct String_And_Flag{ struct String_And_Flag{
char *str; char *str;
@ -125,27 +117,27 @@ is_keyword(Cpp_Token_Type type){
return (type >= CPP_TOKEN_KEY_TYPE && type <= CPP_TOKEN_KEY_OTHER); return (type >= CPP_TOKEN_KEY_TYPE && type <= CPP_TOKEN_KEY_OTHER);
} }
FCPP_LINK Sub_Match_List_Result sub_match_list(Cpp_File file, int pos, String_List list, int sub_size); FCPP_LINK Sub_Match_List_Result sub_match_list(char *data, int size, int pos, String_List list, int sub_size);
FCPP_LINK Seek_Result seek_unescaped_eol(char *data, int size, int pos); FCPP_LINK Seek_Result seek_unescaped_eol(char *data, int size, int pos);
FCPP_LINK Seek_Result seek_unescaped_delim(char *data, int size, int pos, char delim); FCPP_LINK Seek_Result seek_unescaped_delim(char *data, int size, int pos, char delim);
FCPP_LINK Seek_Result seek_block_comment_end(char *data, int size, int pos); FCPP_LINK Seek_Result seek_block_comment_end(char *data, int size, int pos);
FCPP_LINK Cpp_Read_Result cpp_read_whitespace(Cpp_File file, int pos); FCPP_LINK Cpp_Read_Result cpp_read_whitespace(char *data, int size, int pos);
FCPP_LINK Cpp_Read_Result cpp_read_junk_line(Cpp_File file, int pos); FCPP_LINK Cpp_Read_Result cpp_read_junk_line(char *data, int size, int pos);
FCPP_LINK Cpp_Read_Result cpp_read_operator(Cpp_File file, int pos); FCPP_LINK Cpp_Read_Result cpp_read_operator(char *data, int size, int pos);
FCPP_LINK Cpp_Read_Result cpp_read_pp_operator(Cpp_File file, int pos); FCPP_LINK Cpp_Read_Result cpp_read_pp_operator(char *data, int size, int pos);
FCPP_LINK Cpp_Read_Result cpp_read_alpha_numeric(Cpp_File file, int pos, bool in_if_body); FCPP_LINK Cpp_Read_Result cpp_read_alpha_numeric(char *data, int size, int pos, bool in_if_body);
inline Cpp_Read_Result cpp_read_alpha_numeric(Cpp_File file, int pos) { return cpp_read_alpha_numeric(file, pos, 0); } inline Cpp_Read_Result cpp_read_alpha_numeric(char *data, int size, int pos) { return cpp_read_alpha_numeric(data, size, pos, 0); }
FCPP_LINK Cpp_Read_Result cpp_read_number(Cpp_File file, int pos); FCPP_LINK Cpp_Read_Result cpp_read_number(char *data, int size, int pos);
FCPP_LINK Cpp_Read_Result cpp_read_string_litteral(Cpp_File file, int pos); FCPP_LINK Cpp_Read_Result cpp_read_string_litteral(char *data, int size, int pos);
FCPP_LINK Cpp_Read_Result cpp_read_character_litteral(Cpp_File file, int pos); FCPP_LINK Cpp_Read_Result cpp_read_character_litteral(char *data, int size, int pos);
FCPP_LINK Cpp_Read_Result cpp_read_line_comment(Cpp_File file, int pos); FCPP_LINK Cpp_Read_Result cpp_read_line_comment(char *data, int size, int pos);
FCPP_LINK Cpp_Read_Result cpp_read_block_comment(Cpp_File file, int pos); FCPP_LINK Cpp_Read_Result cpp_read_block_comment(char *data, int size, int pos);
FCPP_LINK Cpp_Read_Result cpp_read_preprocessor(Cpp_File file, int pos); FCPP_LINK Cpp_Read_Result cpp_read_preprocessor(char *data, int size, int pos);
FCPP_LINK Cpp_Read_Result cpp_read_pp_include_file(Cpp_File file, int pos); FCPP_LINK Cpp_Read_Result cpp_read_pp_include_file(char *data, int size, int pos);
FCPP_LINK Cpp_Read_Result cpp_read_pp_default_mode(Cpp_File file, int pos, bool in_if_body); FCPP_LINK Cpp_Read_Result cpp_read_pp_default_mode(char *data, int size, int pos, bool in_if_body);
inline Cpp_Read_Result cpp_read_pp_default_mode(Cpp_File file, int pos) { return cpp_read_pp_default_mode(file, pos, 0); } inline Cpp_Read_Result cpp_read_pp_default_mode(char *data, int size, int pos) { return cpp_read_pp_default_mode(data, size, pos, 0); }
FCPP_LINK Cpp_Token_Merge cpp_attempt_token_merge(Cpp_Token prev, Cpp_Token next); FCPP_LINK Cpp_Token_Merge cpp_attempt_token_merge(Cpp_Token prev, Cpp_Token next);
@ -154,18 +146,18 @@ FCPP_LINK bool cpp_push_token_nonalloc(Cpp_Token_Stack *stack, Cpp_Token token);
inline Cpp_Lex_Data cpp_lex_data_zero() { Cpp_Lex_Data data = {(Cpp_Preprocessor_State)0}; return(data); } inline Cpp_Lex_Data cpp_lex_data_zero() { Cpp_Lex_Data data = {(Cpp_Preprocessor_State)0}; return(data); }
FCPP_LINK Cpp_Read_Result cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex); FCPP_LINK Cpp_Read_Result cpp_lex_step(char *data, int size, Cpp_Lex_Data *lex);
FCPP_LINK int cpp_lex_file_token_count(Cpp_File file); FCPP_LINK int cpp_lex_file_token_count(char *data, int size);
FCPP_LINK Cpp_Lex_Data cpp_lex_file_nonalloc(Cpp_File file, Cpp_Token_Stack *stack, Cpp_Lex_Data data); FCPP_LINK Cpp_Lex_Data cpp_lex_file_nonalloc(char *data, int size, Cpp_Token_Stack *stack, Cpp_Lex_Data lex_data);
inline Cpp_Lex_Data cpp_lex_file_nonalloc(Cpp_File file, Cpp_Token_Stack *stack) { return cpp_lex_file_nonalloc(file, stack, cpp_lex_data_zero()); } inline Cpp_Lex_Data cpp_lex_file_nonalloc(char *data, int size, Cpp_Token_Stack *stack) { return cpp_lex_file_nonalloc(data, size, stack, cpp_lex_data_zero()); }
FCPP_LINK Cpp_Get_Token_Result cpp_get_token(Cpp_Token_Stack *stack, int pos); FCPP_LINK Cpp_Get_Token_Result cpp_get_token(Cpp_Token_Stack *stack, int pos);
FCPP_LINK int cpp_get_end_token(Cpp_Token_Stack *stack, int end); FCPP_LINK int cpp_get_end_token(Cpp_Token_Stack *stack, int end);
FCPP_LINK void cpp_shift_token_starts(Cpp_Token_Stack *stack, int from_token, int amount); FCPP_LINK void cpp_shift_token_starts(Cpp_Token_Stack *stack, int from_token, int amount);
FCPP_LINK Cpp_Relex_State cpp_relex_nonalloc_start(Cpp_File file, Cpp_Token_Stack *stack, int start, int end, int amount, int tolerance); FCPP_LINK Cpp_Relex_State cpp_relex_nonalloc_start(char *data, int size, Cpp_Token_Stack *stack, int start, int end, int amount, int tolerance);
FCPP_LINK bool cpp_relex_nonalloc_main(Cpp_Relex_State state, Cpp_Token_Stack *stack); FCPP_LINK bool cpp_relex_nonalloc_main(Cpp_Relex_State state, Cpp_Token_Stack *stack);
#ifndef FCPP_FORBID_MALLOC #ifndef FCPP_FORBID_MALLOC
@ -174,10 +166,10 @@ FCPP_LINK void cpp_free_token_stack(Cpp_Token_Stack stack);
FCPP_LINK void cpp_resize_token_stack(Cpp_Token_Stack *stack, int new_max); FCPP_LINK void cpp_resize_token_stack(Cpp_Token_Stack *stack, int new_max);
FCPP_LINK void cpp_push_token(Cpp_Token_Stack *stack, Cpp_Token token); FCPP_LINK void cpp_push_token(Cpp_Token_Stack *stack, Cpp_Token token);
FCPP_LINK void cpp_lex_file(Cpp_File file, Cpp_Token_Stack *stack); FCPP_LINK void cpp_lex_file(char *data, int size, Cpp_Token_Stack *stack);
FCPP_LINK bool cpp_relex_file_limited(Cpp_File file, Cpp_Token_Stack *stack, int start_i, int end_i, int amount, int extra_tolerance); FCPP_LINK bool cpp_relex_file_limited(char *data, int size, Cpp_Token_Stack *stack, int start_i, int end_i, int amount, int extra_tolerance);
inline void cpp_relex_file(Cpp_File file, Cpp_Token_Stack *stack, int start_i, int end_i, int amount) inline void cpp_relex_file(char *data, int size, Cpp_Token_Stack *stack, int start_i, int end_i, int amount)
{ cpp_relex_file_limited(file, stack, start_i, end_i, amount, -1); } { cpp_relex_file_limited(data, size, stack, start_i, end_i, amount, -1); }
#endif #endif
#define FCPP_STRING_LIST(x) {x, FCPP_COUNT(x)} #define FCPP_STRING_LIST(x) {x, FCPP_COUNT(x)}
@ -393,7 +385,7 @@ FCPP_GLOBAL String_List preprops = FCPP_STRING_LIST(preprop_strings);
#define _TentativeAssert FCPP_ASSERT #define _TentativeAssert FCPP_ASSERT
FCPP_LINK Sub_Match_List_Result FCPP_LINK Sub_Match_List_Result
sub_match_list(Cpp_File file, int pos, String_List list, int sub_size){ sub_match_list(char *data, int size, int pos, String_List list, int sub_size){
Sub_Match_List_Result result; Sub_Match_List_Result result;
String str_main; String str_main;
char *str_check; char *str_check;
@ -401,7 +393,7 @@ sub_match_list(Cpp_File file, int pos, String_List list, int sub_size){
result.index = -1; result.index = -1;
result.new_pos = pos; result.new_pos = pos;
str_main = make_string(file.data + pos, file.size - pos); str_main = make_string(data + pos, size - pos);
if (sub_size > 0){ if (sub_size > 0){
str_main = substr(str_main, 0, sub_size); str_main = substr(str_main, 0, sub_size);
for (i = 0; i < list.count; ++i){ for (i = 0; i < list.count; ++i){
@ -505,11 +497,11 @@ seek_block_comment_end(char *data, int size, int pos){
} }
FCPP_LINK Cpp_Read_Result FCPP_LINK Cpp_Read_Result
cpp_read_whitespace(Cpp_File file, int pos){ cpp_read_whitespace(char *data, int size, int pos){
Cpp_Read_Result result = {}; Cpp_Read_Result result = {};
while (pos < file.size && char_is_whitespace(file.data[pos])){ while (pos < size && char_is_whitespace(data[pos])){
if (file.data[pos] == '\n'){ if (data[pos] == '\n'){
result.newline = 1; result.newline = 1;
} }
++pos; ++pos;
@ -521,16 +513,16 @@ cpp_read_whitespace(Cpp_File file, int pos){
} }
FCPP_LINK Cpp_Read_Result FCPP_LINK Cpp_Read_Result
cpp_read_junk_line(Cpp_File file, int pos){ cpp_read_junk_line(char *data, int size, int pos){
Cpp_Read_Result result = {}; Cpp_Read_Result result = {};
result.token.start = pos; result.token.start = pos;
result.token.type = CPP_TOKEN_JUNK; result.token.type = CPP_TOKEN_JUNK;
bool comment_end = 0; bool comment_end = 0;
while (pos < file.size && file.data[pos] != '\n'){ while (pos < size && data[pos] != '\n'){
if (file.data[pos] == '/' && pos + 1 < file.size){ if (data[pos] == '/' && pos + 1 < size){
if (file.data[pos + 1] == '/' || if (data[pos + 1] == '/' ||
file.data[pos + 1] == '*'){ data[pos + 1] == '*'){
comment_end = 1; comment_end = 1;
break; break;
} }
@ -543,10 +535,10 @@ cpp_read_junk_line(Cpp_File file, int pos){
result.token.size = pos - result.token.start; result.token.size = pos - result.token.start;
} }
else{ else{
while (pos > 0 && file.data[pos - 1] == '\r'){ while (pos > 0 && data[pos - 1] == '\r'){
--pos; --pos;
} }
if (pos > 0 && file.data[pos - 1] == '\\'){ if (pos > 0 && data[pos - 1] == '\\'){
--pos; --pos;
} }
result.pos = pos; result.pos = pos;
@ -557,13 +549,13 @@ cpp_read_junk_line(Cpp_File file, int pos){
} }
FCPP_LINK Cpp_Read_Result FCPP_LINK Cpp_Read_Result
cpp_read_operator(Cpp_File file, int pos){ cpp_read_operator(char *data, int size, int pos){
Cpp_Read_Result result = {}; Cpp_Read_Result result = {};
result.pos = pos; result.pos = pos;
result.token.start = pos; result.token.start = pos;
Sub_Match_List_Result match; Sub_Match_List_Result match;
match = sub_match_list(file, result.token.start, ops, -1); match = sub_match_list(data, size, result.token.start, ops, -1);
if (match.index != -1){ if (match.index != -1){
result.pos = match.new_pos; result.pos = match.new_pos;
@ -581,13 +573,13 @@ cpp_read_operator(Cpp_File file, int pos){
} }
FCPP_LINK Cpp_Read_Result FCPP_LINK Cpp_Read_Result
cpp_read_pp_operator(Cpp_File file, int pos){ cpp_read_pp_operator(char *data, int size, int pos){
Cpp_Read_Result result = {}; Cpp_Read_Result result = {};
result.pos = pos; result.pos = pos;
result.token.start = pos; result.token.start = pos;
Sub_Match_List_Result match; Sub_Match_List_Result match;
match = sub_match_list(file, result.token.start, pp_ops, -1); match = sub_match_list(data, size, result.token.start, pp_ops, -1);
_Assert(match.index != -1); _Assert(match.index != -1);
result.pos = match.new_pos; result.pos = match.new_pos;
@ -598,13 +590,13 @@ cpp_read_pp_operator(Cpp_File file, int pos){
} }
FCPP_LINK Cpp_Read_Result FCPP_LINK Cpp_Read_Result
cpp_read_alpha_numeric(Cpp_File file, int pos, bool in_if_body){ cpp_read_alpha_numeric(char *data, int size, int pos, bool in_if_body){
Cpp_Read_Result result = {}; Cpp_Read_Result result = {};
result.pos = pos; result.pos = pos;
result.token.start = pos; result.token.start = pos;
while (result.pos < file.size && while (result.pos < size &&
char_is_alpha_numeric(file.data[result.pos])){ char_is_alpha_numeric(data[result.pos])){
++result.pos; ++result.pos;
} }
@ -614,7 +606,7 @@ cpp_read_alpha_numeric(Cpp_File file, int pos, bool in_if_body){
if (in_if_body){ if (in_if_body){
String word; String word;
word.size = result.token.size; word.size = result.token.size;
word.str = file.data + result.token.start; word.str = data + result.token.start;
if (match(word, "defined")){ if (match(word, "defined")){
result.token.type = CPP_TOKEN_DEFINED; result.token.type = CPP_TOKEN_DEFINED;
result.token.flags |= CPP_TFLAG_IS_OPERATOR; result.token.flags |= CPP_TFLAG_IS_OPERATOR;
@ -624,14 +616,14 @@ cpp_read_alpha_numeric(Cpp_File file, int pos, bool in_if_body){
if (result.token.type == CPP_TOKEN_JUNK){ if (result.token.type == CPP_TOKEN_JUNK){
Sub_Match_List_Result match; Sub_Match_List_Result match;
match = sub_match_list(file, result.token.start, bool_lits, result.token.size); match = sub_match_list(data, size, result.token.start, bool_lits, result.token.size);
if (match.index != -1){ if (match.index != -1){
result.token.type = CPP_TOKEN_BOOLEAN_CONSTANT; result.token.type = CPP_TOKEN_BOOLEAN_CONSTANT;
result.token.flags |= CPP_TFLAG_IS_KEYWORD; result.token.flags |= CPP_TFLAG_IS_KEYWORD;
} }
else{ else{
match = sub_match_list(file, result.token.start, keywords, result.token.size); match = sub_match_list(data, size, result.token.start, keywords, result.token.size);
if (match.index != -1){ if (match.index != -1){
String_And_Flag data = keywords.data[match.index]; String_And_Flag data = keywords.data[match.index];
@ -648,7 +640,7 @@ cpp_read_alpha_numeric(Cpp_File file, int pos, bool in_if_body){
} }
FCPP_LINK Cpp_Read_Result FCPP_LINK Cpp_Read_Result
cpp_read_number(Cpp_File file, int pos){ cpp_read_number(char *data, int size, int pos){
Cpp_Read_Result result = {}; Cpp_Read_Result result = {};
result.pos = pos; result.pos = pos;
result.token.start = pos; result.token.start = pos;
@ -659,9 +651,9 @@ cpp_read_number(Cpp_File file, int pos){
bool is_hex = 0; bool is_hex = 0;
bool is_zero = 0; bool is_zero = 0;
if (file.data[pos] == '0'){ if (data[pos] == '0'){
if (pos+1 < file.size){ if (pos+1 < size){
char next = file.data[pos+1]; char next = data[pos+1];
if (next == 'x'){ if (next == 'x'){
is_hex = 1; is_hex = 1;
is_integer = 1; is_integer = 1;
@ -684,7 +676,7 @@ cpp_read_number(Cpp_File file, int pos){
is_integer = 1; is_integer = 1;
} }
} }
else if (file.data[pos] == '.'){ else if (data[pos] == '.'){
is_float = 1; is_float = 1;
} }
@ -696,20 +688,20 @@ cpp_read_number(Cpp_File file, int pos){
char character; char character;
do{ do{
++result.pos; ++result.pos;
if (result.pos >= file.size){ if (result.pos >= size){
break; break;
} }
character = file.data[result.pos]; character = data[result.pos];
} while(char_is_hex(character)); } while(char_is_hex(character));
} }
else if (is_oct){ else if (is_oct){
char character; char character;
do{ do{
++result.pos; ++result.pos;
if (result.pos >= file.size){ if (result.pos >= size){
break; break;
} }
character = file.data[result.pos]; character = data[result.pos];
}while(char_is_numeric(character)); }while(char_is_numeric(character));
} }
else{ else{
@ -718,11 +710,11 @@ cpp_read_number(Cpp_File file, int pos){
while (1){ while (1){
++result.pos; ++result.pos;
if (result.pos >= file.size){ if (result.pos >= size){
break; break;
} }
bool is_good = 0; bool is_good = 0;
char character = file.data[result.pos]; char character = data[result.pos];
if (character >= '0' && character <= '9'){ if (character >= '0' && character <= '9'){
is_good = 1; is_good = 1;
} }
@ -744,11 +736,11 @@ cpp_read_number(Cpp_File file, int pos){
while (1){ while (1){
++result.pos; ++result.pos;
if (result.pos >= file.size){ if (result.pos >= size){
break; break;
} }
is_good = 0; is_good = 0;
character = file.data[result.pos]; character = data[result.pos];
if (character >= '0' && character <= '9'){ if (character >= '0' && character <= '9'){
is_good = 1; is_good = 1;
} }
@ -771,7 +763,7 @@ cpp_read_number(Cpp_File file, int pos){
if (is_integer){ if (is_integer){
Sub_Match_List_Result match = Sub_Match_List_Result match =
sub_match_list(file, result.pos, int_sufs, -1); sub_match_list(data, size, result.pos, int_sufs, -1);
if (match.index != -1){ if (match.index != -1){
result.pos = match.new_pos; result.pos = match.new_pos;
} }
@ -780,7 +772,7 @@ cpp_read_number(Cpp_File file, int pos){
} }
else if (is_float){ else if (is_float){
Sub_Match_List_Result match = Sub_Match_List_Result match =
sub_match_list(file, result.pos, float_sufs, -1); sub_match_list(data, size, result.pos, float_sufs, -1);
if (match.index != -1){ if (match.index != -1){
result.pos = match.new_pos; result.pos = match.new_pos;
} }
@ -795,12 +787,12 @@ cpp_read_number(Cpp_File file, int pos){
} }
FCPP_LINK Cpp_Read_Result FCPP_LINK Cpp_Read_Result
cpp_read_string_litteral(Cpp_File file, int pos){ cpp_read_string_litteral(char *data, int size, int pos){
Cpp_Read_Result result = {}; Cpp_Read_Result result = {};
result.token.start = pos; result.token.start = pos;
_Assert(file.data[pos] == '"'); _Assert(data[pos] == '"');
Seek_Result seek = seek_unescaped_delim(file.data, file.size, pos, '"'); Seek_Result seek = seek_unescaped_delim(data, size, pos, '"');
pos = seek.pos; pos = seek.pos;
if (seek.new_line){ if (seek.new_line){
result.token.flags |= CPP_TFLAG_MULTILINE; result.token.flags |= CPP_TFLAG_MULTILINE;
@ -814,12 +806,12 @@ cpp_read_string_litteral(Cpp_File file, int pos){
} }
FCPP_LINK Cpp_Read_Result FCPP_LINK Cpp_Read_Result
cpp_read_character_litteral(Cpp_File file, int pos){ cpp_read_character_litteral(char *data, int size, int pos){
Cpp_Read_Result result = {}; Cpp_Read_Result result = {};
result.token.start = pos; result.token.start = pos;
_Assert(file.data[pos] == '\''); _Assert(data[pos] == '\'');
Seek_Result seek = seek_unescaped_delim(file.data, file.size, pos, '\''); Seek_Result seek = seek_unescaped_delim(data, size, pos, '\'');
pos = seek.pos; pos = seek.pos;
if (seek.new_line){ if (seek.new_line){
result.token.flags |= CPP_TFLAG_MULTILINE; result.token.flags |= CPP_TFLAG_MULTILINE;
@ -833,31 +825,31 @@ cpp_read_character_litteral(Cpp_File file, int pos){
} }
FCPP_LINK Cpp_Read_Result FCPP_LINK Cpp_Read_Result
cpp_read_line_comment(Cpp_File file, int pos){ cpp_read_line_comment(char *data, int size, int pos){
Cpp_Read_Result result = {}; Cpp_Read_Result result = {};
result.token.start = pos; result.token.start = pos;
_Assert(file.data[pos] == '/' && file.data[pos + 1] == '/'); _Assert(data[pos] == '/' && data[pos + 1] == '/');
pos += 2; pos += 2;
while (pos < file.size){ while (pos < size){
if (file.data[pos] == '\n'){ if (data[pos] == '\n'){
break; break;
} }
if (file.data[pos] == '\\'){ if (data[pos] == '\\'){
if (pos + 1 < file.size && if (pos + 1 < size &&
file.data[pos + 1] == '\n'){ data[pos + 1] == '\n'){
++pos; ++pos;
} }
else if (pos + 2 < file.size && else if (pos + 2 < size &&
file.data[pos + 1] == '\r' && data[pos + 1] == '\r' &&
file.data[pos + 2] == '\n'){ data[pos + 2] == '\n'){
pos += 2; pos += 2;
} }
} }
++pos; ++pos;
} }
if (pos > 0 && file.data[pos-1] == '\r'){ if (pos > 0 && data[pos-1] == '\r'){
--pos; --pos;
} }
result.token.size = pos - result.token.start; result.token.size = pos - result.token.start;
@ -867,16 +859,16 @@ cpp_read_line_comment(Cpp_File file, int pos){
} }
FCPP_LINK Cpp_Read_Result FCPP_LINK Cpp_Read_Result
cpp_read_block_comment(Cpp_File file, int pos){ cpp_read_block_comment(char *data, int size, int pos){
Cpp_Read_Result result = {}; Cpp_Read_Result result = {};
result.token.start = pos; result.token.start = pos;
_Assert(file.data[pos] == '/' && file.data[pos + 1] == '*'); _Assert(data[pos] == '/' && data[pos + 1] == '*');
pos += 2; pos += 2;
while (pos < file.size){ while (pos < size){
if (file.data[pos] == '*' && if (data[pos] == '*' &&
pos + 1 < file.size && pos + 1 < size &&
file.data[pos+1] == '/'){ data[pos+1] == '/'){
break; break;
} }
++pos; ++pos;
@ -889,22 +881,22 @@ cpp_read_block_comment(Cpp_File file, int pos){
} }
FCPP_LINK Cpp_Read_Result FCPP_LINK Cpp_Read_Result
cpp_read_preprocessor(Cpp_File file, int pos){ cpp_read_preprocessor(char *data, int size, int pos){
_Assert(file.data[pos] == '#'); _Assert(data[pos] == '#');
Cpp_Read_Result result = {}; Cpp_Read_Result result = {};
result.token.start = pos; result.token.start = pos;
result.token.type = CPP_PP_UNKNOWN; result.token.type = CPP_PP_UNKNOWN;
result.token.flags |= CPP_TFLAG_PP_DIRECTIVE; result.token.flags |= CPP_TFLAG_PP_DIRECTIVE;
++pos; ++pos;
while (pos < file.size && while (pos < size &&
(file.data[pos] == ' ' || (data[pos] == ' ' ||
file.data[pos] == '\t')){ data[pos] == '\t')){
++pos; ++pos;
} }
Sub_Match_List_Result match Sub_Match_List_Result match
= sub_match_list(file, pos, preprops, -1); = sub_match_list(data, size, pos, preprops, -1);
if (match.index != -1){ if (match.index != -1){
result.token.size = match.new_pos - result.token.start; result.token.size = match.new_pos - result.token.start;
@ -912,8 +904,7 @@ cpp_read_preprocessor(Cpp_File file, int pos){
result.pos = match.new_pos; result.pos = match.new_pos;
} }
else{ else{
while (pos < file.size && while (pos < size && !char_is_whitespace(data[pos])){
!char_is_whitespace(file.data[pos])){
++pos; ++pos;
} }
result.token.size = pos - result.token.start; result.token.size = pos - result.token.start;
@ -924,8 +915,8 @@ cpp_read_preprocessor(Cpp_File file, int pos){
} }
FCPP_LINK Cpp_Read_Result FCPP_LINK Cpp_Read_Result
cpp_read_pp_include_file(Cpp_File file, int pos){ cpp_read_pp_include_file(char *data, int size, int pos){
char start = file.data[pos]; char start = data[pos];
_Assert(start == '<' || start == '"'); _Assert(start == '<' || start == '"');
Cpp_Read_Result result = {}; Cpp_Read_Result result = {};
@ -942,22 +933,20 @@ cpp_read_pp_include_file(Cpp_File file, int pos){
} }
++pos; ++pos;
while (pos < file.size && file.data[pos] != end){ while (pos < size && data[pos] != end){
if (file.data[pos] == '\n'){ if (data[pos] == '\n'){
result.token.type = CPP_TOKEN_JUNK; result.token.type = CPP_TOKEN_JUNK;
result.token.flags |= CPP_TFLAG_BAD_ENDING; result.token.flags |= CPP_TFLAG_BAD_ENDING;
break; break;
} }
if (file.data[pos] == '\\'){ if (data[pos] == '\\'){
// TODO(allen): Not sure that this is 100% correct. if (pos + 1 < size && data[pos + 1] == '\n'){
if (pos + 1 < file.size &&
file.data[pos + 1] == '\n'){
++pos; ++pos;
result.token.flags |= CPP_TFLAG_MULTILINE; result.token.flags |= CPP_TFLAG_MULTILINE;
} }
else if (pos + 2 < file.size && else if (pos + 2 < size &&
file.data[pos + 1] == '\r' && data[pos + 1] == '\r' &&
file.data[pos + 2] == '\n'){ data[pos + 2] == '\n'){
pos += 2; pos += 2;
result.token.flags |= CPP_TFLAG_MULTILINE; result.token.flags |= CPP_TFLAG_MULTILINE;
} }
@ -966,7 +955,7 @@ cpp_read_pp_include_file(Cpp_File file, int pos){
} }
if (result.token.type != CPP_TOKEN_JUNK){ if (result.token.type != CPP_TOKEN_JUNK){
if (pos < file.size){ if (pos < size){
++pos; ++pos;
} }
} }
@ -978,55 +967,55 @@ cpp_read_pp_include_file(Cpp_File file, int pos){
} }
FCPP_LINK Cpp_Read_Result FCPP_LINK Cpp_Read_Result
cpp_read_pp_default_mode(Cpp_File file, int pos, bool in_if_body){ cpp_read_pp_default_mode(char *data, int size, int pos, bool in_if_body){
char current = file.data[pos]; char current = data[pos];
Cpp_Read_Result result; Cpp_Read_Result result;
if (char_is_numeric(current)){ if (char_is_numeric(current)){
result = cpp_read_number(file, pos); result = cpp_read_number(data, size, pos);
} }
else if (char_is_alpha(current)){ else if (char_is_alpha(current)){
result = cpp_read_alpha_numeric(file, pos, in_if_body); result = cpp_read_alpha_numeric(data, size, pos, in_if_body);
} }
else if (current == '.'){ else if (current == '.'){
if (pos + 1 < file.size){ if (pos + 1 < size){
char next = file.data[pos + 1]; char next = data[pos + 1];
if (char_is_numeric(next)){ if (char_is_numeric(next)){
result = cpp_read_number(file, pos); result = cpp_read_number(data, size, pos);
} }
else{ else{
result = cpp_read_operator(file, pos); result = cpp_read_operator(data, size, pos);
} }
} }
else{ else{
result = cpp_read_operator(file, pos); result = cpp_read_operator(data, size, pos);
} }
} }
else if (current == '/'){ else if (current == '/'){
if (pos + 1 < file.size){ if (pos + 1 < size){
char next = file.data[pos + 1]; char next = data[pos + 1];
if (next == '/'){ if (next == '/'){
result = cpp_read_line_comment(file, pos); result = cpp_read_line_comment(data, size, pos);
} }
else if (next == '*'){ else if (next == '*'){
result = cpp_read_block_comment(file, pos); result = cpp_read_block_comment(data, size, pos);
} }
else{ else{
result = cpp_read_operator(file, pos); result = cpp_read_operator(data, size, pos);
} }
} }
else{ else{
result = cpp_read_operator(file, pos); result = cpp_read_operator(data, size, pos);
} }
} }
else if (current == '"'){ else if (current == '"'){
result = cpp_read_string_litteral(file, pos); result = cpp_read_string_litteral(data, size, pos);
} }
else if (current == '\''){ else if (current == '\''){
result = cpp_read_character_litteral(file, pos); result = cpp_read_character_litteral(data, size, pos);
} }
else{ else{
result = cpp_read_operator(file, pos); result = cpp_read_operator(data, size, pos);
} }
return result; return result;
@ -1084,16 +1073,16 @@ cpp_push_token_nonalloc(Cpp_Token_Stack *token_stack, Cpp_Token token){
} }
FCPP_LINK Cpp_Read_Result FCPP_LINK Cpp_Read_Result
cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){ cpp_lex_step(char *data, int size, Cpp_Lex_Data *lex_data){
Cpp_Lex_Data lex = *lex_data; Cpp_Lex_Data lex = *lex_data;
Cpp_Read_Result result = {}; Cpp_Read_Result result = {};
bool has_result = 1; bool has_result = 1;
fcpp_u16 state_flags = cpp_token_set_pp_state(0, lex.pp_state); fcpp_u16 state_flags = cpp_token_set_pp_state(0, lex.pp_state);
char current = file.data[lex.pos]; char current = data[lex.pos];
if (char_is_whitespace(current)){ if (char_is_whitespace(current)){
result = cpp_read_whitespace(file, lex.pos); result = cpp_read_whitespace(data, size, lex.pos);
lex.pos = result.pos; lex.pos = result.pos;
if (result.newline && lex.pp_state != CPP_LEX_PP_DEFAULT){ if (result.newline && lex.pp_state != CPP_LEX_PP_DEFAULT){
lex.pp_state = CPP_LEX_PP_DEFAULT; lex.pp_state = CPP_LEX_PP_DEFAULT;
@ -1105,7 +1094,7 @@ cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){
if (lex.pp_state == CPP_LEX_PP_DEFAULT){ if (lex.pp_state == CPP_LEX_PP_DEFAULT){
// TODO(allen): Not first hard of the line? Then it's junk. // TODO(allen): Not first hard of the line? Then it's junk.
if (current == '#'){ if (current == '#'){
result = cpp_read_preprocessor(file, lex.pos); result = cpp_read_preprocessor(data, size, lex.pos);
lex.pos = result.pos; lex.pos = result.pos;
switch (result.token.type){ switch (result.token.type){
case CPP_PP_INCLUDE: case CPP_PP_INCLUDE:
@ -1143,7 +1132,7 @@ cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){
} }
} }
else{ else{
result = cpp_read_pp_default_mode(file, lex.pos); result = cpp_read_pp_default_mode(data, size, lex.pos);
lex.pos = result.pos; lex.pos = result.pos;
} }
} }
@ -1152,10 +1141,10 @@ cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){
if (current == '\\'){ if (current == '\\'){
fcpp_i32 seek = lex.pos; fcpp_i32 seek = lex.pos;
++seek; ++seek;
while (seek < file.size && file.data[seek] == '\r'){ while (seek < size && data[seek] == '\r'){
++seek; ++seek;
} }
if ((seek < file.size && file.data[seek] == '\n') || seek >= file.size){ if ((seek < size && data[seek] == '\n') || seek >= size){
lex.pos = seek + 1; lex.pos = seek + 1;
has_result = 0; has_result = 0;
} }
@ -1177,7 +1166,7 @@ cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){
lex.pp_state = CPP_LEX_PP_JUNK; lex.pp_state = CPP_LEX_PP_JUNK;
} }
else{ else{
result = cpp_read_alpha_numeric(file, lex.pos); result = cpp_read_alpha_numeric(data, size, lex.pos);
result.token.flags |= CPP_TFLAG_PP_BODY; result.token.flags |= CPP_TFLAG_PP_BODY;
lex.pos = result.pos; lex.pos = result.pos;
lex.pp_state = CPP_LEX_PP_JUNK; lex.pp_state = CPP_LEX_PP_JUNK;
@ -1190,7 +1179,7 @@ cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){
lex.pp_state = CPP_LEX_PP_JUNK; lex.pp_state = CPP_LEX_PP_JUNK;
} }
else{ else{
result = cpp_read_alpha_numeric(file, lex.pos); result = cpp_read_alpha_numeric(data, size, lex.pos);
result.token.flags |= CPP_TFLAG_PP_BODY; result.token.flags |= CPP_TFLAG_PP_BODY;
lex.pos = result.pos; lex.pos = result.pos;
lex.pp_state = CPP_LEX_PP_BODY; lex.pp_state = CPP_LEX_PP_BODY;
@ -1203,7 +1192,7 @@ cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){
lex.pp_state = CPP_LEX_PP_JUNK; lex.pp_state = CPP_LEX_PP_JUNK;
} }
else{ else{
result = cpp_read_pp_include_file(file, lex.pos); result = cpp_read_pp_include_file(data, size, lex.pos);
lex.pos = result.pos; lex.pos = result.pos;
lex.pp_state = CPP_LEX_PP_JUNK; lex.pp_state = CPP_LEX_PP_JUNK;
} }
@ -1211,10 +1200,10 @@ cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){
case CPP_LEX_PP_BODY: case CPP_LEX_PP_BODY:
if (current == '#'){ if (current == '#'){
result = cpp_read_pp_operator(file, lex.pos); result = cpp_read_pp_operator(data, size, lex.pos);
} }
else{ else{
result = cpp_read_pp_default_mode(file, lex.pos); result = cpp_read_pp_default_mode(data, size, lex.pos);
} }
lex.pos = result.pos; lex.pos = result.pos;
result.token.flags |= CPP_TFLAG_PP_BODY; result.token.flags |= CPP_TFLAG_PP_BODY;
@ -1222,10 +1211,10 @@ cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){
case CPP_LEX_PP_BODY_IF: case CPP_LEX_PP_BODY_IF:
if (current == '#'){ if (current == '#'){
result = cpp_read_pp_operator(file, lex.pos); result = cpp_read_pp_operator(data, size, lex.pos);
} }
else{ else{
result = cpp_read_pp_default_mode(file, lex.pos, 1); result = cpp_read_pp_default_mode(data, size, lex.pos, 1);
} }
lex.pos = result.pos; lex.pos = result.pos;
result.token.flags |= CPP_TFLAG_PP_BODY; result.token.flags |= CPP_TFLAG_PP_BODY;
@ -1237,7 +1226,7 @@ cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){
lex.pp_state = CPP_LEX_PP_JUNK; lex.pp_state = CPP_LEX_PP_JUNK;
} }
else{ else{
result = cpp_read_number(file, lex.pos); result = cpp_read_number(data, size, lex.pos);
lex.pos = result.pos; lex.pos = result.pos;
result.token.flags |= CPP_TFLAG_PP_BODY; result.token.flags |= CPP_TFLAG_PP_BODY;
lex.pp_state = CPP_LEX_PP_INCLUDE; lex.pp_state = CPP_LEX_PP_INCLUDE;
@ -1245,7 +1234,7 @@ cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){
break; break;
case CPP_LEX_PP_ERROR: case CPP_LEX_PP_ERROR:
result = cpp_read_junk_line(file, lex.pos); result = cpp_read_junk_line(data, size, lex.pos);
lex.pos = result.pos; lex.pos = result.pos;
result.token.type = CPP_TOKEN_ERROR_MESSAGE; result.token.type = CPP_TOKEN_ERROR_MESSAGE;
result.token.flags |= CPP_TFLAG_PP_BODY; result.token.flags |= CPP_TFLAG_PP_BODY;
@ -1254,21 +1243,21 @@ cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){
default: default:
{ {
bool took_comment = 0; bool took_comment = 0;
if (current == '/' && lex.pos + 1 < file.size){ if (current == '/' && lex.pos + 1 < size){
if (file.data[lex.pos + 1] == '/'){ if (data[lex.pos + 1] == '/'){
result = cpp_read_line_comment(file, lex.pos); result = cpp_read_line_comment(data, size, lex.pos);
lex.pp_state = CPP_LEX_PP_DEFAULT; lex.pp_state = CPP_LEX_PP_DEFAULT;
lex.pos = result.pos; lex.pos = result.pos;
took_comment = 1; took_comment = 1;
}else if (file.data[lex.pos + 1] == '*'){ }else if (data[lex.pos + 1] == '*'){
result = cpp_read_block_comment(file, lex.pos); result = cpp_read_block_comment(data, size, lex.pos);
lex.pos = result.pos; lex.pos = result.pos;
took_comment = 1; took_comment = 1;
} }
} }
if (!took_comment){ if (!took_comment){
result = cpp_read_junk_line(file, lex.pos); result = cpp_read_junk_line(data, size, lex.pos);
lex.pos = result.pos; lex.pos = result.pos;
result.token.flags |= CPP_TFLAG_PP_BODY; result.token.flags |= CPP_TFLAG_PP_BODY;
} }
@ -1287,12 +1276,12 @@ cpp_lex_step(Cpp_File file, Cpp_Lex_Data *lex_data){
} }
FCPP_LINK int FCPP_LINK int
cpp_lex_file_token_count(Cpp_File file){ cpp_lex_file_token_count(char *data, int size){
int count = 0; int count = 0;
Cpp_Lex_Data lex = {}; Cpp_Lex_Data lex = {};
Cpp_Token token = {}; Cpp_Token token = {};
while (lex.pos < file.size){ while (lex.pos < size){
Cpp_Read_Result step_result = cpp_lex_step(file, &lex); Cpp_Read_Result step_result = cpp_lex_step(data, size, &lex);
if (step_result.has_result){ if (step_result.has_result){
if (count > 0){ if (count > 0){
@ -1315,21 +1304,21 @@ cpp_lex_file_token_count(Cpp_File file){
} }
FCPP_LINK Cpp_Lex_Data FCPP_LINK Cpp_Lex_Data
cpp_lex_file_nonalloc(Cpp_File file, Cpp_Token_Stack *token_stack_out, Cpp_Lex_Data data){ cpp_lex_file_nonalloc(char *data, int size, Cpp_Token_Stack *token_stack_out, Cpp_Lex_Data lex_data){
while (data.pos < file.size){ while (lex_data.pos < size){
Cpp_Lex_Data prev_lex = data; Cpp_Lex_Data prev_lex = lex_data;
Cpp_Read_Result step_result = cpp_lex_step(file, &data); Cpp_Read_Result step_result = cpp_lex_step(data, size, &lex_data);
if (step_result.has_result){ if (step_result.has_result){
if (!cpp_push_token_nonalloc(token_stack_out, step_result.token)){ if (!cpp_push_token_nonalloc(token_stack_out, step_result.token)){
data = prev_lex; lex_data = prev_lex;
return data; return lex_data;
} }
} }
} }
data.complete = 1; lex_data.complete = 1;
return data; return lex_data;
} }
FCPP_LINK Cpp_Get_Token_Result FCPP_LINK Cpp_Get_Token_Result
@ -1395,10 +1384,11 @@ cpp_shift_token_starts(Cpp_Token_Stack *stack, int from_token_i, int amount){
} }
FCPP_LINK Cpp_Relex_State FCPP_LINK Cpp_Relex_State
cpp_relex_nonalloc_start(Cpp_File file, Cpp_Token_Stack *stack, cpp_relex_nonalloc_start(char *data, int size, Cpp_Token_Stack *stack,
int start, int end, int amount, int tolerance){ int start, int end, int amount, int tolerance){
Cpp_Relex_State state; Cpp_Relex_State state;
state.file = file; state.data = data;
state.size = size;
state.stack = stack; state.stack = stack;
state.start = start; state.start = start;
state.end = end; state.end = end;
@ -1460,12 +1450,12 @@ cpp_relex_nonalloc_main(Cpp_Relex_State *state, Cpp_Token_Stack *relex_stack, in
lex.pos = state->relex_start; lex.pos = state->relex_start;
int relex_end_i = state->end_token_i; int relex_end_i = state->end_token_i;
Cpp_Token match_token = cpp__get_token(stack, tokens, state->file.size, relex_end_i); Cpp_Token match_token = cpp__get_token(stack, tokens, state->size, relex_end_i);
Cpp_Token end_token = match_token; Cpp_Token end_token = match_token;
bool went_too_far = 0; bool went_too_far = 0;
for (;;){ for (;;){
Cpp_Read_Result read = cpp_lex_step(state->file, &lex); Cpp_Read_Result read = cpp_lex_step(state->data, state->size, &lex);
if (read.has_result){ if (read.has_result){
if (read.token.start == end_token.start && if (read.token.start == end_token.start &&
read.token.size == end_token.size && read.token.size == end_token.size &&
@ -1477,14 +1467,14 @@ cpp_relex_nonalloc_main(Cpp_Relex_State *state, Cpp_Token_Stack *relex_stack, in
while (lex.pos > end_token.start && relex_end_i < stack->count){ while (lex.pos > end_token.start && relex_end_i < stack->count){
++relex_end_i; ++relex_end_i;
end_token = cpp__get_token(stack, tokens, state->file.size, relex_end_i); end_token = cpp__get_token(stack, tokens, state->size, relex_end_i);
} }
if (relex_stack->count == relex_stack->max_count){ if (relex_stack->count == relex_stack->max_count){
went_too_far = 1; went_too_far = 1;
break; break;
} }
} }
if (lex.pos >= state->file.size) break; if (lex.pos >= state->size) break;
} }
if (!went_too_far){ if (!went_too_far){
@ -1557,10 +1547,10 @@ cpp_push_token(Cpp_Token_Stack *token_stack, Cpp_Token token){
} }
FCPP_LINK void FCPP_LINK void
cpp_lex_file(Cpp_File file, Cpp_Token_Stack *token_stack_out){ cpp_lex_file(char *data, int size, Cpp_Token_Stack *token_stack_out){
Cpp_Lex_Data lex = {}; Cpp_Lex_Data lex = {};
while (lex.pos < file.size){ while (lex.pos < size){
Cpp_Read_Result step_result = cpp_lex_step(file, &lex); Cpp_Read_Result step_result = cpp_lex_step(data, size, &lex);
if (step_result.has_result){ if (step_result.has_result){
cpp_push_token(token_stack_out, step_result.token); cpp_push_token(token_stack_out, step_result.token);
} }
@ -1568,7 +1558,7 @@ cpp_lex_file(Cpp_File file, Cpp_Token_Stack *token_stack_out){
} }
FCPP_LINK bool FCPP_LINK bool
cpp_relex_file_limited(Cpp_File file, Cpp_Token_Stack *stack, cpp_relex_file_limited(char *data, int size, Cpp_Token_Stack *stack,
int start, int end, int amount, int tolerance){ int start, int end, int amount, int tolerance){
#if 0 #if 0
int start_token_i, end_token_i; int start_token_i, end_token_i;
@ -1624,7 +1614,7 @@ cpp_relex_file_limited(Cpp_File file, Cpp_Token_Stack *stack,
bool went_too_far = 0; bool went_too_far = 0;
while (1){ while (1){
Cpp_Read_Result result = cpp_lex_step(file, &lex); Cpp_Read_Result result = cpp_lex_step(data, size, &lex);
if (result.has_result){ if (result.has_result){
if (end_guess_i < stack->count && if (end_guess_i < stack->count &&
result.token.start == stack->tokens[end_guess_i].start && result.token.start == stack->tokens[end_guess_i].start &&
@ -1642,7 +1632,7 @@ cpp_relex_file_limited(Cpp_File file, Cpp_Token_Stack *stack,
} }
} }
if (lex.pos >= file.size){ if (lex.pos >= size){
break; break;
} }

11
4cpp_lexer_types.h
View File

@ -170,13 +170,6 @@ enum Cpp_Token_Type{
CPP_TOKEN_TYPE_COUNT CPP_TOKEN_TYPE_COUNT
}; };
// TODO(allen): This is a dumb redundant type... probably just
// move towards using String for this everywhere eventually.
struct Cpp_File{
char *data;
int size;
};
struct Cpp_Token{ struct Cpp_Token{
Cpp_Token_Type type; Cpp_Token_Type type;
int start, size; int start, size;
@ -235,7 +228,9 @@ struct Cpp_Get_Token_Result{
}; };
struct Cpp_Relex_State{ struct Cpp_Relex_State{
Cpp_File file; char *data;
int size;
Cpp_Token_Stack *stack; Cpp_Token_Stack *stack;
int start, end, amount; int start, end, amount;
int start_token_i; int start_token_i;

18
4ed_file_view.cpp
View File

@ -1177,9 +1177,8 @@ Job_Callback_Sig(job_full_lex){
Editing_File *file = (Editing_File*)data[0]; Editing_File *file = (Editing_File*)data[0];
General_Memory *general = (General_Memory*)data[1]; General_Memory *general = (General_Memory*)data[1];
Cpp_File cpp_file; char *text_data = file->state.buffer.data;
cpp_file.data = file->state.buffer.data; i32 text_size = file->state.buffer.size;
cpp_file.size = file->state.buffer.size;
i32 buffer_size = file->state.buffer.size; i32 buffer_size = file->state.buffer.size;
buffer_size = (buffer_size + 3)&(~3); buffer_size = (buffer_size + 3)&(~3);
@ -1243,9 +1242,9 @@ Job_Callback_Sig(job_full_lex){
Cpp_Lex_Data status = {}; Cpp_Lex_Data status = {};
do{ do{
for (i32 r = 2048; r > 0 && status.pos < cpp_file.size; --r){ for (i32 r = 2048; r > 0 && status.pos < text_size; --r){
Cpp_Lex_Data prev_lex = status; Cpp_Lex_Data prev_lex = status;
Cpp_Read_Result step_result = cpp_lex_step(cpp_file, &status); Cpp_Read_Result step_result = cpp_lex_step(text_data, text_size, &status);
if (step_result.has_result){ if (step_result.has_result){
if (!cpp_push_token_nonalloc(&tokens, step_result.token)){ if (!cpp_push_token_nonalloc(&tokens, step_result.token)){
@ -1257,7 +1256,7 @@ Job_Callback_Sig(job_full_lex){
} }
} }
if (status.pos >= cpp_file.size){ if (status.pos >= text_size){
status.complete = 1; status.complete = 1;
} }
else{ else{
@ -1361,14 +1360,13 @@ file_relex_parallel(System_Functions *system,
b32 result = true; b32 result = true;
b32 inline_lex = !file->state.still_lexing; b32 inline_lex = !file->state.still_lexing;
if (inline_lex){ if (inline_lex){
Cpp_File cpp_file; char *data = file->state.buffer.data;
cpp_file.data = file->state.buffer.data; i32 size = file->state.buffer.size;
cpp_file.size = file->state.buffer.size;
Cpp_Token_Stack *stack = &file->state.token_stack; Cpp_Token_Stack *stack = &file->state.token_stack;
Cpp_Relex_State state = Cpp_Relex_State state =
cpp_relex_nonalloc_start(cpp_file, stack, cpp_relex_nonalloc_start(data, size, stack,
start_i, end_i, amount, 100); start_i, end_i, amount, 100);
Temp_Memory temp = begin_temp_memory(part); Temp_Memory temp = begin_temp_memory(part);

155
4ed_metagen.cpp
View File

@ -710,14 +710,14 @@ perform_doc_parse(Partition *part, String doc_string, Documentation *doc){
} }
static int static int
get_type_doc_string(Cpp_File file, Cpp_Token *tokens, int i, get_type_doc_string(char *data, Cpp_Token *tokens, int i,
String *doc_string){ String *doc_string){
int result = false; int result = false;
if (i > 0){ if (i > 0){
Cpp_Token *prev_token = tokens + i - 1; Cpp_Token *prev_token = tokens + i - 1;
if (prev_token->type == CPP_TOKEN_COMMENT){ if (prev_token->type == CPP_TOKEN_COMMENT){
*doc_string = make_string(file.data + prev_token->start, prev_token->size); *doc_string = make_string(data + prev_token->start, prev_token->size);
if (check_and_fix_docs(doc_string)){ if (check_and_fix_docs(doc_string)){
result = true; result = true;
} }
@ -729,13 +729,13 @@ get_type_doc_string(Cpp_File file, Cpp_Token *tokens, int i,
static int static int
parse_struct(Partition *part, int is_struct, parse_struct(Partition *part, int is_struct,
Cpp_File file, Cpp_Token *tokens, int count, char *data, Cpp_Token *tokens, int count,
Cpp_Token **token_ptr, Cpp_Token **token_ptr,
Struct_Member *top_member); Struct_Member *top_member);
static int static int
parse_struct_member(Partition *part, parse_struct_member(Partition *part,
Cpp_File file, Cpp_Token *tokens, int count, char *data, Cpp_Token *tokens, int count,
Cpp_Token **token_ptr, Cpp_Token **token_ptr,
Struct_Member *member){ Struct_Member *member){
@ -745,7 +745,7 @@ parse_struct_member(Partition *part,
int i = (int)(token - tokens); int i = (int)(token - tokens);
String doc_string = {0}; String doc_string = {0};
get_type_doc_string(file, tokens, i, &doc_string); get_type_doc_string(data, tokens, i, &doc_string);
int start_i = i; int start_i = i;
Cpp_Token *start_token = token; Cpp_Token *start_token = token;
@ -779,18 +779,18 @@ parse_struct_member(Partition *part,
} }
} }
String name = make_string(file.data + token_j->start, token_j->size); String name = make_string(data + token_j->start, token_j->size);
name = skip_chop_whitespace(name); name = skip_chop_whitespace(name);
int type_start = start_token->start; int type_start = start_token->start;
int type_end = token_j->start; int type_end = token_j->start;
String type = make_string(file.data + type_start, type_end - type_start); String type = make_string(data + type_start, type_end - type_start);
type = skip_chop_whitespace(type); type = skip_chop_whitespace(type);
type_start = token_j->start + token_j->size; type_start = token_j->start + token_j->size;
type_end = token->start; type_end = token->start;
String type_postfix = make_string(file.data + type_start, type_end - type_start); String type_postfix = make_string(data + type_start, type_end - type_start);
type_postfix = skip_chop_whitespace(type_postfix); type_postfix = skip_chop_whitespace(type_postfix);
++token; ++token;
@ -811,7 +811,7 @@ parse_struct_member(Partition *part,
static Struct_Member* static Struct_Member*
parse_struct_next_member(Partition *part, parse_struct_next_member(Partition *part,
Cpp_File file, Cpp_Token *tokens, int count, char *data, Cpp_Token *tokens, int count,
Cpp_Token **token_ptr){ Cpp_Token **token_ptr){
Struct_Member *result = 0; Struct_Member *result = 0;
@ -821,11 +821,11 @@ parse_struct_next_member(Partition *part,
for (; i < count; ++i, ++token){ for (; i < count; ++i, ++token){
if (token->type == CPP_TOKEN_IDENTIFIER || if (token->type == CPP_TOKEN_IDENTIFIER ||
(token->flags & CPP_TFLAG_IS_KEYWORD)){ (token->flags & CPP_TFLAG_IS_KEYWORD)){
String lexeme = make_string(file.data + token->start, token->size); String lexeme = make_string(data + token->start, token->size);
if (match(lexeme, make_lit_string("struct"))){ if (match(lexeme, make_lit_string("struct"))){
Struct_Member *member = push_struct(part, Struct_Member); Struct_Member *member = push_struct(part, Struct_Member);
if (parse_struct(part, true, file, tokens, count, &token, member)){ if (parse_struct(part, true, data, tokens, count, &token, member)){
result = member; result = member;
break; break;
} }
@ -835,7 +835,7 @@ parse_struct_next_member(Partition *part,
} }
else if (match(lexeme, make_lit_string("union"))){ else if (match(lexeme, make_lit_string("union"))){
Struct_Member *member = push_struct(part, Struct_Member); Struct_Member *member = push_struct(part, Struct_Member);
if (parse_struct(part, false, file, tokens, count, &token, member)){ if (parse_struct(part, false, data, tokens, count, &token, member)){
result = member; result = member;
break; break;
} }
@ -845,7 +845,7 @@ parse_struct_next_member(Partition *part,
} }
else{ else{
Struct_Member *member = push_struct(part, Struct_Member); Struct_Member *member = push_struct(part, Struct_Member);
if (parse_struct_member(part, file, tokens, count, &token, member)){ if (parse_struct_member(part, data, tokens, count, &token, member)){
result = member; result = member;
break; break;
} }
@ -866,7 +866,7 @@ parse_struct_next_member(Partition *part,
static int static int
parse_struct(Partition *part, int is_struct, parse_struct(Partition *part, int is_struct,
Cpp_File file, Cpp_Token *tokens, int count, char *data, Cpp_Token *tokens, int count,
Cpp_Token **token_ptr, Cpp_Token **token_ptr,
Struct_Member *top_member){ Struct_Member *top_member){
@ -876,7 +876,7 @@ parse_struct(Partition *part, int is_struct,
int i = (int)(token - tokens); int i = (int)(token - tokens);
String doc_string = {0}; String doc_string = {0};
get_type_doc_string(file, tokens, i, &doc_string); get_type_doc_string(data, tokens, i, &doc_string);
int start_i = i; int start_i = i;
@ -899,7 +899,7 @@ parse_struct(Partition *part, int is_struct,
String name = {0}; String name = {0};
if (j != start_i){ if (j != start_i){
name = make_string(file.data + token_j->start, token_j->size); name = make_string(data + token_j->start, token_j->size);
name = skip_chop_whitespace(name); name = skip_chop_whitespace(name);
} }
@ -913,7 +913,7 @@ parse_struct(Partition *part, int is_struct,
++token; ++token;
Struct_Member *new_member = Struct_Member *new_member =
parse_struct_next_member(part, file, tokens, count, &token); parse_struct_next_member(part, data, tokens, count, &token);
if (new_member){ if (new_member){
top_member->first_child = new_member; top_member->first_child = new_member;
@ -921,7 +921,7 @@ parse_struct(Partition *part, int is_struct,
Struct_Member *head_member = new_member; Struct_Member *head_member = new_member;
for(;;){ for(;;){
new_member = new_member =
parse_struct_next_member(part, file, tokens, count, &token); parse_struct_next_member(part, data, tokens, count, &token);
if (new_member){ if (new_member){
head_member->next_sibling = new_member; head_member->next_sibling = new_member;
head_member = new_member; head_member = new_member;
@ -1124,7 +1124,7 @@ print_see_also(FILE *file, Documentation *doc){
} }
static int static int
parse_enum(Partition *part, Cpp_File file, parse_enum(Partition *part, char *data,
Cpp_Token *tokens, int count, Cpp_Token *tokens, int count,
Cpp_Token **token_ptr, int start_i, Cpp_Token **token_ptr, int start_i,
Enum_Set flag_set, int flag_index){ Enum_Set flag_set, int flag_index){
@ -1143,7 +1143,7 @@ parse_enum(Partition *part, Cpp_File file,
} }
} }
String name = make_string(file.data + token_j->start, token_j->size); String name = make_string(data + token_j->start, token_j->size);
name = skip_chop_whitespace(name); name = skip_chop_whitespace(name);
for (; i < count; ++i, ++token){ for (; i < count; ++i, ++token){
@ -1162,9 +1162,9 @@ parse_enum(Partition *part, Cpp_File file,
} }
else if (token->type == CPP_TOKEN_IDENTIFIER){ else if (token->type == CPP_TOKEN_IDENTIFIER){
String doc_string = {0}; String doc_string = {0};
get_type_doc_string(file, tokens, i, &doc_string); get_type_doc_string(data, tokens, i, &doc_string);
String name = make_string(file.data + token->start, token->size); String name = make_string(data + token->start, token->size);
name = skip_chop_whitespace(name); name = skip_chop_whitespace(name);
String value = {0}; String value = {0};
@ -1185,7 +1185,7 @@ parse_enum(Partition *part, Cpp_File file,
int val_start = start_token->start + start_token->size; int val_start = start_token->start + start_token->size;
int val_end = token->start; int val_end = token->start;
value = make_string(file.data + val_start, val_end - val_start); value = make_string(data + val_start, val_end - val_start);
value = skip_chop_whitespace(value); value = skip_chop_whitespace(value);
--i; --i;
@ -1282,7 +1282,7 @@ allocate_argument_breakdown(int count){
} }
static Argument_Breakdown static Argument_Breakdown
do_parameter_parse(Cpp_File file, Cpp_Token *args_start_token, Cpp_Token *token){ do_parameter_parse(char *data, Cpp_Token *args_start_token, Cpp_Token *token){
int arg_index = 0; int arg_index = 0;
Cpp_Token *arg_token = args_start_token + 1; Cpp_Token *arg_token = args_start_token + 1;
int param_string_start = arg_token->start; int param_string_start = arg_token->start;
@ -1303,7 +1303,7 @@ do_parameter_parse(Cpp_File file, Cpp_Token *args_start_token, Cpp_Token *token)
arg_token->type == CPP_TOKEN_PARENTHESE_CLOSE){ arg_token->type == CPP_TOKEN_PARENTHESE_CLOSE){
int size = arg_token->start - param_string_start; int size = arg_token->start - param_string_start;
String param_string = make_string(file.data + param_string_start, size); String param_string = make_string(data + param_string_start, size);
param_string = chop_whitespace(param_string); param_string = chop_whitespace(param_string);
breakdown.param_string[arg_index] = param_string; breakdown.param_string[arg_index] = param_string;
@ -1313,7 +1313,7 @@ do_parameter_parse(Cpp_File file, Cpp_Token *args_start_token, Cpp_Token *token)
if (param_name_token->type == CPP_TOKEN_IDENTIFIER){ if (param_name_token->type == CPP_TOKEN_IDENTIFIER){
int start = param_name_token->start; int start = param_name_token->start;
int size = param_name_token->size; int size = param_name_token->size;
breakdown.param_name[arg_index] = make_string(file.data + start, size); breakdown.param_name[arg_index] = make_string(data + start, size);
break; break;
} }
} }
@ -1363,7 +1363,7 @@ do_function_parse_check(int *index, Cpp_Token **token_ptr, int count){
static int static int
do_function_get_doc(int *index, Cpp_Token **token_ptr, int count, do_function_get_doc(int *index, Cpp_Token **token_ptr, int count,
Cpp_File file, String *doc_string){ char *data, String *doc_string){
int result = false; int result = false;
int i = *index; int i = *index;
@ -1371,7 +1371,7 @@ do_function_get_doc(int *index, Cpp_Token **token_ptr, int count,
for (; i < count; ++i, ++token){ for (; i < count; ++i, ++token){
if (token->type == CPP_TOKEN_COMMENT){ if (token->type == CPP_TOKEN_COMMENT){
String lexeme = make_string(file.data + token->start, token->size); String lexeme = make_string(data + token->start, token->size);
if (check_and_fix_docs(&lexeme)){ if (check_and_fix_docs(&lexeme)){
*doc_string = lexeme; *doc_string = lexeme;
result = true; result = true;
@ -1391,7 +1391,7 @@ do_function_get_doc(int *index, Cpp_Token **token_ptr, int count,
static int static int
do_function_parse(int *index, Cpp_Token **token_ptr, int count, Cpp_Token *ret_start_token, do_function_parse(int *index, Cpp_Token **token_ptr, int count, Cpp_Token *ret_start_token,
Cpp_File file, Function_Set function_set, int sig_count){ char *data, Function_Set function_set, int sig_count){
int result = false; int result = false;
int i = *index; int i = *index;
@ -1399,10 +1399,10 @@ do_function_parse(int *index, Cpp_Token **token_ptr, int count, Cpp_Token *ret_s
Cpp_Token *args_start_token = token+1; Cpp_Token *args_start_token = token+1;
function_set.name[sig_count] = make_string(file.data + token->start, token->size); function_set.name[sig_count] = make_string(data + token->start, token->size);
int size = token->start - ret_start_token->start; int size = token->start - ret_start_token->start;
String ret = make_string(file.data + ret_start_token->start, size); String ret = make_string(data + ret_start_token->start, size);
ret = chop_whitespace(ret); ret = chop_whitespace(ret);
function_set.ret[sig_count] = ret; function_set.ret[sig_count] = ret;
@ -1415,12 +1415,12 @@ do_function_parse(int *index, Cpp_Token **token_ptr, int count, Cpp_Token *ret_s
if (i < count){ if (i < count){
int size = token->start + token->size - args_start_token->start;; int size = token->start + token->size - args_start_token->start;;
function_set.args[sig_count] = function_set.args[sig_count] =
make_string(file.data + args_start_token->start, size); make_string(data + args_start_token->start, size);
function_set.valid[sig_count] = true; function_set.valid[sig_count] = true;
result = true; result = true;
Argument_Breakdown *breakdown = &function_set.breakdown[sig_count]; Argument_Breakdown *breakdown = &function_set.breakdown[sig_count];
*breakdown = do_parameter_parse(file, args_start_token, token); *breakdown = do_parameter_parse(data, args_start_token, token);
} }
*index = i; *index = i;
@ -1430,7 +1430,7 @@ do_function_parse(int *index, Cpp_Token **token_ptr, int count, Cpp_Token *ret_s
} }
static int static int
do_full_function_parse(int *index, Cpp_Token **token_ptr, int count, Cpp_File file, do_full_function_parse(int *index, Cpp_Token **token_ptr, int count, char *data,
Function_Set function_set, int sig_count){ Function_Set function_set, int sig_count){
int result = false; int result = false;
@ -1438,7 +1438,7 @@ do_full_function_parse(int *index, Cpp_Token **token_ptr, int count, Cpp_File fi
Cpp_Token *token = *token_ptr; Cpp_Token *token = *token_ptr;
{ {
function_set.marker[sig_count] = make_string(file.data + token->start, token->size); function_set.marker[sig_count] = make_string(data + token->start, token->size);
int j = i; int j = i;
Cpp_Token *jtoken = token; Cpp_Token *jtoken = token;
@ -1446,7 +1446,7 @@ do_full_function_parse(int *index, Cpp_Token **token_ptr, int count, Cpp_File fi
if (do_function_parse_check(&j, &jtoken, count)){ if (do_function_parse_check(&j, &jtoken, count)){
if (token->type == CPP_TOKEN_IDENTIFIER){ if (token->type == CPP_TOKEN_IDENTIFIER){
String doc_string = {0}; String doc_string = {0};
if (do_function_get_doc(&j, &jtoken, count, file, &doc_string)){ if (do_function_get_doc(&j, &jtoken, count, data, &doc_string)){
function_set.doc_string[sig_count] = doc_string; function_set.doc_string[sig_count] = doc_string;
} }
} }
@ -1457,7 +1457,7 @@ do_full_function_parse(int *index, Cpp_Token **token_ptr, int count, Cpp_File fi
Cpp_Token *ret_start_token = token; Cpp_Token *ret_start_token = token;
if (do_function_parse_check(&i, &token, count)){ if (do_function_parse_check(&i, &token, count)){
if (do_function_parse(&i, &token, count, ret_start_token, if (do_function_parse(&i, &token, count, ret_start_token,
file, function_set, sig_count)){ data, function_set, sig_count)){
result = true; result = true;
} }
} }
@ -1499,7 +1499,7 @@ do_macro_parse_check(int *index, Cpp_Token **token_ptr, int count){
static int static int
do_macro_parse(int *index, Cpp_Token **token_ptr, int count, do_macro_parse(int *index, Cpp_Token **token_ptr, int count,
Cpp_File file, Function_Set macro_set, int sig_count){ char *data, Function_Set macro_set, int sig_count){
int result = false; int result = false;
int i = *index; int i = *index;
@ -1508,7 +1508,7 @@ do_macro_parse(int *index, Cpp_Token **token_ptr, int count,
if (i > 0){ if (i > 0){
Cpp_Token *doc_token = token-1; Cpp_Token *doc_token = token-1;
String doc_string = make_string(file.data + doc_token->start, doc_token->size); String doc_string = make_string(data + doc_token->start, doc_token->size);
if (check_and_fix_docs(&doc_string)){ if (check_and_fix_docs(&doc_string)){
macro_set.doc_string[sig_count] = doc_string; macro_set.doc_string[sig_count] = doc_string;
@ -1520,7 +1520,7 @@ do_macro_parse(int *index, Cpp_Token **token_ptr, int count,
} }
if (i < count && (token->flags & CPP_TFLAG_PP_BODY)){ if (i < count && (token->flags & CPP_TFLAG_PP_BODY)){
macro_set.name[sig_count] = make_string(file.data + token->start, token->size); macro_set.name[sig_count] = make_string(data + token->start, token->size);
++i, ++token; ++i, ++token;
if (i < count){ if (i < count){
@ -1534,10 +1534,10 @@ do_macro_parse(int *index, Cpp_Token **token_ptr, int count,
if (i < count){ if (i < count){
int start = args_start_token->start; int start = args_start_token->start;
int end = token->start + token->size; int end = token->start + token->size;
macro_set.args[sig_count] = make_string(file.data + start, end - start); macro_set.args[sig_count] = make_string(data + start, end - start);
Argument_Breakdown *breakdown = &macro_set.breakdown[sig_count]; Argument_Breakdown *breakdown = &macro_set.breakdown[sig_count];
*breakdown = do_parameter_parse(file, args_start_token, token); *breakdown = do_parameter_parse(data, args_start_token, token);
++i, ++token; ++i, ++token;
if (i < count){ if (i < count){
@ -1556,7 +1556,7 @@ do_macro_parse(int *index, Cpp_Token **token_ptr, int count,
start = body_start->start; start = body_start->start;
end = body_end->start + body_end->size; end = body_end->start + body_end->size;
macro_set.body[sig_count] = make_string(file.data + start, end - start); macro_set.body[sig_count] = make_string(data + start, end - start);
} }
} }
@ -1745,12 +1745,11 @@ generate_custom_headers(){
String *code = &string_code; String *code = &string_code;
Cpp_Token_Stack *token_stack = &string_tokens; Cpp_Token_Stack *token_stack = &string_tokens;
Cpp_File file; char *data = code->str;
file.data = code->str; int size = code->size;
file.size = code->size;
*token_stack = cpp_make_token_stack(1024); *token_stack = cpp_make_token_stack(1024);
cpp_lex_file(file, token_stack); cpp_lex_file(data, size, token_stack);
int count = token_stack->count; int count = token_stack->count;
@ -1760,7 +1759,7 @@ generate_custom_headers(){
for (int i = 0; i < count; ++i, ++token){ for (int i = 0; i < count; ++i, ++token){
if (token->type == CPP_TOKEN_IDENTIFIER && if (token->type == CPP_TOKEN_IDENTIFIER &&
!(token->flags & CPP_TFLAG_PP_BODY)){ !(token->flags & CPP_TFLAG_PP_BODY)){
String lexeme = make_string(file.data + token->start, token->size); String lexeme = make_string(data + token->start, token->size);
String_Function_Marker marker = String_Function_Marker marker =
do_string_function_marker_check(lexeme); do_string_function_marker_check(lexeme);
@ -1786,9 +1785,7 @@ generate_custom_headers(){
String *code = &string_code; String *code = &string_code;
Cpp_Token_Stack *token_stack = &string_tokens; Cpp_Token_Stack *token_stack = &string_tokens;
Cpp_File file; char *data = code->str;
file.data = code->str;
file.size = code->size;
int count = token_stack->count; int count = token_stack->count;
Cpp_Token *tokens = token_stack->tokens; Cpp_Token *tokens = token_stack->tokens;
@ -1797,21 +1794,21 @@ generate_custom_headers(){
for (int i = 0; i < count; ++i, ++token){ for (int i = 0; i < count; ++i, ++token){
if (token->type == CPP_TOKEN_IDENTIFIER && if (token->type == CPP_TOKEN_IDENTIFIER &&
!(token->flags & CPP_TFLAG_PP_BODY)){ !(token->flags & CPP_TFLAG_PP_BODY)){
String lexeme = make_string(file.data + token->start, token->size); String lexeme = make_string(data + token->start, token->size);
String_Function_Marker marker = String_Function_Marker marker =
do_string_function_marker_check(lexeme); do_string_function_marker_check(lexeme);
if (marker.parse_function){ if (marker.parse_function){
if (do_full_function_parse(&i, &token, count, file, if (do_full_function_parse(&i, &token, count, data,
string_function_set, string_sig_count)){ string_function_set, string_sig_count)){
++string_sig_count; ++string_sig_count;
} }
} }
else if (marker.parse_doc){ else if (marker.parse_doc){
if (do_macro_parse_check(&i, &token, count)){ if (do_macro_parse_check(&i, &token, count)){
do_macro_parse(&i, &token, count, do_macro_parse(&i, &token, count, data,
file, string_function_set, string_sig_count); string_function_set, string_sig_count);
++string_sig_count; ++string_sig_count;
} }
} }
@ -1834,13 +1831,11 @@ generate_custom_headers(){
String *code = &code_data[J]; String *code = &code_data[J];
Parse *parse = &parses[J]; Parse *parse = &parses[J];
// TODO(allen): KILL THIS FUCKIN' Cpp_File FUCKIN' NONSENSE HORSE SHIT!!!!! char *data = code->str;
Cpp_File file; int size = code->size;
file.data = code->str;
file.size = code->size;
parse->tokens = cpp_make_token_stack(512); parse->tokens = cpp_make_token_stack(512);
cpp_lex_file(file, &parse->tokens); cpp_lex_file(data, size, &parse->tokens);
int count = parse->tokens.count; int count = parse->tokens.count;
Cpp_Token *tokens = parse->tokens.tokens; Cpp_Token *tokens = parse->tokens.tokens;
@ -1849,7 +1844,7 @@ generate_custom_headers(){
for (int i = 0; i < count; ++i, ++token){ for (int i = 0; i < count; ++i, ++token){
if (token->type == CPP_TOKEN_IDENTIFIER && if (token->type == CPP_TOKEN_IDENTIFIER &&
!(token->flags & CPP_TFLAG_PP_BODY)){ !(token->flags & CPP_TFLAG_PP_BODY)){
String lexeme = make_string(file.data + token->start, token->size); String lexeme = make_string(data + token->start, token->size);
if (match(lexeme, "API_EXPORT")){ if (match(lexeme, "API_EXPORT")){
if (do_function_parse_check(&i, &token, count)){ if (do_function_parse_check(&i, &token, count)){
++line_count; ++line_count;
@ -1867,10 +1862,7 @@ generate_custom_headers(){
String *code = &code_data[J]; String *code = &code_data[J];
Parse *parse = &parses[J]; Parse *parse = &parses[J];
// TODO(allen): KILL THIS FUCKIN' Cpp_File FUCKIN' NONSENSE HORSE SHIT!!!!! char *data = code->str;
Cpp_File file;
file.data = code->str;
file.size = code->size;
int count = parse->tokens.count; int count = parse->tokens.count;
Cpp_Token *tokens = parse->tokens.tokens; Cpp_Token *tokens = parse->tokens.tokens;
@ -1880,9 +1872,9 @@ generate_custom_headers(){
for (int i = 0; i < count; ++i, ++token){ for (int i = 0; i < count; ++i, ++token){
if (token->type == CPP_TOKEN_IDENTIFIER && if (token->type == CPP_TOKEN_IDENTIFIER &&
!(token->flags & CPP_TFLAG_PP_BODY)){ !(token->flags & CPP_TFLAG_PP_BODY)){
String lexeme = make_string(file.data + token->start, token->size); String lexeme = make_string(data + token->start, token->size);
if (match(lexeme, "API_EXPORT")){ if (match(lexeme, "API_EXPORT")){
do_full_function_parse(&i, &token, count, file, function_set, sig_count); do_full_function_parse(&i, &token, count, data, function_set, sig_count);
if (!function_set.valid[sig_count]){ if (!function_set.valid[sig_count]){
zero_index(function_set, sig_count); zero_index(function_set, sig_count);
// TODO(allen): get warning file name and line numbers // TODO(allen): get warning file name and line numbers
@ -1989,14 +1981,11 @@ generate_custom_headers(){
String type_code = file_dump("4coder_types.h"); String type_code = file_dump("4coder_types.h");
char *data = type_code.str;
// TODO(allen): KILL THIS FUCKIN' Cpp_File FUCKIN' NONSENSE HORSE SHIT!!!!! int size = type_code.size;
Cpp_File type_file;
type_file.data = type_code.str;
type_file.size = type_code.size;
Cpp_Token_Stack types_tokens = cpp_make_token_stack(512); Cpp_Token_Stack types_tokens = cpp_make_token_stack(512);
cpp_lex_file(type_file, &types_tokens); cpp_lex_file(data, size, &types_tokens);
int typedef_count = 0; int typedef_count = 0;
int struct_count = 0; int struct_count = 0;
@ -2021,7 +2010,7 @@ generate_custom_headers(){
(token->type == CPP_TOKEN_KEY_TYPE_DECLARATION || (token->type == CPP_TOKEN_KEY_TYPE_DECLARATION ||
token->type == CPP_TOKEN_IDENTIFIER)){ token->type == CPP_TOKEN_IDENTIFIER)){
String lexeme = make_string(type_file.data + token->start, token->size); String lexeme = make_string(data + token->start, token->size);
int match_index = 0; int match_index = 0;
if (string_set_match(type_spec_keys, ArrayCount(type_spec_keys), if (string_set_match(type_spec_keys, ArrayCount(type_spec_keys),
lexeme, &match_index)){ lexeme, &match_index)){
@ -2077,7 +2066,7 @@ generate_custom_headers(){
(token->type == CPP_TOKEN_KEY_TYPE_DECLARATION || (token->type == CPP_TOKEN_KEY_TYPE_DECLARATION ||
token->type == CPP_TOKEN_IDENTIFIER)){ token->type == CPP_TOKEN_IDENTIFIER)){
String lexeme = make_string(type_file.data + token->start, token->size); String lexeme = make_string(data + token->start, token->size);
int match_index = 0; int match_index = 0;
if (string_set_match(type_spec_keys, ArrayCount(type_spec_keys), if (string_set_match(type_spec_keys, ArrayCount(type_spec_keys),
lexeme, &match_index)){ lexeme, &match_index)){
@ -2085,7 +2074,7 @@ generate_custom_headers(){
case 0: //typedef case 0: //typedef
{ {
String doc_string = {0}; String doc_string = {0};
get_type_doc_string(type_file, tokens, i, &doc_string); get_type_doc_string(data, tokens, i, &doc_string);
int start_i = i; int start_i = i;
Cpp_Token *start_token = token; Cpp_Token *start_token = token;
@ -2105,12 +2094,12 @@ generate_custom_headers(){
} }
} }
String name = make_string(type_file.data + token_j->start, token_j->size); String name = make_string(data + token_j->start, token_j->size);
name = skip_chop_whitespace(name); name = skip_chop_whitespace(name);
int type_start = start_token->start + start_token->size; int type_start = start_token->start + start_token->size;
int type_end = token_j->start; int type_end = token_j->start;
String type = make_string(type_file.data + type_start, type_end - type_start); String type = make_string(data + type_start, type_end - type_start);
type = skip_chop_whitespace(type); type = skip_chop_whitespace(type);
typedef_set.type[typedef_index] = type; typedef_set.type[typedef_index] = type;
@ -2123,7 +2112,7 @@ generate_custom_headers(){
case 1: case 2: //struct/union case 1: case 2: //struct/union
{ {
if (parse_struct(part, (match_index == 1), if (parse_struct(part, (match_index == 1),
type_file, tokens, count, &token, data, tokens, count, &token,
struct_set.structs + struct_index)){ struct_set.structs + struct_index)){
++struct_index; ++struct_index;
} }
@ -2133,7 +2122,7 @@ generate_custom_headers(){
case 3: //ENUM case 3: //ENUM
{ {
String doc_string = {0}; String doc_string = {0};
get_type_doc_string(type_file, tokens, i, &doc_string); get_type_doc_string(data, tokens, i, &doc_string);
int start_i = i; int start_i = i;
@ -2143,7 +2132,7 @@ generate_custom_headers(){
} }
} }
if (parse_enum(part, type_file, if (parse_enum(part, data,
tokens, count, tokens, count,
&token, start_i, &token, start_i,
enum_set, enum_index)){ enum_set, enum_index)){
@ -2156,7 +2145,7 @@ generate_custom_headers(){
case 4: //FLAGENUM case 4: //FLAGENUM
{ {
String doc_string = {0}; String doc_string = {0};
get_type_doc_string(type_file, tokens, i, &doc_string); get_type_doc_string(data, tokens, i, &doc_string);
int start_i = i; int start_i = i;
@ -2166,7 +2155,7 @@ generate_custom_headers(){
} }
} }
if (parse_enum(part, type_file, if (parse_enum(part, data,
tokens, count, tokens, count,
&token, start_i, &token, start_i,
flag_set, flag_index)){ flag_set, flag_index)){

4
build.bat
View File

@ -1,6 +1,6 @@
@echo off @echo off
REM "build_exp.bat" /Zi REM "build_exp.bat" /Zi
REM "build_all.bat" /DFRED_SUPER /DFRED_INTERNAL /Zi "build_all.bat" /DFRED_SUPER /DFRED_INTERNAL /Zi
"build_all.bat" /DFRED_INTERNAL /Zi REM "build_all.bat" /DFRED_INTERNAL /Zi
REM "build_all.bat" /O2 /Zi REM "build_all.bat" /O2 /Zi

2
power/4coder_casey.cpp
View File

@ -687,6 +687,8 @@ CUSTOM_COMMAND_SIG(casey_save_and_make_without_asking)
CLI_OverlapWithConflict); CLI_OverlapWithConflict);
} }
exec_command(app, change_active_panel); exec_command(app, change_active_panel);
prev_location = null_location;
} }
#if 0 #if 0

15
test/4cpp_new_lexer.h
View File

@ -1032,10 +1032,11 @@ cpp_lex_size_nonalloc(Lex_Data *S_ptr,
} }
lexer_link Cpp_Relex_State lexer_link Cpp_Relex_State
cpp_relex_nonalloc_start(Cpp_File file, Cpp_Token_Stack *stack, cpp_relex_nonalloc_start(char *data, int size, Cpp_Token_Stack *stack,
int start, int end, int amount, int tolerance){ int start, int end, int amount, int tolerance){
Cpp_Relex_State state; Cpp_Relex_State state;
state.file = file; state.data = data;
state.size = size;
state.stack = stack; state.stack = stack;
state.start = start; state.start = start;
state.end = end; state.end = end;
@ -1107,7 +1108,7 @@ cpp_relex_nonalloc_main(Cpp_Relex_State *state,
lex.pos = state->relex_start; lex.pos = state->relex_start;
int relex_end_i = state->end_token_i; int relex_end_i = state->end_token_i;
Cpp_Token match_token = cpp__get_token(stack, tokens, state->file.size, relex_end_i); Cpp_Token match_token = cpp__get_token(stack, tokens, state->size, relex_end_i);
Cpp_Token end_token = match_token; Cpp_Token end_token = match_token;
int went_too_far = false; int went_too_far = false;
@ -1115,9 +1116,9 @@ cpp_relex_nonalloc_main(Cpp_Relex_State *state,
for (;;){ for (;;){
int result = int result =
cpp_lex_size_nonalloc(&lex, cpp_lex_size_nonalloc(&lex,
state->file.data, state->data,
state->file.size, state->size,
state->file.size, state->size,
relex_stack, 1); relex_stack, 1);
switch (result){ switch (result){
@ -1134,7 +1135,7 @@ cpp_relex_nonalloc_main(Cpp_Relex_State *state,
while (lex.pos > end_token.start && relex_end_i < stack->count){ while (lex.pos > end_token.start && relex_end_i < stack->count){
++relex_end_i; ++relex_end_i;
end_token = cpp__get_token(stack, tokens, state->file.size, relex_end_i); end_token = cpp__get_token(stack, tokens, state->size, relex_end_i);
} }
} }
break; break;

30
test/experiment.cpp
View File

@ -13,7 +13,7 @@
// TOP // TOP
#include "../4ed_meta.h" #include "../4ed_meta.h"
#define FCPP_STRING_IMPLEMENTATION #define FSTRING_IMPLEMENTATION
#include "../4coder_string.h" #include "../4coder_string.h"
#include "../4cpp_types.h" #include "../4cpp_types.h"
@ -94,12 +94,11 @@ system_set_file_list(File_List *file_list, String directory){
append(&dir, directory); append(&dir, directory);
char trail_str[] = "\\*"; char trail_str[] = "\\*";
append(&dir, trail_str); append(&dir, trail_str);
terminate_with_null(&dir);
char *c_str_dir = make_c_str(dir);
WIN32_FIND_DATA find_data; WIN32_FIND_DATA find_data;
HANDLE search; HANDLE search;
search = FindFirstFileA(c_str_dir, &find_data); search = FindFirstFileA(dir.str, &find_data);
if (search != INVALID_HANDLE_VALUE){ if (search != INVALID_HANDLE_VALUE){
i32 count = 0; i32 count = 0;
@ -127,7 +126,7 @@ system_set_file_list(File_List *file_list, String directory){
file_list->infos = (File_Info*)file_list->block; file_list->infos = (File_Info*)file_list->block;
char *name = (char*)(file_list->infos + file_count); char *name = (char*)(file_list->infos + file_count);
if (file_list->block){ if (file_list->block){
search = FindFirstFileA(c_str_dir, &find_data); search = FindFirstFileA(dir.str, &find_data);
if (search != INVALID_HANDLE_VALUE){ if (search != INVALID_HANDLE_VALUE){
File_Info *info = file_list->infos; File_Info *info = file_list->infos;
@ -143,7 +142,7 @@ system_set_file_list(File_List *file_list, String directory){
info->filename.size = i; info->filename.size = i;
info->filename.memory_size = info->filename.size + 1; info->filename.memory_size = info->filename.size + 1;
*name++ = 0; *name++ = 0;
replace_char(info->filename, '\\', '/'); replace_char(&info->filename, '\\', '/');
++info; ++info;
} }
more_files = FindNextFile(search, &find_data); more_files = FindNextFile(search, &find_data);
@ -212,7 +211,6 @@ run_experiment(Experiment *exp, char *filename, int verbose,
int chunks, int max_tokens){ int chunks, int max_tokens){
String extension = {}; String extension = {};
Data file_data; Data file_data;
Cpp_File file_cpp;
new_lex::Lex_Data ld = {0}; new_lex::Lex_Data ld = {0};
int pass; int pass;
int k, chunk_size, is_last; int k, chunk_size, is_last;
@ -232,8 +230,8 @@ run_experiment(Experiment *exp, char *filename, int verbose,
memset(exp->correct_stack.tokens, 0, TOKEN_ARRAY_SIZE); memset(exp->correct_stack.tokens, 0, TOKEN_ARRAY_SIZE);
memset(exp->testing_stack.tokens, 0, TOKEN_ARRAY_SIZE); memset(exp->testing_stack.tokens, 0, TOKEN_ARRAY_SIZE);
file_cpp.data = (char*)file_data.data; char *data = (char*)file_data.data;
file_cpp.size = file_data.size; int size = file_data.size;
ld.tb = (char*)malloc(file_data.size + 1); ld.tb = (char*)malloc(file_data.size + 1);
@ -241,7 +239,7 @@ run_experiment(Experiment *exp, char *filename, int verbose,
i64 start; i64 start;
start = __rdtsc(); start = __rdtsc();
cpp_lex_file_nonalloc(file_cpp, &exp->correct_stack, lex_data); cpp_lex_file_nonalloc(data, size, &exp->correct_stack, lex_data);
time.handcoded += (__rdtsc() - start); time.handcoded += (__rdtsc() - start);
if (max_tokens == 0){ if (max_tokens == 0){
@ -357,8 +355,8 @@ run_experiment(Experiment *exp, char *filename, int verbose,
" %.*s original %.*s testing\n", " %.*s original %.*s testing\n",
j, j,
correct->start, correct->size, testing->start, testing->size, correct->start, correct->size, testing->start, testing->size,
correct->size, file_cpp.data + correct->start, correct->size, data + correct->start,
testing->size, file_cpp.data + testing->start); testing->size, data + testing->start);
} }
} }
@ -389,15 +387,15 @@ show_time(Times t, int repeats, char *type){
f32 speed_up = ((f32)t.handcoded) / (t.fsm); f32 speed_up = ((f32)t.handcoded) / (t.fsm);
printf( printf(
"\n%s time for %d repeates\n" "\n%s time for %d repeates\n"
OUTLINE("%lld") OUTLINE("%lld")
OUTLINE("%lld") OUTLINE("%lld")
OUTLINE("%f"), OUTLINE("%f"),
type, type,
repeats, repeats,
OUTLINE_VAR(i64, t.handcoded), OUTLINE_VAR(i64, t.handcoded),
OUTLINE_VAR(i64, t.fsm), OUTLINE_VAR(i64, t.fsm),
OUTLINE_VAR(f32, speed_up) OUTLINE_VAR(f32, speed_up)
); );
} }
#define BASE_DIR "w:/4ed/data/test/" #define BASE_DIR "w:/4ed/data/test/"