diff --git a/4coder_default_framework.h b/4coder_default_framework.h index b1d0b50f..38c5e05a 100644 --- a/4coder_default_framework.h +++ b/4coder_default_framework.h @@ -627,6 +627,23 @@ config_array_good(Config_Array_Reader *array_reader){ } +// +// Lexer Helper +// + +static void +lexer_keywords_default_init(Partition *part, Cpp_Keyword_Table *kw_out, Cpp_Keyword_Table *pp_out){ + size_t kw_size = cpp_get_table_memory_size_default(CPP_TABLE_KEYWORDS); + size_t pp_size = cpp_get_table_memory_size_default(CPP_TABLE_PREPROCESSOR_DIRECTIVES); + + void *kw_mem = push_block(part, (i32_4tech)kw_size); + void *pp_mem = push_block(part, (i32_4tech)pp_size); + + *kw_out = cpp_make_table_default(CPP_TABLE_KEYWORDS, kw_mem, kw_size); + *pp_out = cpp_make_table_default(CPP_TABLE_PREPROCESSOR_DIRECTIVES, pp_mem, pp_size); +} + + // // Configuration // @@ -748,7 +765,11 @@ process_config_file(Application_Links *app){ array.max_count = (1 << 20)/sizeof(Cpp_Token); array.tokens = push_array(&global_part, Cpp_Token, array.max_count); - Cpp_Lex_Data S = cpp_lex_data_init(false); + Cpp_Keyword_Table kw_table = {0}; + Cpp_Keyword_Table pp_table = {0}; + lexer_keywords_default_init(part, &kw_table, &pp_table); + + Cpp_Lex_Data S = cpp_lex_data_init(false, kw_table, pp_table); Cpp_Lex_Result result = cpp_lex_step(&S, mem, size+1, HAS_NULL_TERM, &array, NO_OUT_LIMIT); if (result == LexResult_Finished){ diff --git a/4coder_project_commands.cpp b/4coder_project_commands.cpp index 50767165..7b2a6c7c 100644 --- a/4coder_project_commands.cpp +++ b/4coder_project_commands.cpp @@ -199,7 +199,11 @@ load_project_from_file(Application_Links *app, Partition *part, FILE *file, Stri array.max_count = (1 << 20)/sizeof(Cpp_Token); array.tokens = push_array(&global_part, Cpp_Token, array.max_count); - Cpp_Lex_Data S = cpp_lex_data_init(false); + Cpp_Keyword_Table kw_table = {0}; + Cpp_Keyword_Table pp_table = {0}; + lexer_keywords_default_init(part, &kw_table, &pp_table); + + Cpp_Lex_Data S = cpp_lex_data_init(false, kw_table, pp_table); Cpp_Lex_Result result = cpp_lex_step(&S, mem, size+1, HAS_NULL_TERM, &array, NO_OUT_LIMIT); if (result == LexResult_Finished){ diff --git a/4cpp/4cpp_default_keywords.h b/4cpp/4cpp_default_keywords.h new file mode 100644 index 00000000..279f1bd2 --- /dev/null +++ b/4cpp/4cpp_default_keywords.h @@ -0,0 +1,159 @@ + +// For a quick way to extend the default keywords: +// #define FCPP_LEXER_EXTRA_KEYWORDS "my_keywords.h" +// And in the file "my_keywords.h", list the keywords you want. + +// For a quick way to extend the default preprocessor +// directives do the same thing with the macro: +// #define FCPP_LEXER_EXTRA_PREPROPS + +// TOP + +#if !defined(FCPP_DEFAULT_KEYWORDS_H) +#define FCPP_DEFAULT_KEYWORDS_H + +#define make_stafl(s,f) (s), sizeof(s)-1, f + +struct String_And_Flag{ + char *str; + u32_4tech length; + u32_4tech flags; +}; + +static String_And_Flag default_preprops[] = { + {make_stafl("include" , CPP_PP_INCLUDE )} , + {make_stafl("INCLUDE" , CPP_PP_INCLUDE )} , + {make_stafl("version" , CPP_PP_VERSION )} , + {make_stafl("VERSION" , CPP_PP_VERSION )} , + {make_stafl("ifndef" , CPP_PP_IFNDEF )} , + {make_stafl("IFNDEF" , CPP_PP_IFNDEF )} , + {make_stafl("define" , CPP_PP_DEFINE )} , + {make_stafl("DEFINE" , CPP_PP_DEFINE )} , + {make_stafl("import" , CPP_PP_IMPORT )} , + {make_stafl("IMPORT" , CPP_PP_IMPORT )} , + {make_stafl("pragma" , CPP_PP_PRAGMA )} , + {make_stafl("PRAGMA" , CPP_PP_PRAGMA )} , + {make_stafl("undef" , CPP_PP_UNDEF )} , + {make_stafl("UNDEF" , CPP_PP_UNDEF )} , + {make_stafl("endif" , CPP_PP_ENDIF )} , + {make_stafl("ENDIF" , CPP_PP_ENDIF )} , + {make_stafl("error" , CPP_PP_ERROR )} , + {make_stafl("ERROR" , CPP_PP_ERROR )} , + {make_stafl("ifdef" , CPP_PP_IFDEF )} , + {make_stafl("IFDEF" , CPP_PP_IFDEF )} , + {make_stafl("using" , CPP_PP_USING )} , + {make_stafl("USING" , CPP_PP_USING )} , + {make_stafl("else" , CPP_PP_ELSE )} , + {make_stafl("ELSE" , CPP_PP_ELSE )} , + {make_stafl("elif" , CPP_PP_ELIF )} , + {make_stafl("ELIF" , CPP_PP_ELIF )} , + {make_stafl("line" , CPP_PP_LINE )} , + {make_stafl("LINE" , CPP_PP_LINE )} , + {make_stafl("if" , CPP_PP_IF )} , + {make_stafl("IF" , CPP_PP_IF )} , + +#if defined(FCPP_LEXER_EXTRA_PREPROPS) +#include FCPP_LEXER_EXTRA_PREPROPS +#endif +}; +static i32_4tech default_preprops_count = sizeof(default_preprops)/sizeof(default_preprops[0]); + +static String_And_Flag default_keywords[] = { + {make_stafl("true" , CPP_TOKEN_BOOLEAN_CONSTANT)}, + {make_stafl("false" , CPP_TOKEN_BOOLEAN_CONSTANT)}, + + {make_stafl("and" , CPP_TOKEN_AND)}, + {make_stafl("and_eq" , CPP_TOKEN_ANDEQ)}, + {make_stafl("bitand" , CPP_TOKEN_BIT_AND)}, + {make_stafl("bitor" , CPP_TOKEN_BIT_OR)}, + {make_stafl("or" , CPP_TOKEN_OR)}, + {make_stafl("or_eq" , CPP_TOKEN_OREQ)}, + {make_stafl("sizeof" , CPP_TOKEN_SIZEOF)}, + {make_stafl("alignof" , CPP_TOKEN_ALIGNOF)}, + {make_stafl("decltype" , CPP_TOKEN_DECLTYPE)}, + {make_stafl("throw" , CPP_TOKEN_THROW)}, + {make_stafl("new" , CPP_TOKEN_NEW)}, + {make_stafl("delete" , CPP_TOKEN_DELETE)}, + {make_stafl("xor" , CPP_TOKEN_BIT_XOR)}, + {make_stafl("xor_eq" , CPP_TOKEN_XOREQ)}, + {make_stafl("not" , CPP_TOKEN_NOT)}, + {make_stafl("not_eq" , CPP_TOKEN_NOTEQ)}, + {make_stafl("typeid" , CPP_TOKEN_TYPEID)}, + {make_stafl("compl" , CPP_TOKEN_BIT_NOT)}, + + {make_stafl("void" , CPP_TOKEN_KEY_TYPE)}, + {make_stafl("bool" , CPP_TOKEN_KEY_TYPE)}, + {make_stafl("char" , CPP_TOKEN_KEY_TYPE)}, + {make_stafl("int" , CPP_TOKEN_KEY_TYPE)}, + {make_stafl("float" , CPP_TOKEN_KEY_TYPE)}, + {make_stafl("double" , CPP_TOKEN_KEY_TYPE)}, + + {make_stafl("long" , CPP_TOKEN_KEY_MODIFIER)}, + {make_stafl("short" , CPP_TOKEN_KEY_MODIFIER)}, + {make_stafl("unsigned" , CPP_TOKEN_KEY_MODIFIER)}, + + {make_stafl("const" , CPP_TOKEN_KEY_QUALIFIER)}, + {make_stafl("volatile" , CPP_TOKEN_KEY_QUALIFIER)}, + + {make_stafl("asm" , CPP_TOKEN_KEY_CONTROL_FLOW)}, + {make_stafl("break" , CPP_TOKEN_KEY_CONTROL_FLOW)}, + {make_stafl("case" , CPP_TOKEN_KEY_CONTROL_FLOW)}, + {make_stafl("catch" , CPP_TOKEN_KEY_CONTROL_FLOW)}, + {make_stafl("continue" , CPP_TOKEN_KEY_CONTROL_FLOW)}, + {make_stafl("default" , CPP_TOKEN_KEY_CONTROL_FLOW)}, + {make_stafl("do" , CPP_TOKEN_KEY_CONTROL_FLOW)}, + {make_stafl("else" , CPP_TOKEN_KEY_CONTROL_FLOW)}, + {make_stafl("for" , CPP_TOKEN_KEY_CONTROL_FLOW)}, + {make_stafl("goto" , CPP_TOKEN_KEY_CONTROL_FLOW)}, + {make_stafl("if" , CPP_TOKEN_KEY_CONTROL_FLOW)}, + {make_stafl("return" , CPP_TOKEN_KEY_CONTROL_FLOW)}, + {make_stafl("switch" , CPP_TOKEN_KEY_CONTROL_FLOW)}, + {make_stafl("try" , CPP_TOKEN_KEY_CONTROL_FLOW)}, + {make_stafl("while" , CPP_TOKEN_KEY_CONTROL_FLOW)}, + {make_stafl("static_assert" , CPP_TOKEN_KEY_CONTROL_FLOW)}, + + {make_stafl("const_cast" , CPP_TOKEN_KEY_CAST)}, + {make_stafl("dynamic_cast" , CPP_TOKEN_KEY_CAST)}, + {make_stafl("reinterpret_cast" , CPP_TOKEN_KEY_CAST)}, + {make_stafl("static_cast" , CPP_TOKEN_KEY_CAST)}, + + {make_stafl("class" , CPP_TOKEN_KEY_TYPE_DECLARATION)}, + {make_stafl("enum" , CPP_TOKEN_KEY_TYPE_DECLARATION)}, + {make_stafl("struct" , CPP_TOKEN_KEY_TYPE_DECLARATION)}, + {make_stafl("typedef" , CPP_TOKEN_KEY_TYPE_DECLARATION)}, + {make_stafl("union" , CPP_TOKEN_KEY_TYPE_DECLARATION)}, + {make_stafl("template" , CPP_TOKEN_KEY_TYPE_DECLARATION)}, + {make_stafl("typename" , CPP_TOKEN_KEY_TYPE_DECLARATION)}, + + {make_stafl("friend" , CPP_TOKEN_KEY_ACCESS)}, + {make_stafl("namespace" , CPP_TOKEN_KEY_ACCESS)}, + {make_stafl("private" , CPP_TOKEN_KEY_ACCESS)}, + {make_stafl("protected" , CPP_TOKEN_KEY_ACCESS)}, + {make_stafl("public" , CPP_TOKEN_KEY_ACCESS)}, + {make_stafl("using" , CPP_TOKEN_KEY_ACCESS)}, + + {make_stafl("extern" , CPP_TOKEN_KEY_LINKAGE)}, + {make_stafl("export" , CPP_TOKEN_KEY_LINKAGE)}, + {make_stafl("inline" , CPP_TOKEN_KEY_LINKAGE)}, + {make_stafl("static" , CPP_TOKEN_KEY_LINKAGE)}, + {make_stafl("virtual" , CPP_TOKEN_KEY_LINKAGE)}, + + {make_stafl("alignas" , CPP_TOKEN_KEY_OTHER)}, + {make_stafl("explicit" , CPP_TOKEN_KEY_OTHER)}, + {make_stafl("noexcept" , CPP_TOKEN_KEY_OTHER)}, + {make_stafl("nullptr" , CPP_TOKEN_KEY_OTHER)}, + {make_stafl("operator" , CPP_TOKEN_KEY_OTHER)}, + {make_stafl("register" , CPP_TOKEN_KEY_OTHER)}, + {make_stafl("this" , CPP_TOKEN_KEY_OTHER)}, + {make_stafl("thread_local" , CPP_TOKEN_KEY_OTHER)}, + +#if defined(FCPP_LEXER_EXTRA_KEYWORDS) +#include FCPP_LEXER_EXTRA_KEYWORDS +#endif +}; +static i32_4tech default_keywords_count = sizeof(default_keywords)/sizeof(default_keywords[0]); + +#endif + +// BOTTOM + diff --git a/4cpp/4cpp_lexer.h b/4cpp/4cpp_lexer.h index 075743b4..0609a9ba 100644 --- a/4cpp/4cpp_lexer.h +++ b/4cpp/4cpp_lexer.h @@ -1,5 +1,5 @@ /* -4cpp_lexer.h - 1.0.2 +4cpp_lexer.h - 1.0.3 no warranty implied; use at your own risk This software is in the public domain. Where that dedication is not @@ -26,6 +26,8 @@ typedef uint16_t u16_4tech; typedef uint32_t u32_4tech; typedef uint64_t u64_4tech; +typedef u64_4tech umem_4tech; + typedef float f32_4tech; typedef double f64_4tech; @@ -53,144 +55,280 @@ typedef int32_t b32_4tech; #include "4cpp_lexer_types.h" #include "4cpp_lexer_tables.c" +#include "4cpp_default_keywords.h" -// TODO(allen): revisit this keyword data declaration system -struct String_And_Flag{ - char *str; - u32_4tech length; - u32_4tech flags; -}; +//////////////// -#define make_stafl(s,f) (s), sizeof(s)-1, f +FCPP_LINK umem_4tech +cpp_get_table_memory_size_null_terminated(char **str_array, u32_4tech str_count){ + umem_4tech memsize = 0; + for (u32_4tech i = 0; i < str_count; ++i){ + char *str = str_array[i]; + u32_4tech len = 0; + for (; str[len]; ++len); + memsize += 8 + (len+3)&(~3); + } + u32_4tech table_count = (str_count * 3) / 2; + memsize += table_count*sizeof(void*); + return(memsize); +} -static String_And_Flag preprops[] = { - {make_stafl("include" , CPP_PP_INCLUDE )} , - {make_stafl("INCLUDE" , CPP_PP_INCLUDE )} , - {make_stafl("version" , CPP_PP_VERSION )} , - {make_stafl("VERSION" , CPP_PP_VERSION )} , - {make_stafl("ifndef" , CPP_PP_IFNDEF )} , - {make_stafl("IFNDEF" , CPP_PP_IFNDEF )} , - {make_stafl("define" , CPP_PP_DEFINE )} , - {make_stafl("DEFINE" , CPP_PP_DEFINE )} , - {make_stafl("import" , CPP_PP_IMPORT )} , - {make_stafl("IMPORT" , CPP_PP_IMPORT )} , - {make_stafl("pragma" , CPP_PP_PRAGMA )} , - {make_stafl("PRAGMA" , CPP_PP_PRAGMA )} , - {make_stafl("undef" , CPP_PP_UNDEF )} , - {make_stafl("UNDEF" , CPP_PP_UNDEF )} , - {make_stafl("endif" , CPP_PP_ENDIF )} , - {make_stafl("ENDIF" , CPP_PP_ENDIF )} , - {make_stafl("error" , CPP_PP_ERROR )} , - {make_stafl("ERROR" , CPP_PP_ERROR )} , - {make_stafl("ifdef" , CPP_PP_IFDEF )} , - {make_stafl("IFDEF" , CPP_PP_IFDEF )} , - {make_stafl("using" , CPP_PP_USING )} , - {make_stafl("USING" , CPP_PP_USING )} , - {make_stafl("else" , CPP_PP_ELSE )} , - {make_stafl("ELSE" , CPP_PP_ELSE )} , - {make_stafl("elif" , CPP_PP_ELIF )} , - {make_stafl("ELIF" , CPP_PP_ELIF )} , - {make_stafl("line" , CPP_PP_LINE )} , - {make_stafl("LINE" , CPP_PP_LINE )} , - {make_stafl("if" , CPP_PP_IF )} , - {make_stafl("IF" , CPP_PP_IF )} , -}; -static i32_4tech preprops_count = sizeof(preprops)/sizeof(preprops[0]); +FCPP_LINK umem_4tech +cpp_get_table_memory_size_string_lengths(u32_4tech *str_len, u32_4tech byte_stride, u32_4tech str_count){ + umem_4tech memsize = 0; + u8_4tech *length_data = (u8_4tech*)str_len; + for (u32_4tech i = 0; i < str_count; ++i, length_data += byte_stride){ + u32_4tech len = *(u32_4tech*)(length_data); + memsize += 8 + (len+3)&(~3); + } + u32_4tech table_count = (str_count * 3)/2; + memsize += table_count*sizeof(void*); + return(memsize); +} -static String_And_Flag keywords[] = { - {make_stafl("true" , CPP_TOKEN_BOOLEAN_CONSTANT)}, - {make_stafl("false" , CPP_TOKEN_BOOLEAN_CONSTANT)}, +FCPP_LINK void +cpp__write_word_data(char **out_ptr, u32_4tech len, u32_4tech type, char *str){ + char *out = *out_ptr; + *(u32_4tech*)out = len; + out += 4; + *(u32_4tech*)out = type; + out += 4; + for (u32_4tech j = 0; str[j]; ++j){ + out[j] = str[j]; + } + len = (len+3)&(~3); + out += len; + *out_ptr = out; +} + +FCPP_LINK b32_4tech +cpp__match(char *a, i32_4tech a_len, char *b, i32_4tech b_len){ + b32_4tech result = false; + if (a_len == b_len){ + char *a_end = a + a_len; + result = true; + for (; a < a_end; ++a, ++b){ + if (*a != *b){ + result = false; + break; + } + } + } + return(result); +} + +FCPP_LINK void +cpp__fill_table(Cpp_Keyword_Table *table, char *str, u32_4tech str_count){ + u64_4tech *keywords = table->keywords; + u8_4tech *base = (u8_4tech*)keywords; + u32_4tech max = table->max; - {make_stafl("and" , CPP_TOKEN_AND)}, - {make_stafl("and_eq" , CPP_TOKEN_ANDEQ)}, - {make_stafl("bitand" , CPP_TOKEN_BIT_AND)}, - {make_stafl("bitor" , CPP_TOKEN_BIT_OR)}, - {make_stafl("or" , CPP_TOKEN_OR)}, - {make_stafl("or_eq" , CPP_TOKEN_OREQ)}, - {make_stafl("sizeof" , CPP_TOKEN_SIZEOF)}, - {make_stafl("alignof" , CPP_TOKEN_ALIGNOF)}, - {make_stafl("decltype" , CPP_TOKEN_DECLTYPE)}, - {make_stafl("throw" , CPP_TOKEN_THROW)}, - {make_stafl("new" , CPP_TOKEN_NEW)}, - {make_stafl("delete" , CPP_TOKEN_DELETE)}, - {make_stafl("xor" , CPP_TOKEN_BIT_XOR)}, - {make_stafl("xor_eq" , CPP_TOKEN_XOREQ)}, - {make_stafl("not" , CPP_TOKEN_NOT)}, - {make_stafl("not_eq" , CPP_TOKEN_NOTEQ)}, - {make_stafl("typeid" , CPP_TOKEN_TYPEID)}, - {make_stafl("compl" , CPP_TOKEN_BIT_NOT)}, + for (u32_4tech i = 0; i < str_count; ++i){ + u32_4tech str_len = *(u32_4tech*)str; + str += 8; + + u32_4tech hash = 0; + for (u32_4tech i = 0; i < str_len; ++i){ + hash = (hash << 5) + (u32_4tech)(str[i]); + } + + u32_4tech first_index = hash % max; + u32_4tech index = first_index; + for (;;){ + u64_4tech *keyword_ptr = keywords + index; + if (*keyword_ptr == 0){ + *keyword_ptr = (u64_4tech)((str - 8) - (char*)base); + break; + } + else{ + u32_4tech *table_str_len = (u32_4tech*)(*keyword_ptr + base); + char *table_str = (char*)(table_str_len + 2); + if (cpp__match(table_str, *table_str_len, str, str_len)){ + break; + } + } + + ++index; + if (index >= max){ + index = 0; + } + if (index == first_index){ + break; + } + } + + str_len = (str_len+3)&(~3); + str += str_len; + } +} + +FCPP_LINK Cpp_Keyword_Table +cpp_make_table(char **str_array, u32_4tech str_stride, u32_4tech *len_array, u32_4tech len_stride, u32_4tech *type_array, u32_4tech type_stride, u32_4tech str_count, void *memory, umem_4tech memsize){ + Cpp_Keyword_Table table = {0}; + table.mem = memory; + table.memsize = memsize; + table.keywords = (u64_4tech*)memory; + table.max = (str_count * 3)/2; + umem_4tech size_of_table = sizeof(*table.keywords)*table.max; - {make_stafl("void" , CPP_TOKEN_KEY_TYPE)}, - {make_stafl("bool" , CPP_TOKEN_KEY_TYPE)}, - {make_stafl("char" , CPP_TOKEN_KEY_TYPE)}, - {make_stafl("int" , CPP_TOKEN_KEY_TYPE)}, - {make_stafl("float" , CPP_TOKEN_KEY_TYPE)}, - {make_stafl("double" , CPP_TOKEN_KEY_TYPE)}, + { + u8_4tech *ptr = (u8_4tech*)memory; + for (umem_4tech i = memsize; i > 0; --i, ++ptr){ + *ptr = 0; + } + } - {make_stafl("long" , CPP_TOKEN_KEY_MODIFIER)}, - {make_stafl("short" , CPP_TOKEN_KEY_MODIFIER)}, - {make_stafl("unsigned" , CPP_TOKEN_KEY_MODIFIER)}, + char *out_base = ((char*)memory) + size_of_table; + char *out_ptr = out_base; - {make_stafl("const" , CPP_TOKEN_KEY_QUALIFIER)}, - {make_stafl("volatile" , CPP_TOKEN_KEY_QUALIFIER)}, + u8_4tech *str_ptr = (u8_4tech*)str_array; + u8_4tech *len_ptr = (u8_4tech*)len_array; + u8_4tech *type_ptr = (u8_4tech*)type_array; - {make_stafl("asm" , CPP_TOKEN_KEY_CONTROL_FLOW)}, - {make_stafl("break" , CPP_TOKEN_KEY_CONTROL_FLOW)}, - {make_stafl("case" , CPP_TOKEN_KEY_CONTROL_FLOW)}, - {make_stafl("catch" , CPP_TOKEN_KEY_CONTROL_FLOW)}, - {make_stafl("continue" , CPP_TOKEN_KEY_CONTROL_FLOW)}, - {make_stafl("default" , CPP_TOKEN_KEY_CONTROL_FLOW)}, - {make_stafl("do" , CPP_TOKEN_KEY_CONTROL_FLOW)}, - {make_stafl("else" , CPP_TOKEN_KEY_CONTROL_FLOW)}, - {make_stafl("for" , CPP_TOKEN_KEY_CONTROL_FLOW)}, - {make_stafl("goto" , CPP_TOKEN_KEY_CONTROL_FLOW)}, - {make_stafl("if" , CPP_TOKEN_KEY_CONTROL_FLOW)}, - {make_stafl("return" , CPP_TOKEN_KEY_CONTROL_FLOW)}, - {make_stafl("switch" , CPP_TOKEN_KEY_CONTROL_FLOW)}, - {make_stafl("try" , CPP_TOKEN_KEY_CONTROL_FLOW)}, - {make_stafl("while" , CPP_TOKEN_KEY_CONTROL_FLOW)}, - {make_stafl("static_assert" , CPP_TOKEN_KEY_CONTROL_FLOW)}, + if (len_ptr == 0){ + if (type_ptr == 0){ + for (u32_4tech i = 0; i < str_count; ++i){ + char *str_item = *(char**)str_ptr; + str_ptr += str_stride; + u32_4tech len = 0; + for (; str_item[len]; ++len); + cpp__write_word_data(&out_ptr, len, CPP_TOKEN_KEY_OTHER, str_item); + } + } + else{ + for (u32_4tech i = 0; i < str_count; ++i){ + char *str_item = *(char**)str_ptr; + str_ptr += str_stride; + u32_4tech len = 0; + for (; str_item[len]; ++len); + u32_4tech type = *(u32_4tech*)(type_ptr); + cpp__write_word_data(&out_ptr, len, type, str_item); + } + } + } + else{ + if (type_ptr == 0){ + for (u32_4tech i = 0; i < str_count; ++i){ + char *str_item = *(char**)str_ptr; + str_ptr += str_stride; + u32_4tech len = *(u32_4tech*)(len_ptr); + len_ptr += len_stride; + cpp__write_word_data(&out_ptr, len, CPP_TOKEN_KEY_OTHER, str_item); + } + } + else{ + for (u32_4tech i = 0; i < str_count; ++i){ + char *str_item = *(char**)str_ptr; + str_ptr += str_stride; + u32_4tech len = *(u32_4tech*)(len_ptr); + len_ptr += len_stride; + u32_4tech type = *(u32_4tech*)(type_ptr); + type_ptr += type_stride; + cpp__write_word_data(&out_ptr, len, type, str_item); + } + } + } - {make_stafl("const_cast" , CPP_TOKEN_KEY_CAST)}, - {make_stafl("dynamic_cast" , CPP_TOKEN_KEY_CAST)}, - {make_stafl("reinterpret_cast" , CPP_TOKEN_KEY_CAST)}, - {make_stafl("static_cast" , CPP_TOKEN_KEY_CAST)}, + cpp__fill_table(&table, out_base, str_count); - {make_stafl("class" , CPP_TOKEN_KEY_TYPE_DECLARATION)}, - {make_stafl("enum" , CPP_TOKEN_KEY_TYPE_DECLARATION)}, - {make_stafl("struct" , CPP_TOKEN_KEY_TYPE_DECLARATION)}, - {make_stafl("typedef" , CPP_TOKEN_KEY_TYPE_DECLARATION)}, - {make_stafl("union" , CPP_TOKEN_KEY_TYPE_DECLARATION)}, - {make_stafl("template" , CPP_TOKEN_KEY_TYPE_DECLARATION)}, - {make_stafl("typename" , CPP_TOKEN_KEY_TYPE_DECLARATION)}, + return(table); +} + +FCPP_LINK b32_4tech +cpp__table_match(Cpp_Keyword_Table *table, char *s, u32_4tech s_len, u32_4tech **item_ptr_out){ + u32_4tech hash = 0; + for (u32_4tech i = 0; i < s_len; ++i){ + hash = (hash << 5) + (u32_4tech)(s[i]); + } - {make_stafl("friend" , CPP_TOKEN_KEY_ACCESS)}, - {make_stafl("namespace" , CPP_TOKEN_KEY_ACCESS)}, - {make_stafl("private" , CPP_TOKEN_KEY_ACCESS)}, - {make_stafl("protected" , CPP_TOKEN_KEY_ACCESS)}, - {make_stafl("public" , CPP_TOKEN_KEY_ACCESS)}, - {make_stafl("using" , CPP_TOKEN_KEY_ACCESS)}, + u64_4tech *keywords = table->keywords; + u8_4tech *base = (u8_4tech*)keywords; - {make_stafl("extern" , CPP_TOKEN_KEY_LINKAGE)}, - {make_stafl("export" , CPP_TOKEN_KEY_LINKAGE)}, - {make_stafl("inline" , CPP_TOKEN_KEY_LINKAGE)}, - {make_stafl("static" , CPP_TOKEN_KEY_LINKAGE)}, - {make_stafl("virtual" , CPP_TOKEN_KEY_LINKAGE)}, + b32_4tech result = false; + u32_4tech max = table->max; + u32_4tech first_index = hash % max; + u32_4tech index = first_index; + for (;;){ + u64_4tech *keyword_ptr = keywords + index; + if (*keyword_ptr == 0){ + break; + } + + u32_4tech *str_len = (u32_4tech*)(*keyword_ptr + base); + char *str = (char*)(str_len + 2); + if (cpp__match(str, *str_len, s, s_len)){ + *item_ptr_out = (u32_4tech*)(*keyword_ptr + base); + result = true; + break; + } + + ++index; + if (index >= max){ + index = 0; + } + if (index == first_index){ + break; + } + } - {make_stafl("alignas" , CPP_TOKEN_KEY_OTHER)}, - {make_stafl("explicit" , CPP_TOKEN_KEY_OTHER)}, - {make_stafl("noexcept" , CPP_TOKEN_KEY_OTHER)}, - {make_stafl("nullptr" , CPP_TOKEN_KEY_OTHER)}, - {make_stafl("operator" , CPP_TOKEN_KEY_OTHER)}, - {make_stafl("register" , CPP_TOKEN_KEY_OTHER)}, - {make_stafl("this" , CPP_TOKEN_KEY_OTHER)}, - {make_stafl("thread_local" , CPP_TOKEN_KEY_OTHER)}, + return(result); +} + +FCPP_LINK umem_4tech +cpp_get_table_memory_size_default(Cpp_Word_Table_Type type){ + u32_4tech *ptr = 0; + u32_4tech count = 0; -#if defined(FCPP_LEXER_EXTRA_KEYWORDS) -#include FCPP_LEXER_EXTRA_KEYWORDS -#endif -}; -static i32_4tech keywords_count = sizeof(keywords)/sizeof(keywords[0]); + switch (type){ + case CPP_TABLE_KEYWORDS: + { + ptr = &default_keywords->length; + count = default_keywords_count; + }break; + + case CPP_TABLE_PREPROCESSOR_DIRECTIVES: + { + ptr = &default_preprops->length; + count = default_preprops_count; + }break; + } + + u32_4tech stride = sizeof(String_And_Flag); + umem_4tech size = cpp_get_table_memory_size_string_lengths(ptr, stride, count); + return(size); +} + +FCPP_LINK Cpp_Keyword_Table +cpp_make_table_default(Cpp_Word_Table_Type type, void *mem, umem_4tech memsize){ + char **str_ptr = 0; + u32_4tech *len_ptr = 0; + u32_4tech *type_ptr = 0; + u32_4tech count = 0; + + switch (type){ + case CPP_TABLE_KEYWORDS: + { + str_ptr = &default_keywords->str; + len_ptr = &default_keywords->length; + type_ptr = &default_keywords->flags; + count = default_keywords_count; + }break; + + case CPP_TABLE_PREPROCESSOR_DIRECTIVES: + { + str_ptr = &default_preprops->str; + len_ptr = &default_preprops->length; + type_ptr = &default_preprops->flags; + count = default_preprops_count; + }break; + } + + u32_4tech stride = sizeof(String_And_Flag); + Cpp_Keyword_Table table = cpp_make_table(str_ptr, stride, len_ptr, stride, type_ptr, stride, count, mem, memsize); + return(table); +} + +//////////////// API_EXPORT FCPP_LINK Cpp_Get_Token_Result cpp_get_token(Cpp_Token_Array array, i32_4tech pos)/* @@ -262,8 +400,9 @@ DOC_SEE(Cpp_Get_Token_Result) return(result); } +// TODO(allen): eliminate this and just make a table. FCPP_LINK Cpp_Lex_PP_State -cpp_pp_directive_to_state(Cpp_Token_Type type){ +cpp__pp_directive_to_state(Cpp_Token_Type type){ Cpp_Lex_PP_State result = LSPP_default; switch (type){ case CPP_PP_INCLUDE: case CPP_PP_IMPORT: case CPP_PP_USING: @@ -301,37 +440,6 @@ cpp_pp_directive_to_state(Cpp_Token_Type type){ return(result); } -FCPP_LINK b32_4tech -cpp__match(char *a, i32_4tech a_len, char *b, i32_4tech b_len){ - b32_4tech result = false; - if (a_len == b_len){ - char *a_end = a + a_len; - result = true; - for (; a < a_end; ++a, ++b){ - if (*a != *b){ - result = false; - break; - } - } - } - return(result); -} - -FCPP_LINK b32_4tech -cpp__table_match(String_And_Flag *table, i32_4tech count, char *s, i32_4tech len, i32_4tech *index_out){ - b32_4tech result = false; - String_And_Flag *entry = table; - *index_out = -1; - for (i32_4tech i = 0; i < count; ++i, ++entry){ - if (cpp__match(entry->str, entry->length, s, len)){ - result = true; - *index_out = i; - break; - } - } - return(result); -} - #define LEXER_TB(n) ((n) & (sizeof(S.tb)-1)) FCPP_LINK Cpp_Lex_Result @@ -360,6 +468,9 @@ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, i32_4tech s DfrCase(10); } + Assert(S.keyword_table.keywords != 0); + Assert(S.preprops_table.keywords != 0); + for (;;){ S.white_done = 0; for(;;){ @@ -507,12 +618,11 @@ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, i32_4tech s } } - i32_4tech sub_match = -1; - cpp__table_match(keywords, keywords_count, S.tb, S.tb_pos-1, &sub_match); + u32_4tech *item_ptr = 0; + cpp__table_match(&S.keyword_table, S.tb, S.tb_pos-1, &item_ptr); - if (sub_match != -1){ - String_And_Flag data = keywords[sub_match]; - S.token.type = (Cpp_Token_Type)data.flags; + if (item_ptr != 0){ + S.token.type = (Cpp_Token_Type)(item_ptr[1]); S.token.flags = CPP_TFLAG_IS_KEYWORD; break; } @@ -554,14 +664,13 @@ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, i32_4tech s } } - i32_4tech sub_match = -1; - cpp__table_match(preprops, preprops_count, S.tb+i, pos-i, &sub_match); + u32_4tech *item_ptr = 0; + cpp__table_match(&S.preprops_table, S.tb+i, S.tb_pos-i-1, &item_ptr); - if (sub_match != -1){ - String_And_Flag data = preprops[sub_match]; - S.token.type = (Cpp_Token_Type)data.flags; + if (item_ptr != 0){ + S.token.type = (Cpp_Token_Type)(item_ptr[1]); S.token.flags = CPP_TFLAG_PP_DIRECTIVE; - S.pp_state = (u8_4tech)cpp_pp_directive_to_state(S.token.type); + S.pp_state = (u8_4tech)cpp__pp_directive_to_state(S.token.type); break; } } @@ -1181,50 +1290,21 @@ DOC_SEE(Cpp_Lex_Result) } API_EXPORT FCPP_LINK Cpp_Lex_Data -cpp_lex_data_init(b32_4tech ignore_string_delims)/* +cpp_lex_data_init(b32_4tech ignore_string_delims, Cpp_Keyword_Table keywords, Cpp_Keyword_Table preprocessor_words)/* DOC_PARAM(ignore_string_delims, TODO) -DOC_RETURN(A brand new lex state ready to begin lexing a file from the beginning.) +DOC_PARAM(keywords, TODO) +DOC_PARAM(preprocessor_words, TODO) +DOC_RETURN(A brand new lex state setup to lex from the beginning of the file.) -DOC(Creates a new lex state in the form of a Cpp_Lex_Data struct and returns the struct. -The system needs a temporary buffer that is as long as the longest token. 4096 is usually -enough but the buffer is not checked, so to be 100% bullet proof it has to be the same length -as the file being lexed.) +DOC(Creates a new lex state in the form of a Cpp_Lex_Data struct and returns the struct.) */{ Cpp_Lex_Data data = {0}; - data.ignore_string_delims = ignore_string_delims; + data.ignore_string_delims = (b8_4tech)ignore_string_delims; + data.keyword_table = keywords; + data.preprops_table = preprocessor_words; return(data); } -API_EXPORT FCPP_LINK i32_4tech -cpp_lex_data_temp_size(Cpp_Lex_Data *lex_data)/* -DOC_PARAM(lex_data, The lex state from which to get the temporary buffer size.) -DOC(This call gets the current size of the temporary buffer in the lexer state so -that you can move to a new temporary buffer by copying the data over.) -DOC_SEE(cpp_lex_data_temp_read) -DOC_SEE(cpp_lex_data_new_temp) -*/{ - i32_4tech result = lex_data->tb_pos; - Assert(lex_data->tb != 0); - return(result); -} - -API_EXPORT FCPP_LINK void -cpp_lex_data_temp_read(Cpp_Lex_Data *lex_data, char *out_buffer)/* -DOC_PARAM(lex_data, The lex state from which to read the temporary buffer.) -DOC_PARAM(out_buffer, The buffer into which the contents of the temporary buffer will be written. -The size of the buffer must be at least the size as returned by cpp_lex_data_temp_size.) -DOC(This call reads the current contents of the temporary buffer.) -DOC_SEE(cpp_lex_data_temp_size) -DOC_SEE(cpp_lex_data_new_temp) -*/{ - i32_4tech size = lex_data->tb_pos; - char *src = lex_data->tb; - char *end = src + size; - for (; src < end; ++src, ++out_buffer){ - *out_buffer = *src; - } -} - FCPP_LINK char cpp_token_get_pp_state(u16_4tech bitfield){ return (char)(bitfield); @@ -1289,24 +1369,18 @@ The start and end points are based on the edited region of the file before the e } API_EXPORT FCPP_LINK Cpp_Relex_Data -cpp_relex_init(Cpp_Token_Array *array, i32_4tech start_pos, i32_4tech end_pos, i32_4tech character_shift_amount, b32_4tech ignore_string_delims) +cpp_relex_init(Cpp_Token_Array *array, i32_4tech start_pos, i32_4tech end_pos, i32_4tech character_shift_amount, b32_4tech ignore_string_delims, Cpp_Keyword_Table keywords, Cpp_Keyword_Table preprocessor_words) /* -DOC_PARAM(array, A pointer to the token array that will be modified by the relex, -this array should already contain the tokens for the previous state of the file.) -DOC_PARAM(start_pos, The start position of the edited region of the file. -The start and end points are based on the edited region of the file before the edit.) -DOC_PARAM(end_pos, The end position of the edited region of the file. -In particular, end_pos is the first character after the edited region not effected by the edit. -Thus if the edited region contained one character end_pos - start_pos should equal 1. -The start and end points are based on the edited region of the file before the edit.) +DOC_PARAM(array, A pointer to the token array that will be modified by the relex, this array should already contain the tokens for the previous state of the file.) +DOC_PARAM(start_pos, The start position of the edited region of the file. The start and end points are based on the edited region of the file before the edit.) +DOC_PARAM(end_pos, The end position of the edited region of the file. In particular, end_pos is the first character after the edited region not effected by the edit. Thus if the edited region contained one character end_pos - start_pos should equal 1. The start and end points are based on the edited region of the file before the edit.) DOC_PARAM(character_shift_amount, The shift in the characters after the edited region.) DOC_PARAM(ignore_string_delims, TODO) +DOC_PARAM(keywords, TODO) +DOC_PARAM(preprocessor_words, TODO) DOC_RETURN(Returns a partially initialized relex state.) -DOC(This call does the first setup step of initializing a relex state. To finish initializing the relex state -you must tell the state about the positioning of the first chunk it will be fed. There are two methods of doing -this, the direct method is with cpp_relex_declare_first_chunk_position, the method that is often more convenient -is with cpp_relex_is_start_chunk. If the file is not chunked the second step of initialization can be skipped.) +DOC(This call does the first setup step of initializing a relex state. To finish initializing the relex state you must tell the state about the positioning of the first chunk it will be fed. There are two methods of doing this, the direct method is with cpp_relex_declare_first_chunk_position, the method that is often more convenient is with cpp_relex_is_start_chunk. If the file is not chunked the second step of initialization can be skipped.) DOC_SEE(cpp_relex_declare_first_chunk_position) DOC_SEE(cpp_relex_is_start_chunk) @@ -1326,7 +1400,7 @@ DOC_SEE(cpp_relex_is_start_chunk) state.character_shift_amount = character_shift_amount; - state.lex = cpp_lex_data_init(ignore_string_delims); + state.lex = cpp_lex_data_init(ignore_string_delims, keywords, preprocessor_words); state.lex.pp_state = cpp_token_get_pp_state(array->tokens[state.start_token_index].state_flags); state.lex.pos = state.relex_start_position; @@ -1589,8 +1663,7 @@ does the necessary replacement of tokens in the array to make it match the new f array->count += shift_amount; } - cpp__block_move(array->tokens + S_ptr->start_token_index, relex_array->tokens, - sizeof(Cpp_Token)*relex_array->count); + cpp__block_move(array->tokens + S_ptr->start_token_index, relex_array->tokens, sizeof(Cpp_Token)*relex_array->count); } API_EXPORT FCPP_LINK void @@ -1683,7 +1756,16 @@ return(array); ) DOC_SEE(cpp_make_token_array) */{ - Cpp_Lex_Data S = cpp_lex_data_init(false); + umem_4tech keywords_memsize = cpp_get_table_memory_size_default(CPP_TABLE_KEYWORDS); + umem_4tech preprocessor_words_memsize = cpp_get_table_memory_size_default(CPP_TABLE_PREPROCESSOR_DIRECTIVES); + + void *keywords_mem = malloc(keywords_memsize); + void *preprocessor_words_mem = malloc(preprocessor_words_memsize); + + Cpp_Keyword_Table keywords = cpp_make_table_default(CPP_TABLE_KEYWORDS, keywords_mem, keywords_memsize); + Cpp_Keyword_Table preprocessor_words = cpp_make_table_default(CPP_TABLE_PREPROCESSOR_DIRECTIVES, preprocessor_words_mem, preprocessor_words_memsize); + + Cpp_Lex_Data S = cpp_lex_data_init(false, keywords, preprocessor_words); i32_4tech quit = 0; char empty = 0; @@ -1718,6 +1800,9 @@ DOC_SEE(cpp_make_token_array) }break; } } + + free(keywords_mem); + free(preprocessor_words_mem); } #endif diff --git a/4cpp/4cpp_lexer_types.h b/4cpp/4cpp_lexer_types.h index 13864a53..57f3e4dc 100644 --- a/4cpp/4cpp_lexer_types.h +++ b/4cpp/4cpp_lexer_types.h @@ -16,6 +16,11 @@ #define STRUCT struct #endif +ENUM(uint32_t, Cpp_Word_Table_Type){ + CPP_TABLE_KEYWORDS, + CPP_TABLE_PREPROCESSOR_DIRECTIVES, +}; + /* DOC(A Cpp_Token_Type classifies a token to make parsing easier. Some types are not actually output by the lexer, but exist because parsers will also make use of token types in their own output.) */ ENUM(uint32_t, Cpp_Token_Type){ CPP_TOKEN_JUNK = 0, @@ -306,6 +311,13 @@ struct Cpp_Lex_FSM{ }; static Cpp_Lex_FSM null_lex_fsm = {0}; +STRUCT Cpp_Keyword_Table{ + void *mem; + umem_4tech memsize; + u64_4tech *keywords; + u32_4tech max; +}; + /* DOC(Cpp_Lex_Data represents the state of the lexer so that the system may be resumable and the user can manage the lexer state and decide when to resume lexing with it. To create a new lexer state call cpp_lex_data_init. The internals of the lex state should not be treated as a part of the public API.) @@ -332,8 +344,10 @@ STRUCT Cpp_Lex_Data{ b8_4tech str_raw; b8_4tech str_include; + b8_4tech ignore_string_delims; - i32_4tech ignore_string_delims; + Cpp_Keyword_Table keyword_table; + Cpp_Keyword_Table preprops_table; i32_4tech __pc__; }; diff --git a/4ed.cpp b/4ed.cpp index 95a95d74..51b1d9e1 100644 --- a/4ed.cpp +++ b/4ed.cpp @@ -438,7 +438,7 @@ COMMAND_DECL(kill_buffer){ } internal void -case_change_range(System_Functions *system, Mem_Options *mem, View *view, Editing_File *file, u8 a, u8 z, u8 char_delta){ +case_change_range(System_Functions *system, Models *models, View *view, Editing_File *file, u8 a, u8 z, u8 char_delta){ Range range = {0}; range.min = Min(view->edit_pos->cursor.pos, view->edit_pos->mark); range.max = Max(view->edit_pos->cursor.pos, view->edit_pos->mark); @@ -453,7 +453,7 @@ case_change_range(System_Functions *system, Mem_Options *mem, View *view, Editin system->cancel_job(BACKGROUND_THREADS, file->state.lex_job); } - file_update_history_before_edit(mem, file, step, 0, hist_normal); + file_update_history_before_edit(&models->mem, file, step, 0, hist_normal); u8 *data = (u8*)file->state.buffer.data; for (i32 i = range.start; i < range.end; ++i){ @@ -463,7 +463,7 @@ case_change_range(System_Functions *system, Mem_Options *mem, View *view, Editin } if (file->state.token_array.tokens){ - file_relex_parallel(system, mem, file, range.start, range.end, 0); + file_relex_parallel(system, models, file, range.start, range.end, 0); } } } @@ -1131,16 +1131,16 @@ App_Read_Command_Line_Sig(app_read_command_line){ return(out_size); } -extern "C" SCROLL_RULE_SIG(fallback_scroll_rule){ - i32 result = 0; +SCROLL_RULE_SIG(fallback_scroll_rule){ + b32 result = false; if (target_x != *scroll_x){ *scroll_x = target_x; - result = 1; + result = true; } if (target_y != *scroll_y){ *scroll_y = target_y; - result = 1; + result = true; } return(result); @@ -1185,7 +1185,6 @@ App_Init_Sig(app_init){ div->next = 0; models->layout.free_divider = dividers; - { models->live_set = &vars->live_set; @@ -1207,6 +1206,13 @@ App_Init_Sig(app_init){ } } + { + umem memsize = KB(8); + void *mem = push_block(partition, (i32)memsize); + parse_context_init_memory(&models->parse_context_memory, mem, memsize); + parse_context_add_default(&models->parse_context_memory, &models->mem.general); + } + { models->scroll_rule = fallback_scroll_rule; models->hook_open_file = 0; diff --git a/4ed_api_implementation.cpp b/4ed_api_implementation.cpp index 30aa2975..2225337b 100644 --- a/4ed_api_implementation.cpp +++ b/4ed_api_implementation.cpp @@ -899,10 +899,10 @@ DOC_SEE(Buffer_Setting_ID) else{ if (value){ if (!file->settings.virtual_white){ - file_first_lex_parallel(system, &models->mem, file); + file_first_lex_parallel(system, models, file); } else{ - file_first_lex_serial(&models->mem, file); + file_first_lex_serial(models, file); } } } @@ -915,10 +915,10 @@ DOC_SEE(Buffer_Setting_ID) file_kill_tokens(system, &models->mem.general, file); file->settings.tokens_without_strings = (b8)value; if (!file->settings.virtual_white){ - file_first_lex_parallel(system, &models->mem, file); + file_first_lex_parallel(system, models, file); } else{ - file_first_lex_serial(&models->mem, file); + file_first_lex_serial(models, file); } } } @@ -1001,34 +1001,34 @@ DOC_SEE(Buffer_Setting_ID) case BufferSetting_ReadOnly: { if (value){ - file->settings.read_only = 1; + file->settings.read_only = true; } else{ - file->settings.read_only = 0; + file->settings.read_only = false; } }break; case BufferSetting_VirtualWhitespace: { - b32 full_remeasure = 0; + b32 full_remeasure = false; if (value){ if (!file->settings.virtual_white){ if (!file->settings.tokens_exist){ - file_first_lex_serial(&models->mem, file); + file_first_lex_serial(models, file); } if (!file->state.still_lexing){ - file->settings.virtual_white = 1; - full_remeasure = 1; + file->settings.virtual_white = true; + full_remeasure = true; } else{ - result = 0; + result = false; } } } else{ if (file->settings.virtual_white){ - file->settings.virtual_white = 0; - full_remeasure = 1; + file->settings.virtual_white = false; + full_remeasure = true; } } diff --git a/4ed_app_models.h b/4ed_app_models.h index 2f98ac56..ec9b3e5a 100644 --- a/4ed_app_models.h +++ b/4ed_app_models.h @@ -78,6 +78,8 @@ struct Models{ Editing_Layout layout; Working_Set working_set; + Parse_Context_Memory parse_context_memory; + struct Live_Views *live_set; Editing_File *message_buffer; Editing_File *scratch_buffer; diff --git a/4ed_app_target.cpp b/4ed_app_target.cpp index 3bda77fd..67eddfd0 100644 --- a/4ed_app_target.cpp +++ b/4ed_app_target.cpp @@ -55,6 +55,7 @@ #include "4ed_file.cpp" #include "4ed_working_set.cpp" #include "4ed_hot_directory.cpp" +#include "4ed_parse_contexts.cpp" #include "4ed_gui.h" #include "4ed_gui.cpp" diff --git a/4ed_doubly_linked_list.cpp b/4ed_doubly_linked_list.cpp index a2cd7052..a3e6b0ad 100644 --- a/4ed_doubly_linked_list.cpp +++ b/4ed_doubly_linked_list.cpp @@ -7,6 +7,8 @@ * */ +// TOP + // NOTE(allen): These macros are setup to work on structs // with a next and prev pointer where the type of the struct // is the same as the type of the next/prev pointers. @@ -19,5 +21,12 @@ // for(dll_items(iterator, sentinel_ptr)){...} #define dll_items(it, st) ((it) = (st)->next); ((it) != (st)); ((it) = (it)->next) +#define sll_init_sentinel(s) do{ (s)->next=(s); }while(0) +#define sll_insert(p,v) do{ (v)->next=(p)->next; (p)->next = (v); }while(0) +#define sll_remove(p,v) do{ Assert((p)->next == (v)); (p)->next = (v)->next; }while(0) + +// for(sll_items(iterator, sentinel_ptr)){...} +#define sll_items(it, st) ((it) = (st)->next); ((it) != (st)); ((it) = (it)->next) + // BOTTOM diff --git a/4ed_file.cpp b/4ed_file.cpp index 11fc83b4..d8a9ceca 100644 --- a/4ed_file.cpp +++ b/4ed_file.cpp @@ -93,6 +93,7 @@ struct Editing_File_Settings{ i32 display_width; i32 minimum_base_display_width; i32 wrap_indicator; + u32 parse_context_id; b32 dos_write_mode; b32 virtual_white; Font_ID font_id; diff --git a/4ed_file_view.cpp b/4ed_file_view.cpp index cf8ab651..65ed21d1 100644 --- a/4ed_file_view.cpp +++ b/4ed_file_view.cpp @@ -2123,29 +2123,40 @@ struct Shift_Information{ i32 start, end, amount; }; -// TODO(allen): I want this code audited soon internal Job_Callback_Sig(job_full_lex){ Editing_File *file = (Editing_File*)data[0]; General_Memory *general = (General_Memory*)data[1]; + Models *models = (Models*)data[2]; + + Parse_Context parse_context = parse_context_get(&models->parse_context_memory, file->settings.parse_context_id, memory->data, memory->size); + if (!parse_context.valid){ + return; + } Gap_Buffer *buffer = &file->state.buffer; i32 text_size = buffer_size(buffer); i32 aligned_buffer_size = (text_size + 3)&(~3); - while (memory->size < aligned_buffer_size){ + for (;memory->size < aligned_buffer_size + parse_context.memory_size;){ system->grow_thread_memory(memory); } - Cpp_Token_Array tokens; - tokens.tokens = (Cpp_Token*)(memory->data); - tokens.max_count = memory->size / sizeof(Cpp_Token); + u8 *data_ptr = (u8*)memory->data; + umem data_size = memory->size; + + data_ptr += parse_context.memory_size; + data_size -= parse_context.memory_size; + + Cpp_Token_Array tokens = {0}; + tokens.tokens = (Cpp_Token*)(data_ptr); + tokens.max_count = (u32)(data_size / sizeof(Cpp_Token)); tokens.count = 0; - b32 still_lexing = 1; + b32 still_lexing = true; - Cpp_Lex_Data lex = cpp_lex_data_init(file->settings.tokens_without_strings); + Cpp_Lex_Data lex = cpp_lex_data_init(file->settings.tokens_without_strings, parse_context.kw_table, parse_context.pp_table); // TODO(allen): deduplicate this against relex char *chunks[3]; @@ -2189,7 +2200,7 @@ Job_Callback_Sig(job_full_lex){ } }break; - case LexResult_Finished: still_lexing = 0; break; + case LexResult_Finished: still_lexing = false; break; } } while (still_lexing); @@ -2223,8 +2234,8 @@ Job_Callback_Sig(job_full_lex){ // NOTE(allen): These are outside the locked section because I don't // think getting these out of order will cause critical bugs, and I // want to minimize what's done in locked sections. - file->state.tokens_complete = 1; - file->state.still_lexing = 0; + file->state.tokens_complete = true; + file->state.still_lexing = false; } internal void @@ -2245,31 +2256,33 @@ file_kill_tokens(System_Functions *system, General_Memory *general, Editing_File } internal void -file_first_lex_parallel(System_Functions *system, Mem_Options *mem, Editing_File *file){ - General_Memory *general = &mem->general; - file->settings.tokens_exist = 1; +file_first_lex_parallel(System_Functions *system, Models *models, Editing_File *file){ + General_Memory *general = &models->mem.general; + file->settings.tokens_exist = true; if (file->is_loading == 0 && file->state.still_lexing == 0){ Assert(file->state.token_array.tokens == 0); - file->state.tokens_complete = 0; - file->state.still_lexing = 1; + file->state.tokens_complete = false; + file->state.still_lexing = true; Job_Data job; job.callback = job_full_lex; job.data[0] = file; job.data[1] = general; + job.data[2] = models; file->state.lex_job = system->post_job(BACKGROUND_THREADS, job); } } internal void -file_first_lex_serial(Mem_Options *mem, Editing_File *file){ +file_first_lex_serial(Models *models, Editing_File *file){ + Mem_Options *mem = &models->mem; Partition *part = &mem->part; General_Memory *general = &mem->general; - file->settings.tokens_exist = 1; + file->settings.tokens_exist = true; - Assert(file->state.still_lexing == 0); + Assert(!file->state.still_lexing); if (file->is_loading == 0){ Assert(file->state.token_array.tokens == 0); @@ -2277,6 +2290,10 @@ file_first_lex_serial(Mem_Options *mem, Editing_File *file){ { Temp_Memory temp = begin_temp_memory(part); + Parse_Context parse_context = parse_context_get(&models->parse_context_memory, file->settings.parse_context_id, partition_current(part), partition_remaining(part)); + Assert(parse_context.valid); + push_block(part, (i32)parse_context.memory_size); + Gap_Buffer *buffer = &file->state.buffer; i32 text_size = buffer_size(buffer); @@ -2287,9 +2304,9 @@ file_first_lex_serial(Mem_Options *mem, Editing_File *file){ tokens.count = 0; tokens.tokens = push_array(part, Cpp_Token, tokens.max_count); - b32 still_lexing = 1; + b32 still_lexing = true; - Cpp_Lex_Data lex = cpp_lex_data_init(file->settings.tokens_without_strings); + Cpp_Lex_Data lex = cpp_lex_data_init(file->settings.tokens_without_strings, parse_context.kw_table, parse_context.pp_table); // TODO(allen): deduplicate this against relex char *chunks[3]; @@ -2338,7 +2355,7 @@ file_first_lex_serial(Mem_Options *mem, Editing_File *file){ tokens.count = 0; if (result == LexResult_Finished){ - still_lexing = 0; + still_lexing = false; } }break; @@ -2364,21 +2381,22 @@ file_first_lex_serial(Mem_Options *mem, Editing_File *file){ end_temp_memory(temp); } - file->state.tokens_complete = 1; + file->state.tokens_complete = true; } } internal b32 -file_relex_parallel(System_Functions *system, Mem_Options *mem, Editing_File *file, i32 start_i, i32 end_i, i32 shift_amount){ +file_relex_parallel(System_Functions *system, Models *models, Editing_File *file, i32 start_i, i32 end_i, i32 shift_amount){ + Mem_Options *mem = &models->mem; General_Memory *general = &mem->general; Partition *part = &mem->part; if (file->state.token_array.tokens == 0){ - file_first_lex_parallel(system, mem, file); - return(0); + file_first_lex_parallel(system, models, file); + return(false); } - b32 result = 1; + b32 result = true; b32 inline_lex = !file->state.still_lexing; if (inline_lex){ Gap_Buffer *buffer = &file->state.buffer; @@ -2391,6 +2409,10 @@ file_relex_parallel(System_Functions *system, Mem_Options *mem, Editing_File *fi relex_range.end_token_index - relex_range.start_token_index + extra_tolerance; Temp_Memory temp = begin_temp_memory(part); + Parse_Context parse_context = parse_context_get(&models->parse_context_memory, file->settings.parse_context_id, partition_current(part), partition_remaining(part)); + Assert(parse_context.valid); + push_block(part, (i32)parse_context.memory_size); + Cpp_Token_Array relex_array; relex_array.count = 0; relex_array.max_count = relex_space_size; @@ -2398,7 +2420,7 @@ file_relex_parallel(System_Functions *system, Mem_Options *mem, Editing_File *fi i32 size = buffer_size(buffer); - Cpp_Relex_Data state = cpp_relex_init(array, start_i, end_i, shift_amount, file->settings.tokens_without_strings); + Cpp_Relex_Data state = cpp_relex_init(array, start_i, end_i, shift_amount, file->settings.tokens_without_strings, parse_context.kw_table, parse_context.pp_table); char *chunks[3]; i32 chunk_sizes[3]; @@ -2438,7 +2460,7 @@ file_relex_parallel(System_Functions *system, Mem_Options *mem, Editing_File *fi case LexResult_NeedTokenMemory: { - inline_lex = 0; + inline_lex = false; }goto doublebreak; case LexResult_Finished: goto doublebreak; @@ -2485,26 +2507,28 @@ file_relex_parallel(System_Functions *system, Mem_Options *mem, Editing_File *fi } } - file->state.still_lexing = 1; + file->state.still_lexing = true; Job_Data job; job.callback = job_full_lex; job.data[0] = file; job.data[1] = general; + job.data[2] = models; file->state.lex_job = system->post_job(BACKGROUND_THREADS, job); - result = 0; + result = false; } return(result); } internal b32 -file_relex_serial(Mem_Options *mem, Editing_File *file, i32 start_i, i32 end_i, i32 shift_amount){ +file_relex_serial(Models *models, Editing_File *file, i32 start_i, i32 end_i, i32 shift_amount){ + Mem_Options *mem = &models->mem; General_Memory *general = &mem->general; Partition *part = &mem->part; if (file->state.token_array.tokens == 0){ - file_first_lex_serial(mem, file); + file_first_lex_serial(models, file); return(1); } @@ -2514,6 +2538,10 @@ file_relex_serial(Mem_Options *mem, Editing_File *file, i32 start_i, i32 end_i, Cpp_Token_Array *array = &file->state.token_array; Temp_Memory temp = begin_temp_memory(part); + Parse_Context parse_context = parse_context_get(&models->parse_context_memory, file->settings.parse_context_id, partition_current(part), partition_remaining(part)); + Assert(parse_context.valid); + push_block(part, (i32)parse_context.memory_size); + Cpp_Token_Array relex_array; relex_array.count = 0; relex_array.max_count = partition_remaining(part) / sizeof(Cpp_Token); @@ -2521,7 +2549,7 @@ file_relex_serial(Mem_Options *mem, Editing_File *file, i32 start_i, i32 end_i, i32 size = buffer_size(buffer); - Cpp_Relex_Data state = cpp_relex_init(array, start_i, end_i, shift_amount, file->settings.tokens_without_strings); + Cpp_Relex_Data state = cpp_relex_init(array, start_i, end_i, shift_amount, file->settings.tokens_without_strings, parse_context.kw_table, parse_context.pp_table); char *chunks[3]; i32 chunk_sizes[3]; @@ -3328,10 +3356,10 @@ file_do_single_edit(System_Functions *system, Models *models, Editing_File *file // NOTE(allen): token fixing if (file->settings.tokens_exist){ if (!file->settings.virtual_white){ - file_relex_parallel(system, mem, file, start, end, shift_amount); + file_relex_parallel(system, models, file, start, end, shift_amount); } else{ - file_relex_serial(mem, file, start, end, shift_amount); + file_relex_serial(models, file, start, end, shift_amount); } } @@ -3415,10 +3443,10 @@ file_do_batch_edit(System_Functions *system, Models *models, Editing_File *file, Buffer_Edit *last_edit = batch + batch_size - 1; if (!file->settings.virtual_white){ - file_relex_parallel(system, mem, file, first_edit->start, last_edit->end, shift_total); + file_relex_parallel(system, models, file, first_edit->start, last_edit->end, shift_total); } else{ - file_relex_serial(mem, file, first_edit->start, last_edit->end, shift_total); + file_relex_serial(models, file, first_edit->start, last_edit->end, shift_total); } } }break; @@ -3819,13 +3847,12 @@ init_normal_file(System_Functions *system, Models *models, Editing_File *file, c String val = make_string(buffer, size); file_create_from_string(system, models, file, val); - Mem_Options *mem = &models->mem; if (file->settings.tokens_exist && file->state.token_array.tokens == 0){ if (!file->settings.virtual_white){ - file_first_lex_parallel(system, mem, file); + file_first_lex_parallel(system, models, file); } else{ - file_first_lex_serial(mem, file); + file_first_lex_serial(models, file); } } } @@ -3835,18 +3862,16 @@ init_read_only_file(System_Functions *system, Models *models, Editing_File *file String val = null_string; file_create_from_string(system, models, file, val, 1); - Mem_Options *mem = &models->mem; if (file->settings.tokens_exist && file->state.token_array.tokens == 0){ if (!file->settings.virtual_white){ - file_first_lex_parallel(system, mem, file); + file_first_lex_parallel(system, models, file); } else{ - file_first_lex_serial(mem, file); + file_first_lex_serial(models, file); } } } - internal void view_open_file(System_Functions *system, Models *models, View *view, String filename){ Working_Set *working_set = &models->working_set; diff --git a/4ed_parse_contexts.cpp b/4ed_parse_contexts.cpp new file mode 100644 index 00000000..88b85703 --- /dev/null +++ b/4ed_parse_contexts.cpp @@ -0,0 +1,179 @@ +/* + * Mr. 4th Dimention - Allen Webster + * + * 19.05.2017 + * + * Parse contexts allocation and locking. + * + */ + +// TOP + +struct Parser_String_And_Type{ + char *str; + u32 length; + u32 type; +}; + +struct Stored_Parse_Context{ + umem memsize; + u64 *kw_keywords; + u64 *pp_keywords; + u32 kw_max; + u32 pp_max; +}; + +struct Stored_Parse_Context_Slot{ + union{ + Stored_Parse_Context_Slot *next; + Stored_Parse_Context *context; + }; + b32 freed; +}; + +struct Parse_Context_Memory{ + Stored_Parse_Context_Slot *parse_context_array; + u32 parse_context_counter; + u32 parse_context_max; + + Stored_Parse_Context_Slot free_sentinel; +}; + +internal void +parse_context_init_memory(Parse_Context_Memory *parse_mem, void *mem, umem memsize){ + sll_init_sentinel(&parse_mem->free_sentinel); + + parse_mem->parse_context_array = (Stored_Parse_Context_Slot*)mem; + parse_mem->parse_context_counter = 0; + parse_mem->parse_context_max = (u32)(memsize / sizeof(*parse_mem->parse_context_array)); +} + +internal u32 +parse_context_add(Parse_Context_Memory *parse_mem, General_Memory *general, Parser_String_And_Type *kw_sats, u32 kw_count, Parser_String_And_Type *pp_sats, u32 pp_count){ + Stored_Parse_Context_Slot *slot = 0; + if (parse_mem->free_sentinel.next != &parse_mem->free_sentinel){ + slot = parse_mem->free_sentinel.next; + sll_remove(&parse_mem->free_sentinel, slot); + } + else if (parse_mem->parse_context_counter < parse_mem->parse_context_max){ + slot = &parse_mem->parse_context_array[parse_mem->parse_context_counter++]; + } + + u32 result = 0; + if (slot != 0){ + u32 stride = sizeof(*kw_sats); + umem kw_memsize = cpp_get_table_memory_size_string_lengths(&kw_sats->length, stride, kw_count); + umem pp_memsize = cpp_get_table_memory_size_string_lengths(&pp_sats->length, stride, pp_count); + + umem memsize = kw_memsize + pp_memsize + sizeof(Stored_Parse_Context); + void *mem = general_memory_allocate(general, (i32)memsize); + + Stored_Parse_Context *parse_context = (Stored_Parse_Context*)mem; + u8 *kw_mem = (u8*)(parse_context+1); + u8 *pp_mem = kw_mem + kw_memsize; + + Cpp_Keyword_Table kw_table = cpp_make_table(&kw_sats->str, stride, &kw_sats->length, stride, &kw_sats->type, stride, kw_count, kw_mem, kw_memsize); + + Cpp_Keyword_Table pp_table = cpp_make_table(&pp_sats->str, stride, &pp_sats->length, stride, &pp_sats->type, stride, pp_count, pp_mem, pp_memsize); + + parse_context->memsize = memsize; + parse_context->kw_keywords = kw_table.keywords; + parse_context->pp_keywords = pp_table.keywords; + parse_context->kw_max = kw_table.max; + parse_context->pp_max = pp_table.max; + slot->context = parse_context; + slot->freed = false; + + result = (u32)(slot - parse_mem->parse_context_array) + parse_mem->parse_context_max; + } + + return(result); +} + +internal u32 +parse_context_add_default(Parse_Context_Memory *parse_mem, General_Memory *general){ + Stored_Parse_Context_Slot *slot = 0; + if (parse_mem->free_sentinel.next != &parse_mem->free_sentinel){ + slot = parse_mem->free_sentinel.next; + sll_remove(&parse_mem->free_sentinel, slot); + } + else if (parse_mem->parse_context_counter < parse_mem->parse_context_max){ + slot = &parse_mem->parse_context_array[parse_mem->parse_context_counter++]; + } + + u32 result = 0; + if (slot != 0){ + umem kw_memsize = cpp_get_table_memory_size_default(CPP_TABLE_KEYWORDS); + umem pp_memsize = cpp_get_table_memory_size_default(CPP_TABLE_PREPROCESSOR_DIRECTIVES); + + umem memsize = kw_memsize + pp_memsize + sizeof(Stored_Parse_Context); + void *mem = general_memory_allocate(general, (i32)(memsize)); + + Stored_Parse_Context *parse_context = (Stored_Parse_Context*)mem; + u8 *kw_mem = (u8*)(parse_context+1); + u8 *pp_mem = kw_mem + kw_memsize; + + Cpp_Keyword_Table kw_table = cpp_make_table_default(CPP_TABLE_KEYWORDS, kw_mem, kw_memsize); + Cpp_Keyword_Table pp_table = cpp_make_table_default(CPP_TABLE_PREPROCESSOR_DIRECTIVES, pp_mem, pp_memsize); + + parse_context->memsize = memsize; + parse_context->kw_keywords = kw_table.keywords; + parse_context->pp_keywords = pp_table.keywords; + parse_context->kw_max = kw_table.max; + parse_context->pp_max = pp_table.max; + slot->context = parse_context; + slot->freed = false; + + result = (u32)(slot - parse_mem->parse_context_array) + parse_mem->parse_context_max; + } + + return(result); +} + +struct Parse_Context{ + b32 valid; + Cpp_Keyword_Table kw_table; + Cpp_Keyword_Table pp_table; + umem memory_size; +}; + +internal Parse_Context +parse_context_get(Parse_Context_Memory *parse_mem, u32 id, void *mem, umem memsize){ + Parse_Context result = {0}; + + Stored_Parse_Context_Slot *slot = 0; + if (id == 0){ + // do nothing + } + else{ + id -= parse_mem->parse_context_max; + } + + if (id < parse_mem->parse_context_counter){ + slot = &parse_mem->parse_context_array[id]; + if (slot->freed){ + slot = 0; + } + } + + if (slot != 0){ + Stored_Parse_Context *context = slot->context; + if (context->memsize < memsize){ + u8 *base_ptr = (u8*)context; + u8 *kw_keywords = (u8*)mem + (((u8*)context->kw_keywords) - base_ptr); + u8 *pp_keywords = (u8*)mem + (((u8*)context->pp_keywords) - base_ptr); + result.valid = true; + result.kw_table.keywords = (u64*)kw_keywords; + result.pp_table.keywords = (u64*)pp_keywords; + result.kw_table.max = context->kw_max; + result.pp_table.max = context->pp_max; + result.memory_size = context->memsize; + memcpy(mem, context, context->memsize); + } + } + + return(result); +} + +// BOTTOM + diff --git a/4ed_system.h b/4ed_system.h index 6481c9b6..6251ae76 100644 --- a/4ed_system.h +++ b/4ed_system.h @@ -139,11 +139,7 @@ struct Thread_Memory{ i32 size; i32 id; }; -inline Thread_Memory -thread_memory_zero(){ - Thread_Memory memory={0}; - return(memory); -} +global Thread_Memory null_thread_memory = {0}; struct Thread_Exchange; struct System_Functions; @@ -152,12 +148,12 @@ struct System_Functions; System_Functions *system, \ Thread_Context *thread, \ Thread_Memory *memory, \ -void *data[2]) +void *data[4]) typedef Job_Callback_Sig(Job_Callback); struct Job_Data{ Job_Callback *callback; - void *data[2]; + void *data[4]; }; struct Full_Job_Data{ diff --git a/meta/fsm_table_generator.cpp b/meta/fsm_table_generator.cpp index 19587a7d..52132479 100644 --- a/meta/fsm_table_generator.cpp +++ b/meta/fsm_table_generator.cpp @@ -21,6 +21,8 @@ typedef uint16_t u16_4tech; typedef uint32_t u32_4tech; typedef uint64_t u64_4tech; +typedef u64_4tech umem_4tech; + typedef float f32_4tech; typedef double f64_4tech; diff --git a/win32_4ed.cpp b/win32_4ed.cpp index dc270174..3d2d82fe 100644 --- a/win32_4ed.cpp +++ b/win32_4ed.cpp @@ -1975,10 +1975,7 @@ WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPSTR lpCmdLine, int nCmdS Thread_Memory thread_memory[ArrayCount(background)]; win32vars.thread_memory = thread_memory; - win32vars.queues[BACKGROUND_THREADS].semaphore = - Win32Handle(CreateSemaphore(0, 0, - win32vars.groups[BACKGROUND_THREADS].count, - 0)); + win32vars.queues[BACKGROUND_THREADS].semaphore =Win32Handle(CreateSemaphore(0, 0, win32vars.groups[BACKGROUND_THREADS].count, 0)); u32 creation_flag = 0; for (i32 i = 0; i < win32vars.groups[BACKGROUND_THREADS].count; ++i){ @@ -1987,7 +1984,7 @@ WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPSTR lpCmdLine, int nCmdS thread->group_id = BACKGROUND_THREADS; Thread_Memory *memory = win32vars.thread_memory + i; - *memory = thread_memory_zero(); + *memory = null_thread_memory; memory->id = thread->id; thread->queue = &win32vars.queues[BACKGROUND_THREADS];