mr4th-public
/
4coder
1
Fork 0
Code Issues Releases Activity

Merge branch 'master' of https://bitbucket.org/4coder/4coder

master
Allen Webster 2017-05-22 15:22:16 -04:00
parent 326cd19708 a4ef272b1b
commit 4630154eaa
16 changed files with 802 additions and 301 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
4coder_default_framework.h
View File

@ -627,6 +627,23 @@ config_array_good(Config_Array_Reader *array_reader){
}
//
// Lexer Helper
//
static void
lexer_keywords_default_init(Partition *part, Cpp_Keyword_Table *kw_out, Cpp_Keyword_Table *pp_out){
size_t kw_size = cpp_get_table_memory_size_default(CPP_TABLE_KEYWORDS);
size_t pp_size = cpp_get_table_memory_size_default(CPP_TABLE_PREPROCESSOR_DIRECTIVES);
void *kw_mem = push_block(part, (i32_4tech)kw_size);
void *pp_mem = push_block(part, (i32_4tech)pp_size);
*kw_out = cpp_make_table_default(CPP_TABLE_KEYWORDS, kw_mem, kw_size);
*pp_out = cpp_make_table_default(CPP_TABLE_PREPROCESSOR_DIRECTIVES, pp_mem, pp_size);
}
//
// Configuration
//
@ -748,7 +765,11 @@ process_config_file(Application_Links *app){
array.max_count = (1 << 20)/sizeof(Cpp_Token);
array.tokens = push_array(&global_part, Cpp_Token, array.max_count);
Cpp_Lex_Data S = cpp_lex_data_init(false);
Cpp_Keyword_Table kw_table = {0};
Cpp_Keyword_Table pp_table = {0};
lexer_keywords_default_init(part, &kw_table, &pp_table);
Cpp_Lex_Data S = cpp_lex_data_init(false, kw_table, pp_table);
Cpp_Lex_Result result = cpp_lex_step(&S, mem, size+1, HAS_NULL_TERM, &array, NO_OUT_LIMIT);
if (result == LexResult_Finished){

6
4coder_project_commands.cpp
View File

@ -199,7 +199,11 @@ load_project_from_file(Application_Links *app, Partition *part, FILE *file, Stri
array.max_count = (1 << 20)/sizeof(Cpp_Token);
array.tokens = push_array(&global_part, Cpp_Token, array.max_count);
Cpp_Lex_Data S = cpp_lex_data_init(false);
Cpp_Keyword_Table kw_table = {0};
Cpp_Keyword_Table pp_table = {0};
lexer_keywords_default_init(part, &kw_table, &pp_table);
Cpp_Lex_Data S = cpp_lex_data_init(false, kw_table, pp_table);
Cpp_Lex_Result result = cpp_lex_step(&S, mem, size+1, HAS_NULL_TERM, &array, NO_OUT_LIMIT);
if (result == LexResult_Finished){

159
4cpp/4cpp_default_keywords.h Normal file
View File

@ -0,0 +1,159 @@
// For a quick way to extend the default keywords:
// #define FCPP_LEXER_EXTRA_KEYWORDS "my_keywords.h"
// And in the file "my_keywords.h", list the keywords you want.
// For a quick way to extend the default preprocessor
// directives do the same thing with the macro:
// #define FCPP_LEXER_EXTRA_PREPROPS
// TOP
#if !defined(FCPP_DEFAULT_KEYWORDS_H)
#define FCPP_DEFAULT_KEYWORDS_H
#define make_stafl(s,f) (s), sizeof(s)-1, f
struct String_And_Flag{
char *str;
u32_4tech length;
u32_4tech flags;
};
static String_And_Flag default_preprops[] = {
{make_stafl("include" , CPP_PP_INCLUDE )} ,
{make_stafl("INCLUDE" , CPP_PP_INCLUDE )} ,
{make_stafl("version" , CPP_PP_VERSION )} ,
{make_stafl("VERSION" , CPP_PP_VERSION )} ,
{make_stafl("ifndef" , CPP_PP_IFNDEF )} ,
{make_stafl("IFNDEF" , CPP_PP_IFNDEF )} ,
{make_stafl("define" , CPP_PP_DEFINE )} ,
{make_stafl("DEFINE" , CPP_PP_DEFINE )} ,
{make_stafl("import" , CPP_PP_IMPORT )} ,
{make_stafl("IMPORT" , CPP_PP_IMPORT )} ,
{make_stafl("pragma" , CPP_PP_PRAGMA )} ,
{make_stafl("PRAGMA" , CPP_PP_PRAGMA )} ,
{make_stafl("undef" , CPP_PP_UNDEF )} ,
{make_stafl("UNDEF" , CPP_PP_UNDEF )} ,
{make_stafl("endif" , CPP_PP_ENDIF )} ,
{make_stafl("ENDIF" , CPP_PP_ENDIF )} ,
{make_stafl("error" , CPP_PP_ERROR )} ,
{make_stafl("ERROR" , CPP_PP_ERROR )} ,
{make_stafl("ifdef" , CPP_PP_IFDEF )} ,
{make_stafl("IFDEF" , CPP_PP_IFDEF )} ,
{make_stafl("using" , CPP_PP_USING )} ,
{make_stafl("USING" , CPP_PP_USING )} ,
{make_stafl("else" , CPP_PP_ELSE )} ,
{make_stafl("ELSE" , CPP_PP_ELSE )} ,
{make_stafl("elif" , CPP_PP_ELIF )} ,
{make_stafl("ELIF" , CPP_PP_ELIF )} ,
{make_stafl("line" , CPP_PP_LINE )} ,
{make_stafl("LINE" , CPP_PP_LINE )} ,
{make_stafl("if" , CPP_PP_IF )} ,
{make_stafl("IF" , CPP_PP_IF )} ,
#if defined(FCPP_LEXER_EXTRA_PREPROPS)
#include FCPP_LEXER_EXTRA_PREPROPS
#endif
};
static i32_4tech default_preprops_count = sizeof(default_preprops)/sizeof(default_preprops[0]);
static String_And_Flag default_keywords[] = {
{make_stafl("true" , CPP_TOKEN_BOOLEAN_CONSTANT)},
{make_stafl("false" , CPP_TOKEN_BOOLEAN_CONSTANT)},
{make_stafl("and" , CPP_TOKEN_AND)},
{make_stafl("and_eq" , CPP_TOKEN_ANDEQ)},
{make_stafl("bitand" , CPP_TOKEN_BIT_AND)},
{make_stafl("bitor" , CPP_TOKEN_BIT_OR)},
{make_stafl("or" , CPP_TOKEN_OR)},
{make_stafl("or_eq" , CPP_TOKEN_OREQ)},
{make_stafl("sizeof" , CPP_TOKEN_SIZEOF)},
{make_stafl("alignof" , CPP_TOKEN_ALIGNOF)},
{make_stafl("decltype" , CPP_TOKEN_DECLTYPE)},
{make_stafl("throw" , CPP_TOKEN_THROW)},
{make_stafl("new" , CPP_TOKEN_NEW)},
{make_stafl("delete" , CPP_TOKEN_DELETE)},
{make_stafl("xor" , CPP_TOKEN_BIT_XOR)},
{make_stafl("xor_eq" , CPP_TOKEN_XOREQ)},
{make_stafl("not" , CPP_TOKEN_NOT)},
{make_stafl("not_eq" , CPP_TOKEN_NOTEQ)},
{make_stafl("typeid" , CPP_TOKEN_TYPEID)},
{make_stafl("compl" , CPP_TOKEN_BIT_NOT)},
{make_stafl("void" , CPP_TOKEN_KEY_TYPE)},
{make_stafl("bool" , CPP_TOKEN_KEY_TYPE)},
{make_stafl("char" , CPP_TOKEN_KEY_TYPE)},
{make_stafl("int" , CPP_TOKEN_KEY_TYPE)},
{make_stafl("float" , CPP_TOKEN_KEY_TYPE)},
{make_stafl("double" , CPP_TOKEN_KEY_TYPE)},
{make_stafl("long" , CPP_TOKEN_KEY_MODIFIER)},
{make_stafl("short" , CPP_TOKEN_KEY_MODIFIER)},
{make_stafl("unsigned" , CPP_TOKEN_KEY_MODIFIER)},
{make_stafl("const" , CPP_TOKEN_KEY_QUALIFIER)},
{make_stafl("volatile" , CPP_TOKEN_KEY_QUALIFIER)},
{make_stafl("asm" , CPP_TOKEN_KEY_CONTROL_FLOW)},
{make_stafl("break" , CPP_TOKEN_KEY_CONTROL_FLOW)},
{make_stafl("case" , CPP_TOKEN_KEY_CONTROL_FLOW)},
{make_stafl("catch" , CPP_TOKEN_KEY_CONTROL_FLOW)},
{make_stafl("continue" , CPP_TOKEN_KEY_CONTROL_FLOW)},
{make_stafl("default" , CPP_TOKEN_KEY_CONTROL_FLOW)},
{make_stafl("do" , CPP_TOKEN_KEY_CONTROL_FLOW)},
{make_stafl("else" , CPP_TOKEN_KEY_CONTROL_FLOW)},
{make_stafl("for" , CPP_TOKEN_KEY_CONTROL_FLOW)},
{make_stafl("goto" , CPP_TOKEN_KEY_CONTROL_FLOW)},
{make_stafl("if" , CPP_TOKEN_KEY_CONTROL_FLOW)},
{make_stafl("return" , CPP_TOKEN_KEY_CONTROL_FLOW)},
{make_stafl("switch" , CPP_TOKEN_KEY_CONTROL_FLOW)},
{make_stafl("try" , CPP_TOKEN_KEY_CONTROL_FLOW)},
{make_stafl("while" , CPP_TOKEN_KEY_CONTROL_FLOW)},
{make_stafl("static_assert" , CPP_TOKEN_KEY_CONTROL_FLOW)},
{make_stafl("const_cast" , CPP_TOKEN_KEY_CAST)},
{make_stafl("dynamic_cast" , CPP_TOKEN_KEY_CAST)},
{make_stafl("reinterpret_cast" , CPP_TOKEN_KEY_CAST)},
{make_stafl("static_cast" , CPP_TOKEN_KEY_CAST)},
{make_stafl("class" , CPP_TOKEN_KEY_TYPE_DECLARATION)},
{make_stafl("enum" , CPP_TOKEN_KEY_TYPE_DECLARATION)},
{make_stafl("struct" , CPP_TOKEN_KEY_TYPE_DECLARATION)},
{make_stafl("typedef" , CPP_TOKEN_KEY_TYPE_DECLARATION)},
{make_stafl("union" , CPP_TOKEN_KEY_TYPE_DECLARATION)},
{make_stafl("template" , CPP_TOKEN_KEY_TYPE_DECLARATION)},
{make_stafl("typename" , CPP_TOKEN_KEY_TYPE_DECLARATION)},
{make_stafl("friend" , CPP_TOKEN_KEY_ACCESS)},
{make_stafl("namespace" , CPP_TOKEN_KEY_ACCESS)},
{make_stafl("private" , CPP_TOKEN_KEY_ACCESS)},
{make_stafl("protected" , CPP_TOKEN_KEY_ACCESS)},
{make_stafl("public" , CPP_TOKEN_KEY_ACCESS)},
{make_stafl("using" , CPP_TOKEN_KEY_ACCESS)},
{make_stafl("extern" , CPP_TOKEN_KEY_LINKAGE)},
{make_stafl("export" , CPP_TOKEN_KEY_LINKAGE)},
{make_stafl("inline" , CPP_TOKEN_KEY_LINKAGE)},
{make_stafl("static" , CPP_TOKEN_KEY_LINKAGE)},
{make_stafl("virtual" , CPP_TOKEN_KEY_LINKAGE)},
{make_stafl("alignas" , CPP_TOKEN_KEY_OTHER)},
{make_stafl("explicit" , CPP_TOKEN_KEY_OTHER)},
{make_stafl("noexcept" , CPP_TOKEN_KEY_OTHER)},
{make_stafl("nullptr" , CPP_TOKEN_KEY_OTHER)},
{make_stafl("operator" , CPP_TOKEN_KEY_OTHER)},
{make_stafl("register" , CPP_TOKEN_KEY_OTHER)},
{make_stafl("this" , CPP_TOKEN_KEY_OTHER)},
{make_stafl("thread_local" , CPP_TOKEN_KEY_OTHER)},
#if defined(FCPP_LEXER_EXTRA_KEYWORDS)
#include FCPP_LEXER_EXTRA_KEYWORDS
#endif
};
static i32_4tech default_keywords_count = sizeof(default_keywords)/sizeof(default_keywords[0]);
#endif
// BOTTOM

527
4cpp/4cpp_lexer.h
View File

@ -1,5 +1,5 @@
/*
4cpp_lexer.h - 1.0.2
4cpp_lexer.h - 1.0.3
no warranty implied; use at your own risk
This software is in the public domain. Where that dedication is not
@ -26,6 +26,8 @@ typedef uint16_t u16_4tech;
typedef uint32_t u32_4tech;
typedef uint64_t u64_4tech;
typedef u64_4tech umem_4tech;
typedef float f32_4tech;
typedef double f64_4tech;
@ -53,144 +55,280 @@ typedef int32_t b32_4tech;
#include "4cpp_lexer_types.h"
#include "4cpp_lexer_tables.c"
#include "4cpp_default_keywords.h"
// TODO(allen): revisit this keyword data declaration system
struct String_And_Flag{
char *str;
u32_4tech length;
u32_4tech flags;
};
////////////////
#define make_stafl(s,f) (s), sizeof(s)-1, f
FCPP_LINK umem_4tech
cpp_get_table_memory_size_null_terminated(char **str_array, u32_4tech str_count){
umem_4tech memsize = 0;
for (u32_4tech i = 0; i < str_count; ++i){
char *str = str_array[i];
u32_4tech len = 0;
for (; str[len]; ++len);
memsize += 8 + (len+3)&(~3);
}
u32_4tech table_count = (str_count * 3) / 2;
memsize += table_count*sizeof(void*);
return(memsize);
}
static String_And_Flag preprops[] = {
{make_stafl("include" , CPP_PP_INCLUDE )} ,
{make_stafl("INCLUDE" , CPP_PP_INCLUDE )} ,
{make_stafl("version" , CPP_PP_VERSION )} ,
{make_stafl("VERSION" , CPP_PP_VERSION )} ,
{make_stafl("ifndef" , CPP_PP_IFNDEF )} ,
{make_stafl("IFNDEF" , CPP_PP_IFNDEF )} ,
{make_stafl("define" , CPP_PP_DEFINE )} ,
{make_stafl("DEFINE" , CPP_PP_DEFINE )} ,
{make_stafl("import" , CPP_PP_IMPORT )} ,
{make_stafl("IMPORT" , CPP_PP_IMPORT )} ,
{make_stafl("pragma" , CPP_PP_PRAGMA )} ,
{make_stafl("PRAGMA" , CPP_PP_PRAGMA )} ,
{make_stafl("undef" , CPP_PP_UNDEF )} ,
{make_stafl("UNDEF" , CPP_PP_UNDEF )} ,
{make_stafl("endif" , CPP_PP_ENDIF )} ,
{make_stafl("ENDIF" , CPP_PP_ENDIF )} ,
{make_stafl("error" , CPP_PP_ERROR )} ,
{make_stafl("ERROR" , CPP_PP_ERROR )} ,
{make_stafl("ifdef" , CPP_PP_IFDEF )} ,
{make_stafl("IFDEF" , CPP_PP_IFDEF )} ,
{make_stafl("using" , CPP_PP_USING )} ,
{make_stafl("USING" , CPP_PP_USING )} ,
{make_stafl("else" , CPP_PP_ELSE )} ,
{make_stafl("ELSE" , CPP_PP_ELSE )} ,
{make_stafl("elif" , CPP_PP_ELIF )} ,
{make_stafl("ELIF" , CPP_PP_ELIF )} ,
{make_stafl("line" , CPP_PP_LINE )} ,
{make_stafl("LINE" , CPP_PP_LINE )} ,
{make_stafl("if" , CPP_PP_IF )} ,
{make_stafl("IF" , CPP_PP_IF )} ,
};
static i32_4tech preprops_count = sizeof(preprops)/sizeof(preprops[0]);
FCPP_LINK umem_4tech
cpp_get_table_memory_size_string_lengths(u32_4tech *str_len, u32_4tech byte_stride, u32_4tech str_count){
umem_4tech memsize = 0;
u8_4tech *length_data = (u8_4tech*)str_len;
for (u32_4tech i = 0; i < str_count; ++i, length_data += byte_stride){
u32_4tech len = *(u32_4tech*)(length_data);
memsize += 8 + (len+3)&(~3);
}
u32_4tech table_count = (str_count * 3)/2;
memsize += table_count*sizeof(void*);
return(memsize);
}
static String_And_Flag keywords[] = {
{make_stafl("true" , CPP_TOKEN_BOOLEAN_CONSTANT)},
{make_stafl("false" , CPP_TOKEN_BOOLEAN_CONSTANT)},
FCPP_LINK void
cpp__write_word_data(char **out_ptr, u32_4tech len, u32_4tech type, char *str){
char *out = *out_ptr;
*(u32_4tech*)out = len;
out += 4;
*(u32_4tech*)out = type;
out += 4;
for (u32_4tech j = 0; str[j]; ++j){
out[j] = str[j];
}
len = (len+3)&(~3);
out += len;
*out_ptr = out;
}
FCPP_LINK b32_4tech
cpp__match(char *a, i32_4tech a_len, char *b, i32_4tech b_len){
b32_4tech result = false;
if (a_len == b_len){
char *a_end = a + a_len;
result = true;
for (; a < a_end; ++a, ++b){
if (*a != *b){
result = false;
break;
}
}
}
return(result);
}
FCPP_LINK void
cpp__fill_table(Cpp_Keyword_Table *table, char *str, u32_4tech str_count){
u64_4tech *keywords = table->keywords;
u8_4tech *base = (u8_4tech*)keywords;
u32_4tech max = table->max;
{make_stafl("and" , CPP_TOKEN_AND)},
{make_stafl("and_eq" , CPP_TOKEN_ANDEQ)},
{make_stafl("bitand" , CPP_TOKEN_BIT_AND)},
{make_stafl("bitor" , CPP_TOKEN_BIT_OR)},
{make_stafl("or" , CPP_TOKEN_OR)},
{make_stafl("or_eq" , CPP_TOKEN_OREQ)},
{make_stafl("sizeof" , CPP_TOKEN_SIZEOF)},
{make_stafl("alignof" , CPP_TOKEN_ALIGNOF)},
{make_stafl("decltype" , CPP_TOKEN_DECLTYPE)},
{make_stafl("throw" , CPP_TOKEN_THROW)},
{make_stafl("new" , CPP_TOKEN_NEW)},
{make_stafl("delete" , CPP_TOKEN_DELETE)},
{make_stafl("xor" , CPP_TOKEN_BIT_XOR)},
{make_stafl("xor_eq" , CPP_TOKEN_XOREQ)},
{make_stafl("not" , CPP_TOKEN_NOT)},
{make_stafl("not_eq" , CPP_TOKEN_NOTEQ)},
{make_stafl("typeid" , CPP_TOKEN_TYPEID)},
{make_stafl("compl" , CPP_TOKEN_BIT_NOT)},
for (u32_4tech i = 0; i < str_count; ++i){
u32_4tech str_len = *(u32_4tech*)str;
str += 8;
u32_4tech hash = 0;
for (u32_4tech i = 0; i < str_len; ++i){
hash = (hash << 5) + (u32_4tech)(str[i]);
}
u32_4tech first_index = hash % max;
u32_4tech index = first_index;
for (;;){
u64_4tech *keyword_ptr = keywords + index;
if (*keyword_ptr == 0){
*keyword_ptr = (u64_4tech)((str - 8) - (char*)base);
break;
}
else{
u32_4tech *table_str_len = (u32_4tech*)(*keyword_ptr + base);
char *table_str = (char*)(table_str_len + 2);
if (cpp__match(table_str, *table_str_len, str, str_len)){
break;
}
}
++index;
if (index >= max){
index = 0;
}
if (index == first_index){
break;
}
}
str_len = (str_len+3)&(~3);
str += str_len;
}
}
FCPP_LINK Cpp_Keyword_Table
cpp_make_table(char **str_array, u32_4tech str_stride, u32_4tech *len_array, u32_4tech len_stride, u32_4tech *type_array, u32_4tech type_stride, u32_4tech str_count, void *memory, umem_4tech memsize){
Cpp_Keyword_Table table = {0};
table.mem = memory;
table.memsize = memsize;
table.keywords = (u64_4tech*)memory;
table.max = (str_count * 3)/2;
umem_4tech size_of_table = sizeof(*table.keywords)*table.max;
{make_stafl("void" , CPP_TOKEN_KEY_TYPE)},
{make_stafl("bool" , CPP_TOKEN_KEY_TYPE)},
{make_stafl("char" , CPP_TOKEN_KEY_TYPE)},
{make_stafl("int" , CPP_TOKEN_KEY_TYPE)},
{make_stafl("float" , CPP_TOKEN_KEY_TYPE)},
{make_stafl("double" , CPP_TOKEN_KEY_TYPE)},
{
u8_4tech *ptr = (u8_4tech*)memory;
for (umem_4tech i = memsize; i > 0; --i, ++ptr){
*ptr = 0;
}
}
{make_stafl("long" , CPP_TOKEN_KEY_MODIFIER)},
{make_stafl("short" , CPP_TOKEN_KEY_MODIFIER)},
{make_stafl("unsigned" , CPP_TOKEN_KEY_MODIFIER)},
char *out_base = ((char*)memory) + size_of_table;
char *out_ptr = out_base;
{make_stafl("const" , CPP_TOKEN_KEY_QUALIFIER)},
{make_stafl("volatile" , CPP_TOKEN_KEY_QUALIFIER)},
u8_4tech *str_ptr = (u8_4tech*)str_array;
u8_4tech *len_ptr = (u8_4tech*)len_array;
u8_4tech *type_ptr = (u8_4tech*)type_array;
{make_stafl("asm" , CPP_TOKEN_KEY_CONTROL_FLOW)},
{make_stafl("break" , CPP_TOKEN_KEY_CONTROL_FLOW)},
{make_stafl("case" , CPP_TOKEN_KEY_CONTROL_FLOW)},
{make_stafl("catch" , CPP_TOKEN_KEY_CONTROL_FLOW)},
{make_stafl("continue" , CPP_TOKEN_KEY_CONTROL_FLOW)},
{make_stafl("default" , CPP_TOKEN_KEY_CONTROL_FLOW)},
{make_stafl("do" , CPP_TOKEN_KEY_CONTROL_FLOW)},
{make_stafl("else" , CPP_TOKEN_KEY_CONTROL_FLOW)},
{make_stafl("for" , CPP_TOKEN_KEY_CONTROL_FLOW)},
{make_stafl("goto" , CPP_TOKEN_KEY_CONTROL_FLOW)},
{make_stafl("if" , CPP_TOKEN_KEY_CONTROL_FLOW)},
{make_stafl("return" , CPP_TOKEN_KEY_CONTROL_FLOW)},
{make_stafl("switch" , CPP_TOKEN_KEY_CONTROL_FLOW)},
{make_stafl("try" , CPP_TOKEN_KEY_CONTROL_FLOW)},
{make_stafl("while" , CPP_TOKEN_KEY_CONTROL_FLOW)},
{make_stafl("static_assert" , CPP_TOKEN_KEY_CONTROL_FLOW)},
if (len_ptr == 0){
if (type_ptr == 0){
for (u32_4tech i = 0; i < str_count; ++i){
char *str_item = *(char**)str_ptr;
str_ptr += str_stride;
u32_4tech len = 0;
for (; str_item[len]; ++len);
cpp__write_word_data(&out_ptr, len, CPP_TOKEN_KEY_OTHER, str_item);
}
}
else{
for (u32_4tech i = 0; i < str_count; ++i){
char *str_item = *(char**)str_ptr;
str_ptr += str_stride;
u32_4tech len = 0;
for (; str_item[len]; ++len);
u32_4tech type = *(u32_4tech*)(type_ptr);
cpp__write_word_data(&out_ptr, len, type, str_item);
}
}
}
else{
if (type_ptr == 0){
for (u32_4tech i = 0; i < str_count; ++i){
char *str_item = *(char**)str_ptr;
str_ptr += str_stride;
u32_4tech len = *(u32_4tech*)(len_ptr);
len_ptr += len_stride;
cpp__write_word_data(&out_ptr, len, CPP_TOKEN_KEY_OTHER, str_item);
}
}
else{
for (u32_4tech i = 0; i < str_count; ++i){
char *str_item = *(char**)str_ptr;
str_ptr += str_stride;
u32_4tech len = *(u32_4tech*)(len_ptr);
len_ptr += len_stride;
u32_4tech type = *(u32_4tech*)(type_ptr);
type_ptr += type_stride;
cpp__write_word_data(&out_ptr, len, type, str_item);
}
}
}
{make_stafl("const_cast" , CPP_TOKEN_KEY_CAST)},
{make_stafl("dynamic_cast" , CPP_TOKEN_KEY_CAST)},
{make_stafl("reinterpret_cast" , CPP_TOKEN_KEY_CAST)},
{make_stafl("static_cast" , CPP_TOKEN_KEY_CAST)},
cpp__fill_table(&table, out_base, str_count);
{make_stafl("class" , CPP_TOKEN_KEY_TYPE_DECLARATION)},
{make_stafl("enum" , CPP_TOKEN_KEY_TYPE_DECLARATION)},
{make_stafl("struct" , CPP_TOKEN_KEY_TYPE_DECLARATION)},
{make_stafl("typedef" , CPP_TOKEN_KEY_TYPE_DECLARATION)},
{make_stafl("union" , CPP_TOKEN_KEY_TYPE_DECLARATION)},
{make_stafl("template" , CPP_TOKEN_KEY_TYPE_DECLARATION)},
{make_stafl("typename" , CPP_TOKEN_KEY_TYPE_DECLARATION)},
return(table);
}
FCPP_LINK b32_4tech
cpp__table_match(Cpp_Keyword_Table *table, char *s, u32_4tech s_len, u32_4tech **item_ptr_out){
u32_4tech hash = 0;
for (u32_4tech i = 0; i < s_len; ++i){
hash = (hash << 5) + (u32_4tech)(s[i]);
}
{make_stafl("friend" , CPP_TOKEN_KEY_ACCESS)},
{make_stafl("namespace" , CPP_TOKEN_KEY_ACCESS)},
{make_stafl("private" , CPP_TOKEN_KEY_ACCESS)},
{make_stafl("protected" , CPP_TOKEN_KEY_ACCESS)},
{make_stafl("public" , CPP_TOKEN_KEY_ACCESS)},
{make_stafl("using" , CPP_TOKEN_KEY_ACCESS)},
u64_4tech *keywords = table->keywords;
u8_4tech *base = (u8_4tech*)keywords;
{make_stafl("extern" , CPP_TOKEN_KEY_LINKAGE)},
{make_stafl("export" , CPP_TOKEN_KEY_LINKAGE)},
{make_stafl("inline" , CPP_TOKEN_KEY_LINKAGE)},
{make_stafl("static" , CPP_TOKEN_KEY_LINKAGE)},
{make_stafl("virtual" , CPP_TOKEN_KEY_LINKAGE)},
b32_4tech result = false;
u32_4tech max = table->max;
u32_4tech first_index = hash % max;
u32_4tech index = first_index;
for (;;){
u64_4tech *keyword_ptr = keywords + index;
if (*keyword_ptr == 0){
break;
}
u32_4tech *str_len = (u32_4tech*)(*keyword_ptr + base);
char *str = (char*)(str_len + 2);
if (cpp__match(str, *str_len, s, s_len)){
*item_ptr_out = (u32_4tech*)(*keyword_ptr + base);
result = true;
break;
}
++index;
if (index >= max){
index = 0;
}
if (index == first_index){
break;
}
}
{make_stafl("alignas" , CPP_TOKEN_KEY_OTHER)},
{make_stafl("explicit" , CPP_TOKEN_KEY_OTHER)},
{make_stafl("noexcept" , CPP_TOKEN_KEY_OTHER)},
{make_stafl("nullptr" , CPP_TOKEN_KEY_OTHER)},
{make_stafl("operator" , CPP_TOKEN_KEY_OTHER)},
{make_stafl("register" , CPP_TOKEN_KEY_OTHER)},
{make_stafl("this" , CPP_TOKEN_KEY_OTHER)},
{make_stafl("thread_local" , CPP_TOKEN_KEY_OTHER)},
return(result);
}
FCPP_LINK umem_4tech
cpp_get_table_memory_size_default(Cpp_Word_Table_Type type){
u32_4tech *ptr = 0;
u32_4tech count = 0;
#if defined(FCPP_LEXER_EXTRA_KEYWORDS)
#include FCPP_LEXER_EXTRA_KEYWORDS
#endif
};
static i32_4tech keywords_count = sizeof(keywords)/sizeof(keywords[0]);
switch (type){
case CPP_TABLE_KEYWORDS:
{
ptr = &default_keywords->length;
count = default_keywords_count;
}break;
case CPP_TABLE_PREPROCESSOR_DIRECTIVES:
{
ptr = &default_preprops->length;
count = default_preprops_count;
}break;
}
u32_4tech stride = sizeof(String_And_Flag);
umem_4tech size = cpp_get_table_memory_size_string_lengths(ptr, stride, count);
return(size);
}
FCPP_LINK Cpp_Keyword_Table
cpp_make_table_default(Cpp_Word_Table_Type type, void *mem, umem_4tech memsize){
char **str_ptr = 0;
u32_4tech *len_ptr = 0;
u32_4tech *type_ptr = 0;
u32_4tech count = 0;
switch (type){
case CPP_TABLE_KEYWORDS:
{
str_ptr = &default_keywords->str;
len_ptr = &default_keywords->length;
type_ptr = &default_keywords->flags;
count = default_keywords_count;
}break;
case CPP_TABLE_PREPROCESSOR_DIRECTIVES:
{
str_ptr = &default_preprops->str;
len_ptr = &default_preprops->length;
type_ptr = &default_preprops->flags;
count = default_preprops_count;
}break;
}
u32_4tech stride = sizeof(String_And_Flag);
Cpp_Keyword_Table table = cpp_make_table(str_ptr, stride, len_ptr, stride, type_ptr, stride, count, mem, memsize);
return(table);
}
////////////////
API_EXPORT FCPP_LINK Cpp_Get_Token_Result
cpp_get_token(Cpp_Token_Array array, i32_4tech pos)/*
@ -262,8 +400,9 @@ DOC_SEE(Cpp_Get_Token_Result)
return(result);
}
// TODO(allen): eliminate this and just make a table.
FCPP_LINK Cpp_Lex_PP_State
cpp_pp_directive_to_state(Cpp_Token_Type type){
cpp__pp_directive_to_state(Cpp_Token_Type type){
Cpp_Lex_PP_State result = LSPP_default;
switch (type){
case CPP_PP_INCLUDE: case CPP_PP_IMPORT: case CPP_PP_USING:
@ -301,37 +440,6 @@ cpp_pp_directive_to_state(Cpp_Token_Type type){
return(result);
}
FCPP_LINK b32_4tech
cpp__match(char *a, i32_4tech a_len, char *b, i32_4tech b_len){
b32_4tech result = false;
if (a_len == b_len){
char *a_end = a + a_len;
result = true;
for (; a < a_end; ++a, ++b){
if (*a != *b){
result = false;
break;
}
}
}
return(result);
}
FCPP_LINK b32_4tech
cpp__table_match(String_And_Flag *table, i32_4tech count, char *s, i32_4tech len, i32_4tech *index_out){
b32_4tech result = false;
String_And_Flag *entry = table;
*index_out = -1;
for (i32_4tech i = 0; i < count; ++i, ++entry){
if (cpp__match(entry->str, entry->length, s, len)){
result = true;
*index_out = i;
break;
}
}
return(result);
}
#define LEXER_TB(n) ((n) & (sizeof(S.tb)-1))
FCPP_LINK Cpp_Lex_Result
@ -360,6 +468,9 @@ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, i32_4tech s
DfrCase(10);
}
Assert(S.keyword_table.keywords != 0);
Assert(S.preprops_table.keywords != 0);
for (;;){
S.white_done = 0;
for(;;){
@ -507,12 +618,11 @@ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, i32_4tech s
}
}
i32_4tech sub_match = -1;
cpp__table_match(keywords, keywords_count, S.tb, S.tb_pos-1, &sub_match);
u32_4tech *item_ptr = 0;
cpp__table_match(&S.keyword_table, S.tb, S.tb_pos-1, &item_ptr);
if (sub_match != -1){
String_And_Flag data = keywords[sub_match];
S.token.type = (Cpp_Token_Type)data.flags;
if (item_ptr != 0){
S.token.type = (Cpp_Token_Type)(item_ptr[1]);
S.token.flags = CPP_TFLAG_IS_KEYWORD;
break;
}
@ -554,14 +664,13 @@ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, i32_4tech s
}
}
i32_4tech sub_match = -1;
cpp__table_match(preprops, preprops_count, S.tb+i, pos-i, &sub_match);
u32_4tech *item_ptr = 0;
cpp__table_match(&S.preprops_table, S.tb+i, S.tb_pos-i-1, &item_ptr);
if (sub_match != -1){
String_And_Flag data = preprops[sub_match];
S.token.type = (Cpp_Token_Type)data.flags;
if (item_ptr != 0){
S.token.type = (Cpp_Token_Type)(item_ptr[1]);
S.token.flags = CPP_TFLAG_PP_DIRECTIVE;
S.pp_state = (u8_4tech)cpp_pp_directive_to_state(S.token.type);
S.pp_state = (u8_4tech)cpp__pp_directive_to_state(S.token.type);
break;
}
}
@ -1181,50 +1290,21 @@ DOC_SEE(Cpp_Lex_Result)
}
API_EXPORT FCPP_LINK Cpp_Lex_Data
cpp_lex_data_init(b32_4tech ignore_string_delims)/*
cpp_lex_data_init(b32_4tech ignore_string_delims, Cpp_Keyword_Table keywords, Cpp_Keyword_Table preprocessor_words)/*
DOC_PARAM(ignore_string_delims, TODO)
DOC_RETURN(A brand new lex state ready to begin lexing a file from the beginning.)
DOC_PARAM(keywords, TODO)
DOC_PARAM(preprocessor_words, TODO)
DOC_RETURN(A brand new lex state setup to lex from the beginning of the file.)
DOC(Creates a new lex state in the form of a Cpp_Lex_Data struct and returns the struct.
The system needs a temporary buffer that is as long as the longest token. 4096 is usually
enough but the buffer is not checked, so to be 100% bullet proof it has to be the same length
as the file being lexed.)
DOC(Creates a new lex state in the form of a Cpp_Lex_Data struct and returns the struct.)
*/{
Cpp_Lex_Data data = {0};
data.ignore_string_delims = ignore_string_delims;
data.ignore_string_delims = (b8_4tech)ignore_string_delims;
data.keyword_table = keywords;
data.preprops_table = preprocessor_words;
return(data);
}
API_EXPORT FCPP_LINK i32_4tech
cpp_lex_data_temp_size(Cpp_Lex_Data *lex_data)/*
DOC_PARAM(lex_data, The lex state from which to get the temporary buffer size.)
DOC(This call gets the current size of the temporary buffer in the lexer state so
that you can move to a new temporary buffer by copying the data over.)
DOC_SEE(cpp_lex_data_temp_read)
DOC_SEE(cpp_lex_data_new_temp)
*/{
i32_4tech result = lex_data->tb_pos;
Assert(lex_data->tb != 0);
return(result);
}
API_EXPORT FCPP_LINK void
cpp_lex_data_temp_read(Cpp_Lex_Data *lex_data, char *out_buffer)/*
DOC_PARAM(lex_data, The lex state from which to read the temporary buffer.)
DOC_PARAM(out_buffer, The buffer into which the contents of the temporary buffer will be written.
The size of the buffer must be at least the size as returned by cpp_lex_data_temp_size.)
DOC(This call reads the current contents of the temporary buffer.)
DOC_SEE(cpp_lex_data_temp_size)
DOC_SEE(cpp_lex_data_new_temp)
*/{
i32_4tech size = lex_data->tb_pos;
char *src = lex_data->tb;
char *end = src + size;
for (; src < end; ++src, ++out_buffer){
*out_buffer = *src;
}
}
FCPP_LINK char
cpp_token_get_pp_state(u16_4tech bitfield){
return (char)(bitfield);
@ -1289,24 +1369,18 @@ The start and end points are based on the edited region of the file before the e
}
API_EXPORT FCPP_LINK Cpp_Relex_Data
cpp_relex_init(Cpp_Token_Array *array, i32_4tech start_pos, i32_4tech end_pos, i32_4tech character_shift_amount, b32_4tech ignore_string_delims)
cpp_relex_init(Cpp_Token_Array *array, i32_4tech start_pos, i32_4tech end_pos, i32_4tech character_shift_amount, b32_4tech ignore_string_delims, Cpp_Keyword_Table keywords, Cpp_Keyword_Table preprocessor_words)
/*
DOC_PARAM(array, A pointer to the token array that will be modified by the relex,
this array should already contain the tokens for the previous state of the file.)
DOC_PARAM(start_pos, The start position of the edited region of the file.
The start and end points are based on the edited region of the file before the edit.)
DOC_PARAM(end_pos, The end position of the edited region of the file.
In particular, end_pos is the first character after the edited region not effected by the edit.
Thus if the edited region contained one character end_pos - start_pos should equal 1.
The start and end points are based on the edited region of the file before the edit.)
DOC_PARAM(array, A pointer to the token array that will be modified by the relex, this array should already contain the tokens for the previous state of the file.)
DOC_PARAM(start_pos, The start position of the edited region of the file. The start and end points are based on the edited region of the file before the edit.)
DOC_PARAM(end_pos, The end position of the edited region of the file. In particular, end_pos is the first character after the edited region not effected by the edit. Thus if the edited region contained one character end_pos - start_pos should equal 1. The start and end points are based on the edited region of the file before the edit.)
DOC_PARAM(character_shift_amount, The shift in the characters after the edited region.)
DOC_PARAM(ignore_string_delims, TODO)
DOC_PARAM(keywords, TODO)
DOC_PARAM(preprocessor_words, TODO)
DOC_RETURN(Returns a partially initialized relex state.)
DOC(This call does the first setup step of initializing a relex state. To finish initializing the relex state
you must tell the state about the positioning of the first chunk it will be fed. There are two methods of doing
this, the direct method is with cpp_relex_declare_first_chunk_position, the method that is often more convenient
is with cpp_relex_is_start_chunk. If the file is not chunked the second step of initialization can be skipped.)
DOC(This call does the first setup step of initializing a relex state. To finish initializing the relex state you must tell the state about the positioning of the first chunk it will be fed. There are two methods of doing this, the direct method is with cpp_relex_declare_first_chunk_position, the method that is often more convenient is with cpp_relex_is_start_chunk. If the file is not chunked the second step of initialization can be skipped.)
DOC_SEE(cpp_relex_declare_first_chunk_position)
DOC_SEE(cpp_relex_is_start_chunk)
@ -1326,7 +1400,7 @@ DOC_SEE(cpp_relex_is_start_chunk)
state.character_shift_amount = character_shift_amount;
state.lex = cpp_lex_data_init(ignore_string_delims);
state.lex = cpp_lex_data_init(ignore_string_delims, keywords, preprocessor_words);
state.lex.pp_state = cpp_token_get_pp_state(array->tokens[state.start_token_index].state_flags);
state.lex.pos = state.relex_start_position;
@ -1589,8 +1663,7 @@ does the necessary replacement of tokens in the array to make it match the new f
array->count += shift_amount;
}
cpp__block_move(array->tokens + S_ptr->start_token_index, relex_array->tokens,
sizeof(Cpp_Token)*relex_array->count);
cpp__block_move(array->tokens + S_ptr->start_token_index, relex_array->tokens, sizeof(Cpp_Token)*relex_array->count);
}
API_EXPORT FCPP_LINK void
@ -1683,7 +1756,16 @@ return(array);
)
DOC_SEE(cpp_make_token_array)
*/{
Cpp_Lex_Data S = cpp_lex_data_init(false);
umem_4tech keywords_memsize = cpp_get_table_memory_size_default(CPP_TABLE_KEYWORDS);
umem_4tech preprocessor_words_memsize = cpp_get_table_memory_size_default(CPP_TABLE_PREPROCESSOR_DIRECTIVES);
void *keywords_mem = malloc(keywords_memsize);
void *preprocessor_words_mem = malloc(preprocessor_words_memsize);
Cpp_Keyword_Table keywords = cpp_make_table_default(CPP_TABLE_KEYWORDS, keywords_mem, keywords_memsize);
Cpp_Keyword_Table preprocessor_words = cpp_make_table_default(CPP_TABLE_PREPROCESSOR_DIRECTIVES, preprocessor_words_mem, preprocessor_words_memsize);
Cpp_Lex_Data S = cpp_lex_data_init(false, keywords, preprocessor_words);
i32_4tech quit = 0;
char empty = 0;
@ -1718,6 +1800,9 @@ DOC_SEE(cpp_make_token_array)
}break;
}
}
free(keywords_mem);
free(preprocessor_words_mem);
}
#endif

16
4cpp/4cpp_lexer_types.h
View File

@ -16,6 +16,11 @@
#define STRUCT struct
#endif
ENUM(uint32_t, Cpp_Word_Table_Type){
CPP_TABLE_KEYWORDS,
CPP_TABLE_PREPROCESSOR_DIRECTIVES,
};
/* DOC(A Cpp_Token_Type classifies a token to make parsing easier. Some types are not actually output by the lexer, but exist because parsers will also make use of token types in their own output.) */
ENUM(uint32_t, Cpp_Token_Type){
CPP_TOKEN_JUNK = 0,
@ -306,6 +311,13 @@ struct Cpp_Lex_FSM{
};
static Cpp_Lex_FSM null_lex_fsm = {0};
STRUCT Cpp_Keyword_Table{
void *mem;
umem_4tech memsize;
u64_4tech *keywords;
u32_4tech max;
};
/* DOC(Cpp_Lex_Data represents the state of the lexer so that the system may be resumable and the user can manage the lexer state and decide when to resume lexing with it. To create a new lexer state call cpp_lex_data_init.
The internals of the lex state should not be treated as a part of the public API.)
@ -332,8 +344,10 @@ STRUCT Cpp_Lex_Data{
b8_4tech str_raw;
b8_4tech str_include;
b8_4tech ignore_string_delims;
i32_4tech ignore_string_delims;
Cpp_Keyword_Table keyword_table;
Cpp_Keyword_Table preprops_table;
i32_4tech __pc__;
};

22
4ed.cpp
View File

@ -438,7 +438,7 @@ COMMAND_DECL(kill_buffer){
}
internal void
case_change_range(System_Functions *system, Mem_Options *mem, View *view, Editing_File *file, u8 a, u8 z, u8 char_delta){
case_change_range(System_Functions *system, Models *models, View *view, Editing_File *file, u8 a, u8 z, u8 char_delta){
Range range = {0};
range.min = Min(view->edit_pos->cursor.pos, view->edit_pos->mark);
range.max = Max(view->edit_pos->cursor.pos, view->edit_pos->mark);
@ -453,7 +453,7 @@ case_change_range(System_Functions *system, Mem_Options *mem, View *view, Editin
system->cancel_job(BACKGROUND_THREADS, file->state.lex_job);
}
file_update_history_before_edit(mem, file, step, 0, hist_normal);
file_update_history_before_edit(&models->mem, file, step, 0, hist_normal);
u8 *data = (u8*)file->state.buffer.data;
for (i32 i = range.start; i < range.end; ++i){
@ -463,7 +463,7 @@ case_change_range(System_Functions *system, Mem_Options *mem, View *view, Editin
}
if (file->state.token_array.tokens){
file_relex_parallel(system, mem, file, range.start, range.end, 0);
file_relex_parallel(system, models, file, range.start, range.end, 0);
}
}
}
@ -1131,16 +1131,16 @@ App_Read_Command_Line_Sig(app_read_command_line){
return(out_size);
}
extern "C" SCROLL_RULE_SIG(fallback_scroll_rule){
i32 result = 0;
SCROLL_RULE_SIG(fallback_scroll_rule){
b32 result = false;
if (target_x != *scroll_x){
*scroll_x = target_x;
result = 1;
result = true;
}
if (target_y != *scroll_y){
*scroll_y = target_y;
result = 1;
result = true;
}
return(result);
@ -1185,7 +1185,6 @@ App_Init_Sig(app_init){
div->next = 0;
models->layout.free_divider = dividers;
{
models->live_set = &vars->live_set;
@ -1207,6 +1206,13 @@ App_Init_Sig(app_init){
}
}
{
umem memsize = KB(8);
void *mem = push_block(partition, (i32)memsize);
parse_context_init_memory(&models->parse_context_memory, mem, memsize);
parse_context_add_default(&models->parse_context_memory, &models->mem.general);
}
{
models->scroll_rule = fallback_scroll_rule;
models->hook_open_file = 0;

26
4ed_api_implementation.cpp
View File

@ -899,10 +899,10 @@ DOC_SEE(Buffer_Setting_ID)
else{
if (value){
if (!file->settings.virtual_white){
file_first_lex_parallel(system, &models->mem, file);
file_first_lex_parallel(system, models, file);
}
else{
file_first_lex_serial(&models->mem, file);
file_first_lex_serial(models, file);
}
}
}
@ -915,10 +915,10 @@ DOC_SEE(Buffer_Setting_ID)
file_kill_tokens(system, &models->mem.general, file);
file->settings.tokens_without_strings = (b8)value;
if (!file->settings.virtual_white){
file_first_lex_parallel(system, &models->mem, file);
file_first_lex_parallel(system, models, file);
}
else{
file_first_lex_serial(&models->mem, file);
file_first_lex_serial(models, file);
}
}
}
@ -1001,34 +1001,34 @@ DOC_SEE(Buffer_Setting_ID)
case BufferSetting_ReadOnly:
{
if (value){
file->settings.read_only = 1;
file->settings.read_only = true;
}
else{
file->settings.read_only = 0;
file->settings.read_only = false;
}
}break;
case BufferSetting_VirtualWhitespace:
{
b32 full_remeasure = 0;
b32 full_remeasure = false;
if (value){
if (!file->settings.virtual_white){
if (!file->settings.tokens_exist){
file_first_lex_serial(&models->mem, file);
file_first_lex_serial(models, file);
}
if (!file->state.still_lexing){
file->settings.virtual_white = 1;
full_remeasure = 1;
file->settings.virtual_white = true;
full_remeasure = true;
}
else{
result = 0;
result = false;
}
}
}
else{
if (file->settings.virtual_white){
file->settings.virtual_white = 0;
full_remeasure = 1;
file->settings.virtual_white = false;
full_remeasure = true;
}
}

2
4ed_app_models.h
View File

@ -78,6 +78,8 @@ struct Models{
Editing_Layout layout;
Working_Set working_set;
Parse_Context_Memory parse_context_memory;
struct Live_Views *live_set;
Editing_File *message_buffer;
Editing_File *scratch_buffer;

1
4ed_app_target.cpp
View File

@ -55,6 +55,7 @@
#include "4ed_file.cpp"
#include "4ed_working_set.cpp"
#include "4ed_hot_directory.cpp"
#include "4ed_parse_contexts.cpp"
#include "4ed_gui.h"
#include "4ed_gui.cpp"

9
4ed_doubly_linked_list.cpp
View File

@ -7,6 +7,8 @@
*
*/
// TOP
// NOTE(allen): These macros are setup to work on structs
// with a next and prev pointer where the type of the struct
// is the same as the type of the next/prev pointers.
@ -19,5 +21,12 @@
// for(dll_items(iterator, sentinel_ptr)){...}
#define dll_items(it, st) ((it) = (st)->next); ((it) != (st)); ((it) = (it)->next)
#define sll_init_sentinel(s) do{ (s)->next=(s); }while(0)
#define sll_insert(p,v) do{ (v)->next=(p)->next; (p)->next = (v); }while(0)
#define sll_remove(p,v) do{ Assert((p)->next == (v)); (p)->next = (v)->next; }while(0)
// for(sll_items(iterator, sentinel_ptr)){...}
#define sll_items(it, st) ((it) = (st)->next); ((it) != (st)); ((it) = (it)->next)
// BOTTOM

1
4ed_file.cpp
View File

@ -93,6 +93,7 @@ struct Editing_File_Settings{
i32 display_width;
i32 minimum_base_display_width;
i32 wrap_indicator;
u32 parse_context_id;
b32 dos_write_mode;
b32 virtual_white;
Font_ID font_id;

113
4ed_file_view.cpp
View File

@ -2123,29 +2123,40 @@ struct Shift_Information{
i32 start, end, amount;
};
// TODO(allen): I want this code audited soon
internal
Job_Callback_Sig(job_full_lex){
Editing_File *file = (Editing_File*)data[0];
General_Memory *general = (General_Memory*)data[1];
Models *models = (Models*)data[2];
Parse_Context parse_context = parse_context_get(&models->parse_context_memory, file->settings.parse_context_id, memory->data, memory->size);
if (!parse_context.valid){
return;
}
Gap_Buffer *buffer = &file->state.buffer;
i32 text_size = buffer_size(buffer);
i32 aligned_buffer_size = (text_size + 3)&(~3);
while (memory->size < aligned_buffer_size){
for (;memory->size < aligned_buffer_size + parse_context.memory_size;){
system->grow_thread_memory(memory);
}
Cpp_Token_Array tokens;
tokens.tokens = (Cpp_Token*)(memory->data);
tokens.max_count = memory->size / sizeof(Cpp_Token);
u8 *data_ptr = (u8*)memory->data;
umem data_size = memory->size;
data_ptr += parse_context.memory_size;
data_size -= parse_context.memory_size;
Cpp_Token_Array tokens = {0};
tokens.tokens = (Cpp_Token*)(data_ptr);
tokens.max_count = (u32)(data_size / sizeof(Cpp_Token));
tokens.count = 0;
b32 still_lexing = 1;
b32 still_lexing = true;
Cpp_Lex_Data lex = cpp_lex_data_init(file->settings.tokens_without_strings);
Cpp_Lex_Data lex = cpp_lex_data_init(file->settings.tokens_without_strings, parse_context.kw_table, parse_context.pp_table);
// TODO(allen): deduplicate this against relex
char *chunks[3];
@ -2189,7 +2200,7 @@ Job_Callback_Sig(job_full_lex){
}
}break;
case LexResult_Finished: still_lexing = 0; break;
case LexResult_Finished: still_lexing = false; break;
}
} while (still_lexing);
@ -2223,8 +2234,8 @@ Job_Callback_Sig(job_full_lex){
// NOTE(allen): These are outside the locked section because I don't
// think getting these out of order will cause critical bugs, and I
// want to minimize what's done in locked sections.
file->state.tokens_complete = 1;
file->state.still_lexing = 0;
file->state.tokens_complete = true;
file->state.still_lexing = false;
}
internal void
@ -2245,31 +2256,33 @@ file_kill_tokens(System_Functions *system, General_Memory *general, Editing_File
}
internal void
file_first_lex_parallel(System_Functions *system, Mem_Options *mem, Editing_File *file){
General_Memory *general = &mem->general;
file->settings.tokens_exist = 1;
file_first_lex_parallel(System_Functions *system, Models *models, Editing_File *file){
General_Memory *general = &models->mem.general;
file->settings.tokens_exist = true;
if (file->is_loading == 0 && file->state.still_lexing == 0){
Assert(file->state.token_array.tokens == 0);
file->state.tokens_complete = 0;
file->state.still_lexing = 1;
file->state.tokens_complete = false;
file->state.still_lexing = true;
Job_Data job;
job.callback = job_full_lex;
job.data[0] = file;
job.data[1] = general;
job.data[2] = models;
file->state.lex_job = system->post_job(BACKGROUND_THREADS, job);
}
}
internal void
file_first_lex_serial(Mem_Options *mem, Editing_File *file){
file_first_lex_serial(Models *models, Editing_File *file){
Mem_Options *mem = &models->mem;
Partition *part = &mem->part;
General_Memory *general = &mem->general;
file->settings.tokens_exist = 1;
file->settings.tokens_exist = true;
Assert(file->state.still_lexing == 0);
Assert(!file->state.still_lexing);
if (file->is_loading == 0){
Assert(file->state.token_array.tokens == 0);
@ -2277,6 +2290,10 @@ file_first_lex_serial(Mem_Options *mem, Editing_File *file){
{
Temp_Memory temp = begin_temp_memory(part);
Parse_Context parse_context = parse_context_get(&models->parse_context_memory, file->settings.parse_context_id, partition_current(part), partition_remaining(part));
Assert(parse_context.valid);
push_block(part, (i32)parse_context.memory_size);
Gap_Buffer *buffer = &file->state.buffer;
i32 text_size = buffer_size(buffer);
@ -2287,9 +2304,9 @@ file_first_lex_serial(Mem_Options *mem, Editing_File *file){
tokens.count = 0;
tokens.tokens = push_array(part, Cpp_Token, tokens.max_count);
b32 still_lexing = 1;
b32 still_lexing = true;
Cpp_Lex_Data lex = cpp_lex_data_init(file->settings.tokens_without_strings);
Cpp_Lex_Data lex = cpp_lex_data_init(file->settings.tokens_without_strings, parse_context.kw_table, parse_context.pp_table);
// TODO(allen): deduplicate this against relex
char *chunks[3];
@ -2338,7 +2355,7 @@ file_first_lex_serial(Mem_Options *mem, Editing_File *file){
tokens.count = 0;
if (result == LexResult_Finished){
still_lexing = 0;
still_lexing = false;
}
}break;
@ -2364,21 +2381,22 @@ file_first_lex_serial(Mem_Options *mem, Editing_File *file){
end_temp_memory(temp);
}
file->state.tokens_complete = 1;
file->state.tokens_complete = true;
}
}
internal b32
file_relex_parallel(System_Functions *system, Mem_Options *mem, Editing_File *file, i32 start_i, i32 end_i, i32 shift_amount){
file_relex_parallel(System_Functions *system, Models *models, Editing_File *file, i32 start_i, i32 end_i, i32 shift_amount){
Mem_Options *mem = &models->mem;
General_Memory *general = &mem->general;
Partition *part = &mem->part;
if (file->state.token_array.tokens == 0){
file_first_lex_parallel(system, mem, file);
return(0);
file_first_lex_parallel(system, models, file);
return(false);
}
b32 result = 1;
b32 result = true;
b32 inline_lex = !file->state.still_lexing;
if (inline_lex){
Gap_Buffer *buffer = &file->state.buffer;
@ -2391,6 +2409,10 @@ file_relex_parallel(System_Functions *system, Mem_Options *mem, Editing_File *fi
relex_range.end_token_index - relex_range.start_token_index + extra_tolerance;
Temp_Memory temp = begin_temp_memory(part);
Parse_Context parse_context = parse_context_get(&models->parse_context_memory, file->settings.parse_context_id, partition_current(part), partition_remaining(part));
Assert(parse_context.valid);
push_block(part, (i32)parse_context.memory_size);
Cpp_Token_Array relex_array;
relex_array.count = 0;
relex_array.max_count = relex_space_size;
@ -2398,7 +2420,7 @@ file_relex_parallel(System_Functions *system, Mem_Options *mem, Editing_File *fi
i32 size = buffer_size(buffer);
Cpp_Relex_Data state = cpp_relex_init(array, start_i, end_i, shift_amount, file->settings.tokens_without_strings);
Cpp_Relex_Data state = cpp_relex_init(array, start_i, end_i, shift_amount, file->settings.tokens_without_strings, parse_context.kw_table, parse_context.pp_table);
char *chunks[3];
i32 chunk_sizes[3];
@ -2438,7 +2460,7 @@ file_relex_parallel(System_Functions *system, Mem_Options *mem, Editing_File *fi
case LexResult_NeedTokenMemory:
{
inline_lex = 0;
inline_lex = false;
}goto doublebreak;
case LexResult_Finished: goto doublebreak;
@ -2485,26 +2507,28 @@ file_relex_parallel(System_Functions *system, Mem_Options *mem, Editing_File *fi
}
}
file->state.still_lexing = 1;
file->state.still_lexing = true;
Job_Data job;
job.callback = job_full_lex;
job.data[0] = file;
job.data[1] = general;
job.data[2] = models;
file->state.lex_job = system->post_job(BACKGROUND_THREADS, job);
result = 0;
result = false;
}
return(result);
}
internal b32
file_relex_serial(Mem_Options *mem, Editing_File *file, i32 start_i, i32 end_i, i32 shift_amount){
file_relex_serial(Models *models, Editing_File *file, i32 start_i, i32 end_i, i32 shift_amount){
Mem_Options *mem = &models->mem;
General_Memory *general = &mem->general;
Partition *part = &mem->part;
if (file->state.token_array.tokens == 0){
file_first_lex_serial(mem, file);
file_first_lex_serial(models, file);
return(1);
}
@ -2514,6 +2538,10 @@ file_relex_serial(Mem_Options *mem, Editing_File *file, i32 start_i, i32 end_i,
Cpp_Token_Array *array = &file->state.token_array;
Temp_Memory temp = begin_temp_memory(part);
Parse_Context parse_context = parse_context_get(&models->parse_context_memory, file->settings.parse_context_id, partition_current(part), partition_remaining(part));
Assert(parse_context.valid);
push_block(part, (i32)parse_context.memory_size);
Cpp_Token_Array relex_array;
relex_array.count = 0;
relex_array.max_count = partition_remaining(part) / sizeof(Cpp_Token);
@ -2521,7 +2549,7 @@ file_relex_serial(Mem_Options *mem, Editing_File *file, i32 start_i, i32 end_i,
i32 size = buffer_size(buffer);
Cpp_Relex_Data state = cpp_relex_init(array, start_i, end_i, shift_amount, file->settings.tokens_without_strings);
Cpp_Relex_Data state = cpp_relex_init(array, start_i, end_i, shift_amount, file->settings.tokens_without_strings, parse_context.kw_table, parse_context.pp_table);
char *chunks[3];
i32 chunk_sizes[3];
@ -3328,10 +3356,10 @@ file_do_single_edit(System_Functions *system, Models *models, Editing_File *file
// NOTE(allen): token fixing
if (file->settings.tokens_exist){
if (!file->settings.virtual_white){
file_relex_parallel(system, mem, file, start, end, shift_amount);
file_relex_parallel(system, models, file, start, end, shift_amount);
}
else{
file_relex_serial(mem, file, start, end, shift_amount);
file_relex_serial(models, file, start, end, shift_amount);
}
}
@ -3415,10 +3443,10 @@ file_do_batch_edit(System_Functions *system, Models *models, Editing_File *file,
Buffer_Edit *last_edit = batch + batch_size - 1;
if (!file->settings.virtual_white){
file_relex_parallel(system, mem, file, first_edit->start, last_edit->end, shift_total);
file_relex_parallel(system, models, file, first_edit->start, last_edit->end, shift_total);
}
else{
file_relex_serial(mem, file, first_edit->start, last_edit->end, shift_total);
file_relex_serial(models, file, first_edit->start, last_edit->end, shift_total);
}
}
}break;
@ -3819,13 +3847,12 @@ init_normal_file(System_Functions *system, Models *models, Editing_File *file, c
String val = make_string(buffer, size);
file_create_from_string(system, models, file, val);
Mem_Options *mem = &models->mem;
if (file->settings.tokens_exist && file->state.token_array.tokens == 0){
if (!file->settings.virtual_white){
file_first_lex_parallel(system, mem, file);
file_first_lex_parallel(system, models, file);
}
else{
file_first_lex_serial(mem, file);
file_first_lex_serial(models, file);
}
}
}
@ -3835,18 +3862,16 @@ init_read_only_file(System_Functions *system, Models *models, Editing_File *file
String val = null_string;
file_create_from_string(system, models, file, val, 1);
Mem_Options *mem = &models->mem;
if (file->settings.tokens_exist && file->state.token_array.tokens == 0){
if (!file->settings.virtual_white){
file_first_lex_parallel(system, mem, file);
file_first_lex_parallel(system, models, file);
}
else{
file_first_lex_serial(mem, file);
file_first_lex_serial(models, file);
}
}
}
internal void
view_open_file(System_Functions *system, Models *models, View *view, String filename){
Working_Set *working_set = &models->working_set;

179
4ed_parse_contexts.cpp Normal file
View File

@ -0,0 +1,179 @@
/*
* Mr. 4th Dimention - Allen Webster
*
* 19.05.2017
*
* Parse contexts allocation and locking.
*
*/
// TOP
struct Parser_String_And_Type{
char *str;
u32 length;
u32 type;
};
struct Stored_Parse_Context{
umem memsize;
u64 *kw_keywords;
u64 *pp_keywords;
u32 kw_max;
u32 pp_max;
};
struct Stored_Parse_Context_Slot{
union{
Stored_Parse_Context_Slot *next;
Stored_Parse_Context *context;
};
b32 freed;
};
struct Parse_Context_Memory{
Stored_Parse_Context_Slot *parse_context_array;
u32 parse_context_counter;
u32 parse_context_max;
Stored_Parse_Context_Slot free_sentinel;
};
internal void
parse_context_init_memory(Parse_Context_Memory *parse_mem, void *mem, umem memsize){
sll_init_sentinel(&parse_mem->free_sentinel);
parse_mem->parse_context_array = (Stored_Parse_Context_Slot*)mem;
parse_mem->parse_context_counter = 0;
parse_mem->parse_context_max = (u32)(memsize / sizeof(*parse_mem->parse_context_array));
}
internal u32
parse_context_add(Parse_Context_Memory *parse_mem, General_Memory *general, Parser_String_And_Type *kw_sats, u32 kw_count, Parser_String_And_Type *pp_sats, u32 pp_count){
Stored_Parse_Context_Slot *slot = 0;
if (parse_mem->free_sentinel.next != &parse_mem->free_sentinel){
slot = parse_mem->free_sentinel.next;
sll_remove(&parse_mem->free_sentinel, slot);
}
else if (parse_mem->parse_context_counter < parse_mem->parse_context_max){
slot = &parse_mem->parse_context_array[parse_mem->parse_context_counter++];
}
u32 result = 0;
if (slot != 0){
u32 stride = sizeof(*kw_sats);
umem kw_memsize = cpp_get_table_memory_size_string_lengths(&kw_sats->length, stride, kw_count);
umem pp_memsize = cpp_get_table_memory_size_string_lengths(&pp_sats->length, stride, pp_count);
umem memsize = kw_memsize + pp_memsize + sizeof(Stored_Parse_Context);
void *mem = general_memory_allocate(general, (i32)memsize);
Stored_Parse_Context *parse_context = (Stored_Parse_Context*)mem;
u8 *kw_mem = (u8*)(parse_context+1);
u8 *pp_mem = kw_mem + kw_memsize;
Cpp_Keyword_Table kw_table = cpp_make_table(&kw_sats->str, stride, &kw_sats->length, stride, &kw_sats->type, stride, kw_count, kw_mem, kw_memsize);
Cpp_Keyword_Table pp_table = cpp_make_table(&pp_sats->str, stride, &pp_sats->length, stride, &pp_sats->type, stride, pp_count, pp_mem, pp_memsize);
parse_context->memsize = memsize;
parse_context->kw_keywords = kw_table.keywords;
parse_context->pp_keywords = pp_table.keywords;
parse_context->kw_max = kw_table.max;
parse_context->pp_max = pp_table.max;
slot->context = parse_context;
slot->freed = false;
result = (u32)(slot - parse_mem->parse_context_array) + parse_mem->parse_context_max;
}
return(result);
}
internal u32
parse_context_add_default(Parse_Context_Memory *parse_mem, General_Memory *general){
Stored_Parse_Context_Slot *slot = 0;
if (parse_mem->free_sentinel.next != &parse_mem->free_sentinel){
slot = parse_mem->free_sentinel.next;
sll_remove(&parse_mem->free_sentinel, slot);
}
else if (parse_mem->parse_context_counter < parse_mem->parse_context_max){
slot = &parse_mem->parse_context_array[parse_mem->parse_context_counter++];
}
u32 result = 0;
if (slot != 0){
umem kw_memsize = cpp_get_table_memory_size_default(CPP_TABLE_KEYWORDS);
umem pp_memsize = cpp_get_table_memory_size_default(CPP_TABLE_PREPROCESSOR_DIRECTIVES);
umem memsize = kw_memsize + pp_memsize + sizeof(Stored_Parse_Context);
void *mem = general_memory_allocate(general, (i32)(memsize));
Stored_Parse_Context *parse_context = (Stored_Parse_Context*)mem;
u8 *kw_mem = (u8*)(parse_context+1);
u8 *pp_mem = kw_mem + kw_memsize;
Cpp_Keyword_Table kw_table = cpp_make_table_default(CPP_TABLE_KEYWORDS, kw_mem, kw_memsize);
Cpp_Keyword_Table pp_table = cpp_make_table_default(CPP_TABLE_PREPROCESSOR_DIRECTIVES, pp_mem, pp_memsize);
parse_context->memsize = memsize;
parse_context->kw_keywords = kw_table.keywords;
parse_context->pp_keywords = pp_table.keywords;
parse_context->kw_max = kw_table.max;
parse_context->pp_max = pp_table.max;
slot->context = parse_context;
slot->freed = false;
result = (u32)(slot - parse_mem->parse_context_array) + parse_mem->parse_context_max;
}
return(result);
}
struct Parse_Context{
b32 valid;
Cpp_Keyword_Table kw_table;
Cpp_Keyword_Table pp_table;
umem memory_size;
};
internal Parse_Context
parse_context_get(Parse_Context_Memory *parse_mem, u32 id, void *mem, umem memsize){
Parse_Context result = {0};
Stored_Parse_Context_Slot *slot = 0;
if (id == 0){
// do nothing
}
else{
id -= parse_mem->parse_context_max;
}
if (id < parse_mem->parse_context_counter){
slot = &parse_mem->parse_context_array[id];
if (slot->freed){
slot = 0;
}
}
if (slot != 0){
Stored_Parse_Context *context = slot->context;
if (context->memsize < memsize){
u8 *base_ptr = (u8*)context;
u8 *kw_keywords = (u8*)mem + (((u8*)context->kw_keywords) - base_ptr);
u8 *pp_keywords = (u8*)mem + (((u8*)context->pp_keywords) - base_ptr);
result.valid = true;
result.kw_table.keywords = (u64*)kw_keywords;
result.pp_table.keywords = (u64*)pp_keywords;
result.kw_table.max = context->kw_max;
result.pp_table.max = context->pp_max;
result.memory_size = context->memsize;
memcpy(mem, context, context->memsize);
}
}
return(result);
}
// BOTTOM

10
4ed_system.h
View File

@ -139,11 +139,7 @@ struct Thread_Memory{
i32 size;
i32 id;
};
inline Thread_Memory
thread_memory_zero(){
Thread_Memory memory={0};
return(memory);
}
global Thread_Memory null_thread_memory = {0};
struct Thread_Exchange;
struct System_Functions;
@ -152,12 +148,12 @@ struct System_Functions;
System_Functions *system, \
Thread_Context *thread, \
Thread_Memory *memory, \
void *data[2])
void *data[4])
typedef Job_Callback_Sig(Job_Callback);
struct Job_Data{
Job_Callback *callback;
void *data[2];
void *data[4];
};
struct Full_Job_Data{

2
meta/fsm_table_generator.cpp
View File

@ -21,6 +21,8 @@ typedef uint16_t u16_4tech;
typedef uint32_t u32_4tech;
typedef uint64_t u64_4tech;
typedef u64_4tech umem_4tech;
typedef float f32_4tech;
typedef double f64_4tech;

7
win32_4ed.cpp
View File

@ -1975,10 +1975,7 @@ WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPSTR lpCmdLine, int nCmdS
Thread_Memory thread_memory[ArrayCount(background)];
win32vars.thread_memory = thread_memory;
win32vars.queues[BACKGROUND_THREADS].semaphore =
Win32Handle(CreateSemaphore(0, 0,
win32vars.groups[BACKGROUND_THREADS].count,
0));
win32vars.queues[BACKGROUND_THREADS].semaphore =Win32Handle(CreateSemaphore(0, 0, win32vars.groups[BACKGROUND_THREADS].count, 0));
u32 creation_flag = 0;
for (i32 i = 0; i < win32vars.groups[BACKGROUND_THREADS].count; ++i){
@ -1987,7 +1984,7 @@ WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPSTR lpCmdLine, int nCmdS
thread->group_id = BACKGROUND_THREADS;
Thread_Memory *memory = win32vars.thread_memory + i;
*memory = thread_memory_zero();
*memory = null_thread_memory;
memory->id = thread->id;
thread->queue = &win32vars.queues[BACKGROUND_THREADS];