mr4th-public
/
4coder
1
Fork 0
Code Issues Releases Activity

Fixing up lexer build and preprocessor parsing logic

master
Allen Webster 2019-10-01 14:49:05 -07:00
parent 457fc27d14
commit 73fa47a672
7 changed files with 281 additions and 237 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

362
4ed_api_implementation.cpp
View File

File diff suppressed because it is too large Load Diff

5
custom/4coder_base_types.h
View File

@ -172,6 +172,9 @@ typedef void Void_Func(void);
#define stringify_(a) #a
#define stringify(a) stringify_(a)
#define function static
#define api(x)
#define internal static
#define local_persist static
#define global static
@ -297,8 +300,6 @@ global_const f32 epsilon_f32 = 5.96046448e-8f;
#define require(c) Stmnt( if (!(c)){ return(0); } )
#define API_EXPORT
////////////////////////////////
global_const u32 bit_1 = 0x00000001;

15
custom/4coder_file.h
View File

@ -16,25 +16,16 @@
#if OS_WINDOWS
//// WINDOWS BEGIN ////
#undef function
#define UNICODE
#include <Windows.h>
typedef TCHAR Filename_Character;
#define SLASH '\\'
#define function static
//// WINDOWS END ////
#elif OS_LINUX || OS_MAC
//// UNIX BEGIN ////
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <dirent.h>
typedef char Filename_Character;
#define SLASH '/'
//// UNIX END ////
#else
# error metdata generator not supported on this platform
# error 4coder file not defined
#endif
struct Cross_Platform_File_Info{

33
custom/bin/build_one_time.bat
View File

@ -1,32 +1,41 @@
@echo off
REM usage: build_generator <src-file> <binary-output-path> [release]
REM src-file: a relative path to the target of a unity-build for a generator
REM binary-output-path: a relative path where the generator.exe will be written, assumed to be "." if unset
REM usage: <script> <target> [binary-output-path [mode]]
REM src-file : a relative path to the target of a unity-build for a one time executable
REM binary-output-path : a relative path where the generator.exe will be written,
REM assumed to be "." if unset
REM mode : if set to "release" builds with optimizations
set code_home=%~dp0
if %code_home:~-1%==\ (set code_home=%code_home:~0,-1%)
set location=%cd%
set me="%~dp0"
cd %me%
cd ..
set custom_root=%cd%
set custom_bin=%custom_root%\bin
cd %location%
if NOT "%Platform%" == "X64" IF NOT "%Platform%" == "x64" (call "%code_home%\windows_scripts\setup_cl_x64.bat")
set src=%1
if "%src%" == "" (echo error: no input file & exit)
set target=%1
if "%target%" == "" (echo error: no input file & exit)
set full_target=%target%
if NOT "%target:~1,1%" == ":" (set full_target="%cd%\%target%")
set dst=%2
if "%dst%" == "" (set dst=".")
set opts=/W4 /wd4310 /wd4100 /wd4201 /wd4505 /wd4996 /wd4127 /wd4510 /wd4512 /wd4610 /wd4457 /wd4146 /WX
set opts=%opts% /GR- /nologo /FC
set debug=/Zi
set release=/O2 /Zi
set mode=%debug%
if "%3" == "release" (set mode=%release%)
set full_src="%cd%\%src%"
set opts=/W4 /wd4310 /wd4100 /wd4201 /wd4505 /wd4996 /wd4127 /wd4510 /wd4512 /wd4610 /wd4457 /wd4146 /WX
set opts=%opts% /GR- /nologo /FC
set opts=%opts% /I%custom_root%
set opts=%opts% %mode%
pushd %dst%
call cl /I"%code_home%" %opts% %mode% %full_src% /Fegenerator
call cl /I"%code_home%" %opts% %full_target% /Fegenerator
popd

62
custom/4coder_lex_gen_cpp.cpp → custom/languages/4coder_lex_gen_cpp.cpp
View File

@ -235,21 +235,21 @@ build_language_model(void){
Keyword_Set *pp_directive_set = sm_begin_key_set("pp_directives");
sm_select_base_kind(TokenBaseKind_Preprocessor);
sm_key("PPInclude", "#include");
sm_key("PPVersion", "#version");
sm_key("PPDefine", "#define");
sm_key("PPUndef", "#undef");
sm_key("PPIf", "#if");
sm_key("PPIfDef", "#ifdef");
sm_key("PPIfNDef", "#ifndef");
sm_key("PPElse", "#else");
sm_key("PPElIf", "#elif");
sm_key("PPEndIf", "#endif");
sm_key("PPError", "#error");
sm_key("PPImport", "#import");
sm_key("PPUsing", "#using");
sm_key("PPLine", "#line");
sm_key("PPPragma", "#pragma");
sm_key("PPInclude", "include");
sm_key("PPVersion", "version");
sm_key("PPDefine", "define");
sm_key("PPUndef", "undef");
sm_key("PPIf", "if");
sm_key("PPIfDef", "ifdef");
sm_key("PPIfNDef", "ifndef");
sm_key("PPElse", "else");
sm_key("PPElIf", "elif");
sm_key("PPEndIf", "endif");
sm_key("PPError", "error");
sm_key("PPImport", "import");
sm_key("PPUsing", "using");
sm_key("PPLine", "line");
sm_key("PPPragma", "pragma");
sm_select_base_kind(TokenBaseKind_LexError);
sm_key_fallback("PPUnknown");
@ -310,7 +310,9 @@ build_language_model(void){
AddState(LL_number);
AddState(ULL_number);
AddState(pp_directive_whitespace);
AddState(pp_directive);
AddState(pp_directive_emit);
AddState(include_pointy);
AddState(include_quotes);
@ -401,7 +403,7 @@ build_language_model(void){
sm_case_flagged(is_include_body, false, "\"", string);
sm_case("\'", character);
sm_case_flagged(is_pp_body, false, "#", pp_directive);
sm_case_flagged(is_pp_body, false, "#", pp_directive_whitespace);
{
State *operator_state = smo_op_set_lexer_root(pp_ops, root, "LexError");
sm_case_peek_flagged(is_pp_body, true, "#", operator_state);
@ -771,18 +773,40 @@ build_language_model(void){
////
sm_select_state(pp_directive);
sm_set_flag(is_pp_body, true);
sm_select_state(pp_directive_whitespace);
sm_delim_mark_first();
sm_case(" \t\f\v", pp_directive_whitespace);
sm_case("abcdefghijklmnopqrstuvwxyz"
"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
"_"
"0123456789",
pp_directive);
{
Emit_Rule *emit = sm_emit_rule();
sm_emit_handler_direct("LexError");
sm_fallback_peek(emit);
}
////
sm_select_state(pp_directive);
sm_set_flag(is_pp_body, true);
sm_case("abcdefghijklmnopqrstuvwxyz"
"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
"_"
"0123456789",
pp_directive);
sm_fallback_peek(pp_directive_emit);
////
sm_select_state(pp_directive_emit);
sm_delim_mark_one_past_last();
{
Emit_Rule *emit = sm_emit_rule();
sm_emit_check_set_flag("PPInclude", is_include_body, true);
sm_emit_check_set_flag("PPError", is_error_body, true);
sm_emit_handler_keys(pp_directive_set);
sm_emit_handler_keys_delim(pp_directive_set);
sm_fallback_peek(emit);
}

39
custom/lexer_generator/4coder_lex_gen_main.cpp
View File

@ -106,6 +106,7 @@ typedef i32 Emit_Handler_Kind;
enum{
EmitHandlerKind_Direct,
EmitHandlerKind_Keywords,
EmitHandlerKind_KeywordsDelim,
};
struct Emit_Handler{
@ -494,11 +495,10 @@ smi_emit_rule(Arena *arena){
}
internal Emit_Handler*
smi_emit_handler(Arena *arena, Emit_Rule *rule, String_Const_u8 name, Flag *flag_check){
smi_emit_handler__inner(Arena *arena, Emit_Rule *rule, Emit_Handler_Kind kind, Flag *flag_check){
Emit_Handler *handler = push_array_zero(arena, Emit_Handler, 1);
handler->kind = EmitHandlerKind_Direct;
handler->kind = kind;
handler->flag_check = flag_check;
handler->token_name = name;
if (rule != 0){
sll_queue_push(rule->first, rule->last, handler);
rule->count += 1;
@ -506,16 +506,24 @@ smi_emit_handler(Arena *arena, Emit_Rule *rule, String_Const_u8 name, Flag *flag
return(handler);
}
internal Emit_Handler*
smi_emit_handler(Arena *arena, Emit_Rule *rule, String_Const_u8 name, Flag *flag_check){
Emit_Handler *handler = smi_emit_handler__inner(arena, rule, EmitHandlerKind_Direct, flag_check);
handler->token_name = name;
return(handler);
}
internal Emit_Handler*
smi_emit_handler(Arena *arena, Emit_Rule *rule, Keyword_Set *set, Flag *flag_check){
Emit_Handler *handler = push_array_zero(arena, Emit_Handler, 1);
handler->kind = EmitHandlerKind_Keywords;
handler->flag_check = flag_check;
Emit_Handler *handler = smi_emit_handler__inner(arena, rule, EmitHandlerKind_Keywords, flag_check);
handler->keywords = set;
return(handler);
}
internal Emit_Handler*
smi_emit_handler_delim(Arena *arena, Emit_Rule *rule, Keyword_Set *set, Flag *flag_check){
Emit_Handler *handler = smi_emit_handler__inner(arena, rule, EmitHandlerKind_KeywordsDelim, flag_check);
handler->keywords = set;
if (rule != 0){
sll_queue_push(rule->first, rule->last, handler);
rule->count += 1;
}
return(handler);
}
@ -1388,6 +1396,17 @@ sm_emit_handler_keys(Keyword_Set *set){
sm_emit_handler_keys(0, set);
}
internal void
sm_emit_handler_keys_delim(Flag *flag_check, Keyword_Set *set){
Emit_Rule *rule = helper_ctx.selected_emit_rule;
smi_emit_handler(helper_ctx.arena, rule, set, flag_check);
}
internal void
sm_emit_handler_keys_delim(Keyword_Set *set){
sm_emit_handler_keys_delim(0, set);
}
internal Transition*
sm_case(String_Const_u8 str, Flag *flag_check, b32 flag_check_value, State *dst, Transition_Consume_Rule consume_rule, Emit_Rule *emit){
Transition *transition = smi_case(&helper_ctx.primary_ctx, helper_ctx.selected_state, TransitionCaseKind_CharaterArray, str,

2
project.4coder
View File

@ -39,7 +39,7 @@ command_list = {
{ .name = "build C++ lexer generator",
.out = "*compilation*", .footer_panel = true, .save_dirty_files = true,
.cmd = { { "custom\\bin\\build_one_time 4coder_lex_gen_cpp.cpp ..\\build", .os = "win" }, }, },
.cmd = { { "custom\\bin\\build_one_time custom\\languages\\4coder_lex_gen_cpp.cpp ..\\build", .os = "win" }, }, },
{ .name = "build token tester",
.out = "*compilation*", .footer_panel = true, .save_dirty_files = true,