From 7a507ee6e49585842fef766c960d5fce26d41dec Mon Sep 17 00:00:00 2001 From: Allen Webster Date: Thu, 3 Oct 2019 16:00:31 -0700 Subject: [PATCH] API parser --- 4ed_api_definition.cpp | 162 ++++++++---------- 4ed_api_definition.h | 53 ++++++ 4ed_api_definition_main.cpp | 78 +++++++++ 4ed_api_parser.cpp | 252 ++++++++++++++++++++++++++++ 4ed_font_api.cpp | 2 +- 4ed_graphics_api.cpp | 2 +- 4ed_system_api.cpp | 2 +- custom/4coder_file.cpp | 26 +++ custom/4coder_token.cpp | 5 + custom/generated/command_metadata.h | 8 +- project.4coder | 6 +- 11 files changed, 492 insertions(+), 104 deletions(-) create mode 100644 4ed_api_definition.h create mode 100644 4ed_api_definition_main.cpp create mode 100644 4ed_api_parser.cpp create mode 100644 custom/4coder_file.cpp diff --git a/4ed_api_definition.cpp b/4ed_api_definition.cpp index 0936c6d5..9b94089f 100644 --- a/4ed_api_definition.cpp +++ b/4ed_api_definition.cpp @@ -1,7 +1,7 @@ /* * Mr. 4th Dimention - Allen Webster * - * 02.10.2019 + * 03.10.2019 * * System API definition program. * @@ -9,44 +9,6 @@ // TOP -#include "4coder_base_types.h" - -#include "4coder_base_types.cpp" -#include "4coder_stringf.cpp" - -#include "4coder_malloc_allocator.cpp" - -//////////////////////////////// - -struct API_Param{ - API_Param *next; - String_Const_u8 type_name; - String_Const_u8 name; -}; - -struct API_Param_List{ - API_Param *first; - API_Param *last; - i32 count; -}; - -struct API_Call{ - API_Call *next; - String_Const_u8 name; - String_Const_u8 return_type; - API_Param_List params; -}; - -struct API_Definition{ - API_Call *first; - API_Call *last; - i32 count; - - String_Const_u8 name; -}; - -//////////////////////////////// - function API_Definition* begin_api(Arena *arena, char *name){ API_Definition *api = push_array_zero(arena, API_Definition, 1); @@ -55,15 +17,20 @@ begin_api(Arena *arena, char *name){ } function API_Call* -api_call(Arena *arena, API_Definition *api, char *name, char *return_type){ +api_call(Arena *arena, API_Definition *api, String_Const_u8 name, String_Const_u8 return_type){ API_Call *call = push_array_zero(arena, API_Call, 1); sll_queue_push(api->first, api->last, call); api->count += 1; - call->name = SCu8(name); - call->return_type = SCu8(return_type); + call->name = name; + call->return_type = return_type; return(call); } +function API_Call* +api_call(Arena *arena, API_Definition *api, char *name, char *return_type){ + return(api_call(arena, api, SCu8(name), SCu8(return_type))); +} + function API_Param* api_param(Arena *arena, API_Call *call, char *type_name, char *name){ API_Param *param = push_array_zero(arena, API_Param, 1); @@ -74,8 +41,64 @@ api_param(Arena *arena, API_Call *call, char *type_name, char *name){ return(param); } +function void +api_set_param_list(API_Call *call, API_Param_List list){ + call->params = list; +} + +function API_Definition* +api_get_api(Arena *arena, API_Definition_List *list, String_Const_u8 name){ + API_Definition *result = 0; + for (API_Definition *node = list->first; + node != 0; + node = node->next){ + if (string_match(name, node->name)){ + result = node; + break; + } + } + if (result == 0){ + result = push_array_zero(arena, API_Definition, 1); + sll_queue_push(list->first, list->last, result); + list->count += 1; + result->name = name; + } + return(result); +} + //////////////////////////////// +#if !defined(SKIP_STDIO) +#include + +function void +generate_api_master_list(Arena *scratch, API_Definition *api, FILE *out){ + fprintf(out, "// %.*s\n", string_expand(api->name)); + for (API_Call *call = api->first; + call != 0; + call = call->next){ + fprintf(out, "%.*s %.*s(", + string_expand(call->return_type), + string_expand(call->name)); + if (call->params.count == 0){ + fprintf(out, "void"); + } + else{ + for (API_Param *param = call->params.first; + param != 0; + param = param->next){ + fprintf(out, "%.*s %.*s", + string_expand(param->type_name), + string_expand(param->name)); + if (param->next != 0){ + fprintf(out, ", "); + } + } + } + fprintf(out, ");\n"); + } +} + function void generate_header(Arena *scratch, API_Definition *api, FILE *out){ for (API_Call *call = api->first; @@ -217,60 +240,7 @@ generate_cpp(Arena *scratch, API_Definition *api, FILE *out){ fprintf(out, "#endif\n"); } -//////////////////////////////// - -function API_Definition* -define_api(Arena *arena); - -int -main(void){ - Arena arena = make_arena_malloc(); - API_Definition *api = define_api(&arena); - - //////////////////////////////// - - // NOTE(allen): Arrange output files - - String_Const_u8 path_to_self = string_u8_litexpr(__FILE__); - path_to_self = string_remove_last_folder(path_to_self); - - String_Const_u8 fname_h = push_u8_stringf(&arena, "%.*sgenerated/%.*s_api.h", - string_expand(path_to_self), - string_expand(api->name)); - - String_Const_u8 fname_cpp = push_u8_stringf(&arena, "%.*sgenerated/%.*s_api.cpp", - string_expand(path_to_self), - string_expand(api->name)); - - FILE *out_file_h = fopen((char*)fname_h.str, "wb"); - if (out_file_h == 0){ - printf("could not open output file: '%s'\n", fname_h.str); - exit(1); - } - - FILE *out_file_cpp = fopen((char*)fname_cpp.str, "wb"); - if (out_file_cpp == 0){ - printf("could not open output file: '%s'\n", fname_cpp.str); - exit(1); - } - - printf("%s:1:\n", fname_h.str); - printf("%s:1:\n", fname_cpp.str); - - //////////////////////////////// - - // NOTE(allen): Generate output - - generate_header(&arena, api, out_file_h); - generate_cpp(&arena, api, out_file_cpp); - - //////////////////////////////// - - fclose(out_file_h); - fclose(out_file_cpp); - - return(0); -} +#endif // BOTTOM diff --git a/4ed_api_definition.h b/4ed_api_definition.h new file mode 100644 index 00000000..29a100e7 --- /dev/null +++ b/4ed_api_definition.h @@ -0,0 +1,53 @@ +/* + * Mr. 4th Dimention - Allen Webster + * + * 03.10.2019 + * + * System API definition types. + * + */ + +// TOP + +#if !defined(FRED_API_DEFINITION_H) +#define FRED_API_DEFINITION_H + +struct API_Param{ + API_Param *next; + String_Const_u8 type_name; + String_Const_u8 name; +}; + +struct API_Param_List{ + API_Param *first; + API_Param *last; + i32 count; +}; + +struct API_Call{ + API_Call *next; + String_Const_u8 name; + String_Const_u8 return_type; + API_Param_List params; +}; + +struct API_Definition{ + API_Definition *next; + + API_Call *first; + API_Call *last; + i32 count; + + String_Const_u8 name; +}; + +struct API_Definition_List{ + API_Definition *first; + API_Definition *last; + i32 count; +}; + +#endif + +// BOTTOM + diff --git a/4ed_api_definition_main.cpp b/4ed_api_definition_main.cpp new file mode 100644 index 00000000..ecf10a4f --- /dev/null +++ b/4ed_api_definition_main.cpp @@ -0,0 +1,78 @@ +/* + * Mr. 4th Dimention - Allen Webster + * + * 02.10.2019 + * + * System API definition program. + * + */ + +// TOP + +#include "4coder_base_types.h" +#include "4ed_api_definition.h" + +#include "4coder_base_types.cpp" +#include "4ed_api_definition.cpp" +#include "4coder_stringf.cpp" +#include "4coder_malloc_allocator.cpp" + +#include + +//////////////////////////////// + +function API_Definition* +define_api(Arena *arena); + +int +main(void){ + Arena arena = make_arena_malloc(); + API_Definition *api = define_api(&arena); + + //////////////////////////////// + + // NOTE(allen): Arrange output files + + String_Const_u8 path_to_self = string_u8_litexpr(__FILE__); + path_to_self = string_remove_last_folder(path_to_self); + + String_Const_u8 fname_h = push_u8_stringf(&arena, "%.*sgenerated/%.*s_api.h", + string_expand(path_to_self), + string_expand(api->name)); + + String_Const_u8 fname_cpp = push_u8_stringf(&arena, "%.*sgenerated/%.*s_api.cpp", + string_expand(path_to_self), + string_expand(api->name)); + + FILE *out_file_h = fopen((char*)fname_h.str, "wb"); + if (out_file_h == 0){ + printf("could not open output file: '%s'\n", fname_h.str); + exit(1); + } + + FILE *out_file_cpp = fopen((char*)fname_cpp.str, "wb"); + if (out_file_cpp == 0){ + printf("could not open output file: '%s'\n", fname_cpp.str); + exit(1); + } + + printf("%s:1:\n", fname_h.str); + printf("%s:1:\n", fname_cpp.str); + + //////////////////////////////// + + // NOTE(allen): Generate output + + generate_header(&arena, api, out_file_h); + generate_cpp(&arena, api, out_file_cpp); + + //////////////////////////////// + + fclose(out_file_h); + fclose(out_file_cpp); + + return(0); +} + +// BOTTOM + diff --git a/4ed_api_parser.cpp b/4ed_api_parser.cpp new file mode 100644 index 00000000..0fbb49b7 --- /dev/null +++ b/4ed_api_parser.cpp @@ -0,0 +1,252 @@ +/* + * Mr. 4th Dimention - Allen Webster + * + * 03.10.2019 + * + * Parser that extracts an API from C++ source code. + * + */ + +// TOP + +#include "4coder_base_types.h" +#include "4ed_api_definition.h" +#include "4coder_token.h" +#include "generated/lexer_cpp.h" + +#include "4coder_base_types.cpp" +#include "4ed_api_definition.cpp" +#include "4coder_stringf.cpp" +#include "4coder_malloc_allocator.cpp" +#include "4coder_token.cpp" +#include "generated/lexer_cpp.cpp" +#include "4coder_file.cpp" + +//////////////////////////////// + +/* +function: +api ( ) function {*} ( ) + +param_list: +void | + {*} [, {*} ] + +anything_else: +*** + +api_source: +{function|anything_else} EOF +*/ + +function b32 +api_parse__match(Token_Iterator *it, Token_Cpp_Kind sub_kind){ + b32 match = false; + Token *token = token_it_read(it); + if (token->sub_kind == sub_kind){ + if (token_it_inc(it)){ + match = true; + } + } + return(match); +} + +function b32 +api_parse__match_identifier(Token_Iterator *it, String_Const_u8 source, String_Const_u8 *lexeme){ + b32 match = false; + Token *token = token_it_read(it); + if (token->kind == TokenBaseKind_Identifier || + token->kind == TokenBaseKind_Keyword){ + if (token_it_inc(it)){ + *lexeme = string_substring(source, Ii64(token)); + match = true; + } + } + return(match); +} + +function b32 +api_parse__match_identifier(Token_Iterator *it, String_Const_u8 source, char *lexeme){ + b32 match = false; + Token *token = token_it_read(it); + if ((token->kind == TokenBaseKind_Identifier || + token->kind == TokenBaseKind_Keyword) && + string_match(SCu8(lexeme), string_substring(source, Ii64(token)))){ + if (token_it_inc(it)){ + match = true; + } + } + return(match); +} + +function String_Const_u8 +api_parse__type_name_with_stars(Arena *arena, String_Const_u8 type, i32 star_counter){ + if (star_counter > 0){ + i32 type_full_size = type.size + star_counter; + u8 *type_buffer = push_array(arena, u8, type_full_size + 1); + block_copy(type_buffer, type.str, type.size); + block_fill_u8(type_buffer + type.size, star_counter, (u8)'*'); + type_buffer[type_full_size] = 0; + type = SCu8(type_buffer, type_full_size); + } + return(type); +} + +function void +api_parse_add_param(Arena *arena, API_Param_List *list, String_Const_u8 type, i32 star_counter, String_Const_u8 name){ + type = api_parse__type_name_with_stars(arena, type, star_counter); + API_Param *param = push_array(arena, API_Param, 1); + sll_queue_push(list->first, list->last, param); + list->count += 1; + param->type_name = type; + param->name = name; +} + +function void +api_parse_add_function(Arena *arena, API_Definition_List *list, String_Const_u8 api_name, String_Const_u8 func_name, String_Const_u8 type, i32 star_counter, API_Param_List param_list){ + API_Definition *api = api_get_api(arena, list, api_name); + type = api_parse__type_name_with_stars(arena, type, star_counter); + API_Call *call = api_call(arena, api, func_name, type); + api_set_param_list(call, param_list); +} + +function void +api_parse_source_add_to_list(Arena *arena, String_Const_u8 source, API_Definition_List *list){ + Token_List token_list = lex_full_input_cpp(arena, source); + Token_Iterator token_it = token_iterator(token_iterator(0, &token_list)); + + for (;;){ + Token *token = token_it_read(&token_it); + if (token->sub_kind == TokenCppKind_EOF){ + break; + } + + if (api_parse__match_identifier(&token_it, source, "api")){ + String_Const_u8 api_name = {}; + String_Const_u8 ret_type = {}; + i32 ret_type_star_counter = 0; + String_Const_u8 func_name = {}; + API_Param_List param_list = {}; + + b32 success = false; + if (api_parse__match(&token_it, TokenCppKind_ParenOp)){ + if (api_parse__match_identifier(&token_it, source, &api_name)){ + if (api_parse__match(&token_it, TokenCppKind_ParenCl)){ + if (api_parse__match_identifier(&token_it, source, "function")){ + if (api_parse__match_identifier(&token_it, source, &ret_type)){ + for (;api_parse__match(&token_it, TokenCppKind_Star);){ + ret_type_star_counter += 1; + } + if (api_parse__match_identifier(&token_it, source, &func_name)){ + if (api_parse__match(&token_it, TokenCppKind_ParenOp)){ + b32 param_list_success = false; + if (api_parse__match_identifier(&token_it, source, "void")){ + param_list_success = true; + } + else{ + for (;;){ + String_Const_u8 type = {}; + i32 star_counter = 0; + String_Const_u8 name = {}; + if (api_parse__match_identifier(&token_it, source, &type)){ + for (;api_parse__match(&token_it, TokenCppKind_Star);){ + star_counter += 1; + } + if (api_parse__match_identifier(&token_it, source, &name)){ + param_list_success = true; + } + else{ + break; + } + } + else{ + break; + } + if (param_list_success){ + api_parse_add_param(arena, ¶m_list, type, star_counter, name); + } + if (api_parse__match(&token_it, TokenCppKind_Comma)){ + param_list_success = false; + } + else{ + break; + } + } + } + if (param_list_success){ + if (api_parse__match(&token_it, TokenCppKind_ParenCl)){ + success = true; + } + } + } + } + } + } + } + } + } + + if (success){ + api_parse_add_function(arena, list, api_name, func_name, ret_type, ret_type_star_counter, param_list); + } + } + else{ + if (!token_it_inc(&token_it)){ + break; + } + } + } + +} + +function API_Definition_List +api_parse_source(Arena *arena, String_Const_u8 source){ + API_Definition_List list = {}; + api_parse_source_add_to_list(arena, source, &list); + return(list); +} + +//////////////////////////////// + +int +main(int argc, char **argv){ + Arena arena = make_arena_malloc(); + + char *hack[] = { + "nothing", + "../code/4ed_api_implementation.cpp" + }; + argv = hack; + argc = 2; + + if (argc < 2){ + printf("usage: