/* ** C Scripting - Expression Parser */ #define SY__MAIN 1 #include "symbol_set.h" #include "symbol_set.init.h" #include "examples.base.h" #include "expression_parser.h" #include "examples.base.c" //////////////////////////////// // Operator Functions static U32 exp_operator_id_from_string(String8 string){ // TODO(allen): bake accelerator ahead of time for this path U32 operator_id = 0; for (SyEachID(EXP_OPERATOR, id)){ EXP_Operator *op = SyAddressFromID(EXP_OPERATOR, id); if (str8_match(op->token_str, string)){ operator_id = id; break; } } return(operator_id); } //////////////////////////////// // Lexer Functions static EXP_TokenArray exp_lex(Arena *arena, String8 text){ // Static Map of char -> TokenKind static U8 token_kind_table[127] = {0}; static B8 token_kinds_initialized = 0; if (!token_kinds_initialized){ token_kind_table[' '] = token_kind_table['\n'] = token_kind_table['\t'] = token_kind_table['\r'] = token_kind_table['\f'] = token_kind_table['\v'] = EXP_TokenKind_Whitespace; token_kind_table['`'] = token_kind_table['~'] = token_kind_table['!'] = token_kind_table['@'] = token_kind_table['#'] = token_kind_table['$'] = token_kind_table['%'] = token_kind_table['^'] = token_kind_table['&'] = token_kind_table['*'] = token_kind_table['='] = token_kind_table['-'] = token_kind_table['+'] = token_kind_table['\\'] = token_kind_table['|'] = token_kind_table[':'] = token_kind_table[','] = token_kind_table['<'] = token_kind_table['.'] = token_kind_table['>'] = token_kind_table['/'] = token_kind_table['?'] = EXP_TokenKind_Operator; for (U32 i = 'a'; i <= 'z'; i += 1){ token_kind_table[i] = EXP_TokenKind_IDNum; } for (U32 i = 'A'; i <= 'Z'; i += 1){ token_kind_table[i] = EXP_TokenKind_IDNum; } for (U32 i = '0'; i <= '9'; i += 1){ token_kind_table[i] = EXP_TokenKind_IDNum; } token_kind_table['('] = EXP_TokenKind_OpenParen|(1 << 7); token_kind_table[')'] = EXP_TokenKind_CloseParen|(1 << 7); token_kind_table['['] = EXP_TokenKind_OpenBracket|(1 << 7); token_kind_table[']'] = EXP_TokenKind_CloseBracket|(1 << 7); token_kind_table['{'] = EXP_TokenKind_OpenBrace|(1 << 7); token_kind_table['}'] = EXP_TokenKind_CloseBrace|(1 << 7); } // over-allocate tokens U32 token_cap = text.size + 1; EXP_Token *tokens = push_array(arena, EXP_Token, token_cap); // fill tokens array EXP_Token *tp = tokens; U8 *ptr = text.str; U8 *opl = text.str + text.size; for (;ptr < opl;){ U32 kind = EXP_TokenKind_IDNum; if (*ptr < 127){ kind = token_kind_table[*ptr]; } tp->pos = (U32)(ptr - text.str); tp->kind = kind&~(1 << 7); if (kind&(1 << 7)){ ptr += 1; } else{ for (ptr += 1; ptr < opl && token_kind_table[*ptr] == kind; ptr += 1); } tp += 1; } tp->pos = (U32)text.size; tp->kind = EXP_TokenKind_EndOfFile; tp += 1; // put back extra memory U32 token_count = (U32)(tp - tokens); arena_pop_amount(arena, sizeof(*tokens)*(token_cap - token_count)); // arrange as result EXP_TokenArray result = {0}; result.tokens = tokens; result.count = token_count - 1; return(result); } //////////////////////////////// // Parser Functions EXP_PRECEDENCE(1, Multiplicative); EXP_PRECEDENCE(2, Additive); EXP_PRECEDENCE(3, Assignment, .right_to_left = 1); EXP_OPERATOR(Multiply, "*", Multiplicative); EXP_OPERATOR(Divide, "/", Multiplicative); EXP_OPERATOR(Modulo, "%", Multiplicative); EXP_OPERATOR(Add, "+", Additive); EXP_OPERATOR(Subtract, "-", Additive); EXP_OPERATOR(Assign, "=", Assignment); EXP_OPERATOR(MulAssign, "*=", Assignment); EXP_OPERATOR(DivAssign, "/=", Assignment); EXP_OPERATOR(AddAssign, "+=", Assignment); EXP_OPERATOR(SubAssign, "-=", Assignment); static EXP_ParseNode* exp_parse(Arena *arena, String8 text, EXP_TokenArray tokens){ EXP_ParseCtx ctx = {0}; ctx.text = text; ctx.tokens = tokens; ctx.token = tokens.tokens; EXP_ParseNode *result = exp__parse_op(arena, &ctx); return(result); } static EXP_ParseNode* exp__parse_op_rec(Arena *arena, EXP_ParseCtx *ctx, EXP_ParseNode *left, U32 max_precedence_level){ // read first operator exp__parse_skip_whitespace(ctx); EXP_OperatorRead peak_op = exp__parse_read_operator(ctx); for (;;){ EXP_OperatorRead left_op = peak_op; // end loop when precedence level increases (or no operator next) if (left_op.operator_id == 0 || left_op.precedence_level > max_precedence_level){ break; } // consume left op ctx->token += 1; // read right leaf exp__parse_skip_whitespace(ctx); EXP_ParseNode *right = exp__parse_leaf(arena, ctx); // apply stickier right hand operators for (;;){ // read right operator exp__parse_skip_whitespace(ctx); peak_op = exp__parse_read_operator(ctx); // check precedence if (peak_op.operator_id == 0 || peak_op.precedence_level > left_op.precedence_level || (peak_op.precedence_level == left_op.precedence_level && !peak_op.right_to_left)){ break; } // recursively fold onto right right = exp__parse_op_rec(arena, ctx, right, peak_op.precedence_level); } // combine left & right with operator { EXP_ParseNode *node = push_array(arena, EXP_ParseNode, 1); node->kind = EXP_ParseNodeKind_Operator; node->op.operator_id = left_op.operator_id; node->op.child[0] = left; node->op.child[1] = right; left = node; } } done:; return(left); } static EXP_ParseNode* exp__parse_op(Arena *arena, EXP_ParseCtx *ctx){ EXP_ParseNode *left = exp__parse_leaf(arena, ctx); EXP_ParseNode *result = exp__parse_op_rec(arena, ctx, left, 3); return(result); } static EXP_ParseNode* exp__parse_leaf(Arena *arena, EXP_ParseCtx *ctx){ EXP_ParseNode *result = 0; exp__parse_skip_whitespace(ctx); // check next token switch (ctx->token->kind){ case EXP_TokenKind_OpenParen: { ctx->token += 1; result = exp__parse_op(arena, ctx); if (ctx->error) goto done; if (ctx->token->kind == EXP_TokenKind_CloseParen){ ctx->token += 1; } else{ exp__parse_errorf(arena, ctx, "Expected ')'"); goto done; } }break; case EXP_TokenKind_IDNum: { result = push_array(arena, EXP_ParseNode, 1); result->kind = EXP_ParseNodeKind_Leaf; result->leaf.token_idx = (U32)(ctx->token - ctx->tokens.tokens); ctx->token += 1; }break; } done:; return(result); } static void exp__parse_errorf(Arena *arena, EXP_ParseCtx *ctx, char *fmt, ...){ if (ctx->error == 0){ va_list args; va_start(args, fmt); String8 string = str8_pushfv(arena, fmt, args); va_end(args); ctx->error = 1; ctx->error_token_idx = (U32)(ctx->token - ctx->tokens.tokens); ctx->error_message = string; } } static void exp__parse_skip_whitespace(EXP_ParseCtx *ctx){ if (ctx->token->kind == EXP_TokenKind_Whitespace){ ctx->token += 1; } } static EXP_OperatorRead exp__parse_read_operator(EXP_ParseCtx *ctx){ EXP_OperatorRead result = {0}; if (ctx->token->kind == EXP_TokenKind_Operator){ String8 operator_text = str8(ctx->text.str + ctx->token->pos, (ctx->token + 1)->pos - ctx->token->pos); result.operator_id = exp_operator_id_from_string(operator_text); if (result.operator_id != 0){ // TODO(allen): [operator] -> precedence_level accelerator EXP_Operator *op = SyAddressFromID(EXP_OPERATOR, result.operator_id); EXP_Precedence *precedence = SyAddressFromID(EXP_PRECEDENCE, op->precedence_id); result.precedence_level = precedence->order; result.right_to_left = precedence->right_to_left; } } return(result); } static void exp_parse_node_print(String8 text, EXP_TokenArray tokens, EXP_ParseNode *node, S32 indent){ printf("%.*s", indent, " "); switch (node->kind){ case EXP_ParseNodeKind_Operator: { EXP_Operator *op = SyAddressFromID(EXP_OPERATOR, node->op.operator_id); printf("%.*s\n", (U32)op->token_str.size, op->token_str.str); exp_parse_node_print(text, tokens, node->op.child[0], indent + 1); exp_parse_node_print(text, tokens, node->op.child[1], indent + 1); }break; case EXP_ParseNodeKind_Leaf: { EXP_Token *token = &tokens.tokens[node->leaf.token_idx]; String8 string = str8(text.str + token->pos, (token + 1)->pos - token->pos); printf("%.*s\n", (U32)string.size, string.str); }break; } } //////////////////////////////// // Entrpy Point int main(void){ sy__run_init(); Arena arena = arena_alloc((10 << 20)); #if 0 String8 text = str8_const("((FooBar + 100 ** 3 ^;;\r\n\t\t 17){{[[?/:::~`]]}})"); EXP_TokenArray tokens = exp_lex(&arena, text); for (U32 i = 0; i < tokens.count; i += 1){ EXP_Token *token = &tokens.tokens[i]; char *token_kind_str = 0; switch (token->kind){ #define X(N) case EXP_TokenKind_##N: token_kind_str = #N; break; EXP_TokenKindXList(X) #undef X } U32 token_size = (token + 1)->pos - token->pos; printf("TOKEN %s: '%.*s'\n", token_kind_str, token_size, text.str + token->pos); } #endif String8 text = str8_const("1 + 2 = a - b + c /= a + b * c = (a + b) / (x += c) = d"); EXP_TokenArray tokens = exp_lex(&arena, text); EXP_ParseNode *root = exp_parse(&arena, text, tokens); exp_parse_node_print(text, tokens, root, 0); return(0); } /* TODO ** C-Scripted Interpretations ** ** C-Scripted Prefix Postfix Operators ** ** C-Scripted Bracket Operators ** ** C-Scripted Structure Builder */