c-scripting/examples/expression_parser.c

/*
** C Scripting - Expression Parser
*/

#define SY__MAIN 1
#include "symbol_set.h"
#include "symbol_set.init.h"

#include "examples.base.h"
#include "expression_parser.h"

#include "examples.base.c"

////////////////////////////////
// Operator Functions

static U32
exp_operator_id_from_string(String8 string){
  // TODO(allen): bake accelerator ahead of time for this path
  U32 operator_id = 0;
  for (SyEachID(EXP_OPERATOR, id)){
    EXP_Operator *op = SyAddressFromID(EXP_OPERATOR, id);
    if (str8_match(op->token_str, string)){
      operator_id = id;
      break;
    }
  }
  return(operator_id);
}

////////////////////////////////
// Lexer Functions

static EXP_TokenArray
exp_lex(Arena *arena, String8 text){
  // Static Map of char -> TokenKind
  static U8 token_kind_table[127] = {0};
  static B8 token_kinds_initialized = 0;
  if (!token_kinds_initialized){
    token_kind_table[' '] = token_kind_table['\n'] =
      token_kind_table['\t'] = token_kind_table['\r'] =
      token_kind_table['\f'] = token_kind_table['\v'] = EXP_TokenKind_Whitespace;
    
    token_kind_table['`'] =  token_kind_table['~'] =
      token_kind_table['!'] = token_kind_table['@'] =
      token_kind_table['#'] = token_kind_table['$'] =
      token_kind_table['%'] = token_kind_table['^'] =
      token_kind_table['&'] = token_kind_table['*'] =
      token_kind_table['='] = token_kind_table['-'] =
      token_kind_table['+'] = token_kind_table['\\'] =
      token_kind_table['|'] = token_kind_table[':'] =
      token_kind_table[','] = token_kind_table['<'] =
      token_kind_table['.'] = token_kind_table['>'] =
      token_kind_table['/'] = token_kind_table['?'] = EXP_TokenKind_Operator;
    
    for (U32 i = 'a'; i <= 'z'; i += 1){
      token_kind_table[i] = EXP_TokenKind_IDNum;
    }
    for (U32 i = 'A'; i <= 'Z'; i += 1){
      token_kind_table[i] = EXP_TokenKind_IDNum;
    }
    for (U32 i = '0'; i <= '9'; i += 1){
      token_kind_table[i] = EXP_TokenKind_IDNum;
    }
    
    token_kind_table['('] = EXP_TokenKind_OpenParen|(1 << 7);
    token_kind_table[')'] = EXP_TokenKind_CloseParen|(1 << 7);
    token_kind_table['['] = EXP_TokenKind_OpenBracket|(1 << 7);
    token_kind_table[']'] = EXP_TokenKind_CloseBracket|(1 << 7);
    token_kind_table['{'] = EXP_TokenKind_OpenBrace|(1 << 7);
    token_kind_table['}'] = EXP_TokenKind_CloseBrace|(1 << 7);
  }
  
  // over-allocate tokens
  U32 token_cap = text.size + 1;
  EXP_Token *tokens = push_array(arena, EXP_Token, token_cap);
  
  // fill tokens array
  EXP_Token *tp = tokens;
  U8 *ptr = text.str;
  U8 *opl = text.str + text.size;
  for (;ptr < opl;){
    U32 kind = EXP_TokenKind_IDNum;
    if (*ptr < 127){
      kind = token_kind_table[*ptr];
    }
    
    tp->pos = (U32)(ptr - text.str);
    tp->kind = kind&~(1 << 7);
    
    if (kind&(1 << 7)){
      ptr += 1;
    }
    else{
      for (ptr += 1;
           ptr < opl && token_kind_table[*ptr] == kind;
           ptr += 1);
    }
    
    tp += 1;
  }
  
  tp->pos = (U32)text.size;
  tp->kind = EXP_TokenKind_EndOfFile;
  tp += 1;
  
  // put back extra memory
  U32 token_count = (U32)(tp - tokens);
  arena_pop_amount(arena, sizeof(*tokens)*(token_cap - token_count));
  
  // arrange as result
  EXP_TokenArray result = {0};
  result.tokens = tokens;
  result.count = token_count - 1;
  return(result);
}

////////////////////////////////
// Parser Functions

EXP_PRECEDENCE(1, Multiplicative);
EXP_PRECEDENCE(2, Additive);
EXP_PRECEDENCE(3, Assignment, .right_to_left = 1);

EXP_OPERATOR(Multiply, "*", Multiplicative);
EXP_OPERATOR(Divide,   "/", Multiplicative);
EXP_OPERATOR(Modulo,   "%", Multiplicative);
EXP_OPERATOR(Add,      "+", Additive);
EXP_OPERATOR(Subtract, "-", Additive);
EXP_OPERATOR(Assign,   "=", Assignment);
EXP_OPERATOR(MulAssign, "*=", Assignment);
EXP_OPERATOR(DivAssign, "/=", Assignment);
EXP_OPERATOR(AddAssign, "+=", Assignment);
EXP_OPERATOR(SubAssign, "-=", Assignment);

static EXP_ParseNode*
exp_parse(Arena *arena, String8 text, EXP_TokenArray tokens){
  EXP_ParseCtx ctx = {0};
  ctx.text = text;
  ctx.tokens = tokens;
  ctx.token = tokens.tokens;
  
  EXP_ParseNode *result = exp__parse_op(arena, &ctx);
  return(result);
}

static EXP_ParseNode*
exp__parse_op_rec(Arena *arena, EXP_ParseCtx *ctx,
                  EXP_ParseNode *left, U32 max_precedence_level){
  // read first operator
  exp__parse_skip_whitespace(ctx);
  EXP_OperatorRead peak_op = exp__parse_read_operator(ctx);
  
  for (;;){
    EXP_OperatorRead left_op = peak_op;
    
    // end loop when precedence level increases (or no operator next)
    if (left_op.operator_id == 0 ||
        left_op.precedence_level > max_precedence_level){
      break;
    }
    
    // consume left op
    ctx->token += 1;
    
    // read right leaf
    exp__parse_skip_whitespace(ctx);
    EXP_ParseNode *right = exp__parse_leaf(arena, ctx);
    
    // apply stickier right hand operators
    for (;;){
      
      // read right operator
      exp__parse_skip_whitespace(ctx);
      peak_op = exp__parse_read_operator(ctx);
      
      // check precedence
      if (peak_op.operator_id == 0 ||
          peak_op.precedence_level > left_op.precedence_level ||
          (peak_op.precedence_level == left_op.precedence_level && !peak_op.right_to_left)){
        break;
      }
      
      // recursively fold onto right
      right = exp__parse_op_rec(arena, ctx, right, peak_op.precedence_level);
    }
    
    // combine left & right with operator
    {
      EXP_ParseNode *node = push_array(arena, EXP_ParseNode, 1);
      node->kind = EXP_ParseNodeKind_Operator;
      node->op.operator_id = left_op.operator_id;
      node->op.child[0] = left;
      node->op.child[1] = right;
      left = node;
    }
  }
  
  done:;
  return(left);
}

static EXP_ParseNode*
exp__parse_op(Arena *arena, EXP_ParseCtx *ctx){
  EXP_ParseNode *left = exp__parse_leaf(arena, ctx);
  EXP_ParseNode *result = exp__parse_op_rec(arena, ctx, left, 3);
  return(result);
}

static EXP_ParseNode*
exp__parse_leaf(Arena *arena, EXP_ParseCtx *ctx){
  EXP_ParseNode *result = 0;
  
  exp__parse_skip_whitespace(ctx);
  
  // check next token
  switch (ctx->token->kind){
    case EXP_TokenKind_OpenParen: {
      ctx->token += 1;
      result = exp__parse_op(arena, ctx);
      if (ctx->error) goto done;
      if (ctx->token->kind == EXP_TokenKind_CloseParen){
        ctx->token += 1;
      }
      else{
        exp__parse_errorf(arena, ctx, "Expected ')'");
        goto done;
      }
    }break;
    
    case EXP_TokenKind_IDNum: {
      result = push_array(arena, EXP_ParseNode, 1);
      result->kind = EXP_ParseNodeKind_Leaf;
      result->leaf.token_idx = (U32)(ctx->token - ctx->tokens.tokens);
      ctx->token += 1;
    }break;
  }
  
  done:;
  return(result);
}

static void
exp__parse_errorf(Arena *arena, EXP_ParseCtx *ctx, char *fmt, ...){
  if (ctx->error == 0){
    va_list args;
    va_start(args, fmt);
    String8 string = str8_pushfv(arena, fmt, args);
    va_end(args);
    ctx->error = 1;
    ctx->error_token_idx = (U32)(ctx->token - ctx->tokens.tokens);
    ctx->error_message = string;
  }
}

static void
exp__parse_skip_whitespace(EXP_ParseCtx *ctx){
  if (ctx->token->kind == EXP_TokenKind_Whitespace){
    ctx->token += 1;
  }
}

static EXP_OperatorRead
exp__parse_read_operator(EXP_ParseCtx *ctx){
  EXP_OperatorRead result = {0};
  if (ctx->token->kind == EXP_TokenKind_Operator){
    String8 operator_text = str8(ctx->text.str + ctx->token->pos,
                                 (ctx->token + 1)->pos - ctx->token->pos);
    result.operator_id = exp_operator_id_from_string(operator_text);
    
    if (result.operator_id != 0){
      // TODO(allen): [operator] -> precedence_level accelerator
      EXP_Operator *op = SyAddressFromID(EXP_OPERATOR, result.operator_id);
      EXP_Precedence *precedence = SyAddressFromID(EXP_PRECEDENCE, op->precedence_id);
      result.precedence_level = precedence->order;
      result.right_to_left = precedence->right_to_left;
    }
  }
  return(result);
}

static void
exp_parse_node_print(String8 text, EXP_TokenArray tokens,
                     EXP_ParseNode *node, S32 indent){
  printf("%.*s", indent, "                                                                ");
  switch (node->kind){
    case EXP_ParseNodeKind_Operator: {
      EXP_Operator *op = SyAddressFromID(EXP_OPERATOR, node->op.operator_id);
      printf("%.*s\n", (U32)op->token_str.size, op->token_str.str);
      exp_parse_node_print(text, tokens, node->op.child[0], indent + 1);
      exp_parse_node_print(text, tokens, node->op.child[1], indent + 1);
    }break;
    
    case EXP_ParseNodeKind_Leaf: {
      EXP_Token *token = &tokens.tokens[node->leaf.token_idx];
      String8 string = str8(text.str + token->pos, (token + 1)->pos - token->pos);
      printf("%.*s\n", (U32)string.size, string.str);
    }break;
  }
}

////////////////////////////////
// Entrpy Point

int main(void){
  sy__run_init();
  
  Arena arena = arena_alloc((10 << 20));
  
#if 0
  String8 text = str8_const("((FooBar + 100 ** 3 ^;;\r\n\t\t 17){{[[?/:::~`]]}})");
  EXP_TokenArray tokens = exp_lex(&arena, text);
  
  for (U32 i = 0; i < tokens.count; i += 1){
    EXP_Token *token = &tokens.tokens[i];
    char *token_kind_str = 0;
    switch (token->kind){
#define X(N) case EXP_TokenKind_##N: token_kind_str = #N; break;
      EXP_TokenKindXList(X)
#undef X
    }
    U32 token_size = (token + 1)->pos - token->pos;
    printf("TOKEN %s: '%.*s'\n", token_kind_str, token_size, text.str + token->pos);
  }
#endif
  
  String8 text = str8_const("1 + 2 = a - b + c /= a + b * c = (a + b) / (x += c) = d");
  EXP_TokenArray tokens = exp_lex(&arena, text);
  EXP_ParseNode *root = exp_parse(&arena, text, tokens);
  
  exp_parse_node_print(text, tokens, root, 0);
  
  return(0);
}


/* TODO
 ** C-Scripted Interpretations
** 
** C-Scripted Prefix Postfix Operators
** 
** C-Scripted Bracket Operators
** 
** C-Scripted Structure Builder
*/