mr4th-public
/
4coder
1
Fork 0
Code Issues Releases Activity

fixed subtle relex bug

master
Allen Webster 2016-09-07 15:07:10 -04:00
parent 723945bb03
commit 992058578d
6 changed files with 357 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

313
4coder_API.html
View File

File diff suppressed because one or more lines are too long

1
4coder_custom.h
View File

@ -12,6 +12,7 @@
#include "4coder_style.h"
#include "4coder_rect.h"
#include "4coder_mem.h"
#include "4cpp_lexer_types.h"
#ifndef FSTRING_STRUCT
#define FSTRING_STRUCT

2
4coder_default_include.cpp
View File

@ -887,6 +887,8 @@ move_past_lead_whitespace(Application_Links *app, View_Summary *view, Buffer_Sum
}
}
//#include "4coder_auto_indent.cpp"
CUSTOM_COMMAND_SIG(auto_tab_line_at_cursor){
uint32_t access = AccessOpen;
View_Summary view = app->get_active_view(app, access);

89
4cpp_lexer.h
View File

@ -14,6 +14,9 @@
#define FCPP_INTERNAL FCPP_LINK
#include <stdint.h>
#define FSTRING_IMPLEMENTATION
#include "4coder_string.h"
#include "4cpp_lexer_types.h"
#include "4cpp_lexer_tables.c"
@ -433,7 +436,7 @@ cpp_lex_nonalloc_null_end_no_limit(Cpp_Lex_Data *S_ptr, char *chunk, int32_t siz
int32_t sub_match = -1;
string_set_match_table(keywords, sizeof(*keywords), ArrayCount(keywords),
make_string(S.tb, S.tb_pos), &sub_match);
make_string(S.tb, S.tb_pos-1), &sub_match);
if (sub_match != -1){
String_And_Flag data = keywords[sub_match];
@ -1165,6 +1168,8 @@ cpp_shift_token_starts(Cpp_Token_Array *array, int32_t from_token_i, int32_t shi
}
}
// TODO(allen): This relex system is a little bit broken. It doesn't allow for the
// data chunks and it doesn't actually set up the state mid-data stream properly.
FCPP_INTERNAL int32_t
cpp_relex_nonalloc_main(Cpp_Relex_State *state,
Cpp_Token_Array *relex_array,
@ -1175,52 +1180,54 @@ cpp_relex_nonalloc_main(Cpp_Relex_State *state,
cpp_shift_token_starts(array, state->end_token_i, state->amount);
Cpp_Lex_Data lex = cpp_lex_data_init(spare);
lex.pp_state = cpp_token_get_pp_state(tokens[state->start_token_i].state_flags);
lex.pos = state->relex_start;
int32_t relex_end_i = state->end_token_i;
Cpp_Token match_token = cpp_index_array(array, state->size, relex_end_i);
Cpp_Token end_token = match_token;
int32_t went_too_far = false;
// TODO(allen): This can be better I suspect.
for (;;){
int32_t result =
cpp_lex_nonalloc_no_null_out_limit(&lex, state->data,
state->size, state->size,
relex_array, 1);
if (state->relex_start < state->size){
Cpp_Lex_Data lex = cpp_lex_data_init(spare);
lex.pp_state = cpp_token_get_pp_state(tokens[state->start_token_i].state_flags);
lex.pos = state->relex_start;
switch (result){
case LexResult_HitTokenLimit:
{
Cpp_Token token = relex_array->tokens[relex_array->count-1];
if (token.start == end_token.start &&
token.size == end_token.size &&
token.flags == end_token.flags &&
token.state_flags == end_token.state_flags){
--relex_array->count;
goto double_break;
}
// TODO(allen): This can be better I suspect.
for (;;){
int32_t result =
cpp_lex_nonalloc_no_null_out_limit(&lex, state->data,
state->size, state->size,
relex_array, 1);
while (lex.pos > end_token.start && relex_end_i < array->count){
++relex_end_i;
end_token = cpp_index_array(array, state->size, relex_end_i);
switch (result){
case LexResult_HitTokenLimit:
{
Cpp_Token token = relex_array->tokens[relex_array->count-1];
if (token.start == end_token.start &&
token.size == end_token.size &&
token.flags == end_token.flags &&
token.state_flags == end_token.state_flags){
--relex_array->count;
goto double_break;
}
while (lex.pos > end_token.start && relex_end_i < array->count){
++relex_end_i;
end_token = cpp_index_array(array, state->size, relex_end_i);
}
}
break;
case LexResult_NeedChunk: Assert(!"Invalid path"); break;
case LexResult_NeedTokenMemory:
went_too_far = true;
goto double_break;
case LexResult_Finished:
goto double_break;
}
break;
case LexResult_NeedChunk: Assert(!"Invalid path"); break;
case LexResult_NeedTokenMemory:
went_too_far = true;
goto double_break;
case LexResult_Finished:
goto double_break;
}
double_break:;
}
double_break:;
if (!went_too_far){
*relex_end = relex_end_i;
@ -1312,6 +1319,8 @@ DOC_SEE(cpp_make_token_array)
S.tb = (char*)malloc(size);
int32_t quit = 0;
char empty = 0;
token_array_out->count = 0;
for (;!quit;){
int32_t result = cpp_lex_step(&S, data, size, HAS_NULL_TERM, token_array_out, NO_OUT_LIMIT);
@ -1326,10 +1335,10 @@ DOC_SEE(cpp_make_token_array)
Assert(token_array_out->count < token_array_out->max_count);
// NOTE(allen): We told the system we would provide the null
// terminator, but we didn't actually, so provide the null
// terminator via this one byte chunk.
char empty = 0;
cpp_lex_step(&S, &empty, 1, HAS_NULL_TERM, token_array_out, NO_OUT_LIMIT);
// terminator, but as it turned out we didn't actually. So in
// the next iteration pass a 1 byte chunk with the null terminator.
data = &empty;
size = 1;
}break;
case LexResult_NeedTokenMemory:

23
4ed_metagen.cpp
View File

@ -2545,10 +2545,8 @@ generate_custom_headers(){
{make_lit_string("ENUM") , Item_Enum } ,
};
#if 0
Meta_Unit unit = compile_meta_unit(part, type_files, ArrayCount(type_files),
type_keys, ArrayCount(type_keys));
#endif
// NOTE(allen): Output
String out = str_alloc(part, 10 << 20);
@ -2922,9 +2920,9 @@ generate_custom_headers(){
static Section sections[] = {
{"introduction", "Introduction"},
// {"4coder_systems", "4coder Systems"},
// {"types_and_functions", "Types and Functions"},
// {"string_library", "String Library"},
{"4coder_systems", "4coder Systems"},
{"types_and_functions", "Types and Functions"},
{"string_library", "String Library"},
{"lexer_library", "Lexer Library"}
};
@ -2952,6 +2950,8 @@ generate_custom_headers(){
append_sc(&out, sections[msection].display_string);
append_sc(&out, "</h2>");
#if 0
// NOTE(allen): doc intro for lexer standalone
append_sc(&out,
"<div>"
"<p>This is the documentation for the 4cpp lexer version 1.0. "
@ -2964,8 +2964,7 @@ generate_custom_headers(){
"4coder forums hosted on handmade.network at "
"<span style='"CODE_STYLE"'>4coder.handmade.network</span></p>"
"</div>");
#if 0
#endif
append_sc(&out,
"<div>"
@ -2995,6 +2994,7 @@ generate_custom_headers(){
#undef MAJOR_SECTION
#define MAJOR_SECTION "3"
msection = 2;
append_sc(&out, "\n<h2 id='section_");
append_sc(&out, sections[msection].id_string);
@ -3056,6 +3056,7 @@ generate_custom_headers(){
#undef MAJOR_SECTION
#define MAJOR_SECTION "4"
msection = 3;
append_sc(&out, "\n<h2 id='section_");
append_sc(&out, sections[msection].id_string);
@ -3092,13 +3093,7 @@ generate_custom_headers(){
#undef MAJOR_SECTION
#define MAJOR_SECTION "5"
#endif
#undef MAJOR_SECTION
#define MAJOR_SECTION "2"
msection = 1;
msection = 4;
append_sc(&out, "\n<h2 id='section_");
append_sc(&out, sections[msection].id_string);

2
TODO.txt
View File

@ -75,7 +75,6 @@
;
; [] indication on failure to save
; [] history is broken, revist the entire system
;
; [] 8.0\Include\um\dsound.h (not reproduced, get more info)
; [] paste external text from bug report (in email) (not reproduced, get more info)
;
@ -89,6 +88,7 @@
; [] option to not open *messages* every startup
; [] commands for resizing panels
; [] make panel resizing not whacky with child panels
; [] killing compilation panel changes active panel
; [] control over how mouse effects panel focus
; [] API docs as text file
; [] user file bar string