/* * Mr. 4th Dimention - Allen Webster * * 21.1.2015 * * Test for CPP lexer & parser layer for project codename "4ed" * */ // TOP #include "../4ed_meta.h" #define FCPP_STRING_IMPLEMENTATION #include "../4coder_string.h" #include "../4cpp_types.h" #include "../4cpp_lexer_types.h" #define FCPP_LEXER_IMPLEMENTATION #include "../4cpp_lexer.h" #include "4cpp_new_lexer.h" #include #include #include static Data dump_file(char *filename){ Data data = {}; HANDLE file; DWORD hi, lo; file = CreateFile(filename, GENERIC_READ, 0, 0, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, 0); if (file != INVALID_HANDLE_VALUE){ lo = GetFileSize(file, &hi); assert(hi == 0); data.size = (int)lo; data.data = (byte*)malloc(data.size + 1); ReadFile(file, data.data, lo, &lo, 0); assert((int)lo == data.size); CloseHandle(file); } return(data); } typedef struct File_Info{ String filename; int folder; } File_Info; typedef struct File_List{ // Ignore this, it's for internal stuff. void *block; // The list of files and folders. File_Info *infos; int count; // Ignore this, it's for internal stuff. int block_size; } File_List; void* Win32GetMemory(int size){ return (malloc(size)); } void Win32FreeMemory(void *ptr){ free(ptr); } static void system_set_file_list(File_List *file_list, String directory){ if (directory.size > 0){ char dir_space[MAX_PATH + 32]; String dir = make_string(dir_space, 0, MAX_PATH + 32); append(&dir, directory); char trail_str[] = "\\*"; append(&dir, trail_str); char *c_str_dir = make_c_str(dir); WIN32_FIND_DATA find_data; HANDLE search; search = FindFirstFileA(c_str_dir, &find_data); if (search != INVALID_HANDLE_VALUE){ i32 count = 0; i32 file_count = 0; BOOL more_files = 1; do{ if (!match(find_data.cFileName, ".") && !match(find_data.cFileName, "..")){ ++file_count; i32 size = 0; for(;find_data.cFileName[size];++size); count += size + 1; } more_files = FindNextFile(search, &find_data); }while(more_files); FindClose(search); i32 required_size = count + file_count * sizeof(File_Info); if (file_list->block_size < required_size){ Win32FreeMemory(file_list->block); file_list->block = Win32GetMemory(required_size); file_list->block_size = required_size; } file_list->infos = (File_Info*)file_list->block; char *name = (char*)(file_list->infos + file_count); if (file_list->block){ search = FindFirstFileA(c_str_dir, &find_data); if (search != INVALID_HANDLE_VALUE){ File_Info *info = file_list->infos; more_files = 1; do{ if (!match(find_data.cFileName, ".") && !match(find_data.cFileName, "..")){ info->folder = (find_data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0; info->filename.str = name; i32 i = 0; for(;find_data.cFileName[i];++i) *name++ = find_data.cFileName[i]; info->filename.size = i; info->filename.memory_size = info->filename.size + 1; *name++ = 0; replace_char(info->filename, '\\', '/'); ++info; } more_files = FindNextFile(search, &find_data); }while(more_files); FindClose(search); file_list->count = file_count; }else{ Win32FreeMemory(file_list->block); file_list->block = 0; file_list->block_size = 0; } } } } else{ if (directory.str == 0){ Win32FreeMemory(file_list->block); file_list->block = 0; file_list->block_size = 0; } file_list->infos = 0; file_list->count = 0; } } #define TOKEN_MAX (1 << 12) #define TOKEN_ARRAY_SIZE (TOKEN_MAX*sizeof(Cpp_Token)) static void init_test_stack(Cpp_Token_Stack *stack){ stack->tokens = (Cpp_Token*)malloc(TOKEN_ARRAY_SIZE); stack->count = 0; stack->max_count = TOKEN_MAX; } Cpp_Lex_Data lex_data = {}; struct Experiment{ Cpp_Token_Stack correct_stack; Cpp_Token_Stack testing_stack; int passed_total, test_total; }; static void run_experiment(Experiment *exp, char *filename){ String extension = {}; Data file_data; Cpp_File file_cpp; int pass; extension = file_extension(make_string_slowly(filename)); if (match(extension, "cpp") || match(extension, "h")){ pass = 1; printf("testing on file: %s\n", filename); file_data = dump_file(filename); if (file_data.size < (100 << 10)){ exp->test_total++; exp->correct_stack.count = 0; exp->testing_stack.count = 0; memset(exp->correct_stack.tokens, TOKEN_ARRAY_SIZE, 0); memset(exp->testing_stack.tokens, TOKEN_ARRAY_SIZE, 0); file_cpp.data = (char*)file_data.data; file_cpp.size = file_data.size; cpp_lex_file_nonalloc(file_cpp, &exp->correct_stack, lex_data); new_lex::cpp_lex_nonalloc((char*)file_data.data, 0, file_data.size, &exp->testing_stack); if (exp->correct_stack.count != exp->testing_stack.count){ pass = 0; printf("error: stack size mismatch %d original and %d testing\n", exp->correct_stack.count, exp->testing_stack.count); } int min_count = exp->correct_stack.count; if (min_count > exp->testing_stack.count) min_count = exp->testing_stack.count; for (int j = 0; j < min_count; ++j){ Cpp_Token *correct, *testing; correct = exp->correct_stack.tokens + j; testing = exp->testing_stack.tokens + j; if (correct->type != testing->type){ pass = 0; printf("type mismatch at token %d\n", j); } if (correct->start != testing->start || correct->size != testing->size){ pass = 0; printf("token range mismatch at token %d\n" " %d:%d original %d:%d testing\n" " %.*s original %.*s testing\n", j, correct->start, correct->size, testing->start, testing->size, correct->size, file_cpp.data + correct->start, testing->size, file_cpp.data + testing->start); } if (correct->flags != testing->flags){ pass = 0; printf("token flag mismatch at token %d\n", j); } } if (pass){ exp->passed_total++; printf("test passed!\n\n"); } else{ printf("test failed, you failed, fix it now!\n\n"); } } free(file_data.data); } } #define BASE_DIR "w:/4ed/data/test/" int main(){ char test_directory[] = BASE_DIR; File_List all_files = {}; Experiment exp = {}; init_test_stack(&exp.correct_stack); init_test_stack(&exp.testing_stack); AllowLocal(test_directory); AllowLocal(all_files); run_experiment(&exp, BASE_DIR "autotab.cpp"); #if 0 system_set_file_list(&all_files, make_lit_string(test_directory)); for (int i = 0; i < all_files.count; ++i){ if (all_files.infos[i].folder == 0){ run_experiment(&exp, all_files.infos[i].filename.str); } } #endif printf("you passed %d / %d tests\n", exp.passed_total, exp.test_total); return(0); } // BOTTOM