Cleanup; removed 4coder_utf8 library; rewrote win32 u8 <-> u16 layer

4coder_base_types.h

@@ -999,6 +999,12 @@ struct Character_Consume_Result{
     u32 codepoint;
 };
 
+global u32 surrogate_min = 0xD800;
+global u32 surrogate_max = 0xDFFF;
+
+global u32 nonchar_min = 0xFDD0;
+global u32 nonchar_max = 0xFDEF;
+
 struct Data{
     u8 *data;
     umem size;

4coder_default_include.cpp

@@ -15,7 +15,6 @@
 #include "4coder_stringf.cpp"
 #include "4coder_app_links_allocator.cpp"
 
-#include "4coder_lib/4coder_utf8.h"
 #include "4coder_table.h"
 #include "4coder_token.h"
 

4coder_generated/command_metadata.h

@@ -248,12 +248,12 @@ i32 line_number;
 };
 static Command_Metadata fcoder_metacmd_table[226] = {
 { PROC_LINKS(set_bindings_mac_default, 0), "set_bindings_mac_default", 24,  "Remap keybindings using the 'mac-default' mapping rule.", 55, "w:\\4ed\\code\\4coder_remapping_commands.cpp", 41, 62 },
-{ PROC_LINKS(seek_beginning_of_textual_line, 0), "seek_beginning_of_textual_line", 30,  "Seeks the cursor to the beginning of the line across all text.", 62, "w:\\4ed\\code\\4coder_helper.cpp", 29, 2160 },
+{ PROC_LINKS(seek_beginning_of_textual_line, 0), "seek_beginning_of_textual_line", 30,  "Seeks the cursor to the beginning of the line across all text.", 62, "w:\\4ed\\code\\4coder_helper.cpp", 29, 2152 },
-{ PROC_LINKS(seek_end_of_textual_line, 0), "seek_end_of_textual_line", 24,  "Seeks the cursor to the end of the line across all text.", 56, "w:\\4ed\\code\\4coder_helper.cpp", 29, 2166 },
+{ PROC_LINKS(seek_end_of_textual_line, 0), "seek_end_of_textual_line", 24,  "Seeks the cursor to the end of the line across all text.", 56, "w:\\4ed\\code\\4coder_helper.cpp", 29, 2158 },
-{ PROC_LINKS(seek_beginning_of_line, 0), "seek_beginning_of_line", 22,  "Seeks the cursor to the beginning of the visual line.", 53, "w:\\4ed\\code\\4coder_helper.cpp", 29, 2172 },
+{ PROC_LINKS(seek_beginning_of_line, 0), "seek_beginning_of_line", 22,  "Seeks the cursor to the beginning of the visual line.", 53, "w:\\4ed\\code\\4coder_helper.cpp", 29, 2164 },
-{ PROC_LINKS(seek_end_of_line, 0), "seek_end_of_line", 16,  "Seeks the cursor to the end of the visual line.", 47, "w:\\4ed\\code\\4coder_helper.cpp", 29, 2178 },
+{ PROC_LINKS(seek_end_of_line, 0), "seek_end_of_line", 16,  "Seeks the cursor to the end of the visual line.", 47, "w:\\4ed\\code\\4coder_helper.cpp", 29, 2170 },
-{ PROC_LINKS(goto_beginning_of_file, 0), "goto_beginning_of_file", 22,  "Sets the cursor to the beginning of the file.", 45, "w:\\4ed\\code\\4coder_helper.cpp", 29, 2184 },
+{ PROC_LINKS(goto_beginning_of_file, 0), "goto_beginning_of_file", 22,  "Sets the cursor to the beginning of the file.", 45, "w:\\4ed\\code\\4coder_helper.cpp", 29, 2176 },
-{ PROC_LINKS(goto_end_of_file, 0), "goto_end_of_file", 16,  "Sets the cursor to the end of the file.", 39, "w:\\4ed\\code\\4coder_helper.cpp", 29, 2192 },
+{ PROC_LINKS(goto_end_of_file, 0), "goto_end_of_file", 16,  "Sets the cursor to the end of the file.", 39, "w:\\4ed\\code\\4coder_helper.cpp", 29, 2184 },
 { PROC_LINKS(change_active_panel, 0), "change_active_panel", 19,  "Change the currently active panel, moving to the panel with the next highest view_id.", 85, "w:\\4ed\\code\\4coder_default_framework.cpp", 40, 196 },
 { PROC_LINKS(change_active_panel_backwards, 0), "change_active_panel_backwards", 29,  "Change the currently active panel, moving to the panel with the next lowest view_id.", 84, "w:\\4ed\\code\\4coder_default_framework.cpp", 40, 206 },
 { PROC_LINKS(open_panel_vsplit, 0), "open_panel_vsplit", 17,  "Create a new panel by vertically splitting the active panel.", 60, "w:\\4ed\\code\\4coder_default_framework.cpp", 40, 216 },

4coder_helper.cpp

@@ -276,9 +276,9 @@ end_bind_helper_get_buffer(Bind_Helper *helper){
 
 internal u32
 get_key_code(char *buffer){
-    u32 ignore;
+    String_Const_u8 str = SCu8(buffer);
-    u32 result = utf8_to_u32_length_unchecked((u8*)buffer, &ignore);
+    Character_Consume_Result consume = utf8_consume(str.str, str.size);
-    return(result);
+    return(consume.codepoint);
 }
 
 ////////////////////////////////
@@ -1559,21 +1559,13 @@ key_is_unmodified(Key_Event_Data *key){
 }
 
 internal u32
-to_writable_character(User_Input in, uint8_t *character){
+to_writable_character(User_Input in, u8 *space){
-    u32 result = 0;
+    return(utf8_write(space, in.key.character));
-    if (in.key.character != 0){
-        u32_to_utf8_unchecked(in.key.character, character, &result);
-    }
-    return(result);
 }
 
 internal u32
-to_writable_character(Key_Event_Data key, uint8_t *character){
+to_writable_character(Key_Event_Data key, u8 *space){
-    u32 result = 0;
+    return(utf8_write(space, key.character));
-    if (key.character != 0){
-        u32_to_utf8_unchecked(key.character, character, &result);
-    }
-    return(result);
 }
 
 internal String_Const_u8
@@ -1617,7 +1609,7 @@ query_user_general(Application_Links *app, Query_Bar *bar, b32 force_number){
             break;
         }
         
-        uint8_t character[4];
+        u8 character[4];
         u32 length = 0;
         b32 good_character = false;
         if (key_is_unmodified(&in.key)){

4coder_lib/4coder_utf8.h

@@ -1,395 +0,0 @@
-/*
- * Mr. 4th Dimention - Allen Webster
- *
- * 17.02.2017
- *
- * Code for converting to and from utf8 to ANSI and utf16 text encodings.
- *
- */
-
-// TOP
-
-#if !defined(FED_UTF8_CONVERSION_H)
-#define FED_UTF8_CONVERSION_H
-
-static u32 cp_min_by_utf8_length[] = {
-    0x0,
-    0x0,
-    0x80,
-    0x800,
-    0x10000,
-};
-
-static u32 surrogate_min = 0xD800;
-static u32 surrogate_max = 0xDFFF;
-
-static u32 nonchar_min = 0xFDD0;
-static u32 nonchar_max = 0xFDEF;
-
-static b32
-codepoint_is_whitespace(u32 codepoint){
-    b32 result = false;
-    if (codepoint == ' ' || codepoint == '\r' || codepoint == '\n' || codepoint == '\t'){
-        result = true;
-    }
-    return(result);
-}
-
-static u32
-utf8_to_u32_length_unchecked(u8 *buffer, u32 *length_out){
-    u32 result = 0;
-    
-    if (buffer[0] < 0x80){
-        result = (u32)buffer[0];
-        *length_out = 1;
-    }
-    else if (buffer[0] < 0xE0){
-        result = ((u32)((buffer[0])&0x1F)) << 6;
-        result |= ((u32)((buffer[1])&0x3F));
-        *length_out = 2;
-    }
-    else if (buffer[0] < 0xF0){
-        result = ((u32)((buffer[0])&0x0F)) << 12;
-        result |= ((u32)((buffer[1])&0x3F)) << 6;
-        result |= ((u32)((buffer[2])&0x3F));
-        *length_out = 3;
-    }
-    else{
-        result = ((u32)((buffer[0])&0x07)) << 18;
-        result |= ((u32)((buffer[1])&0x3F)) << 12;
-        result |= ((u32)((buffer[2])&0x3F)) << 6;
-        result |= ((u32)((buffer[3])&0x3F));
-        *length_out = 4;
-    }
-    
-    if (result < cp_min_by_utf8_length[*length_out] || (result >= surrogate_min && result <= surrogate_max)){
-        result = 0;
-        *length_out = 0;
-    }
-    
-    return(result);
-}
-
-static u32
-utf8_to_u32_unchecked(u8 *buffer){
-    u32 ignore;
-    u32 result = utf8_to_u32_length_unchecked(buffer, &ignore);
-    return(result);
-}
-
-static u32
-utf8_to_u32(u8 **buffer_ptr, u8 *end){
-    u8 *buffer = *buffer_ptr;
-    u32 limit = (u32)(end - buffer);
-    
-    u32 length = 0;
-    if (buffer[0] < 0x80){
-        length = 1;
-    }
-    else if (buffer[0] < 0xC0){
-        length = 0;
-    }
-    else if (buffer[0] < 0xE0){
-        length = 2;
-    }
-    else if (buffer[0] < 0xF0){
-        length = 3;
-    }
-    else if (buffer[0] < 0xF8){
-        length = 4;
-    }
-    else{
-        length = 0;
-    }
-    
-    for (u32 i = 1; i < length; ++i){
-        if ((buffer[i] & 0xC0) != 0x80){
-            length = 0;
-            break;
-        }
-    }
-    
-    u32 result = 0;
-    if (length != 0 && length <= limit){
-        switch (length){
-            case 1:
-            {
-                result = (u32)buffer[0];
-            }break;
-            
-            case 2:
-            {
-                result = ((u32)((buffer[0])&0x1F)) << 6;
-                result |= ((u32)((buffer[1])&0x3F));
-            }break;
-            
-            case 3:
-            {
-                result = ((u32)((buffer[0])&0x0F)) << 12;
-                result |= ((u32)((buffer[1])&0x3F)) << 6;
-                result |= ((u32)((buffer[2])&0x3F));
-            }break;
-            
-            case 4:
-            {
-                result = ((u32)((buffer[0])&0x07)) << 18;
-                result |= ((u32)((buffer[1])&0x3F)) << 12;
-                result |= ((u32)((buffer[2])&0x3F)) << 6;
-                result |= ((u32)((buffer[3])&0x3F));
-            }break;
-        }
-        
-        if (result < cp_min_by_utf8_length[length] || (result >= surrogate_min && result <= surrogate_max)){
-            result = 0;
-            length = 0;
-        }
-        
-        *buffer_ptr = buffer + length;
-    }
-    else{
-        *buffer_ptr = end;
-    }
-    
-    return(result);
-}
-
-static void
-u32_to_utf8_unchecked(u32 codepoint, u8 *buffer, u32 *length_out){
-    if (codepoint <= 0x7F){
-        buffer[0] = (u8)codepoint;
-        *length_out = 1;
-    }
-    else if (codepoint <= 0x7FF){
-        buffer[0] = (u8)(0xC0 | (codepoint >> 6));
-        buffer[1] = (u8)(0x80 | (codepoint & 0x3F));
-        *length_out = 2;
-    }
-    else if (codepoint <= 0xFFFF){
-        buffer[0] = (u8)(0xE0 | (codepoint >> 12));
-        buffer[1] = (u8)(0x80 | ((codepoint >> 6) & 0x3F));
-        buffer[2] = (u8)(0x80 | (codepoint & 0x3F));
-        *length_out = 3;
-    }
-    else if (codepoint <= 0x10FFFF){
-        codepoint &= 0x001FFFFF;
-        buffer[0] = (u8)(0xF0 | (codepoint >> 18));
-        buffer[1] = (u8)(0x80 | ((codepoint >> 12) & 0x3F));
-        buffer[2] = (u8)(0x80 | ((codepoint >> 6) & 0x3F));
-        buffer[3] = (u8)(0x80 | (codepoint & 0x3F));
-        *length_out = 4;
-    }
-    else{
-        *length_out = 0;
-    }
-}
-
-static umem
-utf8_to_utf16_minimal_checking(u16 *dst, umem max_wchars, u8 *src, umem length, b32 *error){
-    u8 *s = src;
-    u8 *s_end = s + length;
-    
-    u16 *d = dst;
-    u16 *d_end = d + max_wchars;
-    umem limit = length;
-    
-    umem needed_max = 0;
-    u32 advance = 1;
-    
-    *error = false;
-    for(; s < s_end;){
-        u32 codepoint = 0;
-        u32 utf8_size = 0;
-        
-        if (s[0] < 0x80){
-            codepoint = (u32)s[0];
-            utf8_size = 1;
-        }
-        else if (s[0] < 0xE0){
-            if (limit <= 1){
-                *error = true;
-                break;
-            }
-            
-            codepoint = ((u32)((s[0])&0x1F)) << 6;
-            codepoint |= ((u32)((s[1])&0x3F));
-            utf8_size = 2;
-        }
-        else if (s[0] < 0xF0){
-            if (limit <= 2){
-                *error = true;
-                break;
-            }
-            
-            codepoint = ((u32)((s[0])&0x0F)) << 12;
-            codepoint |= ((u32)((s[1])&0x3F)) << 6;
-            codepoint |= ((u32)((s[2])&0x3F));
-            utf8_size = 3;
-        }
-        else if (s[0] < 0xF8){
-            if (limit > 3){
-                *error = true;
-                break;
-            }
-            
-            codepoint = ((u32)((s[0])&0x07)) << 18;
-            codepoint |= ((u32)((s[1])&0x3F)) << 12;
-            codepoint |= ((u32)((s[2])&0x3F)) << 6;
-            codepoint |= ((u32)((s[3])&0x3F));
-            utf8_size = 4;
-        }
-        else{
-            *error = true;
-            break;
-        }
-        
-        if (codepoint < cp_min_by_utf8_length[utf8_size]){
-            *error = true;
-            break;
-        }
-        
-        s += utf8_size;
-        limit -= utf8_size;
-        
-        if (codepoint <= 0xD7FF || (codepoint >= 0xE000 && codepoint <= 0xFFFF)){
-            *d = (u16)(codepoint);
-            d += advance;
-            needed_max += 1;
-        }
-        else if (codepoint >= 0x10000 && codepoint <= 0x10FFFF){
-            codepoint -= 0x10000;
-            
-            u32 high = (codepoint >> 10) & 0x03FF;
-            u32 low = (codepoint) & 0x03FF;
-            
-            high += 0xD800;
-            low += 0xDC00;
-            
-            if (d + advance < d_end){
-                *d = (u16)high;
-                d += advance;
-                *d = (u16)low;
-                d += advance;
-            }
-            else{
-                advance = 0;
-            }
-            
-            needed_max += 2;
-        }
-        else{
-            *error = true;
-            break;
-        }
-        
-        if (d >= d_end){
-            advance = 0;
-        }
-    }
-    
-    return(needed_max);
-}
-
-static umem
-utf16_to_utf8_minimal_checking(u8 *dst, umem max_chars, u16 *src, umem length, b32 *error){
-    u16 *s = src;
-    u16 *s_end = s + length;
-    
-    u8 *d = dst;
-    u8 *d_end = d + max_chars;
-    umem limit = length;
-    
-    umem needed_max = 0;
-    
-    *error = false;
-    
-    for (; s < s_end;){
-        u32 codepoint = 0;
-        u32 utf16_size = 0;
-        
-        if (s[0] <= 0xD7FF || (s[0] >= 0xE000 && s[0] <= 0xFFFF)){
-            codepoint = s[0];
-            utf16_size = 1;
-        }
-        else{
-            if (s[0] >= 0xD800 && s[0] <= 0xDBFF){
-                if (limit <= 1){
-                    *error = true;
-                    break;
-                }
-                
-                u32 high = s[0] - 0xD800;
-                u32 low = s[1] - 0xDC00;
-                codepoint = ((high << 10) | (low)) + 0x10000;
-                utf16_size = 2;
-            }
-            else{
-                *error = true;
-                break;
-            }
-        }
-        
-        s += utf16_size;
-        limit -= utf16_size;
-        
-        u8 d_fill[4];
-        u32 d_fill_count = 0;
-        
-        if (codepoint <= 0x7F){
-            d_fill[0] = (u8)codepoint;
-            d_fill_count = 1;
-        }
-        else if (codepoint <= 0x7FF){
-            d_fill[0] = (u8)(0xC0 | (codepoint >> 6));
-            d_fill[1] = (u8)(0x80 | (codepoint & 0x3F));
-            d_fill_count = 2;
-        }
-        else if (codepoint <= 0xFFFF){
-            d_fill[0] = (u8)(0xE0 | (codepoint >> 12));
-            d_fill[1] = (u8)(0x80 | ((codepoint >> 6) & 0x3F));
-            d_fill[2] = (u8)(0x80 | (codepoint & 0x3F));
-            d_fill_count = 3;
-        }
-        else if (codepoint <= 0x10FFFF){
-            d_fill[0] = (u8)(0xF0 | (codepoint >> 18));
-            d_fill[1] = (u8)(0x80 | ((codepoint >> 12) & 0x3F));
-            d_fill[2] = (u8)(0x80 | ((codepoint >> 6) & 0x3F));
-            d_fill[3] = (u8)(0x80 | (codepoint & 0x3F));
-            d_fill_count = 4;
-        }
-        else{
-            *error = true;
-            break;
-        }
-        
-        if (d + d_fill_count <= d_end){
-            for (u32 i = 0; i < d_fill_count; ++i){
-                *d = d_fill[i];
-                ++d;
-            }
-        }
-        needed_max += d_fill_count;
-    }
-    
-    return(needed_max);
-}
-
-static void
-byte_to_ascii(u8 n, u8 *out){
-    u8 C = '0' + (n / 0x10);
-    if ((n / 0x10) > 0x9){
-        C = ('A' - 0xA) + (n / 0x10);
-    }
-    out[0] = C;
-    
-    n = (n % 0x10);
-    C = '0' + n;
-    if (n > 0x9){
-        C = ('A' - 0xA) + n;
-    }
-    out[1] = C;
-}
-
-#endif
-
-// BOTTOM
-

4ed_app_target.cpp

@@ -30,8 +30,6 @@
 #include "4coder_hash_functions.cpp"
 #include "4coder_table.cpp"
 
-#include "4coder_lib/4coder_utf8.h"
-
 #include "4ed_render_target.h"
 #include "4ed.h"
 #include "4ed_buffer_model.h"

4ed_render_target.cpp

@@ -318,7 +318,19 @@ draw_string(Render_Target *target, Face *face, String_Const_u8 string, Vec2 poin
                     if (color != 0){
                         u8 cs[3];
                         cs[0] = '\\';
-                        byte_to_ascii(n, cs+1);
+                        u8 nh = (n >> 4);
+                        u8 nl = (n & 0xF);
+                        u8 ch = '0' + nh;
+                        u8 cl = '0' + nl;
+                        if (nh > 0x9){
+                            ch = ('A' - 0xA) + nh;
+                        }
+                        if (nl > 0x9){
+                            cl = ('A' - 0xA) + nl;
+                        }
+                        cs[1] = ch;
+                        cs[2] = cl;
+                        
                         Vec2 pp = point;
                         for (u32 j = 0; j < 3; ++j){
                             draw_font_glyph(target, face, cs[j], pp.x, pp.y, color, flags);

4ed_translation.cpp

@@ -105,23 +105,6 @@ translating_consume_byte(Translation_State *tran, u8 ch, u32 i, u32 size, Transl
     }
 }
 
-internal void
-translating_select_emit_rule_ASCII(Translation_State *tran, Translation_Byte_Description desc, Translation_Emit_Rule *type_out){
-    type_out->byte_class = desc.byte_class;
-    type_out->last_byte_handler = desc.last_byte_handler;
-    type_out->emit_type = desc.prelim_emit_type;
-    
-    type_out->codepoint = 0;
-    type_out->codepoint_length = 0;
-    if (desc.prelim_emit_type == BufferModelUnit_Codepoint){
-        u32 cp = utf8_to_u32_length_unchecked(tran->fill_buffer, &type_out->codepoint_length);
-        type_out->codepoint = cp;
-        if (!(cp == '\n' || cp == '\t' || cp == '\r' || (cp >= ' ' && cp <= 255 && cp != 127))){
-            type_out->emit_type = BufferModelUnit_Numbers;
-        }
-    }
-}
-
 internal void
 translating_select_emit_rule_UTF8(Translation_State *tran, Translation_Byte_Description desc, Translation_Emit_Rule *type_out){
     type_out->byte_class = desc.byte_class;
@@ -131,7 +114,12 @@ translating_select_emit_rule_UTF8(Translation_State *tran, Translation_Byte_Desc
     type_out->codepoint = 0;
     type_out->codepoint_length = 0;
     if (desc.prelim_emit_type == BufferModelUnit_Codepoint){
-        u32 cp = utf8_to_u32_length_unchecked(tran->fill_buffer, &type_out->codepoint_length);
+        Character_Consume_Result consume = utf8_consume(tran->fill_buffer, ArrayCount(tran->fill_buffer));
+        u32 cp = consume.codepoint;
+        type_out->codepoint_length = consume.inc;
+        if (cp == max_u32){
+            type_out->codepoint_length = 0;
+        }
         if (type_out->codepoint_length != 0){
             if ((cp >= nonchar_min && cp <= nonchar_max) || ((cp & 0xFFFF) >= 0xFFFE)){
                 type_out->emit_type = BufferModelUnit_Numbers;

manual/manual.man

@@ -1,69 +0,0 @@
-
-$ProductVersion{4.1.0}
-$ManualSubVersion{0}
-$File{4coder_manual}
-
-$Title{4coder 4.1 User's Manual}
-
-$Hd{1}{4coder 4.1. User's Manual}
-$Hd{3}[Allen Webster, 2018-09-07}
-
-$TableOfContents{ALL}
-
-$BeginContents{ALL}
-
-$PushSection{ALL}{}{$Hd{2}{First Time User}}
-
-$PushSection{ALL}{}{$Hd{3}{Organization}}
-4coder's most basic organization of information is by $Link{buffers}{org_buffers} and $Link{views}{views}.  Buffers, described
-simply, are an optimized array of characters encoded in UTF-8, that support reading and writing anywhere in the buffer and can
-be rendered to screen.  A view represents an region of the 4coder window with a self contained state for interactions from the
-user.  
-
-These major objects exist independently, they are created and closed independently.  The only relationship that ever exists
-between a view and a buffer is that a view can set a buffer as it's target for writing, reading, and rendering.  Multiple views
-can target a single buffer.  Any view can target any buffer at any time.  Closing a buffer in a view leaves the view open,
-closing a view that targets a buffer leaves the buffer open.  All of this is to emphasize that these two types of entities
-exist absolutely independently and have only a very weak relationship.
-$PopSection{ALL}
-
-$PushSection{ALL}{org_buffers}{$Hd{3}{Buffer Basics}}
-A buffer is essentially an optimized array of characters encoded in UTF-8, with a lot of extra features attached.  Many
-buffers will be tied directly to a corresponding file and will save their contents out to their file, but a few buffers exist
-just to be a holder for text.  In deafult 4coder behavior all buffers that lack a backing file have a name starting and ending
-with $Code{'*'} characters.
-
-A new buffer with a file association can be opened any time using one of the interactive commands for file system browsing:
-$BeginTable{FileBuf}{1}
-$Row{$Link{cmdref_interactive_open_or_new}{interactive_open_or_new}}
-$Row{$Link{cmdref_interactive_new}{interactive_new}}
-$Row{$Link{cmdref_interactive_open}{interactive_open}}
-$EndContents{FileBuf}
-
-Once you open a new buffer with a file association it will be the dedicated buffer for that file from then on, opening the
-file again will always just bring you back to the buffer you already opened, to create a totally different copy you have to
-manually make a fresh new buffer and then copy the contents over.
-
-To see a list of all open buffers and switch a view to targetting any buffer there is the interactive command
-$Link{cmdref_interactive_switch_buffer}{interactive_switch_buffer}.
-$PopSection{ALL}
-
-$PopSection{ALL}
-
-$PushSection{ALL}{}{$Hd{2}{Commands}}
-$ImportContents{commands.man}{ALL}
-$PopSection{ALL}
-
-$PushSection{ALL}{}{$Hd{2}{Bindings}}
-$ImportContents{bindings.man}{ALL}
-$PopSection{ALL}
-
-$PushSection{ALL}{}{$Hd{2}{Config Files and Project Files}}
-$PopSection{ALL}
-
-$PushSection{ALL}{}{$Hd{2}{Customization}}
-$PopSection{ALL}
-
-$EndContents{ALL}
-
-

platform_win32/win32_4ed.cpp

@@ -25,7 +25,6 @@
 #include "4coder_API/4coder_version.h"
 
 #include <string.h>
-#include "4coder_lib/4coder_utf8.h"
 
 #if defined(FRED_SUPER)
 # include "4coder_base_types.cpp"
@@ -170,8 +169,7 @@ struct Win32_Vars{
     
     String_Const_u8 binary_path;
     
-    u8 *clip_buffer;
+    Arena *clipboard_arena;
-    u32 clip_max;
     String_Const_u8 clipboard_contents;
     b32 next_clipboard_is_self;
     DWORD clipboard_sequence;
@@ -388,57 +386,34 @@ win32_read_clipboard_contents(Arena *scratch){
     if (can_read){
         if (OpenClipboard(win32vars.window_handle)){
             result = true;
-            HANDLE clip_data = 0;
+            String_u8 contents = {};
-            i32 contents_length = 0;
+            Arena *clip_arena = win32vars.clipboard_arena;
             if (has_unicode){
-                clip_data = GetClipboardData(CF_UNICODETEXT);
+                HANDLE clip_data = GetClipboardData(CF_UNICODETEXT);
                 if (clip_data != 0){
-                    u16 *clip_16 = (u16*)GlobalLock(clip_data);
+                    u16 *clip_16_ptr = (u16*)GlobalLock(clip_data);
-                    if (clip_16 != 0){
+                    if (clip_16_ptr != 0){
-                        u32 clip_16_len = 0;
+                        linalloc_clear(clip_arena);
-                        for(;clip_16[clip_16_len];++clip_16_len);
+                        String_Const_u16 clip_16 = SCu16(clip_16_ptr);
-                        
+                        contents = string_u8_from_string_u16(clip_arena, clip_16, StringFill_NullTerminate);
-                        b32 error = false;
-                        u32 clip_8_len = (u32)utf16_to_utf8_minimal_checking(win32vars.clip_buffer, win32vars.clip_max-1, clip_16, clip_16_len, &error);
-                        
-                        for (;clip_8_len >= win32vars.clip_max && !error;){
-                            system_memory_free(win32vars.clip_buffer, win32vars.clip_max);
-                            win32vars.clip_max = round_up_u32(clip_8_len + 1, KB(4));
-                            win32vars.clip_buffer = (u8*)system_memory_allocate(win32vars.clip_max);
-                            clip_8_len = (u32)utf16_to_utf8_minimal_checking(win32vars.clip_buffer, win32vars.clip_max - 1, clip_16, clip_16_len, &error);
-                        }
-                        
-                        if (clip_8_len < win32vars.clip_max && !error){
-                            win32vars.clip_buffer[clip_8_len] = 0;
-                            contents_length = clip_8_len + 1;
-                        }
                     }
                 }
+                GlobalUnlock(clip_data);
             }
             else{
-                clip_data = GetClipboardData(CF_TEXT);
+                HANDLE clip_data = GetClipboardData(CF_TEXT);
                 if (clip_data != 0){
-                    char *clip_ascii = (char*)GlobalLock(clip_data);
+                    char *clip_ascii_ptr = (char*)GlobalLock(clip_data);
-                    if (clip_ascii != 0){
+                    if (clip_ascii_ptr != 0){
-                        u32 clip_ascii_len = 0;
+                        linalloc_clear(clip_arena);
-                        for(;clip_ascii[clip_ascii_len];++clip_ascii_len);
+                        String_Const_char clip_ascii = SCchar(clip_ascii_ptr);
-                        
+                        contents = string_u8_from_string_char(clip_arena, clip_ascii, StringFill_NullTerminate);
-                        if (clip_ascii_len >= win32vars.clip_max){
-                            system_memory_free(win32vars.clip_buffer, win32vars.clip_max);
-                            win32vars.clip_max = round_up_u32(clip_ascii_len + 1, KB(4));
-                            win32vars.clip_buffer = (u8*)system_memory_allocate(win32vars.clip_max);
-                        }
-                        memcpy(win32vars.clip_buffer, clip_ascii, clip_ascii_len + 1);
-                        contents_length = clip_ascii_len + 1;
                     }
                 }
+                GlobalUnlock(clip_data);
             }
             
-            if (contents_length > 0){
+            win32vars.clipboard_contents = contents.string;
-                win32vars.clipboard_contents = SCu8(win32vars.clip_buffer, contents_length - 1);
-            }
-            
-            GlobalUnlock(clip_data);
             
             CloseClipboard();
         }
@@ -1634,8 +1609,7 @@ WinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPSTR lpCmdLine, int nCmdS
         win32_output_error_string(scratch, ErrorString_UseLog);
     }
     
-    win32vars.clip_max = KB(16);
+    win32vars.clipboard_arena = reserve_arena(win32vars.tctx);
-    win32vars.clip_buffer = (u8*)system_memory_allocate(win32vars.clip_max);
     
     win32vars.clipboard_sequence = GetClipboardSequenceNumber();
     if (win32vars.clipboard_sequence == 0){

platform_win32/win32_4ed_functions.cpp

@@ -378,17 +378,9 @@ color_picker_hook(HWND Window, UINT Message, WPARAM WParam, LPARAM LParam){
             Color_Picker *picker = (Color_Picker*)win32_params->lCustData;
             SetWindowLongPtr(Window, GWLP_USERDATA, (LONG_PTR)LParam);
             
-            u16 Temp[256];
+            Scratch_Block scratch(win32vars.tctx);
-            Temp[ArrayCount(Temp) - 1] = 0;
+            String_u16 temp = string_u16_from_string_u8(scratch, picker->title, StringFill_NullTerminate);
-            
+            SetWindowTextW(Window, (LPCWSTR)temp.str);
-            b32 ignored;
-            utf8_to_utf16_minimal_checking(Temp, ArrayCount(Temp), (u8 *)picker->title.str, picker->title.size, &ignored);
-            if(picker->title.size < ArrayCount(Temp))
-            {
-                Temp[picker->title.size] = 0;
-            }
-            
-            SetWindowTextW(Window, (LPCWSTR)Temp);
         } break;
         
         case WM_CTLCOLORSTATIC:

platform_win32/win32_utf8.cpp

@@ -12,50 +12,12 @@
 #if !defined(FRED_WIN32_UTF8_CPP)
 #define FRED_WIN32_UTF8_CPP
 
-// TODO(allen): rewrite _EVERYTHING_ that does Win32 UTF16 <-> UTF8
-internal Win32_UTF16
-input_8_to_16(Arena *scratch, u8 *in, u32 in_length){
-    Win32_UTF16 r = {};
-    u32 utf16_max = (in_length + 1)*2;
-    u16 *utf16 = push_array(scratch, u16, utf16_max);
-    b32 error = false;
-    u32 utf16_len = (u32)utf8_to_utf16_minimal_checking(utf16, utf16_max - 1, in, in_length, &error);
-    if (!error && utf16_len < utf16_max){
-        utf16[utf16_len] = 0;
-        r.success = true;
-        r.utf8_len = in_length;
-        r.utf16_max = utf16_max;
-        r.utf16_len = utf16_len;
-        r.utf16 = utf16;
-    }
-    return(r);
-}
-
-internal Win32_UTF16
-input_8_to_16(Arena *scratch, u8 *in){
-    u32 length = 0;
-    for (;in[length];++length);
-    return(input_8_to_16(scratch, in, length));
-}
-
-internal Win32_UTF16
-input_8_to_16(Arena *scratch, String_Const_u8 in){
-    return(input_8_to_16(scratch, in.str, (u32)in.size));
-}
-
 internal HANDLE
 CreateFile_utf8(Arena *scratch, u8 *name, DWORD access, DWORD share, LPSECURITY_ATTRIBUTES security, DWORD creation, DWORD flags, HANDLE template_file){
-    HANDLE result = INVALID_HANDLE_VALUE;
-    
     Temp_Memory temp = begin_temp(scratch);
-    
+    String_u16 name_16 = string_u16_from_string_u8(scratch, SCu8(name), StringFill_NullTerminate);
-    Win32_UTF16 name_16 = input_8_to_16(scratch, name);
+    HANDLE result = CreateFileW((LPWSTR)name_16.str, access, share, security, creation, flags, template_file);
-    if (name_16.success){
-        result = CreateFileW((LPWSTR)name_16.utf16, access, share, security, creation, flags, template_file);
-    }
-    
     end_temp(temp);
-    
     return(result);
 }
 
@@ -77,10 +39,14 @@ GetFinalPathNameByHandle_utf8(Arena *scratch, HANDLE file, u8 *file_path_out, DW
         
         if (length_16 != 0 && length_16 < path_16_max){
             b32 convert_error = false;
-            u32 path_8_len = (u32)utf16_to_utf8_minimal_checking(file_path_out, path_max-1, path_16, length_16, &convert_error);
+            String_Const_u16 path_16_str = SCu16(path_16, length_16);
-            if (path_8_len < path_max && !convert_error){
+            String_u8 path_8 = string_u8_from_string_u16(scratch, path_16_str, StringFill_NullTerminate);
-                file_path_out[path_8_len] = 0;
+            if (path_8.size + 1 <= path_max && !convert_error){
-                result = path_8_len;
+                block_copy(file_path_out, path_8.str, path_8.size + 1);
+                result = (DWORD)path_8.size;
+            }
+            else{
+                result = (DWORD)path_8.size + 1;
             }
         }
         
@@ -92,48 +58,33 @@ GetFinalPathNameByHandle_utf8(Arena *scratch, HANDLE file, u8 *file_path_out, DW
 
 internal HANDLE
 FindFirstFile_utf8(Arena *scratch, u8 *name, LPWIN32_FIND_DATA find_data){
-    HANDLE result = INVALID_HANDLE_VALUE;
-    
     Temp_Memory temp = begin_temp(scratch);
-    
+    String_u16 name_16 = string_u16_from_string_u8(scratch, SCu8(name), StringFill_NullTerminate);
-    Win32_UTF16 name_16 = input_8_to_16(scratch, name);
+    HANDLE result = FindFirstFileW((LPWSTR)name_16.str, find_data);
-    if (name_16.success){
-        result = FindFirstFileW((LPWSTR)name_16.utf16, find_data);
-    }
-    
     end_temp(temp);
-    
     return(result);
 }
 
 internal DWORD
 GetFileAttributes_utf8(Arena *scratch, u8 *name){
-    DWORD result = 0;
-    
     Temp_Memory temp = begin_temp(scratch);
-    
+    String_u16 name_16 = string_u16_from_string_u8(scratch, SCu8(name), StringFill_NullTerminate);
-    Win32_UTF16 name_16 = input_8_to_16(scratch, name);
+    DWORD result = GetFileAttributesW((LPWSTR)name_16.str);
-    if (name_16.success){
-        result = GetFileAttributesW((LPWSTR)name_16.utf16);
-    }
-    
     end_temp(temp);
-    
     return(result);
 }
 
 internal DWORD
 GetModuleFileName_utf8(Arena *scratch, HMODULE module, u8 *file_out, DWORD max){
-    DWORD result = 0;
     Temp_Memory temp = begin_temp(scratch);
     u32 file_16_max = KB(40);
     u16 *file_16 = push_array(scratch, u16, file_16_max);
     DWORD file_16_len = GetModuleFileNameW(module, (LPWSTR)file_16, file_16_max);
-    b32 convert_error = false;
+    String_u8 file_8 = string_u8_from_string_u16(scratch, SCu16(file_16, file_16_len), StringFill_NullTerminate);
-    u32 file_8_len = (u32)utf16_to_utf8_minimal_checking(file_out, max - 1, file_16, file_16_len, &convert_error);
+    DWORD result = 0;
-    result = file_8_len;
+    if (file_8.size + 1 <= max){
-    if (convert_error || file_8_len >= max){
+        block_copy(file_out, file_8.str, file_8.size + 1);
-        result = 0;
+        result = (DWORD)file_8.size;
     }
     end_temp(temp);
     return(result);
@@ -141,24 +92,12 @@ GetModuleFileName_utf8(Arena *scratch, HMODULE module, u8 *file_out, DWORD max){
 
 internal BOOL
 CreateProcess_utf8(Arena *scratch, u8 *app_name, u8 *command, LPSECURITY_ATTRIBUTES security, LPSECURITY_ATTRIBUTES thread, BOOL inherit_handles, DWORD creation, LPVOID environment, u8 *curdir, LPSTARTUPINFO startup, LPPROCESS_INFORMATION process){
-    BOOL result = false;
-    
     Temp_Memory temp = begin_temp(scratch);
-    
+    String_u16 app_name_16 = string_u16_from_string_u8(scratch, SCu8(app_name), StringFill_NullTerminate);
-    Win32_UTF16 app_name_16 = input_8_to_16(scratch, app_name);
+    String_u16 command_16 = string_u16_from_string_u8(scratch, SCu8(command), StringFill_NullTerminate);
-    
+    String_u16 curdir_16 = string_u16_from_string_u8(scratch, SCu8(curdir), StringFill_NullTerminate);
-    if (app_name_16.success){
+    BOOL result = CreateProcessW((LPWSTR)app_name_16.str, (LPWSTR)command_16.str, security, thread, inherit_handles, creation, environment, (LPWSTR)curdir_16.str, startup, process);
-        Win32_UTF16 command_16 = input_8_to_16(scratch, command);
-        if (command_16.success){
-            Win32_UTF16 curdir_16 = input_8_to_16(scratch, curdir);
-            if (curdir_16.success){
-                result = CreateProcessW((LPWSTR)app_name_16.utf16, (LPWSTR)command_16.utf16, security, thread, inherit_handles, creation, environment, (LPWSTR)curdir_16.utf16, startup, process);
-            }
-        }
-    }
-    
     end_temp(temp);
-    
     return(result);
 }
 
@@ -171,16 +110,19 @@ GetCurrentDirectory_utf8(Arena *scratch, DWORD max, u8 *buffer){
         u32 buffer_16_max = KB(40);
         u16 *buffer_16 = push_array(scratch, u16, buffer_16_max);
         DWORD buffer_16_len = GetCurrentDirectoryW(buffer_16_max, (LPWSTR)buffer_16);
-        b32 error = false;
+        String_u8 curdir_8 = string_u8_from_string_u16(scratch, SCu16(buffer_16, buffer_16_len), StringFill_NullTerminate);
-        u32 buffer_8_len = (u32)utf16_to_utf8_minimal_checking(buffer, max-1, buffer_16, buffer_16_len, &error);
+        if (curdir_8.size + 1 <= max){
-        if (buffer_8_len < max && !error){
+            block_copy(buffer, curdir_8.str, curdir_8.size + 1);
-            buffer[buffer_8_len] = 0;
+            result = (DWORD)curdir_8.size;
-            result = buffer_8_len;
+        }
+        else{
+            result = (DWORD)curdir_8.size + 1;
         }
         end_temp(temp);
     }
     else{
         result = GetCurrentDirectoryW(0, 0);
+        result *= 2;
     }
     
     return(result);
@@ -188,39 +130,28 @@ GetCurrentDirectory_utf8(Arena *scratch, DWORD max, u8 *buffer){
 
 internal int
 MessageBox_utf8(Arena *scratch, HWND owner, u8 *text, u8 *caption, UINT type){
-    int result = 0;
     Temp_Memory temp = begin_temp(scratch);
-    Win32_UTF16 text_16 = input_8_to_16(scratch, text);
+    String_u16 text_16 = string_u16_from_string_u8(scratch, SCu8(text), StringFill_NullTerminate);
-    if (text_16.success){
+    String_u16 caption_16 = string_u16_from_string_u8(scratch, SCu8(caption), StringFill_NullTerminate);
-        Win32_UTF16 caption_16 = input_8_to_16(scratch, caption);
+    int result = MessageBoxW(owner, (LPWSTR)text_16.str, (LPWSTR)caption_16.str, type);
-        if (caption_16.success){
-            result = MessageBoxW(owner, (LPWSTR)text_16.utf16, (LPWSTR)caption_16.utf16, type);
-        }
-    }
     end_temp(temp);
     return(result);
 }
 
 internal BOOL
 SetWindowText_utf8(Arena *scratch, HWND window, u8 *string){
-    BOOL result = FALSE;
     Temp_Memory temp = begin_temp(scratch);
-    Win32_UTF16 string_16 = input_8_to_16(scratch, string);
+    String_u16 string_16 = string_u16_from_string_u8(scratch, SCu8(string), StringFill_NullTerminate);
-    if (string_16.success){
+    BOOL result = SetWindowTextW(window, (LPWSTR)string_16.str);
-        result = SetWindowTextW(window, (LPWSTR)string_16.utf16);
-    }
     end_temp(temp);
     return(result);
 }
 
 internal BOOL
 GetFileAttributesEx_utf8String(Arena *scratch, String_Const_u8 file_name, GET_FILEEX_INFO_LEVELS info_level_id, LPVOID file_info){
-    BOOL result = FALSE;
     Temp_Memory temp = begin_temp(scratch);
-    Win32_UTF16 string_16 = input_8_to_16(scratch, file_name);
+    String_u16 string_16 = string_u16_from_string_u8(scratch, file_name, StringFill_NullTerminate);
-    if (string_16.success){
+    BOOL result = GetFileAttributesExW((LPWSTR)string_16.str, info_level_id, file_info);
-        result = GetFileAttributesExW((LPWSTR)string_16.utf16, info_level_id, file_info);
-    }
     end_temp(temp);
     return(result);
 }

platform_win32/win32_utf8.h

@@ -42,16 +42,6 @@ SetWindowText_utf8(Arena *scratch, HWND window, u8 *string);
 internal BOOL
 GetFileAttributesEx_utf8String(Arena *scratch, String_Const_u8 file_name, GET_FILEEX_INFO_LEVELS info_level_id, LPVOID file_info);
 
-// For implementation
-
-struct Win32_UTF16{
-    b32 success;
-    u32 utf8_len;
-    u32 utf16_max;
-    u32 utf16_len;
-    u16 *utf16;
-};
-
 #endif
 
 // BOTTOM

test_input_scripts/generated/.gitignore

@@ -1,2 +0,0 @@
-*
-!.gitignore

test_input_scripts/test_load_FONT_COURIER_NEW_28_c.4is

@@ -1,6 +0,0 @@
-
-basewait 1
-key o MDFR_CTRL
-type 1 FONT_COURIER_NEW_28
-key key_newline MDFR_NONE
-exit

test_input_scripts/test_load_rome_txt.4is

@@ -1,15 +0,0 @@
-
-basewait 1
-key P MDFR_CTRL
-key o MDFR_CTRL
-type 1 rome.txt
-key key_newline MDFR_NONE
-key key_page_down MDFR_CTRL
-key key_newline MDFR_NONE
-key key_newline MDFR_NONE
-wait 20
-type 0 Hello World!
-wait 2
-exit
-key y MDFR_NONE
-