core/unicode/utf8
utf8
Types
4Accept_Range
Accept_Range :: struct {
lo: u8,
hi: u8,
}SourceGrapheme
Grapheme :: struct {
byte_index: int,
rune_index: int,
width: int,
}SourceGrapheme_Cluster_Sequence
Grapheme_Cluster_Sequence :: enum int {
None = 0,
Indic = 1,
Emoji = 2,
Regional = 3,
}SourceGrapheme_Iterator
Grapheme_Iterator :: struct {
str: string,
curr_offset: int,
grapheme_count: int,
rune_count: int,
width: int,
last_rune: rune,
last_rune_breaks_forward: bool,
last_width: int,
last_grapheme_count: int,
bypass_next_rune: bool,
regional_indicator_counter: int,
current_sequence: Grapheme_Cluster_Sequence,
continue_sequence: bool,
}SourceConstants
26HICB
HICB :: 0b1011_1111SourceLOCB
LOCB :: 0b1000_0000SourceThe default lowest and highest continuation byte.
MASK2
MASK2 :: 0b0001_1111SourceMASK3
MASK3 :: 0b0000_1111SourceMASK4
MASK4 :: 0b0000_0111SourceMASKX
MASKX :: 0b0011_1111SourceMAX_RUNE
MAX_RUNE :: '\U0010ffff'SourceRUNE1_MAX
RUNE1_MAX :: 1<<7 - 1SourceRUNE2_MAX
RUNE2_MAX :: 1<<11 - 1SourceRUNE3_MAX
RUNE3_MAX :: 1<<16 - 1SourceRUNE_BOM
RUNE_BOM :: 0xfeffSourceRUNE_EOF
RUNE_EOF :: _ = ~rune(0)SourceRUNE_ERROR
RUNE_ERROR :: '\ufffd'SourceRUNE_SELF
RUNE_SELF :: 0x80SourceSURROGATE_HIGH_MAX
SURROGATE_HIGH_MAX :: 0xdbffSourceA high/leading surrogate is in range SURROGATE_MIN..SURROGATE_HIGH_MAX, A low/trailing surrogate is in range SURROGATE_LOW_MIN..SURROGATE_MAX.
SURROGATE_LOW_MIN
SURROGATE_LOW_MIN :: 0xdc00SourceSURROGATE_MAX
SURROGATE_MAX :: 0xdfffSourceSURROGATE_MIN
SURROGATE_MIN :: 0xd800SourceT1
T1 :: 0b0000_0000SourceT2
T2 :: 0b1100_0000SourceT3
T3 :: 0b1110_0000SourceT4
T4 :: 0b1111_0000SourceT5
T5 :: 0b1111_1000SourceTX
TX :: 0b1000_0000SourceUTF_MAX
UTF_MAX :: 4SourceZERO_WIDTH_JOINER
ZERO_WIDTH_JOINER :: ZERO_WIDTH_JOINERSourceVariables
2accept_ranges
accept_ranges :: [5]Accept_Range = [5]Accept_Range{
{0x80, 0xbf},
{0xa0, 0xbf},
{0x80, 0x9f},
{0x90, 0xbf},
{0x80, 0x8f},
}Sourceaccept_sizes
accept_sizes :: [256]u8 = [256]u8{
0x00..=0x7f = 0xf0, // ascii, size 1
0x80..=0xc1 = 0xf1, // invalid, size 1
0xc2..=0xdf = 0x02, // accept 1, size 2
0xe0 = 0x13, // accept 1, size 3
0xe1..=0xec = 0x03, // accept 0, size 3
0xed = 0x23, // accept 2, size 3
SourceProcedures
38decode_grapheme_clusters
decode_grapheme_clusters :: proc(str: string, track_graphemes: untyped boolean = true, allocator: mem.Allocator = context.allocator) -> (graphemes: [dynamic]Grapheme, grapheme_count: int, rune_count: int, width: int)SourceDecode the individual graphemes in a UTF-8 string.
Allocates Using Provided Allocator
Inputs:
- str: The input string.
- track_graphemes: Whether or not to allocate and return
graphemeswith extra data about each grapheme. - allocator: (default: context.allocator)
Returns:
- graphemes: Extra data about each grapheme.
- grapheme_count: The number of graphemes in the string.
- rune_count: The number of runes in the string.
- width: The width of the string in number of monospace cells.
decode_grapheme_iterate
decode_grapheme_iterate :: proc(it: ^Grapheme_Iterator) -> (text: string, grapheme: Grapheme, ok: bool)Sourcedecode_grapheme_iterator_make
decode_grapheme_iterator_make :: proc(str: string) -> (it: Grapheme_Iterator)Sourcedecode_last_rune_in_bytes
decode_last_rune_in_bytes :: proc(s: []u8) -> (int, rune)Sourcedecode_last_rune_in_string
decode_last_rune_in_string :: proc(s: string) -> (int, rune)Sourcedecode_rune_in_bytes
decode_rune_in_bytes :: proc(s: []u8) -> (int, rune)Sourcedecode_rune_in_string
decode_rune_in_string :: proc(s: string) -> (int, rune)Sourceencode_rune
encode_rune :: proc(c: rune) -> ([4]u8, int)Sourcefull_rune_in_bytes
full_rune_in_bytes :: proc(b: []u8) -> (bool)Sourcefull_rune_in_bytes reports if the bytes in b begin with a full utf-8 encoding of a rune or not An invalid encoding is considered a full rune since it will convert as an error rune of width 1 (RUNE_ERROR)
full_rune_in_string
full_rune_in_string :: proc(s: string) -> (bool)Sourcefull_rune_in_string reports if the bytes in s begin with a full utf-8 encoding of a rune or not An invalid encoding is considered a full rune since it will convert as an error rune of width 1 (RUNE_ERROR)
grapheme_count
grapheme_count :: proc(str: string) -> (graphemes: int, runes: int, width: int)SourceCount the individual graphemes in a UTF-8 string.
Inputs:
- str: The input string.
Returns:
- graphemes: The number of graphemes in the string.
- runes: The number of runes in the string.
- width: The width of the string in number of monospace cells.
is_control
is_control :: proc(r: rune) -> (bool)Sourceis_emoji_extended_pictographic
is_emoji_extended_pictographic :: proc(r: rune) -> (bool)Sourceis_gcb_extend_class
is_gcb_extend_class :: proc(r: rune) -> (bool)Sourceis_gcb_prepend_class
is_gcb_prepend_class :: proc(r: rune) -> (bool)Sourceis_hangul_syllable_leading
is_hangul_syllable_leading :: proc(r: rune) -> (bool)Sourceis_hangul_syllable_lv
is_hangul_syllable_lv :: proc(r: rune) -> (bool)Sourceis_hangul_syllable_lvt
is_hangul_syllable_lvt :: proc(r: rune) -> (bool)Sourceis_hangul_syllable_trailing
is_hangul_syllable_trailing :: proc(r: rune) -> (bool)Sourceis_hangul_syllable_vowel
is_hangul_syllable_vowel :: proc(r: rune) -> (bool)Sourceis_indic_conjunct_break_consonant
is_indic_conjunct_break_consonant :: proc(r: rune) -> (bool)Sourceis_indic_conjunct_break_extend
is_indic_conjunct_break_extend :: proc(r: rune) -> (bool)Sourceis_indic_conjunct_break_linker
is_indic_conjunct_break_linker :: proc(r: rune) -> (bool)Sourceis_regional_indicator
is_regional_indicator :: proc(r: rune) -> (bool)Sourceis_spacing_mark
is_spacing_mark :: proc(r: rune) -> (bool)Sourcenormalized_east_asian_width
normalized_east_asian_width :: proc(r: rune) -> (int)Sourcerune_at
rune_at :: proc(s: string, byte_index: int) -> (rune)Sourcerune_at_pos
rune_at_pos :: proc(s: string, pos: int) -> (rune)Sourcerune_count_in_bytes
rune_count_in_bytes :: proc(s: []u8) -> (int)Sourcerune_count_in_string
rune_count_in_string :: proc(s: string) -> (int)Sourcerune_offset
rune_offset :: proc(s: string, pos: int, start: int) -> (int)SourceReturns the byte position of rune at position pos in s with an optional start byte position. Returns -1 if it runs out of the string.
rune_size
rune_size :: proc(r: rune) -> (int)Sourcerune_start
rune_start :: proc(b: u8) -> (bool)Sourcerune_string_at_pos
rune_string_at_pos :: proc(s: string, pos: int) -> (string)Sourcerunes_to_string
runes_to_string :: proc(runes: []rune, allocator: mem.Allocator = context.allocator) -> (s: string, err: runtime.Allocator_Error)Sourcestring_to_runes
string_to_runes :: proc(s: string, allocator: mem.Allocator = context.allocator) -> (runes: []rune, err: runtime.Allocator_Error)Sourcevalid_rune
valid_rune :: proc(r: rune) -> (bool)Sourcevalid_string
valid_string :: proc(s: string) -> (bool)SourceProcedure Groups
4decode_last_rune
decode_last_rune :: proc{decode_last_rune_in_string, decode_last_rune_in_bytes}Sourcedecode_rune
decode_rune :: proc{decode_rune_in_string, decode_rune_in_bytes}Sourcefull_rune
full_rune :: proc{full_rune_in_bytes, full_rune_in_string}Sourcefull_rune reports if the bytes in b begin with a full utf-8 encoding of a rune or not An invalid encoding is considered a full rune since it will convert as an error rune of width 1 (RUNE_ERROR)
rune_count
rune_count :: proc{rune_count_in_string, rune_count_in_bytes}Source