core/unicode/tools/ucd
ucd
Types
25Age
Age :: enum u8 {
Age_Unknown = 0,
Age_1_1 = 1,
Age_2_0 = 2,
Age_2_1 = 3,
Age_3_0 = 4,
Age_3_1 = 5,
Age_3_2 = 6,
Age_4_0 = 7,
Age_4_1 = 8,
Age_5_0 = 9,
Age_5_1 = 10,
Age_5_2 = 11,
Age_6_0 = 12,
Age_6_1 = 13,
Age_6_2 = 14,
Age_6_3 = 15,
Age_7_0 = 16,
Age_8_0 = 17,
Age_9_0 = 18,
Age_10_0 = 19,
Age_11_0 = 20,
Age_12_0 = 21,
Age_12_1 = 22,
Age_13_0 = 23,
Age_14_0 = 24,
Age_15_0 = 25,
Age_15_1 = 26,
Age_16_0 = 27,
Age_17_0 = 28,
Age_Unassigned = 29,
}SourceBidi
Bidi :: struct {
bc: Bidi_Class,
bmg: Maybe(rune),
m: bool,
c: bool,
pb: Paired_Bracket_Type,
bpb: rune,
}SourceBidi_Class
Bidi_Class :: enum int {
Unknown = 0,
L = 1, // Left-to-Right LRM
R = 2, // Right-to-Left RLM
AL = 3, // Right-to-Left Arabic ALM
EN = 4, // European Number
ES = 5, // European Number Separator
ET = 6, // European Number Terminator
AN = 7, // Arabic Number
CS = 8, // Common Number Separator
NSM = 9, // Nonspacing Mark
BN = 10, // Boundary Neutral
B = 11, // Paragraph Separator
S = 12, // Segment Separator
WS = 13, // Whitespace
ON = 14, // Other Neutrals
LRE = 15, // Left-to-Right Embedding LRE
LRO = 16, // Left-to-Right Override LRO
RLE = 17, // Right-to-Left Embedding RLE
RLO = 18, // Right-to-Left Override RLO
PDF = 19, // Pop Directional Format PDF
LRI = 20, // Left-to-Right Isolate LRI
RLI = 21, // Right-to-Left Isolate RLI
FSI = 22, // First Strong Isolate FSI
PDI = 23, // Pop Directional Isolate PDI
}SourceBlock
Block :: enum int {
Nil = 0,
Adlam = 1,
Aegean_Numbers = 2,
Ahom = 3,
Alchemical = 4,
Alphabetic_PF = 5,
Anatolian_Hieroglyphs = 6,
Ancient_Greek_Music = 7,
Ancient_Greek_Numbers = 8,
Ancient_Symbols = 9,
Arabic = 10,
Arabic_Ext_A = 11,
Arabic_Ext_B = 12,
Arabic_Ext_C = 13,
Arabic_Math = 14,
Arabic_PF_A = 15,
Arabic_PF_B = 16,
Arabic_Sup = 17,
Armenian = 18,
Arrows = 19,
ASCII = 20,
Avestan = 21,
Balinese = 22,
Bamum = 23,
Bamum_Sup = 24,
Bassa_Vah = 25,
Batak = 26,
Bengali = 27,
Beria_Erfe = 28,
Bhaiksuki = 29,
Block_Elements = 30,
Bopomofo = 31,
Bopomofo_Ext = 32,
Box_Drawing = 33,
Brahmi = 34,
Braille = 35,
Buginese = 36,
Buhid = 37,
Byzantine_Music = 38,
Carian = 39,
Caucasian_Albanian = 40,
Chakma = 41,
Cham = 42,
Cherokee = 43,
Cherokee_Sup = 44,
Chess_Symbols = 45,
Chorasmian = 46,
CJK = 47,
CJK_Compat = 48,
CJK_Compat_Forms = 49,
CJK_Compat_Ideographs = 50,
CJK_Compat_Ideographs_Sup = 51,
CJK_Ext_A = 52,
CJK_Ext_B = 53,
CJK_Ext_C = 54,
CJK_Ext_D = 55,
CJK_Ext_E = 56,
CJK_Ext_F = 57,
CJK_Ext_G = 58,
CJK_Ext_H = 59,
CJK_Ext_I = 60,
CJK_Ext_J = 61,
CJK_Radicals_Sup = 62,
CJK_Strokes = 63,
CJK_Symbols = 64,
Compat_Jamo = 65,
Control_Pictures = 66,
Coptic = 67,
Coptic_Epact_Numbers = 68,
Counting_Rod = 69,
Cuneiform = 70,
Cuneiform_Numbers = 71,
Currency_Symbols = 72,
Cypriot_Syllabary = 73,
Cypro_Minoan = 74,
Cyrillic = 75,
Cyrillic_Ext_A = 76,
Cyrillic_Ext_B = 77,
Cyrillic_Ext_C = 78,
Cyrillic_Ext_D = 79,
Cyrillic_Sup = 80,
Deseret = 81,
Devanagari = 82,
Devanagari_Ext = 83,
Devanagari_Ext_A = 84,
Diacriticals = 85,
Diacriticals_Ext = 86,
Diacriticals_For_Symbols = 87,
Diacriticals_Sup = 88,
Dingbats = 89,
Dives_Akuru = 90,
Dogra = 91,
Domino = 92,
Duployan = 93,
Early_Dynastic_Cuneiform = 94,
Egyptian_Hieroglyph_Format_Controls = 95,
Egyptian_Hieroglyphs = 96,
Egyptian_Hieroglyphs_Ext_A = 97,
Elbasan = 98,
Elymaic = 99,
Emoticons = 100,
Enclosed_Alphanum = 101,
Enclosed_Alphanum_Sup = 102,
Enclosed_CJK = 103,
Enclosed_Ideographic_Sup = 104,
Ethiopic = 105,
Ethiopic_Ext = 106,
Ethiopic_Ext_A = 107,
Ethiopic_Ext_B = 108,
Ethiopic_Sup = 109,
Garay = 110,
Geometric_Shapes = 111,
Geometric_Shapes_Ext = 112,
Georgian = 113,
Georgian_Ext = 114,
Georgian_Sup = 115,
Glagolitic = 116,
Glagolitic_Sup = 117,
Gothic = 118,
Grantha = 119,
Greek = 120,
Greek_Ext = 121,
Gujarati = 122,
Gunjala_Gondi = 123,
Gurmukhi = 124,
Gurung_Khema = 125,
Half_And_Full_Forms = 126,
Half_Marks = 127,
Hangul = 128,
Hanifi_Rohingya = 129,
Hanunoo = 130,
Hatran = 131,
Hebrew = 132,
High_PU_Surrogates = 133,
High_Surrogates = 134,
Hiragana = 135,
IDC = 136,
Ideographic_Symbols = 137,
Imperial_Aramaic = 138,
Indic_Number_Forms = 139,
Indic_Siyaq_Numbers = 140,
Inscriptional_Pahlavi = 141,
Inscriptional_Parthian = 142,
IPA_Ext = 143,
Jamo = 144,
Jamo_Ext_A = 145,
Jamo_Ext_B = 146,
Javanese = 147,
Kaithi = 148,
Kaktovik_Numerals = 149,
Kana_Ext_A = 150,
Kana_Ext_B = 151,
Kana_Sup = 152,
Kanbun = 153,
Kangxi = 154,
Kannada = 155,
Katakana = 156,
Katakana_Ext = 157,
Kawi = 158,
Kayah_Li = 159,
Kharoshthi = 160,
Khitan_Small_Script = 161,
Khmer = 162,
Khmer_Symbols = 163,
Khojki = 164,
Khudawadi = 165,
Kirat_Rai = 166,
Lao = 167,
Latin_1_Sup = 168,
Latin_Ext_A = 169,
Latin_Ext_Additional = 170,
Latin_Ext_B = 171,
Latin_Ext_C = 172,
Latin_Ext_D = 173,
Latin_Ext_E = 174,
Latin_Ext_F = 175,
Latin_Ext_G = 176,
Lepcha = 177,
Letterlike_Symbols = 178,
Limbu = 179,
Linear_A = 180,
Linear_B_Ideograms = 181,
Linear_B_Syllabary = 182,
Lisu = 183,
Lisu_Sup = 184,
Low_Surrogates = 185,
Lycian = 186,
Lydian = 187,
Mahajani = 188,
Mahjong = 189,
Makasar = 190,
Malayalam = 191,
Mandaic = 192,
Manichaean = 193,
Marchen = 194,
Masaram_Gondi = 195,
Math_Alphanum = 196,
Math_Operators = 197,
Mayan_Numerals = 198,
Medefaidrin = 199,
Meetei_Mayek = 200,
Meetei_Mayek_Ext = 201,
Mende_Kikakui = 202,
Meroitic_Cursive = 203,
Meroitic_Hieroglyphs = 204,
Miao = 205,
Misc_Arrows = 206,
Misc_Math_Symbols_A = 207,
Misc_Math_Symbols_B = 208,
Misc_Pictographs = 209,
Misc_Symbols = 210,
Misc_Symbols_Sup = 211,
Misc_Technical = 212,
Modi = 213,
Modifier_Letters = 214,
Modifier_Tone_Letters = 215,
Mongolian = 216,
Mongolian_Sup = 217,
Mro = 218,
Multani = 219,
Music = 220,
Myanmar = 221,
Myanmar_Ext_A = 222,
Myanmar_Ext_B = 223,
Myanmar_Ext_C = 224,
Nabataean = 225,
Nag_Mundari = 226,
Nandinagari = 227,
NB = 228,
New_Tai_Lue = 229,
Newa = 230,
NKo = 231,
Number_Forms = 232,
Nushu = 233,
Nyiakeng_Puachue_Hmong = 234,
OCR = 235,
Ogham = 236,
Ol_Chiki = 237,
Ol_Onal = 238,
Old_Hungarian = 239,
Old_Italic = 240,
Old_North_Arabian = 241,
Old_Permic = 242,
Old_Persian = 243,
Old_Sogdian = 244,
Old_South_Arabian = 245,
Old_Turkic = 246,
Old_Uyghur = 247,
Oriya = 248,
Ornamental_Dingbats = 249,
Osage = 250,
Osmanya = 251,
Ottoman_Siyaq_Numbers = 252,
Pahawh_Hmong = 253,
Palmyrene = 254,
Pau_Cin_Hau = 255,
Phags_Pa = 256,
Phaistos = 257,
Phoenician = 258,
Phonetic_Ext = 259,
Phonetic_Ext_Sup = 260,
Playing_Cards = 261,
Psalter_Pahlavi = 262,
PUA = 263,
Punctuation = 264,
Rejang = 265,
Rumi = 266,
Runic = 267,
Samaritan = 268,
Saurashtra = 269,
Sharada = 270,
Sharada_Sup = 271,
Shavian = 272,
Shorthand_Format_Controls = 273,
Siddham = 274,
Sidetic = 275,
Sinhala = 276,
Sinhala_Archaic_Numbers = 277,
Small_Forms = 278,
Small_Kana_Ext = 279,
Sogdian = 280,
Sora_Sompeng = 281,
Soyombo = 282,
Specials = 283,
Sundanese = 284,
Sundanese_Sup = 285,
Sunuwar = 286,
Sup_Arrows_A = 287,
Sup_Arrows_B = 288,
Sup_Arrows_C = 289,
Sup_Math_Operators = 290,
Sup_PUA_A = 291,
Sup_PUA_B = 292,
Sup_Punctuation = 293,
Sup_Symbols_And_Pictographs = 294,
Super_And_Sub = 295,
Sutton_SignWriting = 296,
Syloti_Nagri = 297,
Symbols_And_Pictographs_Ext_A = 298,
Symbols_For_Legacy_Computing = 299,
Symbols_For_Legacy_Computing_Sup = 300,
Syriac = 301,
Syriac_Sup = 302,
Tagalog = 303,
Tagbanwa = 304,
Tags = 305,
Tai_Le = 306,
Tai_Tham = 307,
Tai_Viet = 308,
Tai_Xuan_Jing = 309,
Tai_Yo = 310,
Takri = 311,
Tamil = 312,
Tamil_Sup = 313,
Tangsa = 314,
Tangut = 315,
Tangut_Components = 316,
Tangut_Components_Sup = 317,
Tangut_Sup = 318,
Telugu = 319,
Thaana = 320,
Thai = 321,
Tibetan = 322,
Tifinagh = 323,
Tirhuta = 324,
Todhri = 325,
Tolong_Siki = 326,
Toto = 327,
Transport_And_Map = 328,
Tulu_Tigalari = 329,
UCAS = 330,
UCAS_Ext = 331,
UCAS_Ext_A = 332,
Ugaritic = 333,
Vai = 334,
Vedic_Ext = 335,
Vertical_Forms = 336,
Vithkuqi = 337,
VS = 338,
VS_Sup = 339,
Wancho = 340,
Warang_Citi = 341,
Yezidi = 342,
Yi_Radicals = 343,
Yi_Syllables = 344,
Yijing = 345,
Zanabazar_Square = 346,
Znamenny_Music = 347,
}SourceChar
Char :: struct {
cp: rune,
name: string,
gc: General_Category,
ccc: Combining_Class,
bc: Bidi_Class,
dt: Decomposition_Type,
dm: Decomposition_Mapping,
nt: Numeric_Type,
nv: Numberic_Value,
bm: bool,
name1: string,
sum: string,
slm: string,
stm: string,
}SourceChar_Range
Char_Range :: struct {
first_cp: rune,
last_cp: rune,
name: string,
gc: General_Category,
ccc: Combining_Class,
bc: Bidi_Class,
dt: Decomposition_Type,
dm: Decomposition_Mapping,
nt: Numeric_Type,
nv: Numberic_Value,
bm: bool,
name1: string,
sum: string,
slm: string,
stm: string,
}SourceChars
Chars :: union {
Char,
Char_Range,
}SourceCombining_Class
Combining_Class :: u8SourceDecomposition
Decomposition :: struct {
dt: Decomposition_Type,
dm: Decomposition_Mapping,
ce: bool,
comp_ex: bool,
nfc_quick_check: Trinary_Bool,
nfd_quick_check: bool,
nfkc_quick_check: Trinary_Bool,
nfkd_quick_check: bool,
}SourceDecomposition_Mapping
Decomposition_Mapping :: [dynamic]runeSourceDecomposition_Type
Decomposition_Type :: enum int {
Nil = 0,
can = 1,
com = 2,
enc = 3,
fin = 4,
font = 5,
fra = 6,
init = 7,
iso = 8,
med = 9,
nar = 10,
nb = 11,
sml = 12,
sqr = 13,
sub = 14,
sup = 15,
vert = 16,
wid = 17,
none = 18,
}SourceDynamic_Range
Dynamic_Range :: struct {
single_16: [dynamic]u16,
ranges_16: [dynamic]Range_u16,
single_32: [dynamic]i32,
ranges_32: [dynamic]Range_i32,
}SourceError
Error :: union {
UCD_Error,
os.Error,
}SourceGeneral_Category
General_Category :: enum int {
Cc = 0, // Control, a C0 or C1 control code
Cf = 1, // Format, a format control character
Cn = 2, // Unassigned, a reserved unassigned code point or a noncharacter
Co = 3, // Private_Use, a private-use character
Cs = 4, // Surrogate, a surrogate code point
Ll = 5, // Lowercase_Letter, a lowercase letter
Lm = 6, // Modifier_Letter, a modifier letter
Lo = 7, // Other_Letter, other letters, including syllables and ideographs
Lt = 8, // Titlecase_Letter, a digraph encoded as a single character, with first part uppercase
Lu = 9, // Uppercase_Letter, an uppercase letter
Mc = 10, // Spacing_Mark, a spacing combining mark (positive advance width)
Me = 11, // Enclosing_Mark, an enclosing combining mark
Mn = 12, // Nonspacing_Mark, a nonspacing combining mark (zero advance width)
Nd = 13, // Decimal_Number, a decimal digit
Nl = 14, // Letter_Number, a letterlike numeric character
No = 15, // Other_Number, a numeric character of other type
Pc = 16, // Connector_Punctuation, a connecting punctuation mark, like a tie
Pd = 17, // Dash_Punctuation, a dash or hyphen punctuation mark
Pe = 18, // Close_Punctuation, a closing punctuation mark (of a pair)
Pf = 19, // Final_Punctuation, a final quotation mark
Pi = 20, // Initial_Punctuation, an initial quotation mark
Po = 21, // Other_Punctuation, a punctuation mark of other type
Ps = 22, // Open_Punctuation, an opening punctuation mark (of a pair)
Sc = 23, // Currency_Symbol, a currency sign
Sk = 24, // Modifier_Symbol, a non-letterlike modifier symbol
Sm = 25, // Math_Symbol, a symbol of mathematical use
So = 26, // Other_Symbol, a symbol of other type
Zl = 27, // Line_Separator, U+2028 LINE SEPARATOR only
Zp = 28, // Paragraph_Separator, U+2029 PARAGRAPH SEPARATOR only
Zs = 29, // Space_Separator, a space character (of various non-zero widths)
}SourceNumberic_Value
Numberic_Value :: struct {
numerator: int,
denominator: int,
}SourceNote: Value is NAN when numberator and denominator ar 0
Numeric_Type
Numeric_Type :: enum int {
None = 0, // None
Decimal = 1, // De
Digit = 2, // Di
Numeric = 3, // Nu
}SourcePaired_Bracket_Type
Paired_Bracket_Type :: enum int {
Unknown = 0,
Open = 1,
Close = 2,
None = 3,
}SourceProp_List
Prop_List :: [Prop_List_Property]Dynamic_RangeSourceData contained in the Unicode fiel PropList.txt
A Prop_List is the data contained in the Unicode Database (UCD) file PropList.txt. It is created with the procedure load_property_list and destroyed with the procedure destroy_property_list.
Prop_List_Property
Prop_List_Property :: enum int {
Unknown = 0,
White_Space = 1,
Bidi_Control = 2,
Join_Control = 3,
Dash = 4,
Hyphen = 5,
Quotation_Mark = 6,
Terminal_Punctuation = 7,
Other_Math = 8,
Hex_Digit = 9,
ASCII_Hex_Digit = 10,
Other_Alphabetic = 11,
Ideographic = 12,
Diacritic = 13,
Extender = 14,
Other_Lowercase = 15,
Other_Uppercase = 16,
Noncharacter_Code_Point = 17,
Other_Grapheme_Extend = 18,
IDS_Binary_Operator = 19,
IDS_Trinary_Operator = 20,
IDS_Unary_Operator = 21,
Radical = 22,
Unified_Ideograph = 23,
Other_Default_Ignorable_Code_Point = 24,
Deprecated = 25,
Soft_Dotted = 26,
Logical_Order_Exception = 27,
Other_ID_Start = 28,
Other_ID_Continue = 29,
ID_Compat_Math_Continue = 30,
ID_Compat_Math_Start = 31,
Sentence_Terminal = 32,
Variation_Selector = 33,
Pattern_White_Space = 34,
Pattern_Syntax = 35,
Prepended_Concatenation_Mark = 36,
Regional_Indicator = 37,
Modifier_Combining_Mark = 38,
}SourceRange_Rune
Range_Rune :: struct {
first: rune,
last: rune,
}SourceRange_i32
Range_i32 :: struct {
first: i32,
last: i32,
}SourceRange_u16
Range_u16 :: struct {
first: u16,
last: u16,
}SourceTrinary_Bool
Trinary_Bool :: enum int {
Maybe = -1,
False = 0,
True = 1,
}SourceUCD_Error
UCD_Error :: enum int {
XML_LOAD_ERROR = 0,
XML_Not_UCD = 1,
Nil_XML_Document = 2,
Element_Not_Repertoire = 3,
Extra_Fields = 4,
Unknown_Property = 5,
Unknown_Bidi_Class = 6,
NO_REPERTOIRE = 7,
UNEXPECTED_STRING = 8,
Invalid_Hex_Number = 9,
Invalid_General_Category = 10,
UnicodeData_6_Too_Long = 11,
UnicodeData_6_Invalid = 12,
UnicodeData_7_Too_Long = 13,
UnicodeData_7_Invalid = 14,
}SourceUnicode_Data
Unicode_Data :: [dynamic]CharsSourceConstants
2GENERATED
GENERATED :: `/*
------ GENERATED ------ DO NOT EDIT ------ GENERATED ------ DO NOT EDIT ------ GENERATED ------
*/
`SourceMESSAGE
MESSAGE :: `/*
This file is generated from UnicodeData.txt and PropList.txt. These files
are part of the Unicode Database (UCD) and are covered by the license
listed further down. They may be downloaded from the following locations;
https://www.unicode.org/PubliSourceProcedures
18append_to_dynamic_range
append_to_dynamic_range :: proc(dr: ^Dynamic_Range, range: Range_Rune, allocator: mem.Allocator = context.allocator)Sourcedecode_rune
decode_rune :: proc(str: string) -> (cp1: rune, cp2: rune, err: Error)Sourcedestroy_dynamic_range
destroy_dynamic_range :: proc(dr: Dynamic_Range)Sourcedestroy_general_category_ranges
destroy_general_category_ranges :: proc(gcr: [30]Dynamic_Range)Sourcedestroy_property_list
destroy_property_list :: proc(props: Prop_List)SourceThis function destroys a Prop_List created by load_property_list.
Inputs:
- props: The Prop_List to destroy
destroy_unicode_data
destroy_unicode_data :: proc(unicode_data: Unicode_Data)Sourceextra_digits
extra_digits :: proc(ud: ^Unicode_Data, allocator: mem.Allocator = context.allocator) -> (Dynamic_Range)Sourcegc_ranges
gc_ranges :: proc(ud: ^Unicode_Data, allocator: mem.Allocator = context.allocator) -> (lst: [30]Dynamic_Range)Sourceload_property_list
load_property_list :: proc(filename: string, allocator: mem.Allocator = context.allocator) -> (props: Prop_List, err: Error)Sourceload_unicode_data
load_unicode_data :: proc(filename: string, allocator: mem.Allocator = context.allocator) -> (unicode_data: Unicode_Data, err: Error)Sourcemain
main :: proc()Sourcestring_to_age
string_to_age :: proc(str: string) -> (age: Age)Sourcestring_to_bidi_class
string_to_bidi_class :: proc(str: string) -> (class: Bidi_Class)Sourcestring_to_general_category
string_to_general_category :: proc(str: string) -> (gc: General_Category, err: Error)Sourcestring_to_paired_bracket_type
string_to_paired_bracket_type :: proc(str: string) -> (pbt: Paired_Bracket_Type)Sourcestring_to_proplist_property
string_to_proplist_property :: proc(str: string) -> (prop: Prop_List_Property, err: Error)Sourcewrite_range
write_range :: proc(writer: io.Writer, name: union {
string,
General_Category,
}, range: Dynamic_Range)Sourcewrite_range_arrays
write_range_arrays :: proc(writer: io.Writer, name: string, range: Dynamic_Range)SourceTable 2-3. Types of Code Points Table 4-4. General_Category Values page 229 Reference https://www.unicode.org/reports/tr44/ Formats a Dynamic_Range into a set of fixed length arrays and writes them to an io.Writer. The value of the parameter name will be used as a prefix to the array names.
If a dynamic array contained in the range is empty, no corresponding fixed length array will be written.
Inputs:
- writer: The
io.Writerto be written to. - name: Prefix to add to any array that is written to
writer - range:
The Dynamic_Rangeto format and write to writer.