core/text/regex/tokenizer
regex_tokenizer
Types
4Error
Error :: enum int {
None = 0,
Illegal_Null_Character = 1,
Illegal_Codepoint = 2,
Illegal_Byte_Order_Mark = 3,
}SourceToken
Token :: struct {
kind: Token_Kind,
text: string,
pos: int,
}SourceToken_Kind
Token_Kind :: enum int {
Invalid = 0,
EOF = 1,
Rune = 2,
Wildcard = 3,
Alternate = 4,
Concatenate = 5,
Repeat_Zero = 6,
Repeat_Zero_Non_Greedy = 7,
Repeat_One = 8,
Repeat_One_Non_Greedy = 9,
Repeat_N = 10,
Optional = 11,
Optional_Non_Greedy = 12,
Rune_Class = 13,
Open_Paren = 14,
Open_Paren_Non_Capture = 15,
Close_Paren = 16,
Anchor_Start = 17,
Anchor_End = 18,
Word_Boundary = 19,
Non_Word_Boundary = 20,
}SourceTokenizer
Tokenizer :: struct {
flags: common.Flags,
src: string,
ch: rune,
offset: int,
read_offset: int,
last_token_kind: Token_Kind,
held_token: Token,
error_state: Error,
paren_depth: int,
}SourceProcedures
9advance_rune
advance_rune :: proc(t: ^Tokenizer) -> (err: Error)Sourceinit
init :: proc(t: ^Tokenizer, str: string, flags: common.Flags)Sourcepeek_byte
peek_byte :: proc(t: ^Tokenizer, offset: untyped integer = 0) -> (u8)Sourcescan
scan :: proc(t: ^Tokenizer) -> (token: Token)Sourcescan_class
scan_class :: proc(t: ^Tokenizer) -> (str: string, ok: bool)Sourcescan_comment
scan_comment :: proc(t: ^Tokenizer)Sourcescan_non_capture_group
scan_non_capture_group :: proc(t: ^Tokenizer) -> (bool)Sourcescan_non_greedy
scan_non_greedy :: proc(t: ^Tokenizer) -> (bool)Sourcescan_repeat
scan_repeat :: proc(t: ^Tokenizer) -> (str: string, ok: bool)Source