core/text/scanner

text_scanner

Types

Position :: struct {
	filename: string,
	offset:   int,
	line:     int,
	column:   int,
}

Source

Position represents a source position A position is valid if line > 0

Scan_Flag :: enum u32 {
	Scan_Idents         = 0,
	Scan_Ints           = 1,
	Scan_C_Int_Prefixes = 2,
	Scan_Floats         = 3, // Includes integers and hexadecimal floats
	Scan_Chars          = 4,
	Scan_Strings        = 5,
	Scan_Raw_Strings    = 6,
	Scan_Comments       = 7,
	Skip_Comments       = 8, // if set with .Scan_Comments, comments become white space
}

Source

Scan_Flags :: bit_set[Scan_Flag; u32]Source

Scanner :: struct {
	src:           string,
	src_pos:       int,
	src_end:       int,
	tok_pos:       int,
	tok_end:       int,
	ch:            rune,
	line:          int,
	column:        int,
	prev_line_len: int,
	prev_char_len: int,
	// error is called for each error encountered
	// If no error procedure is set, the error is reported to os.stderr
	error:         proc(s: ^Scanner, msg: string),
	s:             ^Scanner,
	Scanner:       string,
	msg:           string,
	// error_count is incremented by one for each error encountered
	error_count:   int,
	// flags controls which tokens are recognized
	// e.g. to recognize integers, set the .Scan_Ints flag
	// This field may be changed by the user at any time during scanning
	flags:         Scan_Flags,
	// The whitespace field controls which characters are recognized as white space
	// This field may be changed by the user at any time during scanning
	whitespace:    Whitespace,
	// is_ident_rune is a predicate controlling the characters accepted as the ith rune in an identifier
	// The valid characters must not conflict with the set of white space characters
	// If is_ident_rune is not set, regular Odin-like identifiers are accepted
	// This field may be changed by the user at any time during scanning
	is_ident_rune: proc(ch: rune, i: int) -> (bool),
	ch:            rune,
	i:             int,
	// Start position of most recently scanned token (set by scan(s))
	// Call init or next invalidates the position
	pos:           Position,
}

Source

Scanner allows for the reading of Unicode characters and tokens from a string

Whitespace :: WhitespaceSource

Only allows for ASCII whitespace

Constants

C_Like_Tokens :: Scan_Flags = Scan_Flags{.Scan_Idents, .Scan_Ints, .Scan_C_Int_Prefixes, .Scan_Floats, .Scan_Chars, .Scan_Strings, .Scan_Raw_Strings, .Scan_Comments, .Skip_Comments}

Source

C_Whitespace :: Whitespace = Whitespace{'\t', '\n', '\r', '\v', '\f', ' '}Source

Char :: -5Source

Comment :: -8Source

EOF :: -1Source

Float :: -4Source

Ident :: -2Source

Int :: -3Source

Odin_Like_Tokens :: Scan_Flags = Scan_Flags{.Scan_Idents, .Scan_Ints, .Scan_Floats, .Scan_Chars, .Scan_Strings, .Scan_Raw_Strings, .Scan_Comments, .Skip_Comments}

Source

Odin_Whitespace :: Whitespace = Whitespace{'\t', '\n', '\r', ' '}Source

Odin_Whitespace is the default value for the Scanner's whitespace field

Raw_String :: -7Source

String :: -6Source

Procedures

error :: proc(s: ^Scanner, msg: string)Source

errorf :: proc(s: ^Scanner, format: string, args)Source

init :: proc(s: ^Scanner, src: string, filename: untyped string = "") -> (^Scanner)Source

init initializes a scanner with a new source and returns itself. error_count is set to 0, flags is set to Odin_Like_Tokens, whitespace is set to Odin_Whitespace

next :: proc(s: ^Scanner) -> (rune)Source

next reads and returns the next Unicode character. It returns EOF at the end of the source. next does not update the Scanner's pos field. Use 'position(s)' to get the current position

peek :: proc(s: ^Scanner, n: untyped integer = 0) -> (ch: rune)Source

peek returns the next Unicode character in the source without advancing the scanner It returns EOF if the scanner's position is at least the last character of the source if n > 0, it call next n times and return the nth Unicode character and then restore the Scanner's state

peek_token :: proc(s: ^Scanner, n: untyped integer = 0) -> (tok: rune)Source

peek returns the next token in the source It returns EOF if the scanner's position is at least the last character of the source if n > 0, it call next n times and return the nth token and then restore the Scanner's state

position :: proc(s: ^Scanner) -> (Position)Source

position returns the position of the character immediately after the character or token returns by the previous call to next or scan Use the Scanner's position field for the most recently scanned token position

position_is_valid :: proc(pos: Position) -> (bool)Source

position_is_valid reports where the position is valid

position_to_string :: proc(pos: Position, allocator = context.temp_allocator) -> (string)Source

scan :: proc(s: ^Scanner) -> (tok: rune)Source

scan reads the next token or Unicode character from source and returns it It only recognizes tokens for which the respective flag that is set It returns EOF at the end of the source It reports Scanner errors by calling s.error, if not nil; otherwise it will print the error message to os.stderr

token_string :: proc(tok: rune, allocator: runtime.Allocator) -> (string)Source

token_string returns a printable string for a token or Unicode character By default, it uses the context.temp_allocator to produce the string

token_text :: proc(s: ^Scanner) -> (string)Source

token_text returns the string of the most recently scanned token

text_scanner

Types

Constants

Procedures

Find anything

Settings