std/jule/token

Index

Variables
fn AppendAll(mut buf: []&Token, mut f: &FileSet, opt: int): ([]&Token, []log::Log)
fn IsKeyword(s: string): bool
fn IsPostfix(id: int): bool
fn IsAssign(id: int): bool
struct FileSet
    fn New(path: string): &FileSet
    fn Fill(mut *self, data: []byte)
    fn FillMut(mut *self, mut data: []byte)
    fn Dir(*self): string
    fn Name(*self): string
    fn Addr(*self): uintptr
    fn GetRow(*self, row: int): string
struct Scanner
    fn New(mut f: &FileSet, opt: int): &Scanner
    fn Scan(mut *self): (token: &Token, EOF: bool)
struct Token
    fn Precedence(*self): int

Variables

const InvalidID = -1

The standard invalid FileSet ID. All FileSet implementations must consider this special value as invalid.

---

const (
	Standard = 1 << iota // Standard mode.
	Comments             // Standard mode + comments.
)

Lexer mode.

---

const (
	ILLEGAL = iota // illegal token

	SEMICOLON // ;
	COMMA     // ,
	COLON     // :
	PERIOD    // .
	SEP       // ::
	HASH      // #
	ELLIPSIS  // ...
	QMARK     // ?
	TILDE     // ~

	COMMENT // /*..*/ or //...
	NAME    // identifier

	FLOAT  // floating-point literal
	INT    // integer literal
	STRING // string literal
	RUNE   // rune literal
	IMAG   // imaginary part literal

	RETURN   // return
	CONST    // const
	TYPE     // type
	FOR      // for
	BREAK    // break
	CONTINUE // continue
	IN       // int
	IF       // if
	ELSE     // else
	USE      // use
	GOTO     // goto
	ENUM     // enum
	STRUCT   // struct
	SPAWN    // spawn
	MATCH    // match
	SELF     // self
	TRAIT    // trait
	IMPL     // impl
	CHAN     // chan
	EXTERN   // extern
	FALL     // fall
	FN       // fn
	LET      // let
	UNSAFE   // unsafe
	MUT      // mut
	DEFER    // defer
	ERROR    // error
	THROW    // throw
	MAP      // map
	SELECT   // select
	CATCH    // catch

	EQL  // ==
	NEQ  // !=
	GEQ  // >=
	LEQ  // <=
	LAND // &&
	LOR  // ||

	SHL   // <<
	SHR   // >>
	INC   // ++
	DEC   // --
	ADD   // +
	SUB   // -
	MUL   // *
	QUO   // /
	REM   // %
	AND   // &
	OR    // |
	XOR   // ^
	NOT   // !
	LSS   // <
	GTR   // >
	ARROW // <-

	ASSIGN     // =
	DEFINE     // :=
	ADD_ASSIGN // +=
	SUB_ASSIGN // -=
	MUL_ASSIGN // *=
	QUO_ASSIGN // /=
	REM_ASSIGN // %=
	SHL_ASSIGN // <<=
	SHR_ASSIGN // >>=
	XOR_ASSIGN // ^=
	AND_ASSIGN // &=
	OR_ASSIGN  // |=

	LPAREN // (
	LBRACK // [
	LBRACE // {
	RPAREN // )
	RBRACK // ]
	RBRACE // }
)

Token identities.

---

const (
	LowestPrec  = 0 // non-operators
	UnaryPrec   = 6
	HighestPrec = 7
)

A set of constants for precedence-based expression parsing. Non-operators have lowest precedence, followed by operators starting with precedence 1 up to unary operators. The highest precedence serves as "catch-all" precedence for selector, indexing, and other operator and delimiter tokens.

AppendAll

fn AppendAll(mut buf: []&Token, mut f: &FileSet, opt: int): ([]&Token, []log::Log)

Scans and appends all tokens of the FileSet f to buf and returns it with error logs.

IsKeyword

fn IsKeyword(s: string): bool

Reports whether s is keyword.

IsPostfix

fn IsPostfix(id: int): bool

Reports whether operator kind is postfix operator.

IsAssign

fn IsAssign(id: int): bool

Reports whether operator kind is assignment operator.

FileSet

struct FileSet {
	ID:   int // Unique identity of the FileSet.
	Path: string
	// NOTE: contains filtered hidden or unexported fields
}

Fileset for the lexer.

New

fn New(path: string): &FileSet

Returns new FileSet with path.

Fill

fn Fill(mut *self, data: []byte)

Fills data. Not uses mutable copy of data, allocates new copy.

FillMut

unsafe fn FillMut(mut *self, mut data: []byte)

Fills data. Uses mutable copy of data, not allocated new copy. But it is unsafe, because any mutation on the data may cause inconsistent results. However, it is efficient way to use already allocated data.

Dir

fn Dir(*self): string

Returns directory of file's path.

Name

fn Name(*self): string

Returns filename.

Addr

fn Addr(*self): uintptr

Returns self as uintptr.

GetRow

fn GetRow(*self, row: int): string

Returns line (not include new-line char) by row. Returns empty string if line is not buffer.

Scanner

struct Scanner {
	Logs: []log::Log // Scan logs.
	// NOTE: contains filtered hidden or unexported fields
}

Scanner scans tokens from FileSet data. Any log will be appended to [Scanner.Logs].

New

fn New(mut f: &FileSet, opt: int): &Scanner

Returns new Scanner for the FileSet f.

Scan

fn Scan(mut *self): (token: &Token, EOF: bool)

Scans and returns new token, reports if EOF. If and error appeared or a token ignored, returns nil token and not-EOF.

Token

struct Token {
	ID:        int    // Identity of token.
	FileSetID: int    // ID of the associated FileSet where token appear.
	Row:       int    // Row position of token.
	Column:    int    // Column position of token.
	Kind:      string // Token kind as string.
}

Token.

Precedence

fn Precedence(*self): int

Returns operator precedence of token. Returns 0 if token is not operator or invalid operator for operator precedence. It only reports for the binary operators, otherwise returns LowestPrec.