std/jule/token

Index

Variables
fn IsKeyword(s: str): bool
fn IsPostfix(id: int): bool
fn IsAssign(id: int): bool
fn ScanAll(mut f: &FileSet, opt: int): []log::Log
struct Token
    fn Precedence(*self): int
struct Scanner
    fn New(mut f: &FileSet, opt: int): &Scanner
    fn Scan(mut *self): (token: &Token, EOF: bool)
struct FileSet
    fn New(path: str): &FileSet
    fn Fill(mut *self, data: []byte)
    unsafe fn FillMut(mut *self, mut data: []byte)
    fn Dir(*self): str
    fn Name(*self): str
    fn Addr(*self): uintptr
    fn GetRow(*self, row: int): str

Variables

const (
	ILLEGAL = iota // illegal token

	SEMICOLON // ;
	COMMA     // ,
	COLON     // :
	PERIOD    // .
	SEP       // ::
	HASH      // #
	ELLIPSIS  // ...
	QMARK     // ?

	COMMENT // /*..*/ or //...
	NAME    // identifier

	FLOAT // floating-point literal
	INT   // integer literal
	STR   // string literal
	RUNE  // rune literal

	RET      // ret
	CONST    // const
	TYPE     // type
	FOR      // for
	BREAK    // break
	CONTINUE // continue
	IN       // int
	IF       // if
	ELSE     // else
	USE      // use
	GOTO     // goto
	ENUM     // enum
	STRUCT   // struct
	CO       // co
	MATCH    // match
	SELF     // self
	TRAIT    // trait
	IMPL     // impl
	CHAN     // chan
	BIND     // cpp
	FALL     // fall
	FN       // fn
	LET      // let
	UNSAFE   // unsafe
	MUT      // mut
	DEFER    // defer
	ERROR    // error
	MAP      // map
	SELECT   // select

	EQL  // ==
	NEQ  // !=
	GEQ  // >=
	LEQ  // <=
	LAND // &&
	LOR  // ||

	SHL   // <<
	SHR   // >>
	INC   // ++
	DEC   // --
	ADD   // +
	SUB   // -
	MUL   // *
	QUO   // /
	REM   // %
	AND   // &
	OR    // |
	XOR   // ^
	NOT   // !
	LSS   // <
	GTR   // >
	ARROW // <-

	ASSIGN     // =
	DEFINE     // :=
	ADD_ASSIGN // +=
	SUB_ASSIGN // -=
	MUL_ASSIGN // *=
	QUO_ASSIGN // /=
	REM_ASSIGN // %=
	SHL_ASSIGN // <<=
	SHR_ASSIGN // >>=
	XOR_ASSIGN // ^=
	AND_ASSIGN // &=
	OR_ASSIGN  // |=

	LPAREN // (
	LBRACK // [
	LBRACE // {
	RPAREN // )
	RBRACK // ]
	RBRACE // }
)

Token identities.

---

const (
	LowestPrec  = 0 // non-operators
	UnaryPrec   = 6
	HighestPrec = 7
)

A set of constants for precedence-based expression parsing. Non-operators have lowest precedence, followed by operators starting with precedence 1 up to unary operators. The highest precedence serves as "catch-all" precedence for selector, indexing, and other operator and delimiter tokens.

---

const (
	Standard = 1 << iota // Standard mode.
	Comments             // Standard mode + comments.
)

Lexer mode.

IsKeyword

fn IsKeyword(s: str): bool

Reports whether s is keyword.

IsPostfix

fn IsPostfix(id: int): bool

Reports whether operator kind is postfix operator.

IsAssign

fn IsAssign(id: int): bool

Reports whether operator kind is assignment operator.

ScanAll

fn ScanAll(mut f: &FileSet, opt: int): []log::Log

Scans all tokens into FileSet f and returns error logs.

Token

struct Token {
	ID:     int      // Identity of token.
	File:   &FileSet // Associated FileSet where token appear.
	Row:    int      // Row position of token.
	Column: int      // Column position of token.
	Kind:   str      // Token kind as string.
}

Token.

Precedence

fn Precedence(*self): int

Returns operator precedence of token. Returns 0 if token is not operator or invalid operator for operator precedence. It only reports for the binary operators, otherwise returns LowestPrec.

Scanner

struct Scanner {
	Logs: []log::Log // Scan logs.
	// NOTE: contains filtered hidden or unexported fields
}

Scanner scans tokens from FileSet data. Any log will be appended to [Scanner.Logs].

New

fn New(mut f: &FileSet, opt: int): &Scanner

Returns new Scanner for the FileSet f.

Scan

fn Scan(mut *self): (token: &Token, EOF: bool)

Scans and returns new token, reports if EOF. If and error appeared, returns nil token and not-EOF.

FileSet

struct FileSet {
	Path:   str
	Tokens: []&Token
	// NOTE: contains filtered hidden or unexported fields
}

Fileset for the lexer.

New

fn New(path: str): &FileSet

Returns new FileSet with path.

Fill

fn Fill(mut *self, data: []byte)

Fills data. Not uses mutable copy of data, allocates new copy.

FillMut

unsafe fn FillMut(mut *self, mut data: []byte)

Fills data. Uses mutable copy of data, not allocated new copy. But it is unsafe, because any mutation on the data may cause inconsistent results. However, it is efficient way to use already allocated data.

Dir

fn Dir(*self): str

Returns directory of file's path.

Name

fn Name(*self): str

Returns filename.

Addr

fn Addr(*self): uintptr

Returns self as uintptr.

GetRow

fn GetRow(*self, row: int): str

Returns line (not include new-line char) by row. Returns empty string if line is not buffer.