mirror of
https://github.com/vlang/v.git
synced 2023-08-10 21:13:21 +03:00
375 lines
7.5 KiB
V
375 lines
7.5 KiB
V
// Copyright (c) 2019 Alexander Medvednikov. All rights reserved.
|
|
// Use of this source code is governed by an MIT license
|
|
// that can be found in the LICENSE file.
|
|
module token
|
|
|
|
/*
|
|
struct Token {
|
|
tok TokenKind // the token number/enum; for quick comparisons
|
|
lit string // literal representation of the token
|
|
line_nr int // the line number in the source where the token occured
|
|
//name_idx int // name table index for O(1) lookup
|
|
pos int // the position of the token in scanner text
|
|
}
|
|
*/
|
|
|
|
pub enum Token {
|
|
eof
|
|
name // user
|
|
number // 123
|
|
str // 'foo'
|
|
str_inter // 'name=$user.name'
|
|
chartoken // `A`
|
|
plus
|
|
minus
|
|
mul
|
|
div
|
|
mod
|
|
xor // ^
|
|
pipe // |
|
|
inc // ++
|
|
dec // --
|
|
and // &&
|
|
logical_or
|
|
not
|
|
bit_not
|
|
question
|
|
comma
|
|
semicolon
|
|
colon
|
|
arrow // =>
|
|
amp
|
|
hash
|
|
dollar
|
|
str_dollar
|
|
left_shift
|
|
righ_shift
|
|
// at // @
|
|
assign // =
|
|
decl_assign // :=
|
|
plus_assign // +=
|
|
minus_assign // -=
|
|
div_assign
|
|
mult_assign
|
|
xor_assign
|
|
mod_assign
|
|
or_assign
|
|
and_assign
|
|
righ_shift_assign
|
|
left_shift_assign
|
|
// {} () []
|
|
lcbr
|
|
rcbr
|
|
lpar
|
|
rpar
|
|
lsbr
|
|
rsbr
|
|
// == != <= < >= >
|
|
eq
|
|
ne
|
|
gt
|
|
lt
|
|
ge
|
|
le
|
|
// comments
|
|
line_comment
|
|
mline_comment
|
|
nl
|
|
dot
|
|
dotdot
|
|
ellipsis
|
|
// keywords
|
|
keyword_beg
|
|
key_as
|
|
key_asm
|
|
key_assert
|
|
key_atomic
|
|
key_break
|
|
key_const
|
|
key_continue
|
|
key_defer
|
|
key_else
|
|
key_embed
|
|
key_enum
|
|
key_false
|
|
key_for
|
|
key_fn
|
|
key_global
|
|
key_go
|
|
key_goto
|
|
key_if
|
|
key_import
|
|
key_import_const
|
|
key_in
|
|
key_interface
|
|
// key_it
|
|
key_match
|
|
key_module
|
|
key_mut
|
|
key_none
|
|
key_return
|
|
key_select
|
|
key_sizeof
|
|
key_offsetof
|
|
key_struct
|
|
key_switch
|
|
key_true
|
|
key_type
|
|
// typeof
|
|
key_orelse
|
|
key_union
|
|
key_pub
|
|
key_static
|
|
key_unsafe
|
|
keyword_end
|
|
}
|
|
|
|
const (
|
|
assign_tokens = [Token.assign, .plus_assign, .minus_assign, .mult_assign,
|
|
.div_assign, .xor_assign, .mod_assign, .or_assign, .and_assign,
|
|
.righ_shift_assign, .left_shift_assign]
|
|
|
|
nr_tokens = 141
|
|
)
|
|
|
|
|
|
// build_keys genereates a map with keywords' string values:
|
|
// Keywords['return'] == .key_return
|
|
fn build_keys() map[string]int {
|
|
mut res := map[string]int
|
|
for t := int(Token.keyword_beg) + 1; t < int(Token.keyword_end); t++ {
|
|
key := token_str[t]
|
|
res[key] = t
|
|
}
|
|
return res
|
|
}
|
|
|
|
// TODO remove once we have `enum Token { name('name') if('if') ... }`
|
|
fn build_token_str() []string {
|
|
mut s := [''].repeat(nr_tokens)
|
|
s[Token.keyword_beg] = ''
|
|
s[Token.keyword_end] = ''
|
|
s[Token.eof] = 'eof'
|
|
s[Token.name] = 'name'
|
|
s[Token.number] = 'number'
|
|
s[Token.str] = 'STR'
|
|
s[Token.chartoken] = 'char'
|
|
s[Token.plus] = '+'
|
|
s[Token.minus] = '-'
|
|
s[Token.mul] = '*'
|
|
s[Token.div] = '/'
|
|
s[Token.mod] = '%'
|
|
s[Token.xor] = '^'
|
|
s[Token.bit_not] = '~'
|
|
s[Token.pipe] = '|'
|
|
s[Token.hash] = '#'
|
|
s[Token.amp] = '&'
|
|
s[Token.inc] = '++'
|
|
s[Token.dec] = '--'
|
|
s[Token.and] = '&&'
|
|
s[Token.logical_or] = '||'
|
|
s[Token.not] = '!'
|
|
s[Token.dot] = '.'
|
|
s[Token.dotdot] = '..'
|
|
s[Token.ellipsis] = '...'
|
|
s[Token.comma] = ','
|
|
// s[Token.at] = '@'
|
|
s[Token.semicolon] = ';'
|
|
s[Token.colon] = ':'
|
|
s[Token.arrow] = '=>'
|
|
s[Token.assign] = '='
|
|
s[Token.decl_assign] = ':='
|
|
s[Token.plus_assign] = '+='
|
|
s[Token.minus_assign] = '-='
|
|
s[Token.mult_assign] = '*='
|
|
s[Token.div_assign] = '/='
|
|
s[Token.xor_assign] = '^='
|
|
s[Token.mod_assign] = '%='
|
|
s[Token.or_assign] = '|='
|
|
s[Token.and_assign] = '&='
|
|
s[Token.righ_shift_assign] = '>>='
|
|
s[Token.left_shift_assign] = '<<='
|
|
s[Token.lcbr] = '{'
|
|
s[Token.rcbr] = '}'
|
|
s[Token.lpar] = '('
|
|
s[Token.rpar] = ')'
|
|
s[Token.lsbr] = '['
|
|
s[Token.rsbr] = ']'
|
|
s[Token.eq] = '=='
|
|
s[Token.ne] = '!='
|
|
s[Token.gt] = '>'
|
|
s[Token.lt] = '<'
|
|
s[Token.ge] = '>='
|
|
s[Token.le] = '<='
|
|
s[Token.question] = '?'
|
|
s[Token.left_shift] = '<<'
|
|
s[Token.righ_shift] = '>>'
|
|
s[Token.line_comment] = '// line comment'
|
|
s[Token.mline_comment] = '/* mline comment */'
|
|
s[Token.nl] = 'NLL'
|
|
s[Token.dollar] = '$'
|
|
s[Token.str_dollar] = '$2'
|
|
s[Token.key_assert] = 'assert'
|
|
s[Token.key_struct] = 'struct'
|
|
s[Token.key_if] = 'if'
|
|
// s[Token.key_it] = 'it'
|
|
s[Token.key_else] = 'else'
|
|
s[Token.key_asm] = 'asm'
|
|
s[Token.key_return] = 'return'
|
|
s[Token.key_module] = 'module'
|
|
s[Token.key_sizeof] = 'sizeof'
|
|
s[Token.key_go] = 'go'
|
|
s[Token.key_goto] = 'goto'
|
|
s[Token.key_const] = 'const'
|
|
s[Token.key_mut] = 'mut'
|
|
s[Token.key_type] = 'type'
|
|
s[Token.key_for] = 'for'
|
|
s[Token.key_switch] = 'switch'
|
|
s[Token.key_fn] = 'fn'
|
|
s[Token.key_true] = 'true'
|
|
s[Token.key_false] = 'false'
|
|
s[Token.key_continue] = 'continue'
|
|
s[Token.key_break] = 'break'
|
|
s[Token.key_import] = 'import'
|
|
s[Token.key_embed] = 'embed'
|
|
s[Token.key_unsafe] = 'unsafe'
|
|
// Tokens[key_typeof] = 'typeof'
|
|
s[Token.key_enum] = 'enum'
|
|
s[Token.key_interface] = 'interface'
|
|
s[Token.key_pub] = 'pub'
|
|
s[Token.key_import_const] = 'import_const'
|
|
s[Token.key_in] = 'in'
|
|
s[Token.key_atomic] = 'atomic'
|
|
s[Token.key_orelse] = 'or'
|
|
s[Token.key_global] = '__global'
|
|
s[Token.key_union] = 'union'
|
|
s[Token.key_static] = 'static'
|
|
s[Token.key_as] = 'as'
|
|
s[Token.key_defer] = 'defer'
|
|
s[Token.key_match] = 'match'
|
|
s[Token.key_select] = 'select'
|
|
s[Token.key_none] = 'none'
|
|
s[Token.key_offsetof] = '__offsetof'
|
|
return s
|
|
}
|
|
|
|
const (
|
|
token_str = build_token_str()
|
|
keywords = build_keys()
|
|
)
|
|
|
|
pub fn key_to_token(key string) Token {
|
|
a := Token(keywords[key])
|
|
return a
|
|
}
|
|
|
|
pub fn is_key(key string) bool {
|
|
return int(key_to_token(key)) > 0
|
|
}
|
|
|
|
pub fn is_decl(t Token) bool {
|
|
return t in [.key_enum,
|
|
.key_interface, .key_fn, .key_struct, .key_type, .key_const, .key_import_const,
|
|
.key_pub, .eof]
|
|
}
|
|
|
|
fn (t Token) is_assign() bool {
|
|
return t in assign_tokens
|
|
}
|
|
|
|
fn (t []Token) contains(val Token) bool {
|
|
for tt in t {
|
|
if tt == val {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
pub fn (t Token) str() string {
|
|
lit := 't.lit'
|
|
if t == .number {
|
|
return 'number'
|
|
}
|
|
if t == .chartoken {
|
|
return '`lit`'
|
|
}
|
|
if t == .str {
|
|
return "'lit'"
|
|
}
|
|
if t < .plus {
|
|
return lit // string, number etc
|
|
}
|
|
return token_str[int(t)]
|
|
}
|
|
|
|
|
|
// Representation of highest and lowest precedence
|
|
const (
|
|
lowest_prec = 0
|
|
highest_prec = 7
|
|
)
|
|
|
|
// Precedence returns a tokens precedence if defined, otherwise lowest_prec
|
|
pub fn (tok Token) precedence() int {
|
|
match tok {
|
|
// `*` | `/` | `%` | `<<` | `>>` | `&`
|
|
.mul, .div, .left_shift, .righ_shift, .amp { return 7 }
|
|
// `+` | `-` | `|` | `^`
|
|
.plus, .minus, .pipe, .xor { return 6 }
|
|
// `==` | `!=` | `<` | `<=` | `>` | `>=`
|
|
.eq, .ne, .lt, .le, .gt, .ge { return 5 }
|
|
// `&&`
|
|
.and { return 4 }
|
|
// `||`
|
|
.logical_or { return 3 }
|
|
else { return lowest_prec }
|
|
}
|
|
}
|
|
|
|
// is_scalar returns true if the token is a scalar
|
|
pub fn (tok Token) is_scalar() bool {
|
|
return tok in [.number, .str]
|
|
}
|
|
|
|
// is_unary returns true if the token can be in a unary expression
|
|
pub fn (tok Token) is_unary() bool {
|
|
match tok {
|
|
// `+` | `-` | `!` | `~` | `*` | `&`
|
|
.plus, .minus, .not, .bit_not, .mul, .amp { return true }
|
|
else { return false }
|
|
}
|
|
}
|
|
|
|
// NOTE: do we need this for all tokens (is_left_assoc / is_right_assoc),
|
|
// or only ones with the same precedence?
|
|
|
|
// is_left_assoc returns true if the token is left associative
|
|
pub fn (tok Token) is_left_assoc() bool {
|
|
match tok {
|
|
// .number,
|
|
// `*` | `/` | `%`
|
|
.mul, .div, .mod,
|
|
// `^` | `||` | `&`
|
|
.xor, .logical_or, .and,
|
|
// `,`
|
|
.comma { return true }
|
|
else { return false }
|
|
}
|
|
}
|
|
|
|
// is_right_assoc returns true if the token is right associative
|
|
pub fn (tok Token) is_right_assoc() bool {
|
|
match tok {
|
|
// `+` | `-` | `!` | `++` | `--`
|
|
.plus, .minus, .not, .inc, .dec,
|
|
// `=` | `+=` | `-=` | `*=` | `/=`
|
|
.assign, .plus_assign, .minus_assign, .mult_assign, .div_assign,
|
|
// `%=` | `>>=` | `<<=`
|
|
.mod_assign, .righ_shift_assign, .left_shift_assign,
|
|
// `&=` | `^=` | `|=`
|
|
.and_assign, .xor_assign, .or_assign { return true }
|
|
else { return false }
|
|
}
|
|
}
|