2019-06-23 05:21:30 +03:00
|
|
|
// Copyright (c) 2019 Alexander Medvednikov. All rights reserved.
|
|
|
|
// Use of this source code is governed by an MIT license
|
|
|
|
// that can be found in the LICENSE file.
|
|
|
|
|
2019-06-22 21:20:28 +03:00
|
|
|
module main
|
|
|
|
|
2019-07-12 08:37:54 +03:00
|
|
|
import os
|
|
|
|
import strings
|
|
|
|
|
2019-06-22 21:20:28 +03:00
|
|
|
struct Scanner {
|
|
|
|
mut:
|
|
|
|
file_path string
|
|
|
|
text string
|
|
|
|
pos int
|
|
|
|
line_nr int
|
|
|
|
inside_string bool
|
|
|
|
dollar_start bool // for hacky string interpolation TODO simplify
|
|
|
|
dollar_end bool
|
|
|
|
debug bool
|
|
|
|
line_comment string
|
|
|
|
started bool
|
|
|
|
is_fmt bool
|
|
|
|
// vfmt fields
|
2019-07-03 23:11:27 +03:00
|
|
|
fmt_out strings.Builder
|
2019-06-22 21:20:28 +03:00
|
|
|
fmt_indent int
|
|
|
|
fmt_line_empty bool
|
|
|
|
}
|
|
|
|
|
|
|
|
const (
|
|
|
|
SINGLE_QUOTE = `\'`
|
2019-06-30 15:35:26 +03:00
|
|
|
//QUOTE = `"`
|
2019-06-22 21:20:28 +03:00
|
|
|
)
|
|
|
|
|
|
|
|
fn new_scanner(file_path string) *Scanner {
|
|
|
|
if !os.file_exists(file_path) {
|
2019-06-23 03:02:33 +03:00
|
|
|
panic('"$file_path" doesn\'t exist')
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
2019-07-03 22:07:42 +03:00
|
|
|
//text := os.read_file(file_path)
|
2019-06-26 11:57:13 +03:00
|
|
|
text := os.read_file(file_path) or {
|
|
|
|
panic('scanner: failed to open "$file_path"')
|
|
|
|
return &Scanner{}
|
|
|
|
}
|
2019-06-22 21:20:28 +03:00
|
|
|
scanner := &Scanner {
|
|
|
|
file_path: file_path
|
2019-06-26 11:57:13 +03:00
|
|
|
text: text
|
2019-07-03 23:11:27 +03:00
|
|
|
fmt_out: strings.new_builder(1000)
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
|
|
|
// println('new scanner "$file_path" txt.len=$scanner.text.len')
|
|
|
|
return scanner
|
|
|
|
}
|
|
|
|
|
|
|
|
// TODO remove once multiple return values are implemented
|
|
|
|
struct ScanRes {
|
|
|
|
tok Token
|
|
|
|
lit string
|
|
|
|
}
|
|
|
|
|
|
|
|
fn scan_res(tok Token, lit string) ScanRes {
|
|
|
|
return ScanRes{tok, lit}
|
|
|
|
}
|
|
|
|
|
|
|
|
fn is_white(c byte) bool {
|
|
|
|
return c.is_white()
|
|
|
|
}
|
|
|
|
|
|
|
|
fn is_nl(c byte) bool {
|
2019-06-26 14:54:03 +03:00
|
|
|
return c == `\r` || c == `\n`
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
fn (s mut Scanner) ident_name() string {
|
|
|
|
start := s.pos
|
|
|
|
for {
|
|
|
|
s.pos++
|
2019-06-27 18:05:02 +03:00
|
|
|
if s.pos >= s.text.len {
|
|
|
|
break
|
|
|
|
}
|
2019-06-22 21:20:28 +03:00
|
|
|
c := s.text[s.pos]
|
|
|
|
if !is_name_char(c) && !c.is_digit() {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
name := s.text.substr(start, s.pos)
|
|
|
|
s.pos--
|
|
|
|
return name
|
|
|
|
}
|
|
|
|
|
|
|
|
fn (s mut Scanner) ident_number() string {
|
|
|
|
start := s.pos
|
2019-06-27 18:05:02 +03:00
|
|
|
is_hex := s.pos + 1 < s.text.len && s.text[s.pos] == `0` && s.text[s.pos + 1] == `x`
|
2019-06-22 21:20:28 +03:00
|
|
|
is_oct := !is_hex && s.text[s.pos] == `0`
|
|
|
|
mut is_float := false
|
|
|
|
for {
|
|
|
|
s.pos++
|
2019-06-27 18:05:02 +03:00
|
|
|
if s.pos >= s.text.len {
|
|
|
|
break
|
|
|
|
}
|
2019-06-22 21:20:28 +03:00
|
|
|
c := s.text[s.pos]
|
|
|
|
if c == `.` {
|
|
|
|
is_float = true
|
|
|
|
}
|
2019-07-02 15:45:27 +03:00
|
|
|
is_good_hex := is_hex && (c == `x` || (c >= `a` && c <= `f`) || (c >= `A` && c <= `F`))
|
2019-06-22 21:20:28 +03:00
|
|
|
// 1e+3, 1e-3, 1e3
|
2019-06-27 18:05:02 +03:00
|
|
|
if !is_hex && c == `e` && s.pos + 1 < s.text.len {
|
2019-06-22 21:20:28 +03:00
|
|
|
next := s.text[s.pos + 1]
|
|
|
|
if next == `+` || next == `-` || next.is_digit() {
|
|
|
|
s.pos++
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if !c.is_digit() && c != `.` && !is_good_hex {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
// 1..9
|
2019-06-27 18:05:02 +03:00
|
|
|
if c == `.` && s.pos + 1 < s.text.len && s.text[s.pos + 1] == `.` {
|
2019-06-22 21:20:28 +03:00
|
|
|
break
|
|
|
|
}
|
|
|
|
if is_oct && c >= `8` && !is_float {
|
|
|
|
s.error('malformed octal constant')
|
|
|
|
}
|
|
|
|
}
|
|
|
|
number := s.text.substr(start, s.pos)
|
|
|
|
s.pos--
|
|
|
|
return number
|
|
|
|
}
|
|
|
|
|
2019-07-03 22:53:25 +03:00
|
|
|
fn (s Scanner) has_gone_over_line_end() bool {
|
|
|
|
mut i := s.pos-1
|
|
|
|
for i >= 0 && !is_white(s.text[i]) {
|
|
|
|
i--
|
|
|
|
}
|
|
|
|
for i >= 0 && is_white(s.text[i]) {
|
|
|
|
if is_nl(s.text[i]) {
|
|
|
|
return true
|
|
|
|
}
|
|
|
|
i--
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
2019-06-22 21:20:28 +03:00
|
|
|
fn (s mut Scanner) skip_whitespace() {
|
|
|
|
for s.pos < s.text.len && is_white(s.text[s.pos]) {
|
|
|
|
if is_nl(s.text[s.pos]) {
|
2019-07-05 23:07:44 +03:00
|
|
|
// Count \r\n as one line
|
2019-07-05 23:12:06 +03:00
|
|
|
if !(s.text[s.pos] == `\n` && s.pos > 0 && s.text[s.pos-1] == `\r`) {
|
2019-07-05 23:07:44 +03:00
|
|
|
s.line_nr++
|
|
|
|
}
|
2019-06-22 21:20:28 +03:00
|
|
|
if s.is_fmt {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
s.pos++
|
|
|
|
}
|
|
|
|
// if s.pos == s.text.len {
|
2019-07-07 23:30:15 +03:00
|
|
|
// return scan_res(.eof, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
// }
|
|
|
|
}
|
|
|
|
|
2019-06-26 20:07:58 +03:00
|
|
|
fn (s mut Scanner) get_var_name(pos int) string {
|
2019-06-28 15:19:46 +03:00
|
|
|
mut pos_start := pos
|
2019-06-26 20:07:58 +03:00
|
|
|
|
|
|
|
for ; pos_start >= 0 && s.text[pos_start] != `\n` && s.text[pos_start] != `;`; pos_start-- {}
|
|
|
|
pos_start++
|
2019-06-25 16:26:26 +03:00
|
|
|
return s.text.substr(pos_start, pos)
|
|
|
|
}
|
|
|
|
|
|
|
|
// CAO stands for Compound Assignment Operators (e.g '+=' )
|
|
|
|
fn (s mut Scanner) cao_change(operator string) {
|
2019-06-27 03:33:49 +03:00
|
|
|
s.text = s.text.substr(0, s.pos - operator.len) + ' = ' + s.get_var_name(s.pos - operator.len) + ' ' + operator + ' ' + s.text.substr(s.pos + 1, s.text.len)
|
2019-06-25 16:26:26 +03:00
|
|
|
}
|
|
|
|
|
2019-07-06 16:33:15 +03:00
|
|
|
fn (s mut Scanner) scan() ScanRes {
|
2019-06-22 21:20:28 +03:00
|
|
|
// if s.file_path == 'd.v' {
|
|
|
|
// println('\nscan()')
|
|
|
|
// }
|
2019-07-01 19:07:22 +03:00
|
|
|
if s.started {
|
2019-06-22 21:20:28 +03:00
|
|
|
// || (s.pos == 0 && s.text.len > 0 && s.text[s.pos] == `\n`) {
|
|
|
|
s.pos++
|
|
|
|
}
|
|
|
|
s.started = true
|
|
|
|
if s.pos >= s.text.len {
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.eof, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
|
|
|
// skip whitespace
|
|
|
|
if !s.inside_string {
|
|
|
|
s.skip_whitespace()
|
|
|
|
}
|
|
|
|
if s.is_fmt && s.text[s.pos] == `\n` {
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.nl, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
|
|
|
// End of $var, start next string
|
|
|
|
if !s.is_fmt && s.dollar_end {
|
|
|
|
// fmt.Println("end of $var, get string", s.pos, string(s.text[s.pos]))
|
|
|
|
if s.text[s.pos] == SINGLE_QUOTE {
|
|
|
|
// fmt.Println("ENDDD")
|
|
|
|
s.dollar_end = false
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.strtoken, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
|
|
|
s.dollar_end = false
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.strtoken, s.ident_string())
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
|
|
|
s.skip_whitespace()
|
|
|
|
// end of file
|
|
|
|
if s.pos >= s.text.len {
|
2019-07-07 23:30:15 +03:00
|
|
|
// println('scan(): returning .eof (pos >= len)')
|
|
|
|
return scan_res(.eof, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
|
|
|
// handle each char
|
|
|
|
c := s.text[s.pos]
|
|
|
|
mut nextc := `\0`
|
|
|
|
if s.pos + 1 < s.text.len {
|
|
|
|
nextc = s.text[s.pos + 1]
|
|
|
|
}
|
|
|
|
// name or keyword
|
|
|
|
if is_name_char(c) {
|
|
|
|
name := s.ident_name()
|
2019-06-27 18:05:02 +03:00
|
|
|
// tmp hack to detect . in ${}
|
2019-07-07 23:30:15 +03:00
|
|
|
// Check if not .eof to prevent panic
|
2019-06-27 18:05:02 +03:00
|
|
|
next_char := if s.pos + 1 < s.text.len { s.text[s.pos + 1] } else { `\0` }
|
2019-06-22 21:20:28 +03:00
|
|
|
// println('!!! got name=$name next_char=$next_char')
|
|
|
|
if is_key(name) {
|
|
|
|
// println('IS KEY')
|
|
|
|
// tok := (key_to_token(name))
|
|
|
|
// println(tok.str())
|
|
|
|
return scan_res(key_to_token(name), '')
|
|
|
|
}
|
|
|
|
// 'asdf $b' => "b" is the last name in the string, dont start parsing string
|
|
|
|
// at the next ', skip it
|
|
|
|
if s.inside_string {
|
|
|
|
// println('is_letter inside string! nextc=${nextc.str()}')
|
2019-06-27 18:05:02 +03:00
|
|
|
if next_char == SINGLE_QUOTE {
|
2019-06-22 21:20:28 +03:00
|
|
|
// println('var is last before QUOTE')
|
|
|
|
s.pos++
|
|
|
|
s.dollar_start = false
|
|
|
|
s.inside_string = false
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if s.dollar_start && next_char != `.` {
|
2019-07-07 23:30:15 +03:00
|
|
|
// println('INSIDE .strtoken .dollar var=$name')
|
2019-06-22 21:20:28 +03:00
|
|
|
s.dollar_end = true
|
|
|
|
s.dollar_start = false
|
|
|
|
}
|
2019-06-28 17:15:28 +03:00
|
|
|
if s.pos == 0 && next_char == ` ` {
|
|
|
|
s.pos++
|
|
|
|
//If a single letter name at the start of the file, increment
|
|
|
|
//Otherwise the scanner would be stuck at s.pos = 0
|
|
|
|
}
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.name, name)
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
|
|
|
// number, `.123`
|
|
|
|
else if c.is_digit() || c == `.` && nextc.is_digit() {
|
|
|
|
num := s.ident_number()
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.integer, num)
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
|
|
|
// all other tokens
|
|
|
|
switch c {
|
|
|
|
case `+`:
|
|
|
|
if nextc == `+` {
|
|
|
|
s.pos++
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.inc, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
|
|
|
else if nextc == `=` {
|
|
|
|
s.pos++
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.plus_assign, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.plus, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
case `-`:
|
|
|
|
if nextc == `-` {
|
|
|
|
s.pos++
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.dec, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
|
|
|
else if nextc == `=` {
|
|
|
|
s.pos++
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.minus_assign, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.minus, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
case `*`:
|
|
|
|
if nextc == `=` {
|
|
|
|
s.pos++
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.mult_assign, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.mul, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
case `^`:
|
|
|
|
if nextc == `=` {
|
|
|
|
s.pos++
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.xor_assign, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.xor, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
case `%`:
|
|
|
|
if nextc == `=` {
|
|
|
|
s.pos++
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.mod_assign, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.mod, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
case `?`:
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.question, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
case SINGLE_QUOTE:
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.strtoken, s.ident_string())
|
2019-06-22 21:20:28 +03:00
|
|
|
// TODO allow double quotes
|
|
|
|
// case QUOTE:
|
2019-07-07 23:30:15 +03:00
|
|
|
// return scan_res(.strtoken, s.ident_string())
|
2019-06-22 21:20:28 +03:00
|
|
|
case `\``:
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.chartoken, s.ident_char())
|
2019-06-22 21:20:28 +03:00
|
|
|
case `(`:
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.lpar, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
case `)`:
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.rpar, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
case `[`:
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.lsbr, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
case `]`:
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.rsbr, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
case `{`:
|
|
|
|
// Skip { in ${ in strings
|
|
|
|
if s.inside_string {
|
|
|
|
return s.scan()
|
|
|
|
}
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.lcbr, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
case `$`:
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.dollar, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
case `}`:
|
2019-07-10 10:48:10 +03:00
|
|
|
// s = `hello $name !`
|
|
|
|
// s = `hello ${name} !`
|
2019-06-22 21:20:28 +03:00
|
|
|
if s.inside_string {
|
|
|
|
s.pos++
|
2019-07-07 23:30:15 +03:00
|
|
|
// TODO UN.neEDED?
|
2019-06-22 21:20:28 +03:00
|
|
|
if s.text[s.pos] == SINGLE_QUOTE {
|
|
|
|
s.inside_string = false
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.strtoken, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.strtoken, s.ident_string())
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
|
|
|
else {
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.rcbr, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
|
|
|
case `&`:
|
|
|
|
if nextc == `=` {
|
|
|
|
s.pos++
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.and_assign, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
2019-06-27 18:05:02 +03:00
|
|
|
if nextc == `&` {
|
2019-06-22 21:20:28 +03:00
|
|
|
s.pos++
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.and, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.amp, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
case `|`:
|
2019-06-27 18:05:02 +03:00
|
|
|
if nextc == `|` {
|
2019-06-22 21:20:28 +03:00
|
|
|
s.pos++
|
2019-07-14 12:01:32 +03:00
|
|
|
return scan_res(.logical_or, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
|
|
|
if nextc == `=` {
|
|
|
|
s.pos++
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.or_assign, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.pipe, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
case `,`:
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.comma, '')
|
2019-06-23 06:21:22 +03:00
|
|
|
case `\r`:
|
|
|
|
if nextc == `\n` {
|
|
|
|
s.pos++
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.nl, '')
|
2019-06-23 06:21:22 +03:00
|
|
|
}
|
2019-06-22 21:20:28 +03:00
|
|
|
case `\n`:
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.nl, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
case `.`:
|
2019-06-27 18:05:02 +03:00
|
|
|
if nextc == `.` {
|
2019-06-22 21:20:28 +03:00
|
|
|
s.pos++
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.dotdot, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.dot, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
case `#`:
|
|
|
|
start := s.pos + 1
|
2019-06-27 18:05:02 +03:00
|
|
|
for s.pos < s.text.len && s.text[s.pos] != `\n` {
|
2019-06-22 21:20:28 +03:00
|
|
|
s.pos++
|
|
|
|
}
|
|
|
|
s.line_nr++
|
|
|
|
hash := s.text.substr(start, s.pos)
|
|
|
|
if s.is_fmt {
|
|
|
|
// fmt needs NL after #
|
|
|
|
s.pos--
|
|
|
|
}
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.hash, hash.trim_space())
|
2019-06-22 21:20:28 +03:00
|
|
|
case `>`:
|
2019-06-27 18:05:02 +03:00
|
|
|
if nextc == `=` {
|
2019-06-22 21:20:28 +03:00
|
|
|
s.pos++
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.ge, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
2019-06-27 18:05:02 +03:00
|
|
|
else if nextc == `>` {
|
|
|
|
if s.pos + 2 < s.text.len && s.text[s.pos + 2] == `=` {
|
2019-06-22 21:20:28 +03:00
|
|
|
s.pos += 2
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.righ_shift_assign, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
|
|
|
s.pos++
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.righ_shift, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
|
|
|
else {
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.gt, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
|
|
|
case `<`:
|
2019-06-27 18:05:02 +03:00
|
|
|
if nextc == `=` {
|
2019-06-22 21:20:28 +03:00
|
|
|
s.pos++
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.le, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
2019-06-27 18:05:02 +03:00
|
|
|
else if nextc == `<` {
|
|
|
|
if s.pos + 2 < s.text.len && s.text[s.pos + 2] == `=` {
|
2019-06-22 21:20:28 +03:00
|
|
|
s.pos += 2
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.left_shift_assign, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
|
|
|
s.pos++
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.left_shift, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
|
|
|
else {
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.lt, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
|
|
|
case `=`:
|
2019-06-27 18:05:02 +03:00
|
|
|
if nextc == `=` {
|
2019-06-22 21:20:28 +03:00
|
|
|
s.pos++
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.eq, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
|
|
|
else {
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.assign, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
|
|
|
case `:`:
|
2019-06-27 18:05:02 +03:00
|
|
|
if nextc == `=` {
|
2019-06-22 21:20:28 +03:00
|
|
|
s.pos++
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.decl_assign, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
|
|
|
else {
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.colon, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
|
|
|
case `;`:
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.semicolon, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
case `!`:
|
2019-06-27 18:05:02 +03:00
|
|
|
if nextc == `=` {
|
2019-06-22 21:20:28 +03:00
|
|
|
s.pos++
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.ne, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
|
|
|
else {
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.not, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
|
|
|
case `~`:
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.bit_not, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
case `/`:
|
|
|
|
if nextc == `=` {
|
|
|
|
s.pos++
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.div_assign, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
2019-06-27 18:05:02 +03:00
|
|
|
if nextc == `/` {
|
2019-07-07 23:30:15 +03:00
|
|
|
// debug("!!!!!!.key_goT LI.ne COM")
|
2019-06-22 21:20:28 +03:00
|
|
|
start := s.pos + 1
|
2019-06-27 18:05:02 +03:00
|
|
|
for s.pos < s.text.len && s.text[s.pos] != `\n`{
|
2019-06-22 21:20:28 +03:00
|
|
|
s.pos++
|
|
|
|
}
|
|
|
|
s.line_nr++
|
|
|
|
s.line_comment = s.text.substr(start + 1, s.pos)
|
|
|
|
s.line_comment = s.line_comment.trim_space()
|
|
|
|
s.fgenln('// $s.line_comment')
|
|
|
|
if s.is_fmt {
|
|
|
|
// fmt needs NL after comment
|
|
|
|
s.pos--
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
// Skip comment
|
|
|
|
return s.scan()
|
|
|
|
}
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.line_com, s.line_comment)
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
|
|
|
// Multiline comments
|
2019-06-27 18:05:02 +03:00
|
|
|
if nextc == `*` {
|
2019-06-22 21:20:28 +03:00
|
|
|
start := s.pos
|
2019-07-02 02:49:15 +03:00
|
|
|
mut nest_count := 1
|
2019-06-22 21:20:28 +03:00
|
|
|
// Skip comment
|
2019-07-02 02:49:15 +03:00
|
|
|
for nest_count > 0 {
|
2019-06-22 21:20:28 +03:00
|
|
|
s.pos++
|
|
|
|
if s.pos >= s.text.len {
|
|
|
|
s.line_nr--
|
|
|
|
s.error('comment not terminated')
|
|
|
|
}
|
|
|
|
if s.text[s.pos] == `\n` {
|
|
|
|
s.line_nr++
|
2019-07-02 02:49:15 +03:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
if s.text[s.pos] == `/` && s.text[s.pos + 1] == `*` {
|
|
|
|
nest_count++
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if s.text[s.pos] == `*` && s.text[s.pos + 1] == `/` {
|
|
|
|
nest_count--
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
s.pos++
|
|
|
|
end := s.pos + 1
|
|
|
|
comm := s.text.substr(start, end)
|
|
|
|
s.fgenln(comm)
|
|
|
|
if s.is_fmt {
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.mline_com, comm)
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
|
|
|
// Skip if not in fmt mode
|
|
|
|
return s.scan()
|
|
|
|
}
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.div, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
2019-06-29 18:58:20 +03:00
|
|
|
$if windows {
|
|
|
|
if c == `\0` {
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.eof, '')
|
2019-06-29 18:58:20 +03:00
|
|
|
}
|
|
|
|
}
|
2019-06-23 11:01:55 +03:00
|
|
|
println('(char code=$c) pos=$s.pos len=$s.text.len')
|
2019-07-03 14:20:43 +03:00
|
|
|
mut msg := 'invalid character `${c.str()}`'
|
|
|
|
if c == `"` {
|
|
|
|
msg += ', use \' to denote strings'
|
|
|
|
}
|
|
|
|
s.error(msg)
|
2019-07-07 23:30:15 +03:00
|
|
|
return scan_res(.eof, '')
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
fn (s &Scanner) error(msg string) {
|
|
|
|
file := s.file_path.all_after('/')
|
2019-06-23 11:01:55 +03:00
|
|
|
println('panic: $file:${s.line_nr + 1}')
|
|
|
|
println(msg)
|
2019-06-22 21:20:28 +03:00
|
|
|
// os.print_backtrace()
|
|
|
|
// println(file)
|
|
|
|
// println(s.file_path)
|
2019-06-23 11:12:09 +03:00
|
|
|
exit(1)
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
|
|
|
|
2019-06-23 11:01:55 +03:00
|
|
|
// println('array out of bounds $idx len=$a.len')
|
2019-06-22 21:20:28 +03:00
|
|
|
// This is really bad. It needs a major clean up
|
|
|
|
fn (s mut Scanner) ident_string() string {
|
|
|
|
// println("\nidentString() at char=", string(s.text[s.pos]),
|
|
|
|
// "chard=", s.text[s.pos], " pos=", s.pos, "txt=", s.text[s.pos:s.pos+7])
|
|
|
|
debug := s.file_path.contains('test_test')
|
|
|
|
if debug {
|
|
|
|
println('identStr() $s.file_path line=$s.line_nr pos=$s.pos')
|
|
|
|
}
|
|
|
|
mut start := s.pos
|
|
|
|
s.inside_string = false
|
|
|
|
slash := `\\`
|
|
|
|
for {
|
|
|
|
s.pos++
|
|
|
|
if s.pos >= s.text.len {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
c := s.text[s.pos]
|
|
|
|
if debug {
|
|
|
|
println(c.str())
|
|
|
|
}
|
|
|
|
prevc := s.text[s.pos - 1]
|
|
|
|
// end of string
|
|
|
|
if c == SINGLE_QUOTE && (prevc != slash || (prevc == slash && s.text[s.pos - 2] == slash)) {
|
|
|
|
// handle '123\\' slash at the end
|
|
|
|
break
|
|
|
|
}
|
|
|
|
if c == `\n` {
|
|
|
|
s.line_nr++
|
|
|
|
}
|
|
|
|
// Don't allow \0
|
|
|
|
if c == `0` && s.pos > 2 && s.text[s.pos - 1] == `\\` {
|
|
|
|
s.error('0 character in a string literal')
|
|
|
|
}
|
|
|
|
// Don't allow \x00
|
|
|
|
if c == `0` && s.pos > 5 && s.text[s.pos - 1] == `0` && s.text[s.pos - 2] == `x` &&
|
|
|
|
s.text[s.pos - 3] == `\\` {
|
|
|
|
s.error('0 character in a string literal')
|
|
|
|
}
|
|
|
|
// ${var}
|
|
|
|
if !s.is_fmt && c == `{` && prevc == `$` {
|
|
|
|
s.inside_string = true
|
|
|
|
// fmt.Println("breaking out of is()")
|
|
|
|
// so that s.pos points to $ at the next step
|
|
|
|
s.pos -= 2
|
|
|
|
// fmt.Println("break pos=", s.pos, "c=", string(s.text[s.pos]), "d=", s.text[s.pos])
|
|
|
|
break
|
|
|
|
}
|
|
|
|
// $var
|
|
|
|
// if !s.is_fmt && c != `{` && c != ` ` && ! (c >= `0` && c <= `9`) && prevc == `$` {
|
|
|
|
if !s.is_fmt && (c.is_letter() || c == `_`) && prevc == `$` {
|
|
|
|
s.inside_string = true
|
|
|
|
s.dollar_start = true
|
|
|
|
// println('setting s.dollar=true pos=$s.pos')
|
|
|
|
s.pos -= 2
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
mut lit := ''
|
|
|
|
if s.text[start] == SINGLE_QUOTE {
|
|
|
|
start++
|
|
|
|
}
|
|
|
|
mut end := s.pos
|
|
|
|
if s.inside_string {
|
|
|
|
end++
|
|
|
|
}
|
|
|
|
if start > s.pos{}
|
|
|
|
else {
|
|
|
|
lit = s.text.substr(start, end)
|
|
|
|
}
|
|
|
|
// if lit.contains('\n') {
|
|
|
|
// println('\nstring lit="$lit" pos=$s.pos line=$s.line_nr')
|
|
|
|
// }
|
|
|
|
/*
|
|
|
|
for c in lit {
|
|
|
|
if s.file_path.contains('range_test') {
|
|
|
|
println('!')
|
|
|
|
println(c)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
*/
|
|
|
|
return lit
|
|
|
|
}
|
|
|
|
|
|
|
|
fn (s mut Scanner) ident_char() string {
|
|
|
|
start := s.pos
|
|
|
|
slash := `\\`
|
|
|
|
mut len := 0
|
|
|
|
for {
|
|
|
|
s.pos++
|
|
|
|
if s.pos >= s.text.len {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
if s.text[s.pos] != slash {
|
|
|
|
len++
|
|
|
|
}
|
|
|
|
double_slash := s.text[s.pos - 1] == slash && s.text[s.pos - 2] == slash
|
|
|
|
if s.text[s.pos] == `\`` && (s.text[s.pos - 1] != slash || double_slash) {
|
|
|
|
if double_slash {
|
|
|
|
len++
|
|
|
|
}
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
len--
|
|
|
|
c := s.text.substr(start + 1, s.pos)
|
|
|
|
if len != 1 {
|
2019-07-07 22:46:21 +03:00
|
|
|
u := c.ustring()
|
|
|
|
if u.len != 1 {
|
2019-06-22 21:20:28 +03:00
|
|
|
s.error('invalid character literal (more than one character: $len)')
|
2019-07-07 22:46:21 +03:00
|
|
|
}
|
2019-06-22 21:20:28 +03:00
|
|
|
}
|
|
|
|
return c
|
|
|
|
}
|
|
|
|
|
|
|
|
fn (p mut Parser) peek() Token {
|
|
|
|
for {
|
|
|
|
tok := p.scanner.peek()
|
2019-07-07 23:30:15 +03:00
|
|
|
if tok != .nl {
|
2019-06-22 21:20:28 +03:00
|
|
|
return tok
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
fn (s mut Scanner) peek() Token {
|
|
|
|
pos := s.pos
|
|
|
|
line := s.line_nr
|
|
|
|
inside_string := s.inside_string
|
|
|
|
dollar_start := s.dollar_start
|
|
|
|
dollar_end := s.dollar_end
|
|
|
|
// /////
|
|
|
|
res := s.scan()
|
|
|
|
tok := res.tok
|
|
|
|
s.pos = pos
|
|
|
|
s.line_nr = line
|
|
|
|
s.inside_string = inside_string
|
|
|
|
s.dollar_start = dollar_start
|
|
|
|
s.dollar_end = dollar_end
|
|
|
|
return tok
|
|
|
|
}
|
|
|
|
|
|
|
|
fn (s mut Scanner) debug_tokens() {
|
|
|
|
s.pos = 0
|
|
|
|
fname := s.file_path.all_after('/')
|
|
|
|
println('\n===DEBUG TOKENS $fname ============')
|
|
|
|
// allToks := ''
|
|
|
|
s.debug = true
|
|
|
|
for {
|
|
|
|
res := s.scan()
|
|
|
|
tok := res.tok
|
|
|
|
lit := res.lit
|
|
|
|
// printiln(tok)
|
|
|
|
print(tok.str())
|
|
|
|
// allToks += tok.String()
|
|
|
|
if lit != '' {
|
|
|
|
println(' `$lit`')
|
|
|
|
// allToks += " `" + lit + "`"
|
|
|
|
}
|
|
|
|
else {
|
|
|
|
println('')
|
|
|
|
}
|
|
|
|
// allToks += "\n"
|
2019-07-07 23:30:15 +03:00
|
|
|
if tok == .eof {
|
2019-06-22 21:20:28 +03:00
|
|
|
println('============ END OF DEBUG TOKENS ==================')
|
|
|
|
// fmt.Println("========"+s.file+"========\n", allToks)
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
fn is_name_char(c byte) bool {
|
|
|
|
return c.is_letter() || c == `_`
|
|
|
|
}
|
|
|
|
|
2019-06-27 01:12:55 +03:00
|
|
|
fn (s mut Scanner) get_opening_bracket() int {
|
|
|
|
mut pos := s.pos
|
|
|
|
mut parentheses := 0
|
|
|
|
mut inside_string := false
|
|
|
|
|
|
|
|
for pos > 0 && s.text[pos] != `\n` {
|
|
|
|
if s.text[pos] == `)` && !inside_string {
|
|
|
|
parentheses++
|
|
|
|
}
|
|
|
|
if s.text[pos] == `(` && !inside_string {
|
|
|
|
parentheses--
|
|
|
|
}
|
|
|
|
if s.text[pos] == `\'` && s.text[pos - 1] != `\\` && s.text[pos - 1] != `\`` {
|
|
|
|
inside_string = !inside_string
|
|
|
|
}
|
|
|
|
if parentheses == 0 {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
pos--
|
|
|
|
}
|
|
|
|
return pos
|
|
|
|
}
|
|
|
|
|
2019-06-27 02:55:37 +03:00
|
|
|
// Foo { bar: 3, baz: 'hi' } => '{ bar: 3, baz: "hi" }'
|
2019-06-27 01:12:55 +03:00
|
|
|
fn (s mut Scanner) create_type_string(T Type, name string) {
|
|
|
|
line := s.line_nr
|
|
|
|
inside_string := s.inside_string
|
|
|
|
mut newtext := '\'{ '
|
|
|
|
start := s.get_opening_bracket() + 1
|
|
|
|
end := s.pos
|
|
|
|
for i, field in T.fields {
|
|
|
|
if i != 0 {
|
|
|
|
newtext += ', '
|
|
|
|
}
|
|
|
|
newtext += '$field.name: ' + '$${name}.${field.name}'
|
|
|
|
}
|
|
|
|
newtext += ' }\''
|
|
|
|
s.text = s.text.substr(0, start) + newtext + s.text.substr(end, s.text.len)
|
|
|
|
s.pos = start - 2
|
|
|
|
s.line_nr = line
|
|
|
|
s.inside_string = inside_string
|
|
|
|
}
|
|
|
|
|
|
|
|
fn (p mut Parser) create_type_string(T Type, name string) {
|
|
|
|
p.scanner.create_type_string(T, name)
|
2019-06-27 02:55:37 +03:00
|
|
|
}
|