compiler2: start implementing pratt style parser

vlib/compiler2/ast/ast.v

@@ -10,12 +10,13 @@ import (
 
 struct Foo {}
 
-pub type Expr = Foo | IfExpr | BinaryExpr | IntegerExpr
+// pub type Expr = Foo | IfExpr | BinaryExpr | IntegerExpr
+pub type Expr = Foo | IfExpr | BinaryExpr | ScalarExpr | UnaryExpr
 
-pub struct IntegerExpr {
+// pub struct IntegerExpr {
-pub:
+// pub:
-	val int
+// 	val int
-}
+// }
 
 /*
 pub enum Expr {
@@ -45,6 +46,24 @@ pub:
 	right Expr
 }
 
+pub struct ScalarExpr {
+pub:
+	token token.Token
+	// op    BinaryOp
+	// op    token.Token
+	typ   token.Token
+	val   string
+	left  Expr
+}
+
+pub struct UnaryExpr {
+pub:
+	// token token.Token
+	//op    BinaryOp
+	op    token.Token
+	left  Expr
+}
+
 struct IfExpr {
 	token token.Token
 	cond  Expr
@@ -57,29 +76,44 @@ struct ReturnStmt {
 	results []Expr
 }
 
-enum BinaryOp {
+// string representaiton of expr
-	sum
+pub fn (x Expr) str() string {
-	difference
+	match x {
-	product
+		BinaryExpr {
-	quotient
+			return '(${it.left.str()}$it.op.str()${it.right.str()})'
-	remainder
+		}
-	bitwise_and
+		ScalarExpr {
-	bitwise_or
+			return '${it.left.str()}$it.val'
-	bitwise_xor
+		}
-	left_shift
+		UnaryExpr {
-	right_shift
+			return '${it.left.str()}$it.op.str()'
-
+		}
-	equality
+		else { return '' }
-	inequality
+	}
-	less_than
-	less_than_or_equal
-	more_than
-	more_than_or_equal
-
-	in_check
-
-	//These are suffixed with `bool` to prevent conflict with the keyword `or`
-	and_bool
-	or_bool
 }
 
+// enum BinaryOp {
+// 	sum
+// 	difference
+// 	product
+// 	quotient
+// 	remainder
+// 	bitwise_and
+// 	bitwise_or
+// 	bitwise_xor
+// 	left_shift
+// 	right_shift
+
+// 	equality
+// 	inequality
+// 	less_than
+// 	less_than_or_equal
+// 	more_than
+// 	more_than_or_equal
+
+// 	in_check
+
+// 	//These are suffixed with `bool` to prevent conflict with the keyword `or`
+// 	and_bool
+// 	or_bool
+// }

vlib/compiler2/parser/parser.v

@@ -24,7 +24,8 @@ pub fn parse_expr(text string) ast.Expr {
 		tok: res.tok
 		lit: res.lit
 	}
-	return p.expr()
+	// return p.expr()
+	return p.expr(token.lowest_prec)
 }
 
 fn (p mut Parser) next() {
@@ -34,48 +35,43 @@ fn (p mut Parser) next() {
 	p.lit = res.lit
 }
 
-fn (p mut Parser) expr() ast.Expr {
+// Implementation of Pratt Precedence
-	//println('\n\nexpr()')
+pub fn (p mut Parser) expr(rbp int) ast.Expr {
-	mut node := p.term()
+	// null denotation (prefix)
-	for p.tok == .plus || p.tok == .minus {
+	tok := p.tok
-		op := p.tok
+	lit := p.lit
 	p.next()
-		node = ast.BinaryExpr {
+	mut left := ast.Expr{}
-			left: node
+	match tok {
-			op: op
+		.lpar {
-			right: p.term()
+			left = p.expr(0)
+			if p.tok != .rpar {
+				panic("Parse Error: expected )")
 			}
-	}
-	return node
-}
-
-fn (p mut Parser) term() ast.Expr {
-	mut node := p.factor()
-	for p.tok == .mul || p.tok == .div || p.tok == .mod {
-		op := p.tok
 			p.next()
-		node = ast.BinaryExpr {
+		}
-			left: node
+		else {
-			op: op
+			// TODO: fix bug. note odd conditon instead of else if (same below)
-			right: p.factor()
+			if tok.is_scalar() {
+				left = ast.ScalarExpr{val: lit, typ: tok}
+			}
+			if !tok.is_scalar() && tok.is_unary() {
+				left = ast.UnaryExpr{left: p.expr(token.highest_prec), op: tok}
 			}
 		}
-	return node
-	//return ast.BinaryExpr{}
-	//return ast.Expr.Binary(ast.BinaryExpr{})
 	}
 
-fn (p mut Parser) factor() ast.Expr {
+	// left binding power
-	if p.tok == .number {
+	for rbp < p.tok.precedence() {
-		val := p.lit.int()
+		tok2 := p.tok
 		p.next()
-		return ast.IntegerExpr { val: val }
+		// left denotation (infix)
-	} else {
+		if tok2.is_right_assoc() {
-		println('bad factor token')
+			left = ast.BinaryExpr{left: left, op: tok2, right: p.expr(tok2.precedence() - 1)}
-		println(p.tok)
+		}
-		exit(1)
+		if !tok2.is_right_assoc() && tok2.is_left_assoc() {
+			left = ast.BinaryExpr{left: left, op: tok2, right: p.expr(tok2.precedence())}
 		}
 	}
-
+	return left
-
+}
-

vlib/compiler2/parser/parser_test.v

@@ -5,19 +5,37 @@ import compiler2.ast
 fn test_parser() {
 	//expr := ast.IntegerExpr {val:10}
 	//expr := ast.BinaryExpr{}
+
+	// print using walk
 	expr := parse_expr('3 + 7')
 	walk(expr)
-	println('')
+	println('\n')
+	
+	text_expr := [
+		'4 + 4',
+		'1 + 2 * 5',
+		'(2 * 3) / 2',
+		'3 + (7 * 6)',
+		'2 ^ 8 * (7 * 6)',
+		'(2) + (17*2-30) * (5)+2 - (8/2)*4'
+	]
+	for s in text_expr {
+		// print using str method
+		x := parse_expr(s)
+		println('source: $s')
+		println('parsed: $x')
+		println('===================')
 	}
+}
+
 
 fn walk(node ast.Expr) {
 	//println('walk()')
 	match node {
-		ast.IntegerExpr {
-			print(it.val)
-		}
 		ast.BinaryExpr {
+			print(' (')
 			walk(it.left)
+			// print('$it.op.str()')
 			match it.op {
 				.plus {
 					print(' + ')
@@ -29,6 +47,15 @@ fn walk(node ast.Expr) {
 
 			}
 			walk(it.right)
+			print(') ')
+		}
+		ast.ScalarExpr {
+			walk(it.left)
+			print(' $it.val ')
+		}
+		ast.UnaryExpr {
+			walk(it.left)
+			print(' $it.op ')
 		}
 		else { }
 	}

vlib/compiler2/token/token.v

@@ -303,3 +303,52 @@ pub fn (t Token) str() string {
 	return token_str[int(t)]
 }
 
+
+// Representation of highest and lowest precedence
+const (
+	lowest_prec  = 0
+	highest_prec = 7
+)
+
+// Precedence returns a tokens precedence if defined, otherwise lowest_prec
+pub fn (tok Token) precedence() int {
+	match tok {
+		.plus, .minus { return 4 }
+		.mul, .div { return 4 }
+		.xor { return 6 }
+		.mod {return 7 }
+		else { return lowest_prec }
+	}
+}
+
+// is_scalar returns true if the token is a scalar
+pub fn (tok Token) is_scalar() bool {
+	match tok {
+		.number { return true }
+		else { return false }
+	}
+}
+
+// is_unary returns true if the token can be in a unary expression
+pub fn (tok Token) is_unary() bool {
+	match tok {
+		.plus, .minus { return true }
+		else { return false }
+	}
+}
+
+// is_left_assoc returns true if the token is left associative
+pub fn (tok Token) is_left_assoc() bool {
+	match tok {
+		.number, .plus, .minus, .mul, .div, .mod { return true }
+		else { return false }
+	}
+}
+
+// is_right_assoc returns true if the token is right associative
+pub fn (tok Token) is_right_assoc() bool {
+	match tok {
+		.xor { return true }
+		else { return false }
+	}
+}