transformer: refactor + apply transform to (hopefully) all nodes (#13216)

vlib/v/ast/ast.v

@@ -151,10 +151,10 @@ pub:
 // Stand-alone expression in a statement list.
 pub struct ExprStmt {
 pub:
-	expr     Expr
 	pos      token.Position
 	comments []Comment
 pub mut:
+	expr    Expr
 	is_expr bool
 	typ     Type
 }
@@ -290,11 +290,11 @@ pub struct ConstField {
 pub:
 	mod         string
 	name        string
-	expr        Expr // the value expr of field; everything after `=`
 	is_pub      bool
 	is_markused bool // an explict `[markused]` tag; the const will NOT be removed by `-skip-unused`, no matter what
 	pos         token.Position
 pub mut:
+	expr     Expr      // the value expr of field; everything after `=`
 	typ      Type      // the type of the const field, it can be any type in V
 	comments []Comment // comments before current const field
 	// the comptime_expr_value field is filled by the checker, when it has enough
@@ -387,11 +387,11 @@ pub mut:
 
 pub struct StructInitEmbed {
 pub:
-	expr          Expr
 	pos           token.Position
 	comments      []Comment
 	next_comments []Comment
 pub mut:
+	expr          Expr
 	name          string
 	typ           Type
 	expected_type Type
@@ -792,9 +792,9 @@ pub mut:
 pub struct PostfixExpr {
 pub:
 	op  token.Kind
-	expr Expr
 	pos token.Position
 pub mut:
+	expr        Expr
 	auto_locked string
 }
 
@@ -813,9 +813,9 @@ pub mut:
 pub struct IndexExpr {
 pub:
 	pos token.Position
+pub mut:
 	index     Expr // [0], RangeExpr [start..end] or map[key]
 	or_expr   OrExpr
-pub mut:
 	left      Expr
 	left_type Type // array, map, fixed array
 	is_setter bool
@@ -831,10 +831,10 @@ pub struct IfExpr {
 pub:
 	is_comptime   bool
 	tok_kind      token.Kind
-	left          Expr // `a` in `a := if ...`
 	pos           token.Position
 	post_comments []Comment
 pub mut:
+	left     Expr       // `a` in `a := if ...`
 	branches []IfBranch // includes all `else if` branches
 	is_expr  bool
 	typ      Type
@@ -844,11 +844,11 @@ pub mut:
 
 pub struct IfBranch {
 pub:
-	cond     Expr
 	pos      token.Position
 	body_pos token.Position
 	comments []Comment
 pub mut:
+	cond      Expr
 	pkg_exist bool
 	stmts     []Stmt
 	scope     &Scope
@@ -856,16 +856,17 @@ pub mut:
 
 pub struct UnsafeExpr {
 pub:
-	expr Expr
 	pos token.Position
+pub mut:
+	expr Expr
 }
 
 pub struct LockExpr {
 pub:
-	stmts    []Stmt
 	is_rlock []bool
 	pos      token.Position
 pub mut:
+	stmts    []Stmt
 	lockeds  []Expr // `x`, `y.z` in `lock x, y.z {`
 	comments []Comment
 	is_expr  bool
@@ -876,11 +877,11 @@ pub mut:
 pub struct MatchExpr {
 pub:
 	tok_kind token.Kind
-	cond     Expr
-	branches []MatchBranch
 	pos      token.Position
 	comments []Comment // comments before the first branch
 pub mut:
+	cond          Expr
+	branches      []MatchBranch
 	is_expr       bool // returns a value
 	return_type   Type
 	cond_type     Type // type of `x` in `match x {`
@@ -913,13 +914,14 @@ pub mut:
 
 pub struct SelectBranch {
 pub:
-	stmt          Stmt   // `a := <-ch` or `ch <- a`
-	stmts         []Stmt // right side
 	pos           token.Position
 	comment       Comment // comment above `select {`
 	is_else       bool
 	is_timeout    bool
 	post_comments []Comment
+pub mut:
+	stmt  Stmt   // `a := <-ch` or `ch <- a`
+	stmts []Stmt // right side
 }
 
 pub enum ComptimeForKind {
@@ -942,11 +944,11 @@ pub mut:
 
 pub struct ForStmt {
 pub:
-	cond   Expr
-	stmts  []Stmt
 	is_inf bool // `for {}`
 	pos    token.Position
 pub mut:
+	cond  Expr
+	stmts []Stmt
 	label string // `label: for {`
 	scope &Scope
 }
@@ -974,16 +976,16 @@ pub mut:
 
 pub struct ForCStmt {
 pub:
-	init     Stmt // i := 0;
 	has_init bool
-	cond     Expr // i < 10;
 	has_cond bool
-	inc      Stmt // i++; i += 2
 	has_inc  bool
 	is_multi bool // for a,b := 0,1; a < 10; a,b = a+b, a {...}
-	stmts    []Stmt
 	pos      token.Position
 pub mut:
+	init  Stmt // i := 0;
+	cond  Expr // i < 10;
+	inc   Stmt // i++; i += 2
+	stmts []Stmt
 	label string // `label: for {`
 	scope &Scope
 }
@@ -1031,10 +1033,10 @@ pub mut:
 // `expr as Ident`
 pub struct AsCast {
 pub:
-	expr Expr // from expr: `expr` in `expr as Ident`
 	typ Type // to type
 	pos token.Position
 pub mut:
+	expr      Expr // from expr: `expr` in `expr as Ident`
 	expr_type Type // from type
 }
 
@@ -1124,8 +1126,9 @@ pub mut:
 // `(3+4)`
 pub struct ParExpr {
 pub:
-	expr Expr
 	pos token.Position
+pub mut:
+	expr Expr
 }
 
 pub struct GoExpr {
@@ -1152,20 +1155,20 @@ pub struct ArrayInit {
 pub:
 	pos           token.Position // `[]` in []Type{} position
 	elem_type_pos token.Position // `Type` in []Type{} position
-	exprs         []Expr      // `[expr, expr]` or `[expr]Type{}` for fixed array
 	ecmnts        [][]Comment    // optional iembed comments after each expr
 	pre_cmnts     []Comment
 	is_fixed      bool
 	has_val       bool // fixed size literal `[expr, expr]!`
 	mod           string
-	len_expr      Expr // len: expr
-	cap_expr      Expr // cap: expr
-	default_expr  Expr // init: expr
 	has_len       bool
 	has_cap       bool
 	has_default   bool
 	has_it        bool // true if temp variable it is used
 pub mut:
+	exprs        []Expr // `[expr, expr]` or `[expr]Type{}` for fixed array
+	len_expr     Expr   // len: expr
+	cap_expr     Expr   // cap: expr
+	default_expr Expr   // init: expr
 	expr_types   []Type // [Dog, Cat] // also used for interface_types
 	elem_type    Type   // element type
 	default_type Type   // default value type
@@ -1174,9 +1177,9 @@ pub mut:
 
 pub struct ArrayDecompose {
 pub:
-	expr Expr
 	pos token.Position
 pub mut:
+	expr      Expr
 	expr_type Type
 	arg_type  Type
 }
@@ -1184,9 +1187,9 @@ pub mut:
 pub struct ChanInit {
 pub:
 	pos     token.Position
-	cap_expr Expr
 	has_cap bool
 pub mut:
+	cap_expr  Expr
 	typ       Type
 	elem_type Type
 }
@@ -1194,11 +1197,11 @@ pub mut:
 pub struct MapInit {
 pub:
 	pos       token.Position
-	keys      []Expr
-	vals      []Expr
 	comments  [][]Comment // comments after key-value pairs
 	pre_cmnts []Comment   // comments before the first key-value pair
 pub mut:
+	keys       []Expr
+	vals       []Expr
 	typ        Type
 	key_type   Type
 	value_type Type
@@ -1207,18 +1210,18 @@ pub mut:
 // s[10..20]
 pub struct RangeExpr {
 pub:
-	low      Expr
-	high     Expr
 	has_high bool
 	has_low  bool
 	pos      token.Position
 	is_gated bool // #[] gated array
+pub mut:
+	low  Expr
+	high Expr
 }
 
 pub struct CastExpr {
-pub:
-	arg Expr // `n` in `string(buf, n)`
 pub mut:
+	arg       Expr   // `n` in `string(buf, n)`
 	typ       Type   // `string`
 	expr      Expr   // `buf` in `string(buf, n)` and `&Type(buf)`
 	typname   string // `&Type` in `&Type(buf)`
@@ -1457,9 +1460,9 @@ pub struct Assoc {
 pub:
 	var_name string
 	fields   []string
-	exprs    []Expr
 	pos      token.Position
 pub mut:
+	exprs []Expr
 	typ   Type
 	scope &Scope
 }
@@ -1467,18 +1470,18 @@ pub mut:
 pub struct SizeOf {
 pub:
 	is_type bool
-	expr    Expr // checker uses this to set typ
 	pos     token.Position
 pub mut:
+	expr Expr // checker uses this to set typ
 	typ  Type
 }
 
 pub struct IsRefType {
 pub:
 	is_type bool
-	expr    Expr // checker uses this to set typ
 	pos     token.Position
 pub mut:
+	expr Expr // checker uses this to set typ
 	typ  Type
 }
 
@@ -1491,24 +1494,25 @@ pub:
 
 pub struct Likely {
 pub:
-	expr      Expr
 	pos       token.Position
 	is_likely bool // false for _unlikely_
+pub mut:
+	expr Expr
 }
 
 pub struct TypeOf {
 pub:
-	expr Expr
 	pos token.Position
 pub mut:
+	expr      Expr
 	expr_type Type
 }
 
 pub struct DumpExpr {
 pub:
-	expr Expr
 	pos token.Position
 pub mut:
+	expr      Expr
 	expr_type Type
 	cname     string // filled in the checker
 }
@@ -1542,11 +1546,11 @@ pub mut:
 pub struct ComptimeSelector {
 pub:
 	has_parens bool // if $() is used, for vfmt
-	left       Expr
-	field_expr Expr
 	pos        token.Position
 pub mut:
+	left       Expr
 	left_type  Type
+	field_expr Expr
 	typ        Type
 }
 
@@ -1616,19 +1620,19 @@ pub struct SqlExpr {
 pub:
 	typ        Type
 	is_count   bool
-	db_expr     Expr // `db` in `sql db {`
 	has_where  bool
-	has_offset  bool
-	offset_expr Expr
 	has_order  bool
-	order_expr  Expr
+	has_limit  bool
+	has_offset bool
 	has_desc   bool
 	is_array   bool
 	pos        token.Position
-	has_limit   bool
-	limit_expr  Expr
 pub mut:
+	db_expr     Expr // `db` in `sql db {`
 	where_expr  Expr
+	order_expr  Expr
+	limit_expr  Expr
+	offset_expr Expr
 	table_expr  TypeNode
 	fields      []StructField
 	sub_structs map[int]SqlExpr
@@ -1780,9 +1784,10 @@ pub fn (stmt Stmt) check_c_expr() ? {
 pub struct CTempVar {
 pub:
 	name   string // the name of the C temporary variable; used by g.expr(x)
-	orig   Expr   // the original expression, which produced the C temp variable; used by x.str()
 	typ    Type   // the type of the original expression
 	is_ptr bool   // whether the type is a pointer
+pub mut:
+	orig Expr // the original expression, which produced the C temp variable; used by x.str()
 }
 
 pub fn (node Node) position() token.Position {

vlib/v/checker/checker.v

@@ -1821,7 +1821,7 @@ fn (mut c Checker) stmt(node ast.Stmt) {
 			node.typ = c.expr(node.expr)
 			c.expected_type = ast.void_type
 			mut or_typ := ast.void_type
-			match node.expr {
+			match mut node.expr {
 				ast.IndexExpr {
 					if node.expr.or_expr.kind != .absent {
 						node.is_expr = true
@@ -1837,7 +1837,7 @@ fn (mut c Checker) stmt(node ast.Stmt) {
 				else {}
 			}
 			if !c.pref.is_repl && (c.stmt_level == 1 || (c.stmt_level > 1 && !c.is_last_stmt)) {
-				if node.expr is ast.InfixExpr {
+				if mut node.expr is ast.InfixExpr {
 					if node.expr.op == .left_shift {
 						left_sym := c.table.final_sym(node.expr.left_type)
 						if left_sym.kind != .array {
@@ -2422,7 +2422,7 @@ pub fn (mut c Checker) expr(node ast.Expr) ast.Type {
 				c.error('expected `string` instead of `$expr_sym.name` (e.g. `field.name`)',
 					node.field_expr.position())
 			}
-			if node.field_expr is ast.SelectorExpr {
+			if mut node.field_expr is ast.SelectorExpr {
 				left_pos := node.field_expr.expr.position()
 				if c.comptime_fields_type.len == 0 {
 					c.error('compile time field access can only be used when iterating over `T.fields`',
@@ -3025,7 +3025,7 @@ pub fn (mut c Checker) ident(mut node ast.Ident) ast.Type {
 						c.inside_const = false
 						c.mod = old_c_mod
 
-						if obj.expr is ast.CallExpr {
+						if mut obj.expr is ast.CallExpr {
 							if obj.expr.or_block.kind != .absent {
 								typ = typ.clear_flag(.optional)
 							}

vlib/v/checker/for.v

@@ -147,10 +147,10 @@ fn (mut c Checker) for_stmt(mut node ast.ForStmt) {
 	if !node.is_inf && typ.idx() != ast.bool_type_idx && !c.pref.translated {
 		c.error('non-bool used as for condition', node.pos)
 	}
-	if node.cond is ast.InfixExpr {
+	if mut node.cond is ast.InfixExpr {
 		infix := node.cond
 		if infix.op == .key_is {
-			if infix.left in [ast.Ident, ast.SelectorExpr] && infix.right is ast.TypeNode {
+			if infix.right is ast.TypeNode && infix.left in [ast.Ident, ast.SelectorExpr] {
 				is_variable := if mut infix.left is ast.Ident {
 					infix.left.kind == .variable
 				} else {

vlib/v/checker/if.v

@@ -50,7 +50,7 @@ pub fn (mut c Checker) if_expr(mut node ast.IfExpr) ast.Type {
 		if node.is_comptime { // Skip checking if needed
 			// smartcast field type on comptime if
 			mut comptime_field_name := ''
-			if branch.cond is ast.InfixExpr {
+			if mut branch.cond is ast.InfixExpr {
 				if branch.cond.op == .key_is {
 					if branch.cond.right !is ast.TypeNode {
 						c.error('invalid `\$if` condition: expected a type', branch.cond.right.position())
@@ -106,7 +106,7 @@ pub fn (mut c Checker) if_expr(mut node ast.IfExpr) ast.Type {
 				}
 			} else if c.pref.output_cross_c {
 				mut is_freestanding_block := false
-				if branch.cond is ast.Ident {
+				if mut branch.cond is ast.Ident {
 					if branch.cond.name == 'freestanding' {
 						is_freestanding_block = true
 					}

vlib/v/checker/match.v

@@ -8,7 +8,7 @@ import strings
 pub fn (mut c Checker) match_expr(mut node ast.MatchExpr) ast.Type {
 	node.is_expr = c.expected_type != ast.void_type
 	node.expected_type = c.expected_type
-	if node.cond is ast.ParExpr && !c.pref.translated {
+	if mut node.cond is ast.ParExpr && !c.pref.translated {
 		c.error('unnecessary `()` in `match` condition, use `match expr {` instead of `match (expr) {`.',
 			node.cond.pos)
 	}

vlib/v/parser/parse_type.v

@@ -27,13 +27,13 @@ pub fn (mut p Parser) parse_array_type(expecting token.Kind) ast.Type {
 				}
 				ast.Ident {
 					mut show_non_const_error := false
-					if const_field := p.table.global_scope.find_const('${p.mod}.$size_expr.name') {
-						if const_field.expr is ast.IntegerLiteral {
+					if mut const_field := p.table.global_scope.find_const('${p.mod}.$size_expr.name') {
+						if mut const_field.expr is ast.IntegerLiteral {
 							fixed_size = const_field.expr.val.int()
 						} else {
-							if const_field.expr is ast.InfixExpr {
+							if mut const_field.expr is ast.InfixExpr {
 								mut t := transformer.new_transformer(p.pref)
-								folded_expr := t.infix_expr(const_field.expr)
+								folded_expr := t.infix_expr(mut const_field.expr)
 
 								if folded_expr is ast.IntegerLiteral {
 									fixed_size = folded_expr.val.int()

vlib/v/transformer/index_state.v

@@ -0,0 +1,123 @@
+module transformer
+
+struct KeyVal {
+	key   string
+	value int
+}
+
+[if debug_bounds_checking ?]
+fn debug_bounds_checking(str string) {
+	println(str)
+}
+
+// IndexState is used to track the index analysis performed when parsing the code
+// `IndexExpr` nodes are annotated with `is_direct`, indicating that the array index can be safely directly accessed.
+
+// The c_gen code check will handle this annotation and perform this direct memory access. The following cases are considered valid for this optimisation:
+// 1. the array size is known and has a `len` larger than the index requested
+// 2. the array was previously accessed with a higher value which would have reported the issue already
+// 3. the array was created from a range expression a := range[10..13] and the offset'ed indexes are safe
+
+// Current limitations:
+//  * any function using break/continue or goto/label stopped from being optimised as soon as the relevant AST nodes are found as the code can not be ensured to be sequential
+//  * `enum` and `const` indexes are not optimised (they could probably be looked up)
+//  * for loops with multiple var in their init and/or inc are not analysed
+//  * mut array are not analysed as their size can be reduced, but self-assignment in a single line
+
+pub struct IndexState {
+mut:
+	// max_index has the biggest array index accessed for then named array
+	// so if a[2] was set or read, it will be 2
+	// A new array with no .len will recorded as -1 (accessing a[0] would be invalid)
+	// the value -2 is used to indicate that the array should not be analysed
+	// this is used for a mut array
+	max_index map[string]int
+	// We need to snapshot when entering `if` and `for` blocks and restore on exit
+	// as the statements may not be run. This is managed by indent() & unindent().
+	saved_disabled []bool
+	saved_key_vals [][]KeyVal
+pub mut:
+	// on encountering goto/break/continue statements we stop any analysis
+	// for the current function (as the code is not linear anymore)
+	disabled bool
+	level    int
+}
+
+// we are remembering the last array accessed and checking if the value is safe
+// the node is updated with this information which can then be used by the code generators
+fn (mut i IndexState) safe_access(key string, new int) bool {
+	$if no_bounds_checking {
+		return false
+	}
+	if i.disabled {
+		return false
+	}
+	old := i.max_index[key] or {
+		debug_bounds_checking('$i.level ${key}.len = $new')
+		i.max_index[key] = new
+		return false
+	}
+	if new > old {
+		if old < -1 {
+			debug_bounds_checking('$i.level $key[$new] unsafe (mut array)')
+			return false
+		}
+		debug_bounds_checking('$i.level $key[$new] unsafe (index was $old)')
+		i.max_index[key] = new
+		return false
+	}
+	debug_bounds_checking('$i.level $key[$new] safe (index is $old)')
+	return true
+}
+
+// safe_offset returns for a previvous array what was the highest
+// offset we ever accessed for that identifier
+fn (mut i IndexState) safe_offset(key string) int {
+	$if no_bounds_checking {
+		return -2
+	}
+	if i.disabled {
+		return -2
+	}
+	return i.max_index[key] or { -1 }
+}
+
+// indent is used for when encountering new code blocks (if, for and functions)
+// The code analysis needs to take into consideration blocks of code which
+// may not run at runtime (if/for) and therefore even if a new maximum for an
+// index access is found on an if branch it can not be used within the parent
+// code. The same is true with for blocks. indent() snapshot the current state,
+// to allow restoration with unindent()
+// Also within a function, analysis must be `disabled` when goto or break are
+// encountered as the code flow is then not lineear, and only restart when a
+// new function analysis is started.
+[if !no_bounds_checking]
+fn (mut i IndexState) indent(is_function bool) {
+	mut kvs := []KeyVal{cap: i.max_index.len}
+	for k, v in i.max_index {
+		kvs << KeyVal{k, v}
+	}
+	i.saved_disabled << i.disabled
+	i.saved_key_vals << kvs
+	if is_function {
+		i.disabled = false
+	}
+	i.level += 1
+}
+
+// restoring the data as it was before the if/for/unsafe block
+[if !no_bounds_checking]
+fn (mut i IndexState) unindent() {
+	i.level -= 1
+	mut keys := []string{cap: i.max_index.len}
+	for k, _ in i.max_index {
+		keys << k
+	}
+	for k in keys {
+		i.max_index.delete(k)
+	}
+	for saved in i.saved_key_vals.pop() {
+		i.max_index[saved.key] = saved.value
+	}
+	i.disabled = i.saved_disabled.pop()
+}