strings,builder: reduce V cgen and parser memory consumption, fix strings.Builder leak (#12342)

vlib/builtin/array.v

@@ -238,6 +238,11 @@ pub fn (mut a array) delete_many(i int, size int) {
 		vmemcpy(&byte(a.data) + i * a.element_size, &byte(old_data) + (i + size) * a.element_size,
 			(a.len - i - size) * a.element_size)
 	}
+	if a.flags.has(.noslices) {
+		unsafe {
+			free(old_data)
+		}
+	}
 	a.len = new_size
 	a.cap = new_cap
 }

vlib/strings/builder.c.v

@@ -11,7 +11,9 @@ pub type Builder = []byte
 
 // new_builder returns a new string builder, with an initial capacity of `initial_size`
 pub fn new_builder(initial_size int) Builder {
-	return Builder([]byte{cap: initial_size})
+	mut res := Builder([]byte{cap: initial_size})
+	unsafe { res.flags.set(.noslices) }
+	return res
 }
 
 // write_ptr writes `len` bytes provided byteptr to the accumulated buffer
@@ -161,9 +163,8 @@ pub fn (mut b Builder) str() string {
 pub fn (mut b Builder) free() {
 	if b.data != 0 {
 		unsafe { free(b.data) }
-		mut pd := &b.data
 		unsafe {
-			(*pd) = voidptr(0)
+			b.data = voidptr(0)
 		}
 	}
 }

vlib/v/depgraph/depgraph.v

@@ -25,6 +25,12 @@ mut:
 	data map[string][]string
 }
 
+pub fn new_ordered_dependency_map() OrderedDepMap {
+	mut res := OrderedDepMap{}
+	unsafe { res.keys.flags.set(.noslices) }
+	return res
+}
+
 pub fn (mut o OrderedDepMap) set(name string, deps []string) {
 	if name !in o.data {
 		o.keys << name
@@ -92,8 +98,8 @@ pub fn (mut graph DepGraph) add(mod string, deps []string) {
 }
 
 pub fn (graph &DepGraph) resolve() &DepGraph {
-	mut node_names := OrderedDepMap{}
-	mut node_deps := OrderedDepMap{}
+	mut node_names := new_ordered_dependency_map()
+	mut node_deps := new_ordered_dependency_map()
 	for node in graph.nodes {
 		node_names.add(node.name, node.deps)
 		node_deps.add(node.name, node.deps)

vlib/v/parser/parser.v

@@ -111,7 +111,9 @@ pub fn parse_comptime(text string, table &ast.Table, pref &pref.Preferences, sco
 		errors: []errors.Error{}
 		warnings: []errors.Warning{}
 	}
-	return p.parse()
+	res := p.parse()
+	unsafe { p.free_scanner() }
+	return res
 }
 
 pub fn parse_text(text string, path string, table &ast.Table, comments_mode scanner.CommentsMode, pref &pref.Preferences) &ast.File {
@@ -128,13 +130,23 @@ pub fn parse_text(text string, path string, table &ast.Table, comments_mode scan
 		warnings: []errors.Warning{}
 	}
 	p.set_path(path)
-	return p.parse()
+	res := p.parse()
+	unsafe { p.free_scanner() }
+	return res
 }
 
 [unsafe]
 pub fn (mut p Parser) free() {
+	unsafe { p.free_scanner() }
+}
+
+[unsafe]
+pub fn (mut p Parser) free_scanner() {
 	unsafe {
-		p.scanner.free()
+		if p.scanner != 0 {
+			p.scanner.free()
+			p.scanner = &scanner.Scanner(0)
+		}
 	}
 }
 
@@ -190,7 +202,9 @@ pub fn parse_file(path string, table &ast.Table, comments_mode scanner.CommentsM
 		warnings: []errors.Warning{}
 	}
 	p.set_path(path)
-	return p.parse()
+	res := p.parse()
+	unsafe { p.free_scanner() }
+	return res
 }
 
 pub fn parse_vet_file(path string, table_ &ast.Table, pref &pref.Preferences) (&ast.File, []vet.Error) {
@@ -225,6 +239,7 @@ pub fn parse_vet_file(path string, table_ &ast.Table, pref &pref.Preferences) (&
 	}
 	p.vet_errors << p.scanner.vet_errors
 	file := p.parse()
+	unsafe { p.free_scanner() }
 	return file, p.vet_errors
 }
 
@@ -367,7 +382,7 @@ pub fn parse_files(paths []string, table &ast.Table, pref &pref.Preferences) []&
 		}
 		*/
 	}
-	mut files := []&ast.File{}
+	mut files := []&ast.File{cap: paths.len}
 	for path in paths {
 		timers.start('parse_file $path')
 		files << parse_file(path, table, .skip_comments, pref)

vlib/v/scanner/scanner.v

@@ -151,7 +151,11 @@ fn (mut s Scanner) init_scanner() {
 [unsafe]
 pub fn (mut s Scanner) free() {
 	unsafe {
-		s.text.free()
+		// NB: s.text is not freed here, because it is shared with all other util.read_file instances,
+		// and strings are not reference counted yet:
+		// s.text.free()
+		// .all_tokens however are not shared with anything, and can be freed:
+		s.all_tokens.free()
 	}
 }
 
@@ -1455,15 +1459,19 @@ pub fn verror(s string) {
 	util.verror('scanner error', s)
 }
 
+// codegen allows you to generate V code, so that it can be parsed,
+// checked, markused, cgen-ed etc further, just like user's V code.
 pub fn (mut s Scanner) codegen(newtext string) {
 	$if debug_codegen ? {
 		eprintln('scanner.codegen:\n $newtext')
 	}
-	// codegen makes sense only during normal compilation
-	// feeding code generated V code to vfmt or vdoc will
-	// cause them to output/document ephemeral stuff.
 	if s.comments_mode == .skip_comments {
-		s.all_tokens.delete_last() // remove .eof from end of .all_tokens
+		// Calling codegen makes sense only during normal compilation, since
+		// feeding code generated V code to vfmt or vdoc will cause them to
+		// output/document ephemeral stuff.
+		for s.all_tokens.len > 0 && s.all_tokens.last().kind == .eof {
+			s.all_tokens.delete_last()
+		}
 		s.text += newtext
 		old_tidx := s.tidx
 		s.tidx = s.all_tokens.len

vlib/v/tests/valgrind/strings_builder.v

@@ -0,0 +1,22 @@
+import strings
+
+fn main() {
+	mut sb := strings.new_builder(4)
+	sb.write_string('abcd')
+	sb.write_string('abcd')
+	sb.write_string('abcd')
+	sb.write_string('abcd')
+	sb.write_string('abcd')
+	sb.write_string('abcd')
+	sb.write_string('abcd')
+	sb.write_string('abcd')
+	sb.write_string('abcd')
+	sb.write_string('abcd')
+	sb.write_string('abcd')
+	sb.write_string('abcd')
+	sb.write_string('abcd')
+	sb.write_string('abcd')
+	sb.write_string('abcd')
+	sb.write_string('abcd')
+	println(sb.str())
+}