From a7d423633787cca7d15faf28aeb02dac58e7668c Mon Sep 17 00:00:00 2001 From: Delyan Angelov Date: Fri, 29 Oct 2021 21:01:07 +0300 Subject: [PATCH] sync,strings,cgen: reduce memory consumption in cgen --- vlib/strings/builder.c.v | 8 +- vlib/sync/pool/pool.v | 19 +++-- vlib/sync/pool/pool_test.v | 10 +++ vlib/v/gen/c/cgen.v | 154 +++++++++++++++++++++++++------------ 4 files changed, 133 insertions(+), 58 deletions(-) diff --git a/vlib/strings/builder.c.v b/vlib/strings/builder.c.v index c8aa6ab490..b1fd3d6fff 100644 --- a/vlib/strings/builder.c.v +++ b/vlib/strings/builder.c.v @@ -159,5 +159,11 @@ pub fn (mut b Builder) str() string { // free is for manually freeing the contents of the buffer [unsafe] pub fn (mut b Builder) free() { - unsafe { free(b.data) } + if b.data != 0 { + unsafe { free(b.data) } + mut pd := &b.data + unsafe { + (*pd) = voidptr(0) + } + } } diff --git a/vlib/sync/pool/pool.v b/vlib/sync/pool/pool.v index b2c5340cdd..43b02eaa22 100644 --- a/vlib/sync/pool/pool.v +++ b/vlib/sync/pool/pool.v @@ -81,12 +81,10 @@ pub fn (mut pool PoolProcessor) work_on_pointers(items []voidptr) { if pool.njobs > 0 { njobs = pool.njobs } - pool.items = [] - pool.results = [] - pool.thread_contexts = [] + pool.thread_contexts = []voidptr{len: items.len} + pool.results = []voidptr{len: items.len} + pool.items = []voidptr{cap: items.len} pool.items << items - pool.results = []voidptr{len: (pool.items.len)} - pool.thread_contexts << []voidptr{len: (pool.items.len)} pool.waitgroup.add(njobs) for i := 0; i < njobs; i++ { if njobs > 1 { @@ -129,13 +127,22 @@ pub fn (pool &PoolProcessor) get_result(idx int) T { // get_results - get a list of type safe results in the main thread. pub fn (pool &PoolProcessor) get_results() []T { - mut res := []T{} + mut res := []T{cap: pool.results.len} for i in 0 .. pool.results.len { res << *(&T(pool.results[i])) } return res } +// get_results_ref - get a list of type safe results in the main thread. +pub fn (pool &PoolProcessor) get_results_ref() []&T { + mut res := []&T{cap: pool.results.len} + for i in 0 .. pool.results.len { + res << &T(pool.results[i]) + } + return res +} + // set_shared_context - can be called during the setup so that you can // provide a context that is shared between all worker threads, like // common options/settings. diff --git a/vlib/sync/pool/pool_test.v b/vlib/sync/pool/pool_test.v index 629b5244b0..eabaeda825 100644 --- a/vlib/sync/pool/pool_test.v +++ b/vlib/sync/pool/pool_test.v @@ -34,6 +34,11 @@ fn test_work_on_strings() { println(x.s) assert x.s.len > 1 } + println('---------- pool_s.get_results_ref: --------------') + for x in pool_s.get_results_ref() { + println(x.s) + assert x.s.len > 1 + } } fn test_work_on_ints() { @@ -49,4 +54,9 @@ fn test_work_on_ints() { println(x.i) assert x.i > 100 } + println('---------- pool_i.get_results_ref: --------------') + for x in pool_i.get_results_ref() { + println(x.i) + assert x.i > 100 + } } diff --git a/vlib/v/gen/c/cgen.v b/vlib/v/gen/c/cgen.v index 79581e360d..eb74902d15 100644 --- a/vlib/v/gen/c/cgen.v +++ b/vlib/v/gen/c/cgen.v @@ -260,58 +260,12 @@ pub fn gen(files []&ast.File, table &ast.Table, pref &pref.Preferences) string { global_g.timers.show('cgen init') global_g.tests_inited = false if !pref.no_parallel { - mut pp := pool.new_pool_processor( - callback: fn (p &pool.PoolProcessor, idx int, wid int) &Gen { - file := p.get_item<&ast.File>(idx) - mut global_g := &Gen(p.get_shared_context()) - mut g := &Gen{ - file: file - out: strings.new_builder(512000) - cheaders: strings.new_builder(15000) - includes: strings.new_builder(100) - typedefs: strings.new_builder(100) - typedefs2: strings.new_builder(100) - type_definitions: strings.new_builder(100) - definitions: strings.new_builder(100) - gowrappers: strings.new_builder(100) - stringliterals: strings.new_builder(100) - auto_str_funcs: strings.new_builder(100) - comptime_defines: strings.new_builder(100) - pcs_declarations: strings.new_builder(100) - hotcode_definitions: strings.new_builder(100) - embedded_data: strings.new_builder(1000) - options: strings.new_builder(100) - shared_types: strings.new_builder(100) - shared_functions: strings.new_builder(100) - channel_definitions: strings.new_builder(100) - json_forward_decls: strings.new_builder(100) - enum_typedefs: strings.new_builder(100) - sql_buf: strings.new_builder(100) - init: strings.new_builder(100) - global_init: strings.new_builder(0) - cleanup: strings.new_builder(100) - table: global_g.table - pref: global_g.pref - fn_decl: 0 - indent: -1 - module_built: global_g.module_built - timers: util.new_timers(global_g.timers_should_print) - inner_loop: &ast.EmptyStmt{} - field_data_type: ast.Type(global_g.table.find_type_idx('FieldData')) - array_sort_fn: global_g.array_sort_fn - threaded_fns: global_g.threaded_fns - done_optionals: global_g.done_optionals - is_autofree: global_g.pref.autofree - } - g.gen_file() - return g - } - ) + mut pp := pool.new_pool_processor(callback: cgen_process_one_file_cb) pp.set_shared_context(global_g) // TODO: make global_g shared pp.work_on_items(files) global_g.timers.start('cgen unification') // tg = thread gen - for g in pp.get_results() { + for g in pp.get_results_ref() { global_g.embedded_files << g.embedded_files global_g.out.write(g.out) or { panic(err) } global_g.cheaders.write(g.cheaders) or { panic(err) } @@ -381,16 +335,23 @@ pub fn gen(files []&ast.File, table &ast.Table, pref &pref.Preferences) string { global_g.pcs << g.pcs global_g.json_types << g.json_types global_g.hotcode_fn_names << g.hotcode_fn_names + unsafe { g.free_builders() } } } else { for file in files { global_g.file = file global_g.gen_file() + global_g.inits[file.mod.name].write(global_g.init) or { panic(err) } + unsafe { global_g.init.free() } global_g.init = strings.new_builder(100) + global_g.cleanups[file.mod.name].write(global_g.cleanup) or { panic(err) } + unsafe { global_g.cleanup.free() } global_g.cleanup = strings.new_builder(100) + global_g.global_inits[file.mod.name].write(global_g.global_init) or { panic(err) } + unsafe { global_g.global_init.free() } global_g.global_init = strings.new_builder(100) } global_g.timers.start('cgen unification') @@ -523,7 +484,98 @@ pub fn gen(files []&ast.File, table &ast.Table, pref &pref.Preferences) string { b.write_string(g.out.str()) b.writeln('\n// THE END.') g.timers.show('cgen common') - return b.str() + res := b.str() + unsafe { b.free() } + unsafe { g.free_builders() } + return res +} + +fn cgen_process_one_file_cb(p &pool.PoolProcessor, idx int, wid int) &Gen { + file := p.get_item<&ast.File>(idx) + mut global_g := &Gen(p.get_shared_context()) + mut g := &Gen{ + file: file + out: strings.new_builder(512000) + cheaders: strings.new_builder(15000) + includes: strings.new_builder(100) + typedefs: strings.new_builder(100) + typedefs2: strings.new_builder(100) + type_definitions: strings.new_builder(100) + definitions: strings.new_builder(100) + gowrappers: strings.new_builder(100) + stringliterals: strings.new_builder(100) + auto_str_funcs: strings.new_builder(100) + comptime_defines: strings.new_builder(100) + pcs_declarations: strings.new_builder(100) + hotcode_definitions: strings.new_builder(100) + embedded_data: strings.new_builder(1000) + options: strings.new_builder(100) + shared_types: strings.new_builder(100) + shared_functions: strings.new_builder(100) + channel_definitions: strings.new_builder(100) + json_forward_decls: strings.new_builder(100) + enum_typedefs: strings.new_builder(100) + sql_buf: strings.new_builder(100) + init: strings.new_builder(100) + global_init: strings.new_builder(0) + cleanup: strings.new_builder(100) + table: global_g.table + pref: global_g.pref + fn_decl: 0 + indent: -1 + module_built: global_g.module_built + timers: util.new_timers(global_g.timers_should_print) + inner_loop: &ast.EmptyStmt{} + field_data_type: ast.Type(global_g.table.find_type_idx('FieldData')) + array_sort_fn: global_g.array_sort_fn + threaded_fns: global_g.threaded_fns + done_optionals: global_g.done_optionals + is_autofree: global_g.pref.autofree + } + g.gen_file() + return g +} + +// free_builders should be called only when a Gen would NOT be used anymore +// it frees the bulk of the memory that is private to the Gen instance +// (the various string builders) +[unsafe] +pub fn (mut g Gen) free_builders() { + unsafe { + g.out.free() + g.cheaders.free() + g.includes.free() + g.typedefs.free() + g.typedefs2.free() + g.type_definitions.free() + g.definitions.free() + g.global_init.free() + g.init.free() + g.cleanup.free() + g.gowrappers.free() + g.stringliterals.free() + g.auto_str_funcs.free() + g.comptime_defines.free() + g.pcs_declarations.free() + g.hotcode_definitions.free() + g.embedded_data.free() + g.shared_types.free() + g.shared_functions.free() + g.channel_definitions.free() + g.options.free() + g.json_forward_decls.free() + g.enum_typedefs.free() + g.sql_buf.free() + for _, mut v in g.global_inits { + v.free() + } + for _, mut v in g.inits { + v.free() + } + for _, mut v in g.cleanups { + v.free() + } + } } pub fn (mut g Gen) gen_file() { @@ -6203,7 +6255,7 @@ fn (mut g Gen) write_builtin_types() { // Sort the types, make sure types that are referenced by other types // are added before them. fn (mut g Gen) write_sorted_types() { - mut types := []ast.TypeSymbol{} // structs that need to be sorted + mut types := []ast.TypeSymbol{cap: g.table.type_symbols.len} // structs that need to be sorted for typ in g.table.type_symbols { if typ.name !in c.builtins { types << typ @@ -6445,7 +6497,7 @@ fn (g &Gen) sort_structs(typesa []ast.TypeSymbol) []ast.TypeSymbol { '\nif you feel this is an error, please create a new issue here: https://github.com/vlang/v/issues and tag @joe-conigliaro') } // sort types - mut types_sorted := []ast.TypeSymbol{} + mut types_sorted := []ast.TypeSymbol{cap: dep_graph_sorted.nodes.len} for node in dep_graph_sorted.nodes { types_sorted << g.table.type_symbols[g.table.type_idxs[node.name]] }