mirror/v
1
0
Fork 0
mirror of https://github.com/vlang/v.git synced 2023-08-10 21:13:21 +03:00
Code Issues Projects Releases Wiki Activity

sync,strings,cgen: reduce memory consumption in cgen

Browse Source
This commit is contained in:
Delyan Angelov 2021-10-29 21:01:07 +03:00
parent 7e0f2fcda9
commit a7d4236337
No known key found for this signature in database
GPG Key ID: 66886C0F12D595ED
4 changed files with 133 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
vlib/strings/builder.c.v
View File

@ -159,5 +159,11 @@ pub fn (mut b Builder) str() string {
// free is for manually freeing the contents of the buffer
[unsafe]
pub fn (mut b Builder) free() {
unsafe { free(b.data) }
if b.data != 0 {
unsafe { free(b.data) }
mut pd := &b.data
unsafe {
(*pd) = voidptr(0)
}
}
}

19
vlib/sync/pool/pool.v
View File

@ -81,12 +81,10 @@ pub fn (mut pool PoolProcessor) work_on_pointers(items []voidptr) {
if pool.njobs > 0 {
njobs = pool.njobs
}
pool.items = []
pool.results = []
pool.thread_contexts = []
pool.thread_contexts = []voidptr{len: items.len}
pool.results = []voidptr{len: items.len}
pool.items = []voidptr{cap: items.len}
pool.items << items
pool.results = []voidptr{len: (pool.items.len)}
pool.thread_contexts << []voidptr{len: (pool.items.len)}
pool.waitgroup.add(njobs)
for i := 0; i < njobs; i++ {
if njobs > 1 {
@ -129,13 +127,22 @@ pub fn (pool &PoolProcessor) get_result<T>(idx int) T {
// get_results - get a list of type safe results in the main thread.
pub fn (pool &PoolProcessor) get_results<T>() []T {
mut res := []T{}
mut res := []T{cap: pool.results.len}
for i in 0 .. pool.results.len {
res << *(&T(pool.results[i]))
}
return res
}
// get_results_ref - get a list of type safe results in the main thread.
pub fn (pool &PoolProcessor) get_results_ref<T>() []&T {
mut res := []&T{cap: pool.results.len}
for i in 0 .. pool.results.len {
res << &T(pool.results[i])
}
return res
}
// set_shared_context - can be called during the setup so that you can
// provide a context that is shared between all worker threads, like
// common options/settings.

10
vlib/sync/pool/pool_test.v
View File

@ -34,6 +34,11 @@ fn test_work_on_strings() {
println(x.s)
assert x.s.len > 1
}
println('---------- pool_s.get_results_ref: --------------')
for x in pool_s.get_results_ref<SResult>() {
println(x.s)
assert x.s.len > 1
}
}
fn test_work_on_ints() {
@ -49,4 +54,9 @@ fn test_work_on_ints() {
println(x.i)
assert x.i > 100
}
println('---------- pool_i.get_results_ref: --------------')
for x in pool_i.get_results_ref<IResult>() {
println(x.i)
assert x.i > 100
}
}

154
vlib/v/gen/c/cgen.v
View File

@ -260,58 +260,12 @@ pub fn gen(files []&ast.File, table &ast.Table, pref &pref.Preferences) string {
global_g.timers.show('cgen init')
global_g.tests_inited = false
if !pref.no_parallel {
mut pp := pool.new_pool_processor(
callback: fn (p &pool.PoolProcessor, idx int, wid int) &Gen {
file := p.get_item<&ast.File>(idx)
mut global_g := &Gen(p.get_shared_context())
mut g := &Gen{
file: file
out: strings.new_builder(512000)
cheaders: strings.new_builder(15000)
includes: strings.new_builder(100)
typedefs: strings.new_builder(100)
typedefs2: strings.new_builder(100)
type_definitions: strings.new_builder(100)
definitions: strings.new_builder(100)
gowrappers: strings.new_builder(100)
stringliterals: strings.new_builder(100)
auto_str_funcs: strings.new_builder(100)
comptime_defines: strings.new_builder(100)
pcs_declarations: strings.new_builder(100)
hotcode_definitions: strings.new_builder(100)
embedded_data: strings.new_builder(1000)
options: strings.new_builder(100)
shared_types: strings.new_builder(100)
shared_functions: strings.new_builder(100)
channel_definitions: strings.new_builder(100)
json_forward_decls: strings.new_builder(100)
enum_typedefs: strings.new_builder(100)
sql_buf: strings.new_builder(100)
init: strings.new_builder(100)
global_init: strings.new_builder(0)
cleanup: strings.new_builder(100)
table: global_g.table
pref: global_g.pref
fn_decl: 0
indent: -1
module_built: global_g.module_built
timers: util.new_timers(global_g.timers_should_print)
inner_loop: &ast.EmptyStmt{}
field_data_type: ast.Type(global_g.table.find_type_idx('FieldData'))
array_sort_fn: global_g.array_sort_fn
threaded_fns: global_g.threaded_fns
done_optionals: global_g.done_optionals
is_autofree: global_g.pref.autofree
}
g.gen_file()
return g
}
)
mut pp := pool.new_pool_processor(callback: cgen_process_one_file_cb)
pp.set_shared_context(global_g) // TODO: make global_g shared
pp.work_on_items(files)
global_g.timers.start('cgen unification')
// tg = thread gen
for g in pp.get_results<Gen>() {
for g in pp.get_results_ref<Gen>() {
global_g.embedded_files << g.embedded_files
global_g.out.write(g.out) or { panic(err) }
global_g.cheaders.write(g.cheaders) or { panic(err) }
@ -381,16 +335,23 @@ pub fn gen(files []&ast.File, table &ast.Table, pref &pref.Preferences) string {
global_g.pcs << g.pcs
global_g.json_types << g.json_types
global_g.hotcode_fn_names << g.hotcode_fn_names
unsafe { g.free_builders() }
}
} else {
for file in files {
global_g.file = file
global_g.gen_file()
global_g.inits[file.mod.name].write(global_g.init) or { panic(err) }
unsafe { global_g.init.free() }
global_g.init = strings.new_builder(100)
global_g.cleanups[file.mod.name].write(global_g.cleanup) or { panic(err) }
unsafe { global_g.cleanup.free() }
global_g.cleanup = strings.new_builder(100)
global_g.global_inits[file.mod.name].write(global_g.global_init) or { panic(err) }
unsafe { global_g.global_init.free() }
global_g.global_init = strings.new_builder(100)
}
global_g.timers.start('cgen unification')
@ -523,7 +484,98 @@ pub fn gen(files []&ast.File, table &ast.Table, pref &pref.Preferences) string {
b.write_string(g.out.str())
b.writeln('\n// THE END.')
g.timers.show('cgen common')
return b.str()
res := b.str()
unsafe { b.free() }
unsafe { g.free_builders() }
return res
}
fn cgen_process_one_file_cb(p &pool.PoolProcessor, idx int, wid int) &Gen {
file := p.get_item<&ast.File>(idx)
mut global_g := &Gen(p.get_shared_context())
mut g := &Gen{
file: file
out: strings.new_builder(512000)
cheaders: strings.new_builder(15000)
includes: strings.new_builder(100)
typedefs: strings.new_builder(100)
typedefs2: strings.new_builder(100)
type_definitions: strings.new_builder(100)
definitions: strings.new_builder(100)
gowrappers: strings.new_builder(100)
stringliterals: strings.new_builder(100)
auto_str_funcs: strings.new_builder(100)
comptime_defines: strings.new_builder(100)
pcs_declarations: strings.new_builder(100)
hotcode_definitions: strings.new_builder(100)
embedded_data: strings.new_builder(1000)
options: strings.new_builder(100)
shared_types: strings.new_builder(100)
shared_functions: strings.new_builder(100)
channel_definitions: strings.new_builder(100)
json_forward_decls: strings.new_builder(100)
enum_typedefs: strings.new_builder(100)
sql_buf: strings.new_builder(100)
init: strings.new_builder(100)
global_init: strings.new_builder(0)
cleanup: strings.new_builder(100)
table: global_g.table
pref: global_g.pref
fn_decl: 0
indent: -1
module_built: global_g.module_built
timers: util.new_timers(global_g.timers_should_print)
inner_loop: &ast.EmptyStmt{}
field_data_type: ast.Type(global_g.table.find_type_idx('FieldData'))
array_sort_fn: global_g.array_sort_fn
threaded_fns: global_g.threaded_fns
done_optionals: global_g.done_optionals
is_autofree: global_g.pref.autofree
}
g.gen_file()
return g
}
// free_builders should be called only when a Gen would NOT be used anymore
// it frees the bulk of the memory that is private to the Gen instance
// (the various string builders)
[unsafe]
pub fn (mut g Gen) free_builders() {
unsafe {
g.out.free()
g.cheaders.free()
g.includes.free()
g.typedefs.free()
g.typedefs2.free()
g.type_definitions.free()
g.definitions.free()
g.global_init.free()
g.init.free()
g.cleanup.free()
g.gowrappers.free()
g.stringliterals.free()
g.auto_str_funcs.free()
g.comptime_defines.free()
g.pcs_declarations.free()
g.hotcode_definitions.free()
g.embedded_data.free()
g.shared_types.free()
g.shared_functions.free()
g.channel_definitions.free()
g.options.free()
g.json_forward_decls.free()
g.enum_typedefs.free()
g.sql_buf.free()
for _, mut v in g.global_inits {
v.free()
}
for _, mut v in g.inits {
v.free()
}
for _, mut v in g.cleanups {
v.free()
}
}
}
pub fn (mut g Gen) gen_file() {
@ -6203,7 +6255,7 @@ fn (mut g Gen) write_builtin_types() {
// Sort the types, make sure types that are referenced by other types
// are added before them.
fn (mut g Gen) write_sorted_types() {
mut types := []ast.TypeSymbol{} // structs that need to be sorted
mut types := []ast.TypeSymbol{cap: g.table.type_symbols.len} // structs that need to be sorted
for typ in g.table.type_symbols {
if typ.name !in c.builtins {
types << typ
@ -6445,7 +6497,7 @@ fn (g &Gen) sort_structs(typesa []ast.TypeSymbol) []ast.TypeSymbol {
'\nif you feel this is an error, please create a new issue here: https://github.com/vlang/v/issues and tag @joe-conigliaro')
}
// sort types
mut types_sorted := []ast.TypeSymbol{}
mut types_sorted := []ast.TypeSymbol{cap: dep_graph_sorted.nodes.len}
for node in dep_graph_sorted.nodes {
types_sorted << g.table.type_symbols[g.table.type_idxs[node.name]]
}