mirror/v
1
0
Fork 0
mirror of https://github.com/vlang/v.git synced 2023-08-10 21:13:21 +03:00
Code Issues Projects Releases Wiki Activity

cbuilder: fix hardcoded v_parallel for -parallel-cc, use a sync.pool for the compilation threads

Browse Source
This commit is contained in:
Delyan Angelov
2022-10-01 13:50:39 +03:00
parent aa02a9cd3b
commit 90599c7f85
1 changed files with 34 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
vlib/v/builder/cbuilder/parallel_cc.v
View File

@@ -5,10 +5,11 @@ import time
import sync import sync
import v.util import v.util
import v.builder import v.builder
import sync.pool
fn parallel_cc(mut b builder.Builder, header string, res string, out_str string, out_fn_start_pos []int) { fn parallel_cc(mut b builder.Builder, header string, res string, out_str string, out_fn_start_pos []int) {
nr_jobs := util.nr_jobs c_files := util.nr_jobs
println('len=$nr_jobs') println('> c_files: $c_files | util.nr_jobs: $util.nr_jobs')
out_h := header.replace_once('static char * v_typeof_interface_IError', 'char * v_typeof_interface_IError') out_h := header.replace_once('static char * v_typeof_interface_IError', 'char * v_typeof_interface_IError')
os.write_file('out.h', out_h) or { panic(err) } os.write_file('out.h', out_h) or { panic(err) }
// Write generated stuff in `g.out` before and after the `out_fn_start_pos` locations, // Write generated stuff in `g.out` before and after the `out_fn_start_pos` locations,
@@ -21,9 +22,9 @@ fn parallel_cc(mut b builder.Builder, header string, res string, out_str string,
} }
mut prev_fn_pos := 0 mut prev_fn_pos := 0
mut out_files := []os.File{len: nr_jobs} mut out_files := []os.File{len: c_files}
mut fnames := []string{} mut fnames := []string{}
for i in 0 .. nr_jobs { for i in 0 .. c_files {
fname := 'out_${i + 1}.c' fname := 'out_${i + 1}.c'
fnames << fname fnames << fname
out_files[i] = os.create(fname) or { panic(err) } out_files[i] = os.create(fname) or { panic(err) }
@@ -32,7 +33,7 @@ fn parallel_cc(mut b builder.Builder, header string, res string, out_str string,
// out_fn_start_pos.sort() // out_fn_start_pos.sort()
for i, fn_pos in out_fn_start_pos { for i, fn_pos in out_fn_start_pos {
if prev_fn_pos >= out_str.len || fn_pos >= out_str.len || prev_fn_pos > fn_pos { if prev_fn_pos >= out_str.len || fn_pos >= out_str.len || prev_fn_pos > fn_pos {
println('EXITING i=$i out of $out_fn_start_pos.len prev_pos=$prev_fn_pos fn_pos=$fn_pos') println('> EXITING i=$i out of $out_fn_start_pos.len prev_pos=$prev_fn_pos fn_pos=$fn_pos')
break break
} }
if i == 0 { if i == 0 {
@@ -41,42 +42,50 @@ fn parallel_cc(mut b builder.Builder, header string, res string, out_str string,
continue continue
} }
fn_text := out_str[prev_fn_pos..fn_pos] fn_text := out_str[prev_fn_pos..fn_pos]
out_files[i % nr_jobs].writeln(fn_text + '\n//////////////////////////////////////\n\n') or { out_files[i % c_files].writeln(fn_text + '\n//////////////////////////////////////\n\n') or {
panic(err) panic(err)
} }
prev_fn_pos = fn_pos prev_fn_pos = fn_pos
} }
for i in 0 .. nr_jobs { for i in 0 .. c_files {
out_files[i].close() out_files[i].close()
} }
t := time.now() //
mut wg := sync.new_waitgroup() sw := time.new_stopwatch()
mut o_postfixes := []string{}
for i in ['0', 'x'] { for i in ['0', 'x'] {
wg.add(1) o_postfixes << i
go build_o(i, mut wg)
} }
for i in 0 .. nr_jobs { for i in 0 .. c_files {
wg.add(1) o_postfixes << (i + 1).str()
go build_o((i + 1).str(), mut wg)
} }
wg.wait() mut pp := pool.new_pool_processor(callback: build_o_cb)
println(time.now() - t) nthreads := c_files + 2
link_cmd := '${os.quoted_path(cbuilder.cc_compiler)} -o v_parallel out_0.o ${fnames.map(it.replace('.c', pp.set_max_jobs(nthreads)
pp.work_on_items(o_postfixes)
eprintln('> C compilation on $nthreads threads, working on $o_postfixes.len files took: $sw.elapsed().milliseconds() ms')
dump(b.pref.out_name)
link_cmd := '${os.quoted_path(cbuilder.cc_compiler)} -o ${os.quoted_path(b.pref.out_name)} out_0.o ${fnames.map(it.replace('.c',
'.o')).join(' ')} out_x.o -lpthread $cbuilder.cc_ldflags' '.o')).join(' ')} out_x.o -lpthread $cbuilder.cc_ldflags'
sw_link := time.new_stopwatch()
link_res := os.execute(link_cmd) link_res := os.execute(link_cmd)
println('> link_cmd: $link_cmd => $link_res.exit_code') eprint_time('link_cmd', link_cmd, link_res, sw_link)
println(time.now() - t)
if link_res.exit_code != 0 {
println(link_res.output)
}
} }
fn build_o(postfix string, mut wg sync.WaitGroup) { fn build_o_cb(p &pool.PoolProcessor, idx int, wid int) voidptr {
postfix := p.get_item<string>(idx)
sw := time.new_stopwatch() sw := time.new_stopwatch()
cmd := '${os.quoted_path(cbuilder.cc_compiler)} $cbuilder.cc_cflags -c -w -o out_${postfix}.o out_${postfix}.c' cmd := '${os.quoted_path(cbuilder.cc_compiler)} $cbuilder.cc_cflags -c -w -o out_${postfix}.o out_${postfix}.c'
res := os.execute(cmd) res := os.execute(cmd)
wg.done() eprint_time('c cmd', cmd, res, sw)
println('cmd: `$cmd` => $res.exit_code , $sw.elapsed().milliseconds() ms') return unsafe { nil }
}
fn eprint_time(label string, cmd string, res os.Result, sw time.StopWatch) {
eprintln('> $label: `$cmd` => $res.exit_code , $sw.elapsed().milliseconds() ms')
if res.exit_code != 0 {
eprintln(res.output)
}
} }
const cc_compiler = os.getenv_opt('CC') or { 'cc' } const cc_compiler = os.getenv_opt('CC') or { 'cc' }