mirror of
https://github.com/vlang/v.git
synced 2023-08-10 21:13:21 +03:00
sync: implement pool.work_on_items to process a list of items in parallel
This commit is contained in:
Alexander Medvednikov
committed by
GitHub
GitHub
parent
136aa763a3
commit
b0ece3a9d8
@ -5,22 +5,17 @@ import (
|
||||
term
|
||||
benchmark
|
||||
filepath
|
||||
runtime
|
||||
sync
|
||||
v.pref
|
||||
)
|
||||
|
||||
pub struct TestSession {
|
||||
pub mut:
|
||||
files []string
|
||||
vexe string
|
||||
vargs string
|
||||
failed bool
|
||||
benchmark benchmark.Benchmark
|
||||
|
||||
ntask int // writing to this should be locked by mu.
|
||||
ntask_mtx &sync.Mutex
|
||||
waitgroup &sync.WaitGroup
|
||||
files []string
|
||||
vexe string
|
||||
vargs string
|
||||
failed bool
|
||||
benchmark benchmark.Benchmark
|
||||
show_ok_tests bool
|
||||
}
|
||||
|
||||
@ -28,11 +23,6 @@ pub fn new_test_session(vargs string) TestSession {
|
||||
return TestSession{
|
||||
vexe: pref.vexe_path()
|
||||
vargs: vargs
|
||||
|
||||
ntask: 0
|
||||
ntask_mtx: sync.new_mutex()
|
||||
waitgroup: sync.new_waitgroup()
|
||||
|
||||
show_ok_tests: !vargs.contains('-silent')
|
||||
}
|
||||
}
|
||||
@ -69,109 +59,96 @@ pub fn (ts mut TestSession) test() {
|
||||
}
|
||||
remaining_files << dot_relative_file
|
||||
}
|
||||
|
||||
ts.files = remaining_files
|
||||
ts.benchmark.set_total_expected_steps(remaining_files.len)
|
||||
|
||||
mut njobs := runtime.nr_jobs()
|
||||
mut pool_of_test_runners := sync.new_pool_processor({
|
||||
callback: worker_trunner
|
||||
})
|
||||
pool_of_test_runners.set_shared_context(ts)
|
||||
$if msvc {
|
||||
// NB: MSVC can not be launched in parallel, without giving it
|
||||
// the option /FS because it uses a shared PDB file, which should
|
||||
// be locked, but that makes writing slower...
|
||||
// See: https://docs.microsoft.com/en-us/cpp/build/reference/fs-force-synchronous-pdb-writes?view=vs-2019
|
||||
// Instead, just run tests on 1 core for now.
|
||||
njobs = 1
|
||||
pool_of_test_runners.set_max_jobs(1)
|
||||
}
|
||||
ts.waitgroup.add( njobs )
|
||||
for i:=0; i < njobs; i++ {
|
||||
go process_in_thread(ts)
|
||||
}
|
||||
ts.waitgroup.wait()
|
||||
pool_of_test_runners.work_on_pointers(remaining_files.pointers())
|
||||
ts.benchmark.stop()
|
||||
eprintln(term.h_divider('-'))
|
||||
}
|
||||
|
||||
|
||||
fn process_in_thread(ts mut TestSession){
|
||||
ts.process_files()
|
||||
ts.waitgroup.done()
|
||||
}
|
||||
|
||||
fn (ts mut TestSession) process_files() {
|
||||
fn worker_trunner(p mut sync.PoolProcessor, idx int, thread_id int) voidptr {
|
||||
mut ts := &TestSession(p.get_shared_context())
|
||||
tmpd := os.tmpdir()
|
||||
show_stats := '-stats' in ts.vargs.split(' ')
|
||||
|
||||
mut tls_bench := benchmark.new_benchmark() // tls_bench is used to format the step messages/timings
|
||||
tls_bench.set_total_expected_steps( ts.benchmark.nexpected_steps )
|
||||
for {
|
||||
|
||||
ts.ntask_mtx.lock()
|
||||
ts.ntask++
|
||||
idx := ts.ntask-1
|
||||
ts.ntask_mtx.unlock()
|
||||
|
||||
if idx >= ts.files.len { break }
|
||||
tls_bench.cstep = idx
|
||||
|
||||
dot_relative_file := ts.files[ idx ]
|
||||
relative_file := dot_relative_file.replace('./', '')
|
||||
file := os.realpath(relative_file)
|
||||
// Ensure that the generated binaries will be stored in the temporary folder.
|
||||
// Remove them after a test passes/fails.
|
||||
fname := filepath.filename(file)
|
||||
generated_binary_fname := if os.user_os() == 'windows' { fname.replace('.v', '.exe') } else { fname.replace('.v', '') }
|
||||
generated_binary_fpath := filepath.join(tmpd,generated_binary_fname)
|
||||
if os.exists(generated_binary_fpath) {
|
||||
os.rm(generated_binary_fpath)
|
||||
}
|
||||
mut cmd_options := [ts.vargs]
|
||||
if !ts.vargs.contains('fmt') {
|
||||
cmd_options << ' -o "$generated_binary_fpath"'
|
||||
}
|
||||
cmd := '"${ts.vexe}" ' + cmd_options.join(' ') + ' "${file}"'
|
||||
// eprintln('>>> v cmd: $cmd')
|
||||
ts.benchmark.step()
|
||||
tls_bench.step()
|
||||
if show_stats {
|
||||
eprintln(term.h_divider('-'))
|
||||
status := os.system(cmd)
|
||||
if status == 0 {
|
||||
ts.benchmark.ok()
|
||||
tls_bench.ok()
|
||||
}
|
||||
else {
|
||||
ts.failed = true
|
||||
ts.benchmark.fail()
|
||||
tls_bench.fail()
|
||||
continue
|
||||
}
|
||||
// tls_bench is used to format the step messages/timings
|
||||
mut tls_bench := &benchmark.Benchmark(p.get_thread_context(idx))
|
||||
if isnil(tls_bench) {
|
||||
tls_bench = benchmark.new_benchmark_pointer()
|
||||
tls_bench.set_total_expected_steps(ts.benchmark.nexpected_steps)
|
||||
p.set_thread_context(idx, tls_bench)
|
||||
}
|
||||
tls_bench.cstep = idx
|
||||
dot_relative_file := p.get_string_item(idx)
|
||||
relative_file := dot_relative_file.replace('./', '')
|
||||
file := os.realpath(relative_file)
|
||||
// Ensure that the generated binaries will be stored in the temporary folder.
|
||||
// Remove them after a test passes/fails.
|
||||
fname := filepath.filename(file)
|
||||
generated_binary_fname := if os.user_os() == 'windows' { fname.replace('.v', '.exe') } else { fname.replace('.v', '') }
|
||||
generated_binary_fpath := filepath.join(tmpd,generated_binary_fname)
|
||||
if os.exists(generated_binary_fpath) {
|
||||
os.rm(generated_binary_fpath)
|
||||
}
|
||||
mut cmd_options := [ts.vargs]
|
||||
if !ts.vargs.contains('fmt') {
|
||||
cmd_options << ' -o "$generated_binary_fpath"'
|
||||
}
|
||||
cmd := '"${ts.vexe}" ' + cmd_options.join(' ') + ' "${file}"'
|
||||
// eprintln('>>> v cmd: $cmd')
|
||||
ts.benchmark.step()
|
||||
tls_bench.step()
|
||||
if show_stats {
|
||||
eprintln(term.h_divider('-'))
|
||||
status := os.system(cmd)
|
||||
if status == 0 {
|
||||
ts.benchmark.ok()
|
||||
tls_bench.ok()
|
||||
}
|
||||
else {
|
||||
r := os.exec(cmd) or {
|
||||
ts.failed = true
|
||||
ts.benchmark.fail()
|
||||
tls_bench.fail()
|
||||
eprintln(tls_bench.step_message_fail(relative_file))
|
||||
continue
|
||||
}
|
||||
if r.exit_code != 0 {
|
||||
ts.failed = true
|
||||
ts.benchmark.fail()
|
||||
tls_bench.fail()
|
||||
eprintln(tls_bench.step_message_fail('${relative_file}\n`$file`\n (\n$r.output\n)'))
|
||||
}
|
||||
else {
|
||||
ts.benchmark.ok()
|
||||
tls_bench.ok()
|
||||
if ts.show_ok_tests {
|
||||
eprintln(tls_bench.step_message_ok(relative_file))
|
||||
}
|
||||
}
|
||||
}
|
||||
if os.exists(generated_binary_fpath) {
|
||||
os.rm(generated_binary_fpath)
|
||||
ts.failed = true
|
||||
ts.benchmark.fail()
|
||||
tls_bench.fail()
|
||||
return sync.no_result
|
||||
}
|
||||
}
|
||||
else {
|
||||
r := os.exec(cmd) or {
|
||||
ts.failed = true
|
||||
ts.benchmark.fail()
|
||||
tls_bench.fail()
|
||||
eprintln(tls_bench.step_message_fail(relative_file))
|
||||
return sync.no_result
|
||||
}
|
||||
if r.exit_code != 0 {
|
||||
ts.failed = true
|
||||
ts.benchmark.fail()
|
||||
tls_bench.fail()
|
||||
eprintln(tls_bench.step_message_fail('${relative_file}\n`$file`\n (\n$r.output\n)'))
|
||||
}
|
||||
else {
|
||||
ts.benchmark.ok()
|
||||
tls_bench.ok()
|
||||
if ts.show_ok_tests {
|
||||
eprintln(tls_bench.step_message_ok(relative_file))
|
||||
}
|
||||
}
|
||||
}
|
||||
if os.exists(generated_binary_fpath) {
|
||||
os.rm(generated_binary_fpath)
|
||||
}
|
||||
return sync.no_result
|
||||
}
|
||||
|
||||
pub fn vlib_should_be_present(parent_dir string) {
|
||||
@ -193,17 +170,17 @@ pub fn v_build_failing(zargs string, folder string) bool {
|
||||
eprintln('v compiler args: "$vargs"')
|
||||
mut session := new_test_session(vargs)
|
||||
files := os.walk_ext(filepath.join(parent_dir,folder), '.v')
|
||||
mut mains := files.filter(!it.contains('modules') && !it.contains('preludes'))
|
||||
$if windows {
|
||||
// skip pico example on windows
|
||||
// there was a bug using filter here
|
||||
mut mains_filtered := []string
|
||||
for file in mains {
|
||||
if !file.ends_with('examples\\pico\\pico.v') {
|
||||
mains_filtered << file
|
||||
mut mains := []string
|
||||
for f in files {
|
||||
if !f.contains('modules') && !f.contains('preludes') {
|
||||
$if windows {
|
||||
// skip pico example on windows
|
||||
if f.ends_with('examples\\pico\\pico.v') {
|
||||
continue
|
||||
}
|
||||
}
|
||||
mains << f
|
||||
}
|
||||
mains = mains_filtered
|
||||
}
|
||||
session.files << mains
|
||||
session.test()
|
||||
@ -257,9 +234,9 @@ pub fn building_any_v_binaries_failed() bool {
|
||||
}
|
||||
|
||||
pub fn eheader(msg string) {
|
||||
eprintln(term.header(msg,'-'))
|
||||
eprintln(term.header(msg, '-'))
|
||||
}
|
||||
|
||||
pub fn header(msg string) {
|
||||
println(term.header(msg,'-'))
|
||||
println(term.header(msg, '-'))
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user