mirror/v
1
0
Fork 0
mirror of https://github.com/vlang/v.git synced 2023-08-10 21:13:21 +03:00
Code Issues Projects Releases Wiki Activity

repeat: implement -nmaxs and -nmins to eliminate flukes

Browse Source
This commit is contained in:
Delyan Angelov 2020-12-04 19:18:58 +02:00
parent ac7440cf92
commit 9ea2608372
No known key found for this signature in database
GPG Key ID: 66886C0F12D595ED
2 changed files with 31 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/ci.yml vendored
View File

@ -63,9 +63,9 @@ jobs:
- name: Build the repeat tool
run: ./v cmd/tools/repeat.v
- name: Repeat -o hw.c examples/hello_world.v
run: cmd/tools/repeat --max_time 150 --series 3 --count 15 --warmup 3 --fail_percent 10 './v -show-timings -o hw.c examples/hello_world.v' './vmaster/v -show-timings -o hw.c examples/hello_world.v'
run: cmd/tools/repeat --max_time 150 --series 3 --count 10 --nmaxs 5 --warmup 3 --fail_percent 10 -t '{T} -show-timings -o hw.c examples/hello_world.v' ./v ./vmaster/v
- name: Repeat -o v.c cmd/v
run: cmd/tools/repeat --max_time 750 --series 3 --count 15 --warmup 3 --fail_percent 10 './v -show-timings -o v.c cmd/v' './vmaster/v -show-timings -o v.c cmd/v'
run: cmd/tools/repeat --max_time 750 --series 3 --count 15 --nmaxs 5 --warmup 3 --fail_percent 10 -t '{T} -show-timings -o v.c cmd/v' ./v ./vmaster/v
ubuntu-tcc:
runs-on: ubuntu-18.04

39
cmd/tools/repeat.v
View File

@ -37,6 +37,8 @@ mut:
cmd_params map[string][]string
cline string // a terminal clearing line
cgoback string
nmins int // number of minimums to discard
nmaxs int // number of maximums to discard
}
struct Aints {
@ -46,15 +48,28 @@ mut:
imax int
average f64
stddev f64
nmins int // number of discarded fastest results
nmaxs int // number of discarded slowest results
}
fn new_aints(vals []int) Aints {
fn new_aints(ovals []int, extreme_mins int, extreme_maxs int) Aints {
mut res := Aints{
values: vals
values: ovals // remember the original values
nmins: extreme_mins
nmaxs: extreme_maxs
}
mut sum := i64(0)
mut imin := math.max_i32
mut imax := -math.max_i32
// discard the extremes:
mut vals := ovals.clone()
vals.sort()
if vals.len > extreme_mins + extreme_maxs {
vals = vals[extreme_mins..vals.len - extreme_maxs]
} else {
vals = []
}
// statistical processing of the remaining values:
for i in vals {
sum += i
if i < imin {
@ -76,16 +91,17 @@ fn new_aints(vals []int) Aints {
devsum += (x * x)
}
res.stddev = math.sqrt(devsum / f64(vals.len))
// eprintln('\novals: $ovals\n vals: $vals\n vals.len: $vals.len | res.imin: $res.imin | res.imax: $res.imax | res.average: $res.average | res.stddev: $res.stddev')
return res
}
fn (a Aints) str() string {
return util.bold('${a.average:9.3f}') +
'ms ± σ: ${a.stddev:-5.1f}ms, min … max: ${a.imin}ms … ${a.imax}ms'
return util.bold('${a.average:6.2f}') +
'ms ± σ: ${a.stddev:4.1f}ms, min: ${a.imin:4}ms, max: ${a.imax:4}ms, runs:${a.values.len:3}, nmins:${a.nmins:2}, nmaxs:${a.nmaxs:2}'
}
const (
max_fail_percent = 100000
max_fail_percent = 100 * 1000
max_time = 60 * 1000 // ms
performance_regression_label = 'Performance regression detected, failing since '
)
@ -109,13 +125,15 @@ fn (mut context Context) parse_options() {
context.series = fp.int('series', `s`, 2, 'Series count. `-s 2 -c 4 a b` => aaaabbbbaaaabbbb, while `-s 3 -c 2 a b` => aabbaabbaabb.')
context.warmup = fp.int('warmup', `w`, 2, 'Warmup runs. These are done *only at the start*, and are ignored.')
context.show_help = fp.bool('help', `h`, false, 'Show this help screen.')
context.use_newline = fp.bool('newline', `N`, false, 'Use \\n, do not overwrite the last line. Produces more output, but easier to diagnose.')
context.use_newline = fp.bool('newline', `n`, false, 'Use \\n, do not overwrite the last line. Produces more output, but easier to diagnose.')
context.show_output = fp.bool('output', `O`, false, 'Show command stdout/stderr in the progress indicator for each command. NB: slower, for verbose commands.')
context.verbose = fp.bool('verbose', `v`, false, 'Be more verbose.')
context.fail_on_maxtime = fp.int('max_time', `m`, max_time, 'Fail with exit code 2, when first cmd takes above M milliseconds (regression).')
context.fail_on_regress_percent = fp.int('fail_percent', `f`, max_fail_percent, 'Fail with exit code 3, when first cmd is X% slower than the rest (regression).')
context.cmd_template = fp.string('template', `t`, '{T}', 'Command template. {T} will be substituted with the current command.')
cmd_params := fp.string_multi('parameter', `p`, 'A parameter substitution list. `{p}=val1,val2,val2` means that {p} in the template, will be substituted with each of val1, val2, val3.')
context.nmins = fp.int('nmins', `i`, 0, 'Ignore the BOTTOM X results (minimum execution time). Makes the results more robust to performance flukes.')
context.nmaxs = fp.int('nmaxs', `a`, 1, 'Ignore the TOP X results (maximum execution time). Makes the results more robust to performance flukes.')
for p in cmd_params {
parts := p.split(':')
if parts.len > 1 {
@ -217,7 +235,8 @@ fn (mut context Context) run() {
context.results[icmd].cmd = cmd
context.results[icmd].icmd = icmd
context.results[icmd].runs += runs
context.results[icmd].atiming = new_aints(context.results[icmd].timings)
context.results[icmd].atiming = new_aints(context.results[icmd].timings, context.nmins,
context.nmaxs)
context.clear_line()
print(context.cgoback)
mut m := map[string][]int{}
@ -232,7 +251,7 @@ fn (mut context Context) run() {
mut summary := map[string]Aints{}
for k, v in m {
// show a temporary summary for the current series/cmd cycle
s := new_aints(v)
s := new_aints(v, context.nmins, context.nmaxs)
println(' $k: $s')
summary[k] = s
}
@ -255,7 +274,7 @@ fn (mut context Context) run() {
for icmd in 0 .. context.results.len {
mut new_full_summary := map[string]Aints{}
for k, v in context.results[icmd].oms {
new_full_summary[k] = new_aints(v)
new_full_summary[k] = new_aints(v, context.nmins, context.nmaxs)
}
context.results[icmd].summary = new_full_summary
}
@ -280,7 +299,7 @@ fn (mut context Context) show_diff_summary() {
if r.icmd == 0 {
first_cmd_percentage = cpercent
}
println(' $first_marker${(i + 1):3} | ${cpercent:6.1f}% slower | ${r.cmd:-55s} | $r.atiming')
println(' $first_marker${(i + 1):3} | ${cpercent:5.1f}% slower | ${r.cmd:-57s} | $r.atiming')
}
$if debugcontext ? {
println('context: $context')