v/cmd/tools/repeat.v

module main

import os
import flag
import time
import term
import math
import scripting
import v.util

struct CmdResult {
mut:
	runs    int
	cmd     string
	icmd    int
	outputs []string
	oms     map[string][]int
	summary map[string]Aints
	timings []int
	atiming Aints
}

struct Context {
mut:
	count                   int
	series                  int
	warmup                  int
	show_help               bool
	show_output             bool
	fail_on_regress_percent int
	fail_on_maxtime         int // in ms
	verbose                 bool
	commands                []string
	results                 []CmdResult
	cline                   string // a terminal clearing line
}

struct Aints {
	values  []int
mut:
	imin    int
	imax    int
	average f64
	stddev  f64
}

fn new_aints(vals []int) Aints {
	mut res := Aints{
		values: vals
	}
	mut sum := i64(0)
	mut imin := math.max_i32
	mut imax := -math.max_i32
	for i in vals {
		sum += i
		if i < imin {
			imin = i
		}
		if i > imax {
			imax = i
		}
	}
	res.imin = imin
	res.imax = imax
	if vals.len > 0 {
		res.average = sum / f64(vals.len)
	}
	//
	mut devsum := f64(0.0)
	for i in vals {
		x := f64(i) - res.average
		devsum += (x * x)
	}
	res.stddev = math.sqrt(devsum / f64(vals.len))
	return res
}

fn (a Aints) str() string {
	return util.bold('${a.average:9.3f}') +
		'ms ± σ: ${a.stddev:-5.1f}ms, min … max: ${a.imin}ms … ${a.imax}ms'
}

const (
	max_fail_percent             = 100000
	max_time                     = 60 * 1000 // ms
	performance_regression_label = 'Performance regression detected, failing since '
)

fn main() {
	mut context := Context{}
	context.parse_options()
	context.run()
	context.show_diff_summary()
}

fn (mut context Context) parse_options() {
	mut fp := flag.new_flag_parser(os.args)
	fp.application(os.file_name(os.executable()))
	fp.version('0.0.1')
	fp.description('Repeat command(s) and collect statistics. NB: you have to quote each command, if it contains spaces.')
	fp.arguments_description('CMD1 CMD2 ...')
	fp.skip_executable()
	fp.limit_free_args_to_at_least(1)
	context.count = fp.int('count', `c`, 10, 'Repetition count.')
	context.series = fp.int('series', `s`, 2, 'Series count. `-s 2 -c 4 a b` => aaaabbbbaaaabbbb, while `-s 3 -c 2 a b` => aabbaabbaabb.')
	context.warmup = fp.int('warmup', `w`, 2, 'Warmup runs. These are done *only at the start*, and are ignored.')
	context.show_help = fp.bool('help', `h`, false, 'Show this help screen.')
	context.show_output = fp.bool('output', `O`, false, 'Show command stdout/stderr in the progress indicator for each command. NB: slower, for verbose commands.')
	context.verbose = fp.bool('verbose', `v`, false, 'Be more verbose.')
	context.fail_on_maxtime = fp.int('max_time', `m`, max_time, 'Fail with exit code 2, when first cmd takes above M milliseconds (regression).')
	context.fail_on_regress_percent = fp.int('fail_percent', `f`, max_fail_percent, 'Fail with exit code 3, when first cmd is X% slower than the rest (regression).')
	if context.show_help {
		println(fp.usage())
		exit(0)
	}
	if context.verbose {
		scripting.set_verbose(true)
	}
	context.commands = fp.finalize() or {
		eprintln('Error: ' + err)
		exit(1)
	}
	context.results = []CmdResult{len: context.commands.len, init: CmdResult{}}
	context.cline = '\r' + term.h_divider('')
}

fn (mut context Context) clear_line() {
	print(context.cline)
}

fn (mut context Context) run() {
	mut run_warmups := 0
	for si in 1 .. context.series + 1 {
		for icmd, cmd in context.commands {
			mut runs := 0
			mut duration := 0
			mut sum := 0
			mut oldres := ''
			println('Series: ${si:4}/${context.series:-4}, command: $cmd')
			if context.warmup > 0 && run_warmups < context.commands.len {
				for i in 1 .. context.warmup + 1 {
					print('\r warming up run: ${i:4}/${context.warmup:-4} for ${cmd:-50s} took ${duration:6} ms ...')
					mut sw := time.new_stopwatch({})
					os.exec(cmd) or {
						continue
					}
					duration = int(sw.elapsed().milliseconds())
				}
				run_warmups++
			}
			context.clear_line()
			for i in 1 .. (context.count + 1) {
				avg := f64(sum) / f64(i)
				print('\rAverage: ${avg:9.3f}ms | run: ${i:4}/${context.count:-4} | took ${duration:6} ms')
				if context.show_output {
					print(' | result: ${oldres:s}')
				}
				mut sw := time.new_stopwatch({})
				res := scripting.exec(cmd) or {
					continue
				}
				duration = int(sw.elapsed().milliseconds())
				if res.exit_code != 0 {
					eprintln('${i:10} non 0 exit code for cmd: $cmd')
					continue
				}
				context.results[icmd].outputs <<
					res.output.trim_right('\r\n').replace('\r\n', '\n').split('\n')
				context.results[icmd].timings << duration
				sum += duration
				runs++
				oldres = res.output.replace('\n', ' ')
			}
			context.results[icmd].cmd = cmd
			context.results[icmd].icmd = icmd
			context.results[icmd].runs += runs
			context.results[icmd].atiming = new_aints(context.results[icmd].timings)
			context.clear_line()
			print('\r')
			mut m := map[string][]int{}
			for o in context.results[icmd].outputs {
				x := o.split(':')
				if x.len > 1 {
					k := x[0]
					v := x[1].trim_left(' ').int()
					m[k] << v
				}
			}
			mut summary := map[string]Aints{}
			for k, v in m {
				// show a temporary summary for the current series/cmd cycle
				s := new_aints(v)
				println('  $k: $s')
				summary[k] = s
			}
			// merge current raw results to the previous ones
			old_oms := context.results[icmd].oms
			mut new_oms := map[string][]int{}
			for k, v in m {
				if old_oms[k].len == 0 {
					new_oms[k] = v
				} else {
					new_oms[k] << old_oms[k]
					new_oms[k] << v
				}
			}
			context.results[icmd].oms = new_oms
			// println('')
		}
	}
	// create full summaries, taking account of all runs
	for icmd in 0 .. context.results.len {
		mut new_full_summary := map[string]Aints{}
		for k, v in context.results[icmd].oms {
			new_full_summary[k] = new_aints(v)
		}
		context.results[icmd].summary = new_full_summary
	}
}

fn (mut context Context) show_diff_summary() {
	context.results.sort_with_compare(fn (a &CmdResult, b &CmdResult) int {
		if a.atiming.average < b.atiming.average {
			return -1
		}
		if a.atiming.average > b.atiming.average {
			return 1
		}
		return 0
	})
	println('Summary (commands are ordered by ascending mean time), after $context.series series of $context.count repetitions:')
	base := context.results[0].atiming.average
	mut first_cmd_percentage := f64(100.0)
	for i, r in context.results {
		cpercent := (r.atiming.average / base) * 100 - 100
		first_marker := if r.icmd == 0 { util.bold('>') } else { ' ' }
		if r.icmd == 0 {
			first_cmd_percentage = cpercent
		}
		println(' $first_marker${(i+1):3} | ${cpercent:6.1f}% slower | ${r.cmd:-55s} | $r.atiming')
	}
	$if debugcontext ? {
		println('context: $context')
	}
	eprintln('base: $base | context.fail_on_maxtime: $context.fail_on_maxtime')
	if int(base) > context.fail_on_maxtime {
		print(performance_regression_label)
		println('average time: ${base:6.1f} ms > $context.fail_on_maxtime ms threshold.')
		exit(2)
	}
	if context.fail_on_regress_percent == max_fail_percent || context.results.len < 2 {
		return
	}
	fail_threshold_max := f64(context.fail_on_regress_percent)
	if first_cmd_percentage > fail_threshold_max {
		print(performance_regression_label)
		println('${first_cmd_percentage:5.1f}% > ${fail_threshold_max:5.1f}% threshold.')
		exit(3)
	}
}