repeat.v: support --max_time limit for the average time of a command

.github/workflows/ci.yml

@@ -39,9 +39,9 @@ jobs:
     - name: Build the repeat tool
       run: ./v cmd/tools/repeat.v
     - name: Repeat -o hw.c examples/hello_world.v
-      run: cmd/tools/repeat --series 3 --count 15 --warmup 3 --fail_percent 10 './v -show-timings -o hw.c examples/hello_world.v' './vmaster/v -show-timings -o hw.c examples/hello_world.v'
+      run: cmd/tools/repeat --max_time 150 --series 3 --count 15 --warmup 3 --fail_percent 10 './v -show-timings -o hw.c examples/hello_world.v' './vmaster/v -show-timings -o hw.c examples/hello_world.v'
     - name: Repeat -o v.c cmd/v
-      run: cmd/tools/repeat --series 3 --count 15 --warmup 3 --fail_percent 10 './v -show-timings -o v.c cmd/v' './vmaster/v -show-timings -o v.c cmd/v'
+      run: cmd/tools/repeat --max_time 750 --series 3 --count 15 --warmup 3 --fail_percent 10 './v -show-timings -o v.c cmd/v' './vmaster/v -show-timings -o v.c cmd/v'
 
   ubuntu-tcc:
     runs-on: ubuntu-18.04

cmd/tools/modules/scripting/scripting.v

@@ -61,6 +61,16 @@ pub fn rmrf(path string) {
 	}
 }
 
+pub fn exec(cmd string) ?os.Result {
+	verbose_trace(@FN, cmd)
+	x := os.exec(cmd) or {
+		verbose_trace(@FN, '## failed.')
+		return error(err)
+	}
+	verbose_trace_exec_result(x)
+	return x
+}
+
 pub fn run(cmd string) string {
 	verbose_trace(@FN, cmd)
 	x := os.exec(cmd) or {

cmd/tools/repeat.v

@@ -27,6 +27,7 @@ mut:
 	show_help bool
 	show_output bool
 	fail_on_regress_percent int
+	fail_on_maxtime int // in ms
 	verbose bool
 	commands []string
 	results []CmdResult
@@ -73,6 +74,8 @@ fn (a Aints) str() string { return util.bold('${a.average:9.3f}') + 'ms ± σ: $
 
 const (
 	max_fail_percent = 100000
+	max_time = 60*1000 // ms
+	performance_regression_label = 'Performance regression detected, failing since '
 )
 fn main(){
 	mut context := Context{}
@@ -95,7 +98,8 @@ fn (mut context Context) parse_options() {
 	context.show_help = fp.bool('help', `h`, false, 'Show this help screen.')
 	context.show_output = fp.bool('output', `O`, false, 'Show command stdout/stderr in the progress indicator for each command. NB: slower, for verbose commands.')
 	context.verbose = fp.bool('verbose', `v`, false, 'Be more verbose.')
-	context.fail_on_regress_percent = fp.int('fail_percent', `f`, max_fail_percent, 'Fail with 1 exit code, when first cmd is X% slower than the rest (regression).')
+	context.fail_on_maxtime = fp.int('max_time', `m`, max_time, 'Fail with exit code 2, when first cmd takes above M milliseconds (regression).')
+	context.fail_on_regress_percent = fp.int('fail_percent', `f`, max_fail_percent, 'Fail with exit code 3, when first cmd is X% slower than the rest (regression).')
 	if context.show_help {
 		println(fp.usage())
 		exit(0)
@@ -141,8 +145,7 @@ fn (mut context Context) run() {
 					print(' | result: ${oldres:-s}')
 				}
 				mut sw := time.new_stopwatch({})
-				res := os.exec(cmd) or {
+				res := scripting.exec(cmd) or {
-					eprintln('${i:10} failed runnning cmd: $cmd')
 					continue
 				}
 				duration = int(sw.elapsed().milliseconds())
@@ -170,7 +173,7 @@ fn (mut context Context) run() {
 					v := x[1].trim_left(' ').int()
 					m[k] << v
 				}
-			}            
+			}
 			mut summary := map[string]Aints{}
 			for k,v in m {
 				// show a temporary summary for the current series/cmd cycle
@@ -223,16 +226,22 @@ fn (mut context Context) show_diff_summary() {
 		}
 		println(' ${first_marker}${(i+1):3} | ${cpercent:6.1f}% slower | ${r.cmd:-55s} | ${r.atiming}')
 	}
+	$if debugcontext ? {
+		println('context: $context')
+	}
+	eprintln('base: $base | context.fail_on_maxtime: $context.fail_on_maxtime')
+	if int(base) > context.fail_on_maxtime {
+		print(performance_regression_label)
+		println('average time: ${base:6.1f} ms > ${context.fail_on_maxtime} ms threshold.')
+		exit(2)
+	}
 	if context.fail_on_regress_percent == max_fail_percent || context.results.len < 2 {
 		return
 	}
-	$if debugcontext ? {
-		println('context: $context')    
-	}
 	fail_threshold_max := f64(context.fail_on_regress_percent)
 	if first_cmd_percentage > fail_threshold_max {
-		print('Performance regression detected, failing since ')
+		print(performance_regression_label)
 		println('${first_cmd_percentage:5.1f}% > ${fail_threshold_max:5.1f}% threshold.')
-		exit(1)
+		exit(3)
 	}
 }