tests: support for marking _test.v files with // vtest flaky: true, to reduce false positives from the CI

cmd/tools/modules/testing/common.v

@@ -19,6 +19,8 @@ pub const hide_oks = os.getenv('VTEST_HIDE_OK') == '1'
 
 pub const fail_fast = os.getenv('VTEST_FAIL_FAST') == '1'
 
+pub const fail_flaky = os.getenv('VTEST_FAIL_FLAKY') == '1'
+
 pub const test_only = os.getenv('VTEST_ONLY').split_any(',')
 
 pub const test_only_fn = os.getenv('VTEST_ONLY_FN').split_any(',')
@@ -362,7 +364,7 @@ fn worker_trunner(mut p pool.PoolProcessor, idx int, thread_id int) voidptr {
 			details := get_test_details(file)
 			os.setenv('VTEST_RETRY_MAX', '$details.retry', true)
 			for retry := 1; retry <= details.retry; retry++ {
-				ts.append_message(.info, '                 retrying $retry/$details.retry of $relative_file ...')
+				ts.append_message(.info, '  [stats]        retrying $retry/$details.retry of $relative_file ; known flaky: $details.flaky ...')
 				os.setenv('VTEST_RETRY', '$retry', true)
 				status = os.system(cmd)
 				if status == 0 {
@@ -372,6 +374,10 @@ fn worker_trunner(mut p pool.PoolProcessor, idx int, thread_id int) voidptr {
 				}
 				time.sleep(500 * time.millisecond)
 			}
+			if details.flaky && !testing.fail_flaky {
+				ts.append_message(.info, '   *FAILURE* of the known flaky test file $relative_file is ignored, since VTEST_FAIL_FLAKY is 0 . Retry count: $details.retry .')
+				goto test_passed_system
+			}
 			ts.failed = true
 			ts.benchmark.fail()
 			tls_bench.fail()
@@ -399,7 +405,7 @@ fn worker_trunner(mut p pool.PoolProcessor, idx int, thread_id int) voidptr {
 			details := get_test_details(file)
 			os.setenv('VTEST_RETRY_MAX', '$details.retry', true)
 			for retry := 1; retry <= details.retry; retry++ {
-				ts.append_message(.info, '                 retrying $retry/$details.retry of $relative_file ...')
+				ts.append_message(.info, '                 retrying $retry/$details.retry of $relative_file ; known flaky: $details.flaky ...')
 				os.setenv('VTEST_RETRY', '$retry', true)
 				r = os.execute(cmd)
 				if r.exit_code == 0 {
@@ -408,6 +414,10 @@ fn worker_trunner(mut p pool.PoolProcessor, idx int, thread_id int) voidptr {
 					}
 				}
 			}
+			if details.flaky && !testing.fail_flaky {
+				ts.append_message(.info, '   *FAILURE* of the known flaky test file $relative_file is ignored, since VTEST_FAIL_FLAKY is 0 . Retry count: $details.retry .')
+				goto test_passed_system
+			}
 			ts.failed = true
 			ts.benchmark.fail()
 			tls_bench.fail()
@@ -562,6 +572,7 @@ pub fn setup_new_vtmp_folder() string {
 pub struct TestDetails {
 pub mut:
 	retry int
+	flaky bool // when flaky tests fail, the whole run is still considered successfull, unless VTEST_FAIL_FLAKY is 1
 }
 
 pub fn get_test_details(file string) TestDetails {
@@ -571,6 +582,9 @@ pub fn get_test_details(file string) TestDetails {
 		if line.starts_with('// vtest retry:') {
 			res.retry = line.all_after(':').trim_space().int()
 		}
+		if line.starts_with('// vtest flaky:') {
+			res.flaky = line.all_after(':').trim_space().bool()
+		}
 	}
 	return res
 }

vlib/context/cancel_test.v

@@ -1,3 +1,4 @@
+// vtest flaky: true
 // vtest retry: 3
 import context
 

vlib/net/tcp_test.v

@@ -1,3 +1,4 @@
+// vtest flaky: true
 // vtest retry: 8
 import net
 import os

vlib/os/notify/notify_test.v

@@ -1,3 +1,4 @@
+// vtest flaky: true
 // vtest retry: 3
 import os
 import os.notify

vlib/os/process_test.v

@@ -1,3 +1,4 @@
+// vtest flaky: true
 // vtest retry: 3
 import os
 import time

vlib/sync/channel_select_6_test.v

@@ -1,3 +1,4 @@
+// vtest flaky: true
 // vtest retry: 15
 
 // This test case runs concurrent 3 instances of `do_select` that

vlib/sync/channel_select_test.v

@@ -1,5 +1,6 @@
 module sync
 
+// vtest flaky: true
 // vtest retry: 6
 
 // ATTENTION! Do not use this file as an example!

vlib/v/live/live_test.v

@@ -1,6 +1,7 @@
 import os
 import time
 
+// vtest flaky: true
 // vtest retry: 4
 
 /*

vlib/v/pref/options_test.v

@@ -1,3 +1,4 @@
+// vtest flaky: true
 // vtest retry: 3
 import os
 import time

vlib/v/tests/go_anon_fn_variable_call_test.v

@@ -1,3 +1,4 @@
+// vtest flaky: true
 // vtest retry: 3
 
 fn sum1(a int, b int) int {

vlib/v/tests/go_array_wait_test.v

@@ -1,3 +1,4 @@
+// vtest flaky: true
 // vtest retry: 3
 
 fn f(x f64) f64 {

vlib/v/tests/go_wait_option_test.v

@@ -1,3 +1,4 @@
+// vtest flaky: true
 // vtest retry: 3
 
 fn f(n int) ?f64 {

vlib/v/tests/init_global_test.v

@@ -1,3 +1,4 @@
+// vtest flaky: true
 // vtest retry: 4
 import sync
 

vlib/v/tests/keep_args_alive_test.v

@@ -1,3 +1,4 @@
+// vtest flaky: true
 // vtest retry: 4
 /*
 * To verify the effect of "[keep_args_alive]", this attribute may be commented out.

vlib/v/tests/semaphore_timed_test.v

@@ -1,3 +1,4 @@
+// vtest flaky: true
 // vtest retry: 3
 import sync
 import time