cgen,pref,preludes: implement v -assert continues file_test.v (#15976)

cmd/tools/test_if_v_test_system_works.v

@@ -6,7 +6,7 @@ import os
 import rand
 
 const (
-	vexe  = get_vexe_path()
+	vexe  = os.quoted_path(get_vexe_path())
 	vroot = os.dir(vexe)
 	tdir  = new_tdir()
 )
@@ -38,34 +38,57 @@ fn cleanup_tdir() {
 	os.rmdir_all(tdir) or { eprintln(err) }
 }
 
+type MyResult = string
+
+[noreturn]
+fn (result MyResult) fail(reason string) {
+	eprintln('> $reason, but it does not. Result:\n$result')
+	exit(1)
+}
+
+fn (result MyResult) has(sub string) MyResult {
+	if !result.contains(sub) {
+		result.fail(' result should have the substring `$sub`')
+	}
+	return result
+}
+
+fn (result MyResult) matches(gpattern string) MyResult {
+	if !result.match_glob(gpattern) {
+		result.fail('result should match the glob pattern `$gpattern`')
+	}
+	return result
+}
+
 fn create_test(tname string, tcontent string) ?string {
 	tpath := os.join_path(tdir, tname)
 	os.write_file(tpath, tcontent)?
 	eprintln('>>>>>>>> tpath: $tpath | tcontent: $tcontent')
-	return tpath
+	return os.quoted_path(tpath)
 }
 
-fn main() {
-	defer {
-		os.chdir(os.wd_at_startup) or {}
+fn check_assert_continues_works() ? {
+	os.chdir(tdir)?
+	create_test('assert_continues_option_works_test.v', 'fn test_fail1() { assert 2==4\nassert 2==1\nassert 2==0 }\nfn test_ok(){ assert true }\nfn test_fail2() { assert false }')?
+	result := check_fail('$vexe -assert continues assert_continues_option_works_test.v')
+	result.has('assert_continues_option_works_test.v:1: fn test_fail1')
+	result.has('assert_continues_option_works_test.v:2: fn test_fail1')
+	result.has('assert_continues_option_works_test.v:3: fn test_fail1')
+	result.has('assert_continues_option_works_test.v:5: fn test_fail2')
+	result.has('> assert 2 == 4').has('> assert 2 == 1').has('> assert 2 == 0')
+	// Check if a test function, tagged with [assert_continues], has the same behaviour, without needing additional options
+	create_test('assert_continues_tag_works_test.v', '[assert_continues]fn test_fail1() { assert 2==4\nassert 2==1\nassert 2==0 }\nfn test_ok(){ assert true }\nfn test_fail2() { assert false\n assert false }')?
+	tag_res := check_fail('$vexe assert_continues_tag_works_test.v')
+	tag_res.has('assert_continues_tag_works_test.v:1: fn test_fail1')
+	tag_res.has('assert_continues_tag_works_test.v:2: fn test_fail1')
+	tag_res.has('assert_continues_tag_works_test.v:3: fn test_fail1')
+	tag_res.has('assert_continues_tag_works_test.v:5: fn test_fail2')
+	if tag_res.contains('assert_continues_tag_works_test.v:6: fn test_fail2') {
+		exit(1)
 	}
-	println('> vroot: $vroot | vexe: $vexe | tdir: $tdir')
-	ok_fpath := create_test('a_single_ok_test.v', 'fn test_ok(){ assert true }')?
-	check_ok('"$vexe" "$ok_fpath"')
-	check_ok('"$vexe" test "$ok_fpath"')
-	check_ok('"$vexe" test "$tdir"')
-	fail_fpath := create_test('a_single_failing_test.v', 'fn test_fail(){ assert 1 == 2 }')?
-	check_fail('"$vexe" "$fail_fpath"')
-	check_fail('"$vexe" test "$fail_fpath"')
-	check_fail('"$vexe" test "$tdir"')
-	rel_dir := os.join_path(tdir, rand.ulid())
-	os.mkdir(rel_dir)?
-	os.chdir(rel_dir)?
-	check_ok('"$vexe" test "..${os.path_separator + os.base(ok_fpath)}"')
-	println('> all done')
 }
 
-fn check_ok(cmd string) string {
+fn check_ok(cmd string) MyResult {
 	println('>   check_ok cmd: $cmd')
 	res := os.execute(cmd)
 	if res.exit_code != 0 {
@@ -75,7 +98,7 @@ fn check_ok(cmd string) string {
 	return res.output
 }
 
-fn check_fail(cmd string) string {
+fn check_fail(cmd string) MyResult {
 	println('> check_fail cmd: $cmd')
 	res := os.execute(cmd)
 	if res.exit_code == 0 {
@@ -84,3 +107,29 @@ fn check_fail(cmd string) string {
 	}
 	return res.output
 }
+
+fn main() {
+	defer {
+		os.chdir(os.wd_at_startup) or {}
+	}
+	println('> vroot: $vroot | vexe: $vexe | tdir: $tdir')
+	ok_fpath := create_test('a_single_ok_test.v', 'fn test_ok(){ assert true }')?
+	if check_ok('$vexe $ok_fpath') != '' {
+		exit(1)
+	}
+	check_ok('$vexe test $ok_fpath').matches('*OK*a_single_ok_test.v*')
+	check_ok('$vexe test "$tdir"').matches('*OK*a_single_ok_test.v*')
+	//
+	fail_fpath := create_test('a_single_failing_test.v', 'fn test_fail(){ assert 1 == 2 }')?
+	check_fail('$vexe $fail_fpath').has('> assert 1 == 2').has('a_single_failing_test.v:1: fn test_fail')
+	check_fail('$vexe test $fail_fpath').has('> assert 1 == 2').has('a_single_failing_test.v:1: fn test_fail')
+	check_fail('$vexe test "$tdir"').has('> assert 1 == 2')
+	rel_dir := os.join_path(tdir, rand.ulid())
+	os.mkdir(rel_dir)?
+	os.chdir(rel_dir)?
+	relative_path := '..' + os.path_separator + 'a_single_ok_test.v'
+	check_ok('$vexe test ${os.quoted_path(relative_path)}').has('OK').has('a_single_ok_test.v')
+	//
+	check_assert_continues_works()?
+	println('> all done')
+}

cmd/v/help/build-c.txt

@@ -291,6 +291,10 @@ see also `v help build`.
       backtraces are not implemented yet on all combinations of
       platform/compiler.
 
+   -assert continues
+      Just prints the failed assertion then continues. Useful if you want to see
+      the failures of many assertions that are all in the same test_ function.
+
    -thread-stack-size 4194304
       Set the thread stack size to 4MB. Use multiples of 4096.
       The default is 8MB, which is enough for compiling V programs, with deeply

doc/docs.md

@@ -96,6 +96,11 @@ To do so, run the command `v up`.
 	* [Decoding JSON](#decoding-json)
 	* [Encoding JSON](#encoding-json)
 * [Testing](#testing)
+    * [Asserts](#asserts)
+    * [Asserts with an extra message](#asserts-with-an-extra-message)
+    * [Asserts that do not abort your program](#asserts-that-do-not-abort-your-program)
+    * [Test files](#test-files)
+    * [Running tests](#running-tests)
 * [Memory management](#memory-management)
     * [Stack and Heap](#stack-and-heap)
 * [ORM](#orm)
@@ -4085,10 +4090,13 @@ foo(mut v)
 assert v[0] < 4
 ```
 An `assert` statement checks that its expression evaluates to `true`. If an assert fails,
-the program will abort. Asserts should only be used to detect programming errors. When an
+the program will usually abort. Asserts should only be used to detect programming errors. When an
 assert fails it is reported to *stderr*, and the values on each side of a comparison operator
 (such as `<`, `==`) will be printed when possible. This is useful to easily find an
-unexpected value. Assert statements can be used in any function.
+unexpected value. Assert statements can be used in any function, not just test ones,
+which is handy when developing new functionality, to keep your invariants in check.
+
+Note: all `assert` statements are *removed*, when you compile your program with the `-prod` flag.
 
 ### Asserts with an extra message
 
@@ -4104,6 +4112,37 @@ fn test_assertion_with_extra_message_failure() {
 }
 ```
 
+### Asserts that do not abort your program
+When initially prototyping functionality and tests, it is sometimes desirable to
+have asserts, that do not stop the program, but just print their failures. That can
+be achieved by tagging your assert containing functions with an `[assert_continues]`
+tag, for example running this program:
+```v
+[assert_continues]
+fn abc(ii int) {
+	assert ii == 2
+}
+
+for i in 0 .. 4 {
+	abc(i)
+}
+```
+... will produce this output:
+```
+assert_continues_example.v:3: FAIL: fn main.abc: assert ii == 2
+   left value: ii = 0
+   right value: 2
+assert_continues_example.v:3: FAIL: fn main.abc: assert ii == 2
+   left value: ii = 1
+  right value: 2
+assert_continues_example.v:3: FAIL: fn main.abc: assert ii == 2
+   left value: ii = 3
+  right value: 2
+```			   
+
+Note: V also supports a command line flag `-assert continues`, which will change the
+behaviour of all asserts globally, as if you had tagged every function with `[assert_continues]`.
+
 ### Test files
 
 ```v

examples/js_dom_draw_bechmark_chart/chart/main.v

@@ -104,12 +104,12 @@ pub fn (mut app App) controller_get_all_task() ?vweb.Result {
 
 		for key, values in framework_platform[orm_stmt_kind] {
 			attribute_names[orm_stmt_kind] << key
-			maxs[orm_stmt_kind] << arrays.max(values)?
+			maxs[orm_stmt_kind] << arrays.max(values) or { continue }
 		}
 
-		max_benchmark[orm_stmt_kind] = arrays.max(maxs[orm_stmt_kind])?
 		from_framework[orm_stmt_kind] = json.encode(framework_platform[orm_stmt_kind])
 		table[orm_stmt_kind] = gen_table_info(attribute_names[orm_stmt_kind], framework_platform[orm_stmt_kind])
+		max_benchmark[orm_stmt_kind] = arrays.max(maxs[orm_stmt_kind]) or { continue }
 	}
 
 	return $vweb.html()

vlib/v/gen/c/assert.v

@@ -32,9 +32,6 @@ fn (mut g Gen) assert_stmt(original_assert_statement ast.AssertStmt) {
 		metaname_fail := g.gen_assert_metainfo(node)
 		g.writeln('\tmain__TestRunner_name_table[test_runner._typ]._method_assert_fail(test_runner._object, &$metaname_fail);')
 		g.gen_assert_postfailure_mode(node)
-		g.writeln('\tlongjmp(g_jump_buffer, 1);')
-		g.writeln('\t// TODO')
-		g.writeln('\t// Maybe print all vars in a test function if it fails?')
 		g.writeln('}')
 	} else {
 		g.write('if (!(')
@@ -45,7 +42,6 @@ fn (mut g Gen) assert_stmt(original_assert_statement ast.AssertStmt) {
 		metaname_panic := g.gen_assert_metainfo(node)
 		g.writeln('\t__print_assert_failure(&$metaname_panic);')
 		g.gen_assert_postfailure_mode(node)
-		g.writeln('\t_v_panic(_SLIT("Assertion failed..."));')
 		g.writeln('}')
 	}
 }
@@ -84,14 +80,24 @@ fn (mut g Gen) assert_subexpression_to_ctemp(expr ast.Expr, expr_type ast.Type)
 
 fn (mut g Gen) gen_assert_postfailure_mode(node ast.AssertStmt) {
 	g.write_v_source_line_info(node.pos)
-	match g.pref.assert_failure_mode {
-		.default {}
-		.aborts {
-			g.writeln('\tabort();')
-		}
-		.backtraces {
-			g.writeln('\tprint_backtrace();')
-		}
+	if g.pref.assert_failure_mode == .continues
+		|| g.fn_decl.attrs.any(it.name == 'assert_continues') {
+		return
+	}
+	if g.pref.assert_failure_mode == .aborts || g.fn_decl.attrs.any(it.name == 'assert_aborts') {
+		g.writeln('\tabort();')
+	}
+	if g.pref.assert_failure_mode == .backtraces
+		|| g.fn_decl.attrs.any(it.name == 'assert_backtraces') {
+		g.writeln('\tprint_backtrace();')
+	}
+	if g.pref.is_test {
+		g.writeln('\tlongjmp(g_jump_buffer, 1);')
+	}
+	g.writeln('\t// TODO')
+	g.writeln('\t// Maybe print all vars in a test function if it fails?')
+	if g.pref.assert_failure_mode != .continues {
+		g.writeln('\t_v_panic(_SLIT("Assertion failed..."));')
 	}
 }
 

vlib/v/pref/pref.v

@@ -21,6 +21,7 @@ pub enum AssertFailureMode {
 	default
 	aborts
 	backtraces
+	continues
 }
 
 pub enum GarbageCollectionMode {
@@ -276,10 +277,14 @@ pub fn parse_args_and_show_errors(known_external_commands []string, args []strin
 					'backtraces' {
 						res.assert_failure_mode = .backtraces
 					}
+					'continues' {
+						res.assert_failure_mode = .continues
+					}
 					else {
 						eprintln('unknown assert mode `-gc $assert_mode`, supported modes are:`')
 						eprintln('  `-assert aborts`     .... calls abort() after assertion failure')
 						eprintln('  `-assert backtraces` .... calls print_backtrace() after assertion failure')
+						eprintln('  `-assert continues`  .... does not call anything, just continue after an assertion failure')
 						exit(1)
 					}
 				}

vlib/v/preludes/test_runner_normal.v

@@ -70,6 +70,9 @@ fn (mut runner NormalTestRunner) exit_code() int {
 	if runner.fn_fails > 0 {
 		return 1
 	}
+	if runner.total_assert_fails > 0 {
+		return 2
+	}
 	return 0
 }
 

vlib/v/preludes/test_runner_simple.v

@@ -44,6 +44,9 @@ fn (mut runner SimpleTestRunner) exit_code() int {
 	if runner.fn_fails > 0 {
 		return 1
 	}
+	if runner.total_assert_fails > 0 {
+		return 2
+	}
 	return 0
 }
 

vlib/v/preludes/test_runner_tap.v

@@ -67,6 +67,9 @@ fn (mut runner TAPTestRunner) exit_code() int {
 	if runner.fn_fails > 0 {
 		return 1
 	}
+	if runner.total_assert_fails > 0 {
+		return 2
+	}
 	return 0
 }