From 88d69d7d54f751798c35106f37e5ca791384bd6d Mon Sep 17 00:00:00 2001 From: Delyan Angelov Date: Thu, 6 Oct 2022 17:20:32 +0300 Subject: [PATCH] cgen,pref,preludes: implement `v -assert continues file_test.v` (#15976) --- cmd/tools/test_if_v_test_system_works.v | 91 ++++++++++++++----- cmd/v/help/build-c.txt | 4 + doc/docs.md | 43 ++++++++- .../js_dom_draw_bechmark_chart/chart/main.v | 4 +- vlib/v/gen/c/assert.v | 30 +++--- vlib/v/pref/pref.v | 5 + vlib/v/preludes/test_runner_normal.v | 3 + vlib/v/preludes/test_runner_simple.v | 3 + vlib/v/preludes/test_runner_tap.v | 3 + 9 files changed, 149 insertions(+), 37 deletions(-) diff --git a/cmd/tools/test_if_v_test_system_works.v b/cmd/tools/test_if_v_test_system_works.v index 909838aab2..9799b6a4bf 100644 --- a/cmd/tools/test_if_v_test_system_works.v +++ b/cmd/tools/test_if_v_test_system_works.v @@ -6,7 +6,7 @@ import os import rand const ( - vexe = get_vexe_path() + vexe = os.quoted_path(get_vexe_path()) vroot = os.dir(vexe) tdir = new_tdir() ) @@ -38,34 +38,57 @@ fn cleanup_tdir() { os.rmdir_all(tdir) or { eprintln(err) } } +type MyResult = string + +[noreturn] +fn (result MyResult) fail(reason string) { + eprintln('> $reason, but it does not. Result:\n$result') + exit(1) +} + +fn (result MyResult) has(sub string) MyResult { + if !result.contains(sub) { + result.fail(' result should have the substring `$sub`') + } + return result +} + +fn (result MyResult) matches(gpattern string) MyResult { + if !result.match_glob(gpattern) { + result.fail('result should match the glob pattern `$gpattern`') + } + return result +} + fn create_test(tname string, tcontent string) ?string { tpath := os.join_path(tdir, tname) os.write_file(tpath, tcontent)? eprintln('>>>>>>>> tpath: $tpath | tcontent: $tcontent') - return tpath + return os.quoted_path(tpath) } -fn main() { - defer { - os.chdir(os.wd_at_startup) or {} +fn check_assert_continues_works() ? { + os.chdir(tdir)? + create_test('assert_continues_option_works_test.v', 'fn test_fail1() { assert 2==4\nassert 2==1\nassert 2==0 }\nfn test_ok(){ assert true }\nfn test_fail2() { assert false }')? + result := check_fail('$vexe -assert continues assert_continues_option_works_test.v') + result.has('assert_continues_option_works_test.v:1: fn test_fail1') + result.has('assert_continues_option_works_test.v:2: fn test_fail1') + result.has('assert_continues_option_works_test.v:3: fn test_fail1') + result.has('assert_continues_option_works_test.v:5: fn test_fail2') + result.has('> assert 2 == 4').has('> assert 2 == 1').has('> assert 2 == 0') + // Check if a test function, tagged with [assert_continues], has the same behaviour, without needing additional options + create_test('assert_continues_tag_works_test.v', '[assert_continues]fn test_fail1() { assert 2==4\nassert 2==1\nassert 2==0 }\nfn test_ok(){ assert true }\nfn test_fail2() { assert false\n assert false }')? + tag_res := check_fail('$vexe assert_continues_tag_works_test.v') + tag_res.has('assert_continues_tag_works_test.v:1: fn test_fail1') + tag_res.has('assert_continues_tag_works_test.v:2: fn test_fail1') + tag_res.has('assert_continues_tag_works_test.v:3: fn test_fail1') + tag_res.has('assert_continues_tag_works_test.v:5: fn test_fail2') + if tag_res.contains('assert_continues_tag_works_test.v:6: fn test_fail2') { + exit(1) } - println('> vroot: $vroot | vexe: $vexe | tdir: $tdir') - ok_fpath := create_test('a_single_ok_test.v', 'fn test_ok(){ assert true }')? - check_ok('"$vexe" "$ok_fpath"') - check_ok('"$vexe" test "$ok_fpath"') - check_ok('"$vexe" test "$tdir"') - fail_fpath := create_test('a_single_failing_test.v', 'fn test_fail(){ assert 1 == 2 }')? - check_fail('"$vexe" "$fail_fpath"') - check_fail('"$vexe" test "$fail_fpath"') - check_fail('"$vexe" test "$tdir"') - rel_dir := os.join_path(tdir, rand.ulid()) - os.mkdir(rel_dir)? - os.chdir(rel_dir)? - check_ok('"$vexe" test "..${os.path_separator + os.base(ok_fpath)}"') - println('> all done') } -fn check_ok(cmd string) string { +fn check_ok(cmd string) MyResult { println('> check_ok cmd: $cmd') res := os.execute(cmd) if res.exit_code != 0 { @@ -75,7 +98,7 @@ fn check_ok(cmd string) string { return res.output } -fn check_fail(cmd string) string { +fn check_fail(cmd string) MyResult { println('> check_fail cmd: $cmd') res := os.execute(cmd) if res.exit_code == 0 { @@ -84,3 +107,29 @@ fn check_fail(cmd string) string { } return res.output } + +fn main() { + defer { + os.chdir(os.wd_at_startup) or {} + } + println('> vroot: $vroot | vexe: $vexe | tdir: $tdir') + ok_fpath := create_test('a_single_ok_test.v', 'fn test_ok(){ assert true }')? + if check_ok('$vexe $ok_fpath') != '' { + exit(1) + } + check_ok('$vexe test $ok_fpath').matches('*OK*a_single_ok_test.v*') + check_ok('$vexe test "$tdir"').matches('*OK*a_single_ok_test.v*') + // + fail_fpath := create_test('a_single_failing_test.v', 'fn test_fail(){ assert 1 == 2 }')? + check_fail('$vexe $fail_fpath').has('> assert 1 == 2').has('a_single_failing_test.v:1: fn test_fail') + check_fail('$vexe test $fail_fpath').has('> assert 1 == 2').has('a_single_failing_test.v:1: fn test_fail') + check_fail('$vexe test "$tdir"').has('> assert 1 == 2') + rel_dir := os.join_path(tdir, rand.ulid()) + os.mkdir(rel_dir)? + os.chdir(rel_dir)? + relative_path := '..' + os.path_separator + 'a_single_ok_test.v' + check_ok('$vexe test ${os.quoted_path(relative_path)}').has('OK').has('a_single_ok_test.v') + // + check_assert_continues_works()? + println('> all done') +} diff --git a/cmd/v/help/build-c.txt b/cmd/v/help/build-c.txt index 64f693bb55..87b70d7b75 100644 --- a/cmd/v/help/build-c.txt +++ b/cmd/v/help/build-c.txt @@ -291,6 +291,10 @@ see also `v help build`. backtraces are not implemented yet on all combinations of platform/compiler. + -assert continues + Just prints the failed assertion then continues. Useful if you want to see + the failures of many assertions that are all in the same test_ function. + -thread-stack-size 4194304 Set the thread stack size to 4MB. Use multiples of 4096. The default is 8MB, which is enough for compiling V programs, with deeply diff --git a/doc/docs.md b/doc/docs.md index 825b58556e..e58cc5a796 100644 --- a/doc/docs.md +++ b/doc/docs.md @@ -96,6 +96,11 @@ To do so, run the command `v up`. * [Decoding JSON](#decoding-json) * [Encoding JSON](#encoding-json) * [Testing](#testing) + * [Asserts](#asserts) + * [Asserts with an extra message](#asserts-with-an-extra-message) + * [Asserts that do not abort your program](#asserts-that-do-not-abort-your-program) + * [Test files](#test-files) + * [Running tests](#running-tests) * [Memory management](#memory-management) * [Stack and Heap](#stack-and-heap) * [ORM](#orm) @@ -4085,10 +4090,13 @@ foo(mut v) assert v[0] < 4 ``` An `assert` statement checks that its expression evaluates to `true`. If an assert fails, -the program will abort. Asserts should only be used to detect programming errors. When an +the program will usually abort. Asserts should only be used to detect programming errors. When an assert fails it is reported to *stderr*, and the values on each side of a comparison operator (such as `<`, `==`) will be printed when possible. This is useful to easily find an -unexpected value. Assert statements can be used in any function. +unexpected value. Assert statements can be used in any function, not just test ones, +which is handy when developing new functionality, to keep your invariants in check. + +Note: all `assert` statements are *removed*, when you compile your program with the `-prod` flag. ### Asserts with an extra message @@ -4104,6 +4112,37 @@ fn test_assertion_with_extra_message_failure() { } ``` +### Asserts that do not abort your program +When initially prototyping functionality and tests, it is sometimes desirable to +have asserts, that do not stop the program, but just print their failures. That can +be achieved by tagging your assert containing functions with an `[assert_continues]` +tag, for example running this program: +```v +[assert_continues] +fn abc(ii int) { + assert ii == 2 +} + +for i in 0 .. 4 { + abc(i) +} +``` +... will produce this output: +``` +assert_continues_example.v:3: FAIL: fn main.abc: assert ii == 2 + left value: ii = 0 + right value: 2 +assert_continues_example.v:3: FAIL: fn main.abc: assert ii == 2 + left value: ii = 1 + right value: 2 +assert_continues_example.v:3: FAIL: fn main.abc: assert ii == 2 + left value: ii = 3 + right value: 2 +``` + +Note: V also supports a command line flag `-assert continues`, which will change the +behaviour of all asserts globally, as if you had tagged every function with `[assert_continues]`. + ### Test files ```v diff --git a/examples/js_dom_draw_bechmark_chart/chart/main.v b/examples/js_dom_draw_bechmark_chart/chart/main.v index 056d17d31c..eccf37d042 100644 --- a/examples/js_dom_draw_bechmark_chart/chart/main.v +++ b/examples/js_dom_draw_bechmark_chart/chart/main.v @@ -104,12 +104,12 @@ pub fn (mut app App) controller_get_all_task() ?vweb.Result { for key, values in framework_platform[orm_stmt_kind] { attribute_names[orm_stmt_kind] << key - maxs[orm_stmt_kind] << arrays.max(values)? + maxs[orm_stmt_kind] << arrays.max(values) or { continue } } - max_benchmark[orm_stmt_kind] = arrays.max(maxs[orm_stmt_kind])? from_framework[orm_stmt_kind] = json.encode(framework_platform[orm_stmt_kind]) table[orm_stmt_kind] = gen_table_info(attribute_names[orm_stmt_kind], framework_platform[orm_stmt_kind]) + max_benchmark[orm_stmt_kind] = arrays.max(maxs[orm_stmt_kind]) or { continue } } return $vweb.html() diff --git a/vlib/v/gen/c/assert.v b/vlib/v/gen/c/assert.v index eed8662197..1f3ef6ffbf 100644 --- a/vlib/v/gen/c/assert.v +++ b/vlib/v/gen/c/assert.v @@ -32,9 +32,6 @@ fn (mut g Gen) assert_stmt(original_assert_statement ast.AssertStmt) { metaname_fail := g.gen_assert_metainfo(node) g.writeln('\tmain__TestRunner_name_table[test_runner._typ]._method_assert_fail(test_runner._object, &$metaname_fail);') g.gen_assert_postfailure_mode(node) - g.writeln('\tlongjmp(g_jump_buffer, 1);') - g.writeln('\t// TODO') - g.writeln('\t// Maybe print all vars in a test function if it fails?') g.writeln('}') } else { g.write('if (!(') @@ -45,7 +42,6 @@ fn (mut g Gen) assert_stmt(original_assert_statement ast.AssertStmt) { metaname_panic := g.gen_assert_metainfo(node) g.writeln('\t__print_assert_failure(&$metaname_panic);') g.gen_assert_postfailure_mode(node) - g.writeln('\t_v_panic(_SLIT("Assertion failed..."));') g.writeln('}') } } @@ -84,14 +80,24 @@ fn (mut g Gen) assert_subexpression_to_ctemp(expr ast.Expr, expr_type ast.Type) fn (mut g Gen) gen_assert_postfailure_mode(node ast.AssertStmt) { g.write_v_source_line_info(node.pos) - match g.pref.assert_failure_mode { - .default {} - .aborts { - g.writeln('\tabort();') - } - .backtraces { - g.writeln('\tprint_backtrace();') - } + if g.pref.assert_failure_mode == .continues + || g.fn_decl.attrs.any(it.name == 'assert_continues') { + return + } + if g.pref.assert_failure_mode == .aborts || g.fn_decl.attrs.any(it.name == 'assert_aborts') { + g.writeln('\tabort();') + } + if g.pref.assert_failure_mode == .backtraces + || g.fn_decl.attrs.any(it.name == 'assert_backtraces') { + g.writeln('\tprint_backtrace();') + } + if g.pref.is_test { + g.writeln('\tlongjmp(g_jump_buffer, 1);') + } + g.writeln('\t// TODO') + g.writeln('\t// Maybe print all vars in a test function if it fails?') + if g.pref.assert_failure_mode != .continues { + g.writeln('\t_v_panic(_SLIT("Assertion failed..."));') } } diff --git a/vlib/v/pref/pref.v b/vlib/v/pref/pref.v index 2fec3b4ee1..93886e28f4 100644 --- a/vlib/v/pref/pref.v +++ b/vlib/v/pref/pref.v @@ -21,6 +21,7 @@ pub enum AssertFailureMode { default aborts backtraces + continues } pub enum GarbageCollectionMode { @@ -276,10 +277,14 @@ pub fn parse_args_and_show_errors(known_external_commands []string, args []strin 'backtraces' { res.assert_failure_mode = .backtraces } + 'continues' { + res.assert_failure_mode = .continues + } else { eprintln('unknown assert mode `-gc $assert_mode`, supported modes are:`') eprintln(' `-assert aborts` .... calls abort() after assertion failure') eprintln(' `-assert backtraces` .... calls print_backtrace() after assertion failure') + eprintln(' `-assert continues` .... does not call anything, just continue after an assertion failure') exit(1) } } diff --git a/vlib/v/preludes/test_runner_normal.v b/vlib/v/preludes/test_runner_normal.v index d853d962d5..dcc7f129dd 100644 --- a/vlib/v/preludes/test_runner_normal.v +++ b/vlib/v/preludes/test_runner_normal.v @@ -70,6 +70,9 @@ fn (mut runner NormalTestRunner) exit_code() int { if runner.fn_fails > 0 { return 1 } + if runner.total_assert_fails > 0 { + return 2 + } return 0 } diff --git a/vlib/v/preludes/test_runner_simple.v b/vlib/v/preludes/test_runner_simple.v index e9a03b54a4..2c5b3aa2b1 100644 --- a/vlib/v/preludes/test_runner_simple.v +++ b/vlib/v/preludes/test_runner_simple.v @@ -44,6 +44,9 @@ fn (mut runner SimpleTestRunner) exit_code() int { if runner.fn_fails > 0 { return 1 } + if runner.total_assert_fails > 0 { + return 2 + } return 0 } diff --git a/vlib/v/preludes/test_runner_tap.v b/vlib/v/preludes/test_runner_tap.v index 0bb2f788e0..e050d442d8 100644 --- a/vlib/v/preludes/test_runner_tap.v +++ b/vlib/v/preludes/test_runner_tap.v @@ -67,6 +67,9 @@ fn (mut runner TAPTestRunner) exit_code() int { if runner.fn_fails > 0 { return 1 } + if runner.total_assert_fails > 0 { + return 2 + } return 0 }