From db2111235e2633393c009183fc9bd54582ab2f1b Mon Sep 17 00:00:00 2001 From: Makhnev Petr <51853996+i582@users.noreply.github.com> Date: Thu, 15 Dec 2022 11:29:09 +0400 Subject: [PATCH] tests: add a teamcity output format for V's test runner (#16681) --- cmd/tools/modules/testing/common.v | 3 + cmd/tools/modules/testing/output_teamcity.v | 57 ++++++++ cmd/v/help/test.txt | 7 +- vlib/v/pref/pref.v | 2 +- vlib/v/preludes/test_runner_teamcity.v | 136 ++++++++++++++++++++ 5 files changed, 201 insertions(+), 4 deletions(-) create mode 100644 cmd/tools/modules/testing/output_teamcity.v create mode 100644 vlib/v/preludes/test_runner_teamcity.v diff --git a/cmd/tools/modules/testing/common.v b/cmd/tools/modules/testing/common.v index e937ca13a0..62eb196ad6 100644 --- a/cmd/tools/modules/testing/common.v +++ b/cmd/tools/modules/testing/common.v @@ -292,6 +292,9 @@ fn (mut ts TestSession) handle_test_runner_option() { 'dump' { ts.reporter = DumpReporter{} } + 'teamcity' { + ts.reporter = TeamcityReporter{} + } else { dump('just set ts.reporter to an instance of your own struct here') } diff --git a/cmd/tools/modules/testing/output_teamcity.v b/cmd/tools/modules/testing/output_teamcity.v new file mode 100644 index 0000000000..9b08741a99 --- /dev/null +++ b/cmd/tools/modules/testing/output_teamcity.v @@ -0,0 +1,57 @@ +module testing + +// TeamcityReporter implements the interface `testing.Reporter`. +// It is used by `v -test-runner teamcity test .` +pub struct TeamcityReporter { +} + +pub fn (r TeamcityReporter) session_start(message string, mut ts TestSession) { +} + +pub fn (r TeamcityReporter) session_stop(message string, mut ts TestSession) { +} + +pub fn (r TeamcityReporter) report(index int, message LogMessage) { + name := r.get_test_suite_name_by_file(message.file) + match message.kind { + .cmd_begin { + eprintln("##teamcity[testSuiteStarted name='${name}' flowId='${message.flow_id}']") + } + .cmd_end { + eprintln("##teamcity[testSuiteFinished name='${name}' flowId='${message.flow_id}' duration='${message.took}']") + } + else {} + } +} + +pub fn (r TeamcityReporter) get_test_suite_name_by_file(path string) string { + file_name := path.replace('\\', '/').split('/').last() + return file_name.split('.').first() +} + +pub fn (r TeamcityReporter) report_stop() { +} + +pub fn (r TeamcityReporter) progress(index int, message string) { +} + +pub fn (r TeamcityReporter) update_last_line(index int, message string) { +} + +pub fn (r TeamcityReporter) update_last_line_and_move_to_next(index int, message string) { +} + +pub fn (r TeamcityReporter) message(index int, message string) { +} + +pub fn (r TeamcityReporter) divider() { +} + +pub fn (r TeamcityReporter) worker_threads_start(files []string, mut ts TestSession) { +} + +pub fn (r TeamcityReporter) worker_threads_finish(mut ts TestSession) { +} + +pub fn (r TeamcityReporter) list_of_failed_commands(failed_cmds []string) { +} diff --git a/cmd/v/help/test.txt b/cmd/v/help/test.txt index e44bda92f5..a7ca5c5e97 100644 --- a/cmd/v/help/test.txt +++ b/cmd/v/help/test.txt @@ -35,9 +35,10 @@ You can use several alternative test result formats, using `-test-runner name`, or by setting VTEST_RUNNER (the command line option has higher priority). The names of the available test runners are: - `simple` Fastest, does not import additional modules, does no processing. - `tap` Format the output as required by the Test Anything Protocol (TAP). - `normal` Supports color output, nicest/most human readable, the default. + `simple` Fastest, does not import additional modules, does no processing. + `tap` Format the output as required by the Test Anything Protocol (TAP). + `normal` Supports color output, nicest/most human readable, the default. + `teamcity` Format the output as required by the Teamcity and JetBrains IDEs. You can also implement your own custom test runner, by providing the path to your .v file, that implements it to this option. For example, see: diff --git a/vlib/v/pref/pref.v b/vlib/v/pref/pref.v index 8130bf4b65..116f00b39e 100644 --- a/vlib/v/pref/pref.v +++ b/vlib/v/pref/pref.v @@ -90,7 +90,7 @@ pub enum Arch { pub const list_of_flags_with_param = ['o', 'd', 'define', 'b', 'backend', 'cc', 'os', 'cf', 'cflags', 'path', 'arch'] -pub const supported_test_runners = ['normal', 'simple', 'tap', 'dump'] +pub const supported_test_runners = ['normal', 'simple', 'tap', 'dump', 'teamcity'] [heap; minify] pub struct Preferences { diff --git a/vlib/v/preludes/test_runner_teamcity.v b/vlib/v/preludes/test_runner_teamcity.v new file mode 100644 index 0000000000..f801991740 --- /dev/null +++ b/vlib/v/preludes/test_runner_teamcity.v @@ -0,0 +1,136 @@ +module main + +import time + +// Provide a Teamcity implementation of the TestRunner interface. +// Used in Teamcity and JetBrains IDEs for nice test reporting. + +fn vtest_init() { + change_test_runner(&TestRunner(TeamcityTestRunner{})) +} + +struct TeamcityTestRunner { +mut: + fname string + start_time time.Time + + file_test_info VTestFileMetaInfo + fn_test_info VTestFnMetaInfo + fn_assert_passes u64 + fn_passes u64 + fn_fails u64 + + total_assert_passes u64 + total_assert_fails u64 +} + +fn (mut runner TeamcityTestRunner) free() { + unsafe { + runner.fname.free() + runner.fn_test_info.free() + runner.file_test_info.free() + } +} + +fn normalise_fname(name string) string { + return name.replace('__', '.').replace('main.', '') +} + +fn (mut runner TeamcityTestRunner) start(ntests int) { +} + +fn (mut runner TeamcityTestRunner) finish() { +} + +fn (mut runner TeamcityTestRunner) exit_code() int { + if runner.fn_fails > 0 { + return 1 + } + if runner.total_assert_fails > 0 { + return 1 + } + return 0 +} + +// + +fn (mut runner TeamcityTestRunner) fn_start() bool { + runner.fn_assert_passes = 0 + runner.start_time = time.now() + runner.fname = normalise_fname(runner.fn_test_info.name) + println("Start '${runner.fname}'") + eprintln("##teamcity[testStarted name='${runner.fname}' locationHint='v_qn://${runner.file_test_info.file}:${runner.fname}']") + return true +} + +fn (mut runner TeamcityTestRunner) fn_pass() { + runner.fn_passes++ + duration := runner.test_duration() + eprintln("##teamcity[testFinished name='${runner.fname}' duration='${duration}']") + println('\n') +} + +fn (mut runner TeamcityTestRunner) fn_fail() { + runner.fn_fails++ + duration := runner.test_duration() + eprintln("##teamcity[testFailed name='${runner.fname}' duration='${duration}' message='assertion failed']") + println('\n') +} + +fn (mut runner TeamcityTestRunner) fn_error(line_nr int, file string, mod string, fn_name string, errmsg string) { + eprintln('>>> TeamcityTestRunner fn_error ${runner.fname}, line_nr: ${line_nr}, file: ${file}, mod: ${mod}, fn_name: ${fn_name}, errmsg: ${errmsg}') +} + +fn (mut runner TeamcityTestRunner) test_duration() i64 { + return time.now().unix_time_milli() - runner.start_time.unix_time_milli() +} + +// + +fn (mut runner TeamcityTestRunner) assert_pass(i &VAssertMetaInfo) { + runner.total_assert_passes++ + runner.fn_assert_passes++ + + filepath := i.fpath.clone() + println('>>> assert_pass ${filepath}:${i.line_nr + 1}') + + unsafe { i.free() } +} + +fn (mut runner TeamcityTestRunner) assert_fail(i &VAssertMetaInfo) { + runner.total_assert_fails++ + + filepath := i.fpath.clone() + mut final_filepath := filepath + ':${i.line_nr + 1}:' + mut final_funcname := 'fn ' + i.fn_name.replace('main.', '').replace('__', '.') + final_src := 'assert ' + i.src + eprintln('${final_filepath} ${final_funcname}') + if i.op.len > 0 && i.op != 'call' { + mut lvtitle := ' Left value:' + mut rvtitle := ' Right value:' + mut slvalue := '${i.lvalue}' + mut srvalue := '${i.rvalue}' + cutoff_limit := 30 + if slvalue.len > cutoff_limit || srvalue.len > cutoff_limit { + eprintln(' > ${final_src}') + eprintln(lvtitle) + eprintln(' ${slvalue}') + eprintln(rvtitle) + eprintln(' ${srvalue}') + } else { + eprintln(' > ${final_src}') + eprintln(' ${lvtitle} ${slvalue}') + eprintln('${rvtitle} ${srvalue}') + } + } else { + eprintln(' ${final_src}') + } + if i.has_msg { + mut mtitle := ' Message:' + mut mvalue := '${i.message}' + eprintln('${mtitle} ${mvalue}') + } + eprintln('') + + unsafe { i.free() } +}