2019-12-01 12:50:13 +03:00
|
|
|
module testing
|
|
|
|
|
2020-04-26 09:32:05 +03:00
|
|
|
import os
|
2022-12-02 14:45:33 +03:00
|
|
|
import os.cmdline
|
2020-07-06 15:08:38 +03:00
|
|
|
import time
|
2020-04-26 09:32:05 +03:00
|
|
|
import term
|
|
|
|
import benchmark
|
2022-09-16 04:56:19 +03:00
|
|
|
import sync
|
2021-02-11 11:55:23 +03:00
|
|
|
import sync.pool
|
2020-04-26 09:32:05 +03:00
|
|
|
import v.pref
|
2020-08-05 19:34:27 +03:00
|
|
|
import v.util.vtest
|
2021-10-11 13:10:55 +03:00
|
|
|
import runtime
|
2019-12-01 12:50:13 +03:00
|
|
|
|
2021-11-15 12:44:54 +03:00
|
|
|
pub const github_job = os.getenv('GITHUB_JOB')
|
2021-01-03 18:56:55 +03:00
|
|
|
|
2021-11-15 12:44:54 +03:00
|
|
|
pub const show_start = os.getenv('VTEST_SHOW_START') == '1'
|
2021-01-25 17:29:56 +03:00
|
|
|
|
2021-11-15 12:44:54 +03:00
|
|
|
pub const hide_skips = os.getenv('VTEST_HIDE_SKIP') == '1'
|
2021-05-08 13:32:29 +03:00
|
|
|
|
2021-11-15 12:44:54 +03:00
|
|
|
pub const hide_oks = os.getenv('VTEST_HIDE_OK') == '1'
|
|
|
|
|
|
|
|
pub const fail_fast = os.getenv('VTEST_FAIL_FAST') == '1'
|
2021-05-08 13:32:29 +03:00
|
|
|
|
2022-04-30 13:27:50 +03:00
|
|
|
pub const fail_flaky = os.getenv('VTEST_FAIL_FLAKY') == '1'
|
|
|
|
|
2021-12-20 18:22:02 +03:00
|
|
|
pub const test_only = os.getenv('VTEST_ONLY').split_any(',')
|
|
|
|
|
|
|
|
pub const test_only_fn = os.getenv('VTEST_ONLY_FN').split_any(',')
|
|
|
|
|
2021-12-07 22:31:29 +03:00
|
|
|
pub const is_node_present = os.execute('node --version').exit_code == 0
|
|
|
|
|
2022-10-17 10:24:01 +03:00
|
|
|
pub const all_processes = get_all_processes()
|
|
|
|
|
|
|
|
fn get_all_processes() []string {
|
|
|
|
$if windows {
|
|
|
|
// TODO
|
|
|
|
return []
|
|
|
|
} $else {
|
|
|
|
return os.execute('ps ax').output.split_any('\r\n')
|
|
|
|
}
|
|
|
|
}
|
2021-12-07 22:31:29 +03:00
|
|
|
|
2019-12-01 12:50:13 +03:00
|
|
|
pub struct TestSession {
|
|
|
|
pub mut:
|
2020-03-04 22:28:42 +03:00
|
|
|
files []string
|
2020-03-21 11:48:02 +03:00
|
|
|
skip_files []string
|
2020-03-04 22:28:42 +03:00
|
|
|
vexe string
|
2020-03-21 11:48:02 +03:00
|
|
|
vroot string
|
2020-12-01 18:43:34 +03:00
|
|
|
vtmp_dir string
|
2020-03-04 22:28:42 +03:00
|
|
|
vargs string
|
2021-11-15 12:44:54 +03:00
|
|
|
fail_fast bool
|
2020-03-04 22:28:42 +03:00
|
|
|
benchmark benchmark.Benchmark
|
2020-12-01 18:43:34 +03:00
|
|
|
rm_binaries bool = true
|
2020-10-18 20:02:47 +03:00
|
|
|
silent_mode bool
|
2022-12-02 14:45:33 +03:00
|
|
|
show_stats bool
|
2020-10-18 20:02:47 +03:00
|
|
|
progress_mode bool
|
2020-09-18 18:35:38 +03:00
|
|
|
root_relative bool // used by CI runs, so that the output is stable everywhere
|
2020-11-04 14:38:05 +03:00
|
|
|
nmessages chan LogMessage // many publishers, single consumer/printer
|
2022-12-02 14:45:33 +03:00
|
|
|
nmessage_idx int // currently printed message index
|
2021-07-15 09:52:22 +03:00
|
|
|
failed_cmds shared []string
|
2022-12-02 14:45:33 +03:00
|
|
|
reporter Reporter = Reporter(NormalReporter{})
|
2020-11-04 14:38:05 +03:00
|
|
|
}
|
|
|
|
|
2021-07-15 09:52:22 +03:00
|
|
|
pub fn (mut ts TestSession) add_failed_cmd(cmd string) {
|
|
|
|
lock ts.failed_cmds {
|
|
|
|
ts.failed_cmds << cmd
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn (mut ts TestSession) show_list_of_failed_tests() {
|
2022-12-02 14:45:33 +03:00
|
|
|
rlock ts.failed_cmds {
|
|
|
|
ts.reporter.list_of_failed_commands(ts.failed_cmds)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
struct MessageThreadContext {
|
|
|
|
mut:
|
|
|
|
file string
|
|
|
|
flow_id string
|
|
|
|
}
|
|
|
|
|
|
|
|
fn (mut ts TestSession) append_message(kind MessageKind, msg string, mtc MessageThreadContext) {
|
|
|
|
ts.nmessages <- LogMessage{
|
|
|
|
file: mtc.file
|
|
|
|
flow_id: mtc.flow_id
|
|
|
|
message: msg
|
|
|
|
kind: kind
|
|
|
|
when: time.now()
|
2021-07-15 09:52:22 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-12-02 14:45:33 +03:00
|
|
|
fn (mut ts TestSession) append_message_with_duration(kind MessageKind, msg string, d time.Duration, mtc MessageThreadContext) {
|
2020-11-04 14:38:05 +03:00
|
|
|
ts.nmessages <- LogMessage{
|
2022-12-02 14:45:33 +03:00
|
|
|
file: mtc.file
|
|
|
|
flow_id: mtc.flow_id
|
2020-11-04 14:38:05 +03:00
|
|
|
message: msg
|
|
|
|
kind: kind
|
2022-12-02 14:45:33 +03:00
|
|
|
when: time.now()
|
|
|
|
took: d
|
2020-11-04 14:38:05 +03:00
|
|
|
}
|
2020-10-18 18:16:33 +03:00
|
|
|
}
|
|
|
|
|
2022-12-02 14:45:33 +03:00
|
|
|
pub fn (mut ts TestSession) session_start(message string) {
|
|
|
|
ts.reporter.session_start(message, mut ts)
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn (mut ts TestSession) session_stop(message string) {
|
|
|
|
ts.reporter.session_stop(message, mut ts)
|
|
|
|
}
|
|
|
|
|
2020-10-18 18:16:33 +03:00
|
|
|
pub fn (mut ts TestSession) print_messages() {
|
2022-12-03 20:41:22 +03:00
|
|
|
mut test_idx := 0
|
2021-07-20 11:17:08 +03:00
|
|
|
mut print_msg_time := time.new_stopwatch()
|
2020-10-18 18:16:33 +03:00
|
|
|
for {
|
2020-10-18 20:02:47 +03:00
|
|
|
// get a message from the channel of messages to be printed:
|
|
|
|
mut rmessage := <-ts.nmessages
|
2022-12-03 20:41:22 +03:00
|
|
|
ts.nmessage_idx++
|
|
|
|
|
|
|
|
// first sent *all events* to the output reporter, so it can then process them however it wants:
|
|
|
|
ts.reporter.report(ts.nmessage_idx, rmessage)
|
|
|
|
|
|
|
|
if rmessage.kind in [.cmd_begin, .cmd_end] {
|
|
|
|
// The following events, are sent before the test framework has determined,
|
|
|
|
// what the full completion status is. They can also be repeated multiple times,
|
|
|
|
// for tests that are flaky and need repeating.
|
|
|
|
continue
|
|
|
|
}
|
2020-11-04 14:38:05 +03:00
|
|
|
if rmessage.kind == .sentinel {
|
2020-10-18 19:00:32 +03:00
|
|
|
// a sentinel for stopping the printing thread
|
2020-10-18 20:02:47 +03:00
|
|
|
if !ts.silent_mode && ts.progress_mode {
|
2022-12-02 14:45:33 +03:00
|
|
|
ts.reporter.report_stop()
|
2020-10-18 19:00:32 +03:00
|
|
|
}
|
|
|
|
return
|
|
|
|
}
|
2021-01-25 18:58:01 +03:00
|
|
|
if rmessage.kind != .info {
|
2022-12-03 20:41:22 +03:00
|
|
|
// info events can also be repeated, and should be ignored when determining
|
|
|
|
// the total order of the current test file, in the following replacements:
|
|
|
|
test_idx++
|
2021-01-25 18:58:01 +03:00
|
|
|
}
|
2020-12-02 20:47:07 +03:00
|
|
|
msg := rmessage.message.replace_each([
|
|
|
|
'TMP1',
|
2022-12-03 20:41:22 +03:00
|
|
|
'${test_idx:1d}',
|
2020-12-02 20:47:07 +03:00
|
|
|
'TMP2',
|
2022-12-03 20:41:22 +03:00
|
|
|
'${test_idx:2d}',
|
2020-12-02 20:47:07 +03:00
|
|
|
'TMP3',
|
2022-12-03 20:41:22 +03:00
|
|
|
'${test_idx:3d}',
|
2020-12-02 20:47:07 +03:00
|
|
|
'TMP4',
|
2022-12-03 20:41:22 +03:00
|
|
|
'${test_idx:4d}',
|
2020-12-02 20:47:07 +03:00
|
|
|
])
|
2020-11-04 14:38:05 +03:00
|
|
|
is_ok := rmessage.kind == .ok
|
2020-10-18 20:02:47 +03:00
|
|
|
//
|
|
|
|
time_passed := print_msg_time.elapsed().seconds()
|
|
|
|
if time_passed > 10 && ts.silent_mode && is_ok {
|
|
|
|
// Even if OK tests are suppressed,
|
|
|
|
// show *at least* 1 result every 10 seconds,
|
|
|
|
// otherwise the CI can seem stuck ...
|
2022-12-02 14:45:33 +03:00
|
|
|
ts.reporter.progress(ts.nmessage_idx, msg)
|
2020-10-18 20:02:47 +03:00
|
|
|
print_msg_time.restart()
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
if ts.progress_mode {
|
|
|
|
if is_ok && !ts.silent_mode {
|
2022-12-02 14:45:33 +03:00
|
|
|
ts.reporter.update_last_line(ts.nmessage_idx, msg)
|
2020-10-18 19:00:32 +03:00
|
|
|
} else {
|
2022-12-02 14:45:33 +03:00
|
|
|
ts.reporter.update_last_line_and_move_to_next(ts.nmessage_idx, msg)
|
2020-10-18 19:00:32 +03:00
|
|
|
}
|
2020-10-18 20:02:47 +03:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
if !ts.silent_mode || !is_ok {
|
|
|
|
// normal expanded mode, or failures in -silent mode
|
2022-12-02 14:45:33 +03:00
|
|
|
ts.reporter.message(ts.nmessage_idx, msg)
|
2020-10-18 20:02:47 +03:00
|
|
|
continue
|
2020-10-18 19:00:32 +03:00
|
|
|
}
|
2020-10-18 18:16:33 +03:00
|
|
|
}
|
2020-04-18 18:44:21 +03:00
|
|
|
}
|
|
|
|
|
2021-05-08 13:32:29 +03:00
|
|
|
pub fn new_test_session(_vargs string, will_compile bool) TestSession {
|
2020-04-26 14:49:31 +03:00
|
|
|
mut skip_files := []string{}
|
2021-05-08 13:32:29 +03:00
|
|
|
if will_compile {
|
2022-11-05 09:08:01 +03:00
|
|
|
// Skip the call_v_from_c files. They need special instructions for compilation.
|
|
|
|
// Check the README.md for detailed information.
|
|
|
|
skip_files << 'examples/call_v_from_c/v_test_print.v'
|
|
|
|
skip_files << 'examples/call_v_from_c/v_test_math.v'
|
2021-07-30 21:46:59 +03:00
|
|
|
$if msvc {
|
|
|
|
skip_files << 'vlib/v/tests/const_comptime_eval_before_vinit_test.v' // _constructor used
|
|
|
|
}
|
2021-05-08 13:32:29 +03:00
|
|
|
$if solaris {
|
|
|
|
skip_files << 'examples/gg/gg2.v'
|
|
|
|
skip_files << 'examples/pico/pico.v'
|
|
|
|
skip_files << 'examples/sokol/fonts.v'
|
|
|
|
skip_files << 'examples/sokol/drawing.v'
|
|
|
|
}
|
|
|
|
$if macos {
|
|
|
|
skip_files << 'examples/database/mysql.v'
|
|
|
|
skip_files << 'examples/database/orm.v'
|
2021-06-07 18:22:45 +03:00
|
|
|
skip_files << 'examples/database/psql/customer.v'
|
2021-05-08 13:32:29 +03:00
|
|
|
}
|
|
|
|
$if windows {
|
|
|
|
skip_files << 'examples/database/mysql.v'
|
|
|
|
skip_files << 'examples/database/orm.v'
|
2022-02-16 10:18:51 +03:00
|
|
|
skip_files << 'examples/smtp/mail.v' // requires OpenSSL
|
2021-05-08 13:32:29 +03:00
|
|
|
skip_files << 'examples/websocket/ping.v' // requires OpenSSL
|
|
|
|
skip_files << 'examples/websocket/client-server/client.v' // requires OpenSSL
|
|
|
|
skip_files << 'examples/websocket/client-server/server.v' // requires OpenSSL
|
2022-09-22 08:00:03 +03:00
|
|
|
skip_files << 'vlib/v/tests/websocket_logger_interface_should_compile_test.v' // requires OpenSSL
|
2021-05-08 13:32:29 +03:00
|
|
|
$if tinyc {
|
|
|
|
skip_files << 'examples/database/orm.v' // try fix it
|
|
|
|
}
|
|
|
|
}
|
2022-02-12 20:38:07 +03:00
|
|
|
$if windows {
|
2022-10-17 10:24:01 +03:00
|
|
|
// TODO: remove when closures on windows are supported...
|
2022-02-12 20:38:07 +03:00
|
|
|
skip_files << 'examples/pendulum-simulation/animation.v'
|
|
|
|
skip_files << 'examples/pendulum-simulation/full.v'
|
|
|
|
skip_files << 'examples/pendulum-simulation/parallel.v'
|
|
|
|
skip_files << 'examples/pendulum-simulation/parallel_with_iw.v'
|
|
|
|
skip_files << 'examples/pendulum-simulation/sequential.v'
|
|
|
|
}
|
2022-10-17 10:24:01 +03:00
|
|
|
if testing.github_job == 'ubuntu-docker-musl' {
|
|
|
|
skip_files << 'vlib/net/openssl/openssl_compiles_test.v'
|
|
|
|
}
|
|
|
|
if testing.github_job == 'tests-sanitize-memory-clang' {
|
|
|
|
skip_files << 'vlib/net/openssl/openssl_compiles_test.v'
|
|
|
|
}
|
|
|
|
if testing.github_job == 'windows-tcc' {
|
|
|
|
// TODO: fix these by adding declarations for the missing functions in the prebuilt tcc
|
|
|
|
skip_files << 'vlib/net/mbedtls/mbedtls_compiles_test.v'
|
|
|
|
skip_files << 'vlib/net/ssl/ssl_compiles_test.v'
|
|
|
|
}
|
2021-05-08 13:32:29 +03:00
|
|
|
if testing.github_job != 'sokol-shaders-can-be-compiled' {
|
|
|
|
// These examples need .h files that are produced from the supplied .glsl files,
|
|
|
|
// using by the shader compiler tools in https://github.com/floooh/sokol-tools-bin/archive/pre-feb2021-api-changes.tar.gz
|
2022-01-27 22:16:00 +03:00
|
|
|
skip_files << 'examples/sokol/simple_shader_glsl/simple_shader.v'
|
2021-05-08 13:32:29 +03:00
|
|
|
skip_files << 'examples/sokol/02_cubes_glsl/cube_glsl.v'
|
|
|
|
skip_files << 'examples/sokol/03_march_tracing_glsl/rt_glsl.v'
|
|
|
|
skip_files << 'examples/sokol/04_multi_shader_glsl/rt_glsl.v'
|
|
|
|
skip_files << 'examples/sokol/05_instancing_glsl/rt_glsl.v'
|
|
|
|
// Skip obj_viewer code in the CI
|
|
|
|
skip_files << 'examples/sokol/06_obj_viewer/show_obj.v'
|
|
|
|
}
|
|
|
|
if testing.github_job != 'ubuntu-tcc' {
|
|
|
|
skip_files << 'examples/c_interop_wkhtmltopdf.v' // needs installation of wkhtmltopdf from https://github.com/wkhtmltopdf/packaging/releases
|
2022-01-09 20:53:36 +03:00
|
|
|
skip_files << 'examples/call_v_from_python/test.v' // the example only makes sense to be compiled, when python is installed
|
2021-05-08 13:32:29 +03:00
|
|
|
// the ttf_test.v is not interactive, but needs X11 headers to be installed, which is done only on ubuntu-tcc for now
|
|
|
|
skip_files << 'vlib/x/ttf/ttf_test.v'
|
2021-05-11 17:47:43 +03:00
|
|
|
skip_files << 'vlib/vweb/vweb_app_test.v' // imports the `sqlite` module, which in turn includes sqlite3.h
|
2021-05-08 13:32:29 +03:00
|
|
|
}
|
|
|
|
if testing.github_job != 'audio-examples' {
|
|
|
|
skip_files << 'examples/sokol/sounds/melody.v'
|
|
|
|
skip_files << 'examples/sokol/sounds/wav_player.v'
|
|
|
|
skip_files << 'examples/sokol/sounds/simple_sin_tones.v'
|
2021-02-11 03:23:03 +03:00
|
|
|
}
|
2022-07-18 09:42:45 +03:00
|
|
|
$if !macos {
|
|
|
|
skip_files << 'examples/macos_tray/tray.v'
|
|
|
|
}
|
2021-01-03 18:56:55 +03:00
|
|
|
}
|
2022-12-02 14:45:33 +03:00
|
|
|
vargs := _vargs.replace('-progress', '')
|
2020-03-21 11:48:02 +03:00
|
|
|
vexe := pref.vexe_path()
|
2021-01-10 21:14:41 +03:00
|
|
|
vroot := os.dir(vexe)
|
2020-12-01 18:43:34 +03:00
|
|
|
new_vtmp_dir := setup_new_vtmp_folder()
|
2021-01-13 12:07:12 +03:00
|
|
|
if term.can_show_color_on_stderr() {
|
|
|
|
os.setenv('VCOLORS', 'always', true)
|
|
|
|
}
|
2022-12-02 14:45:33 +03:00
|
|
|
mut ts := TestSession{
|
2020-03-21 11:48:02 +03:00
|
|
|
vexe: vexe
|
2021-01-10 21:14:41 +03:00
|
|
|
vroot: vroot
|
2020-03-21 11:48:02 +03:00
|
|
|
skip_files: skip_files
|
2021-11-15 12:44:54 +03:00
|
|
|
fail_fast: testing.fail_fast
|
2022-12-02 14:45:33 +03:00
|
|
|
show_stats: '-stats' in vargs.split(' ')
|
2019-12-01 12:50:13 +03:00
|
|
|
vargs: vargs
|
2020-12-01 18:43:34 +03:00
|
|
|
vtmp_dir: new_vtmp_dir
|
2020-10-18 20:02:47 +03:00
|
|
|
silent_mode: _vargs.contains('-silent')
|
|
|
|
progress_mode: _vargs.contains('-progress')
|
2019-12-01 12:50:13 +03:00
|
|
|
}
|
2022-12-02 14:45:33 +03:00
|
|
|
ts.handle_test_runner_option()
|
|
|
|
return ts
|
|
|
|
}
|
|
|
|
|
|
|
|
fn (mut ts TestSession) handle_test_runner_option() {
|
|
|
|
test_runner := cmdline.option(os.args, '-test-runner', 'normal')
|
|
|
|
if test_runner !in pref.supported_test_runners {
|
|
|
|
eprintln('v test: `-test-runner ${test_runner}` is not using one of the supported test runners: ${pref.supported_test_runners_list()}')
|
|
|
|
}
|
|
|
|
test_runner_implementation_file := os.join_path(ts.vroot, 'cmd/tools/modules/testing/output_${test_runner}.v')
|
|
|
|
if !os.exists(test_runner_implementation_file) {
|
|
|
|
eprintln('v test: using `-test-runner ${test_runner}` needs ${test_runner_implementation_file} to exist, and contain a valid testing.Reporter implementation for that runner. See `cmd/tools/modules/testing/output_dump.v` for an example.')
|
|
|
|
exit(1)
|
|
|
|
}
|
|
|
|
match test_runner {
|
|
|
|
'normal' {
|
|
|
|
// default, nothing to do
|
|
|
|
}
|
|
|
|
'dump' {
|
|
|
|
ts.reporter = DumpReporter{}
|
|
|
|
}
|
2022-12-15 10:29:09 +03:00
|
|
|
'teamcity' {
|
|
|
|
ts.reporter = TeamcityReporter{}
|
|
|
|
}
|
2022-12-02 14:45:33 +03:00
|
|
|
else {
|
|
|
|
dump('just set ts.reporter to an instance of your own struct here')
|
|
|
|
}
|
|
|
|
}
|
2019-12-01 12:50:13 +03:00
|
|
|
}
|
|
|
|
|
2020-05-17 14:51:18 +03:00
|
|
|
pub fn (mut ts TestSession) init() {
|
2020-09-26 10:11:28 +03:00
|
|
|
ts.files.sort()
|
2020-04-18 05:49:03 +03:00
|
|
|
ts.benchmark = benchmark.new_benchmark_no_cstep()
|
2019-12-01 12:50:13 +03:00
|
|
|
}
|
|
|
|
|
2021-01-08 13:25:22 +03:00
|
|
|
pub fn (mut ts TestSession) add(file string) {
|
|
|
|
ts.files << file
|
|
|
|
}
|
|
|
|
|
2020-05-17 14:51:18 +03:00
|
|
|
pub fn (mut ts TestSession) test() {
|
2020-07-06 15:08:38 +03:00
|
|
|
// Ensure that .tmp.c files generated from compiling _test.v files,
|
|
|
|
// are easy to delete at the end, *without* affecting the existing ones.
|
2020-09-18 18:35:38 +03:00
|
|
|
current_wd := os.getwd()
|
|
|
|
if current_wd == os.wd_at_startup && current_wd == ts.vroot {
|
|
|
|
ts.root_relative = true
|
|
|
|
}
|
2020-07-06 15:08:38 +03:00
|
|
|
//
|
2019-12-01 12:50:13 +03:00
|
|
|
ts.init()
|
2020-04-26 14:49:31 +03:00
|
|
|
mut remaining_files := []string{}
|
2019-12-01 12:50:13 +03:00
|
|
|
for dot_relative_file in ts.files {
|
2021-09-06 03:11:58 +03:00
|
|
|
file := os.real_path(dot_relative_file)
|
2019-12-01 12:50:13 +03:00
|
|
|
$if windows {
|
2020-01-18 09:27:50 +03:00
|
|
|
if file.contains('sqlite') || file.contains('httpbin') {
|
2019-12-27 19:59:04 +03:00
|
|
|
continue
|
|
|
|
}
|
2019-12-01 12:50:13 +03:00
|
|
|
}
|
2019-12-06 02:11:39 +03:00
|
|
|
$if !macos {
|
2019-12-27 19:59:04 +03:00
|
|
|
if file.contains('customer') {
|
|
|
|
continue
|
|
|
|
}
|
2019-12-06 02:11:39 +03:00
|
|
|
}
|
2019-12-01 12:50:13 +03:00
|
|
|
$if msvc {
|
2019-12-27 19:59:04 +03:00
|
|
|
if file.contains('asm') {
|
|
|
|
continue
|
|
|
|
}
|
2019-12-01 12:50:13 +03:00
|
|
|
}
|
2019-12-30 07:23:54 +03:00
|
|
|
remaining_files << dot_relative_file
|
|
|
|
}
|
2020-12-21 20:51:20 +03:00
|
|
|
remaining_files = vtest.filter_vtest_only(remaining_files, fix_slashes: false)
|
2020-01-20 01:33:16 +03:00
|
|
|
ts.files = remaining_files
|
2019-12-30 07:23:54 +03:00
|
|
|
ts.benchmark.set_total_expected_steps(remaining_files.len)
|
2021-12-21 03:51:26 +03:00
|
|
|
mut njobs := runtime.nr_jobs()
|
|
|
|
if remaining_files.len < njobs {
|
|
|
|
njobs = remaining_files.len
|
|
|
|
}
|
|
|
|
ts.benchmark.njobs = njobs
|
2021-02-11 11:55:23 +03:00
|
|
|
mut pool_of_test_runners := pool.new_pool_processor(callback: worker_trunner)
|
2022-12-02 14:45:33 +03:00
|
|
|
// ensure that the nmessages queue/channel, has enough capacity for handling many messages across threads, without blocking
|
2020-11-04 14:38:05 +03:00
|
|
|
ts.nmessages = chan LogMessage{cap: 10000}
|
2020-10-18 18:16:33 +03:00
|
|
|
ts.nmessage_idx = 0
|
2022-12-02 14:45:33 +03:00
|
|
|
printing_thread := spawn ts.print_messages()
|
2020-03-04 22:28:42 +03:00
|
|
|
pool_of_test_runners.set_shared_context(ts)
|
2022-12-02 14:45:33 +03:00
|
|
|
ts.reporter.worker_threads_start(remaining_files, mut ts)
|
|
|
|
// all the testing happens here:
|
2021-02-27 00:55:09 +03:00
|
|
|
pool_of_test_runners.work_on_pointers(unsafe { remaining_files.pointers() })
|
2022-12-02 14:45:33 +03:00
|
|
|
//
|
2020-01-20 01:33:16 +03:00
|
|
|
ts.benchmark.stop()
|
2022-12-03 20:41:22 +03:00
|
|
|
ts.append_message(.sentinel, '', MessageThreadContext{ flow_id: '-1' }) // send the sentinel
|
2022-12-02 14:45:33 +03:00
|
|
|
printing_thread.wait()
|
|
|
|
ts.reporter.worker_threads_finish(mut ts)
|
|
|
|
ts.reporter.divider()
|
2022-09-16 04:56:19 +03:00
|
|
|
ts.show_list_of_failed_tests()
|
2021-12-24 11:47:48 +03:00
|
|
|
// cleanup generated .tmp.c files after successful tests:
|
2020-07-06 15:08:38 +03:00
|
|
|
if ts.benchmark.nfail == 0 {
|
2020-12-01 18:43:34 +03:00
|
|
|
if ts.rm_binaries {
|
2021-09-15 15:15:46 +03:00
|
|
|
os.rmdir_all(ts.vtmp_dir) or {}
|
2020-12-01 18:43:34 +03:00
|
|
|
}
|
2020-07-06 15:08:38 +03:00
|
|
|
}
|
2022-09-16 04:56:19 +03:00
|
|
|
// remove empty session folders:
|
|
|
|
if os.ls(ts.vtmp_dir) or { [] }.len == 0 {
|
|
|
|
os.rmdir_all(ts.vtmp_dir) or {}
|
|
|
|
}
|
2020-01-20 01:33:16 +03:00
|
|
|
}
|
|
|
|
|
2021-02-11 11:55:23 +03:00
|
|
|
fn worker_trunner(mut p pool.PoolProcessor, idx int, thread_id int) voidptr {
|
2023-01-06 09:28:01 +03:00
|
|
|
mut ts := unsafe { &TestSession(p.get_shared_context()) }
|
2021-11-15 12:44:54 +03:00
|
|
|
if ts.fail_fast {
|
2022-05-15 12:54:22 +03:00
|
|
|
if ts.failed_cmds.len > 0 {
|
2021-11-15 12:44:54 +03:00
|
|
|
return pool.no_result
|
|
|
|
}
|
|
|
|
}
|
2020-12-01 18:43:34 +03:00
|
|
|
tmpd := ts.vtmp_dir
|
2020-03-04 22:28:42 +03:00
|
|
|
// tls_bench is used to format the step messages/timings
|
2023-01-06 09:28:01 +03:00
|
|
|
mut tls_bench := unsafe { &benchmark.Benchmark(p.get_thread_context(idx)) }
|
2020-03-04 22:28:42 +03:00
|
|
|
if isnil(tls_bench) {
|
|
|
|
tls_bench = benchmark.new_benchmark_pointer()
|
|
|
|
tls_bench.set_total_expected_steps(ts.benchmark.nexpected_steps)
|
|
|
|
p.set_thread_context(idx, tls_bench)
|
|
|
|
}
|
2020-04-18 05:49:03 +03:00
|
|
|
tls_bench.no_cstep = true
|
2021-10-11 13:10:55 +03:00
|
|
|
tls_bench.njobs = ts.benchmark.njobs
|
2022-11-26 19:23:26 +03:00
|
|
|
mut relative_file := os.real_path(p.get_item[string](idx))
|
2021-07-27 20:14:30 +03:00
|
|
|
mut cmd_options := [ts.vargs]
|
2021-09-29 15:33:14 +03:00
|
|
|
mut run_js := false
|
|
|
|
|
|
|
|
is_fmt := ts.vargs.contains('fmt')
|
2022-04-09 13:03:52 +03:00
|
|
|
is_vet := ts.vargs.contains('vet')
|
|
|
|
produces_file_output := !(is_fmt || is_vet)
|
2021-09-29 15:33:14 +03:00
|
|
|
|
|
|
|
if relative_file.ends_with('js.v') {
|
2022-04-09 13:03:52 +03:00
|
|
|
if produces_file_output {
|
2021-09-29 15:33:14 +03:00
|
|
|
cmd_options << ' -b js'
|
2022-04-09 13:03:52 +03:00
|
|
|
run_js = true
|
2021-09-29 15:33:14 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if relative_file.contains('global') && !is_fmt {
|
2021-07-27 20:14:30 +03:00
|
|
|
cmd_options << ' -enable-globals'
|
|
|
|
}
|
2020-09-18 18:35:38 +03:00
|
|
|
if ts.root_relative {
|
2020-09-18 19:20:01 +03:00
|
|
|
relative_file = relative_file.replace(ts.vroot + os.path_separator, '')
|
2020-09-18 18:35:38 +03:00
|
|
|
}
|
2020-03-20 18:41:18 +03:00
|
|
|
file := os.real_path(relative_file)
|
2022-12-02 14:45:33 +03:00
|
|
|
mtc := MessageThreadContext{
|
|
|
|
file: file
|
|
|
|
flow_id: thread_id.str()
|
|
|
|
}
|
2021-05-31 15:56:36 +03:00
|
|
|
normalised_relative_file := relative_file.replace('\\', '/')
|
2020-03-04 22:28:42 +03:00
|
|
|
// Ensure that the generated binaries will be stored in the temporary folder.
|
|
|
|
// Remove them after a test passes/fails.
|
2020-03-19 17:49:07 +03:00
|
|
|
fname := os.file_name(file)
|
2021-09-29 15:33:14 +03:00
|
|
|
generated_binary_fname := if os.user_os() == 'windows' && !run_js {
|
2021-02-11 11:55:23 +03:00
|
|
|
fname.replace('.v', '.exe')
|
2021-09-29 15:33:14 +03:00
|
|
|
} else if !run_js {
|
|
|
|
fname.replace('.v', '')
|
2021-02-11 11:55:23 +03:00
|
|
|
} else {
|
|
|
|
fname.replace('.v', '')
|
|
|
|
}
|
2021-11-22 22:42:43 +03:00
|
|
|
generated_binary_fpath := os.join_path_single(tmpd, generated_binary_fname)
|
2022-04-09 13:03:52 +03:00
|
|
|
if produces_file_output {
|
2022-09-16 04:56:19 +03:00
|
|
|
if ts.rm_binaries {
|
|
|
|
os.rm(generated_binary_fpath) or {}
|
2020-12-01 18:43:34 +03:00
|
|
|
}
|
2021-09-29 15:33:14 +03:00
|
|
|
|
2022-01-22 22:56:01 +03:00
|
|
|
cmd_options << ' -o ${os.quoted_path(generated_binary_fpath)}'
|
2020-03-04 22:28:42 +03:00
|
|
|
}
|
2022-01-22 22:56:01 +03:00
|
|
|
cmd := '${os.quoted_path(ts.vexe)} ' + cmd_options.join(' ') + ' ${os.quoted_path(file)}'
|
2020-03-04 22:28:42 +03:00
|
|
|
ts.benchmark.step()
|
|
|
|
tls_bench.step()
|
2020-04-11 17:57:02 +03:00
|
|
|
if relative_file.replace('\\', '/') in ts.skip_files {
|
2020-11-04 14:38:05 +03:00
|
|
|
ts.benchmark.skip()
|
|
|
|
tls_bench.skip()
|
2021-05-08 13:32:29 +03:00
|
|
|
if !testing.hide_skips {
|
2022-12-02 14:45:33 +03:00
|
|
|
ts.append_message(.skip, tls_bench.step_message_skip(normalised_relative_file),
|
|
|
|
mtc)
|
2021-05-08 13:32:29 +03:00
|
|
|
}
|
2021-02-11 11:55:23 +03:00
|
|
|
return pool.no_result
|
2020-03-21 11:48:02 +03:00
|
|
|
}
|
2022-12-02 14:45:33 +03:00
|
|
|
if ts.show_stats {
|
|
|
|
ts.reporter.divider()
|
2022-12-03 20:41:22 +03:00
|
|
|
|
|
|
|
ts.append_message(.cmd_begin, cmd, mtc)
|
|
|
|
d_cmd := time.new_stopwatch()
|
2021-09-18 11:33:05 +03:00
|
|
|
mut status := os.system(cmd)
|
2022-12-03 20:41:22 +03:00
|
|
|
mut cmd_duration := d_cmd.elapsed()
|
|
|
|
ts.append_message_with_duration(.cmd_end, '', cmd_duration, mtc)
|
|
|
|
|
2021-09-18 11:33:05 +03:00
|
|
|
if status != 0 {
|
|
|
|
details := get_test_details(file)
|
2022-11-15 16:53:13 +03:00
|
|
|
os.setenv('VTEST_RETRY_MAX', '${details.retry}', true)
|
2021-09-18 11:33:05 +03:00
|
|
|
for retry := 1; retry <= details.retry; retry++ {
|
2022-12-02 14:45:33 +03:00
|
|
|
ts.append_message(.info, ' [stats] retrying ${retry}/${details.retry} of ${relative_file} ; known flaky: ${details.flaky} ...',
|
|
|
|
mtc)
|
2022-11-15 16:53:13 +03:00
|
|
|
os.setenv('VTEST_RETRY', '${retry}', true)
|
2022-12-03 20:41:22 +03:00
|
|
|
|
|
|
|
ts.append_message(.cmd_begin, cmd, mtc)
|
|
|
|
d_cmd_2 := time.new_stopwatch()
|
2021-09-18 11:33:05 +03:00
|
|
|
status = os.system(cmd)
|
2022-12-03 20:41:22 +03:00
|
|
|
cmd_duration = d_cmd_2.elapsed()
|
|
|
|
ts.append_message_with_duration(.cmd_end, '', cmd_duration, mtc)
|
|
|
|
|
2021-09-18 11:33:05 +03:00
|
|
|
if status == 0 {
|
|
|
|
unsafe {
|
|
|
|
goto test_passed_system
|
|
|
|
}
|
|
|
|
}
|
2022-01-27 18:43:55 +03:00
|
|
|
time.sleep(500 * time.millisecond)
|
2021-09-18 11:33:05 +03:00
|
|
|
}
|
2022-04-30 13:27:50 +03:00
|
|
|
if details.flaky && !testing.fail_flaky {
|
2022-12-02 14:45:33 +03:00
|
|
|
ts.append_message(.info, ' *FAILURE* of the known flaky test file ${relative_file} is ignored, since VTEST_FAIL_FLAKY is 0 . Retry count: ${details.retry} .',
|
|
|
|
mtc)
|
2022-04-30 13:49:36 +03:00
|
|
|
unsafe {
|
|
|
|
goto test_passed_system
|
|
|
|
}
|
2022-04-30 13:27:50 +03:00
|
|
|
}
|
2020-03-04 22:28:42 +03:00
|
|
|
ts.benchmark.fail()
|
|
|
|
tls_bench.fail()
|
2021-07-15 09:52:22 +03:00
|
|
|
ts.add_failed_cmd(cmd)
|
2021-02-11 11:55:23 +03:00
|
|
|
return pool.no_result
|
2021-09-18 11:33:05 +03:00
|
|
|
} else {
|
|
|
|
test_passed_system:
|
|
|
|
ts.benchmark.ok()
|
|
|
|
tls_bench.ok()
|
2019-12-27 19:59:04 +03:00
|
|
|
}
|
2020-11-04 14:38:05 +03:00
|
|
|
} else {
|
2021-01-25 17:29:56 +03:00
|
|
|
if testing.show_start {
|
2022-12-02 14:45:33 +03:00
|
|
|
ts.append_message(.info, ' starting ${relative_file} ...',
|
|
|
|
mtc)
|
2021-01-25 17:29:56 +03:00
|
|
|
}
|
2022-12-03 20:41:22 +03:00
|
|
|
ts.append_message(.cmd_begin, cmd, mtc)
|
2022-12-02 14:45:33 +03:00
|
|
|
d_cmd := time.new_stopwatch()
|
2021-09-18 11:33:05 +03:00
|
|
|
mut r := os.execute(cmd)
|
2022-12-03 20:41:22 +03:00
|
|
|
mut cmd_duration := d_cmd.elapsed()
|
|
|
|
ts.append_message_with_duration(.cmd_end, r.output, cmd_duration, mtc)
|
|
|
|
|
2021-03-08 21:52:13 +03:00
|
|
|
if r.exit_code < 0 {
|
2020-03-04 22:28:42 +03:00
|
|
|
ts.benchmark.fail()
|
|
|
|
tls_bench.fail()
|
2022-12-02 14:45:33 +03:00
|
|
|
ts.append_message_with_duration(.fail, tls_bench.step_message_fail(normalised_relative_file),
|
|
|
|
cmd_duration, mtc)
|
2021-07-15 09:52:22 +03:00
|
|
|
ts.add_failed_cmd(cmd)
|
2021-02-11 11:55:23 +03:00
|
|
|
return pool.no_result
|
2020-03-04 22:28:42 +03:00
|
|
|
}
|
|
|
|
if r.exit_code != 0 {
|
2021-09-18 11:33:05 +03:00
|
|
|
details := get_test_details(file)
|
2022-11-15 16:53:13 +03:00
|
|
|
os.setenv('VTEST_RETRY_MAX', '${details.retry}', true)
|
2021-09-18 11:33:05 +03:00
|
|
|
for retry := 1; retry <= details.retry; retry++ {
|
2022-12-02 14:45:33 +03:00
|
|
|
ts.append_message(.info, ' retrying ${retry}/${details.retry} of ${relative_file} ; known flaky: ${details.flaky} ...',
|
|
|
|
mtc)
|
2022-11-15 16:53:13 +03:00
|
|
|
os.setenv('VTEST_RETRY', '${retry}', true)
|
2022-12-03 20:41:22 +03:00
|
|
|
|
|
|
|
ts.append_message(.cmd_begin, cmd, mtc)
|
|
|
|
d_cmd_2 := time.new_stopwatch()
|
2021-09-18 11:33:05 +03:00
|
|
|
r = os.execute(cmd)
|
2022-12-03 20:41:22 +03:00
|
|
|
cmd_duration = d_cmd_2.elapsed()
|
|
|
|
ts.append_message_with_duration(.cmd_end, r.output, cmd_duration, mtc)
|
|
|
|
|
2021-09-18 11:33:05 +03:00
|
|
|
if r.exit_code == 0 {
|
|
|
|
unsafe {
|
|
|
|
goto test_passed_execute
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2022-04-30 13:27:50 +03:00
|
|
|
if details.flaky && !testing.fail_flaky {
|
2022-12-02 14:45:33 +03:00
|
|
|
ts.append_message(.info, ' *FAILURE* of the known flaky test file ${relative_file} is ignored, since VTEST_FAIL_FLAKY is 0 . Retry count: ${details.retry} .',
|
|
|
|
mtc)
|
2022-04-30 13:49:36 +03:00
|
|
|
unsafe {
|
|
|
|
goto test_passed_execute
|
|
|
|
}
|
2022-04-30 13:27:50 +03:00
|
|
|
}
|
2020-03-04 22:28:42 +03:00
|
|
|
ts.benchmark.fail()
|
|
|
|
tls_bench.fail()
|
2021-01-13 12:07:12 +03:00
|
|
|
ending_newline := if r.output.ends_with('\n') { '\n' } else { '' }
|
2022-12-02 14:45:33 +03:00
|
|
|
ts.append_message_with_duration(.fail, tls_bench.step_message_fail('${normalised_relative_file}\n${r.output.trim_space()}${ending_newline}'),
|
|
|
|
cmd_duration, mtc)
|
2021-07-15 09:52:22 +03:00
|
|
|
ts.add_failed_cmd(cmd)
|
2020-11-04 14:38:05 +03:00
|
|
|
} else {
|
2021-09-18 11:33:05 +03:00
|
|
|
test_passed_execute:
|
2020-03-04 22:28:42 +03:00
|
|
|
ts.benchmark.ok()
|
|
|
|
tls_bench.ok()
|
2021-05-08 13:32:29 +03:00
|
|
|
if !testing.hide_oks {
|
2022-12-02 14:45:33 +03:00
|
|
|
ts.append_message_with_duration(.ok, tls_bench.step_message_ok(normalised_relative_file),
|
|
|
|
cmd_duration, mtc)
|
2021-05-08 13:32:29 +03:00
|
|
|
}
|
2019-12-01 12:50:13 +03:00
|
|
|
}
|
|
|
|
}
|
2022-09-16 04:56:19 +03:00
|
|
|
if produces_file_output && ts.rm_binaries {
|
2022-04-09 13:03:52 +03:00
|
|
|
os.rm(generated_binary_fpath) or {}
|
2020-03-04 22:28:42 +03:00
|
|
|
}
|
2021-02-11 11:55:23 +03:00
|
|
|
return pool.no_result
|
2019-12-01 12:50:13 +03:00
|
|
|
}
|
|
|
|
|
2019-12-27 19:59:04 +03:00
|
|
|
pub fn vlib_should_be_present(parent_dir string) {
|
2021-11-22 22:42:43 +03:00
|
|
|
vlib_dir := os.join_path_single(parent_dir, 'vlib')
|
2019-12-27 19:59:04 +03:00
|
|
|
if !os.is_dir(vlib_dir) {
|
2022-11-15 16:53:13 +03:00
|
|
|
eprintln('${vlib_dir} is missing, it must be next to the V executable')
|
2019-12-01 12:50:13 +03:00
|
|
|
exit(1)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-12-01 18:43:34 +03:00
|
|
|
pub fn prepare_test_session(zargs string, folder string, oskipped []string, main_label string) TestSession {
|
2020-02-20 13:33:01 +03:00
|
|
|
vexe := pref.vexe_path()
|
2020-03-08 00:26:26 +03:00
|
|
|
parent_dir := os.dir(vexe)
|
2019-12-27 19:59:04 +03:00
|
|
|
vlib_should_be_present(parent_dir)
|
2019-12-05 14:22:54 +03:00
|
|
|
vargs := zargs.replace(vexe, '')
|
2020-02-08 19:01:10 +03:00
|
|
|
eheader(main_label)
|
2020-10-18 19:00:32 +03:00
|
|
|
if vargs.len > 0 {
|
2022-11-15 16:53:13 +03:00
|
|
|
eprintln('v compiler args: "${vargs}"')
|
2020-10-18 19:00:32 +03:00
|
|
|
}
|
2021-05-08 13:32:29 +03:00
|
|
|
mut session := new_test_session(vargs, true)
|
2021-11-22 22:42:43 +03:00
|
|
|
files := os.walk_ext(os.join_path_single(parent_dir, folder), '.v')
|
2020-04-26 14:49:31 +03:00
|
|
|
mut mains := []string{}
|
2020-12-20 17:33:55 +03:00
|
|
|
mut skipped := oskipped.clone()
|
2021-01-08 13:25:22 +03:00
|
|
|
next_file: for f in files {
|
2021-06-02 21:19:03 +03:00
|
|
|
fnormalised := f.replace('\\', '/')
|
2022-03-06 20:01:22 +03:00
|
|
|
// Note: a `testdata` folder, is the preferred name of a folder, containing V code,
|
2021-06-02 21:19:03 +03:00
|
|
|
// that you *do not want* the test framework to find incidentally for various reasons,
|
|
|
|
// for example module import tests, or subtests, that are compiled/run by other parent tests
|
|
|
|
// in specific configurations, etc.
|
|
|
|
if fnormalised.contains('testdata/') || fnormalised.contains('modules/')
|
2022-05-04 23:30:44 +03:00
|
|
|
|| fnormalised.contains('preludes/') {
|
2021-01-08 13:25:22 +03:00
|
|
|
continue
|
|
|
|
}
|
|
|
|
$if windows {
|
2021-02-07 14:46:22 +03:00
|
|
|
// skip pico and process/command examples on windows
|
2021-06-02 21:19:03 +03:00
|
|
|
if fnormalised.ends_with('examples/pico/pico.v')
|
|
|
|
|| fnormalised.ends_with('examples/process/command.v') {
|
2020-06-02 16:49:43 +03:00
|
|
|
continue
|
|
|
|
}
|
2021-01-08 13:25:22 +03:00
|
|
|
}
|
2021-03-01 02:18:14 +03:00
|
|
|
c := os.read_file(f) or { panic(err) }
|
2022-01-09 20:53:36 +03:00
|
|
|
maxc := if c.len > 500 { 500 } else { c.len }
|
2021-01-08 13:25:22 +03:00
|
|
|
start := c[0..maxc]
|
|
|
|
if start.contains('module ') && !start.contains('module main') {
|
2021-11-22 22:42:43 +03:00
|
|
|
skipped_f := f.replace(os.join_path_single(parent_dir, ''), '')
|
2021-01-08 13:25:22 +03:00
|
|
|
skipped << skipped_f
|
|
|
|
}
|
|
|
|
for skip_prefix in oskipped {
|
2022-05-04 23:30:44 +03:00
|
|
|
skip_folder := skip_prefix + '/'
|
|
|
|
if fnormalised.starts_with(skip_folder) {
|
2021-01-08 13:25:22 +03:00
|
|
|
continue next_file
|
2020-07-01 14:29:58 +03:00
|
|
|
}
|
2020-01-23 14:58:47 +03:00
|
|
|
}
|
2021-01-08 13:25:22 +03:00
|
|
|
mains << f
|
2020-01-23 14:58:47 +03:00
|
|
|
}
|
2019-12-27 19:59:04 +03:00
|
|
|
session.files << mains
|
2020-05-17 00:14:06 +03:00
|
|
|
session.skip_files << skipped
|
2020-12-01 18:43:34 +03:00
|
|
|
return session
|
|
|
|
}
|
|
|
|
|
2021-12-12 00:20:08 +03:00
|
|
|
pub type FnTestSetupCb = fn (mut session TestSession)
|
|
|
|
|
|
|
|
pub fn v_build_failing_skipped(zargs string, folder string, oskipped []string, cb FnTestSetupCb) bool {
|
2022-11-15 16:53:13 +03:00
|
|
|
main_label := 'Building ${folder} ...'
|
|
|
|
finish_label := 'building ${folder}'
|
2020-12-01 18:43:34 +03:00
|
|
|
mut session := prepare_test_session(zargs, folder, oskipped, main_label)
|
2021-12-12 00:20:08 +03:00
|
|
|
cb(mut session)
|
2019-12-01 12:50:13 +03:00
|
|
|
session.test()
|
2019-12-27 19:59:04 +03:00
|
|
|
eprintln(session.benchmark.total_message(finish_label))
|
2022-05-15 12:54:22 +03:00
|
|
|
return session.failed_cmds.len > 0
|
2019-12-01 12:50:13 +03:00
|
|
|
}
|
2019-12-01 16:12:51 +03:00
|
|
|
|
2019-12-27 19:59:04 +03:00
|
|
|
pub fn build_v_cmd_failed(cmd string) bool {
|
2021-03-08 21:52:13 +03:00
|
|
|
res := os.execute(cmd)
|
|
|
|
if res.exit_code < 0 {
|
|
|
|
return true
|
|
|
|
}
|
2019-12-01 16:12:51 +03:00
|
|
|
if res.exit_code != 0 {
|
|
|
|
eprintln('')
|
2019-12-27 19:59:04 +03:00
|
|
|
eprintln(res.output)
|
2019-12-01 16:12:51 +03:00
|
|
|
return true
|
|
|
|
}
|
|
|
|
return false
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn building_any_v_binaries_failed() bool {
|
2020-02-08 19:01:10 +03:00
|
|
|
eheader('Building V binaries...')
|
2019-12-01 16:12:51 +03:00
|
|
|
eprintln('VFLAGS is: "' + os.getenv('VFLAGS') + '"')
|
2020-02-20 13:33:01 +03:00
|
|
|
vexe := pref.vexe_path()
|
2020-03-08 00:26:26 +03:00
|
|
|
parent_dir := os.dir(vexe)
|
2020-11-04 14:38:05 +03:00
|
|
|
vlib_should_be_present(parent_dir)
|
2021-08-28 09:39:18 +03:00
|
|
|
os.chdir(parent_dir) or { panic(err) }
|
2019-12-06 02:11:39 +03:00
|
|
|
mut failed := false
|
2022-11-15 16:53:13 +03:00
|
|
|
v_build_commands := ['${vexe} -o v_g -g cmd/v',
|
|
|
|
'${vexe} -o v_prod_g -prod -g cmd/v', '${vexe} -o v_cg -cg cmd/v',
|
|
|
|
'${vexe} -o v_prod_cg -prod -cg cmd/v', '${vexe} -o v_prod -prod cmd/v']
|
2019-12-01 16:12:51 +03:00
|
|
|
mut bmark := benchmark.new_benchmark()
|
2019-12-06 02:11:39 +03:00
|
|
|
for cmd in v_build_commands {
|
2019-12-01 16:12:51 +03:00
|
|
|
bmark.step()
|
|
|
|
if build_v_cmd_failed(cmd) {
|
|
|
|
bmark.fail()
|
|
|
|
failed = true
|
2022-11-15 16:53:13 +03:00
|
|
|
eprintln(bmark.step_message_fail('command: ${cmd} . See details above ^^^^^^^'))
|
2019-12-01 16:12:51 +03:00
|
|
|
eprintln('')
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
bmark.ok()
|
2021-05-08 13:32:29 +03:00
|
|
|
if !testing.hide_oks {
|
2022-11-15 16:53:13 +03:00
|
|
|
eprintln(bmark.step_message_ok('command: ${cmd}'))
|
2021-05-08 13:32:29 +03:00
|
|
|
}
|
2019-12-01 16:12:51 +03:00
|
|
|
}
|
|
|
|
bmark.stop()
|
2020-01-28 07:18:19 +03:00
|
|
|
eprintln(term.h_divider('-'))
|
2019-12-27 19:59:04 +03:00
|
|
|
eprintln(bmark.total_message('building v binaries'))
|
2019-12-01 16:12:51 +03:00
|
|
|
return failed
|
|
|
|
}
|
2020-02-08 19:01:10 +03:00
|
|
|
|
2022-11-03 10:24:52 +03:00
|
|
|
// setup_new_vtmp_folder creates a new nested folder inside VTMP, then resets VTMP to it,
|
|
|
|
// so that V programs/tests will write their temporary files to new location.
|
|
|
|
// The new nested folder, and its contents, will get removed after all tests/programs succeed.
|
2020-10-14 18:20:19 +03:00
|
|
|
pub fn setup_new_vtmp_folder() string {
|
|
|
|
now := time.sys_mono_now()
|
2022-11-15 16:53:13 +03:00
|
|
|
new_vtmp_dir := os.join_path(os.vtmp_dir(), 'tsession_${sync.thread_id().hex()}_${now}')
|
2021-03-01 02:18:14 +03:00
|
|
|
os.mkdir_all(new_vtmp_dir) or { panic(err) }
|
2020-10-14 18:20:19 +03:00
|
|
|
os.setenv('VTMP', new_vtmp_dir, true)
|
|
|
|
return new_vtmp_dir
|
|
|
|
}
|
2021-09-18 11:33:05 +03:00
|
|
|
|
|
|
|
pub struct TestDetails {
|
|
|
|
pub mut:
|
|
|
|
retry int
|
2022-04-30 13:27:50 +03:00
|
|
|
flaky bool // when flaky tests fail, the whole run is still considered successfull, unless VTEST_FAIL_FLAKY is 1
|
2021-09-18 11:33:05 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
pub fn get_test_details(file string) TestDetails {
|
|
|
|
mut res := TestDetails{}
|
|
|
|
lines := os.read_lines(file) or { [] }
|
|
|
|
for line in lines {
|
|
|
|
if line.starts_with('// vtest retry:') {
|
|
|
|
res.retry = line.all_after(':').trim_space().int()
|
|
|
|
}
|
2022-04-30 13:27:50 +03:00
|
|
|
if line.starts_with('// vtest flaky:') {
|
|
|
|
res.flaky = line.all_after(':').trim_space().bool()
|
|
|
|
}
|
2021-09-18 11:33:05 +03:00
|
|
|
}
|
|
|
|
return res
|
|
|
|
}
|
2021-12-07 22:31:29 +03:00
|
|
|
|
|
|
|
pub fn find_started_process(pname string) ?string {
|
|
|
|
for line in testing.all_processes {
|
|
|
|
if line.contains(pname) {
|
|
|
|
return line
|
|
|
|
}
|
|
|
|
}
|
2022-11-15 16:53:13 +03:00
|
|
|
return error('could not find process matching ${pname}')
|
2021-12-07 22:31:29 +03:00
|
|
|
}
|
2022-12-02 14:45:33 +03:00
|
|
|
|
|
|
|
pub fn eheader(msg string) {
|
|
|
|
eprintln(term.header_left(msg, '-'))
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn header(msg string) {
|
|
|
|
println(term.header_left(msg, '-'))
|
|
|
|
flush_stdout()
|
|
|
|
}
|