From 366c50674c452b05fd6b4dfcba9b832ae428bccc Mon Sep 17 00:00:00 2001 From: Delyan Angelov Date: Sat, 28 Sep 2019 14:17:16 +0300 Subject: [PATCH] tooling: add tools/compare_v_performance_between_commits easily compare v performance/size across commits. * fix eprintln on linux (it now uses stderr, and flushes it). * flag: cleaner usage information. --- .gitignore | 2 + tools/compare_v_performance_between_commits | 98 ++++++++++++ tools/performance_compare.v | 158 ++++++++++++++++++++ vlib/builtin/builtin.v | 15 +- vlib/flag/flag.v | 116 ++++++++++---- vlib/flag/flag_test.v | 16 +- 6 files changed, 361 insertions(+), 44 deletions(-) create mode 100755 tools/compare_v_performance_between_commits create mode 100644 tools/performance_compare.v diff --git a/.gitignore b/.gitignore index 1b8db30b9b..597841fa1a 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,8 @@ /v.exe /tools/vget /tools/vget.exe +/tools/performance_compare +/tools/performance_compare.exe *.exe *.o .*.c diff --git a/tools/compare_v_performance_between_commits b/tools/compare_v_performance_between_commits new file mode 100755 index 0000000000..e6c849f483 --- /dev/null +++ b/tools/compare_v_performance_between_commits @@ -0,0 +1,98 @@ +#!/bin/sh + +set -e + +msg() { + printf '%s\n' "$*"; +} + +if [ $# -ne 2 ]; then + msg "Usage: compare_v_to_c_performance COMMIT_BEFORE COMMIT_AFTER" + exit 1 +fi + +depend_on() { + type "$1" >/dev/null 2>&1 || { + printf 'ERR: missing tool "%s"\n' "$1" >&2; exit 1; + } +} + +depend_on sh +depend_on cp +depend_on rm +depend_on wc +depend_on head +depend_on cc +depend_on strip +depend_on git +depend_on upx +depend_on make +depend_on hyperfine + +###################################################################### +## NB: cc should be a working, recent, sane C99 compiler +## cc is used by the Makefile to bootstrap v (both gcc/clang work) +## +## If you are a C/V developer in a unix environment, you most probably +## already have the above installed, with the possible exception of: +## https://github.com/sharkdp/hyperfine +## +## Installing them is out of scope of this tool. +###################################################################### + +COMMIT_B="$1" +COMMIT_A="$2" + +CWD="$(pwd)" +WORKDIR="/tmp" + +B="$WORKDIR/v_at_$COMMIT_B" +A="$WORKDIR/v_at_$COMMIT_A" + +prepare_v() { + msg + msg "Cloning current v source to $1 ..." + git clone --quiet "$CWD" "$1" + + cd "$1" + git checkout --quiet "$2" + + msg "Making v and vprod compilers in $1" + make > /dev/null + ./v -o v compiler + ./v -prod -o vprod compiler + cp v v_stripped + cp vprod vprod_stripped + strip *_stripped + cp v_stripped v_stripped_upxed + cp vprod_stripped vprod_stripped_upxed + upx -qqq --lzma v_stripped_upxed + upx -qqq --lzma vprod_stripped_upxed + wc -c "$1/v" "$1/v_stripped" "$1/v_stripped_upxed" "$1/vprod" "$1/vprod_stripped" "$1/vprod_stripped_upxed" | head -n -1 + VVERSION="$($1/v --version)" + GVERSION="$(git rev-parse --short --verify HEAD)" + msg "V version is: $VVERSION , local source commit: $GVERSION" +} + +compare_v_performance() { + CMD="$1" + msg "---------------------------------------------------------------------------------" + msg "Compare '$CMD'" + hyperfine --warmup=3 "cd '$B/' && $CMD " "cd '$A/' && $CMD " + msg +} + +############################################################################## +# Cleanup artifacts from previous runs of this tool: +cd "$WORKDIR" +rm -rf "$A/" "$B/" +############################################################################## + +msg "Comparing v compiler performance of commit $COMMIT_B (before) vs commit $COMMIT_A (after) ..." +prepare_v "$B" "$COMMIT_B" +prepare_v "$A" "$COMMIT_A" + +cd "$WORKDIR" +compare_v_performance "./v -o x.c compiler" +compare_v_performance "./vprod -o x.c compiler" +compare_v_performance "./vprod -o x compiler" diff --git a/tools/performance_compare.v b/tools/performance_compare.v new file mode 100644 index 0000000000..939088690e --- /dev/null +++ b/tools/performance_compare.v @@ -0,0 +1,158 @@ +import os + +import flag + +const ( + tool_version = '0.0.3' + tool_description = '' + + ' Compares V executable size and performance,\n' + + ' between 2 commits from V\'s local git history.\n' + + ' When only one commit is given, it is compared to master.' +) + +struct Context { + cwd string // current working folder +mut: + workdir string // the working folder (typically /tmp), where the tool will write + a string // the full path to the 'after' folder inside workdir + b string // the full path to the 'before' folder inside workdir + commit_before string // the git commit for the 'before' state + commit_after string // the git commit for the 'after' state +} +fn new_context() Context { + return Context{ cwd: os.getwd(), commit_after: 'master' } +} + +////// The stuff in this block may be reusable for other v cli tools? ///////////////// +fn run(cmd string) string { + x := os.exec(cmd) or { return '' } + if x.exit_code == 0 { return x.output } + return '' +} + +fn command_exits_with_zero_status(cmd string) bool { + x := os.exec(cmd) or { return false } + if x.exit_code == 0 { return true } + return false +} + +fn tool_must_exist(toolcmd string) { + if command_exits_with_zero_status( 'type $toolcmd' ) { return } + eprintln('Missing tool: $toolcmd') + eprintln('Please try again after you install it.') + exit(1) +} + +fn used_tools_must_exist(tools []string) { + for t in tools { + tool_must_exist(t) + } +} +////////////////////////////////////////////////////////////////////////// + +fn (c Context) compare_versions() { + // Input is validated at this point... + //Cleanup artifacts from previous runs of this tool: + os.chdir( c.workdir ) + run('rm -rf "$c.a" "$c.b" ') + + println('Comparing v compiler performance of commit $c.commit_before (before) vs commit $c.commit_after (after) ...') + c.prepare_v( c.b , c.commit_before ) + c.prepare_v( c.a , c.commit_after ) + + os.chdir( c.workdir ) + c.compare_v_performance( 'v -o source.c compiler' ) + c.compare_v_performance( 'vprod -o source.c compiler' ) + c.compare_v_performance( 'vprod -o binary compiler' ) +} + +fn show_sizes_of_files(files []string) { + for f in files { + size := os.file_size(f) + println('${size:10d} $f') + } +} + +fn (c &Context) prepare_v( cdir string, commit string ) { + println('') + println('Cloning current v source to $cdir ...') + os.system('git clone --quiet \'$c.cwd\' \'$cdir\' ') + os.chdir( cdir ) + os.system('git checkout --quiet \'$commit\' ') + + println('Making v and vprod compilers in $cdir') + run('make') + run('./v -o v compiler/ ') + run('./v -prod -o vprod compiler/ ') + run('cp v v_stripped') + run('cp vprod vprod_stripped') + run('strip *_stripped') + run('cp v_stripped v_stripped_upxed') + run('cp vprod_stripped vprod_stripped_upxed') + run('upx -qqq --lzma v_stripped_upxed') + run('upx -qqq --lzma vprod_stripped_upxed') + show_sizes_of_files(["$cdir/v", "$cdir/v_stripped", "$cdir/v_stripped_upxed"]) + show_sizes_of_files(["$cdir/vprod", "$cdir/vprod_stripped", "$cdir/vprod_stripped_upxed"]) + println("V version is: " + run("$cdir/v --version") + " , local source commit: " + run("git rev-parse --short --verify HEAD") ) +} + + +fn (c Context) compare_v_performance( cmd string ) { + println('---------------------------------------------------------------------------------') + println('Compare \'$cmd\'') + comparison_cmd := 'hyperfine --warmup=3 \'cd $c.b ; ./$cmd \' \'cd $c.a ; ./$cmd \' ' + os.system( comparison_cmd ) + println('') +} + +fn (c Context) normalized_workpath_for_commit( commit string ) string { + nc := 'v_at_' + commit.replace('^','_').replace('-','_').replace('/','_') + return os.realpath( c.workdir + os.PathSeparator + nc ) +} + +fn validate_commit_exists( commit string ){ + cmd := 'git cat-file -t ' + "'" + commit + "'" + if !command_exits_with_zero_status(cmd) { + eprintln("Commit: '" + commit + "' does not exist in the current repository.") + exit(3) + } +} + +fn main(){ + used_tools_must_exist(['cp','rm','strip','make','git','upx','cc','hyperfine']) + mut context := new_context() + mut fp := flag.new_flag_parser(os.args) + fp.application(os.filename(os.executable())) + fp.version( tool_version ) + fp.description( tool_description ) + fp.arguments_description('COMMIT_BEFORE [COMMIT_AFTER]') + fp.skip_executable() + fp.limit_free_args(1,2) + show_help:=fp.bool('help', false, 'Show this help screen') + context.workdir = os.realpath( fp.string('workdir', '/tmp', 'A writable folder, where the comparison will be done.') ) + if( show_help ){ + println( fp.usage() ) + exit(0) + } + commits := fp.finalize() or { + eprintln('Error: ' + err) + exit(1) + } + + context.commit_before = commits[0] + if commits.len > 1 { context.commit_after = commits[1] } + + validate_commit_exists( context.commit_before ) + validate_commit_exists( context.commit_after ) + + context.b = context.normalized_workpath_for_commit( context.commit_before ) + context.a = context.normalized_workpath_for_commit( context.commit_after ) + + if !os.is_dir( context.workdir ) { + msg := 'Work folder: ' + context.workdir + ' , does not exist.' + eprintln(msg) + exit(2) + } + + context.compare_versions() +} diff --git a/vlib/builtin/builtin.v b/vlib/builtin/builtin.v index 9d0708d9d2..f8cdde0def 100644 --- a/vlib/builtin/builtin.v +++ b/vlib/builtin/builtin.v @@ -91,14 +91,19 @@ pub fn println(s string) { pub fn eprintln(s string) { if isnil(s.str) { panic('eprintln(NIL)') - } + } $if mac { C.fprintf(stderr, '%.*s\n', s.len, s.str) - } + C.fflush(stderr) + return + } + $if linux { + C.fprintf(stderr, '%.*s\n', s.len, s.str) + C.fflush(stderr) + return + } // TODO issues with stderr and cross compiling for Linux - $else { - println(s) - } + println(s) } pub fn print(s string) { diff --git a/vlib/flag/flag.v b/vlib/flag/flag.v index 4cb6538e09..f93e7f30ae 100644 --- a/vlib/flag/flag.v +++ b/vlib/flag/flag.v @@ -1,3 +1,4 @@ +module flag // module flag for command-line flag parsing // @@ -43,15 +44,14 @@ // } // ``` -module flag - // data object storing information about a defined flag struct Flag { pub: name string // name as it appears on command line abbr byte // shortcut usage string // help message - val_desc string // something like '' that appears in usage + val_desc string // something like '' that appears in usage, + // and also the default value, when the flag is not given } // @@ -66,12 +66,20 @@ pub mut: min_free_args int max_free_args int + args_description string } +const ( + // used for formating usage message + SPACE = ' ' + UNDERLINE = '-----------------------------------------------' + MAX_ARGS_NUMBER = 4048 +) + // create a new flag set for parsing command line arguments // TODO use INT_MAX some how pub fn new_flag_parser(args []string) &FlagParser { - return &FlagParser{args:args, max_free_args: 4048} + return &FlagParser{args:args, max_free_args: MAX_ARGS_NUMBER} } // change the application name to be used in 'usage' output @@ -171,7 +179,7 @@ fn (fs mut FlagParser) parse_bool_value(n string, ab byte) ?string { // version with abbreviation //TODO error handling for invalid string to bool conversion pub fn (fs mut FlagParser) bool_(n string, a byte, v bool, u string) bool { - fs.add_flag(n, a, u, '') + fs.add_flag(n, a, u, ':'+v.str()) parsed := fs.parse_bool_value(n, a) or { return v } @@ -196,7 +204,7 @@ pub fn (fs mut FlagParser) bool(n string, v bool, u string) bool { // version with abbreviation //TODO error handling for invalid string to int conversion pub fn (fs mut FlagParser) int_(n string, a byte, i int, u string) int { - fs.add_flag(n, a, u, '') + fs.add_flag(n, a, u, ':$i') parsed := fs.parse_value(n, a) or { return i } @@ -221,7 +229,7 @@ pub fn (fs mut FlagParser) int(n string, i int, u string) int { // version with abbreviation //TODO error handling for invalid string to float conversion pub fn (fs mut FlagParser) float_(n string, a byte, f f32, u string) f32 { - fs.add_flag(n, a, u, '') + fs.add_flag(n, a, u, ':$f') parsed := fs.parse_value(n, a) or { return f } @@ -245,7 +253,7 @@ pub fn (fs mut FlagParser) float(n string, f f32, u string) f32 { // the default value is returned // version with abbreviation pub fn (fs mut FlagParser) string_(n string, a byte, v, u string) string { - fs.add_flag(n, a, u, '') + fs.add_flag(n, a, u, ':$v') parsed := fs.parse_value(n, a) or { return v } @@ -261,6 +269,27 @@ pub fn (fs mut FlagParser) string(n, v, u string) string { return fs.string_(n, `\0`, v, u) } +pub fn (fs mut FlagParser) limit_free_args_to_at_least(n int) { + if n > MAX_ARGS_NUMBER { + panic('flag.limit_free_args_to_at_least expect n to be smaller than $MAX_ARGS_NUMBER') + } + if n <= 0 { + panic('flag.limit_free_args_to_at_least expect n to be a positive number') + } + fs.min_free_args = n +} + +pub fn (fs mut FlagParser) limit_free_args_to_exactly(n int) { + if n > MAX_ARGS_NUMBER { + panic('flag.limit_free_args_to_exactly expect n to be smaller than $MAX_ARGS_NUMBER') + } + if n < 0 { + panic('flag.limit_free_args_to_exactly expect n to be a non negative number') + } + fs.min_free_args = n + fs.max_free_args = n +} + // this will cause an error in finalize() if free args are out of range // (min, ..., max) pub fn (fs mut FlagParser) limit_free_args(min, max int) { @@ -271,19 +300,50 @@ pub fn (fs mut FlagParser) limit_free_args(min, max int) { fs.max_free_args = max } -const ( - // used for formating usage message - SPACE = ' ' -) +pub fn (fs mut FlagParser) arguments_description(description string){ + fs.args_description = description +} // collect all given information and pub fn (fs FlagParser) usage() string { - mut use := '\n' - use += 'usage ${fs.application_name} [options] [ARGS]\n' + + positive_min_arg := ( fs.min_free_args > 0 ) + positive_max_arg := ( fs.max_free_args > 0 && fs.max_free_args != MAX_ARGS_NUMBER ) + no_arguments := ( fs.min_free_args == 0 && fs.max_free_args == 0 ) + + mut adesc := if fs.args_description.len > 0 { fs.args_description } else { '[ARGS]' } + if no_arguments { adesc = '' } + + mut use := '' + use += '$fs.application_name $fs.application_version\n' + use += '$UNDERLINE\n' + use += 'Usage: ${fs.application_name} [options] $adesc\n' use += '\n' + if fs.application_description != '' { + use += 'Description:\n' + use += '$fs.application_description' + use += '\n\n' + } + + // show a message about the [ARGS]: + if positive_min_arg || positive_max_arg || no_arguments { + if no_arguments { + use += 'This application does not expect any arguments\n\n' + goto end_of_arguments_handling + } + mut s:= []string + if positive_min_arg { s << 'at least $fs.min_free_args' } + if positive_max_arg { s << 'at most $fs.max_free_args' } + if positive_min_arg && positive_max_arg && fs.min_free_args == fs.max_free_args { + s = ['exactly $fs.min_free_args'] + } + sargs := s.join(' and ') + use += 'The arguments should be $sargs in number.\n\n' + } + end_of_arguments_handling: if fs.flags.len > 0 { - use += 'options:\n' + use += 'Options:\n' for f in fs.flags { flag_desc := ' --$f.name $f.val_desc' space := if flag_desc.len > SPACE.len-2 { @@ -292,17 +352,10 @@ pub fn (fs FlagParser) usage() string { SPACE.right(flag_desc.len) } abbr_desc := if f.abbr == `\0` { '' } else { ' -${tos(f.abbr, 1)}\n' } - use += '$abbr_desc$flag_desc$space$f.usage\n' + use += '${abbr_desc}${flag_desc}${space}${f.usage}\n' } } - - use += '\n' - use += '$fs.application_name $fs.application_version\n' - if fs.application_description != '' { - use += '\n' - use += 'description:\n' - use += '$fs.application_description' - } + return use } @@ -319,15 +372,14 @@ pub fn (fs FlagParser) finalize() ?[]string { return error('Unknown argument \'${a.right(2)}\'') } } - if fs.args.len < fs.min_free_args { - return error('Expect at least ${fs.min_free_args} arguments') + if fs.args.len < fs.min_free_args && fs.min_free_args > 0 { + return error('Expected at least ${fs.min_free_args} arguments, but given $fs.args.len') } - if fs.args.len >= fs.max_free_args { - if fs.max_free_args > 0 { - return error('Expect at most ${fs.max_free_args} arguments') - } else { - return error('Expect no arguments') - } + if fs.args.len > fs.max_free_args && fs.max_free_args > 0 { + return error('Expected at most ${fs.max_free_args} arguments, but given $fs.args.len') + } + if fs.args.len > 0 && fs.max_free_args == 0 && fs.min_free_args == 0 { + return error('Expected no arguments, but given $fs.args.len') } return fs.args } diff --git a/vlib/flag/flag_test.v b/vlib/flag/flag_test.v index 663275129d..10ccbe8286 100644 --- a/vlib/flag/flag_test.v +++ b/vlib/flag/flag_test.v @@ -145,6 +145,7 @@ fn test_finalize_returns_error_for_unknown_flags() { fn test_allow_to_build_usage_message() { mut fp := flag.new_flag_parser([]string) + fp.limit_free_args(1, 4) fp.application('flag_tool') fp.version('v0.0.0') fp.description('some short information about this tool') @@ -158,13 +159,14 @@ fn test_allow_to_build_usage_message() { usage := fp.usage() mut all_strings_found := true for s in ['flag_tool', 'v0.0.0', - 'an_int ', 'a_bool', 'bool_without', 'a_float ', 'a_string ', + 'an_int ', 'a_bool', 'bool_without', 'a_float ', 'a_string :not_stuff', 'some int to define', 'some bool to define', 'this should appear on the next line', 'some float as well', 'your credit card number', - 'usage', 'options:', 'description:', + 'The arguments should be at least 1 and at most 4 in number.', + 'Usage', 'Options:', 'Description:', 'some short information about this tool'] { if !usage.contains(s) { eprintln(' missing \'$s\' in usage message') @@ -181,7 +183,7 @@ fn test_if_no_description_given_usage_message_does_not_contain_descpription() { fp.bool('a_bool', false, '') - assert !fp.usage().contains('description:') + assert !fp.usage().contains('Description:') } fn test_if_no_options_given_usage_message_does_not_contain_options() { @@ -189,7 +191,7 @@ fn test_if_no_options_given_usage_message_does_not_contain_options() { fp.application('flag_tool') fp.version('v0.0.0') - assert !fp.usage().contains('options:') + assert !fp.usage().contains('Options:') } fn test_free_args_could_be_limited() { @@ -206,7 +208,7 @@ fn test_error_for_to_few_free_args() { mut fp1 := flag.new_flag_parser(['a', 'b', 'c']) fp1.limit_free_args(5, 6) args := fp1.finalize() or { - assert err == 'Expect at least 5 arguments' + assert err.starts_with('Expected at least 5 arguments') return } assert args.len < 0 // expect an error and need to use args @@ -216,7 +218,7 @@ fn test_error_for_to_much_free_args() { mut fp1 := flag.new_flag_parser(['a', 'b', 'c']) fp1.limit_free_args(1, 2) args := fp1.finalize() or { - assert err == 'Expect at most 2 arguments' + assert err.starts_with('Expected at most 2 arguments') return } assert args.len < 0 // expect an error and need to use args @@ -226,7 +228,7 @@ fn test_could_expect_no_free_args() { mut fp1 := flag.new_flag_parser(['a']) fp1.limit_free_args(0, 0) args := fp1.finalize() or { - assert err == 'Expect no arguments' + assert err.starts_with('Expected no arguments') return } assert args.len < 0 // expect an error and need to use args