tooling: add tools/compare_v_performance_between_commits

easily compare v performance/size across commits.

* fix eprintln on linux (it now uses stderr, and flushes it).

* flag: cleaner usage information.

.gitignore

@@ -8,6 +8,8 @@
 /v.exe
 /tools/vget
 /tools/vget.exe
+/tools/performance_compare
+/tools/performance_compare.exe
 *.exe
 *.o
 .*.c

tools/compare_v_performance_between_commits

@@ -0,0 +1,98 @@
+#!/bin/sh
+
+set -e 
+
+msg() { 
+    printf '%s\n' "$*"; 
+}
+
+if [ $# -ne 2 ]; then
+    msg "Usage: compare_v_to_c_performance COMMIT_BEFORE COMMIT_AFTER"
+    exit 1
+fi
+
+depend_on() { 
+    type "$1" >/dev/null 2>&1 || {
+        printf 'ERR: missing tool "%s"\n' "$1" >&2; exit 1;
+    } 
+}
+
+depend_on sh
+depend_on cp
+depend_on rm
+depend_on wc
+depend_on head
+depend_on cc
+depend_on strip
+depend_on git
+depend_on upx
+depend_on make
+depend_on hyperfine
+
+######################################################################
+## NB: cc should be a working, recent, sane C99 compiler 
+##     cc is used by the Makefile to bootstrap v (both gcc/clang work)
+##
+## If you are a C/V developer in a unix environment, you most probably 
+## already have the above installed, with the possible exception of:
+## https://github.com/sharkdp/hyperfine
+##
+## Installing them is out of scope of this tool.
+######################################################################
+
+COMMIT_B="$1"
+COMMIT_A="$2"
+
+CWD="$(pwd)"
+WORKDIR="/tmp"
+
+B="$WORKDIR/v_at_$COMMIT_B"
+A="$WORKDIR/v_at_$COMMIT_A"
+
+prepare_v() {
+    msg 
+    msg "Cloning current v source to $1 ..."
+    git clone --quiet "$CWD" "$1"
+    
+    cd "$1"
+    git checkout --quiet "$2"
+    
+    msg "Making v and vprod compilers in $1"
+    make > /dev/null
+    ./v -o v compiler 
+    ./v -prod -o vprod compiler
+    cp v     v_stripped 
+    cp vprod vprod_stripped 
+    strip *_stripped
+    cp v_stripped      v_stripped_upxed
+    cp vprod_stripped  vprod_stripped_upxed
+    upx -qqq --lzma v_stripped_upxed 
+    upx -qqq --lzma vprod_stripped_upxed
+    wc -c "$1/v" "$1/v_stripped" "$1/v_stripped_upxed" "$1/vprod" "$1/vprod_stripped" "$1/vprod_stripped_upxed" | head -n -1
+    VVERSION="$($1/v --version)"
+    GVERSION="$(git rev-parse --short  --verify HEAD)"
+    msg "V version is: $VVERSION , local source commit: $GVERSION"
+}
+
+compare_v_performance() {
+    CMD="$1"
+    msg "---------------------------------------------------------------------------------"
+    msg "Compare '$CMD'"
+    hyperfine --warmup=3 "cd '$B/' && $CMD "  "cd '$A/' && $CMD "
+    msg
+}
+
+##############################################################################
+# Cleanup artifacts from previous runs of this tool:
+cd "$WORKDIR"
+rm -rf "$A/" "$B/"
+##############################################################################
+
+msg "Comparing v compiler performance of commit $COMMIT_B (before) vs commit $COMMIT_A (after) ..."
+prepare_v "$B" "$COMMIT_B"
+prepare_v "$A" "$COMMIT_A"
+
+cd "$WORKDIR"
+compare_v_performance "./v     -o x.c compiler"
+compare_v_performance "./vprod -o x.c compiler"
+compare_v_performance "./vprod -o x   compiler"

tools/performance_compare.v

@@ -0,0 +1,158 @@
+import os
+
+import flag
+
+const (
+	tool_version = '0.0.3'
+	tool_description = '' +
+		'  Compares V executable size and performance,\n' +
+		'  between 2 commits from V\'s local git history.\n' + 
+		'  When only one commit is given, it is compared to master.'
+)
+
+struct Context {
+	cwd           string // current working folder
+mut:
+	workdir       string // the working folder (typically /tmp), where the tool will write
+	a             string // the full path to the 'after' folder inside workdir
+	b             string // the full path to the 'before' folder inside workdir
+	commit_before string // the git commit for the 'before' state
+	commit_after  string // the git commit for the 'after' state
+}
+fn new_context() Context {
+	return Context{ cwd: os.getwd(), commit_after: 'master' }
+}
+
+////// The stuff in this block may be reusable for other v cli tools? /////////////////
+fn run(cmd string) string {
+	x := os.exec(cmd) or { return '' }
+	if x.exit_code == 0 { return x.output }
+	return ''
+}
+
+fn command_exits_with_zero_status(cmd string) bool {
+	x := os.exec(cmd) or { return false }
+	if x.exit_code == 0 { return true }
+	return false
+}
+
+fn tool_must_exist(toolcmd string) {
+	if command_exits_with_zero_status( 'type $toolcmd' ) { return }
+	eprintln('Missing tool: $toolcmd')
+	eprintln('Please try again after you install it.')
+	exit(1)
+}
+
+fn used_tools_must_exist(tools []string) {
+	for t in tools { 
+		tool_must_exist(t)
+	}
+}
+//////////////////////////////////////////////////////////////////////////
+
+fn (c Context) compare_versions() {
+	// Input is validated at this point...
+	//Cleanup artifacts from previous runs of this tool:
+	os.chdir( c.workdir )
+	run('rm -rf "$c.a" "$c.b" ')
+	
+	println('Comparing v compiler performance of commit $c.commit_before (before) vs commit $c.commit_after (after) ...')
+	c.prepare_v( c.b , c.commit_before )
+	c.prepare_v( c.a , c.commit_after  )
+	
+	os.chdir( c.workdir )
+	c.compare_v_performance( 'v     -o source.c compiler' )
+	c.compare_v_performance( 'vprod -o source.c compiler' )
+	c.compare_v_performance( 'vprod -o binary   compiler' )
+}
+
+fn show_sizes_of_files(files []string) {
+	for f in files {
+		size := os.file_size(f)
+		println('${size:10d} $f')
+	}
+}
+
+fn (c &Context) prepare_v( cdir string, commit string ) {
+	println('')
+	println('Cloning current v source to $cdir ...')
+	os.system('git clone --quiet \'$c.cwd\' \'$cdir\' ')
+	os.chdir( cdir )
+	os.system('git checkout --quiet \'$commit\' ')
+	
+	println('Making v and vprod compilers in $cdir')
+	run('make')
+	run('./v       -o v     compiler/ ')
+	run('./v -prod -o vprod compiler/ ')
+	run('cp v     v_stripped')
+	run('cp vprod vprod_stripped')
+	run('strip *_stripped')
+	run('cp v_stripped      v_stripped_upxed')
+	run('cp vprod_stripped  vprod_stripped_upxed')
+	run('upx -qqq --lzma v_stripped_upxed')
+	run('upx -qqq --lzma vprod_stripped_upxed')
+	show_sizes_of_files(["$cdir/v",     "$cdir/v_stripped",      "$cdir/v_stripped_upxed"])
+	show_sizes_of_files(["$cdir/vprod", "$cdir/vprod_stripped",  "$cdir/vprod_stripped_upxed"])
+	println("V version is: " + run("$cdir/v --version") + " , local source commit: " + run("git rev-parse --short  --verify HEAD") )
+}
+
+
+fn (c Context) compare_v_performance( cmd string ) {
+	println('---------------------------------------------------------------------------------')
+	println('Compare \'$cmd\'')
+	comparison_cmd := 'hyperfine --warmup=3 \'cd $c.b ; ./$cmd \' \'cd $c.a ; ./$cmd \' '
+	os.system( comparison_cmd )
+	println('')
+}
+
+fn (c Context) normalized_workpath_for_commit( commit string ) string {
+	nc := 'v_at_' + commit.replace('^','_').replace('-','_').replace('/','_')
+	return os.realpath( c.workdir + os.PathSeparator + nc )
+}
+
+fn validate_commit_exists( commit string ){
+	cmd := 'git cat-file -t ' + "'" + commit + "'"
+	if !command_exits_with_zero_status(cmd) {
+		eprintln("Commit: '" +  commit + "' does not exist in the current repository.")
+		exit(3)
+	}
+}
+
+fn main(){
+	used_tools_must_exist(['cp','rm','strip','make','git','upx','cc','hyperfine'])
+	mut context := new_context()  
+	mut fp := flag.new_flag_parser(os.args)
+	fp.application(os.filename(os.executable()))
+	fp.version( tool_version )
+	fp.description( tool_description )
+	fp.arguments_description('COMMIT_BEFORE [COMMIT_AFTER]')
+	fp.skip_executable()
+	fp.limit_free_args(1,2)
+	show_help:=fp.bool('help',  false, 'Show this help screen')
+	context.workdir = os.realpath( fp.string('workdir', '/tmp', 'A writable folder, where the comparison will be done.') )
+	if( show_help ){
+		println( fp.usage() )
+		exit(0)
+	}
+	commits := fp.finalize() or {
+		eprintln('Error: ' + err)
+		exit(1)
+	}
+	
+	context.commit_before = commits[0]
+	if commits.len > 1 { context.commit_after = commits[1] }
+	
+	validate_commit_exists( context.commit_before )
+	validate_commit_exists( context.commit_after )
+	
+	context.b = context.normalized_workpath_for_commit( context.commit_before )
+	context.a = context.normalized_workpath_for_commit( context.commit_after )
+    
+	if !os.is_dir( context.workdir ) {
+		msg := 'Work folder: ' + context.workdir + ' , does not exist.'
+		eprintln(msg)
+		exit(2)
+	}
+	
+	context.compare_versions()
+}

vlib/builtin/builtin.v

@@ -94,11 +94,16 @@ pub fn eprintln(s string) {
 	}  
 	$if mac {
 		C.fprintf(stderr, '%.*s\n', s.len, s.str)
+		C.fflush(stderr)
+		return 
+	} 
+	$if linux {
+		C.fprintf(stderr, '%.*s\n', s.len, s.str)
+		C.fflush(stderr)
+		return
 	}  
 	// TODO issues with stderr and cross compiling for Linux
-	$else {
+	println(s)
-		println(s)
-	}
 }
 
 pub fn print(s string) {

vlib/flag/flag.v

@@ -1,3 +1,4 @@
+module flag
 
 // module flag for command-line flag parsing
 //
@@ -43,15 +44,14 @@
 //  }
 //  ```
 
-module flag
-
 // data object storing information about a defined flag
 struct Flag {
 pub:
   name     string // name as it appears on command line
   abbr     byte   // shortcut
   usage    string // help message
-  val_desc string // something like '<arg>' that appears in usage
+  val_desc string // something like '<arg>' that appears in usage, 
+                  // and also the default value, when the flag is not given
 }
 
 // 
@@ -66,12 +66,20 @@ pub mut:
 
   min_free_args int
   max_free_args int
+  args_description        string
 }
 
+const (
+  // used for formating usage message
+  SPACE = '                            '
+  UNDERLINE = '-----------------------------------------------'
+  MAX_ARGS_NUMBER = 4048
+)
+
 // create a new flag set for parsing command line arguments
 // TODO use INT_MAX some how
 pub fn new_flag_parser(args []string) &FlagParser {
-  return &FlagParser{args:args, max_free_args: 4048}
+  return &FlagParser{args:args, max_free_args: MAX_ARGS_NUMBER}
 }
 
 // change the application name to be used in 'usage' output
@@ -171,7 +179,7 @@ fn (fs mut FlagParser) parse_bool_value(n string, ab byte) ?string {
 // version with abbreviation
 //TODO error handling for invalid string to bool conversion
 pub fn (fs mut FlagParser) bool_(n string, a byte, v bool, u string) bool {
-  fs.add_flag(n, a, u, '')
+  fs.add_flag(n, a, u, '<bool>:'+v.str())
   parsed := fs.parse_bool_value(n, a) or {
     return v
   }
@@ -196,7 +204,7 @@ pub fn (fs mut FlagParser) bool(n string, v bool, u string) bool {
 // version with abbreviation
 //TODO error handling for invalid string to int conversion
 pub fn (fs mut FlagParser) int_(n string, a byte, i int, u string) int {
-  fs.add_flag(n, a, u, '<int>')
+  fs.add_flag(n, a, u, '<int>:$i')
   parsed := fs.parse_value(n, a) or {
     return i
   }
@@ -221,7 +229,7 @@ pub fn (fs mut FlagParser) int(n string, i int, u string) int {
 // version with abbreviation
 //TODO error handling for invalid string to float conversion
 pub fn (fs mut FlagParser) float_(n string, a byte, f f32, u string) f32 {
-  fs.add_flag(n, a, u, '<float>')
+  fs.add_flag(n, a, u, '<float>:$f')
   parsed := fs.parse_value(n, a) or {
     return f
   }
@@ -245,7 +253,7 @@ pub fn (fs mut FlagParser) float(n string, f f32, u string) f32 {
 //      the default value is returned
 // version with abbreviation
 pub fn (fs mut FlagParser) string_(n string, a byte, v, u string) string {
-  fs.add_flag(n, a, u, '<arg>')
+  fs.add_flag(n, a, u, '<string>:$v')
   parsed := fs.parse_value(n, a) or {
     return v
   }
@@ -261,6 +269,27 @@ pub fn (fs mut FlagParser) string(n, v, u string) string {
   return fs.string_(n, `\0`, v, u)
 }
 
+pub fn (fs mut FlagParser) limit_free_args_to_at_least(n int) {
+	if n > MAX_ARGS_NUMBER {
+	panic('flag.limit_free_args_to_at_least expect n to be smaller than $MAX_ARGS_NUMBER')
+  }
+	if n <= 0 {
+	panic('flag.limit_free_args_to_at_least expect n to be a positive number')
+  }
+  fs.min_free_args = n
+}
+
+pub fn (fs mut FlagParser) limit_free_args_to_exactly(n int) {
+	if n > MAX_ARGS_NUMBER {
+	panic('flag.limit_free_args_to_exactly expect n to be smaller than $MAX_ARGS_NUMBER')
+  }
+	if n < 0 {
+	panic('flag.limit_free_args_to_exactly expect n to be a non negative number')
+  }
+  fs.min_free_args = n 
+  fs.max_free_args = n
+}
+
 // this will cause an error in finalize() if free args are out of range
 // (min, ..., max)
 pub fn (fs mut FlagParser) limit_free_args(min, max int) {
@@ -271,19 +300,50 @@ pub fn (fs mut FlagParser) limit_free_args(min, max int) {
   fs.max_free_args = max
 }
 
-const (
+pub fn (fs mut FlagParser) arguments_description(description string){
-  // used for formating usage message
+  fs.args_description = description
-  SPACE = '                            '
+}
-)
 
 // collect all given information and 
 pub fn (fs FlagParser) usage() string {
-  mut use := '\n'
+
-  use += 'usage ${fs.application_name} [options] [ARGS]\n'
+  positive_min_arg := ( fs.min_free_args > 0 )
+  positive_max_arg := ( fs.max_free_args > 0 && fs.max_free_args != MAX_ARGS_NUMBER )
+  no_arguments := ( fs.min_free_args == 0 && fs.max_free_args == 0 )
+  
+  mut adesc := if fs.args_description.len > 0 { fs.args_description } else { '[ARGS]' }
+  if no_arguments { adesc = '' }  
+  
+  mut use := ''
+  use += '$fs.application_name $fs.application_version\n'
+  use += '$UNDERLINE\n'
+  use += 'Usage: ${fs.application_name} [options] $adesc\n'
   use += '\n'
+  if fs.application_description != '' {
+    use += 'Description:\n'
+    use += '$fs.application_description'
+    use += '\n\n'
+  }
+  
+  // show a message about the [ARGS]:
+  if positive_min_arg || positive_max_arg || no_arguments {
+    if no_arguments {
+      use += 'This application does not expect any arguments\n\n'
+      goto end_of_arguments_handling
+    }
+    mut s:= []string
+    if positive_min_arg { s << 'at least $fs.min_free_args' }
+    if positive_max_arg { s << 'at most $fs.max_free_args' }
+    if positive_min_arg && positive_max_arg && fs.min_free_args == fs.max_free_args {
+      s = ['exactly $fs.min_free_args']
+    }
+    sargs := s.join(' and ')
+    use += 'The arguments should be $sargs in number.\n\n'
+  }
+  end_of_arguments_handling:
 
   if fs.flags.len > 0 {
-    use += 'options:\n'
+    use += 'Options:\n'
     for f in fs.flags {
       flag_desc := '  --$f.name $f.val_desc'
       space := if flag_desc.len > SPACE.len-2 {
@@ -292,17 +352,10 @@ pub fn (fs FlagParser) usage() string {
         SPACE.right(flag_desc.len)
       }
       abbr_desc := if f.abbr == `\0` { '' } else { '  -${tos(f.abbr, 1)}\n' }
-      use += '$abbr_desc$flag_desc$space$f.usage\n'
+      use += '${abbr_desc}${flag_desc}${space}${f.usage}\n'
     }
   }
   
-  use += '\n'
-  use += '$fs.application_name $fs.application_version\n'
-  if fs.application_description != '' {
-    use += '\n'
-    use += 'description:\n'
-    use += '$fs.application_description'
-  }
   return use
 }
 
@@ -319,15 +372,14 @@ pub fn (fs FlagParser) finalize() ?[]string {
       return error('Unknown argument \'${a.right(2)}\'')
     }
   }
-  if fs.args.len < fs.min_free_args {
+  if fs.args.len < fs.min_free_args && fs.min_free_args > 0 {
-    return error('Expect at least ${fs.min_free_args} arguments')
+    return error('Expected at least ${fs.min_free_args} arguments, but given $fs.args.len')
   }
-  if fs.args.len >= fs.max_free_args {
+  if fs.args.len > fs.max_free_args && fs.max_free_args > 0 {
-    if fs.max_free_args > 0 {
+    return error('Expected at most ${fs.max_free_args} arguments, but given $fs.args.len')
-      return error('Expect at most ${fs.max_free_args} arguments')
+  }
-    } else {
+  if fs.args.len > 0 && fs.max_free_args == 0 && fs.min_free_args == 0 {
-      return error('Expect no arguments')
+    return error('Expected no arguments, but given $fs.args.len')
-    }
   }
   return fs.args
 }

vlib/flag/flag_test.v

@@ -145,6 +145,7 @@ fn test_finalize_returns_error_for_unknown_flags() {
 
 fn test_allow_to_build_usage_message() {
   mut fp := flag.new_flag_parser([]string)
+  fp.limit_free_args(1, 4)
   fp.application('flag_tool')
   fp.version('v0.0.0')
   fp.description('some short information about this tool')
@@ -158,13 +159,14 @@ fn test_allow_to_build_usage_message() {
   usage := fp.usage()
   mut all_strings_found := true
   for s in ['flag_tool', 'v0.0.0', 
-            'an_int <int>', 'a_bool', 'bool_without', 'a_float <float>', 'a_string <arg>', 
+            'an_int <int>', 'a_bool', 'bool_without', 'a_float <float>', 'a_string <string>:not_stuff',
             'some int to define',
             'some bool to define',
             'this should appear on the next line',
             'some float as well',
             'your credit card number',
-            'usage', 'options:', 'description:',
+            'The arguments should be at least 1 and at most 4 in number.',
+            'Usage', 'Options:', 'Description:',
             'some short information about this tool'] {
     if !usage.contains(s) {
       eprintln(' missing \'$s\' in usage message')
@@ -181,7 +183,7 @@ fn test_if_no_description_given_usage_message_does_not_contain_descpription() {
 
   fp.bool('a_bool', false, '')
   
-  assert !fp.usage().contains('description:')
+  assert !fp.usage().contains('Description:')
 }
 
 fn test_if_no_options_given_usage_message_does_not_contain_options() {
@@ -189,7 +191,7 @@ fn test_if_no_options_given_usage_message_does_not_contain_options() {
   fp.application('flag_tool')
   fp.version('v0.0.0')
   
-  assert !fp.usage().contains('options:')
+  assert !fp.usage().contains('Options:')
 }
 
 fn test_free_args_could_be_limited() {
@@ -206,7 +208,7 @@ fn test_error_for_to_few_free_args() {
   mut fp1 := flag.new_flag_parser(['a', 'b', 'c'])
   fp1.limit_free_args(5, 6)
   args := fp1.finalize() or {
-    assert err == 'Expect at least 5 arguments'
+    assert err.starts_with('Expected at least 5 arguments')
     return
   }
   assert args.len < 0 // expect an error and need to use args 
@@ -216,7 +218,7 @@ fn test_error_for_to_much_free_args() {
   mut fp1 := flag.new_flag_parser(['a', 'b', 'c'])
   fp1.limit_free_args(1, 2)
   args := fp1.finalize() or {
-    assert err == 'Expect at most 2 arguments'
+    assert err.starts_with('Expected at most 2 arguments')
     return
   }
   assert args.len < 0 // expect an error and need to use args 
@@ -226,7 +228,7 @@ fn test_could_expect_no_free_args() {
   mut fp1 := flag.new_flag_parser(['a'])
   fp1.limit_free_args(0, 0)
   args := fp1.finalize() or {
-    assert err == 'Expect no arguments'
+    assert err.starts_with('Expected no arguments')
     return
   }
   assert args.len < 0 // expect an error and need to use args