v fmt: process .v files from a module too

tools/vfmt.v

@@ -5,6 +5,7 @@ module main
 
 import (
 	os
+	os.cmdline
 	filepath
 	compiler
 )
@@ -14,69 +15,141 @@ struct FormatOptions {
 	is_diff    bool
 	is_verbose bool
 	is_all     bool
+	is_worker  bool
 }
 
 fn main() {
-	foptions := FormatOptions{
-		is_w: '-w' in os.args
-		is_diff: '-diff' in os.args
-		is_verbose: '-verbose' in os.args || '--verbose' in os.args
-		is_all: '-all' in os.args || '--all' in os.args
-	}
 	toolexe := os.executable()
 	compiler.set_vroot_folder(filepath.dir(filepath.dir(toolexe)))
 	args := compiler.env_vflags_and_os_args()
+	foptions := FormatOptions{
+		is_w: '-w' in args
+		is_diff: '-diff' in args
+		is_verbose: '-verbose' in args || '--verbose' in args
+		is_all: '-all' in args || '--all' in args
+		is_worker: '-worker' in args
+	}
+	if foptions.is_worker {
+		// -worker should be added by a parent vfmt process.
+		// We launch a sub process for each file because
+		// the v compiler can do an early exit if it detects
+		// a syntax error, but we want to process ALL passed
+		// files if possible.
+		foptions.format_file(cmdline.option(args, '-worker', ''))
+		exit(0)
+	}
+	// we are NOT a worker at this stage, i.e. we are a parent vfmt process
+	possible_files := cmdline.only_non_options(cmdline.after(args, ['fmt']))
 	if foptions.is_verbose {
 		eprintln('vfmt toolexe: $toolexe')
 		eprintln('vfmt args: ' + os.args.str())
 		eprintln('vfmt env_vflags_and_os_args: ' + args.str())
+		eprintln('vfmt possible_files: ' + possible_files.str())
+		eprintln('vfmt foptions: $foptions')
 	}
 	mut files := []string
-	for i := 1; i < args.len; i++ {
+	for file in possible_files {
-		a := args[i]
-		if a == 'fmt' {
-			continue
-		}
-		if !a.starts_with('-') {
-			file := a
 		if !os.exists(file) {
 			compiler.verror('"$file" does not exist.')
 		}
 		if !file.ends_with('.v') {
 			compiler.verror('v fmt can only be used on .v files.\nOffending file: "$file" .')
 		}
-			files << a
+		files << file
-		}
 	}
 	if files.len == 0 {
 		usage()
 		exit(0)
 	}
-	if foptions.is_all {
+	mut cli_args_no_files := []string
-		os.setenv('VFMT_OPTION_ALL', 'yes', true)
+	for a in os.args {
+		if !a in files {
+			cli_args_no_files << a
 		}
+	}
+	mut errors := 0
 	for file in files {
-		format_file(file, foptions)
+		fpath := os.realpath(file)
+		mut worker_command_array := cli_args_no_files.clone()
+		worker_command_array << ['-worker', fpath]
+		worker_cmd := worker_command_array.join(' ')
+		if foptions.is_verbose {
+			eprintln('vfmt worker_cmd: $worker_cmd')
+		}
+		cmdcode := os.system(worker_cmd)
+		if cmdcode != 0 {
+			eprintln('vfmt error while formatting file: $file .')
+			errors++
+		}
+	}
+	if errors > 0 {
+		eprintln('Encountered a total of: ${errors} errors.')
+		exit(1)
 	}
 }
 
-fn format_file(file string, foptions FormatOptions) {
+fn (foptions &FormatOptions) format_file(file string) {
-	mut v := compiler.new_v_compiler_with_args([file])
+	tmpfolder := os.tmpdir()
-	if foptions.is_verbose {
+	mut compiler_params := []string
-		eprintln('vfmt format_file: $file | v.dir: $v.dir')
+	mut cfile := file
-	}
+	mut mod_folder_parent := tmpfolder
-	v.compile()
+	fcontent := os.read_file(file) or {
-	formatted_file_path := os.getenv('VFMT_FILE_RESULT')
-	// eprintln('File: $file .')
-	// eprintln('Formatted file is: $formatted_file_path .')
-	if foptions.is_diff {
-		if find_diff:=os.exec('diff -v'){
-			os.system('diff "$formatted_file_path" "$file" ')
 		return
 	}
+	is_test_file := file.ends_with('_test.v')
+	is_module_file := fcontent.contains('module ') && !fcontent.contains('module main\n')
+	use_tmp_main_program := is_module_file && !is_test_file
+	mod_folder := filepath.basedir(file)
+	mut mod_name := 'main'
+	if is_module_file {
+		mod_name = filepath.filename(mod_folder)
+	}
+	if use_tmp_main_program {
+		// TODO: remove the need for this
+		// This makes a small program that imports the module,
+		// so that the module files will get processed by the
+		// vfmt implementation.
+		mod_folder_parent = filepath.basedir(mod_folder)
+		mut main_program_content := 'import ${mod_name} \n fn main(){}'
+		if fcontent.contains('module builtin\n') {
+			main_program_content = 'fn main(){}'
+		}
+		main_program_file := filepath.join(tmpfolder,'vfmt_tmp_${mod_name}_program.v')
+		if os.exists(main_program_file) {
+			os.rm(main_program_file)
+		}
+		os.write_file(main_program_file, main_program_content)
+		cfile = main_program_file
+		compiler_params << ['-user_mod_path', mod_folder_parent]
+	}
+	compiler_params << cfile
+	if foptions.is_verbose {
+		eprintln('vfmt format_file: file: $file')
+		eprintln('vfmt format_file: cfile: $cfile')
+		eprintln('vfmt format_file: is_test_file: $is_test_file')
+		eprintln('vfmt format_file: is_module_file: $is_module_file')
+		eprintln('vfmt format_file: mod_name: $mod_name')
+		eprintln('vfmt format_file: mod_folder: $mod_folder')
+		eprintln('vfmt format_file: mod_folder_parent: $mod_folder_parent')
+		eprintln('vfmt format_file: use_tmp_main_program: $use_tmp_main_program')
+		eprintln('vfmt format_file: compiler_params: $compiler_params')
+		eprintln('-------------------------------------------')
+	}
+	formatted_file_path := foptions.compile_file(file, compiler_params)
+	if use_tmp_main_program {
+		os.rm(cfile)
+	}
+	if formatted_file_path.len == 0 {
+		return
+	}
+	if foptions.is_diff {
+		diff_cmd := find_working_diff_command() or {
 			eprintln('No working "diff" CLI command found.')
 			return
 		}
+		os.system('$diff_cmd --minimal  --text   --unified=2 --show-function-line="fn " "$file" "$formatted_file_path" ')
+		return
+	}
 	if foptions.is_w {
 		os.mv_by_cp(formatted_file_path, file) or {
 			panic(err)
@@ -92,10 +165,36 @@ fn format_file(file string, foptions FormatOptions) {
 }
 
 fn usage() {
-	print('Usage: tools/vfmt [flags] path_to_source.v [path_to_other_source.v]
+	print('Usage: tools/vfmt [flags] fmt path_to_source.v [path_to_other_source.v]
 Formats the given V source files, and prints their formatted source to stdout.
 Options:
   -diff display only diffs between the formatted source and the original source.
   -w    write result to (source) file(s) instead of to stdout.
 ')
 }
+
+fn find_working_diff_command() ?string {
+	for diffcmd in ['colordiff', 'diff', 'colordiff.exe', 'diff.exe'] {
+		p := os.exec('$diffcmd --version') or {
+			continue
+		}
+		if p.exit_code == 0 {
+			return diffcmd
+		}
+	}
+	return error('no working diff command found')
+}
+
+fn (foptions &FormatOptions) compile_file(file string, compiler_params []string) string {
+	mut v := compiler.new_v_compiler_with_args(compiler_params)
+	v.v_fmt_file = file
+	if foptions.is_all {
+		v.v_fmt_all = true
+	}
+	v.compile()
+	return v.v_fmt_file_result
+}
+
+pub fn (f FormatOptions) str() string {
+	return 'FormatOptions{ ' + ' is_w: $f.is_w' + ' is_diff: $f.is_diff' + ' is_verbose: $f.is_verbose' + ' is_all: $f.is_all' + ' is_worker: $f.is_worker' + ' }'
+}

tools/vtest.v

@@ -2,6 +2,7 @@ module main
 
 import (
 	os
+	os.cmdline
 	testing
 )
 
@@ -20,18 +21,18 @@ pub fn main() {
 		return
 	}
 
-	args_string := args[1..].join(' ')
+	args_to_executable := args[1..]
-	args_before := args_string.all_before('test ')
+	args_before := cmdline.before(args_to_executable, ['test'])
-	args_after  := args_string.all_after('test ')
+	args_after := cmdline.after(args_to_executable, ['test'])
 
-	if args_after == 'v' {
+	if args_after.join(' ') == 'v' {
 		eprintln('`v test v` has been deprecated.')
 		eprintln('Use `v test-compiler` instead.')
 		exit(1)
 	}
 
-	mut ts := testing.new_test_session(args_before)
+	mut ts := testing.new_test_session(args_before.join(' '))
-	for targ in args_after.split(' ') {
+	for targ in args_after {
 		if os.exists(targ) && targ.ends_with('_test.v') {
 			ts.files << targ
 			continue

vlib/compiler/cc.v

@@ -5,6 +5,7 @@ module compiler
 
 import (
 	os
+	os.cmdline
 	time
 	filepath
 )
@@ -467,9 +468,9 @@ fn (c &V) build_thirdparty_obj_files() {
 }
 
 fn find_c_compiler() string {
-	args := env_vflags_and_os_args().join(' ')
+	args := env_vflags_and_os_args()
 	defaultcc := find_c_compiler_default()
-	return get_arg(args, 'cc', defaultcc)
+	return cmdline.option(args, '-cc', defaultcc)
 }
 
 fn find_c_compiler_default() string {
@@ -486,7 +487,7 @@ fn find_c_compiler_default() string {
 
 fn find_c_compiler_thirdparty_options() string {
 	fullargs := env_vflags_and_os_args()
-	mut cflags := get_cmdline_multiple_values(fullargs,'-cflags')
+	mut cflags := cmdline.many_values(fullargs,'-cflags')
 	$if !windows {
 		cflags << '-fPIC'
 	}
@@ -496,16 +497,6 @@ fn find_c_compiler_thirdparty_options() string {
 	return cflags.join(' ')
 }
 
-fn get_cmdline_multiple_values(args []string, optname string) []string {
-	mut flags := []string
-	for ci, cv in args {
-		if cv == optname {
-			flags << args[ci + 1]
-		}
-	}
-	return flags
-}
-
 fn parse_defines(defines []string) ([]string,[]string) {
 	// '-d abc -d xyz=1 -d qwe=0' should produce:
 	// compile_defines:      ['abc','xyz']

vlib/compiler/main.v

@@ -5,6 +5,7 @@ module compiler
 
 import (
 	os
+	os.cmdline
 	strings
 	filepath
 	compiler.x64
@@ -80,6 +81,10 @@ pub mut:
 	// -d vfmt and -d another=0 for `$if vfmt { will execute }` and `$if another { will NOT get here }`
 	compile_defines     []string // just ['vfmt']
 	compile_defines_all []string // contains both: ['vfmt','another']
+
+	v_fmt_all           bool   // << input set by tools/vfmt.v
+	v_fmt_file          string // << file given by the user from tools/vfmt.v
+	v_fmt_file_result   string // >> file with formatted output generated by vlib/compiler/vfmt.v
 }
 
 struct Preferences {
@@ -580,20 +585,10 @@ pub fn (v V) run_compiled_executable_and_exit() {
 		println('============ running $v.out_name ============')
 	}
 	mut cmd := '"' + final_target_out_name(v.out_name).replace('.exe', '') + '"'
-	mut args_after := ' '
+	args_after_no_options := cmdline.only_non_options( cmdline.after(args,['run','test']) )
-	for i, a in args {
+	if args_after_no_options.len > 1 {
-		if i == 0 {
+		cmd += ' ' + args_after_no_options[1..].join(' ')
-			continue
 	}
-		if a.starts_with('-') {
-			continue
-		}
-		if a in ['run', 'test'] {
-			args_after += args[i + 2..].join(' ')
-			break
-		}
-	}
-	cmd += args_after
 	if v.pref.is_test {
 		ret := os.system(cmd)
 		if ret != 0 {
@@ -876,36 +871,6 @@ pub fn (v mut V) parse_lib_imports() {
 	}
 }
 
-pub fn get_arg(joined_args, arg, def string) string {
-	return get_param_after(joined_args, '-$arg', def)
-}
-
-pub fn get_param_after(joined_args, arg, def string) string {
-	key := '$arg '
-	mut pos := joined_args.index(key) or {
-		return def
-	}
-	pos += key.len
-	mut space := joined_args.index_after(' ', pos)
-	if space == -1 {
-		space = joined_args.len
-	}
-	res := joined_args[pos..space]
-	return res
-}
-
-pub fn get_cmdline_option(args []string, param string, def string) string {
-	mut found := false
-	for arg in args {
-		if found {
-			return arg
-		}
-		else if param == arg {
-			found = true
-		}
-	}
-	return def
-}
 
 pub fn (v &V) log(s string) {
 	if !v.pref.is_verbose {
@@ -925,19 +890,19 @@ pub fn new_v(args []string) &V {
 		}
 	}
 	// optional, custom modules search path
-	user_mod_path := get_cmdline_option(args, '-user_mod_path', '')
+	user_mod_path := cmdline.option(args, '-user_mod_path', '')
 	// Location of all vlib files
 	vroot := filepath.dir(vexe_path())
-	vlib_path := get_cmdline_option(args, '-vlib-path', filepath.join(vroot,'vlib'))
+	vlib_path := cmdline.option(args, '-vlib-path', filepath.join(vroot,'vlib'))
-	vpath := get_cmdline_option(args, '-vpath', v_modules_path)
+	vpath := cmdline.option(args, '-vpath', v_modules_path)
 	mut vgen_buf := strings.new_builder(1000)
 	vgen_buf.writeln('module vgen\nimport strings')
-	joined_args := args.join(' ')
+	target_os := cmdline.option(args, '-os', '')
-	target_os := get_arg(joined_args, 'os', '')
+	mut out_name := cmdline.option(args, '-o', 'a.out')
-	mut out_name := get_arg(joined_args, 'o', 'a.out')
 	mut dir := args.last()
 	if 'run' in args {
-		dir = get_param_after(joined_args, 'run', '')
+		args_after_run := cmdline.only_non_options( cmdline.after(args,['run']) )
+		dir = if args_after_run.len>0 { args_after_run[0] } else { '' }
 	}
 	if dir.ends_with(os.path_separator) {
 		dir = dir.all_before_last(os.path_separator)
@@ -948,9 +913,11 @@ pub fn new_v(args []string) &V {
 	if args.len < 2 {
 		dir = ''
 	}
+	
 	// build mode
 	mut build_mode := BuildMode.default_mode
 	mut mod := ''	
+	joined_args := args.join(' ')
 	if joined_args.contains('build module ') {
 		build_mode = .build_module
 		os.chdir(vroot)
@@ -1058,9 +1025,9 @@ pub fn new_v(args []string) &V {
 		exit(1)
 	}
 	mut out_name_c := get_vtmp_filename(out_name, '.tmp.c')
-	cflags := get_cmdline_multiple_values(args, '-cflags').join(' ')
+	cflags := cmdline.many_values(args, '-cflags').join(' ')
   
-	defines := get_cmdline_multiple_values(args, '-d')
+	defines := cmdline.many_values(args, '-d')
 	compile_defines, compile_defines_all := parse_defines( defines )
 	
 	rdir := os.realpath(dir)

vlib/compiler/query.v

@@ -47,12 +47,16 @@ fn (p mut Parser) select_query(fn_ph int) string {
 	p.sql_types = []
 	mut q := 'select '
 	p.check(.key_select)
+	p.fspace()
 	n := p.check_name()
+	p.fspace()
 	if n == 'count' {
 		q += 'count(*) from '
 		p.check_name()
+		p.fspace()
 	}
 	table_name := p.check_name()
+	p.fspace()
 	// Register this type's fields as variables so they can be used in `where`
 	// expressions
 	typ := p.table.find_type(table_name)
@@ -105,6 +109,7 @@ fn (p mut Parser) select_query(fn_ph int) string {
 	// `where` statement
 	if p.tok == .name && p.lit == 'where' {
 		p.next()
+		p.fspace()
 		p.is_sql = true
 		_,expr := p.tmp_expr()
 		p.is_sql = false
@@ -113,9 +118,12 @@ fn (p mut Parser) select_query(fn_ph int) string {
 	// limit?
 	mut query_one := false
 	if p.tok == .name && p.lit == 'limit' {
+		p.fspace()
 		p.next()
+		p.fspace()
 		p.is_sql = true
 		_,limit := p.tmp_expr()
+		p.fspace()
 		p.is_sql = false
 		q += ' limit ' + limit
 		// `limit 1` means we are getting `?User`, not `[]User`
@@ -264,12 +272,15 @@ fn (p mut Parser) insert_query(fn_ph int) {
 fn (p mut Parser) update_query(fn_ph int) {
 	println('update query')
 	p.check_name()
+	p.fspace()
 	table_name := p.check_name()
+	p.fspace()
 	typ := p.table.find_type(table_name)
 	if typ.name == '' {
 		p.error('unknown type `$table_name`')
 	}
 	set := p.check_name()
+	p.fspace()
 	if set != 'set' {
 		p.error('expected `set`')
 	}
@@ -280,7 +291,9 @@ fn (p mut Parser) update_query(fn_ph int) {
 		p.error('V orm: `id int` must be the first field in `$typ.name`')
 	}
 	field := p.check_name()
+	p.fspace()
 	p.check(.assign)
+  p.fspace()
 	for f in typ.fields {
 		if !(f.typ in ['string', 'int', 'bool']) {
 			println('orm: skipping $f.name')
@@ -312,6 +325,7 @@ fn (p mut Parser) update_query(fn_ph int) {
 	// where
 	if p.tok == .name && p.lit == 'where' {
 		p.next()
+		p.fspace()
 		p.is_sql = true
 		_,wexpr := p.tmp_expr()
 		p.is_sql = false

vlib/compiler/vfmt.v

@@ -236,8 +236,9 @@ fn (p mut Parser) fnext() {
 
 [if vfmt]
 fn (p mut Parser) fremove_last() {
+	if p.scanner.fmt_lines.len > 0 {
 		p.scanner.fmt_lines[p.scanner.fmt_lines.len-1] = ''
-
+	}
 }
 
 [if vfmt]
@@ -249,15 +250,16 @@ fn (p &Parser) gen_fmt() {
 	if p.file_name == '' {
 		return
 	}
-	is_all := os.getenv('VFMT_OPTION_ALL') == 'yes'
+	is_all := p.v.v_fmt_all
-	if p.file_path != p.v.dir && !is_all {
+	vfmt_file := p.v.v_fmt_file
+	if p.file_path != vfmt_file && !is_all {
 		// skip everything except the last file (given by the CLI argument)
 		return
 	}
 	//s := p.scanner.fmt_out.str().replace('\n\n\n', '\n').trim_space()
 	//s := p.scanner.fmt_out.str().trim_space()
 	//p.scanner.fgenln('// nice')
-	s := p.scanner.fmt_lines.join('')
+	s1 := p.scanner.fmt_lines.join('')
 /*.replace_each([
 		'\n\n\n\n', '\n\n',
 		' \n', '\n',
@@ -265,11 +267,13 @@ fn (p &Parser) gen_fmt() {
 	])
 	*/
 	//.replace('\n\n\n\n', '\n\n')
-	.replace_each([
+  
-		' \n', '\n',
+	s2 := s1.replace(' \n', '\n')
-		') or{', ') or {',
+	s3 := s2.replace(') or{', ') or {')
-		')or{', ') or {',
+	s4 := s3.replace(')or{', ') or {')
-	])
+	s5 := s4.replace('or{', 'or {')
+
+	s := s5
 
 	if s == '' {
 		return
@@ -284,9 +288,10 @@ fn (p &Parser) gen_fmt() {
 			eprintln('Written fmt file to: $p.file_path')
 		}
 	}
-	if p.file_path == p.v.dir {
+	if p.file_path == vfmt_file {
 		res_path := write_formatted_source( p.file_name, s )
-		os.setenv('VFMT_FILE_RESULT', res_path, true )
+		mut vv := p.v
+		vv.v_fmt_file_result = res_path
 	}
 }
 

vlib/os/cmdline/cmdline.v

@@ -0,0 +1,62 @@
+module cmdline
+
+pub fn many_values(args []string, optname string) []string {
+	mut flags := []string
+	for ci, cv in args {
+		if cv == optname {
+			if ci + 1 < args.len {
+				flags << args[ci + 1]
+			}
+		}
+	}
+	return flags
+}
+
+pub fn option(args []string, param string, def string) string {
+	mut found := false
+	for arg in args {
+		if found {
+			return arg
+		}
+		else if param == arg {
+			found = true
+		}
+	}
+	return def
+}
+
+pub fn before(args []string, what []string) []string {
+	mut found := false
+	mut args_before := []string
+	for a in args {
+		if a in what {
+			found = true
+			break
+		}
+		args_before << a
+	}
+	return args_before
+}
+
+pub fn after(args []string, what []string) []string {
+	mut found := false
+	mut args_after := []string
+	for a in args {
+		if a in what {
+			found = true
+			continue
+		}
+		if found {
+			args_after << a
+		}
+	}
+	return args_after
+}
+
+pub fn only_non_options(args []string) []string {
+	return args.filter(!it.starts_with('-'))
+}
+
+pub fn only_options(args []string) []string {
+	return args.filter(it.starts_with('-'))
+}