v: add VCACHE support for thirdparty object files and for v build-module

vlib/v/builder/cc.v

@@ -118,6 +118,21 @@ fn (mut v Builder) post_process_c_compiler_output(res os.Result) {
 	verror(c_error_info)
 }
 
+fn (mut v Builder) rebuild_cached_module(vexe string, imp_path string) string {
+	res := v.pref.cache_manager.exists('.o', imp_path) or {
+		println('Cached $imp_path .o file not found... Building .o file for $imp_path')
+		boptions := v.pref.build_options.join(' ')
+		rebuild_cmd := '$vexe $boptions build-module $imp_path'
+		// eprintln('>> rebuild_cmd: $rebuild_cmd')
+		os.system(rebuild_cmd)
+		rebuilded_o := v.pref.cache_manager.exists('.o', imp_path) or {
+			panic('could not rebuild cache module for $imp_path, error: $err')
+		}
+		return rebuilded_o
+	}
+	return res
+}
+
 fn (mut v Builder) cc() {
 	if os.executable().contains('vfmt') {
 		return
@@ -260,14 +275,9 @@ fn (mut v Builder) cc() {
 		linker_flags << '-nostdlib'
 	}
 	if v.pref.build_mode == .build_module {
-		// Create the modules & out directory if it's not there.
+		v.pref.out_name = v.pref.cache_manager.postfix_with_key2cpath('.o', v.pref.path) // v.out_name
-		out_dir := os.join_path(pref.default_module_path, 'cache', v.pref.path)
+		println('Building $v.pref.path to $v.pref.out_name ...')
-		pdir := out_dir.all_before_last(os.path_separator)
+		v.pref.cache_manager.save('.description.txt', v.pref.path, '${v.pref.path:-30} @ $v.pref.cache_manager.vopts\n')
-		if !os.is_dir(pdir) {
-			os.mkdir_all(pdir)
-		}
-		v.pref.out_name = '${out_dir}.o' // v.out_name
-		println('Building ${v.pref.out_name}...')
 		// println('v.table.imports:')
 		// println(v.table.imports)
 	}
@@ -349,10 +359,7 @@ fn (mut v Builder) cc() {
 		args << '-c'
 	} else if v.pref.use_cache {
 		mut built_modules := []string{}
-		builtin_obj_path := os.join_path(pref.default_module_path, 'cache', 'vlib', 'builtin.o')
+		builtin_obj_path := v.rebuild_cached_module(vexe, 'vlib/builtin')
-		if !os.exists(builtin_obj_path) {
-			os.system('$vexe build-module vlib/builtin')
-		}
 		libs += ' ' + builtin_obj_path
 		for ast_file in v.parsed_files {
 			for imp_stmt in ast_file.imports {
@@ -382,14 +389,8 @@ fn (mut v Builder) cc() {
 				// continue
 				// }
 				imp_path := os.join_path('vlib', mod_path)
-				cache_path := os.join_path(pref.default_module_path, 'cache')
+				obj_path := v.rebuild_cached_module(vexe, imp_path)
-				obj_path := os.join_path(cache_path, '${imp_path}.o')
+				libs += ' ' + obj_path
-				if os.exists(obj_path) {
-					libs += ' ' + obj_path
-				} else {
-					println('$obj_path not found... building module $imp')
-					os.system('$vexe build-module $imp_path')
-				}
 				if obj_path.ends_with('vlib/ui.o') {
 					args << '-framework Cocoa -framework Carbon'
 				}
@@ -777,21 +778,19 @@ fn (mut v Builder) build_thirdparty_obj_file(path string, moduleflags []cflag.CF
 	if v.pref.os == .windows {
 		// Cross compiling for Windows
 		$if !windows {
-			if os.exists(obj_path) {
-				os.rm(obj_path)
-			}
 			v.pref.ccompiler = mingw_cc
 		}
 	}
-	if os.exists(obj_path) {
+	opath := v.pref.cache_manager.postfix_with_key2cpath('.o', obj_path)
+	if os.exists(opath) {
 		return
 	}
-	println('$obj_path not found, building it...')
+	println('$obj_path not found, building it in $opath ...')
 	cfile := '${path[..path.len - 2]}.c'
 	btarget := moduleflags.c_options_before_target()
 	atarget := moduleflags.c_options_after_target()
 	cppoptions := if v.pref.ccompiler.contains('++') { ' -fpermissive -w ' } else { '' }
-	cmd := '$v.pref.ccompiler $cppoptions $v.pref.third_party_option $btarget -c -o "$obj_path" "$cfile" $atarget'
+	cmd := '$v.pref.ccompiler $cppoptions $v.pref.third_party_option $btarget -c -o "$opath" "$cfile" $atarget'
 	res := os.exec(cmd) or {
 		eprintln('exec failed for thirdparty object build cmd:\n$cmd')
 		verror(err)
@@ -802,6 +801,7 @@ fn (mut v Builder) build_thirdparty_obj_file(path string, moduleflags []cflag.CF
 		verror(res.output)
 		return
 	}
+	v.pref.cache_manager.save('.description.txt', obj_path, '${obj_path:-30} @ $cmd\n')
 	println(res.output)
 }
 

vlib/v/builder/cflags.v

@@ -1,15 +1,19 @@
 module builder
 
+import os
 import v.cflag
 
 // get flags for current os
-fn (v &Builder) get_os_cflags() []cflag.CFlag {
+fn (mut v Builder) get_os_cflags() []cflag.CFlag {
 	mut flags := []cflag.CFlag{}
 	mut ctimedefines := []string{}
 	if v.pref.compile_defines.len > 0 {
 		ctimedefines << v.pref.compile_defines
 	}
-	for flag in v.table.cflags {
+	for mut flag in v.table.cflags {
+		if flag.value.ends_with('.o') {
+			flag.cached = v.pref.cache_manager.postfix_with_key2cpath('.o', os.real_path(flag.value))
+		}
 		if flag.os == '' ||
 			(flag.os == 'linux' && v.pref.os == .linux) ||
 			(flag.os == 'macos' && v.pref.os == .macos) ||
@@ -27,7 +31,7 @@ fn (v &Builder) get_os_cflags() []cflag.CFlag {
 	return flags
 }
 
-fn (v &Builder) get_rest_of_module_cflags(c &cflag.CFlag) []cflag.CFlag {
+fn (mut v Builder) get_rest_of_module_cflags(c &cflag.CFlag) []cflag.CFlag {
 	mut flags := []cflag.CFlag{}
 	cflags := v.get_os_cflags()
 	for flag in cflags {

vlib/v/cflag/cflags.v

@@ -8,19 +8,24 @@ import os
 // parsed cflag
 pub struct CFlag {
 pub:
-	mod   string // the module in which the flag was given
+	mod    string // the module in which the flag was given
-	os    string // eg. windows | darwin | linux
+	os     string // eg. windows | darwin | linux
-	name  string // eg. -I
+	name   string // eg. -I
-	value string // eg. /path/to/include
+	value  string // eg. /path/to/include
+pub mut:
+	cached string // eg. ~/.vmodules/cache/ea/ea9878886727367672163.o (for .o files)
 }
 
 pub fn (c &CFlag) str() string {
-	return 'CFlag{ name: "$c.name" value: "$c.value" mod: "$c.mod" os: "$c.os" }'
+	return 'CFlag{ name: "$c.name" value: "$c.value" mod: "$c.mod" os: "$c.os" cached: "$c.cached" }'
 }
 
 // format flag
 pub fn (cf &CFlag) format() string {
 	mut value := cf.value
+	if cf.cached != '' {
+		value = cf.cached
+	}
 	if cf.name in ['-l', '-Wa', '-Wl', '-Wp'] && value.len > 0 {
 		return '$cf.name$value'.trim_space()
 	}

vlib/v/pref/default.v

@@ -71,6 +71,17 @@ pub fn (mut p Preferences) fill_with_defaults() {
 			}
 		}
 	}
+	// Prepare the cache manager. All options that can affect the generated cached .c files
+	// should go into res.cache_manager.vopts, which is used as a salt for the cache hash.
+	p.cache_manager = new_cache_manager([
+		'$p.backend | $p.os | $p.ccompiler',
+		p.cflags.trim_space(),
+		p.third_party_option.trim_space(),
+		'$p.compile_defines_all',
+		'$p.compile_defines',
+		'$p.lookup_path',
+	])
+	// eprintln('prefs.cache_manager: $p')
 }
 
 fn default_c_compiler() string {

vlib/v/pref/pref.v

@@ -129,6 +129,8 @@ pub mut:
 	is_ios_simulator    bool
 	is_apk              bool // build as Android .apk format
 	cleanup_files       []string // list of temporary *.tmp.c and *.tmp.c.rsp files. Cleaned up on successfull builds.
+	build_options       []string // list of options, that should be passed down to `build-module`, if needed for -usecache
+	cache_manager       CacheManager
 }
 
 pub fn parse_args(args []string) (&Preferences, string) {
@@ -142,6 +144,7 @@ pub fn parse_args(args []string) (&Preferences, string) {
 		match arg {
 			'-apk' {
 				res.is_apk = true
+				res.build_options << arg
 			}
 			'-show-timings' {
 				res.show_timings = true
@@ -167,10 +170,12 @@ pub fn parse_args(args []string) (&Preferences, string) {
 			'-g' {
 				res.is_debug = true
 				res.is_vlines = true
+				res.build_options << arg
 			}
 			'-cg' {
 				res.is_debug = true
 				res.is_vlines = false
+				res.build_options << arg
 			}
 			'-repl' {
 				res.is_repl = true
@@ -190,19 +195,23 @@ pub fn parse_args(args []string) (&Preferences, string) {
 			}
 			'-autofree' {
 				res.autofree = true
+				res.build_options << arg
 			}
 			'-compress' {
 				res.compress = true
 			}
 			'-freestanding' {
 				res.is_bare = true
+				res.build_options << arg
 			}
 			'-no-preludes' {
 				res.no_preludes = true
+				res.build_options << arg
 			}
 			'-prof', '-profile' {
 				res.profile_file = cmdline.option(current_args, '-profile', '-')
 				res.is_prof = true
+				res.build_options << '$arg $res.profile_file'
 				i++
 			}
 			'-profile-no-inline' {
@@ -210,6 +219,7 @@ pub fn parse_args(args []string) (&Preferences, string) {
 			}
 			'-prod' {
 				res.is_prod = true
+				res.build_options << arg
 			}
 			'-simulator' {
 				res.is_ios_simulator = true
@@ -240,6 +250,7 @@ pub fn parse_args(args []string) (&Preferences, string) {
 			}
 			'-prealloc' {
 				res.prealloc = true
+				res.build_options << arg
 			}
 			'-keepc' {
 				eprintln('-keepc is deprecated. V always keeps the generated .tmp.c files now.')
@@ -249,6 +260,7 @@ pub fn parse_args(args []string) (&Preferences, string) {
 			}
 			'-x64' {
 				res.backend = .x64
+				res.build_options << arg
 			}
 			'-W' {
 				res.warns_are_errors = true
@@ -277,6 +289,7 @@ pub fn parse_args(args []string) (&Preferences, string) {
 					exit(1)
 				}
 				res.os = target_os_kind
+				res.build_options << '$arg $target_os'
 			}
 			'-printfn' {
 				res.printfn_list << cmdline.option(current_args, '-printfn', '')
@@ -284,6 +297,7 @@ pub fn parse_args(args []string) (&Preferences, string) {
 			}
 			'-cflags' {
 				res.cflags += ' ' + cmdline.option(current_args, '-cflags', '')
+				res.build_options << '$arg "$res.cflags.trim_space()"'
 				i++
 			}
 			'-define', '-d' {
@@ -295,6 +309,7 @@ pub fn parse_args(args []string) (&Preferences, string) {
 			}
 			'-cc' {
 				res.ccompiler = cmdline.option(current_args, '-cc', 'cc')
+				res.build_options << '$arg "$res.ccompiler"'
 				i++
 			}
 			'-o' {
@@ -302,7 +317,9 @@ pub fn parse_args(args []string) (&Preferences, string) {
 				i++
 			}
 			'-b' {
-				b := backend_from_string(cmdline.option(current_args, '-b', 'c')) or {
+				sbackend := cmdline.option(current_args, '-b', 'c')
+				res.build_options << '$arg $sbackend'
+				b := backend_from_string(sbackend) or {
 					continue
 				}
 				res.backend = b
@@ -310,11 +327,13 @@ pub fn parse_args(args []string) (&Preferences, string) {
 			}
 			'-path' {
 				path := cmdline.option(current_args, '-path', '')
+				res.build_options << '$arg "$path"'
 				res.lookup_path = path.replace('|', os.path_delimiter).split(os.path_delimiter)
 				i++
 			}
 			'-custom-prelude' {
 				path := cmdline.option(current_args, '-custom-prelude', '')
+				res.build_options << '$arg $path'
 				prelude := os.read_file(path) or {
 					eprintln('cannot open custom prelude file: $err')
 					exit(1)
@@ -390,6 +409,13 @@ pub fn parse_args(args []string) (&Preferences, string) {
 		res.build_mode = .build_module
 		res.path = args[command_pos + 1]
 	}
+	// keep only the unique res.build_options:
+	mut m := map[string]string{}
+	for x in res.build_options {
+		m[x] = ''
+	}
+	res.build_options = m.keys()
+	// eprintln('>> res.build_options: $res.build_options')
 	res.fill_with_defaults()
 	return res, command
 }

vlib/v/pref/vcache.v

@@ -0,0 +1,85 @@
+module pref
+
+import os
+import crypto.md5
+
+// Using a 2 level cache, ensures a more even distribution of cache entries,
+// so there will not be cramped folders that contain many thousands of them.
+// Most filesystems can not handle performantly such folders, and slow down.
+// The first level will contain a max of 256 folders, named from 00/ to ff/.
+// Each of them will contain many entries, but hopefully < 1000.
+// NB: using a hash here, makes the cache storage immune to special
+// characters in the keys, like quotes, spaces and so on.
+// Cleanup of the cache is simple: just delete the $VCACHE folder.
+// The cache tree will look like this:
+// │ $VCACHE
+// │ ├── 0f
+// │ │   ├── 0f004f983ab9c487b0d7c1a0a73840a5.txt
+// │ │   ├── 0f599edf5e16c2756fbcdd4c865087ac.description.txt <-- build details
+// │ │   └── 0f599edf5e16c2756fbcdd4c865087ac.vh
+// │ ├── 29
+// │ │   ├── 294717dd02a1cca5f2a0393fca2c5c22.o
+// │ │   └── 294717dd02a1cca5f2a0393fca2c5c22.description.txt <-- build details
+// │ ├── 62
+// │ │   └── 620d60d6b81fdcb3cab030a37fd86996.h
+// │ └── 76
+// │     └── 7674f983ab9c487b0d7c1a0ad73840a5.c
+pub struct CacheManager {
+pub:
+	basepath string
+pub mut:
+	vopts    string
+	k2cpath  map[string]string // key -> filesystem cache path for the object
+}
+
+fn new_cache_manager(opts []string) CacheManager {
+	mut vcache_basepath := os.getenv('VCACHE')
+	if vcache_basepath == '' {
+		vcache_basepath = os.join_path(os.home_dir(), '.vmodules', 'cache')
+	}
+	return CacheManager{
+		basepath: vcache_basepath
+		vopts: opts.join('|')
+	}
+}
+
+pub fn (mut cm CacheManager) key2cpath(key string) string {
+	mut cpath := cm.k2cpath[key]
+	if cpath == '' {
+		hk := cm.vopts + key
+		hash := md5.sum(hk.bytes()).hex()
+		prefix := hash[0..2]
+		cprefix_folder := os.join_path(cm.basepath, prefix)
+		cpath = os.join_path(cprefix_folder, hash)
+		if !os.is_dir(cprefix_folder) {
+			os.mkdir_all(cprefix_folder)
+			os.chmod(cprefix_folder, 0o777)
+		}
+		cm.k2cpath[key] = cpath
+	}
+	return cpath
+}
+
+pub fn (mut cm CacheManager) postfix_with_key2cpath(postfix string, key string) string {
+	return cm.key2cpath(key) + postfix
+}
+
+pub fn (mut cm CacheManager) exists(postfix string, key string) ?string {
+	fpath := cm.postfix_with_key2cpath(postfix, key)
+	if !os.exists(fpath) {
+		return error('does not exist yet')
+	}
+	return fpath
+}
+
+pub fn (mut cm CacheManager) save(postfix string, key string, content string) ?string {
+	fpath := cm.postfix_with_key2cpath(postfix, key)
+	os.write_file(fpath, content) ?
+	return fpath
+}
+
+pub fn (mut cm CacheManager) load(postfix string, key string) ?string {
+	fpath := cm.exists(postfix, key) ?
+	content := os.read_file(fpath) ?
+	return content
+}