live: use mostly pure V code for reloading, eases customization

cmd/tools/preludes/live.v

@@ -0,0 +1,5 @@
+module main
+
+// This prelude is loaded in every v program compiled with -live,
+// in both the main executable, and in the shared library.
+import live

cmd/tools/preludes/live_main.v

@@ -1,11 +1,5 @@
 module main
 
-import os
+// This prelude is loaded in every v program compiled with -live,
-import time
+// but only for the main executable.
-import dl
+import live.executable
-
-const (
-	os_used = os.MAX_PATH
-	time_used = time.now()
-	dl_used = dl.version
-)

cmd/tools/preludes/live_shared.v

@@ -1,9 +1,5 @@
 module main
 
-import os
+// This prelude is loaded in every v program compiled with -live,
-import time
+// but only for the shared library.
-
+import live.shared
-const (
-	os_used = os.MAX_PATH
-	time_used = time.now()
-)

examples/hot_reload/bounce.v

@@ -79,8 +79,8 @@ const (
 [live]
 fn (game &Game) draw() {
 	game.gg.draw_rect(game.x, game.y, width, width, blue)
-	game.gg.draw_rect(550 - game.x + 10, 200 - game.y + 50, width, width, gx.rgb(128, 10, 255))
+	game.gg.draw_rect(550 - game.x + 10, 200 - game.y + 50, width, width, gx.rgb(228, 10, 55))
-	game.gg.draw_rect(game.x - 20, 250 - game.y, width, width, gx.rgb(128, 240, 155))
+	game.gg.draw_rect(game.x - 25, 250 - game.y, width, width, gx.rgb(28, 240, 55))
 }
 
 [live]

examples/hot_reload/message.v

@@ -1,13 +1,13 @@
 module main
 
-// Build this example with
+// Build this example with `v -live message.v`
-// v -live message.v
-
 import time
+import live
 
 [live]
 fn print_message() {
-	println('Hello! Modify this message while the program is running.')
+	info := live.info()
+	println('OK reloads: ${info.reloads_ok:4d} | Total reloads: ${info.reloads:4d} | Hello! Modify this message while the program is running.')
 }
 
 fn main() {

vlib/dl/dl_nix.c.v

@@ -4,6 +4,7 @@ module dl
 
 pub const (
 	RTLD_NOW = C.RTLD_NOW
+	RTLD_LAZY = C.RTLD_LAZY
 	DL_EXT   = '.so'
 )
 

vlib/dl/dl_windows.c.v

@@ -2,6 +2,7 @@ module dl
 
 pub const (
 	RTLD_NOW = 0
+	RTLD_LAZY = 0
 	DL_EXT   = '.dll'
 )
 

vlib/live/common.v

@@ -0,0 +1,60 @@
+module live
+
+pub type FNLinkLiveSymbols = fn (linkcb voidptr)
+
+pub type FNLiveReloadCB = fn (info &LiveReloadInfo)
+
+pub struct LiveReloadInfo {
+pub:
+	vexe              string // full path to the v compiler
+	vopts             string // v compiler options for a live shared library
+	original          string // full path to the original source file, compiled with -live
+	live_fn_mutex     voidptr // the address of the C mutex, that locks the [live] fns during reloads.
+	live_linkfn       FNLinkLiveSymbols // generated C callback; receives a dlopen handle
+	so_extension      string // .so or .dll
+	so_name_template  string // a sprintf template for the shared libraries location
+mut:
+	live_lib          voidptr // the result of dl.open
+	reloads           int // how many times a reloading was tried
+	reloads_ok        int // how many times the reloads succeeded
+	reload_time_ms    int // how much time the last reload took (compilation + loading)
+	last_mod_ts       int // a timestamp for when the original was last changed
+	recheck_period_ms int = 100 // how often do you want to check for changes
+	cb_recheck        FNLiveReloadCB = 0 // executed periodically
+	cb_compile_failed FNLiveReloadCB = 0 // executed when a reload compilation failed
+	cb_before         FNLiveReloadCB = 0 // executed before a reload try happens
+	cb_after          FNLiveReloadCB = 0 // executed after a reload try happened, even if failed
+	cb_locked_before  FNLiveReloadCB = 0 // executed before lib reload, in the mutex section
+	cb_locked_after   FNLiveReloadCB = 0 // executed after lib reload, in the mutex section
+}
+
+// LiveReloadInfo.live_linkfn should be called by the reloader
+// to dlsym all live functions. TODO: research a way to implement
+// live_linkfn in pure V, without complicating live code generation
+// too much.
+//
+// The callbacks: cb_compile_fail, cb_before, cb_after will be
+// executed outside the mutex protected section, so be careful,
+// if you modify your data inside them. They can race with your
+// [live] functions.
+//
+// cb_locked_before and cb_locked_after will be executed *inside*
+// the mutex protected section. They can NOT race with your [live]
+// functions. They should be very quick in what they do though,
+// otherwise your live functions can be delayed.
+//
+// live.info - give user access to program's LiveReloadInfo struct,
+// so that the user can set callbacks, read meta information, etc.
+pub fn info() &LiveReloadInfo {
+	if C.g_live_info != 0 {
+		return C.g_live_info
+	}
+	// When the current program is not compiled with -live, simply
+	// return a new empty struct LiveReloadInfo in order to prevent
+	// crashes. In this case, the background reloader thread is not
+	// started, and the structure LiveReloadInfo will not get updated.
+	// All its fields will be 0, but still safe to access.
+	mut x := &LiveReloadInfo{}
+	C.g_live_info = voidptr(x)
+	return x
+}

vlib/live/executable/reloader.v

@@ -0,0 +1,164 @@
+module executable
+
+import os
+import time
+import dl
+import strconv
+import live
+
+// The live reloader code is implemented here.
+
+fn C.pthread_mutex_unlock(mtx voidptr)
+fn C.pthread_mutex_lock(mtx voidptr)
+
+// NB: new_live_reload_info will be called by generated C code inside main()
+pub fn new_live_reload_info(original string, vexe string, vopts string, live_fn_mutex voidptr, live_linkfn live.FNLinkLiveSymbols) &live.LiveReloadInfo {
+	file_base := os.file_name(original).replace('.v', '')
+	so_dir := os.cache_dir()
+	so_extension := dl.DL_EXT
+	/* $if msvc { so_extension = '.dll' } $else { so_extension = '.so' } */
+	return &live.LiveReloadInfo{
+		original: original
+		vexe: vexe
+		vopts: vopts
+		live_fn_mutex: live_fn_mutex
+		live_linkfn: live_linkfn
+		so_extension: so_extension
+		so_name_template: '${so_dir}/tmp.%d.${file_base}'
+		live_lib: 0
+		reloads: 0
+		reload_time_ms: 0
+	}
+}
+
+// NB: start_reloader will be called by generated code inside main(), to start
+// the hot code reloader thread. start_reloader is executed in the context of
+// the original main thread.
+pub fn start_reloader(r mut live.LiveReloadInfo) {
+	// The shared library should be loaded once in the main thread
+	// If that fails, the program would crash anyway, just provide
+	// an error message to the user and exit:
+    r.reloads++
+	_ := compile_and_reload_shared_lib(r) or {
+		eprintln( err )
+		exit(1)
+	}
+	go reloader(r)
+}
+
+[if debuglive]
+fn elog(r mut live.LiveReloadInfo, s string){
+	eprintln(s)
+}
+
+fn compile_and_reload_shared_lib(r mut live.LiveReloadInfo) ?bool {
+	sw := time.new_stopwatch()
+	new_lib_path := compile_lib(r) or {
+		return error('errors while compiling $r.original')
+	}
+	elog(r,'> compile_and_reload_shared_lib compiled: ${new_lib_path}')
+	load_lib(r, new_lib_path )
+	r.reload_time_ms = sw.elapsed().milliseconds()
+	return true
+}
+
+fn compile_lib(r mut live.LiveReloadInfo) ?string {
+	new_lib_path, new_lib_path_with_extension := current_shared_library_path(r)
+	cmd := '$r.vexe $r.vopts -o $new_lib_path $r.original'
+	elog(r,'>       compilation cmd: $cmd')
+	cwatch := time.new_stopwatch()
+	recompilation_result := os.exec( cmd ) or {
+		eprintln('recompilation failed')
+		return none
+	}
+	elog(r,'compilation took: ${cwatch.elapsed().milliseconds()}ms')
+	if recompilation_result.exit_code != 0 {
+		eprintln('recompilation error:')
+		eprintln( recompilation_result.output )
+		return none
+	}
+	if !os.exists( new_lib_path_with_extension ) {
+		eprintln('new_lib_path: $new_lib_path_with_extension does not exist')
+		return none
+	}
+	return new_lib_path_with_extension
+}
+
+fn current_shared_library_path(r mut live.LiveReloadInfo) (string, string) {
+	lib_path := strconv.v_sprintf(r.so_name_template.replace('\\', '\\\\'), r.reloads)
+	lib_path_with_extension := lib_path + r.so_extension
+	return lib_path, lib_path_with_extension
+}
+
+fn load_lib(r mut live.LiveReloadInfo, new_lib_path string) {
+	elog(r,'live mutex locking...')
+	C.pthread_mutex_lock(r.live_fn_mutex)
+	elog(r,'live mutex locked')
+	//
+	if r.cb_locked_before != 0 {
+		r.cb_locked_before( r )
+	}
+	//
+	protected_load_lib(r, new_lib_path)
+	//
+	r.reloads_ok++
+	if r.cb_locked_after != 0 {
+		r.cb_locked_after( r )
+	}
+	//
+	elog(r,'live mutex unlocking...')
+	C.pthread_mutex_unlock(r.live_fn_mutex)
+	elog(r,'live mutex unlocked')
+}
+
+fn protected_load_lib(r mut live.LiveReloadInfo, new_lib_path string) {
+	if r.live_lib != 0 {
+		dl.close( r.live_lib )
+		r.live_lib = 0
+	}
+	r.live_lib = dl.open(new_lib_path, dl.RTLD_LAZY)
+	if r.live_lib == 0 {
+		eprintln('opening $new_lib_path failed')
+		exit(1)
+	}
+	r.live_linkfn( r.live_lib )
+	elog(r,'> load_lib OK, new live_lib: $r.live_lib')
+	// removing the .so file from the filesystem after dlopen-ing 
+    // it is safe, since it will still be mapped in memory
+	os.rm( new_lib_path )
+}
+
+// NB: r.reloader() is executed in a new, independent thread
+fn reloader(r mut live.LiveReloadInfo) {
+	elog(r,'reloader, r: $r')
+	mut last_ts := os.file_last_mod_unix( r.original )
+	for {
+		if r.cb_recheck != 0 {
+			r.cb_recheck( r )
+		}
+		now_ts := os.file_last_mod_unix( r.original )
+		if last_ts != now_ts {
+			r.reloads++
+			last_ts = now_ts
+			r.last_mod_ts = last_ts
+			if r.cb_before != 0 {
+				r.cb_before( r )
+			}
+			compile_and_reload_shared_lib(r) or {
+				if r.cb_compile_failed != 0 {
+					r.cb_compile_failed( r )
+				}
+				if r.cb_after != 0 {
+					r.cb_after( r )
+				}
+				continue
+			}
+			if r.cb_after != 0 {
+				r.cb_after( r )
+			}
+		}
+		if r.recheck_period_ms > 0 {
+			time.sleep_ms(r.recheck_period_ms)
+		}
+	}
+}

vlib/live/shared/live_sharedlib.v

@@ -0,0 +1,3 @@
+module shared
+
+import live

vlib/v/builder/compile.v

@@ -151,6 +151,9 @@ pub fn (v Builder) get_user_files() []string {
 	// See cmd/tools/preludes/README.md for more info about what preludes are
 	vroot := os.dir(pref.vexe_path())
 	preludes_path := os.join_path(vroot, 'cmd', 'tools', 'preludes')
+	if v.pref.is_livemain || v.pref.is_liveshared {
+		user_files << os.join_path(preludes_path, 'live.v')
+    }
 	if v.pref.is_livemain {
 		user_files << os.join_path(preludes_path, 'live_main.v')
 	}

vlib/v/gen/cheaders.v

@@ -171,6 +171,8 @@ extern wchar_t **_wenviron;
 #include <pthread.h>
 #endif
 
+// g_live_info is used by live.info()
+void* g_live_info = NULL;
 
 //============================== HELPER C MACROS =============================*/
 //#define tos4(s, slen) ((string){.str=(s), .len=(slen)})

vlib/v/gen/live.v

@@ -6,69 +6,38 @@ import v.pref
 import v.util
 
 fn (g &Gen) generate_hotcode_reloading_declarations() {
-	if g.pref.os != .windows {
+	if g.pref.os == .windows {
-		if g.pref.is_livemain {
-			g.hotcode_definitions.writeln('pthread_mutex_t live_fn_mutex = PTHREAD_MUTEX_INITIALIZER;')
-		}
-		if g.pref.is_liveshared {
-			g.hotcode_definitions.writeln('pthread_mutex_t live_fn_mutex;')
-		}
-	} else {
 		if g.pref.is_livemain {
 			g.hotcode_definitions.writeln('HANDLE live_fn_mutex = 0;')
 		}
 		if g.pref.is_liveshared {
 			g.hotcode_definitions.writeln('HANDLE live_fn_mutex;')
-			g.hotcode_definitions.writeln('
+		}
+		g.hotcode_definitions.writeln('
 void pthread_mutex_lock(HANDLE *m) {
 	WaitForSingleObject(*m, INFINITE);
 }
-
 void pthread_mutex_unlock(HANDLE *m) {
 	ReleaseMutex(*m);
-}')
+}
+')
+	} else {
+		if g.pref.is_livemain {
+			g.hotcode_definitions.writeln('pthread_mutex_t live_fn_mutex = PTHREAD_MUTEX_INITIALIZER;')
+		}
+		if g.pref.is_liveshared {
+			g.hotcode_definitions.writeln('pthread_mutex_t live_fn_mutex;')
 		}
 	}
 }
 
 fn (g &Gen) generate_hotcode_reloader_code() {
 	if g.pref.is_liveshared {
-		g.hotcode_definitions.writeln('')
-		g.hotcode_definitions.writeln('int load_so(byteptr path) { return 0; }')
 		g.hotcode_definitions.writeln('')
 		return
 	}
 	// Hot code reloading
 	if g.pref.is_livemain {
-		mut file := os.real_path(g.pref.path)
-		file_base := os.file_name(file).replace('.v', '')
-		// Need to build .so file before building the live application
-		// The live app needs to load this .so file on initialization.
-		mut vexe := pref.vexe_path()
-		mut so_dir := os.cache_dir()
-		if os.user_os() == 'windows' {
-			vexe = util.cescaped_path(vexe)
-			file = util.cescaped_path(file)
-			so_dir = util.cescaped_path(so_dir)
-		}
-		mut msvc := ''
-		if g.pref.ccompiler == 'msvc' {
-			msvc = '-cc msvc'
-		}
-		so_debug_flag := if g.pref.is_debug { '-cg' } else { '' }
-		cmd_compile_shared_library := '$vexe $msvc -cg -keepc $so_debug_flag -o ${so_dir}/${file_base} -sharedlive -shared $file'
-		if g.pref.is_verbose {
-			println(cmd_compile_shared_library)
-		}
-		ticks := time.ticks()
-		so_compilation_result := os.system(cmd_compile_shared_library)
-		if g.pref.is_verbose {
-			diff := time.ticks() - ticks
-			println('compiling shared library took $diff ms')
-		}
-		if so_compilation_result != 0 {
-			exit(1)
-		}		
 		mut phd := ''
 		mut load_code := []string{}
 		if g.pref.os != .windows {
@@ -83,181 +52,55 @@ fn (g &Gen) generate_hotcode_reloader_code() {
 			phd = windows_hotcode_definitions_1
 		}
 		g.hotcode_definitions.writeln(phd.replace('@LOAD_FNS@', load_code.join('\n')))
-		g.hotcode_definitions.writeln('
-
-void lfnmutex_print(char *s){
-#if 0
-	fflush(stderr);
-	fprintf(stderr,">> live_fn_mutex: %p | %s\\n", &live_fn_mutex, s);
-	fflush(stderr);
-#endif
-}
-
-void remove_so_file(char *s){
-	// removing the .so file from the filesystem after dlopen-ing it is safe, since it will still be mapped in memory.
-	#ifndef _WIN32
-		unlink(s); 
-	#else
-		_unlink(s);
-	#endif
-}
-
-int _live_reloads = 0;
-void reload_so() {
-	char new_so_base[PATH_MAX] = {0};
-	char new_so_name[PATH_MAX] = {0};
-	char compile_cmd[PATH_MAX] = {0};
-	int last = os__file_last_mod_unix(tos2("$file"));
-	while (1) {
-		// TODO use inotify
-		int now = os__file_last_mod_unix(tos2("$file"));
-		if (now != last) {
-			last = now;
-			_live_reloads++;
-
-			//v -o bounce -sharedlive -shared bounce.v
-			snprintf(new_so_base, sizeof (new_so_base), "${so_dir}/tmp.%d.${file_base}", _live_reloads);
-			#ifdef _MSC_VER
-			snprintf(new_so_name, sizeof (new_so_name), "%s.dll", new_so_base);
-			#else
-			snprintf(new_so_name, sizeof (new_so_name), "%s.so", new_so_base);
-			#endif
-			snprintf(compile_cmd, sizeof (compile_cmd), "$vexe $msvc $so_debug_flag -cg -keepc -o %s -sharedlive -shared $file", new_so_base);
-			os__system(tos2(compile_cmd));
-
-			if( !os__exists(tos2(new_so_name)) ) {
-				puts("Errors while compiling $file\\n");
-				continue;
-			}
-
-			lfnmutex_print("reload_so locking...");
-			pthread_mutex_lock(&live_fn_mutex);
-			lfnmutex_print("reload_so locked");
-
-			load_so(new_so_name);
-			remove_so_file( new_so_name );
-
-			lfnmutex_print("reload_so unlocking...");
-			pthread_mutex_unlock(&live_fn_mutex);
-			lfnmutex_print("reload_so unlocked");
-		}
-		time__sleep_ms(100);
 	}
 }
-')
-	}
-}
-
 
 const (
 	posix_hotcode_definitions_1 = '
-#include <limits.h>
+void v_bind_live_symbols(void* live_lib){
-#ifndef PATH_MAX
-#define PATH_MAX 1024
-#endif
-void* live_lib = 0;
-int load_so(byteptr path) {
-	char cpath[PATH_MAX] = {0};
-	int res = snprintf(cpath, sizeof(cpath), "%s", path);
-	if (res >= sizeof (cpath)) {
-		fprintf (stderr, "path is too long");
-		return 0;
-	}
-	if (live_lib) {
-		int closing_status = dlclose(live_lib);
-		//fprintf(stderr, "Closing status: %d\\n", closing_status);
-		live_lib = 0;
-	}
-	//fprintf (stderr, "Opening shared library at: %s\\n", cpath);
-	live_lib = dlopen(cpath, RTLD_LAZY);
-	if (!live_lib) {
-		fprintf(stderr, "open failed, reason: %s\\n", dlerror());
-		exit(1);
-		return 0;
-	}
-	dlerror(); // clear errors
-	//fprintf(stderr, "live_lib: %p\\n", live_lib);
-
     @LOAD_FNS@
-
-	char *dlsym_error = dlerror(); 
-	if (dlsym_error != NULL) {
-		fprintf(stderr, "dlsym failed, reason: %s\\n", dlsym_error);
-	}
-    return 1;
 }
 '
 	windows_hotcode_definitions_1 = '
-#ifndef PATH_MAX
+void v_bind_live_symbols(void* live_lib){
-#define PATH_MAX 1024
-#endif
-void pthread_mutex_lock(HANDLE *m) {
-	WaitForSingleObject(*m, INFINITE);
-}
-void pthread_mutex_unlock(HANDLE *m) {
-	ReleaseMutex(*m);
-}
-void* live_lib = NULL;
-int load_so(byteptr path) {
-	char cpath[PATH_MAX];
-	int res = snprintf(cpath, sizeof(cpath), "%s", path);
-	if (res >= sizeof(cpath)) {
-		puts("path is too long\\n");
-		exit(1);
-		return 0;
-	}
-	if (live_lib) FreeLibrary(live_lib);
-	live_lib = LoadLibraryA(cpath);
-	if (!live_lib) {
-		puts("open failed\\n");
-		exit(1);
-		return 0;
-	}
     @LOAD_FNS@
-    return 1;
 }
 '
 )
 
-//
-
 fn (g &Gen) generate_hotcode_reloading_main_caller() {
 	if !g.pref.is_livemain {
 		return
 	}
 	g.writeln('')
 	// We are in live code reload mode, so start the .so loader in the background
-	file_base := os.file_name(g.pref.path).replace('.v', '')
-	mut so_dir := os.cache_dir()
-	if os.user_os() == 'windows' {
-		so_dir = util.cescaped_path(so_dir)
-	}
-
 	g.writeln('\t// live code initialization section:')
 	g.writeln('\t{')
 	g.writeln('\t\t// initialization of live function pointers')
 	for fname in g.hotcode_fn_names {
 		g.writeln('\t\timpl_live_${fname} = 0;')
 	}
-	
+	vexe := util.cescaped_path( pref.vexe_path() )
+	file := util.cescaped_path( g.pref.path )
+	msvc := if g.pref.ccompiler == 'msvc' { '-cc msvc' } else { '' }
+	so_debug_flag := if g.pref.is_debug { '-cg' } else { '' }
+	vopts := '$msvc $so_debug_flag -keepc -sharedlive -shared'
+	//
 	g.writeln('\t\t// start background reloading thread')
-	if g.pref.os != .windows {
+	if g.pref.os == .windows {
-		// unix:
-		so_name := file_base + '.so'
-		g.writeln('\t\tchar *live_library_name = "${so_dir}/$so_name";')
-		g.writeln('\t\tload_so(live_library_name);')
-		g.writeln('\t\tpthread_t _thread_so;')
-		g.writeln('\t\tpthread_create(&_thread_so , NULL, (void *)&reload_so, live_library_name);')
-	} else {
-		// windows:
-		so_extension := if g.pref.ccompiler == 'msvc' { '.dll' } else { '.so' }
-		so_name := file_base + so_extension
-		g.writeln('\t\tchar *live_library_name = "${so_dir}/$so_name";')
 		g.writeln('\t\tlive_fn_mutex = CreateMutexA(0, 0, 0);')
-		g.writeln('\t\tload_so(live_library_name);')
-		g.writeln('\t\tunsigned long _thread_so;')
-		g.writeln('\t\t_thread_so = CreateThread(0, 0, (LPTHREAD_START_ROUTINE)&reload_so, 0, 0, 0);')
 	}
+	g.writeln('\t\tlive__LiveReloadInfo* live_info = live__executable__new_live_reload_info(')
+	g.writeln('\t\t\t\t\t tos2("$file"),')
+	g.writeln('\t\t\t\t\t tos2("$vexe"),')
+	g.writeln('\t\t\t\t\t tos2("$vopts"),')
+	g.writeln('\t\t\t\t\t &live_fn_mutex,')
+	g.writeln('\t\t\t\t\t v_bind_live_symbols')
+	g.writeln('\t\t);')
+	// g_live_info gives access to the LiveReloadInfo methods,
+	// to the custom user code, through calling v_live_info()
+	g.writeln('\t\t   g_live_info = (void*)live_info;')
+	g.writeln('\t\tlive__executable__start_reloader(live_info);')
 	g.writeln('\t}\t// end of live code initialization section')
 	g.writeln('')
 }