From 845ffb59a69963d672755fa3fa52bd6bde903711 Mon Sep 17 00:00:00 2001 From: Delyan Angelov Date: Sun, 3 May 2020 18:59:11 +0300 Subject: [PATCH] live: use mostly pure V code for reloading, eases customization --- cmd/tools/preludes/live.v | 5 + cmd/tools/preludes/live_main.v | 12 +- cmd/tools/preludes/live_shared.v | 10 +- examples/hot_reload/bounce.v | 4 +- examples/hot_reload/message.v | 8 +- vlib/dl/dl_nix.c.v | 1 + vlib/dl/dl_windows.c.v | 1 + vlib/live/common.v | 60 ++++++++ vlib/live/executable/reloader.v | 164 ++++++++++++++++++++++ vlib/live/shared/live_sharedlib.v | 3 + vlib/v/builder/compile.v | 3 + vlib/v/gen/cheaders.v | 2 + vlib/v/gen/live.v | 219 +++++------------------------- 13 files changed, 282 insertions(+), 210 deletions(-) create mode 100644 cmd/tools/preludes/live.v create mode 100644 vlib/live/common.v create mode 100644 vlib/live/executable/reloader.v create mode 100644 vlib/live/shared/live_sharedlib.v diff --git a/cmd/tools/preludes/live.v b/cmd/tools/preludes/live.v new file mode 100644 index 0000000000..2b4cffec80 --- /dev/null +++ b/cmd/tools/preludes/live.v @@ -0,0 +1,5 @@ +module main + +// This prelude is loaded in every v program compiled with -live, +// in both the main executable, and in the shared library. +import live diff --git a/cmd/tools/preludes/live_main.v b/cmd/tools/preludes/live_main.v index b9a06cc9ff..62037ec288 100644 --- a/cmd/tools/preludes/live_main.v +++ b/cmd/tools/preludes/live_main.v @@ -1,11 +1,5 @@ module main -import os -import time -import dl - -const ( - os_used = os.MAX_PATH - time_used = time.now() - dl_used = dl.version -) +// This prelude is loaded in every v program compiled with -live, +// but only for the main executable. +import live.executable diff --git a/cmd/tools/preludes/live_shared.v b/cmd/tools/preludes/live_shared.v index 0463ce6b6a..23827f4316 100644 --- a/cmd/tools/preludes/live_shared.v +++ b/cmd/tools/preludes/live_shared.v @@ -1,9 +1,5 @@ module main -import os -import time - -const ( - os_used = os.MAX_PATH - time_used = time.now() -) +// This prelude is loaded in every v program compiled with -live, +// but only for the shared library. +import live.shared diff --git a/examples/hot_reload/bounce.v b/examples/hot_reload/bounce.v index 43b27f0712..eed53b5b62 100644 --- a/examples/hot_reload/bounce.v +++ b/examples/hot_reload/bounce.v @@ -79,8 +79,8 @@ const ( [live] fn (game &Game) draw() { game.gg.draw_rect(game.x, game.y, width, width, blue) - game.gg.draw_rect(550 - game.x + 10, 200 - game.y + 50, width, width, gx.rgb(128, 10, 255)) - game.gg.draw_rect(game.x - 20, 250 - game.y, width, width, gx.rgb(128, 240, 155)) + game.gg.draw_rect(550 - game.x + 10, 200 - game.y + 50, width, width, gx.rgb(228, 10, 55)) + game.gg.draw_rect(game.x - 25, 250 - game.y, width, width, gx.rgb(28, 240, 55)) } [live] diff --git a/examples/hot_reload/message.v b/examples/hot_reload/message.v index e933c726ea..0539d0f7b2 100644 --- a/examples/hot_reload/message.v +++ b/examples/hot_reload/message.v @@ -1,13 +1,13 @@ module main -// Build this example with -// v -live message.v - +// Build this example with `v -live message.v` import time +import live [live] fn print_message() { - println('Hello! Modify this message while the program is running.') + info := live.info() + println('OK reloads: ${info.reloads_ok:4d} | Total reloads: ${info.reloads:4d} | Hello! Modify this message while the program is running.') } fn main() { diff --git a/vlib/dl/dl_nix.c.v b/vlib/dl/dl_nix.c.v index 6b910005ab..af2dacbc2f 100644 --- a/vlib/dl/dl_nix.c.v +++ b/vlib/dl/dl_nix.c.v @@ -4,6 +4,7 @@ module dl pub const ( RTLD_NOW = C.RTLD_NOW + RTLD_LAZY = C.RTLD_LAZY DL_EXT = '.so' ) diff --git a/vlib/dl/dl_windows.c.v b/vlib/dl/dl_windows.c.v index ec971447c2..13af0a1cf7 100644 --- a/vlib/dl/dl_windows.c.v +++ b/vlib/dl/dl_windows.c.v @@ -2,6 +2,7 @@ module dl pub const ( RTLD_NOW = 0 + RTLD_LAZY = 0 DL_EXT = '.dll' ) diff --git a/vlib/live/common.v b/vlib/live/common.v new file mode 100644 index 0000000000..9705f80448 --- /dev/null +++ b/vlib/live/common.v @@ -0,0 +1,60 @@ +module live + +pub type FNLinkLiveSymbols = fn (linkcb voidptr) + +pub type FNLiveReloadCB = fn (info &LiveReloadInfo) + +pub struct LiveReloadInfo { +pub: + vexe string // full path to the v compiler + vopts string // v compiler options for a live shared library + original string // full path to the original source file, compiled with -live + live_fn_mutex voidptr // the address of the C mutex, that locks the [live] fns during reloads. + live_linkfn FNLinkLiveSymbols // generated C callback; receives a dlopen handle + so_extension string // .so or .dll + so_name_template string // a sprintf template for the shared libraries location +mut: + live_lib voidptr // the result of dl.open + reloads int // how many times a reloading was tried + reloads_ok int // how many times the reloads succeeded + reload_time_ms int // how much time the last reload took (compilation + loading) + last_mod_ts int // a timestamp for when the original was last changed + recheck_period_ms int = 100 // how often do you want to check for changes + cb_recheck FNLiveReloadCB = 0 // executed periodically + cb_compile_failed FNLiveReloadCB = 0 // executed when a reload compilation failed + cb_before FNLiveReloadCB = 0 // executed before a reload try happens + cb_after FNLiveReloadCB = 0 // executed after a reload try happened, even if failed + cb_locked_before FNLiveReloadCB = 0 // executed before lib reload, in the mutex section + cb_locked_after FNLiveReloadCB = 0 // executed after lib reload, in the mutex section +} + +// LiveReloadInfo.live_linkfn should be called by the reloader +// to dlsym all live functions. TODO: research a way to implement +// live_linkfn in pure V, without complicating live code generation +// too much. +// +// The callbacks: cb_compile_fail, cb_before, cb_after will be +// executed outside the mutex protected section, so be careful, +// if you modify your data inside them. They can race with your +// [live] functions. +// +// cb_locked_before and cb_locked_after will be executed *inside* +// the mutex protected section. They can NOT race with your [live] +// functions. They should be very quick in what they do though, +// otherwise your live functions can be delayed. +// +// live.info - give user access to program's LiveReloadInfo struct, +// so that the user can set callbacks, read meta information, etc. +pub fn info() &LiveReloadInfo { + if C.g_live_info != 0 { + return C.g_live_info + } + // When the current program is not compiled with -live, simply + // return a new empty struct LiveReloadInfo in order to prevent + // crashes. In this case, the background reloader thread is not + // started, and the structure LiveReloadInfo will not get updated. + // All its fields will be 0, but still safe to access. + mut x := &LiveReloadInfo{} + C.g_live_info = voidptr(x) + return x +} diff --git a/vlib/live/executable/reloader.v b/vlib/live/executable/reloader.v new file mode 100644 index 0000000000..312758e81a --- /dev/null +++ b/vlib/live/executable/reloader.v @@ -0,0 +1,164 @@ +module executable + +import os +import time +import dl +import strconv +import live + +// The live reloader code is implemented here. + +fn C.pthread_mutex_unlock(mtx voidptr) +fn C.pthread_mutex_lock(mtx voidptr) + +// NB: new_live_reload_info will be called by generated C code inside main() +pub fn new_live_reload_info(original string, vexe string, vopts string, live_fn_mutex voidptr, live_linkfn live.FNLinkLiveSymbols) &live.LiveReloadInfo { + file_base := os.file_name(original).replace('.v', '') + so_dir := os.cache_dir() + so_extension := dl.DL_EXT + /* $if msvc { so_extension = '.dll' } $else { so_extension = '.so' } */ + return &live.LiveReloadInfo{ + original: original + vexe: vexe + vopts: vopts + live_fn_mutex: live_fn_mutex + live_linkfn: live_linkfn + so_extension: so_extension + so_name_template: '${so_dir}/tmp.%d.${file_base}' + live_lib: 0 + reloads: 0 + reload_time_ms: 0 + } +} + +// NB: start_reloader will be called by generated code inside main(), to start +// the hot code reloader thread. start_reloader is executed in the context of +// the original main thread. +pub fn start_reloader(r mut live.LiveReloadInfo) { + // The shared library should be loaded once in the main thread + // If that fails, the program would crash anyway, just provide + // an error message to the user and exit: + r.reloads++ + _ := compile_and_reload_shared_lib(r) or { + eprintln( err ) + exit(1) + } + go reloader(r) +} + +[if debuglive] +fn elog(r mut live.LiveReloadInfo, s string){ + eprintln(s) +} + +fn compile_and_reload_shared_lib(r mut live.LiveReloadInfo) ?bool { + sw := time.new_stopwatch() + new_lib_path := compile_lib(r) or { + return error('errors while compiling $r.original') + } + elog(r,'> compile_and_reload_shared_lib compiled: ${new_lib_path}') + load_lib(r, new_lib_path ) + r.reload_time_ms = sw.elapsed().milliseconds() + return true +} + +fn compile_lib(r mut live.LiveReloadInfo) ?string { + new_lib_path, new_lib_path_with_extension := current_shared_library_path(r) + cmd := '$r.vexe $r.vopts -o $new_lib_path $r.original' + elog(r,'> compilation cmd: $cmd') + cwatch := time.new_stopwatch() + recompilation_result := os.exec( cmd ) or { + eprintln('recompilation failed') + return none + } + elog(r,'compilation took: ${cwatch.elapsed().milliseconds()}ms') + if recompilation_result.exit_code != 0 { + eprintln('recompilation error:') + eprintln( recompilation_result.output ) + return none + } + if !os.exists( new_lib_path_with_extension ) { + eprintln('new_lib_path: $new_lib_path_with_extension does not exist') + return none + } + return new_lib_path_with_extension +} + +fn current_shared_library_path(r mut live.LiveReloadInfo) (string, string) { + lib_path := strconv.v_sprintf(r.so_name_template.replace('\\', '\\\\'), r.reloads) + lib_path_with_extension := lib_path + r.so_extension + return lib_path, lib_path_with_extension +} + +fn load_lib(r mut live.LiveReloadInfo, new_lib_path string) { + elog(r,'live mutex locking...') + C.pthread_mutex_lock(r.live_fn_mutex) + elog(r,'live mutex locked') + // + if r.cb_locked_before != 0 { + r.cb_locked_before( r ) + } + // + protected_load_lib(r, new_lib_path) + // + r.reloads_ok++ + if r.cb_locked_after != 0 { + r.cb_locked_after( r ) + } + // + elog(r,'live mutex unlocking...') + C.pthread_mutex_unlock(r.live_fn_mutex) + elog(r,'live mutex unlocked') +} + +fn protected_load_lib(r mut live.LiveReloadInfo, new_lib_path string) { + if r.live_lib != 0 { + dl.close( r.live_lib ) + r.live_lib = 0 + } + r.live_lib = dl.open(new_lib_path, dl.RTLD_LAZY) + if r.live_lib == 0 { + eprintln('opening $new_lib_path failed') + exit(1) + } + r.live_linkfn( r.live_lib ) + elog(r,'> load_lib OK, new live_lib: $r.live_lib') + // removing the .so file from the filesystem after dlopen-ing + // it is safe, since it will still be mapped in memory + os.rm( new_lib_path ) +} + +// NB: r.reloader() is executed in a new, independent thread +fn reloader(r mut live.LiveReloadInfo) { + elog(r,'reloader, r: $r') + mut last_ts := os.file_last_mod_unix( r.original ) + for { + if r.cb_recheck != 0 { + r.cb_recheck( r ) + } + now_ts := os.file_last_mod_unix( r.original ) + if last_ts != now_ts { + r.reloads++ + last_ts = now_ts + r.last_mod_ts = last_ts + if r.cb_before != 0 { + r.cb_before( r ) + } + compile_and_reload_shared_lib(r) or { + if r.cb_compile_failed != 0 { + r.cb_compile_failed( r ) + } + if r.cb_after != 0 { + r.cb_after( r ) + } + continue + } + if r.cb_after != 0 { + r.cb_after( r ) + } + } + if r.recheck_period_ms > 0 { + time.sleep_ms(r.recheck_period_ms) + } + } +} diff --git a/vlib/live/shared/live_sharedlib.v b/vlib/live/shared/live_sharedlib.v new file mode 100644 index 0000000000..b79ee9cc9e --- /dev/null +++ b/vlib/live/shared/live_sharedlib.v @@ -0,0 +1,3 @@ +module shared + +import live diff --git a/vlib/v/builder/compile.v b/vlib/v/builder/compile.v index 47afc998eb..de366cb2dd 100644 --- a/vlib/v/builder/compile.v +++ b/vlib/v/builder/compile.v @@ -151,6 +151,9 @@ pub fn (v Builder) get_user_files() []string { // See cmd/tools/preludes/README.md for more info about what preludes are vroot := os.dir(pref.vexe_path()) preludes_path := os.join_path(vroot, 'cmd', 'tools', 'preludes') + if v.pref.is_livemain || v.pref.is_liveshared { + user_files << os.join_path(preludes_path, 'live.v') + } if v.pref.is_livemain { user_files << os.join_path(preludes_path, 'live_main.v') } diff --git a/vlib/v/gen/cheaders.v b/vlib/v/gen/cheaders.v index 74aa9fb244..9a7e92a3e3 100644 --- a/vlib/v/gen/cheaders.v +++ b/vlib/v/gen/cheaders.v @@ -171,6 +171,8 @@ extern wchar_t **_wenviron; #include #endif +// g_live_info is used by live.info() +void* g_live_info = NULL; //============================== HELPER C MACROS =============================*/ //#define tos4(s, slen) ((string){.str=(s), .len=(slen)}) diff --git a/vlib/v/gen/live.v b/vlib/v/gen/live.v index 67c65048cb..2f307f1c22 100644 --- a/vlib/v/gen/live.v +++ b/vlib/v/gen/live.v @@ -6,69 +6,38 @@ import v.pref import v.util fn (g &Gen) generate_hotcode_reloading_declarations() { - if g.pref.os != .windows { - if g.pref.is_livemain { - g.hotcode_definitions.writeln('pthread_mutex_t live_fn_mutex = PTHREAD_MUTEX_INITIALIZER;') - } - if g.pref.is_liveshared { - g.hotcode_definitions.writeln('pthread_mutex_t live_fn_mutex;') - } - } else { + if g.pref.os == .windows { if g.pref.is_livemain { g.hotcode_definitions.writeln('HANDLE live_fn_mutex = 0;') } if g.pref.is_liveshared { g.hotcode_definitions.writeln('HANDLE live_fn_mutex;') - g.hotcode_definitions.writeln(' + } + g.hotcode_definitions.writeln(' void pthread_mutex_lock(HANDLE *m) { WaitForSingleObject(*m, INFINITE); } - void pthread_mutex_unlock(HANDLE *m) { ReleaseMutex(*m); -}') +} +') + } else { + if g.pref.is_livemain { + g.hotcode_definitions.writeln('pthread_mutex_t live_fn_mutex = PTHREAD_MUTEX_INITIALIZER;') + } + if g.pref.is_liveshared { + g.hotcode_definitions.writeln('pthread_mutex_t live_fn_mutex;') } } } fn (g &Gen) generate_hotcode_reloader_code() { if g.pref.is_liveshared { - g.hotcode_definitions.writeln('') - g.hotcode_definitions.writeln('int load_so(byteptr path) { return 0; }') g.hotcode_definitions.writeln('') return } // Hot code reloading if g.pref.is_livemain { - mut file := os.real_path(g.pref.path) - file_base := os.file_name(file).replace('.v', '') - // Need to build .so file before building the live application - // The live app needs to load this .so file on initialization. - mut vexe := pref.vexe_path() - mut so_dir := os.cache_dir() - if os.user_os() == 'windows' { - vexe = util.cescaped_path(vexe) - file = util.cescaped_path(file) - so_dir = util.cescaped_path(so_dir) - } - mut msvc := '' - if g.pref.ccompiler == 'msvc' { - msvc = '-cc msvc' - } - so_debug_flag := if g.pref.is_debug { '-cg' } else { '' } - cmd_compile_shared_library := '$vexe $msvc -cg -keepc $so_debug_flag -o ${so_dir}/${file_base} -sharedlive -shared $file' - if g.pref.is_verbose { - println(cmd_compile_shared_library) - } - ticks := time.ticks() - so_compilation_result := os.system(cmd_compile_shared_library) - if g.pref.is_verbose { - diff := time.ticks() - ticks - println('compiling shared library took $diff ms') - } - if so_compilation_result != 0 { - exit(1) - } mut phd := '' mut load_code := []string{} if g.pref.os != .windows { @@ -83,181 +52,55 @@ fn (g &Gen) generate_hotcode_reloader_code() { phd = windows_hotcode_definitions_1 } g.hotcode_definitions.writeln(phd.replace('@LOAD_FNS@', load_code.join('\n'))) - g.hotcode_definitions.writeln(' - -void lfnmutex_print(char *s){ -#if 0 - fflush(stderr); - fprintf(stderr,">> live_fn_mutex: %p | %s\\n", &live_fn_mutex, s); - fflush(stderr); -#endif -} - -void remove_so_file(char *s){ - // removing the .so file from the filesystem after dlopen-ing it is safe, since it will still be mapped in memory. - #ifndef _WIN32 - unlink(s); - #else - _unlink(s); - #endif -} - -int _live_reloads = 0; -void reload_so() { - char new_so_base[PATH_MAX] = {0}; - char new_so_name[PATH_MAX] = {0}; - char compile_cmd[PATH_MAX] = {0}; - int last = os__file_last_mod_unix(tos2("$file")); - while (1) { - // TODO use inotify - int now = os__file_last_mod_unix(tos2("$file")); - if (now != last) { - last = now; - _live_reloads++; - - //v -o bounce -sharedlive -shared bounce.v - snprintf(new_so_base, sizeof (new_so_base), "${so_dir}/tmp.%d.${file_base}", _live_reloads); - #ifdef _MSC_VER - snprintf(new_so_name, sizeof (new_so_name), "%s.dll", new_so_base); - #else - snprintf(new_so_name, sizeof (new_so_name), "%s.so", new_so_base); - #endif - snprintf(compile_cmd, sizeof (compile_cmd), "$vexe $msvc $so_debug_flag -cg -keepc -o %s -sharedlive -shared $file", new_so_base); - os__system(tos2(compile_cmd)); - - if( !os__exists(tos2(new_so_name)) ) { - puts("Errors while compiling $file\\n"); - continue; - } - - lfnmutex_print("reload_so locking..."); - pthread_mutex_lock(&live_fn_mutex); - lfnmutex_print("reload_so locked"); - - load_so(new_so_name); - remove_so_file( new_so_name ); - - lfnmutex_print("reload_so unlocking..."); - pthread_mutex_unlock(&live_fn_mutex); - lfnmutex_print("reload_so unlocked"); - } - time__sleep_ms(100); } } -') - } -} - const ( posix_hotcode_definitions_1 = ' -#include -#ifndef PATH_MAX -#define PATH_MAX 1024 -#endif -void* live_lib = 0; -int load_so(byteptr path) { - char cpath[PATH_MAX] = {0}; - int res = snprintf(cpath, sizeof(cpath), "%s", path); - if (res >= sizeof (cpath)) { - fprintf (stderr, "path is too long"); - return 0; - } - if (live_lib) { - int closing_status = dlclose(live_lib); - //fprintf(stderr, "Closing status: %d\\n", closing_status); - live_lib = 0; - } - //fprintf (stderr, "Opening shared library at: %s\\n", cpath); - live_lib = dlopen(cpath, RTLD_LAZY); - if (!live_lib) { - fprintf(stderr, "open failed, reason: %s\\n", dlerror()); - exit(1); - return 0; - } - dlerror(); // clear errors - //fprintf(stderr, "live_lib: %p\\n", live_lib); - +void v_bind_live_symbols(void* live_lib){ @LOAD_FNS@ - - char *dlsym_error = dlerror(); - if (dlsym_error != NULL) { - fprintf(stderr, "dlsym failed, reason: %s\\n", dlsym_error); - } - return 1; } ' windows_hotcode_definitions_1 = ' -#ifndef PATH_MAX -#define PATH_MAX 1024 -#endif -void pthread_mutex_lock(HANDLE *m) { - WaitForSingleObject(*m, INFINITE); -} -void pthread_mutex_unlock(HANDLE *m) { - ReleaseMutex(*m); -} -void* live_lib = NULL; -int load_so(byteptr path) { - char cpath[PATH_MAX]; - int res = snprintf(cpath, sizeof(cpath), "%s", path); - if (res >= sizeof(cpath)) { - puts("path is too long\\n"); - exit(1); - return 0; - } - if (live_lib) FreeLibrary(live_lib); - live_lib = LoadLibraryA(cpath); - if (!live_lib) { - puts("open failed\\n"); - exit(1); - return 0; - } +void v_bind_live_symbols(void* live_lib){ @LOAD_FNS@ - return 1; } ' ) -// - fn (g &Gen) generate_hotcode_reloading_main_caller() { if !g.pref.is_livemain { return } g.writeln('') // We are in live code reload mode, so start the .so loader in the background - file_base := os.file_name(g.pref.path).replace('.v', '') - mut so_dir := os.cache_dir() - if os.user_os() == 'windows' { - so_dir = util.cescaped_path(so_dir) - } - g.writeln('\t// live code initialization section:') g.writeln('\t{') g.writeln('\t\t// initialization of live function pointers') for fname in g.hotcode_fn_names { g.writeln('\t\timpl_live_${fname} = 0;') } - + vexe := util.cescaped_path( pref.vexe_path() ) + file := util.cescaped_path( g.pref.path ) + msvc := if g.pref.ccompiler == 'msvc' { '-cc msvc' } else { '' } + so_debug_flag := if g.pref.is_debug { '-cg' } else { '' } + vopts := '$msvc $so_debug_flag -keepc -sharedlive -shared' + // g.writeln('\t\t// start background reloading thread') - if g.pref.os != .windows { - // unix: - so_name := file_base + '.so' - g.writeln('\t\tchar *live_library_name = "${so_dir}/$so_name";') - g.writeln('\t\tload_so(live_library_name);') - g.writeln('\t\tpthread_t _thread_so;') - g.writeln('\t\tpthread_create(&_thread_so , NULL, (void *)&reload_so, live_library_name);') - } else { - // windows: - so_extension := if g.pref.ccompiler == 'msvc' { '.dll' } else { '.so' } - so_name := file_base + so_extension - g.writeln('\t\tchar *live_library_name = "${so_dir}/$so_name";') + if g.pref.os == .windows { g.writeln('\t\tlive_fn_mutex = CreateMutexA(0, 0, 0);') - g.writeln('\t\tload_so(live_library_name);') - g.writeln('\t\tunsigned long _thread_so;') - g.writeln('\t\t_thread_so = CreateThread(0, 0, (LPTHREAD_START_ROUTINE)&reload_so, 0, 0, 0);') } + g.writeln('\t\tlive__LiveReloadInfo* live_info = live__executable__new_live_reload_info(') + g.writeln('\t\t\t\t\t tos2("$file"),') + g.writeln('\t\t\t\t\t tos2("$vexe"),') + g.writeln('\t\t\t\t\t tos2("$vopts"),') + g.writeln('\t\t\t\t\t &live_fn_mutex,') + g.writeln('\t\t\t\t\t v_bind_live_symbols') + g.writeln('\t\t);') + // g_live_info gives access to the LiveReloadInfo methods, + // to the custom user code, through calling v_live_info() + g.writeln('\t\t g_live_info = (void*)live_info;') + g.writeln('\t\tlive__executable__start_reloader(live_info);') g.writeln('\t}\t// end of live code initialization section') g.writeln('') }