From 8fbfceed3084fa578d12d109057f3997ba709d0f Mon Sep 17 00:00:00 2001 From: joe-conigliaro Date: Fri, 22 Nov 2019 16:22:11 +1100 Subject: [PATCH] compiler: generics - support across modules/files --- examples/vweb/test_vweb_app.v | 8 +- vlib/compiler/expression.v | 3 - vlib/compiler/fn.v | 262 ++++++++++++----------------- vlib/compiler/main.v | 29 +++- vlib/compiler/modules.v | 2 +- vlib/compiler/parser.v | 17 +- vlib/compiler/tests/generic_test.v | 2 +- vlib/vweb/vweb.v | 28 +-- 8 files changed, 156 insertions(+), 195 deletions(-) diff --git a/examples/vweb/test_vweb_app.v b/examples/vweb/test_vweb_app.v index eedcbcc9b0..6df5b9cbec 100644 --- a/examples/vweb/test_vweb_app.v +++ b/examples/vweb/test_vweb_app.v @@ -13,8 +13,8 @@ pub mut: } fn main() { - mut app := App{} - vweb.run(app, port) + app := App{} + vweb.run(mut app, port) //vweb.run(Port) } @@ -22,7 +22,7 @@ pub fn (app mut App) init() { app.vweb.handle_static('.') } -pub fn (app mut App) json_endpoint() { +pub fn (app & App) json_endpoint() { app.vweb.json('{"a": 3}') } @@ -34,7 +34,7 @@ pub fn (app mut App) index() { } */ -pub fn (app mut App) text() { +pub fn (app & App) text() { app.vweb.text('hello world') } diff --git a/vlib/compiler/expression.v b/vlib/compiler/expression.v index bf345ebecf..db4b29a88f 100644 --- a/vlib/compiler/expression.v +++ b/vlib/compiler/expression.v @@ -240,9 +240,6 @@ fn (p mut Parser) name_expr() string { } p.gen('(') mut typ := name - if typ in p.cur_fn.dispatch_of.inst.keys() { - typ = p.cur_fn.dispatch_of.inst[typ] - } p.cast(typ) p.gen(')') for p.tok == .dot { diff --git a/vlib/compiler/fn.v b/vlib/compiler/fn.v index bdb8f7c75c..74a6b65280 100644 --- a/vlib/compiler/fn.v +++ b/vlib/compiler/fn.v @@ -39,8 +39,8 @@ mut: defer_text []string type_pars []string type_inst []TypeInst - dispatch_of TypeInst // current type inst of this generic instance - body_idx int // idx of the first body statement + // dispatch_of TypeInst // current type inst of this generic instance + generic_tmpl []Token fn_name_token_idx int // used by error reporting comptime_define string is_used bool // so that we can skip unused fns in resulting C code @@ -292,7 +292,7 @@ fn (p mut Parser) fn_decl() { f.is_c = true } else if !p.pref.translated { - if contains_capital(f.name) && !p.fileis('view.v') { + if contains_capital(f.name) && !p.fileis('view.v') && !p.is_vgen { println('`$f.name`') p.error('function names cannot contain uppercase letters, use snake_case instead') } @@ -402,27 +402,19 @@ fn (p mut Parser) fn_decl() { // Generic functions are inserted as needed from the call site if f.is_generic { if p.first_pass() { - f.body_idx = p.cur_tok_index()+1 + p.save_generic_tmpl(mut f, p.cur_tok_index()) if f.is_method { rcv := p.table.find_type(receiver_typ) if p.first_pass() && rcv.name == '' { - r := Type { - name: rcv.name.replace('*', '') - mod: p.mod - is_placeholder: true - } - p.table.register_type2(r) + p.error('cannot currently add generic method to a type declared after it or in another module') } - // println('added generic method $rcv.name $f.name') + // println('added generic method r:$rcv.name f:$f.name') p.add_method(rcv.name, f) } else { p.table.register_fn(f) } } - if f.is_method { p.mark_var_changed(f.args[0]) } - p.check_unused_variables() p.set_current_fn( EmptyFn ) - p.returns = false p.skip_fn_body() return } else { @@ -1239,65 +1231,32 @@ fn (p mut Parser) extract_type_inst(f &Fn, args_ []string) TypeInst { } for tp in f.type_pars { if r.inst[tp] == '' { - p.error_with_token_index('unused type parameter `$tp`', f.body_idx-2) + // p.error_with_token_index('unused type parameter `$tp`', f.body_idx-2) + p.error('unused type parameter `$tp`') } } return r } -// Replace type params of a given generic function using a TypeInst -fn (p mut Parser) replace_type_params(f &Fn, ti TypeInst) []string { - mut sig := []string - for a in f.args { - sig << a.typ - } - sig << f.typ - mut r := []string - for _, a in sig { - mut fi := a - mut fr := '' - if fi.starts_with('fn (') { - fr += 'fn (' - mut fn_args := fi[4..].all_before(') ').split(',') - fn_args << fi.all_after(') ') - for i, fa_ in fn_args { - mut fna := fa_.trim_space() - for fna.starts_with('array_') { - fna = fna[6..] - fr += 'array_' - } - if fna in ti.inst.keys() { - fr += ti.inst[fna] - } else { - fr += fna - } - if i <= fn_args.len-3 { - fr += ',' - } else if i == fn_args.len-2 { - fr += ') ' - } +// Replace function type and type of params for a given generic function using a TypeInst +fn (p mut Parser) replace_type_params(f mut Fn, ti TypeInst) { + mut args2 := []Var + mut args := f.args + for i, _ in args { + mut arg := args[i] + for k, v in ti.inst { + for arg.typ.contains(k) { + arg.typ = arg.typ.replace(k, v) } - r << fr - continue } - for fi.starts_with('array_') { - fi = fi[6..] - fr += 'array_' - } - if fi.starts_with('varg_') { - fi = fi[5..] - fr += 'varg_' - } - if fi in ti.inst.keys() { - mut t := ti.inst[fi] - fr += t - // println("replaced $a => $fr") - } else { - fr += fi - } - r << fr + args2 << arg } - return r + for k, v in ti.inst { + for f.typ.contains(k) { + f.typ = f.typ.replace(k, v) + } + } + f.args = args2 } fn (p mut Parser) register_vargs_stuct(typ string, len int) string { @@ -1393,6 +1352,35 @@ fn (p mut Parser) register_multi_return_stuct(types []string) string { return typ } +fn (p mut Parser) save_generic_tmpl(f mut Fn, pos int) { + mut cbr_depth := 1 + mut tokens := []Token + for i in pos..p.tokens.len-1 { + tok := p.tokens[i] + if tok.tok == .lcbr { cbr_depth++ } + if tok.tok == .rcbr { + cbr_depth-- + if cbr_depth == 0 { break } + } + tokens << tok + } + f.generic_tmpl = tokens +} + +fn (f &Fn) generic_tmpl_to_inst(ti TypeInst) string { + mut fn_body := '' + for tok in f.generic_tmpl { + mut toks := tok.str() + if toks in ti.inst { + for k,v in ti.inst { + toks = toks.replace(k, v) + } + } + fn_body += ' $toks' + } + return fn_body +} + fn (p mut Parser) rename_generic_fn_instance(f mut Fn, ti TypeInst) { if f.is_method { f.name = f.receiver_typ + '_' + f.name @@ -1411,113 +1399,44 @@ fn (p mut Parser) dispatch_generic_fn_instance(f mut Fn, ti TypeInst) { break } } - if !new_inst { p.rename_generic_fn_instance(mut f, ti) _f := p.table.find_fn(f.name) or { - p.error('function instance `$f.name` not found') + // p.error('function instance `$f.name` not found') return } - f.args = _f.args - f.typ = _f.typ - f.is_generic = false - f.type_inst = [] - if false {} - f.dispatch_of = ti - // println('using existing inst $f.name(${f.str_args(p.table)}) $f.typ') + // println('using existing inst ${p.fn_signature_v(f)}') return } - f.type_inst << ti p.table.register_fn(f) - // Remember current scanner position, go back here for each type instance - // TODO remove this once tokens are cached in `new_parser()` - saved_tok_idx := p.cur_tok_index() - saved_fn := p.cur_fn - saved_var_idx := p.var_idx - saved_local_vars := p.local_vars - p.clear_vars() - saved_line := p.cgen.cur_line - saved_lines := p.cgen.lines - saved_is_tmp := p.cgen.is_tmp - saved_tmp_line := p.cgen.tmp_line - returns := p.returns // should be always false - + p.rename_generic_fn_instance(mut f, ti) - f.is_generic = false // the instance is a normal function - f.type_inst = [] - if false {} - f.scope_level = 0 - f.dispatch_of = ti - - // TODO this is done to prevent a crash as a result of this not being - // properly initialised. This is a bug somewhere futher upstream - f.defer_text = [] - if false {} - old_args := f.args - new_types := p.replace_type_params(f, ti) - f.args = [] - for i in 0..new_types.len-1 { - mut v := old_args[i] - v.typ = new_types[i] - f.args << v - } - f.typ = new_types.last() - if f.typ in f.type_pars { f.typ = '_ANYTYPE_' } - - if f.typ in ti.inst { - f.typ = ti.inst[f.typ] - } + p.replace_type_params(mut f, ti) + // TODO: Handle + // if f.typ in f.type_pars { f.typ = '_ANYTYPE_' } + // if f.typ in ti.inst { + // f.typ = ti.inst[f.typ] + // } + f.is_generic = false if f.is_method { p.add_method(f.args[0].name, f) } else { p.table.register_fn(f) } - // println("generating gen inst $f.name(${f.str_args(p.table)}) $f.typ : $ti.inst") - - p.cgen.is_tmp = false - p.returns = false - p.cgen.tmp_line = '' - p.cgen.cur_line = '' - p.cgen.lines = [] - unsafe { // TODO - p.cur_fn = *f + mut fn_code := '${p.fn_signature_v(f)} {\n${f.generic_tmpl_to_inst(ti)}\n}' + if f.mod in p.v.gen_parser_idx { + pidx := p.v.gen_parser_idx[f.mod] + p.v.parsers[pidx].add_text(fn_code) + for mod in p.table.imports { + if p.v.parsers[pidx].import_table.known_import(mod) { continue } + p.v.parsers[pidx].register_import(mod, 0) + } + } else { + // TODO: add here after I work out bug } - for arg in f.args { - p.register_var(arg) - } - p.token_idx = f.body_idx-1 - p.next() // re-initializes the parser properly - str_args := f.str_args(p.table) - - p.in_dispatch = true - p.genln('${p.get_linkage_prefix()}$f.typ $f.name($str_args) {') - // p.genln('/* generic fn instance $f.name : $ti.inst */') - p.statements() - p.in_dispatch = false - - if f.typ == '_ANYTYPE_' { - f.typ = p.cur_fn.typ - f.name = f.name.replace('_ANYTYPE_', type_to_safe_str(f.typ)) - p.cgen.lines[0] = p.cgen.lines[0].replace('_ANYTYPE_', f.typ) - p.table.register_fn(f) - } - for l in p.cgen.lines { - p.cgen.fns << l - } - - p.token_idx = saved_tok_idx-1 - p.next() - p.check(.rpar) // end of the arg list which caused this dispatch - p.cur_fn = saved_fn - p.var_idx = saved_var_idx - p.local_vars = saved_local_vars - p.cgen.lines = saved_lines - p.cgen.cur_line = saved_line - p.cgen.is_tmp = saved_is_tmp - p.cgen.tmp_line = saved_tmp_line - p.returns = false + p.cgen.fns << '${p.fn_signature(f)};' } // "fn (int, string) int" @@ -1576,6 +1495,19 @@ fn (f &Fn) str_args(table &Table) string { return s } +fn (f &Fn) str_args_v(table &Table) string { + mut str_args := '' + for i, arg in f.args { + if f.is_method && i == 0 { continue } + mut arg_typ := arg.typ.replace('array_', '[]').replace('map_', 'map[string]') + if arg.is_mut { arg_typ = 'mut '+arg_typ.trim('*') } + else if arg_typ.ends_with('*') || arg.ptr { arg_typ = '&'+arg_typ.trim_right('*') } + str_args += '$arg.name $arg_typ' + if i < f.args.len-1 { str_args += ','} + } + return str_args +} + // find local function variable with closest name to `name` fn (p &Parser) find_misspelled_local_var(name string, min_match f32) string { mut closest := f32(0) @@ -1603,6 +1535,26 @@ fn (fns []Fn) contains(f Fn) bool { } return false } +fn (p &Parser) fn_signature(f &Fn) string { + return '$f.typ $f.name(${f.str_args(p.table)})' +} + +fn (p &Parser) fn_signature_v(f &Fn) string { + mut method := '' + mut f_name := f.name.all_after('__') + if f.is_method { + receiver_arg := f.args[0] + receiver_type := receiver_arg.typ.trim('*') + f_name = f_name.all_after('${receiver_type}_') + mut rcv_typ := receiver_arg.typ.replace('array_', '[]').replace('map_', 'map[string]') + if receiver_arg.is_mut { rcv_typ = 'mut '+rcv_typ.trim('*') } + else if rcv_typ.ends_with('*') || receiver_arg.ptr { rcv_typ = '&'+rcv_typ.trim_right('&*') } + method = '($receiver_arg.name $rcv_typ) ' + } + vis := if f.is_public { 'pub ' } else { '' } + f_type := if f.typ == 'void' { '' } else { f.typ } + return '${vis}fn $method$f_name(${f.str_args_v(p.table)}) $f_type' +} pub fn (f &Fn) v_fn_module() string { return f.mod diff --git a/vlib/compiler/main.v b/vlib/compiler/main.v index c31f1b3f17..1d93bff8ce 100644 --- a/vlib/compiler/main.v +++ b/vlib/compiler/main.v @@ -71,6 +71,7 @@ pub mut: parsers []Parser // file parsers vgen_buf strings.Builder // temporary buffer for generated V code (.str() etc) file_parser_idx map[string]int // map absolute file path to v.parsers index + gen_parser_idx map[string]int cached_mods []string } @@ -289,19 +290,22 @@ pub fn (v mut V) compile() { // new vfmt is not ready yet //} } + + // add parser generated V code (str() methods etc) + mut vgen_parser := v.new_parser_from_string(v.vgen_buf.str()) + // free the string builder which held the generated methods + v.vgen_buf.free() + vgen_parser.is_vgen = true + v.add_parser(vgen_parser) + // run vgen / generic parsers + for i, _ in v.parsers { + if !v.parsers[i].is_vgen { continue } + v.parsers[i].parse(.main) + } // Generate .vh if we are building a module if v.pref.build_mode == .build_module { generate_vh(v.dir) } - - // parse generated V code (str() methods etc) - mut vgen_parser := v.new_parser_from_string(v.vgen_buf.str()) - // free the string builder which held the generated methods - vgen_parser.is_vgen = true - v.vgen_buf.free() - vgen_parser.parse(.main) - // v.parsers.add(vgen_parser) - // All definitions mut def := strings.new_builder(10000)// Avoid unnecessary allocations $if !js { @@ -633,6 +637,13 @@ pub fn (v mut V) add_v_files_to_compile() { // resolve deps and add imports in correct order imported_mods := v.resolve_deps().imports() for mod in imported_mods { + // TODO: work out bug and only add when needed in fn.v + if !mod in v.gen_parser_idx { + mut gp := v.new_parser_from_string('module '+mod.all_after('.')+'\n') + gp.is_vgen = true + gp.mod = mod + v.gen_parser_idx[mod] = v.add_parser(gp) + } if mod == 'builtin' || mod == 'main' { // builtin already added // main files will get added last diff --git a/vlib/compiler/modules.v b/vlib/compiler/modules.v index b25e9c6760..320cafc29d 100644 --- a/vlib/compiler/modules.v +++ b/vlib/compiler/modules.v @@ -49,7 +49,7 @@ fn (p mut Parser) register_import_alias(alias string, mod string, tok_idx int) { if alias in p.import_table.imports && p.import_table.imports[alias] != mod { p.error('cannot import $mod as $alias: import name $alias already in use"') } - if mod.contains('.internal.') { + if mod.contains('.internal.') && !p.is_vgen { mod_parts := mod.split('.') mut internal_mod_parts := []string for part in mod_parts { diff --git a/vlib/compiler/parser.v b/vlib/compiler/parser.v index a2bca31493..87916c8fc9 100644 --- a/vlib/compiler/parser.v +++ b/vlib/compiler/parser.v @@ -262,6 +262,14 @@ fn (p &Parser) log(s string) { */ } +pub fn (p mut Parser) add_text(text string) { + if p.tokens.len > 1 && p.tokens[p.tokens.len-1].tok == .eof { + p.tokens.delete(p.tokens.len-1) + } + p.scanner.text = p.scanner.text + '\n' + text + p.scan_tokens() +} + fn (p mut Parser) parse(pass Pass) { p.cgen.line = 0 p.cgen.file = cescaped_path(os.realpath(p.file_path)) @@ -883,14 +891,7 @@ fn (p mut Parser) get_type() string { nr_muls++ p.check(.amp) } - // Generic type check - ti := p.cur_fn.dispatch_of.inst - if p.lit in ti.keys() { - typ += ti[p.lit] - // println('cur dispatch: $p.lit => $typ') - } else { - typ += p.lit - } + typ += p.lit // C.Struct import if p.lit == 'C' && p.peek() == .dot { p.next() diff --git a/vlib/compiler/tests/generic_test.v b/vlib/compiler/tests/generic_test.v index d3179985e1..2b077d79a1 100644 --- a/vlib/compiler/tests/generic_test.v +++ b/vlib/compiler/tests/generic_test.v @@ -46,7 +46,7 @@ fn assert_eq(a, b T) { fn print_nice(x T, indent int) { mut space := '' - for i in 0..indent { + for _ in 0..indent { space = space + ' ' } println('$space$x') diff --git a/vlib/vweb/vweb.v b/vlib/vweb/vweb.v index 6ec8263b8f..c02fbfd882 100644 --- a/vlib/vweb/vweb.v +++ b/vlib/vweb/vweb.v @@ -44,23 +44,23 @@ mut: } pub fn (ctx Context) html(html string) { - ctx.conn.write('HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n$ctx.headers\r\n\r\n$html') + ctx.conn.write('HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n$ctx.headers\r\n\r\n$html') or { panic(err) } } pub fn (ctx Context) text(s string) { - ctx.conn.write('HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n$ctx.headers\r\n\r\n $s') + ctx.conn.write('HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n$ctx.headers\r\n\r\n $s') or { panic(err) } } pub fn (ctx Context) json(s string) { - ctx.conn.write('HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n$ctx.headers\r\n\r\n$s') + ctx.conn.write('HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n$ctx.headers\r\n\r\n$s') or { panic(err) } } pub fn (ctx Context) redirect(url string) { - ctx.conn.write('HTTP/1.1 302 Found\r\nLocation: $url\r\n\r\n$ctx.headers') + ctx.conn.write('HTTP/1.1 302 Found\r\nLocation: $url\r\n\r\n$ctx.headers') or { panic(err) } } pub fn (ctx Context) not_found(s string) { - ctx.conn.write(HTTP_404) + ctx.conn.write(HTTP_404) or { panic(err) } } pub fn (ctx mut Context) set_cookie(key, val string) { // TODO support directives, escape cookie value (https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie) @@ -89,7 +89,7 @@ fn (ctx mut Context) get_header(key string) string { } //pub fn run(port int) { -pub fn run(app T, port int) { +pub fn run(app mut T, port int) { println('Running vweb app on http://localhost:$port ...') l := net.listen(port) or { panic('failed to listen') } //mut app := T{} @@ -102,8 +102,8 @@ pub fn run(app T, port int) { // TODO move this to handle_conn(conn, app) s := conn.read_line() if s == '' { - conn.write(HTTP_500) - conn.close() + conn.write(HTTP_500) or {} + conn.close() or {} return } // Parse the first line @@ -112,8 +112,8 @@ pub fn run(app T, port int) { vals := first_line.split(' ') if vals.len < 2 { println('no vals for http') - conn.write(HTTP_500) - conn.close() + conn.write(HTTP_500) or {} + conn.close() or {} return } mut action := vals[1][1..].all_before('/') @@ -149,7 +149,7 @@ pub fn run(app T, port int) { $if debug { println('no vals for http') } - conn.close() + conn.close() or {} continue } @@ -161,9 +161,9 @@ pub fn run(app T, port int) { // Call the right action app.$action() or { - conn.write(HTTP_404) + conn.write(HTTP_404) or {} } - conn.close() + conn.close() or {} } } @@ -234,7 +234,7 @@ pub fn (ctx mut Context) handle_static(directory_path string) bool { if static_file != '' { data := os.read_file(static_file) or { return false } - ctx.conn.write('HTTP/1.1 200 OK\r\nContent-Type: $mime_type\r\n\r\n$data') + ctx.conn.write('HTTP/1.1 200 OK\r\nContent-Type: $mime_type\r\n\r\n$data') or { panic(err) } return true } return false