From 5f38ba896ef1c8744676aed4ddf86fef21435cc2 Mon Sep 17 00:00:00 2001 From: Delyan Angelov Date: Tue, 25 Jan 2022 10:58:23 +0200 Subject: [PATCH] parser,cgen: handle `const x = $embed_file("file.txt").to_string()` --- CONTRIBUTING.md | 1 + vlib/v/ast/embed_file.v | 10 ++ vlib/v/embed_file/tests/embed_file_test.v | 8 ++ vlib/v/gen/c/cgen.v | 7 +- vlib/v/gen/c/embed.v | 98 ++++++++++++------- vlib/v/gen/c/testdata/embed.c.must_have | 4 +- .../c/testdata/embed_with_prod.c.must_have | 8 +- ...with_prod_and_several_decoders.c.must_have | 6 +- .../testdata/embed_with_prod_zlib.c.must_have | 8 +- vlib/v/parser/expr.v | 3 +- 10 files changed, 102 insertions(+), 51 deletions(-) create mode 100644 vlib/v/ast/embed_file.v diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f06443a215..e268b16edf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -199,3 +199,4 @@ to create a copy of the compiler rather than replacing it with `v self`. | `trace_parser` | Prints details about parsed statements and expressions | | `trace_thirdparty_obj_files` | Prints details about built thirdparty obj files | | `trace_usecache` | Prints details when -usecache is used | +| `trace_embed_file` | Prints details when $embed_file is used | diff --git a/vlib/v/ast/embed_file.v b/vlib/v/ast/embed_file.v new file mode 100644 index 0000000000..6096659220 --- /dev/null +++ b/vlib/v/ast/embed_file.v @@ -0,0 +1,10 @@ +// Copyright (c) 2019-2022 Alexander Medvednikov. All rights reserved. +// Use of this source code is governed by an MIT license +// that can be found in the LICENSE file. +module ast + +import hash.fnv1a + +pub fn (e EmbeddedFile) hash() u64 { + return fnv1a.sum64_string('$e.apath, $e.compression_type, $e.is_compressed, $e.len') +} diff --git a/vlib/v/embed_file/tests/embed_file_test.v b/vlib/v/embed_file/tests/embed_file_test.v index f3cc8223af..4608512c54 100644 --- a/vlib/v/embed_file/tests/embed_file_test.v +++ b/vlib/v/embed_file/tests/embed_file_test.v @@ -1,5 +1,13 @@ const const_file = $embed_file('v.png') +const src = $embed_file('embed_file_test.v').to_string() + +fn test_const_embed_file_to_string() { + assert src.len > 0 + assert src.split_into_lines()[0].starts_with('const const_file') + assert src.split_into_lines().last() == '}' +} + fn test_const_embed_file() { mut file := const_file eprintln('file: $file') diff --git a/vlib/v/gen/c/cgen.v b/vlib/v/gen/c/cgen.v index 981f3cbda4..078214fc94 100644 --- a/vlib/v/gen/c/cgen.v +++ b/vlib/v/gen/c/cgen.v @@ -762,8 +762,11 @@ pub fn (mut g Gen) finish() { if g.pref.is_livemain || g.pref.is_liveshared { g.generate_hotcode_reloader_code() } - if g.embed_file_is_prod_mode() && g.embedded_files.len > 0 { - g.gen_embedded_data() + if g.embedded_files.len > 0 { + if g.embed_file_is_prod_mode() { + g.gen_embedded_data() + } + g.gen_embedded_metadata() } if g.pref.is_test { g.gen_c_main_for_tests() diff --git a/vlib/v/gen/c/embed.v b/vlib/v/gen/c/embed.v index 0bbb42cb9f..4ca70b9eae 100644 --- a/vlib/v/gen/c/embed.v +++ b/vlib/v/gen/c/embed.v @@ -14,6 +14,9 @@ fn (mut g Gen) embed_file_is_prod_mode() bool { // gen_embed_file_struct generates C code for `$embed_file('...')` calls. fn (mut g Gen) gen_embed_file_init(mut node ast.ComptimeCall) { + $if trace_embed_file ? { + eprintln('> gen_embed_file_init $node.embed_file.apath') + } if g.embed_file_is_prod_mode() { file_bytes := os.read_bytes(node.embed_file.apath) or { panic('unable to read file: "$node.embed_file.rpath') @@ -31,7 +34,11 @@ fn (mut g Gen) gen_embed_file_init(mut node ast.ComptimeCall) { cache_path := os.join_path(cache_dir, cache_key) vexe := pref.vexe_path() - result := os.execute('${os.quoted_path(vexe)} compress $node.embed_file.compression_type ${os.quoted_path(node.embed_file.apath)} ${os.quoted_path(cache_path)}') + compress_cmd := '${os.quoted_path(vexe)} compress $node.embed_file.compression_type ${os.quoted_path(node.embed_file.apath)} ${os.quoted_path(cache_path)}' + $if trace_embed_file ? { + eprintln('> gen_embed_file_init, compress_cmd: $compress_cmd') + } + result := os.execute(compress_cmd) if result.exit_code != 0 { eprintln('unable to compress file "$node.embed_file.rpath": $result.output') node.embed_file.bytes = file_bytes @@ -57,41 +64,62 @@ fn (mut g Gen) gen_embed_file_init(mut node ast.ComptimeCall) { } node.embed_file.len = file_bytes.len } - - g.writeln('(v__embed_file__EmbedFileData){') - g.writeln('\t\t.path = ${ctoslit(node.embed_file.rpath)},') - if g.embed_file_is_prod_mode() { - // apath is not needed in production and may leak information - g.writeln('\t\t.apath = ${ctoslit('')},') - } else { - g.writeln('\t\t.apath = ${ctoslit(node.embed_file.apath)},') - } - if g.embed_file_is_prod_mode() { - // use function generated in Gen.gen_embedded_data() - if node.embed_file.is_compressed { - g.writeln('\t\t.compression_type = ${ctoslit(node.embed_file.compression_type)},') - g.writeln('\t\t.compressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, ${ctoslit(node.embed_file.rpath)}, ${ctoslit(node.embed_file.compression_type)})->data,') - g.writeln('\t\t.uncompressed = NULL,') - } else { - g.writeln('\t\t.uncompressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, ${ctoslit(node.embed_file.rpath)}, ${ctoslit(node.embed_file.compression_type)})->data,') - } - } else { - g.writeln('\t\t.uncompressed = NULL,') - } - g.writeln('\t\t.free_compressed = 0,') - g.writeln('\t\t.free_uncompressed = 0,') - if g.embed_file_is_prod_mode() { - g.writeln('\t\t.len = $node.embed_file.len') - } else { - file_size := os.file_size(node.embed_file.apath) - if file_size > 5242880 { - eprintln('Warning: embedding of files >= ~5MB is currently not supported') - } - g.writeln('\t\t.len = $file_size') - } - g.writeln('} // \$embed_file("$node.embed_file.apath")') - + ef_idx := node.embed_file.hash() + g.write('_v_embed_file_metadata($ef_idx)') g.file.embedded_files << node.embed_file + $if trace_embed_file ? { + eprintln('> gen_embed_file_init => _v_embed_file_metadata(${ef_idx:-25}) | ${node.embed_file.apath:-50} | compression: $node.embed_file.compression_type | len: $node.embed_file.len') + } +} + +// gen_embedded_metadata embeds all of the deduplicated metadata in g.embedded_files, into the V target executable, +// into a single generated function _v_embed_file_metadata, that accepts a hash of the absolute path of the embedded +// files. +fn (mut g Gen) gen_embedded_metadata() { + g.embedded_data.writeln('v__embed_file__EmbedFileData _v_embed_file_metadata(u64 ef_hash) {') + g.embedded_data.writeln('\tv__embed_file__EmbedFileData res;') + g.embedded_data.writeln('\tmemset(&res, 0, sizeof(res));') + g.embedded_data.writeln('\tswitch(ef_hash) {') + for emfile in g.embedded_files { + ef_idx := emfile.hash() + g.embedded_data.writeln('\t\tcase $ef_idx: {') + g.embedded_data.writeln('\t\t\tres.path = ${ctoslit(emfile.rpath)};') + if g.embed_file_is_prod_mode() { + // apath is not needed in production and may leak information + g.embedded_data.writeln('\t\t\tres.apath = ${ctoslit('')};') + } else { + g.embedded_data.writeln('\t\t\tres.apath = ${ctoslit(emfile.apath)};') + } + if g.embed_file_is_prod_mode() { + // use function generated in Gen.gen_embedded_data() + if emfile.is_compressed { + g.embedded_data.writeln('\t\t\tres.compression_type = ${ctoslit(emfile.compression_type)};') + g.embedded_data.writeln('\t\t\tres.compressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, ${ctoslit(emfile.rpath)}, ${ctoslit(emfile.compression_type)})->data;') + g.embedded_data.writeln('\t\t\tres.uncompressed = NULL;') + } else { + g.embedded_data.writeln('\t\t\tres.uncompressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, ${ctoslit(emfile.rpath)}, ${ctoslit(emfile.compression_type)})->data;') + } + } else { + g.embedded_data.writeln('\t\t\tres.uncompressed = NULL;') + } + g.embedded_data.writeln('\t\t\tres.free_compressed = 0;') + g.embedded_data.writeln('\t\t\tres.free_uncompressed = 0;') + if g.embed_file_is_prod_mode() { + g.embedded_data.writeln('\t\t\tres.len = $emfile.len;') + } else { + file_size := os.file_size(emfile.apath) + if file_size > 5242880 { + eprintln('Warning: embedding of files >= ~5MB is currently not supported') + } + g.embedded_data.writeln('\t\t\tres.len = $file_size;') + } + g.embedded_data.writeln('\t\t\tbreak;') + g.embedded_data.writeln('\t\t} // case $ef_idx') + } + g.embedded_data.writeln('\t\tdefault: _v_panic(_SLIT("unknown embed file"));') + g.embedded_data.writeln('\t} // switch') + g.embedded_data.writeln('\treturn res;') + g.embedded_data.writeln('}') } // gen_embedded_data embeds data into the V target executable. diff --git a/vlib/v/gen/c/testdata/embed.c.must_have b/vlib/v/gen/c/testdata/embed.c.must_have index fd6037a518..f659b3e29c 100644 --- a/vlib/v/gen/c/testdata/embed.c.must_have +++ b/vlib/v/gen/c/testdata/embed.c.must_have @@ -13,7 +13,7 @@ string v__embed_file__EmbedFileData_str(v__embed_file__EmbedFileData ed) { string v__embed_file__EmbedFileData_to_string(v__embed_file__EmbedFileData* original) { v__embed_file__EmbedFileIndexEntry* v__embed_file__find_index_entry_by_path(voidptr start, string path, string algo) { -v__embed_file__EmbedFileData my_source = (v__embed_file__EmbedFileData){ -.path = _SLIT("embed.vv"), +v__embed_file__EmbedFileData my_source = _v_embed_file_metadata( +res.path = _SLIT("embed.vv"); // Initializations for module v.embed_file : diff --git a/vlib/v/gen/c/testdata/embed_with_prod.c.must_have b/vlib/v/gen/c/testdata/embed_with_prod.c.must_have index c245ab9049..95d84eb22a 100644 --- a/vlib/v/gen/c/testdata/embed_with_prod.c.must_have +++ b/vlib/v/gen/c/testdata/embed_with_prod.c.must_have @@ -24,9 +24,9 @@ string v__embed_file__EmbedFileData_str(v__embed_file__EmbedFileData ed) { string v__embed_file__EmbedFileData_to_string(v__embed_file__EmbedFileData* original) { v__embed_file__EmbedFileIndexEntry* v__embed_file__find_index_entry_by_path(voidptr start, string path, string algo) { -v__embed_file__EmbedFileData my_source = (v__embed_file__EmbedFileData){ -.path = _SLIT("embed.vv"), -.apath = _SLIT(""), -.uncompressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, _SLIT("embed.vv"), _SLIT("none"))->data, +v__embed_file__EmbedFileData my_source = _v_embed_file_metadata( +res.path = _SLIT("embed.vv"); +res.apath = _SLIT(""); +res.uncompressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, _SLIT("embed.vv"), _SLIT("none"))->data; // Initializations for module v.embed_file : diff --git a/vlib/v/gen/c/testdata/embed_with_prod_and_several_decoders.c.must_have b/vlib/v/gen/c/testdata/embed_with_prod_and_several_decoders.c.must_have index 5d8c9aebf0..7826463c12 100644 --- a/vlib/v/gen/c/testdata/embed_with_prod_and_several_decoders.c.must_have +++ b/vlib/v/gen/c/testdata/embed_with_prod_and_several_decoders.c.must_have @@ -6,6 +6,6 @@ VV_LOCAL_SYMBOL void v__preludes__embed_file__zlib__init(void); VV_LOCAL_SYMBOL Option_Array_byte v__preludes__embed_file__zlib__ZLibDecoder_decompress(v__preludes__embed_file__zlib__ZLibDecoder _d1, Array_byte data) { = compress__zlib__decompress(data); -.compressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, _SLIT("embed.vv"), _SLIT("zlib"))->data, -.compression_type = _SLIT("zlib") -.uncompressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, _SLIT("embed.vv"), _SLIT("none"))->data, +res.compressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, _SLIT("embed.vv"), _SLIT("zlib"))->data; +res.compression_type = _SLIT("zlib"); +res.uncompressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, _SLIT("embed.vv"), _SLIT("none"))->data; diff --git a/vlib/v/gen/c/testdata/embed_with_prod_zlib.c.must_have b/vlib/v/gen/c/testdata/embed_with_prod_zlib.c.must_have index f08654a425..dff10fadd0 100644 --- a/vlib/v/gen/c/testdata/embed_with_prod_zlib.c.must_have +++ b/vlib/v/gen/c/testdata/embed_with_prod_zlib.c.must_have @@ -24,9 +24,9 @@ string v__embed_file__EmbedFileData_str(v__embed_file__EmbedFileData ed) { string v__embed_file__EmbedFileData_to_string(v__embed_file__EmbedFileData* original) { v__embed_file__EmbedFileIndexEntry* v__embed_file__find_index_entry_by_path(voidptr start, string path, string algo) { -v__embed_file__EmbedFileData my_source = (v__embed_file__EmbedFileData){ -.path = _SLIT("embed.vv"), -.apath = _SLIT(""), -.compressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, _SLIT("embed.vv"), _SLIT("zlib"))->data, +v__embed_file__EmbedFileData my_source = _v_embed_file_metadata( +res.path = _SLIT("embed.vv"); +res.apath = _SLIT(""); +res.compressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, _SLIT("embed.vv"), _SLIT("zlib"))->data; // Initializations for module v.embed_file : diff --git a/vlib/v/parser/expr.v b/vlib/v/parser/expr.v index 7d319e0d7e..990720126e 100644 --- a/vlib/v/parser/expr.v +++ b/vlib/v/parser/expr.v @@ -78,7 +78,8 @@ pub fn (mut p Parser) check_expr(precedence int) ?ast.Expr { .dollar { match p.peek_tok.kind { .name { - return p.comptime_call() + node = p.comptime_call() + p.is_stmt_ident = is_stmt_ident } .key_if { return p.if_expr(true)