1
0
mirror of https://github.com/vlang/v.git synced 2023-08-10 21:13:21 +03:00

parser,cgen: handle const x = $embed_file("file.txt").to_string()

This commit is contained in:
Delyan Angelov 2022-01-25 10:58:23 +02:00
parent 32f2f0dfa1
commit 5f38ba896e
No known key found for this signature in database
GPG Key ID: 66886C0F12D595ED
10 changed files with 102 additions and 51 deletions

View File

@ -199,3 +199,4 @@ to create a copy of the compiler rather than replacing it with `v self`.
| `trace_parser` | Prints details about parsed statements and expressions | | `trace_parser` | Prints details about parsed statements and expressions |
| `trace_thirdparty_obj_files` | Prints details about built thirdparty obj files | | `trace_thirdparty_obj_files` | Prints details about built thirdparty obj files |
| `trace_usecache` | Prints details when -usecache is used | | `trace_usecache` | Prints details when -usecache is used |
| `trace_embed_file` | Prints details when $embed_file is used |

10
vlib/v/ast/embed_file.v Normal file
View File

@ -0,0 +1,10 @@
// Copyright (c) 2019-2022 Alexander Medvednikov. All rights reserved.
// Use of this source code is governed by an MIT license
// that can be found in the LICENSE file.
module ast
import hash.fnv1a
pub fn (e EmbeddedFile) hash() u64 {
return fnv1a.sum64_string('$e.apath, $e.compression_type, $e.is_compressed, $e.len')
}

View File

@ -1,5 +1,13 @@
const const_file = $embed_file('v.png') const const_file = $embed_file('v.png')
const src = $embed_file('embed_file_test.v').to_string()
fn test_const_embed_file_to_string() {
assert src.len > 0
assert src.split_into_lines()[0].starts_with('const const_file')
assert src.split_into_lines().last() == '}'
}
fn test_const_embed_file() { fn test_const_embed_file() {
mut file := const_file mut file := const_file
eprintln('file: $file') eprintln('file: $file')

View File

@ -762,8 +762,11 @@ pub fn (mut g Gen) finish() {
if g.pref.is_livemain || g.pref.is_liveshared { if g.pref.is_livemain || g.pref.is_liveshared {
g.generate_hotcode_reloader_code() g.generate_hotcode_reloader_code()
} }
if g.embed_file_is_prod_mode() && g.embedded_files.len > 0 { if g.embedded_files.len > 0 {
g.gen_embedded_data() if g.embed_file_is_prod_mode() {
g.gen_embedded_data()
}
g.gen_embedded_metadata()
} }
if g.pref.is_test { if g.pref.is_test {
g.gen_c_main_for_tests() g.gen_c_main_for_tests()

View File

@ -14,6 +14,9 @@ fn (mut g Gen) embed_file_is_prod_mode() bool {
// gen_embed_file_struct generates C code for `$embed_file('...')` calls. // gen_embed_file_struct generates C code for `$embed_file('...')` calls.
fn (mut g Gen) gen_embed_file_init(mut node ast.ComptimeCall) { fn (mut g Gen) gen_embed_file_init(mut node ast.ComptimeCall) {
$if trace_embed_file ? {
eprintln('> gen_embed_file_init $node.embed_file.apath')
}
if g.embed_file_is_prod_mode() { if g.embed_file_is_prod_mode() {
file_bytes := os.read_bytes(node.embed_file.apath) or { file_bytes := os.read_bytes(node.embed_file.apath) or {
panic('unable to read file: "$node.embed_file.rpath') panic('unable to read file: "$node.embed_file.rpath')
@ -31,7 +34,11 @@ fn (mut g Gen) gen_embed_file_init(mut node ast.ComptimeCall) {
cache_path := os.join_path(cache_dir, cache_key) cache_path := os.join_path(cache_dir, cache_key)
vexe := pref.vexe_path() vexe := pref.vexe_path()
result := os.execute('${os.quoted_path(vexe)} compress $node.embed_file.compression_type ${os.quoted_path(node.embed_file.apath)} ${os.quoted_path(cache_path)}') compress_cmd := '${os.quoted_path(vexe)} compress $node.embed_file.compression_type ${os.quoted_path(node.embed_file.apath)} ${os.quoted_path(cache_path)}'
$if trace_embed_file ? {
eprintln('> gen_embed_file_init, compress_cmd: $compress_cmd')
}
result := os.execute(compress_cmd)
if result.exit_code != 0 { if result.exit_code != 0 {
eprintln('unable to compress file "$node.embed_file.rpath": $result.output') eprintln('unable to compress file "$node.embed_file.rpath": $result.output')
node.embed_file.bytes = file_bytes node.embed_file.bytes = file_bytes
@ -57,41 +64,62 @@ fn (mut g Gen) gen_embed_file_init(mut node ast.ComptimeCall) {
} }
node.embed_file.len = file_bytes.len node.embed_file.len = file_bytes.len
} }
ef_idx := node.embed_file.hash()
g.writeln('(v__embed_file__EmbedFileData){') g.write('_v_embed_file_metadata($ef_idx)')
g.writeln('\t\t.path = ${ctoslit(node.embed_file.rpath)},')
if g.embed_file_is_prod_mode() {
// apath is not needed in production and may leak information
g.writeln('\t\t.apath = ${ctoslit('')},')
} else {
g.writeln('\t\t.apath = ${ctoslit(node.embed_file.apath)},')
}
if g.embed_file_is_prod_mode() {
// use function generated in Gen.gen_embedded_data()
if node.embed_file.is_compressed {
g.writeln('\t\t.compression_type = ${ctoslit(node.embed_file.compression_type)},')
g.writeln('\t\t.compressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, ${ctoslit(node.embed_file.rpath)}, ${ctoslit(node.embed_file.compression_type)})->data,')
g.writeln('\t\t.uncompressed = NULL,')
} else {
g.writeln('\t\t.uncompressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, ${ctoslit(node.embed_file.rpath)}, ${ctoslit(node.embed_file.compression_type)})->data,')
}
} else {
g.writeln('\t\t.uncompressed = NULL,')
}
g.writeln('\t\t.free_compressed = 0,')
g.writeln('\t\t.free_uncompressed = 0,')
if g.embed_file_is_prod_mode() {
g.writeln('\t\t.len = $node.embed_file.len')
} else {
file_size := os.file_size(node.embed_file.apath)
if file_size > 5242880 {
eprintln('Warning: embedding of files >= ~5MB is currently not supported')
}
g.writeln('\t\t.len = $file_size')
}
g.writeln('} // \$embed_file("$node.embed_file.apath")')
g.file.embedded_files << node.embed_file g.file.embedded_files << node.embed_file
$if trace_embed_file ? {
eprintln('> gen_embed_file_init => _v_embed_file_metadata(${ef_idx:-25}) | ${node.embed_file.apath:-50} | compression: $node.embed_file.compression_type | len: $node.embed_file.len')
}
}
// gen_embedded_metadata embeds all of the deduplicated metadata in g.embedded_files, into the V target executable,
// into a single generated function _v_embed_file_metadata, that accepts a hash of the absolute path of the embedded
// files.
fn (mut g Gen) gen_embedded_metadata() {
g.embedded_data.writeln('v__embed_file__EmbedFileData _v_embed_file_metadata(u64 ef_hash) {')
g.embedded_data.writeln('\tv__embed_file__EmbedFileData res;')
g.embedded_data.writeln('\tmemset(&res, 0, sizeof(res));')
g.embedded_data.writeln('\tswitch(ef_hash) {')
for emfile in g.embedded_files {
ef_idx := emfile.hash()
g.embedded_data.writeln('\t\tcase $ef_idx: {')
g.embedded_data.writeln('\t\t\tres.path = ${ctoslit(emfile.rpath)};')
if g.embed_file_is_prod_mode() {
// apath is not needed in production and may leak information
g.embedded_data.writeln('\t\t\tres.apath = ${ctoslit('')};')
} else {
g.embedded_data.writeln('\t\t\tres.apath = ${ctoslit(emfile.apath)};')
}
if g.embed_file_is_prod_mode() {
// use function generated in Gen.gen_embedded_data()
if emfile.is_compressed {
g.embedded_data.writeln('\t\t\tres.compression_type = ${ctoslit(emfile.compression_type)};')
g.embedded_data.writeln('\t\t\tres.compressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, ${ctoslit(emfile.rpath)}, ${ctoslit(emfile.compression_type)})->data;')
g.embedded_data.writeln('\t\t\tres.uncompressed = NULL;')
} else {
g.embedded_data.writeln('\t\t\tres.uncompressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, ${ctoslit(emfile.rpath)}, ${ctoslit(emfile.compression_type)})->data;')
}
} else {
g.embedded_data.writeln('\t\t\tres.uncompressed = NULL;')
}
g.embedded_data.writeln('\t\t\tres.free_compressed = 0;')
g.embedded_data.writeln('\t\t\tres.free_uncompressed = 0;')
if g.embed_file_is_prod_mode() {
g.embedded_data.writeln('\t\t\tres.len = $emfile.len;')
} else {
file_size := os.file_size(emfile.apath)
if file_size > 5242880 {
eprintln('Warning: embedding of files >= ~5MB is currently not supported')
}
g.embedded_data.writeln('\t\t\tres.len = $file_size;')
}
g.embedded_data.writeln('\t\t\tbreak;')
g.embedded_data.writeln('\t\t} // case $ef_idx')
}
g.embedded_data.writeln('\t\tdefault: _v_panic(_SLIT("unknown embed file"));')
g.embedded_data.writeln('\t} // switch')
g.embedded_data.writeln('\treturn res;')
g.embedded_data.writeln('}')
} }
// gen_embedded_data embeds data into the V target executable. // gen_embedded_data embeds data into the V target executable.

View File

@ -13,7 +13,7 @@ string v__embed_file__EmbedFileData_str(v__embed_file__EmbedFileData ed) {
string v__embed_file__EmbedFileData_to_string(v__embed_file__EmbedFileData* original) { string v__embed_file__EmbedFileData_to_string(v__embed_file__EmbedFileData* original) {
v__embed_file__EmbedFileIndexEntry* v__embed_file__find_index_entry_by_path(voidptr start, string path, string algo) { v__embed_file__EmbedFileIndexEntry* v__embed_file__find_index_entry_by_path(voidptr start, string path, string algo) {
v__embed_file__EmbedFileData my_source = (v__embed_file__EmbedFileData){ v__embed_file__EmbedFileData my_source = _v_embed_file_metadata(
.path = _SLIT("embed.vv"), res.path = _SLIT("embed.vv");
// Initializations for module v.embed_file : // Initializations for module v.embed_file :

View File

@ -24,9 +24,9 @@ string v__embed_file__EmbedFileData_str(v__embed_file__EmbedFileData ed) {
string v__embed_file__EmbedFileData_to_string(v__embed_file__EmbedFileData* original) { string v__embed_file__EmbedFileData_to_string(v__embed_file__EmbedFileData* original) {
v__embed_file__EmbedFileIndexEntry* v__embed_file__find_index_entry_by_path(voidptr start, string path, string algo) { v__embed_file__EmbedFileIndexEntry* v__embed_file__find_index_entry_by_path(voidptr start, string path, string algo) {
v__embed_file__EmbedFileData my_source = (v__embed_file__EmbedFileData){ v__embed_file__EmbedFileData my_source = _v_embed_file_metadata(
.path = _SLIT("embed.vv"), res.path = _SLIT("embed.vv");
.apath = _SLIT(""), res.apath = _SLIT("");
.uncompressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, _SLIT("embed.vv"), _SLIT("none"))->data, res.uncompressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, _SLIT("embed.vv"), _SLIT("none"))->data;
// Initializations for module v.embed_file : // Initializations for module v.embed_file :

View File

@ -6,6 +6,6 @@ VV_LOCAL_SYMBOL void v__preludes__embed_file__zlib__init(void);
VV_LOCAL_SYMBOL Option_Array_byte v__preludes__embed_file__zlib__ZLibDecoder_decompress(v__preludes__embed_file__zlib__ZLibDecoder _d1, Array_byte data) { VV_LOCAL_SYMBOL Option_Array_byte v__preludes__embed_file__zlib__ZLibDecoder_decompress(v__preludes__embed_file__zlib__ZLibDecoder _d1, Array_byte data) {
= compress__zlib__decompress(data); = compress__zlib__decompress(data);
.compressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, _SLIT("embed.vv"), _SLIT("zlib"))->data, res.compressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, _SLIT("embed.vv"), _SLIT("zlib"))->data;
.compression_type = _SLIT("zlib") res.compression_type = _SLIT("zlib");
.uncompressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, _SLIT("embed.vv"), _SLIT("none"))->data, res.uncompressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, _SLIT("embed.vv"), _SLIT("none"))->data;

View File

@ -24,9 +24,9 @@ string v__embed_file__EmbedFileData_str(v__embed_file__EmbedFileData ed) {
string v__embed_file__EmbedFileData_to_string(v__embed_file__EmbedFileData* original) { string v__embed_file__EmbedFileData_to_string(v__embed_file__EmbedFileData* original) {
v__embed_file__EmbedFileIndexEntry* v__embed_file__find_index_entry_by_path(voidptr start, string path, string algo) { v__embed_file__EmbedFileIndexEntry* v__embed_file__find_index_entry_by_path(voidptr start, string path, string algo) {
v__embed_file__EmbedFileData my_source = (v__embed_file__EmbedFileData){ v__embed_file__EmbedFileData my_source = _v_embed_file_metadata(
.path = _SLIT("embed.vv"), res.path = _SLIT("embed.vv");
.apath = _SLIT(""), res.apath = _SLIT("");
.compressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, _SLIT("embed.vv"), _SLIT("zlib"))->data, res.compressed = v__embed_file__find_index_entry_by_path((voidptr)_v_embed_file_index, _SLIT("embed.vv"), _SLIT("zlib"))->data;
// Initializations for module v.embed_file : // Initializations for module v.embed_file :

View File

@ -78,7 +78,8 @@ pub fn (mut p Parser) check_expr(precedence int) ?ast.Expr {
.dollar { .dollar {
match p.peek_tok.kind { match p.peek_tok.kind {
.name { .name {
return p.comptime_call() node = p.comptime_call()
p.is_stmt_ident = is_stmt_ident
} }
.key_if { .key_if {
return p.if_expr(true) return p.if_expr(true)