From 031629faa17421174dad6341c69d17860c084ff1 Mon Sep 17 00:00:00 2001 From: Delyan Angelov Date: Thu, 2 Jun 2022 13:35:13 +0300 Subject: [PATCH] tools: add cmd/tools/measure/scanner_speed.v and cmd/tools/measure/parser_speed.v --- cmd/tools/measure/parser_speed.v | 67 +++++++++++++++++++++++++++++++ cmd/tools/measure/scanner_speed.v | 42 +++++++++++++++++++ vlib/v/parser/parser.v | 11 ++--- 3 files changed, 115 insertions(+), 5 deletions(-) create mode 100644 cmd/tools/measure/parser_speed.v create mode 100644 cmd/tools/measure/scanner_speed.v diff --git a/cmd/tools/measure/parser_speed.v b/cmd/tools/measure/parser_speed.v new file mode 100644 index 0000000000..7b549a7a93 --- /dev/null +++ b/cmd/tools/measure/parser_speed.v @@ -0,0 +1,67 @@ +import os +import time +import v.ast +import v.pref +import v.parser +import v.errors +import v.scanner + +fn main() { + files := os.args#[1..] + if files.len > 0 && files[0].starts_with('@') { + lst_path := files[0].all_after('@') + listed_files := os.read_file(lst_path)?.split('\n') + process_files(listed_files)? + return + } + process_files(files)? +} + +fn process_files(files []string) ? { + mut table := ast.new_table() + mut pref := pref.new_preferences() + pref.is_fmt = true + pref.skip_warnings = true + pref.output_mode = .silent + mut sw := time.new_stopwatch() + mut total_us := i64(0) + mut total_bytes := i64(0) + mut total_tokens := i64(0) + for f in files { + if f == '' { + continue + } + if f.ends_with('_test.v') { + continue + } + // do not measure the scanning, but only the parsing: + mut p := new_parser(f, .skip_comments, table, pref) + /// + sw.restart() + _ := p.parse() + f_us := sw.elapsed().microseconds() + /// + total_us += f_us + total_bytes += p.scanner.text.len + total_tokens += p.scanner.all_tokens.len + println('${f_us:10}us ${p.scanner.all_tokens.len:10} ${p.scanner.text.len:10} ${(f64(p.scanner.text.len) / p.scanner.all_tokens.len):7.3} ${p.errors.len:4} $f') + } + println('${total_us:10}us ${total_tokens:10} ${total_bytes:10} ${(f64(total_tokens) / total_bytes):7.3} | speed: ${(f64(total_bytes) / total_us):2.5f} MB/s') +} + +fn new_parser(path string, comments_mode scanner.CommentsMode, table &ast.Table, pref &pref.Preferences) &parser.Parser { + mut p := &parser.Parser{ + scanner: scanner.new_scanner_file(path, comments_mode, pref) or { panic(err) } + comments_mode: comments_mode + table: table + pref: pref + scope: &ast.Scope{ + start_pos: 0 + parent: table.global_scope + } + errors: []errors.Error{} + warnings: []errors.Warning{} + } + p.set_path(path) + return p +} diff --git a/cmd/tools/measure/scanner_speed.v b/cmd/tools/measure/scanner_speed.v new file mode 100644 index 0000000000..5697bd5ab2 --- /dev/null +++ b/cmd/tools/measure/scanner_speed.v @@ -0,0 +1,42 @@ +import os +import time +import v.scanner +import v.pref + +fn main() { + files := os.args#[1..] + if files.len > 0 && files[0].starts_with('@') { + lst_path := files[0].all_after('@') + listed_files := os.read_file(lst_path)?.split('\n') + process_files(listed_files)? + return + } + process_files(files)? +} + +fn process_files(files []string) ? { + mut pref := pref.new_preferences() + pref.is_fmt = true + pref.skip_warnings = true + pref.output_mode = .silent + mut sw := time.new_stopwatch() + mut total_us := i64(0) + mut total_bytes := i64(0) + mut total_tokens := i64(0) + for f in files { + if f == '' { + continue + } + if f.ends_with('_test.v') { + continue + } + sw.restart() + s := scanner.new_scanner_file(f, .skip_comments, pref)? + f_us := sw.elapsed().microseconds() + total_us += f_us + total_bytes += s.text.len + total_tokens += s.all_tokens.len + println('${f_us:10}us ${s.all_tokens.len:10} ${s.text.len:10} ${(f64(s.text.len) / s.all_tokens.len):7.3f} $f') + } + println('${total_us:10}us ${total_tokens:10} ${total_bytes:10} ${(f64(total_tokens) / total_bytes):7.3f} | speed: ${(f64(total_bytes) / total_us):2.5f} MB/s') +} diff --git a/vlib/v/parser/parser.v b/vlib/v/parser/parser.v index 2280f7a903..3b66a435f7 100644 --- a/vlib/v/parser/parser.v +++ b/vlib/v/parser/parser.v @@ -23,7 +23,6 @@ mut: file_name_dir string // "/home/user" unique_prefix string // a hash of p.file_name, used for making anon fn generation unique file_backend_mode ast.Language // .c for .c.v|.c.vv|.c.vsh files; .js for .js.v files, .amd64/.rv32/other arches for .amd64.v/.rv32.v/etc. files, .v otherwise. - scanner &scanner.Scanner comments_mode scanner.CommentsMode = .skip_comments // see comment in parse_file tok token.Token @@ -78,10 +77,6 @@ mut: returns bool is_stmt_ident bool // true while the beginning of a statement is an ident/selector expecting_type bool // `is Type`, expecting type - errors []errors.Error - warnings []errors.Warning - notices []errors.Notice - vet_errors []vet.Error cur_fn_name string label_names []string name_error bool // indicates if the token is not a name or the name is on another line @@ -95,6 +90,12 @@ mut: if_cond_comments []ast.Comment script_mode bool script_mode_start_token token.Token +pub mut: + scanner &scanner.Scanner + errors []errors.Error + warnings []errors.Warning + notices []errors.Notice + vet_errors []vet.Error } __global codegen_files = []&ast.File{}