From 6af082e70ef5fd2a96cdb5b2fe285fa50c085e35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20D=C3=A4schle?= Date: Tue, 1 Dec 2020 03:58:39 +0100 Subject: [PATCH] doc: detailed documentation of the compiler pipeline (#7043) --- vlib/v/README.md | 109 +++++++++++++++++++++++++++++++++++++++ vlib/v/parser/comptime.v | 2 +- vlib/v/parser/parser.v | 26 ++++++++-- 3 files changed, 132 insertions(+), 5 deletions(-) create mode 100644 vlib/v/README.md diff --git a/vlib/v/README.md b/vlib/v/README.md new file mode 100644 index 0000000000..ccf277beb0 --- /dev/null +++ b/vlib/v/README.md @@ -0,0 +1,109 @@ +# Compiler pipeline +A simple high level explanation +how the compiler pipeline (`parser` -> `checker` -> `generator`) works. + +## Reading files +### Getting builtin files +To load all builtin files, +a preference `Preferences.lookup_path` for the path where to look for exists. +See `Builder.get_builtin_files` as example. +If the file is a `.vsh` file and the backend is C, `vlib/os` will also be loaded as builtin. + +### Getting project files +Either there is a specific file: `my_file.v` or a directory containing V files. +In the last case it scans that directory for all files. +See `Builder.v_files_from_dir` as the helper method. +This list of files needs to be filtered so that only `*.v` files exist. + +Skips the following file types: +- `*_test.v` +- either `*.c.v` or `*.c.js` depending on the backend +- all files that doesn't end with `.v` +- Files that are not defined in `Preferences.compile_defines` +or `Preferences.compile_defines_all` **if any file is defined**. + +## Parsing files +To parse something a new template is created as the first step: +```v +import v.table +table := table.new_table() +``` +a new preference is created: +```v +import v.pref +pref := pref.Preferences{} +``` +and a new scope is created: +```v +import v.ast +scope := ast.Scope{ + parent: 0 +} +``` +after that, you can parse your files. + +## Parse text +If you want to parse only text which isn't saved on the disk you can use this function. +```v oksyntax +import v.parser +code := '' +// table, pref and scope needs to be passed as reference +parsed_file := parser.parse_text(code, table, .parse_comments, &pref, &scope) +``` + +## Parse a single file +For parsing files on disk, a path needs to be provided. +The paths are collected one step earlier. +```v oksyntax +import v.parser +path := '' +// table, pref and scope needs to be passed as reference +parsed_file := parser.parse_file(path, table, .parse_comments, &pref, &scope) +``` + +## Parse a set of files +If you have a batch of paths available which should be parsed, +there is also a function which does all the work. +```v oksyntax +import v.parser +paths := [''] +// table, pref and scope needs to be passed as reference +parsed_files := parser.parse_files(paths, table, &pref, &scope) +``` + +## Parse imports +A file often contains imports. These imports might need to be parsed as well. +The builder contains a method which does this: `Builder.parse_imports`. + +If the module which is imported isn't parsed already, +you have to collect it relatively from the main file. +For this the `ast.File` contains a list of imports. +Those imports needs to be found on disk. +`.` is just replaced with seperators in the relative location of the main file. +Then all files from that directory are collected and parsed again like the previous steps explained. + +## Checking AST +A new checker is created: +```v oksyntax +import v.checker +mut checker := checker.new_checker(table, &pref) +``` + +After checking your files in `checker.errors` and `checker.warnings` you can see the results. + +### Check `ast.File` +```v oksyntax +checker.check(parsed_file) +``` + +### Check a list of `ast.File` +```v oksyntax +checker.check_files(parsed_files) +``` + +## Generate target from AST +Generating C code works just as this: +```v oksyntax +import v.gen +res := gen.cgen(parsed_files, table, &pref) +``` diff --git a/vlib/v/parser/comptime.v b/vlib/v/parser/comptime.v index 9e9be823c6..e247271b3d 100644 --- a/vlib/v/parser/comptime.v +++ b/vlib/v/parser/comptime.v @@ -113,7 +113,7 @@ fn (mut p Parser) vweb() ast.ComptimeCall { println('>>> end of vweb template END') println('\n\n') } - mut file := parse_text(v_code, p.table, p.pref, scope, p.global_scope) + mut file := parse_comptime(v_code, p.table, p.pref, scope, p.global_scope) file = { file | path: tmpl_path diff --git a/vlib/v/parser/parser.v b/vlib/v/parser/parser.v index 574dde5a3a..d24903c0c2 100644 --- a/vlib/v/parser/parser.v +++ b/vlib/v/parser/parser.v @@ -81,11 +81,11 @@ pub fn parse_stmt(text string, table &table.Table, scope &ast.Scope) ast.Stmt { return p.stmt(false) } -pub fn parse_text(text string, b_table &table.Table, pref &pref.Preferences, scope &ast.Scope, global_scope &ast.Scope) ast.File { +pub fn parse_comptime(text string, table &table.Table, pref &pref.Preferences, scope &ast.Scope, global_scope &ast.Scope) ast.File { s := scanner.new_scanner(text, .skip_comments, pref) mut p := Parser{ scanner: s - table: b_table + table: table pref: pref scope: scope errors: []errors.Error{} @@ -95,7 +95,25 @@ pub fn parse_text(text string, b_table &table.Table, pref &pref.Preferences, sco return p.parse() } -pub fn parse_file(path string, b_table &table.Table, comments_mode scanner.CommentsMode, pref &pref.Preferences, global_scope &ast.Scope) ast.File { +pub fn parse_text(text string, table &table.Table, comments_mode scanner.CommentsMode, pref &pref.Preferences, global_scope &ast.Scope) ast.File { + s := scanner.new_scanner(text, comments_mode, pref) + mut p := Parser{ + scanner: s + comments_mode: comments_mode + table: table + pref: pref + scope: &ast.Scope{ + start_pos: 0 + parent: global_scope + } + errors: []errors.Error{} + warnings: []errors.Warning{} + global_scope: global_scope + } + return p.parse() +} + +pub fn parse_file(path string, table &table.Table, comments_mode scanner.CommentsMode, pref &pref.Preferences, global_scope &ast.Scope) ast.File { // NB: when comments_mode == .toplevel_comments, // the parser gives feedback to the scanner about toplevel statements, so that the scanner can skip // all the tricky inner comments. This is needed because we do not have a good general solution @@ -107,7 +125,7 @@ pub fn parse_file(path string, b_table &table.Table, comments_mode scanner.Comme mut p := Parser{ scanner: scanner.new_scanner_file(path, comments_mode, pref) comments_mode: comments_mode - table: b_table + table: table file_name: path file_base: os.base(path) file_name_dir: os.dir(path)