doc: add comment example support (#7924)

vlib/v/doc/comment.v

@@ -0,0 +1,23 @@
+module doc
+
+const (
+	example_pattern = '\x01 Example: '
+)
+
+pub struct DocComment {
+pub mut:
+	text     string // Raw text content of the comment, excluding the comment token chars ('//, /*, */')
+	is_multi bool   // Is a block / multi-line comment
+	pos      DocPos = DocPos{-1, -1, 0}
+}
+
+// is_example returns true if the contents of this comment is a doc example.
+// The current convention is '// Example: <content>'
+pub fn (dc DocComment) is_example() bool {
+	return dc.text.starts_with(example_pattern)
+}
+
+// example returns the content of the example body
+pub fn (dc DocComment) example() string {
+	return dc.text.all_after(example_pattern)
+}

vlib/v/doc/doc.v

@@ -80,7 +80,7 @@ pub struct DocNode {
 pub mut:
 	name        string
 	content     string
-	comment     string
+	comments    []DocComment
 	pos         DocPos = DocPos{-1, -1, 0}
 	file_path   string
 	kind        SymbolKind
@@ -128,7 +128,6 @@ pub fn (mut d Doc) stmt(stmt ast.Stmt, filename string) ?DocNode {
 	mut node := DocNode{
 		name: d.stmt_name(stmt)
 		content: d.stmt_signature(stmt)
-		comment: ''
 		pos: d.convert_pos(filename, stmt.position())
 		file_path: os.join_path(d.base_path, filename)
 		is_pub: d.stmt_pub(stmt)
@@ -139,7 +138,7 @@ pub fn (mut d Doc) stmt(stmt ast.Stmt, filename string) ?DocNode {
 	if node.name.starts_with(d.orig_mod_name + '.') {
 		node.name = node.name.all_after(d.orig_mod_name + '.')
 	}
-	if node.name.len == 0 && node.comment.len == 0 && node.content.len == 0 {
+	if node.name.len == 0 && node.comments.len == 0 && node.content.len == 0 {
 		return error('empty stmt')
 	}
 	match stmt {
@@ -246,12 +245,13 @@ pub fn (mut d Doc) file_ast(file_ast ast.File) map[string]DocNode {
 			last_import_stmt_idx = sidx
 		}
 	}
-	mut prev_comments := []ast.Comment{}
+	mut preceeding_comments := []DocComment{}
 	mut imports_section := true
 	for sidx, stmt in stmts {
 		if stmt is ast.ExprStmt {
+			// Collect comments
 			if stmt.expr is ast.Comment {
-				prev_comments << stmt.expr
+				preceeding_comments << ast_comment_to_doc_comment(stmt.expr)
 				continue
 			}
 		}
@@ -261,37 +261,41 @@ pub fn (mut d Doc) file_ast(file_ast ast.File) map[string]DocNode {
 				continue
 			}
 			// the previous comments were probably a copyright/license one
-			module_comment := get_comment_block_right_before(prev_comments)
-			prev_comments = []
+			module_comment := merge_doc_comments(preceeding_comments)
+			preceeding_comments = []
 			if !d.is_vlib && !module_comment.starts_with('Copyright (c)') {
-				if module_comment in ['', d.head.comment] {
+				if module_comment == '' {
 					continue
 				}
+				/*
 				if d.head.comment != '' {
 					d.head.comment += '\n'
 				}
-				d.head.comment += module_comment
+				*/
+				d.head.comments << preceeding_comments //+= module_comment
 			}
 			continue
 		}
 		if last_import_stmt_idx > 0 && sidx == last_import_stmt_idx {
 			// the accumulated comments were interspersed before/between the imports;
-			// just add them all to the module comment:
+			// just add them all to the module comments:
 			if d.with_head {
-				import_comments := merge_comments(prev_comments)
+				// import_comments := merge_comments(preceeding_comments)
+				/*
 				if d.head.comment != '' {
 					d.head.comment += '\n'
 				}
-				d.head.comment += import_comments
+				*/
+				d.head.comments << preceeding_comments //+= import_comments
 			}
-			prev_comments = []
+			preceeding_comments = []
 			imports_section = false
 		}
 		if stmt is ast.Import {
 			continue
 		}
 		mut node := d.stmt(stmt, os.base(file_ast.path)) or {
-			prev_comments = []
+			preceeding_comments = []
 			continue
 		}
 		if node.parent_name !in contents {
@@ -305,18 +309,20 @@ pub fn (mut d Doc) file_ast(file_ast ast.File) map[string]DocNode {
 				kind: parent_node_kind
 			}
 		}
-		if d.with_comments && (prev_comments.len > 0) {
+		if d.with_comments && (preceeding_comments.len > 0) {
 			// last_comment := contents[contents.len - 1].comment
-			// cmt := last_comment + '\n' + get_comment_block_right_before(prev_comments)
-			mut cmt := get_comment_block_right_before(prev_comments)
+			// cmt := last_comment + '\n' + merge_doc_comments(preceeding_comments)
+			/*
+			mut cmt := merge_doc_comments(preceeding_comments)
 			len := node.name.len
 			// fixed-width symbol name at start of comment
 			if cmt.starts_with(node.name) && cmt.len > len && cmt[len] == ` ` {
 				cmt = '`${cmt[..len]}`' + cmt[len..]
 			}
-			node.comment = cmt
+			*/
+			node.comments << preceeding_comments //= cmt
 		}
-		prev_comments = []
+		preceeding_comments = []
 		if node.parent_name.len > 0 {
 			parent_name := node.parent_name
 			if node.parent_name == 'Constants' {

vlib/v/doc/node.v

@@ -45,3 +45,26 @@ pub fn (cnts map[string]DocNode) arr() []DocNode {
 	contents.sort_by_kind()
 	return contents
 }
+
+// merge_comments returns a `string` with the combined contents of `DocNode.comments`.
+pub fn (dc DocNode) merge_comments() string {
+	return merge_doc_comments(dc.comments)
+}
+
+// merge_comments_without_examples returns a `string` with the
+// combined contents of `DocNode.comments` - excluding any examples.
+pub fn (dc DocNode) merge_comments_without_examples() string {
+	sans_examples := dc.comments.filter(!it.is_example())
+	return merge_doc_comments(sans_examples)
+}
+
+// examples returns a `[]string` containing examples parsed from `DocNode.comments`.
+pub fn (dn DocNode) examples() []string {
+	mut output := []string{}
+	for comment in dn.comments {
+		if comment.is_example() {
+			output << comment.example()
+		}
+	}
+	return output
+}

vlib/v/doc/utils.v

@@ -16,9 +16,33 @@ pub fn merge_comments(comments []ast.Comment) string {
 	return res.join('\n')
 }
 
-// get_comment_block_right_before merges all the comments starting from
+// ast_comment_to_doc_comment converts an `ast.Comment` node type to a `DocComment`
+pub fn ast_comment_to_doc_comment(ast_node ast.Comment) DocComment {
+	text := ast_node.text // TODO .trim_left('\x01') // BUG why are this byte here in the first place?
+	return DocComment{
+		text: text
+		is_multi: ast_node.is_multi
+		pos: DocPos{
+			line: ast_node.pos.line_nr - 1
+			col: 0 // ast_node.pos.pos - ast_node.text.len
+			len: text.len
+		}
+	}
+}
+
+// ast_comments_to_doc_comments converts an array of `ast.Comment` nodes to
+// an array of `DocComment` nodes
+pub fn ast_comments_to_doc_comments(ast_nodes []ast.Comment) []DocComment {
+	mut doc_comments := []DocComment{len: ast_nodes.len}
+	for ast_comment in ast_nodes {
+		doc_comments << ast_comment_to_doc_comment(ast_comment)
+	}
+	return doc_comments
+}
+
+// merge_doc_comments merges all the comments starting from
 // the last up to the first item of the array.
-pub fn get_comment_block_right_before(comments []ast.Comment) string {
+pub fn merge_doc_comments(comments []DocComment) string {
 	if comments.len == 0 {
 		return ''
 	}
@@ -26,7 +50,7 @@ pub fn get_comment_block_right_before(comments []ast.Comment) string {
 	mut last_comment_line_nr := 0
 	for i := comments.len - 1; i >= 0; i-- {
 		cmt := comments[i]
-		if last_comment_line_nr != 0 && cmt.pos.line_nr < last_comment_line_nr - 1 {
+		if last_comment_line_nr != 0 && cmt.pos.line < last_comment_line_nr - 1 {
 			// skip comments that are not part of a continuous block,
 			// located right above the top level statement.
 			// break
@@ -58,7 +82,7 @@ pub fn get_comment_block_right_before(comments []ast.Comment) string {
 		// eprintln('cmt: $cmt')
 		cseparator := if cmt_content.starts_with('```') { '\n' } else { ' ' }
 		comment = cmt_content + cseparator + comment
-		last_comment_line_nr = cmt.pos.line_nr
+		last_comment_line_nr = cmt.pos.line
 	}
 	return comment
 }