token: fix a new_keywords_matcher_from_array_trie bug (first word with idx 0 was ignored); add tests

vlib/v/token/keywords_matcher_trie.v

@@ -5,7 +5,7 @@ module token
 // See the module description for more details.
 [heap]
 pub struct KeywordsMatcherTrie {
-mut:
+pub mut:
 	nodes   []&TrieNode
 	min_len int = 999999
 	max_len int
@@ -13,9 +13,9 @@ mut:
 
 // TrieNode is a single node from a trie, used by KeywordsMatcherTrie
 pub struct TrieNode {
-mut:
+pub mut:
 	children [123]&TrieNode
-	value    int // when positive, it is a leaf node representing a match
+	value    int = -1 // when != -1, it is a leaf node representing a match
 }
 
 // find tries to find the given `word` in the set of all previously added words
@@ -38,6 +38,11 @@ pub fn (km &KeywordsMatcherTrie) find(word string) int {
 	return node.find(word)
 }
 
+[inline]
+pub fn (km &KeywordsMatcherTrie) matches(word string) bool {
+	return km.find(word) != -1
+}
+
 // add_word adds the given word to the KeywordsMatcherTrie instance. It associates a non
 // negative integer value to it, so later `find` could return the value, when it succeeds.
 [direct_array_access]
@@ -92,13 +97,14 @@ pub fn new_trie_node() &TrieNode {
 
 // show displays the information in `node`, in a more compact/readable format (recursively)
 pub fn (node &TrieNode) show(level int) {
-	mut non_nil_children := 0
-	for x in node.children {
+	mut non_nil_children := []int{}
+	for idx, x in node.children {
 		if x != unsafe { nil } {
-			non_nil_children++
+			non_nil_children << idx
 		}
 	}
-	eprintln('> level: ${level:2} | value: ${node.value:12} | non_nil_children: ${non_nil_children:2}')
+	children := non_nil_children.map(u8(it).ascii_str())
+	eprintln('> level: ${level:2} | value: ${node.value:12} | non_nil_children: ${non_nil_children.len:2} | $children')
 	for x in node.children {
 		if x != unsafe { nil } {
 			x.show(level + 1)
@@ -134,7 +140,7 @@ pub fn (root &TrieNode) find(word string) int {
 		// eprintln('> match_keyword: `${word:20}` | node: ${ptr_str(node)} | idx: ${idx:3}')
 		if idx == wlen {
 			k := node.value
-			if k > 0 {
+			if k != -1 {
 				// node.show(0)
 				return k
 			}

vlib/v/token/keywords_matcher_trie_test.v

@@ -0,0 +1,13 @@
+import v.token
+
+fn test_new_keywords_matcher_from_array_trie_works() {
+	method_names := ['filter', 'clone', 'repeat', 'reverse', 'map', 'slice', 'sort', 'contains',
+		'index', 'wait', 'any', 'all', 'first', 'last', 'pop']
+	matcher := token.new_keywords_matcher_from_array_trie(method_names)
+	for word in [method_names.first(), method_names.last(), 'something', 'another', 'x', 'y', '',
+		'---', '123', 'abc.def', 'xyz !@#'] {
+		res1 := matcher.find(word) != -1
+		res2 := word in method_names
+		assert res1 == res2
+	}
+}