toml: add UTF header support, add BOM tests (#12326)

vlib/toml/tests/testdata/toml_with_utf8_bom.toml

@@ -0,0 +1,33 @@
+# This is a TOML document with an UTF-8 BOM header.
+
+title = "TOML Example"
+
+[owner]
+name = "Tom Preston-Werner"
+dob = 1979-05-27T07:32:00-08:00 # First class dates
+
+[database]
+server = "192.168.1.1"
+ports = [ 8000, 8001, 8002 ]
+connection_max = 5000
+enabled = true
+
+[servers]
+
+  # Indentation (tabs and/or spaces) is allowed but not required
+  [servers.alpha]
+  ip = "10.0.0.1"
+  dc = "eqdc10"
+
+  [servers.beta]
+  ip = "10.0.0.2"
+  dc = "eqdc10"
+
+[clients]
+data = [ ["gamma", "delta"], [1, 2] ]
+
+# Line breaks are OK when inside arrays
+hosts = [
+  "alpha",
+  "omega"
+]

vlib/toml/tests/toml_bom_test.v

@@ -0,0 +1,49 @@
+import os
+import toml
+import toml.ast
+
+const empty_toml_document = toml.Doc{
+	ast: &ast.Root(0)
+}
+
+const (
+	toml_text_with_utf8_bom  = os.read_file(os.real_path(os.join_path(os.dir(@FILE), 'testdata',
+		'toml_with_utf8_bom' + '.toml'))) or { panic(err) }
+	toml_text_with_utf16_bom = os.read_file(os.real_path(os.join_path(os.dir(@FILE), 'testdata',
+		'toml_with_utf16_bom' + '.toml'))) or { panic(err) }
+	toml_text_with_utf32_bom = os.read_file(os.real_path(os.join_path(os.dir(@FILE), 'testdata',
+		'toml_with_utf32_bom' + '.toml'))) or { panic(err) }
+)
+
+fn test_toml_with_bom() {
+	toml_doc := toml.parse(toml_text_with_utf8_bom) or { panic(err) }
+	toml_json := toml_doc.to_json()
+
+	title := toml_doc.value('title')
+	assert title == toml.Any('TOML Example')
+	assert title as string == 'TOML Example'
+
+	owner := toml_doc.value('owner') as map[string]toml.Any
+	any_name := owner.value('name') or { panic(err) }
+	assert any_name.string() == 'Tom Preston-Werner'
+
+	database := toml_doc.value('database') as map[string]toml.Any
+	db_serv := database['server'] or {
+		panic('could not access "server" index in "database" variable')
+	}
+	assert db_serv as string == '192.168.1.1'
+
+	// Re-cycle bad_toml_doc
+	mut bad_toml_doc := empty_toml_document
+	bad_toml_doc = toml.parse(toml_text_with_utf16_bom) or {
+		println('     $err.msg')
+		assert true
+		empty_toml_document
+	}
+
+	bad_toml_doc = toml.parse(toml_text_with_utf32_bom) or {
+		println('     $err.msg')
+		assert true
+		empty_toml_document
+	}
+}