sqlite: expose SQLite's VFS layer (#16359)

cmd/tools/vtest-self.v

@@ -122,6 +122,7 @@ const (
 		'vlib/orm/orm_sql_or_blocks_test.v',
 		'vlib/sqlite/sqlite_test.v',
 		'vlib/sqlite/sqlite_orm_test.v',
+		'vlib/sqlite/sqlite_vfs_lowlevel_test.v',
 		'vlib/v/tests/orm_sub_struct_test.v',
 		'vlib/v/tests/orm_sub_array_struct_test.v',
 		'vlib/v/tests/orm_joined_tables_select_test.v',
@@ -167,6 +168,7 @@ const (
 		'vlib/net/websocket/ws_test.v',
 		'vlib/sqlite/sqlite_test.v',
 		'vlib/sqlite/sqlite_orm_test.v',
+		'vlib/sqlite/sqlite_vfs_lowlevel_test.v',
 		'vlib/orm/orm_test.v',
 		'vlib/orm/orm_sql_or_blocks_test.v',
 		'vlib/v/tests/orm_sub_struct_test.v',

vlib/sqlite/sqlite.v

@@ -14,11 +14,28 @@ $if windows {
 
 #include "sqlite3.h"
 
+// https://www.sqlite.org/rescode.html
 pub const (
-	sqlite_ok    = 0
-	sqlite_error = 1
-	sqlite_row   = 100
-	sqlite_done  = 101
+	sqlite_ok                 = 0
+	sqlite_error              = 1
+	sqlite_row                = 100
+	sqlite_done               = 101
+	sqlite_cantopen           = 14
+	sqlite_ioerr_read         = 266
+	sqlite_ioerr_short_read   = 522
+	sqlite_ioerr_write        = 778
+	sqlite_ioerr_fsync        = 1034
+	sqlite_ioerr_fstat        = 1802
+	sqlite_ioerr_delete       = 2570
+
+	sqlite_open_main_db       = 0x00000100
+	sqlite_open_temp_db       = 0x00000200
+	sqlite_open_transient_db  = 0x00000400
+	sqlite_open_main_journal  = 0x00000800
+	sqlite_open_temp_journal  = 0x00001000
+	sqlite_open_subjournal    = 0x00002000
+	sqlite_open_super_journal = 0x00004000
+	sqlite_open_wal           = 0x00080000
 )
 
 pub enum SyncMode {

vlib/sqlite/sqlite_vfs_lowlevel_test.v

@@ -0,0 +1,284 @@
+import sqlite
+import rand
+
+const (
+	max_file_name_len = 256
+)
+
+fn test_vfs_register() {
+	org_default_vfs := sqlite.get_default_vfs()?
+
+	assert org_default_vfs.zName != 0
+
+	vfs_name := 'sometest'
+	mut vfs_descr := &sqlite.Sqlite3_vfs{
+		zName: vfs_name.str
+		iVersion: 2
+	}
+
+	if _ := sqlite.get_vfs(vfs_name) {
+		panic('expected that vfs is not known')
+	}
+
+	vfs_descr.register_as_nondefault() or { panic('vfs register failed $err') }
+
+	sqlite.get_vfs(vfs_name)?
+
+	now_default_vfs := sqlite.get_default_vfs()?
+
+	assert now_default_vfs.zName == org_default_vfs.zName
+
+	vfs_descr.unregister() or { panic('vfs unregister failed $err') }
+
+	if _ := sqlite.get_vfs(vfs_name) {
+		panic('vfs supposedly unregistered yet somehow still foundable')
+	}
+}
+
+// minimal vfs based on example https://www.sqlite.org/src/doc/trunk/src/test_demovfs.c
+fn test_verify_vfs_is_actually_used() {
+	wrapped := sqlite.get_default_vfs()?
+
+	vfs_name := 'sometest'
+	mut vfs_state := &ExampleVfsState{
+		log: []string{cap: 100}
+	}
+	mut vfs_descr := &sqlite.Sqlite3_vfs{
+		iVersion: 2
+		szOsFile: int(sizeof(ExampleVfsOpenedFile))
+		mxPathname: max_file_name_len
+		zName: vfs_name.str
+		pAppData: vfs_state
+		xOpen: example_vfs_open
+		xDelete: example_vfs_delete
+		xAccess: example_vfs_access
+		xFullPathname: example_vfs_fullpathname
+		xDlOpen: wrapped.xDlOpen
+		xDlError: wrapped.xDlError
+		xDlSym: wrapped.xDlSym
+		xDlClose: wrapped.xDlClose
+		xRandomness: wrapped.xRandomness
+		xSleep: wrapped.xSleep
+		xCurrentTime: wrapped.xCurrentTime
+		xGetLastError: example_vfs_getlasterror
+		xCurrentTimeInt64: wrapped.xCurrentTimeInt64
+	}
+
+	vfs_descr.register_as_nondefault()?
+
+	// normally this would be written to disk
+	mut db := sqlite.connect_full('foo.db', [.readwrite, .create], vfs_name)!
+	assert ['fullpathname from=foo.db to=foo.db}', 'open temp?=false name=foo.db', 'read file=foo.db'] == vfs_state.log
+	vfs_state.log.clear()
+
+	db.close()!
+	assert ['close file=foo.db'] == vfs_state.log
+}
+
+struct ExampleVfsState {
+mut:
+	log []string
+}
+
+struct ExampleVfsOpenedFile {
+mut:
+	base      sqlite.Sqlite3_file
+	name      string
+	vfs_state &ExampleVfsState
+}
+
+fn to_vfsstate(t &sqlite.Sqlite3_vfs) &ExampleVfsState {
+	unsafe {
+		p := t.pAppData
+		if p == 0 {
+			assert false, 'p should not be 0'
+		}
+		return &ExampleVfsState(p)
+	}
+}
+
+fn to_vfsopenedfile(t &sqlite.Sqlite3_file) &ExampleVfsOpenedFile {
+	unsafe {
+		if t == 0 {
+			assert false, 't should not be 0'
+		}
+		return &ExampleVfsOpenedFile(t)
+	}
+}
+
+fn example_vfs_fullpathname(vfs &sqlite.Sqlite3_vfs, input &char, size_of_output int, output &char) int {
+	println('fullpathname called')
+
+	mut vfs_state := to_vfsstate(vfs)
+
+	from := unsafe { cstring_to_vstring(input) }
+
+	unsafe {
+		vmemcpy(output, input, from.len)
+		output[from.len] = u8(0)
+	}
+	result := unsafe { cstring_to_vstring(output) }
+
+	vfs_state.log << 'fullpathname from=$from to=$result}'
+
+	return sqlite.sqlite_ok
+}
+
+fn example_vfs_access(vfs &sqlite.Sqlite3_vfs, zPath &char, flags int, pResOut &int) int {
+	println('access called')
+	mut vfs_state := &ExampleVfsState{}
+
+	unsafe {
+		assert 0 != vfs.pAppData
+		vfs_state = &ExampleVfsState(vfs.pAppData)
+	}
+	vfs_state.log << 'accessed'
+
+	return sqlite.sqlite_ok
+}
+
+fn example_vfs_open(vfs &sqlite.Sqlite3_vfs, file_name_or_null_for_tempfile &char, vfs_opened_file &sqlite.Sqlite3_file, in_flags int, out_flags &int) int {
+	println('open called')
+
+	mut is_temp := false
+	mut file_name := ''
+
+	unsafe {
+		if file_name_or_null_for_tempfile == nil {
+			is_temp = true
+			file_name = rand.uuid_v4()
+		} else {
+			file_name = cstring_to_vstring(file_name_or_null_for_tempfile)
+		}
+	}
+	mut vfs_state := to_vfsstate(vfs)
+
+	unsafe {
+		mut outp := to_vfsopenedfile(vfs_opened_file)
+		outp.base.pMethods = &sqlite.Sqlite3_io_methods{
+			iVersion: 1
+			xClose: example_vfsfile_close
+			xRead: example_vfsfile_read
+			xWrite: example_vfsfile_write
+			xTruncate: example_vfsfile_truncate
+			xSync: example_vfsfile_sync
+			xFileSize: example_vfsfile_size
+			xLock: example_vfsfile_lock
+			xUnlock: example_vfsfile_unlock
+			xCheckReservedLock: example_vfsfile_checkreservedlock
+			xFileControl: example_vfsfile_filecontrol
+			xSectorSize: example_vfsfile_sectorsize
+			xDeviceCharacteristics: example_vfsfile_devicecharacteristics
+		}
+
+		outp.name = file_name.clone()
+		outp.vfs_state = vfs_state
+	}
+	vfs_state.log << 'open temp?=$is_temp name=$file_name'
+
+	return sqlite.sqlite_ok
+}
+
+fn example_vfsfile_checkreservedlock(file &sqlite.Sqlite3_file, pResOut &int) int {
+	println('file checkreservedlock')
+
+	unsafe {
+		*pResOut = 0
+	}
+	return sqlite.sqlite_ok
+}
+
+fn example_vfsfile_filecontrol(file &sqlite.Sqlite3_file, op int, arg voidptr) int {
+	println('file filecontrol')
+
+	return 0
+}
+
+fn example_vfsfile_devicecharacteristics(file &sqlite.Sqlite3_file) int {
+	println('file devicecharacteristics')
+
+	return 0
+}
+
+fn example_vfsfile_size(file &sqlite.Sqlite3_file, result &i64) int {
+	println('file size')
+
+	return sqlite.sqlite_ok
+}
+
+fn example_vfsfile_read(file &sqlite.Sqlite3_file, output voidptr, amount int, offset i64) int {
+	println('file read')
+
+	assert amount > 0
+
+	mut vfsfile := to_vfsopenedfile(file)
+
+	vfsfile.vfs_state.log << 'read file=$vfsfile.name'
+
+	unsafe {
+		C.memset(output, 0, amount)
+	}
+
+	return sqlite.sqlite_ioerr_short_read
+}
+
+fn example_vfsfile_truncate(file &sqlite.Sqlite3_file, size i64) int {
+	println('file truncate')
+
+	return sqlite.sqlite_ok
+}
+
+fn example_vfsfile_sectorsize(file &sqlite.Sqlite3_file) int {
+	println('file sectorsize')
+
+	return 0
+}
+
+fn example_vfsfile_sync(file &sqlite.Sqlite3_file, flags int) int {
+	println('file sync called')
+
+	return sqlite.sqlite_ok
+}
+
+fn example_vfsfile_lock(file &sqlite.Sqlite3_file, elock int) int {
+	println('file lock called')
+
+	return sqlite.sqlite_ok
+}
+
+fn example_vfsfile_unlock(file &sqlite.Sqlite3_file, elock int) int {
+	println('file unlock called')
+
+	return sqlite.sqlite_ok
+}
+
+fn example_vfsfile_write(file &sqlite.Sqlite3_file, buf voidptr, amount int, offset i64) int {
+	println('file write called')
+
+	return sqlite.sqlite_ok
+}
+
+fn example_vfsfile_close(file &sqlite.Sqlite3_file) int {
+	println('file close called')
+
+	mut vfsfile := to_vfsopenedfile(file)
+
+	vfsfile.vfs_state.log << 'close file=$vfsfile.name'
+
+	return sqlite.sqlite_ok
+}
+
+fn example_vfs_delete(vfs &sqlite.Sqlite3_vfs, name &char, sync_dir int) int {
+	println('vfs delete called')
+
+	return sqlite.sqlite_ok
+}
+
+fn example_vfs_getlasterror(vfs &sqlite.Sqlite3_vfs, i int, o &char) int {
+	println('vfs getlasterror called')
+
+	unsafe {
+		*o = 0
+	}
+	return sqlite.sqlite_ok
+}

vlib/sqlite/vfs_lowlevel.v

@@ -0,0 +1,171 @@
+module sqlite
+
+type Sig1 = fn (&C.sqlite3_file, &i64) int // https://github.com/vlang/v/issues/16291
+
+type Sig2 = fn (&Sqlite3_file, &int) int // https://github.com/vlang/v/issues/16291
+
+pub type Sqlite3_file = C.sqlite3_file
+
+// https://www.sqlite.org/c3ref/file.html
+struct C.sqlite3_file {
+pub mut:
+	pMethods &C.sqlite3_io_methods // Methods for an open file
+}
+
+// https://www.sqlite.org/c3ref/io_methods.html
+[heap]
+struct C.sqlite3_io_methods {
+mut:
+	// version 1 and later fields
+	iVersion int
+
+	xClose                 fn (&Sqlite3_file) int
+	xRead                  fn (&Sqlite3_file, voidptr, int, i64) int
+	xWrite                 fn (&Sqlite3_file, voidptr, int, i64) int
+	xTruncate              fn (&Sqlite3_file, i64) int
+	xSync                  fn (&Sqlite3_file, int) int
+	xFileSize              Sig1
+	xLock                  fn (&Sqlite3_file, int) int
+	xUnlock                fn (&Sqlite3_file, int) int
+	xCheckReservedLock     Sig2
+	xFileControl           fn (&Sqlite3_file, int, voidptr) int
+	xSectorSize            fn (&Sqlite3_file) int
+	xDeviceCharacteristics fn (&Sqlite3_file) int
+	// version 2 and later fields
+	xShmMap     fn (&Sqlite3_file, int, int, int, &voidptr) int
+	xShmLock    fn (&Sqlite3_file, int, int, int) int
+	xShmBarrier fn (&Sqlite3_file)
+	xShmUnmap   fn (&Sqlite3_file, int) int
+	// version 3 and later fields
+	xFetch   fn (&Sqlite3_file, i64, int, &voidptr) int
+	xUnfetch fn (&Sqlite3_file, i64, voidptr) int
+}
+
+pub type Sqlite3_io_methods = C.sqlite3_io_methods
+
+// https://www.sqlite.org/c3ref/vfs.html
+type Fn_sqlite3_syscall_ptr = fn ()
+
+pub type Sqlite3_vfs = C.sqlite3_vfs
+
+[heap]
+struct C.sqlite3_vfs {
+pub mut:
+	// version 1 and later fields
+	iVersion   int // Structure version number (currently 3)
+	szOsFile   int // Size of subclassed sqlite3_file
+	mxPathname int // Maximum file pathname length
+	pNext      &Sqlite3_vfs // Next registered VFS
+	zName      &char        // Name of this virtual file system
+	pAppData   voidptr      // Pointer to application-specific data
+
+	xOpen   fn (&Sqlite3_vfs, &char, &Sqlite3_file, int, &int) int
+	xDelete fn (&Sqlite3_vfs, &char, int) int
+
+	xAccess       fn (&Sqlite3_vfs, &char, int, &int) int
+	xFullPathname fn (&Sqlite3_vfs, &char, int, &char) int
+	xDlOpen       fn (&Sqlite3_vfs, &char) voidptr
+	xDlError      fn (&Sqlite3_vfs, int, &char)
+	xDlSym        fn (&Sqlite3_vfs, voidptr, &char) voidptr // to fn accepting void and returning
+	xDlClose      fn (&Sqlite3_vfs, voidptr)
+	xRandomness   fn (&Sqlite3_vfs, int, &char) int
+	xSleep        fn (&Sqlite3_vfs, int) int
+	xCurrentTime  fn (&Sqlite3_vfs, &f64) int
+	xGetLastError fn (&Sqlite3_vfs, int, &char) int
+	// version two and later only fields
+	xCurrentTimeInt64 fn (&Sqlite3_vfs, &i64) int
+	// version three and later only fields
+	xSetSystemCall  fn (&Sqlite3_vfs, &char, Fn_sqlite3_syscall_ptr) int
+	xGetSystemCall  fn (&Sqlite3_vfs, &char) Fn_sqlite3_syscall_ptr
+	xNextSystemCall fn (&Sqlite3_vfs, &char) &char
+}
+
+// https://www.sqlite.org/c3ref/vfs_find.html
+fn C.sqlite3_vfs_find(&char) &C.sqlite3_vfs
+fn C.sqlite3_vfs_register(&C.sqlite3_vfs, int) int
+fn C.sqlite3_vfs_unregister(&C.sqlite3_vfs) int
+
+// get_vfs Requests sqlite to return instance of VFS with given name.
+// when such vfs is not known, `none` is returned
+pub fn get_vfs(name string) ?&Sqlite3_vfs {
+	res := C.sqlite3_vfs_find(name.str)
+
+	unsafe {
+		if res == nil {
+			return none
+		} else {
+			return res
+		}
+	}
+}
+
+// get_default_vfs Asks sqlite for default VFS instance
+pub fn get_default_vfs() ?&Sqlite3_vfs {
+	unsafe {
+		res := C.sqlite3_vfs_find(nil)
+		if res == nil {
+			return none
+		} else {
+			return res
+		}
+	}
+}
+
+// register_as_nondefault Asks sqlite to register VFS passed in receiver argument as the known VFS.
+// more info about VFS: https://www.sqlite.org/c3ref/vfs.html
+// 'not TODOs' to prevent corruption: https://sqlite.org/howtocorrupt.html
+// example VFS: https://www.sqlite.org/src/doc/trunk/src/test_demovfs.c
+pub fn (mut v Sqlite3_vfs) register_as_nondefault() ? {
+	res := C.sqlite3_vfs_register(v, 0)
+
+	return if sqlite_ok == res { none } else { error('sqlite3_vfs_register returned $res') }
+}
+
+// unregister Requests sqlite to stop using VFS as passed in receiver argument
+pub fn (mut v Sqlite3_vfs) unregister() ? {
+	res := C.sqlite3_vfs_unregister(v)
+
+	return if sqlite_ok == res { none } else { error('sqlite3_vfs_unregister returned $res') }
+}
+
+// https://www.sqlite.org/c3ref/open.html
+fn C.sqlite3_open_v2(&char, &&C.sqlite3, int, &char) int
+
+// https://www.sqlite.org/c3ref/c_open_autoproxy.html
+pub enum OpenModeFlag {
+	readonly = 0x00000001
+	readwrite = 0x00000002
+	create = 0x00000004
+	uri = 0x00000040
+	memory = 0x00000080
+	nomutex = 0x00008000
+	fullmutex = 0x00010000
+	sharedcache = 0x00020000
+	privatecache = 0x00040000
+	exrescode = 0x02000000
+	nofollow = 0x01000000
+}
+
+// connect_full Opens connection to sqlite database. It gives more control than `open`.
+// Flags give control over readonly and create decisions. Specific VFS can be chosen.
+pub fn connect_full(path string, mode_flags []OpenModeFlag, vfs_name string) !DB {
+	db := &C.sqlite3(0)
+
+	mut flags := 0
+
+	for flag in mode_flags {
+		flags = flags | int(flag)
+	}
+
+	code := C.sqlite3_open_v2(&char(path.str), &db, flags, vfs_name.str)
+	if code != 0 {
+		return &SQLError{
+			msg: unsafe { cstring_to_vstring(&char(C.sqlite3_errstr(code))) }
+			code: code
+		}
+	}
+	return DB{
+		conn: db
+		is_open: true
+	}
+}