From 2634b997695053b751a7fe6723bb9b515d98740c Mon Sep 17 00:00:00 2001 From: Dominik Pytlewski Date: Wed, 9 Nov 2022 18:57:06 +0100 Subject: [PATCH] sqlite: expose SQLite's VFS layer (#16359) --- cmd/tools/vtest-self.v | 2 + vlib/sqlite/sqlite.v | 25 ++- vlib/sqlite/sqlite_vfs_lowlevel_test.v | 284 +++++++++++++++++++++++++ vlib/sqlite/vfs_lowlevel.v | 171 +++++++++++++++ 4 files changed, 478 insertions(+), 4 deletions(-) create mode 100644 vlib/sqlite/sqlite_vfs_lowlevel_test.v create mode 100644 vlib/sqlite/vfs_lowlevel.v diff --git a/cmd/tools/vtest-self.v b/cmd/tools/vtest-self.v index 7b073e7e66..84ddf02ceb 100644 --- a/cmd/tools/vtest-self.v +++ b/cmd/tools/vtest-self.v @@ -122,6 +122,7 @@ const ( 'vlib/orm/orm_sql_or_blocks_test.v', 'vlib/sqlite/sqlite_test.v', 'vlib/sqlite/sqlite_orm_test.v', + 'vlib/sqlite/sqlite_vfs_lowlevel_test.v', 'vlib/v/tests/orm_sub_struct_test.v', 'vlib/v/tests/orm_sub_array_struct_test.v', 'vlib/v/tests/orm_joined_tables_select_test.v', @@ -167,6 +168,7 @@ const ( 'vlib/net/websocket/ws_test.v', 'vlib/sqlite/sqlite_test.v', 'vlib/sqlite/sqlite_orm_test.v', + 'vlib/sqlite/sqlite_vfs_lowlevel_test.v', 'vlib/orm/orm_test.v', 'vlib/orm/orm_sql_or_blocks_test.v', 'vlib/v/tests/orm_sub_struct_test.v', diff --git a/vlib/sqlite/sqlite.v b/vlib/sqlite/sqlite.v index ed308844cf..84bce9684d 100644 --- a/vlib/sqlite/sqlite.v +++ b/vlib/sqlite/sqlite.v @@ -14,11 +14,28 @@ $if windows { #include "sqlite3.h" +// https://www.sqlite.org/rescode.html pub const ( - sqlite_ok = 0 - sqlite_error = 1 - sqlite_row = 100 - sqlite_done = 101 + sqlite_ok = 0 + sqlite_error = 1 + sqlite_row = 100 + sqlite_done = 101 + sqlite_cantopen = 14 + sqlite_ioerr_read = 266 + sqlite_ioerr_short_read = 522 + sqlite_ioerr_write = 778 + sqlite_ioerr_fsync = 1034 + sqlite_ioerr_fstat = 1802 + sqlite_ioerr_delete = 2570 + + sqlite_open_main_db = 0x00000100 + sqlite_open_temp_db = 0x00000200 + sqlite_open_transient_db = 0x00000400 + sqlite_open_main_journal = 0x00000800 + sqlite_open_temp_journal = 0x00001000 + sqlite_open_subjournal = 0x00002000 + sqlite_open_super_journal = 0x00004000 + sqlite_open_wal = 0x00080000 ) pub enum SyncMode { diff --git a/vlib/sqlite/sqlite_vfs_lowlevel_test.v b/vlib/sqlite/sqlite_vfs_lowlevel_test.v new file mode 100644 index 0000000000..f769f237b9 --- /dev/null +++ b/vlib/sqlite/sqlite_vfs_lowlevel_test.v @@ -0,0 +1,284 @@ +import sqlite +import rand + +const ( + max_file_name_len = 256 +) + +fn test_vfs_register() { + org_default_vfs := sqlite.get_default_vfs()? + + assert org_default_vfs.zName != 0 + + vfs_name := 'sometest' + mut vfs_descr := &sqlite.Sqlite3_vfs{ + zName: vfs_name.str + iVersion: 2 + } + + if _ := sqlite.get_vfs(vfs_name) { + panic('expected that vfs is not known') + } + + vfs_descr.register_as_nondefault() or { panic('vfs register failed $err') } + + sqlite.get_vfs(vfs_name)? + + now_default_vfs := sqlite.get_default_vfs()? + + assert now_default_vfs.zName == org_default_vfs.zName + + vfs_descr.unregister() or { panic('vfs unregister failed $err') } + + if _ := sqlite.get_vfs(vfs_name) { + panic('vfs supposedly unregistered yet somehow still foundable') + } +} + +// minimal vfs based on example https://www.sqlite.org/src/doc/trunk/src/test_demovfs.c +fn test_verify_vfs_is_actually_used() { + wrapped := sqlite.get_default_vfs()? + + vfs_name := 'sometest' + mut vfs_state := &ExampleVfsState{ + log: []string{cap: 100} + } + mut vfs_descr := &sqlite.Sqlite3_vfs{ + iVersion: 2 + szOsFile: int(sizeof(ExampleVfsOpenedFile)) + mxPathname: max_file_name_len + zName: vfs_name.str + pAppData: vfs_state + xOpen: example_vfs_open + xDelete: example_vfs_delete + xAccess: example_vfs_access + xFullPathname: example_vfs_fullpathname + xDlOpen: wrapped.xDlOpen + xDlError: wrapped.xDlError + xDlSym: wrapped.xDlSym + xDlClose: wrapped.xDlClose + xRandomness: wrapped.xRandomness + xSleep: wrapped.xSleep + xCurrentTime: wrapped.xCurrentTime + xGetLastError: example_vfs_getlasterror + xCurrentTimeInt64: wrapped.xCurrentTimeInt64 + } + + vfs_descr.register_as_nondefault()? + + // normally this would be written to disk + mut db := sqlite.connect_full('foo.db', [.readwrite, .create], vfs_name)! + assert ['fullpathname from=foo.db to=foo.db}', 'open temp?=false name=foo.db', 'read file=foo.db'] == vfs_state.log + vfs_state.log.clear() + + db.close()! + assert ['close file=foo.db'] == vfs_state.log +} + +struct ExampleVfsState { +mut: + log []string +} + +struct ExampleVfsOpenedFile { +mut: + base sqlite.Sqlite3_file + name string + vfs_state &ExampleVfsState +} + +fn to_vfsstate(t &sqlite.Sqlite3_vfs) &ExampleVfsState { + unsafe { + p := t.pAppData + if p == 0 { + assert false, 'p should not be 0' + } + return &ExampleVfsState(p) + } +} + +fn to_vfsopenedfile(t &sqlite.Sqlite3_file) &ExampleVfsOpenedFile { + unsafe { + if t == 0 { + assert false, 't should not be 0' + } + return &ExampleVfsOpenedFile(t) + } +} + +fn example_vfs_fullpathname(vfs &sqlite.Sqlite3_vfs, input &char, size_of_output int, output &char) int { + println('fullpathname called') + + mut vfs_state := to_vfsstate(vfs) + + from := unsafe { cstring_to_vstring(input) } + + unsafe { + vmemcpy(output, input, from.len) + output[from.len] = u8(0) + } + result := unsafe { cstring_to_vstring(output) } + + vfs_state.log << 'fullpathname from=$from to=$result}' + + return sqlite.sqlite_ok +} + +fn example_vfs_access(vfs &sqlite.Sqlite3_vfs, zPath &char, flags int, pResOut &int) int { + println('access called') + mut vfs_state := &ExampleVfsState{} + + unsafe { + assert 0 != vfs.pAppData + vfs_state = &ExampleVfsState(vfs.pAppData) + } + vfs_state.log << 'accessed' + + return sqlite.sqlite_ok +} + +fn example_vfs_open(vfs &sqlite.Sqlite3_vfs, file_name_or_null_for_tempfile &char, vfs_opened_file &sqlite.Sqlite3_file, in_flags int, out_flags &int) int { + println('open called') + + mut is_temp := false + mut file_name := '' + + unsafe { + if file_name_or_null_for_tempfile == nil { + is_temp = true + file_name = rand.uuid_v4() + } else { + file_name = cstring_to_vstring(file_name_or_null_for_tempfile) + } + } + mut vfs_state := to_vfsstate(vfs) + + unsafe { + mut outp := to_vfsopenedfile(vfs_opened_file) + outp.base.pMethods = &sqlite.Sqlite3_io_methods{ + iVersion: 1 + xClose: example_vfsfile_close + xRead: example_vfsfile_read + xWrite: example_vfsfile_write + xTruncate: example_vfsfile_truncate + xSync: example_vfsfile_sync + xFileSize: example_vfsfile_size + xLock: example_vfsfile_lock + xUnlock: example_vfsfile_unlock + xCheckReservedLock: example_vfsfile_checkreservedlock + xFileControl: example_vfsfile_filecontrol + xSectorSize: example_vfsfile_sectorsize + xDeviceCharacteristics: example_vfsfile_devicecharacteristics + } + + outp.name = file_name.clone() + outp.vfs_state = vfs_state + } + vfs_state.log << 'open temp?=$is_temp name=$file_name' + + return sqlite.sqlite_ok +} + +fn example_vfsfile_checkreservedlock(file &sqlite.Sqlite3_file, pResOut &int) int { + println('file checkreservedlock') + + unsafe { + *pResOut = 0 + } + return sqlite.sqlite_ok +} + +fn example_vfsfile_filecontrol(file &sqlite.Sqlite3_file, op int, arg voidptr) int { + println('file filecontrol') + + return 0 +} + +fn example_vfsfile_devicecharacteristics(file &sqlite.Sqlite3_file) int { + println('file devicecharacteristics') + + return 0 +} + +fn example_vfsfile_size(file &sqlite.Sqlite3_file, result &i64) int { + println('file size') + + return sqlite.sqlite_ok +} + +fn example_vfsfile_read(file &sqlite.Sqlite3_file, output voidptr, amount int, offset i64) int { + println('file read') + + assert amount > 0 + + mut vfsfile := to_vfsopenedfile(file) + + vfsfile.vfs_state.log << 'read file=$vfsfile.name' + + unsafe { + C.memset(output, 0, amount) + } + + return sqlite.sqlite_ioerr_short_read +} + +fn example_vfsfile_truncate(file &sqlite.Sqlite3_file, size i64) int { + println('file truncate') + + return sqlite.sqlite_ok +} + +fn example_vfsfile_sectorsize(file &sqlite.Sqlite3_file) int { + println('file sectorsize') + + return 0 +} + +fn example_vfsfile_sync(file &sqlite.Sqlite3_file, flags int) int { + println('file sync called') + + return sqlite.sqlite_ok +} + +fn example_vfsfile_lock(file &sqlite.Sqlite3_file, elock int) int { + println('file lock called') + + return sqlite.sqlite_ok +} + +fn example_vfsfile_unlock(file &sqlite.Sqlite3_file, elock int) int { + println('file unlock called') + + return sqlite.sqlite_ok +} + +fn example_vfsfile_write(file &sqlite.Sqlite3_file, buf voidptr, amount int, offset i64) int { + println('file write called') + + return sqlite.sqlite_ok +} + +fn example_vfsfile_close(file &sqlite.Sqlite3_file) int { + println('file close called') + + mut vfsfile := to_vfsopenedfile(file) + + vfsfile.vfs_state.log << 'close file=$vfsfile.name' + + return sqlite.sqlite_ok +} + +fn example_vfs_delete(vfs &sqlite.Sqlite3_vfs, name &char, sync_dir int) int { + println('vfs delete called') + + return sqlite.sqlite_ok +} + +fn example_vfs_getlasterror(vfs &sqlite.Sqlite3_vfs, i int, o &char) int { + println('vfs getlasterror called') + + unsafe { + *o = 0 + } + return sqlite.sqlite_ok +} diff --git a/vlib/sqlite/vfs_lowlevel.v b/vlib/sqlite/vfs_lowlevel.v new file mode 100644 index 0000000000..24f614284c --- /dev/null +++ b/vlib/sqlite/vfs_lowlevel.v @@ -0,0 +1,171 @@ +module sqlite + +type Sig1 = fn (&C.sqlite3_file, &i64) int // https://github.com/vlang/v/issues/16291 + +type Sig2 = fn (&Sqlite3_file, &int) int // https://github.com/vlang/v/issues/16291 + +pub type Sqlite3_file = C.sqlite3_file + +// https://www.sqlite.org/c3ref/file.html +struct C.sqlite3_file { +pub mut: + pMethods &C.sqlite3_io_methods // Methods for an open file +} + +// https://www.sqlite.org/c3ref/io_methods.html +[heap] +struct C.sqlite3_io_methods { +mut: + // version 1 and later fields + iVersion int + + xClose fn (&Sqlite3_file) int + xRead fn (&Sqlite3_file, voidptr, int, i64) int + xWrite fn (&Sqlite3_file, voidptr, int, i64) int + xTruncate fn (&Sqlite3_file, i64) int + xSync fn (&Sqlite3_file, int) int + xFileSize Sig1 + xLock fn (&Sqlite3_file, int) int + xUnlock fn (&Sqlite3_file, int) int + xCheckReservedLock Sig2 + xFileControl fn (&Sqlite3_file, int, voidptr) int + xSectorSize fn (&Sqlite3_file) int + xDeviceCharacteristics fn (&Sqlite3_file) int + // version 2 and later fields + xShmMap fn (&Sqlite3_file, int, int, int, &voidptr) int + xShmLock fn (&Sqlite3_file, int, int, int) int + xShmBarrier fn (&Sqlite3_file) + xShmUnmap fn (&Sqlite3_file, int) int + // version 3 and later fields + xFetch fn (&Sqlite3_file, i64, int, &voidptr) int + xUnfetch fn (&Sqlite3_file, i64, voidptr) int +} + +pub type Sqlite3_io_methods = C.sqlite3_io_methods + +// https://www.sqlite.org/c3ref/vfs.html +type Fn_sqlite3_syscall_ptr = fn () + +pub type Sqlite3_vfs = C.sqlite3_vfs + +[heap] +struct C.sqlite3_vfs { +pub mut: + // version 1 and later fields + iVersion int // Structure version number (currently 3) + szOsFile int // Size of subclassed sqlite3_file + mxPathname int // Maximum file pathname length + pNext &Sqlite3_vfs // Next registered VFS + zName &char // Name of this virtual file system + pAppData voidptr // Pointer to application-specific data + + xOpen fn (&Sqlite3_vfs, &char, &Sqlite3_file, int, &int) int + xDelete fn (&Sqlite3_vfs, &char, int) int + + xAccess fn (&Sqlite3_vfs, &char, int, &int) int + xFullPathname fn (&Sqlite3_vfs, &char, int, &char) int + xDlOpen fn (&Sqlite3_vfs, &char) voidptr + xDlError fn (&Sqlite3_vfs, int, &char) + xDlSym fn (&Sqlite3_vfs, voidptr, &char) voidptr // to fn accepting void and returning + xDlClose fn (&Sqlite3_vfs, voidptr) + xRandomness fn (&Sqlite3_vfs, int, &char) int + xSleep fn (&Sqlite3_vfs, int) int + xCurrentTime fn (&Sqlite3_vfs, &f64) int + xGetLastError fn (&Sqlite3_vfs, int, &char) int + // version two and later only fields + xCurrentTimeInt64 fn (&Sqlite3_vfs, &i64) int + // version three and later only fields + xSetSystemCall fn (&Sqlite3_vfs, &char, Fn_sqlite3_syscall_ptr) int + xGetSystemCall fn (&Sqlite3_vfs, &char) Fn_sqlite3_syscall_ptr + xNextSystemCall fn (&Sqlite3_vfs, &char) &char +} + +// https://www.sqlite.org/c3ref/vfs_find.html +fn C.sqlite3_vfs_find(&char) &C.sqlite3_vfs +fn C.sqlite3_vfs_register(&C.sqlite3_vfs, int) int +fn C.sqlite3_vfs_unregister(&C.sqlite3_vfs) int + +// get_vfs Requests sqlite to return instance of VFS with given name. +// when such vfs is not known, `none` is returned +pub fn get_vfs(name string) ?&Sqlite3_vfs { + res := C.sqlite3_vfs_find(name.str) + + unsafe { + if res == nil { + return none + } else { + return res + } + } +} + +// get_default_vfs Asks sqlite for default VFS instance +pub fn get_default_vfs() ?&Sqlite3_vfs { + unsafe { + res := C.sqlite3_vfs_find(nil) + if res == nil { + return none + } else { + return res + } + } +} + +// register_as_nondefault Asks sqlite to register VFS passed in receiver argument as the known VFS. +// more info about VFS: https://www.sqlite.org/c3ref/vfs.html +// 'not TODOs' to prevent corruption: https://sqlite.org/howtocorrupt.html +// example VFS: https://www.sqlite.org/src/doc/trunk/src/test_demovfs.c +pub fn (mut v Sqlite3_vfs) register_as_nondefault() ? { + res := C.sqlite3_vfs_register(v, 0) + + return if sqlite_ok == res { none } else { error('sqlite3_vfs_register returned $res') } +} + +// unregister Requests sqlite to stop using VFS as passed in receiver argument +pub fn (mut v Sqlite3_vfs) unregister() ? { + res := C.sqlite3_vfs_unregister(v) + + return if sqlite_ok == res { none } else { error('sqlite3_vfs_unregister returned $res') } +} + +// https://www.sqlite.org/c3ref/open.html +fn C.sqlite3_open_v2(&char, &&C.sqlite3, int, &char) int + +// https://www.sqlite.org/c3ref/c_open_autoproxy.html +pub enum OpenModeFlag { + readonly = 0x00000001 + readwrite = 0x00000002 + create = 0x00000004 + uri = 0x00000040 + memory = 0x00000080 + nomutex = 0x00008000 + fullmutex = 0x00010000 + sharedcache = 0x00020000 + privatecache = 0x00040000 + exrescode = 0x02000000 + nofollow = 0x01000000 +} + +// connect_full Opens connection to sqlite database. It gives more control than `open`. +// Flags give control over readonly and create decisions. Specific VFS can be chosen. +pub fn connect_full(path string, mode_flags []OpenModeFlag, vfs_name string) !DB { + db := &C.sqlite3(0) + + mut flags := 0 + + for flag in mode_flags { + flags = flags | int(flag) + } + + code := C.sqlite3_open_v2(&char(path.str), &db, flags, vfs_name.str) + if code != 0 { + return &SQLError{ + msg: unsafe { cstring_to_vstring(&char(C.sqlite3_errstr(code))) } + code: code + } + } + return DB{ + conn: db + is_open: true + } +}