Adiantum encrypting VFS improvements. (#80)

Encrypt temporary files.

Author: ncruces

embed/exports.txt

@@ -51,6 +51,7 @@ sqlite3_create_collation_go
 sqlite3_create_function_go
 sqlite3_create_module_go
 sqlite3_create_window_function_go
+sqlite3_database_file_object
 sqlite3_db_config
 sqlite3_db_name
 sqlite3_db_readonly
@@ -61,9 +62,6 @@ sqlite3_errmsg
 sqlite3_error_offset
 sqlite3_errstr
 sqlite3_exec
-sqlite3_filename_database
-sqlite3_filename_journal
-sqlite3_filename_wal
 sqlite3_finalize
 sqlite3_get_autocommit
 sqlite3_get_auxdata

embed/sqlite3.wasm

@@ -16,14 +16,24 @@ In general, any HBSH construction can be used to wrap any VFS.
 
 The default Adiantum construction uses XChaCha12 for its stream cipher,
 AES for its block cipher, and NH and Poly1305 for hashing.
-It uses Argon2id to derive 256-bit keys from plain text.
+Additionally, we use Argon2id to derive 256-bit keys from plain text.
 
-The VFS encrypts database files, rollback and statement journals, and WAL files.
+The VFS encrypts all files _except_
+[super journals](https://sqlite.org/tempfiles.html#super_journal_files):
+these _never_ contain database data, only filenames,
+and padding them to the block size is problematic.
+
+Temporary files _are_ encrypted with **random** keys,
+as they _may_ contain database data.
+To avoid the overhead of encrypting temporary files,
+keep them in memory:
+
+    PRAGMA temp_store = memory;
 
 > [!IMPORTANT]
 > Adiantum is typically used for disk encryption.
 > The standard threat model for disk encryption considers an adversary
 > that can read multiple snapshots of a disk.
-> The security property that disk encryption provides is that
-> the only information such an adversary can determine is
-> whether the data in a sector has or has not changed over time.
+> The only security property that disk encryption (and this package)
+> provides is that the only information such an adversary can determine
+> is whether the data in a sector has or has not changed over time.

vfs/adiantum/README.md

@@ -1,6 +1,7 @@
 package adiantum
 
 import (
+	"crypto/rand"
 	"sync"
 
 	"golang.org/x/crypto/argon2"
@@ -20,6 +21,12 @@ func (adiantumCreator) HBSH(key []byte) *hbsh.HBSH {
 }
 
 func (adiantumCreator) KDF(text string) []byte {
+	if text == "" {
+		key := make([]byte, 32)
+		n, _ := rand.Read(key)
+		return key[:n]
+	}
+
 	if key := keyCacheGet(text); key != nil {
 		return key[:]
 	}

vfs/adiantum/adiantum.go

@@ -33,18 +33,20 @@ func Register(name string, base vfs.VFS, cipher HBSHCreator) {
 	if cipher == nil {
 		cipher = adiantumCreator{}
 	}
-	vfs.Register("adiantum", &hbshVFS{
+	vfs.Register(name, &hbshVFS{
 		VFS:  base,
 		hbsh: cipher,
 	})
 }
 
-// HBSHCreator creates an [hbsh.HBSH] cipher,
+// HBSHCreator creates an [hbsh.HBSH] cipher
 // given key material.
 type HBSHCreator interface {
-	// KDF maps a secret (text) to a key of the appropriate size.
-	KDF(text string) (key []byte)
+	// KDF derives an HBSH key from a secret.
+	// If no secret is given, a random key is generated.
+	KDF(secret string) (key []byte)
 
-	// HBSH creates an HBSH cipher given an appropriate key.
+	// HBSH creates an HBSH cipher given a key.
+	// If key is not appropriate, nil is returned.
 	HBSH(key []byte) *hbsh.HBSH
 }

vfs/adiantum/api.go

@@ -18,25 +18,23 @@ type hbshVFS struct {
 }
 
 func (h *hbshVFS) Open(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, error) {
-	return h.OpenParams(name, flags, url.Values{})
+	return h.OpenParams(name, flags, nil)
 }
 
 func (h *hbshVFS) OpenParams(name string, flags vfs.OpenFlag, params url.Values) (file vfs.File, _ vfs.OpenFlag, err error) {
-	encrypt := flags&(0|
-		vfs.OPEN_MAIN_DB|
-		vfs.OPEN_MAIN_JOURNAL|
-		vfs.OPEN_SUBJOURNAL|
-		vfs.OPEN_WAL) != 0
-
 	var hbsh *hbsh.HBSH
-	if encrypt {
+
+	// Encrypt everything except super journals.
+	if flags&vfs.OPEN_SUPER_JOURNAL == 0 {
 		var key []byte
 		if t, ok := params["key"]; ok {
 			key = []byte(t[0])
 		} else if t, ok := params["hexkey"]; ok {
 			key, _ = hex.DecodeString(t[0])
 		} else if t, ok := params["textkey"]; ok {
 			key = h.hbsh.KDF(t[0])
+		} else if name == "" {
+			key = h.hbsh.KDF("")
 		}
 
 		if hbsh = h.hbsh.HBSH(key); hbsh == nil {
@@ -45,15 +43,14 @@ func (h *hbshVFS) OpenParams(name string, flags vfs.OpenFlag, params url.Values)
 	}
 
 	if h, ok := h.VFS.(vfs.VFSParams); ok {
-		delete(params, "vfs")
 		delete(params, "key")
 		delete(params, "hexkey")
 		delete(params, "textkey")
 		file, flags, err = h.OpenParams(name, flags, params)
 	} else {
 		file, flags, err = h.Open(name, flags)
 	}
-	if err != nil || hbsh == nil {
+	if err != nil || hbsh == nil || flags&vfs.OPEN_MEMORY != 0 {
 		return file, flags, err
 	}
 	return &hbshFile{File: file, hbsh: hbsh}, flags, err

vfs/adiantum/hbsh.go

@@ -24,6 +24,15 @@ type VFSParams interface {
 	OpenParams(name string, flags OpenFlag, params url.Values) (File, OpenFlag, error)
 }
 
+// VFSJournal extends VFS with the ability to open journals
+// that need a reference to their corresponding database files.
+//
+// https://sqlite.org/c3ref/database_file_object.html
+type VFSJournal interface {
+	VFS
+	OpenJournal(name string, flags OpenFlag, db File) (File, OpenFlag, error)
+}
+
 // A File represents an open file in the OS interface layer.
 //
 // Use sqlite3.ErrorCode or sqlite3.ExtendedErrorCode to return specific error codes to SQLite.

vfs/api.go

@@ -11,12 +11,22 @@ import (
 	"github.com/ncruces/go-sqlite3/vfs"
 )
 
+// Must be a multiple of 64K (the largest page size).
+const sectorSize = 65536
+
 type memVFS struct{}
 
 func (memVFS) Open(name string, flags vfs.OpenFlag) (vfs.File, vfs.OpenFlag, error) {
-	// Allowed file types:
+	// For simplicity, we do not support reading or writing data
+	// across "sector" boundaries.
+	//
+	// This is not a problem for most SQLite file types:
 	// - databases, which only do page aligned reads/writes;
-	// - temp journals, used by the sorter, which does the same.
+	// - temp journals, as used by the sorter, which does the same:
+	//   https://sqlite.org/src/artifact/237840?ln=409-412
+	//
+	// We refuse to open all other file types,
+	// but returning OPEN_MEMORY means SQLite won't ask us to.
 	const types = vfs.OPEN_MAIN_DB |
 		vfs.OPEN_TRANSIENT_DB |
 		vfs.OPEN_TEMP_DB |
@@ -61,9 +71,6 @@ func (memVFS) FullPathname(name string) (string, error) {
 	return name, nil
 }
 
-// Must be a multiple of 64K (the largest page size).
-const sectorSize = 65536
-
 type memDB struct {
 	// +checklocks:lockMtx
 	pending *memFile

vfs/memdb/memdb.go

@@ -23,6 +23,7 @@ func Find(name string) VFS {
 }
 
 // Register registers a VFS.
+// Empty and "os" are reserved names.
 //
 // https://sqlite.org/c3ref/vfs_find.html
 func Register(name string, vfs VFS) {

vfs/registry.go

@@ -16,11 +16,11 @@ WASI_SDK="$ROOT/tools/wasi-sdk-22.0/bin"
 	-fno-stack-protector -fno-stack-clash-protection \
 	-Wl,--stack-first \
 	-Wl,--import-undefined \
-	-D_HAVE_SQLITE_CONFIG_H -DHAVE_USLEEP \
+	-D_HAVE_SQLITE_CONFIG_H -DSQLITE_USE_URI \
 	-DSQLITE_DEFAULT_SYNCHRONOUS=0 \
 	-DSQLITE_DEFAULT_LOCKING_MODE=0 \
 	-DSQLITE_NO_SYNC -DSQLITE_THREADSAFE=0 \
-	-DSQLITE_OMIT_LOAD_EXTENSION -DSQLITE_USE_URI \
+	-DSQLITE_OMIT_LOAD_EXTENSION -DHAVE_USLEEP \
 	-D_WASI_EMULATED_GETPID -lwasi-emulated-getpid \
 	$(awk '{print "-Wl,--export="$0}' exports.txt)