Skip to content

Commit a57ce87

Browse files
ncrucesncruces
authored
Windows shared memory. (#181)
1 parent 81e7a94 commit a57ce87

File tree

12 files changed

+380
-95
lines changed

12 files changed

+380
-95
lines changed

‎.github/workflows/build-test.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,6 @@ echo 'set -eu' > test.sh
66
for p in $(go list ./...); do
77
dir=".${p#github.com/ncruces/go-sqlite3}"
88
name="$(basename "$p").test"
9-
(cd ${dir}; go test -c)
10-
[ -f "${dir}/${name}" ] && echo "(cd ${dir}; ./${name} ${TESTFLAGS})" >> test.sh
9+
(cd ${dir}; go test -c ${BUILDFLAGS:-})
10+
[ -f "${dir}/${name}" ] && echo "(cd ${dir}; ./${name} ${TESTFLAGS:-})" >> test.sh
1111
done

‎.github/workflows/test.yml

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -52,17 +52,17 @@ jobs:
5252
- name: Test
5353
run: go test -v ./... -bench . -benchtime=1x
5454

55+
- name: Test no locks
56+
run: go test -v -tags sqlite3_nosys ./...
57+
if: matrix.os == 'ubuntu-latest'
58+
5559
- name: Test BSD locks
5660
run: go test -v -tags sqlite3_flock ./...
5761
if: matrix.os == 'macos-latest'
5862

5963
- name: Test dot locks
6064
run: go test -v -tags sqlite3_dotlk ./...
61-
if: matrix.os == 'macos-latest'
62-
63-
- name: Test no locks
64-
run: go test -v -tags sqlite3_nosys ./...
65-
if: matrix.os == 'ubuntu-latest'
65+
if: matrix.os != 'windows-latest'
6666

6767
- name: Test GORM
6868
shell: bash
@@ -194,6 +194,7 @@ jobs:
194194
env:
195195
GOOS: solaris
196196
TESTFLAGS: '-test.v -test.short'
197+
BUILDFLAGS: '-tags sqlite3_dotlk'
197198
run: .github/workflows/build-test.sh
198199

199200
- name: Test Solaris

‎internal/testcfg/testcfg.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,8 @@ func init() {
1919
path := filepath.Join(os.TempDir(), "wazero")
2020
if err := os.MkdirAll(path, 0777); err == nil {
2121
if cache, err := wazero.NewCompilationCacheWithDir(path); err == nil {
22-
sqlite3.RuntimeConfig.WithCompilationCache(cache)
22+
sqlite3.RuntimeConfig = sqlite3.RuntimeConfig.
23+
WithCompilationCache(cache)
2324
}
2425
}
2526
}

‎internal/util/mmap.go

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -55,10 +55,10 @@ type MappedRegion struct {
5555
used bool
5656
}
5757

58-
func MapRegion(ctx context.Context, mod api.Module, f *os.File, offset int64, size int32, prot int) (*MappedRegion, error) {
58+
func MapRegion(ctx context.Context, mod api.Module, f *os.File, offset int64, size int32, readOnly bool) (*MappedRegion, error) {
5959
s := ctx.Value(moduleKey{}).(*moduleState)
6060
r := s.new(ctx, mod, size)
61-
err := r.mmap(f, offset, prot)
61+
err := r.mmap(f, offset, readOnly)
6262
if err != nil {
6363
return nil, err
6464
}
@@ -75,7 +75,11 @@ func (r *MappedRegion) Unmap() error {
7575
return err
7676
}
7777

78-
func (r *MappedRegion) mmap(f *os.File, offset int64, prot int) error {
78+
func (r *MappedRegion) mmap(f *os.File, offset int64, readOnly bool) error {
79+
prot := unix.PROT_READ
80+
if !readOnly {
81+
prot |= unix.PROT_WRITE
82+
}
7983
_, err := unix.MmapPtr(int(f.Fd()), offset, r.addr, uintptr(r.size),
8084
prot, unix.MAP_SHARED|unix.MAP_FIXED)
8185
r.used = err == nil

‎internal/util/mmap_windows.go

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
//go:build !sqlite3_nosys
2+
3+
package util
4+
5+
import (
6+
"context"
7+
"os"
8+
"reflect"
9+
"unsafe"
10+
11+
"github.com/tetratelabs/wazero/api"
12+
"golang.org/x/sys/windows"
13+
)
14+
15+
type MappedRegion struct {
16+
windows.Handle
17+
Data []byte
18+
addr uintptr
19+
}
20+
21+
func MapRegion(ctx context.Context, mod api.Module, f *os.File, offset int64, size int32) (*MappedRegion, error) {
22+
h, err := windows.CreateFileMapping(windows.Handle(f.Fd()), nil, windows.PAGE_READWRITE, 0, 0, nil)
23+
if h == 0 {
24+
return nil, err
25+
}
26+
27+
a, err := windows.MapViewOfFile(h, windows.FILE_MAP_WRITE,
28+
uint32(offset>>32), uint32(offset), uintptr(size))
29+
if a == 0 {
30+
windows.CloseHandle(h)
31+
return nil, err
32+
}
33+
34+
res := &MappedRegion{Handle: h, addr: a}
35+
// SliceHeader, although deprecated, avoids a go vet warning.
36+
sh := (*reflect.SliceHeader)(unsafe.Pointer(&res.Data))
37+
sh.Len = int(size)
38+
sh.Cap = int(size)
39+
sh.Data = a
40+
return res, nil
41+
}
42+
43+
func (r *MappedRegion) Unmap() error {
44+
if r.Data == nil {
45+
return nil
46+
}
47+
err := windows.UnmapViewOfFile(r.addr)
48+
if err != nil {
49+
return err
50+
}
51+
r.Data = nil
52+
return windows.CloseHandle(r.Handle)
53+
}

‎vfs/shm.go

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
//go:build ((darwin || linux || freebsd || openbsd || netbsd || dragonfly || illumos) && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !sqlite3_nosys) || sqlite3_flock || sqlite3_dotlk
1+
//go:build ((linux || darwin || windows || freebsd || openbsd || netbsd || dragonfly || illumos) && (386 || arm || amd64 || arm64 || riscv64 || ppc64le) && !sqlite3_nosys) || sqlite3_flock || sqlite3_dotlk
22

33
package vfs
44

@@ -22,8 +22,5 @@ func NewSharedMemory(path string, flags OpenFlag) SharedMemory {
2222
if flags&OPEN_MAIN_DB == 0 || flags&(OPEN_DELETEONCLOSE|OPEN_MEMORY) != 0 {
2323
return nil
2424
}
25-
return &vfsShm{
26-
path: path,
27-
readOnly: flags&OPEN_READONLY != 0,
28-
}
25+
return &vfsShm{path: path}
2926
}

‎vfs/shm_bsd.go

Lines changed: 16 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,9 @@ type vfsShmFile struct {
1818
*os.File
1919
info os.FileInfo
2020

21-
// +checklocks:vfsShmFilesMtx
22-
refs int
21+
refs int // +checklocks:vfsShmFilesMtx
2322

24-
// +checklocks:Mutex
25-
lock [_SHM_NLOCK]int16
23+
lock [_SHM_NLOCK]int16 // +checklocks:Mutex
2624
sync.Mutex
2725
}
2826

@@ -34,10 +32,9 @@ var (
3432

3533
type vfsShm struct {
3634
*vfsShmFile
37-
path string
38-
lock [_SHM_NLOCK]bool
39-
regions []*util.MappedRegion
40-
readOnly bool
35+
path string
36+
lock [_SHM_NLOCK]bool
37+
regions []*util.MappedRegion
4138
}
4239

4340
func (s *vfsShm) Close() error {
@@ -69,7 +66,7 @@ func (s *vfsShm) Close() error {
6966
panic(util.AssertErr())
7067
}
7168

72-
func (s *vfsShm) shmOpen() (rc _ErrorCode) {
69+
func (s *vfsShm) shmOpen() _ErrorCode {
7370
if s.vfsShmFile != nil {
7471
return _OK
7572
}
@@ -100,17 +97,13 @@ func (s *vfsShm) shmOpen() (rc _ErrorCode) {
10097
}
10198
}
10299

103-
// Lock and truncate the file, if not readonly.
100+
// Lock and truncate the file.
104101
// The lock is only released by closing the file.
105-
if s.readOnly {
106-
rc = _READONLY_CANTINIT
107-
} else {
108-
if rc := osLock(f, unix.LOCK_EX|unix.LOCK_NB, _IOERR_LOCK); rc != _OK {
109-
return rc
110-
}
111-
if err := f.Truncate(0); err != nil {
112-
return _IOERR_SHMOPEN
113-
}
102+
if rc := osLock(f, unix.LOCK_EX|unix.LOCK_NB, _IOERR_LOCK); rc != _OK {
103+
return rc
104+
}
105+
if err := f.Truncate(0); err != nil {
106+
return _IOERR_SHMOPEN
114107
}
115108

116109
// Add the new shared file.
@@ -122,11 +115,11 @@ func (s *vfsShm) shmOpen() (rc _ErrorCode) {
122115
for i, g := range vfsShmFiles {
123116
if g == nil {
124117
vfsShmFiles[i] = s.vfsShmFile
125-
return rc
118+
return _OK
126119
}
127120
}
128121
vfsShmFiles = append(vfsShmFiles, s.vfsShmFile)
129-
return rc
122+
return _OK
130123
}
131124

132125
func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, extend bool) (uint32, _ErrorCode) {
@@ -148,25 +141,16 @@ func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, ext
148141
if !extend {
149142
return 0, _OK
150143
}
151-
if s.readOnly || osAllocate(s.File, n) != nil {
144+
if osAllocate(s.File, n) != nil {
152145
return 0, _IOERR_SHMSIZE
153146
}
154147
}
155148

156-
var prot int
157-
if s.readOnly {
158-
prot = unix.PROT_READ
159-
} else {
160-
prot = unix.PROT_READ | unix.PROT_WRITE
161-
}
162-
r, err := util.MapRegion(ctx, mod, s.File, int64(id)*int64(size), size, prot)
149+
r, err := util.MapRegion(ctx, mod, s.File, int64(id)*int64(size), size, false)
163150
if err != nil {
164151
return 0, _IOERR_SHMMAP
165152
}
166153
s.regions = append(s.regions, r)
167-
if s.readOnly {
168-
return r.Ptr, _READONLY
169-
}
170154
return r.Ptr, _OK
171155
}
172156

‎vfs/shm_copy.go renamed to ‎vfs/shm_dotlk.go

Lines changed: 36 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,16 @@ package vfs
44

55
import (
66
"context"
7+
"errors"
8+
"io/fs"
9+
"os"
710
"sync"
811
"unsafe"
912

1013
"github.com/ncruces/go-sqlite3/internal/util"
1114
"github.com/tetratelabs/wazero/api"
1215
)
1316

14-
const _WALINDEX_PGSZ = 32768
15-
1617
type vfsShmBuffer struct {
1718
shared []byte // +checklocks:Mutex
1819
refs int // +checklocks:vfsShmBuffersMtx
@@ -29,15 +30,14 @@ var (
2930

3031
type vfsShm struct {
3132
*vfsShmBuffer
32-
mod api.Module
33-
alloc api.Function
34-
free api.Function
35-
path string
36-
shadow []byte
37-
ptrs []uint32
38-
stack [1]uint64
39-
lock [_SHM_NLOCK]bool
40-
readOnly bool
33+
mod api.Module
34+
alloc api.Function
35+
free api.Function
36+
path string
37+
shadow []byte
38+
ptrs []uint32
39+
stack [1]uint64
40+
lock [_SHM_NLOCK]bool
4141
}
4242

4343
func (s *vfsShm) Close() error {
@@ -58,13 +58,18 @@ func (s *vfsShm) Close() error {
5858
return nil
5959
}
6060

61+
err := os.Remove(s.path)
62+
if err != nil && !errors.Is(err, fs.ErrNotExist) {
63+
return _IOERR_UNLOCK
64+
}
6165
delete(vfsShmBuffers, s.path)
66+
s.vfsShmBuffer = nil
6267
return nil
6368
}
6469

65-
func (s *vfsShm) shmOpen() {
70+
func (s *vfsShm) shmOpen() _ErrorCode {
6671
if s.vfsShmBuffer != nil {
67-
return
72+
return _OK
6873
}
6974

7075
vfsShmBuffersMtx.Lock()
@@ -74,12 +79,23 @@ func (s *vfsShm) shmOpen() {
7479
if g, ok := vfsShmBuffers[s.path]; ok {
7580
s.vfsShmBuffer = g
7681
g.refs++
77-
return
82+
return _OK
83+
}
84+
85+
// Create a directory on disk to ensure only this process
86+
// uses this path to register a shared memory.
87+
err := os.Mkdir(s.path, 0777)
88+
if errors.Is(err, fs.ErrExist) {
89+
return _BUSY
90+
}
91+
if err != nil {
92+
return _IOERR_LOCK
7893
}
7994

8095
// Add the new shared buffer.
8196
s.vfsShmBuffer = &vfsShmBuffer{}
8297
vfsShmBuffers[s.path] = s.vfsShmBuffer
98+
return _OK
8399
}
84100

85101
func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, extend bool) (uint32, _ErrorCode) {
@@ -91,8 +107,10 @@ func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, ext
91107
s.free = mod.ExportedFunction("sqlite3_free")
92108
s.alloc = mod.ExportedFunction("sqlite3_malloc64")
93109
}
110+
if rc := s.shmOpen(); rc != _OK {
111+
return 0, rc
112+
}
94113

95-
s.shmOpen()
96114
s.Lock()
97115
defer s.Unlock()
98116
defer s.shmAcquire()
@@ -131,7 +149,7 @@ func (s *vfsShm) shmLock(offset, n int32, flags _ShmFlag) _ErrorCode {
131149

132150
switch {
133151
case flags&_SHM_LOCK != 0:
134-
s.shmAcquire()
152+
defer s.shmAcquire()
135153
case flags&_SHM_EXCLUSIVE != 0:
136154
s.shmRelease()
137155
}
@@ -228,6 +246,8 @@ func (s *vfsShm) shmBarrier() {
228246
//
229247
// https://sqlite.org/walformat.html#the_wal_index_file_format
230248

249+
const _WALINDEX_PGSZ = 32768
250+
231251
// +checklocks:s.Mutex
232252
func (s *vfsShm) shmAcquire() {
233253
// Copies modified words from shared to private memory.

0 commit comments

Comments
 (0)