Skip to content

Commit 6ae1042

Browse files
waderwader
committed
wip
1 parent b8dd58a commit 6ae1042

File tree

7 files changed

+133
-1143
lines changed

7 files changed

+133
-1143
lines changed

‎format/sqlite3/sqlite3.go‎

Lines changed: 127 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -41,25 +41,40 @@ import (
4141
"embed"
4242

4343
"github.com/wader/fq/format"
44-
"github.com/wader/fq/format/registry"
45-
"github.com/wader/fq/internal/mathextra"
44+
"github.com/wader/fq/internal/mathex"
4645
"github.com/wader/fq/pkg/bitio"
4746
"github.com/wader/fq/pkg/decode"
47+
"github.com/wader/fq/pkg/interp"
4848
"github.com/wader/fq/pkg/scalar"
4949
)
5050

5151
//go:embed *.jq
5252
var sqlite3FS embed.FS
5353

5454
func init() {
55-
registry.MustRegister(decode.Format{
55+
interp.RegisterFormat(decode.Format{
5656
Name: format.SQLITE3,
5757
Description: "SQLite v3 database",
5858
Groups: []string{format.PROBE},
5959
DecodeFn: sqlite3Decode,
6060
Functions: []string{"torepr"},
61-
Files: sqlite3FS,
6261
})
62+
interp.RegisterFS(sqlite3FS)
63+
}
64+
65+
type intStack struct {
66+
s []int
67+
}
68+
69+
func (s *intStack) Push(n int) { s.s = append(s.s, n) }
70+
71+
func (s *intStack) Pop() (int, bool) {
72+
if len(s.s) == 0 {
73+
return 0, false
74+
}
75+
var n int
76+
n, s.s = s.s[0], s.s[1:]
77+
return n, true
6378
}
6479

6580
const sqlite3HeaderSize = 100
@@ -79,7 +94,7 @@ const (
7994
serialTypeInternal11 = 11
8095
)
8196

82-
var serialTypeMap = scalar.SToSymStr{
97+
var serialTypeMap = scalar.UintMapSymStr{
8398
serialTypeNULL: "null",
8499
serialTypeS8: "int8",
85100
serialTypeSBE16: "int16",
@@ -94,8 +109,8 @@ var serialTypeMap = scalar.SToSymStr{
94109
serialTypeInternal11: "internal11",
95110
}
96111

97-
var serialTypeMapper = scalar.Fn(func(s scalar.S) (scalar.S, error) {
98-
typ := s.ActualS()
112+
var serialTypeMapper = scalar.SintFn(func(s scalar.Sint) (scalar.Sint, error) {
113+
typ := uint64(s.Actual)
99114
if st, ok := serialTypeMap[typ]; ok {
100115
s.Description = st
101116
} else if typ >= 12 && typ%2 == 0 {
@@ -106,23 +121,25 @@ var serialTypeMapper = scalar.Fn(func(s scalar.S) (scalar.S, error) {
106121
return s, nil
107122
})
108123

124+
type pageType int
125+
109126
const (
110-
pageTypePtrmap = 0x00
111-
pageTypeBTreeIndexInterior = 0x02
112-
pageTypeBTreeTableInterior = 0x05
113-
pageTypeBTreeIndexLeaf = 0x0a
114-
pageTypeBTreeTableLeaf = 0x0d
127+
pageTypePtrmap pageType = 0x00
128+
pageTypeBTreeIndexInterior = 0x02
129+
pageTypeBTreeTableInterior = 0x05
130+
pageTypeBTreeIndexLeaf = 0x0a
131+
pageTypeBTreeTableLeaf = 0x0d
115132
)
116133

117-
var pageTypeMap = scalar.UToSymStr{
134+
var pageTypeMap = scalar.UintMapSymStr{
118135
// pageTypePtrmap: "ptrmap",
119136
pageTypeBTreeIndexInterior: "index_interior",
120137
pageTypeBTreeTableInterior: "table_interior",
121138
pageTypeBTreeIndexLeaf: "index_leaf",
122139
pageTypeBTreeTableLeaf: "table_leaf",
123140
}
124141

125-
var ptrmapTypeMap = scalar.UToSymStr{
142+
var ptrmapTypeMap = scalar.UintMapSymStr{
126143
1: "rootpage",
127144
2: "freepage",
128145
3: "overflow1",
@@ -136,13 +153,13 @@ const (
136153
textEncodingUTF16BE = 3
137154
)
138155

139-
var textEncodingMap = scalar.UToSymStr{
156+
var textEncodingMap = scalar.UintMapSymStr{
140157
textEncodingUTF8: "utf8",
141158
textEncodingUTF16LE: "utf16le",
142159
textEncodingUTF16BE: "utf16be",
143160
}
144161

145-
var versionMap = scalar.UToSymStr{
162+
var versionMap = scalar.UintMapSymStr{
146163
1: "legacy",
147164
2: "wal",
148165
}
@@ -164,13 +181,13 @@ func varintDecode(d *decode.D) int64 {
164181
break
165182
}
166183
}
167-
return mathextra.TwosComplement(64, n)
184+
return mathex.TwosComplement(64, n)
168185
}
169186

170187
func sqlite3DecodeSerialType(d *decode.D, h sqlite3Header, typ int64) {
171188
switch typ {
172189
case serialTypeNULL:
173-
d.FieldValueNil("value")
190+
d.FieldValueAny("value", nil)
174191
case serialTypeS8:
175192
d.FieldS8("value")
176193
case serialTypeSBE16:
@@ -186,9 +203,9 @@ func sqlite3DecodeSerialType(d *decode.D, h sqlite3Header, typ int64) {
186203
case serialTypeFloatBE64:
187204
d.FieldF64("value")
188205
case serialTypeInteger0:
189-
d.FieldValueU("value", 0)
206+
d.FieldValueAny("value", 0)
190207
case serialTypeInteger1:
191-
d.FieldValueU("value", 1)
208+
d.FieldValueAny("value", 1)
192209
case 10, 11:
193210
// internal, should not appear in wellformed file
194211
default:
@@ -228,15 +245,15 @@ func sqlite3DecodeCellFreeblock(d *decode.D) uint64 {
228245

229246
func sqlite3CellPayloadDecode(d *decode.D, h sqlite3Header) {
230247
lengthStart := d.Pos()
231-
length := d.FieldSFn("length", varintDecode)
248+
length := d.FieldSintFn("length", varintDecode)
232249
lengthBits := d.Pos() - lengthStart
233250
var serialTypes []int64
234251
d.FramedFn((length)*8-lengthBits, func(d *decode.D) {
235252
d.FieldArray("serials", func(d *decode.D) {
236253
for !d.End() {
237254
serialTypes = append(
238255
serialTypes,
239-
d.FieldSFn("serial", varintDecode, serialTypeMapper),
256+
d.FieldSintFn("serial", varintDecode, serialTypeMapper),
240257
)
241258
}
242259
})
@@ -276,18 +293,18 @@ func sqlite3DecodeTreePage(d *decode.D, h sqlite3Header, x int64, payLoadLen int
276293
d.FieldStruct("overflow_page", func(d *decode.D) {
277294
br := d.FieldRawLen("data", firstPayLoadLen*8)
278295
nextPage = d.FieldS32("next_page")
279-
d.MustCopyBits(payLoadBB, br)
296+
d.CopyBits(payLoadBB, br)
280297
})
281298

282299
payLoadLenLeft := payLoadLen - firstPayLoadLen
283300
for nextPage != 0 {
284301
d.SeekAbs(((nextPage - 1) * h.pageSize) * 8)
285302
d.FieldStruct("overflow_page", func(d *decode.D) {
286303
nextPage = d.FieldS32("next_page")
287-
overflowSize := mathextra.MinInt64(h.pageSize-4, payLoadLenLeft)
304+
overflowSize := mathex.Min(h.pageSize-4, payLoadLenLeft)
288305
br := d.FieldRawLen("data", overflowSize*8)
289306
payLoadLenLeft -= overflowSize
290-
d.MustCopyBits(payLoadBB, br)
307+
d.CopyBits(payLoadBB, br)
291308
})
292309
}
293310
})
@@ -299,26 +316,34 @@ func sqlite3DecodeTreePage(d *decode.D, h sqlite3Header, x int64, payLoadLen int
299316
}
300317
}
301318

319+
func sqlite3SeekPage(d *decode.D, h sqlite3Header, i int) {
320+
pageOffset := h.pageSize * int64(i)
321+
if i == 0 {
322+
pageOffset += sqlite3HeaderSize
323+
}
324+
d.SeekAbs(pageOffset * 8)
325+
}
326+
302327
func sqlite3Decode(d *decode.D, in interface{}) interface{} {
303328
var h sqlite3Header
304329

305330
d.FieldStruct("header", func(d *decode.D) {
306-
d.FieldUTF8("magic", 16, d.AssertStr("SQLite format 3\x00"))
307-
pageSizeS := d.FieldScalarU16("page_size", scalar.UToSymU{1: 65536}) // in bytes. Must be a power of two between 512 and 32768 inclusive, or the value 1 representing a page size of 65536.
308-
d.FieldU8("write_version", versionMap) // 1 for legacy; 2 for WAL.
309-
d.FieldU8("read_version", versionMap) // . 1 for legacy; 2 for WAL.
310-
d.FieldU8("unused_space") // at the end of each page. Usually 0.
311-
d.FieldU8("maximum_embedded_payload_fraction") // . Must be 64.
312-
d.FieldU8("minimum_embedded_payload_fraction") // . Must be 32.
313-
d.FieldU8("leaf_payload_fraction") // . Must be 32.
314-
d.FieldU32("file_change_counter") //
315-
databaseSizePages := int(d.FieldU32("database_size_pages")) // . The "in-header database size".
316-
d.FieldU32("page_number_freelist") // of the first freelist trunk page.
317-
d.FieldU32("total_number_freelist") // pages.
318-
d.FieldU32("schema_cookie") // .
319-
d.FieldU32("schema_format_number") // . Supported schema formats are 1, 2, 3, and 4.
320-
d.FieldU32("default_page_cache_size") // .
321-
d.FieldU32("page_number_largest_root_btree") // page when in auto-vacuum or incremental-vacuum modes, or zero otherwise.
331+
d.FieldUTF8("magic", 16, d.StrAssert("SQLite format 3\x00"))
332+
pageSizeS := d.FieldScalarU16("page_size", scalar.UintMapSymUint{1: 65536}) // in bytes. Must be a power of two between 512 and 32768 inclusive, or the value 1 representing a page size of 65536.
333+
d.FieldU8("write_version", versionMap) // 1 for legacy; 2 for WAL.
334+
d.FieldU8("read_version", versionMap) // . 1 for legacy; 2 for WAL.
335+
d.FieldU8("unused_space") // at the end of each page. Usually 0.
336+
d.FieldU8("maximum_embedded_payload_fraction") // . Must be 64.
337+
d.FieldU8("minimum_embedded_payload_fraction") // . Must be 32.
338+
d.FieldU8("leaf_payload_fraction") // . Must be 32.
339+
d.FieldU32("file_change_counter") //
340+
databaseSizePages := int(d.FieldU32("database_size_pages")) // . The "in-header database size".
341+
d.FieldU32("page_number_freelist") // of the first freelist trunk page.
342+
d.FieldU32("total_number_freelist") // pages.
343+
d.FieldU32("schema_cookie") // .
344+
d.FieldU32("schema_format_number") // . Supported schema formats are 1, 2, 3, and 4.
345+
d.FieldU32("default_page_cache_size") // .
346+
d.FieldU32("page_number_largest_root_btree") // page when in auto-vacuum or incremental-vacuum modes, or zero otherwise.
322347
textEncoding := int(d.FieldU32("text_encoding", textEncodingMap))
323348
d.FieldU32("user_version") // " as read and set by the user_version pragma.
324349
d.FieldU32("incremental_vacuum_mode") // False (zero) otherwise.
@@ -328,9 +353,9 @@ func sqlite3Decode(d *decode.D, in interface{}) interface{} {
328353
d.FieldU32("sqlite_version_number") //
329354

330355
// TODO: nicer API for fallback?
331-
pageSize := int64(pageSizeS.ActualU())
356+
pageSize := int64(pageSizeS.Actual)
332357
if pageSizeS.Sym != nil {
333-
pageSize = int64(pageSizeS.SymU())
358+
pageSize = int64(pageSizeS.SymUint())
334359
}
335360

336361
h = sqlite3Header{
@@ -340,21 +365,73 @@ func sqlite3Decode(d *decode.D, in interface{}) interface{} {
340365
}
341366
})
342367

343-
d.FieldArray("pages", func(d *decode.D) {
344-
d.RangeSorted = false
368+
// pageTypes := map[int]pageType{}
369+
// pageVisitStack := &intStack{}
370+
// pageVisitStack.Push(0)
371+
372+
// for {
373+
// i, ok := pageVisitStack.Pop()
374+
// if !ok {
375+
// break
376+
// }
377+
// if _, ok := pageTypes[i]; ok {
378+
// d.Fatalf("page %d already visited", i)
379+
// }
345380

381+
// sqlite3SeekPage(d, h, i)
382+
// typ := d.U8()
383+
384+
// switch typ {
385+
// case pageTypeBTreeIndexInterior,
386+
// pageTypeBTreeTableInterior:
387+
388+
// d.U16() // start_free_blocks
389+
// d.U16() // cell_start
390+
// d.U8() // cell_fragments
391+
// rightPointer := d.U32()
392+
393+
// pageCells := d.U16()
394+
// for i := uint64(0); i < pageCells; i++ {
395+
396+
// }
397+
398+
// switch typ {
399+
// case pageTypeBTreeIndexInterior:
400+
401+
// }
402+
403+
// default:
404+
// d.Fatalf("asd")
405+
// }
406+
407+
// }
408+
409+
// return nil
410+
411+
d.FieldArray("pages", func(d *decode.D) {
346412
// add a filler entry to make real pages start at index 1
347413
d.FieldStruct("page", func(d *decode.D) {
348414
d.FieldValueStr("type", "page0_index_fill")
349415
})
350416

417+
// for {
418+
// i, ok := pageStack.Pop()
419+
// if !ok {
420+
// break
421+
// }
422+
// if _, ok := pageSeen[i]; ok {
423+
// d.Fatalf("page %d already visited", i)
424+
// }
425+
// pageSeen[i] = struct{}{}
426+
351427
for i := 0; i < h.databaseSizePages; i++ {
352428
pageOffset := h.pageSize * int64(i)
353429
d.SeekAbs(pageOffset * 8)
354430
// skip header for first page
355431
if i == 0 {
356432
d.SeekRel(sqlite3HeaderSize * 8)
357433
}
434+
sqlite3SeekPage(d, h, i)
358435

359436
d.FieldStruct("page", func(d *decode.D) {
360437
typ := d.FieldU8("type", pageTypeMap)
@@ -413,18 +490,18 @@ func sqlite3Decode(d *decode.D, in interface{}) interface{} {
413490
switch typ {
414491
case pageTypeBTreeIndexInterior:
415492
d.FieldU32("left_child")
416-
payLoadLen := d.FieldSFn("payload_len", varintDecode)
493+
payLoadLen := d.FieldSintFn("payload_len", varintDecode)
417494
// formula for x from sqlite format spec
418495
sqlite3DecodeTreePage(d, h, ((h.pageSize-12)*64/255)-23, payLoadLen)
419496
case pageTypeBTreeTableInterior:
420497
d.FieldU32("left_child")
421-
d.FieldSFn("rowid", varintDecode)
498+
d.FieldSintFn("rowid", varintDecode)
422499
case pageTypeBTreeIndexLeaf:
423-
payLoadLen := d.FieldSFn("payload_len", varintDecode)
500+
payLoadLen := d.FieldSintFn("payload_len", varintDecode)
424501
sqlite3DecodeTreePage(d, h, ((h.pageSize-12)*64/255)-23, payLoadLen)
425502
case pageTypeBTreeTableLeaf:
426-
payLoadLen := d.FieldSFn("payload_len", varintDecode)
427-
d.FieldSFn("rowid", varintDecode)
503+
payLoadLen := d.FieldSintFn("payload_len", varintDecode)
504+
d.FieldSintFn("rowid", varintDecode)
428505
sqlite3DecodeTreePage(d, h, h.pageSize-35, payLoadLen)
429506
}
430507
})

‎format/sqlite3/testdata/multi_tables.sql‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,17 @@
11
CREATE TABLE aaa (
2-
cint int primary key,
3-
ctext text
2+
cint INT PRIMARY KEY,
3+
ctext TEXT
44
);
55
INSERT INTO aaa VALUES(123, "AAAAAAA");
66

77
CREATE TABLE bbb (
8-
cint int primary key,
9-
ctext text
8+
cint INT PRIMARY KEY,
9+
ctext TEXT
1010
);
1111
INSERT INTO bbb VALUES(456, "BBBBBBB");
1212

1313
CREATE TABLE ccc (
14-
cint int primary key,
15-
ctext text
14+
cint INT PRIMARY KEY,
15+
ctext TEXT
1616
);
1717
INSERT INTO ccc VALUES(789, "CCCCCCC");

‎format/sqlite3/testdata/multi_tables.sql.db‎

0 Bytes
Binary file not shown.

0 commit comments

Comments
 (0)