Skip to content

Commit 3d975b7

Browse files
klauspostklauspost
authored
Clean up exported funcs (#29)
Unexport useless consts Rename most consts and funcs to not have underscores.
1 parent 4009cbf commit 3d975b7

11 files changed

+127
-118
lines changed

‎find_subroutines_amd64.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,7 @@ func _find_structural_bits_in_slice(buf unsafe.Pointer, len uint64, p3 unsafe.Po
152152
func find_structural_bits_in_slice(buf []byte, prev_iter_ends_odd_backslash *uint64,
153153
prev_iter_inside_quote, error_mask *uint64,
154154
prev_iter_ends_pseudo_pred *uint64,
155-
indexes *[INDEX_SIZE]uint32, index *int, carried *uint64, position *uint64,
155+
indexes *[indexSize]uint32, index *int, carried *uint64, position *uint64,
156156
ndjson uint64) (processed uint64) {
157157

158158
if len(buf) == 0 {
@@ -167,7 +167,7 @@ func find_structural_bits_in_slice(buf []byte, prev_iter_ends_odd_backslash *uin
167167
unsafe.Pointer(prev_iter_inside_quote), unsafe.Pointer(&quote_bits), unsafe.Pointer(error_mask),
168168
unsafe.Pointer(&whitespace), unsafe.Pointer(&structurals),
169169
unsafe.Pointer(prev_iter_ends_pseudo_pred),
170-
unsafe.Pointer(&(*indexes)[0]), unsafe.Pointer(index), INDEX_SIZE_WITH_SAFETY_BUFFER,
170+
unsafe.Pointer(&(*indexes)[0]), unsafe.Pointer(index), indexSizeWithSafetyBuffer,
171171
unsafe.Pointer(carried), unsafe.Pointer(position),
172172
ndjson)
173173
}
@@ -183,7 +183,7 @@ func _find_structural_bits_in_slice_avx512(buf unsafe.Pointer, len uint64, p3 un
183183
func find_structural_bits_in_slice_avx512(buf []byte, prev_iter_ends_odd_backslash *uint64,
184184
prev_iter_inside_quote, error_mask *uint64,
185185
prev_iter_ends_pseudo_pred *uint64,
186-
indexes *[INDEX_SIZE]uint32, index *int, carried *uint64, position *uint64,
186+
indexes *[indexSize]uint32, index *int, carried *uint64, position *uint64,
187187
ndjson uint64) (processed uint64) {
188188

189189
if len(buf) == 0 {
@@ -193,7 +193,7 @@ func find_structural_bits_in_slice_avx512(buf []byte, prev_iter_ends_odd_backsla
193193
return _find_structural_bits_in_slice_avx512(unsafe.Pointer(&buf[0]), uint64(len(buf)), unsafe.Pointer(prev_iter_ends_odd_backslash),
194194
unsafe.Pointer(prev_iter_inside_quote), unsafe.Pointer(error_mask),
195195
unsafe.Pointer(prev_iter_ends_pseudo_pred),
196-
unsafe.Pointer(&(*indexes)[0]), unsafe.Pointer(index), INDEX_SIZE_WITH_SAFETY_BUFFER,
196+
unsafe.Pointer(&(*indexes)[0]), unsafe.Pointer(index), indexSizeWithSafetyBuffer,
197197
unsafe.Pointer(carried), unsafe.Pointer(position),
198198
ndjson)
199199
}
@@ -227,6 +227,6 @@ func __flatten_bits_incremental()
227227
//go:noescape
228228
func _flatten_bits_incremental(base_ptr, pbase unsafe.Pointer, mask uint64, carried unsafe.Pointer, position unsafe.Pointer)
229229

230-
func flatten_bits_incremental(base *[INDEX_SIZE]uint32, base_index *int, mask uint64, carried *int, position *uint64) {
230+
func flatten_bits_incremental(base *[indexSize]uint32, base_index *int, mask uint64, carried *int, position *uint64) {
231231
_flatten_bits_incremental(unsafe.Pointer(&(*base)[0]), unsafe.Pointer(base_index), mask, unsafe.Pointer(carried), unsafe.Pointer(position))
232232
}

‎find_subroutines_amd64_test.go‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -356,7 +356,7 @@ func TestFindStructuralBits(t *testing.T) {
356356
}
357357
}
358358

359-
func testFindStructuralBitsWhitespacePadding(t *testing.T, f func([]byte, *uint64, *uint64, *uint64, *uint64, *[INDEX_SIZE]uint32, *int, *uint64, *uint64, uint64) uint64) {
359+
func testFindStructuralBitsWhitespacePadding(t *testing.T, f func([]byte, *uint64, *uint64, *uint64, *uint64, *[indexSize]uint32, *int, *uint64, *uint64, uint64) uint64) {
360360

361361
// Test whitespace padding (for partial load of last 64 bytes) with
362362
// string full of structural characters
@@ -372,7 +372,7 @@ func testFindStructuralBitsWhitespacePadding(t *testing.T, f func([]byte, *uint6
372372
position := ^uint64(0)
373373

374374
index := indexChan{}
375-
index.indexes = &[INDEX_SIZE]uint32{}
375+
index.indexes = &[indexSize]uint32{}
376376

377377
processed := find_structural_bits_in_slice([]byte(msg[:l]), &prev_iter_ends_odd_backslash,
378378
&prev_iter_inside_quote, &error_mask,
@@ -414,7 +414,7 @@ func TestFindStructuralBitsWhitespacePadding(t *testing.T) {
414414
}
415415
}
416416

417-
func testFindStructuralBitsLoop(t *testing.T, f func([]byte, *uint64, *uint64, *uint64, *uint64, *[INDEX_SIZE]uint32, *int, *uint64, *uint64, uint64) uint64) {
417+
func testFindStructuralBitsLoop(t *testing.T, f func([]byte, *uint64, *uint64, *uint64, *uint64, *[indexSize]uint32, *int, *uint64, *uint64, uint64) uint64) {
418418
msg := loadCompressed(t, "twitter")
419419

420420
prev_iter_ends_odd_backslash := uint64(0)
@@ -428,7 +428,7 @@ func testFindStructuralBitsLoop(t *testing.T, f func([]byte, *uint64, *uint64, *
428428

429429
for processed := uint64(0); processed < uint64(len(msg)); {
430430
index := indexChan{}
431-
index.indexes = &[INDEX_SIZE]uint32{}
431+
index.indexes = &[indexSize]uint32{}
432432

433433
processed += f(msg[processed:], &prev_iter_ends_odd_backslash,
434434
&prev_iter_inside_quote, &error_mask,
@@ -501,7 +501,7 @@ func BenchmarkFindStructuralBits(b *testing.B) {
501501
}
502502
}
503503

504-
func benchmarkFindStructuralBitsLoop(b *testing.B, f func([]byte, *uint64, *uint64, *uint64, *uint64, *[INDEX_SIZE]uint32, *int, *uint64, *uint64, uint64) uint64) {
504+
func benchmarkFindStructuralBitsLoop(b *testing.B, f func([]byte, *uint64, *uint64, *uint64, *uint64, *[indexSize]uint32, *int, *uint64, *uint64, uint64) uint64) {
505505

506506
msg := loadCompressed(b, "twitter")
507507

@@ -520,7 +520,7 @@ func benchmarkFindStructuralBitsLoop(b *testing.B, f func([]byte, *uint64, *uint
520520

521521
for processed := uint64(0); processed < uint64(len(msg)); {
522522
index := indexChan{}
523-
index.indexes = &[INDEX_SIZE]uint32{}
523+
index.indexes = &[indexSize]uint32{}
524524

525525
processed += f(msg[processed:], &prev_iter_ends_odd_backslash,
526526
&prev_iter_inside_quote, &error_mask,
@@ -541,7 +541,7 @@ func BenchmarkFindStructuralBitsLoop(b *testing.B) {
541541
}
542542
}
543543

544-
func benchmarkFindStructuralBitsParallelLoop(b *testing.B, f func([]byte, *uint64, *uint64, *uint64, *uint64, *[INDEX_SIZE]uint32, *int, *uint64, *uint64, uint64) uint64) {
544+
func benchmarkFindStructuralBitsParallelLoop(b *testing.B, f func([]byte, *uint64, *uint64, *uint64, *uint64, *[indexSize]uint32, *int, *uint64, *uint64, uint64) uint64) {
545545

546546
msg := loadCompressed(b, "twitter")
547547
cpus := runtime.NumCPU()
@@ -563,7 +563,7 @@ func benchmarkFindStructuralBitsParallelLoop(b *testing.B, f func([]byte, *uint6
563563

564564
for processed := uint64(0); processed < uint64(len(msg)); {
565565
index := indexChan{}
566-
index.indexes = &[INDEX_SIZE]uint32{}
566+
index.indexes = &[indexSize]uint32{}
567567

568568
processed += f(msg[processed:], &prev_iter_ends_odd_backslash,
569569
&prev_iter_inside_quote, &error_mask,
@@ -708,7 +708,7 @@ func TestFlattenBitsIncremental(t *testing.T) {
708708
for i, tc := range testCases {
709709

710710
index := indexChan{}
711-
index.indexes = &[INDEX_SIZE]uint32{}
711+
index.indexes = &[indexSize]uint32{}
712712
carried := 0
713713
position := ^uint64(0)
714714

@@ -758,7 +758,7 @@ func BenchmarkFlattenBits(b *testing.B) {
758758
b.ResetTimer()
759759

760760
index := indexChan{}
761-
index.indexes = &[INDEX_SIZE]uint32{}
761+
index.indexes = &[indexSize]uint32{}
762762
carried := 0
763763
position := ^uint64(0)
764764

‎ndjson_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -241,7 +241,7 @@ func verifyDemoNdjson(pj internalParsedJson, t *testing.T, object int) {
241241
//}
242242
//fmt.Printf("{%s, 0x%x},\n", c, tp&0xffffffffffffff)
243243
expected := tc.expected[ii].val | (uint64(tc.expected[ii].c) << 56)
244-
if !ALWAYS_COPY_STRINGS && tp != expected {
244+
if !alwaysCopyStrings && tp != expected {
245245
t.Errorf("verifyDemoNdjson(%d): got: %016x want: %016x", ii, tp, expected)
246246
}
247247
}

‎parse_json_amd64.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,8 @@ func (pj *internalParsedJson) initialize(size int) {
4242
pj.Strings = make([]byte, 0, stringsSize)
4343
}
4444
pj.Strings = pj.Strings[:0]
45-
if cap(pj.containing_scope_offset) < DEFAULTDEPTH {
46-
pj.containing_scope_offset = make([]uint64, 0, DEFAULTDEPTH)
45+
if cap(pj.containing_scope_offset) < maxdepth {
46+
pj.containing_scope_offset = make([]uint64, 0, maxdepth)
4747
}
4848
pj.containing_scope_offset = pj.containing_scope_offset[:0]
4949
}
@@ -75,18 +75,18 @@ func (pj *internalParsedJson) parseMessageInternal(msg []byte, ndjson bool) (err
7575
// Make the capacity of the channel smaller than the number of slots.
7676
// This way the sender will automatically block until the consumer
7777
// has finished the slot it is working on.
78-
pj.index_chan = make(chan indexChan, INDEX_SLOTS-2)
78+
pj.index_chan = make(chan indexChan, indexSlots-2)
7979
pj.buffers_offset = ^uint64(0)
8080

8181
var errStage1 error
8282
go func() {
83-
if !find_structural_indices(pj.Message, pj) {
83+
if !findStructuralIndices(pj.Message, pj) {
8484
errStage1 = errors.New("Failed to find all structural indices for stage 1")
8585
}
8686
wg.Done()
8787
}()
8888
go func() {
89-
if !unified_machine(pj.Message, pj) {
89+
if !unifiedMachine(pj.Message, pj) {
9090
err = errors.New("Bad parsing while executing stage 2")
9191
// drain the channel until empty
9292
for range pj.index_chan {

‎parse_json_amd64_test.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ func BenchmarkNdjsonStage1(b *testing.B) {
9797
for i := 0; i < b.N; i++ {
9898
// Create new channel (large enough so we won't block)
9999
pj.index_chan = make(chan indexChan, 128*10240)
100-
find_structural_indices([]byte(ndjson), &pj)
100+
findStructuralIndices([]byte(ndjson), &pj)
101101
}
102102
}
103103

@@ -514,9 +514,9 @@ func TestParseString(t *testing.T) {
514514
t.Run(tt.name, func(t *testing.T) {
515515
// enclose test string in quotes (as validated by stage 1)
516516
buf := []byte(fmt.Sprintf(`"%s"`, tt.str))
517-
dest := make([]byte, 0, len(buf)+32 /* safety margin as parse_string writes full AVX2 words */)
517+
dest := make([]byte, 0, len(buf)+32 /* safety margin as parseString writes full AVX2 words */)
518518

519-
success := parse_string_simd(buf, &dest)
519+
success := parseStringSimd(buf, &dest)
520520

521521
if success != tt.success {
522522
t.Errorf("TestParseString() got = %v, want %v", success, tt.success)
@@ -544,7 +544,7 @@ func TestParseStringValidateOnly(t *testing.T) {
544544
dst_length := uint64(0)
545545
need_copy := false
546546
l := uint64(len(buf))
547-
success := parse_string_simd_validate_only(buf, &l, &dst_length, &need_copy)
547+
success := parseStringSimdValidateOnly(buf, &l, &dst_length, &need_copy)
548548

549549
if success != tt.success {
550550
t.Errorf("TestParseString() got = %v, want %v", success, tt.success)
@@ -567,7 +567,7 @@ func TestParseStringValidateOnlyBeyondBuffer(t *testing.T) {
567567
dst_length := uint64(0)
568568
need_copy := false
569569
l := uint64(len(buf)) + 32
570-
success := parse_string_simd_validate_only(buf, &l, &dst_length, &need_copy)
570+
success := parseStringSimdValidateOnly(buf, &l, &dst_length, &need_copy)
571571
if !success {
572572
t.Errorf("TestParseStringValidateOnlyBeyondBuffer() got = %v, want %v", success, false)
573573
}

‎parse_string_amd64.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,20 +33,20 @@ func _parse_string(src, dst, pcurrent_string_buf_loc unsafe.Pointer) (res uint64
3333

3434
// Disable new -d=checkptr behaviour for Go 1.14
3535
//go:nocheckptr
36-
func parse_string_simd_validate_only(buf []byte, maxStringSize, dst_length *uint64, need_copy *bool) bool {
36+
func parseStringSimdValidateOnly(buf []byte, maxStringSize, dst_length *uint64, need_copy *bool) bool {
3737

3838
src := uintptr(unsafe.Pointer(&buf[1])) // Use buf[1] in order to skip opening quote
3939
src_length := uint64(0)
4040

4141
success := _parse_string_validate_only(unsafe.Pointer(src), unsafe.Pointer(&maxStringSize), unsafe.Pointer(&src_length), unsafe.Pointer(dst_length))
4242

43-
*need_copy = ALWAYS_COPY_STRINGS || src_length != *dst_length
43+
*need_copy = alwaysCopyStrings || src_length != *dst_length
4444
return success != 0
4545
}
4646

4747
// Disable new -d=checkptr behaviour for Go 1.14
4848
//go:nocheckptr
49-
func parse_string_simd(buf []byte, stringbuf *[]byte) bool {
49+
func parseStringSimd(buf []byte, stringbuf *[]byte) bool {
5050

5151
sh := (*reflect.SliceHeader)(unsafe.Pointer(stringbuf))
5252

‎parsed_json.go‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -33,14 +33,14 @@ import (
3333
// strings (not just those transformed anyway for unicode escape characters) into the separate
3434
// Strings buffer (at the expense of using more memory and less performance).
3535
//
36-
const ALWAYS_COPY_STRINGS = true
36+
const alwaysCopyStrings = true
3737

3838
const JSONVALUEMASK = 0xffffffffffffff
3939
const JSONTAGMASK = 0xff << 56
4040
const STRINGBUFBIT = 0x80000000000000
4141
const STRINGBUFMASK = 0x7fffffffffffff
4242

43-
const DEFAULTDEPTH = 128
43+
const maxdepth = 128
4444

4545
type ParsedJson struct {
4646
Message []byte
@@ -51,14 +51,14 @@ type ParsedJson struct {
5151
internal *internalParsedJson
5252
}
5353

54-
const INDEX_SLOTS = 16
55-
const INDEX_SIZE = 1536 // Seems to be a good size for the index buffering
56-
const INDEX_SIZE_WITH_SAFETY_BUFFER = INDEX_SIZE - 128 // Make sure we never write beyond buffer
54+
const indexSlots = 16
55+
const indexSize = 1536 // Seems to be a good size for the index buffering
56+
const indexSizeWithSafetyBuffer = indexSize - 128 // Make sure we never write beyond buffer
5757

5858
type indexChan struct {
5959
index int
6060
length int
61-
indexes *[INDEX_SIZE]uint32
61+
indexes *[indexSize]uint32
6262
}
6363

6464
type internalParsedJson struct {
@@ -67,7 +67,7 @@ type internalParsedJson struct {
6767
isvalid bool
6868
index_chan chan indexChan
6969
indexesChan indexChan
70-
buffers [INDEX_SLOTS][INDEX_SIZE]uint32
70+
buffers [indexSlots][indexSize]uint32
7171
buffers_offset uint64
7272
ndjson uint64
7373
}

‎stage1_find_marks_amd64.go‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,20 @@ import (
2626
"github.com/klauspost/cpuid/v2"
2727
)
2828

29-
func json_markup(b byte) bool {
30-
return b == '{' || b == '}' || b == '[' || b == ']' || b == ',' || b == ':'
29+
var jsonMarkupTable = [256]bool{
30+
'{': true,
31+
'}': true,
32+
'[': true,
33+
']': true,
34+
',': true,
35+
':': true,
3136
}
3237

33-
func find_structural_indices(buf []byte, pj *internalParsedJson) bool {
38+
func jsonMarkup(b byte) bool {
39+
return jsonMarkupTable[b]
40+
}
41+
42+
func findStructuralIndices(buf []byte, pj *internalParsedJson) bool {
3443

3544
f := find_structural_bits_in_slice
3645
if cpuid.CPU.Has(cpuid.AVX512F) {
@@ -66,7 +75,7 @@ func find_structural_indices(buf []byte, pj *internalParsedJson) bool {
6675

6776
index := indexChan{}
6877
offset := atomic.AddUint64(&pj.buffers_offset, 1)
69-
index.indexes = &pj.buffers[offset%INDEX_SLOTS]
78+
index.indexes = &pj.buffers[offset%indexSlots]
7079

7180
// In case last index during previous round was stripped back, put it back
7281
if stripped_index != ^uint64(0) {
@@ -108,7 +117,7 @@ func find_structural_indices(buf []byte, pj *internalParsedJson) bool {
108117
error_mask = ^uint64(0)
109118
break
110119
}
111-
} else if !json_markup(buf[position]) {
120+
} else if !jsonMarkup(buf[position]) {
112121
// There may be a dangling quote at the end of the index buffer
113122
// Strip it from current index buffer and save for next round
114123
stripped_index = uint64(index.indexes[index.length-1])

‎stage1_find_marks_amd64_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,7 @@ func TestFindStructuralIndices(t *testing.T) {
141141
pj.index_chan = make(chan indexChan, 16)
142142

143143
// No need to spawn go-routine since the channel is large enough
144-
find_structural_indices([]byte(demo_json), &pj)
144+
findStructuralIndices([]byte(demo_json), &pj)
145145

146146
ipos, pos := 0, ^uint64(0)
147147
for ic := range pj.index_chan {
@@ -169,6 +169,6 @@ func BenchmarkStage1(b *testing.B) {
169169
for i := 0; i < b.N; i++ {
170170
// Create new channel (large enough so we won't block)
171171
pj.index_chan = make(chan indexChan, 128)
172-
find_structural_indices([]byte(msg), &pj)
172+
findStructuralIndices([]byte(msg), &pj)
173173
}
174174
}

0 commit comments

Comments
 (0)