Skip to content

Commit d95f712

Browse files
klauspostklauspost
authored
Add github actions (#53)
* Add github actions * Skip tests on non-avx2 * Reset error mask * Remove amd64 tags for parse_number. Remove unused asm. * Update download URL * Print correct error on download failure. * Use smaller payload for NDJSON tests.
1 parent e885dec commit d95f712

24 files changed

+215
-1170
lines changed

‎.github/workflows/go.yml‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
name: Go
2+
3+
on:
4+
push:
5+
branches: [ master ]
6+
pull_request:
7+
branches: [ master ]
8+
9+
10+
jobs:
11+
build:
12+
strategy:
13+
matrix:
14+
go-version: [1.15.x, 1.16.x, 1.17.x]
15+
os: [ubuntu-latest, macos-latest, windows-latest]
16+
env:
17+
CGO_ENABLED: 0
18+
runs-on: ${{ matrix.os }}
19+
steps:
20+
- name: Set up Go
21+
uses: actions/setup-go@v2
22+
with:
23+
go-version: ${{ matrix.go-version }}
24+
25+
- name: Checkout code
26+
uses: actions/checkout@v2
27+
28+
- name: Test
29+
run: go test ./...
30+
31+
- name: Test Noasm
32+
run: go test -tags=noasm ./...
33+
34+
- name: Test Race
35+
env:
36+
CGO_ENABLED: 1
37+
run: go test -short -race ./...
38+
39+
build-special:
40+
env:
41+
CGO_ENABLED: 0
42+
runs-on: ubuntu-latest
43+
steps:
44+
- name: Set up Go
45+
uses: actions/setup-go@v2
46+
with:
47+
go-version: 1.17.x
48+
49+
- name: Checkout code
50+
uses: actions/checkout@v2
51+
52+
- name: fmt
53+
run: diff <(gofmt -d .) <(printf "")
54+
55+
- name: Vet
56+
run: go vet ./...
57+
58+
- name: Test 386
59+
run: GOOS=linux GOARCH=386 go test -short ./...

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -269,7 +269,7 @@ To serialize a block of parsed data use the [`Serialize`](https://pkg.go.dev/git
269269
To read back use the [`Deserialize`](https://pkg.go.dev/github.com/minio/simdjson-go#Serializer.Deserialize) method.
270270
For deserializing the compression mode does not need to match since it is read from the stream.
271271

272-
Example of speed for serializer/deserializer on [`parking-citations-1M`](https://files.klauspost.com/compress/parking-citations-1M.json.zst).
272+
Example of speed for serializer/deserializer on [`parking-citations-1M`](https://dl.minio.io/assets/parking-citations-1M.json.zst).
273273

274274
| Compress Mode | % of JSON size | Serialize Speed | Deserialize Speed |
275275
|---------------|----------------|-----------------|-------------------|

‎find_quote_mask_and_bits_amd64.s‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -89,16 +89,16 @@ TEXT ·_find_quote_mask_and_bits_avx512(SB), $0-48
8989
MOVQ odd_ends+8(FP), DX
9090
MOVQ prev_iter_inside_quote+16(FP), CX
9191

92-
KORQ K_ERRORMASK, K_ERRORMASK, K_ERRORMASK
92+
KXORQ K_ERRORMASK, K_ERRORMASK, K_ERRORMASK
9393

9494
VMOVDQU32 (DI), Z8
9595

9696
CALL ·__init_quote_mask_and_bits_avx512(SB)
9797
CALL ·__find_quote_mask_and_bits_avx512(SB)
9898

99-
VZEROUPPER
10099
KMOVQ K_ERRORMASK, error_mask+24(FP)
101100
KMOVQ K_QUOTEBITS, quote_bits+32(FP)
101+
VZEROUPPER
102102
MOVQ AX, quote_mask+40(FP)
103103
RET
104104

‎find_subroutines_amd64.go‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
1-
//+build !noasm
2-
//+build !appengine
3-
//+build gc
1+
//go:build !noasm && !appengine && gc
2+
// +build !noasm,!appengine,gc
43

54
/*
65
* MinIO Cloud Storage, (C) 2020 MinIO, Inc.

‎find_subroutines_amd64_test.go‎

Lines changed: 49 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
1-
//+build !noasm
2-
//+build !appengine
3-
//+build gc
1+
//go:build !noasm && !appengine && gc
2+
// +build !noasm,!appengine,gc
43

54
/*
65
* MinIO Cloud Storage, (C) 2020 MinIO, Inc.
@@ -31,7 +30,9 @@ import (
3130
)
3231

3332
func TestFinalizeStructurals(t *testing.T) {
34-
33+
if !SupportedCPU() {
34+
t.SkipNow()
35+
}
3536
testCases := []struct {
3637
structurals uint64
3738
whitespace uint64
@@ -92,6 +93,9 @@ func testFindNewlineDelimiters(t *testing.T, f func([]byte, uint64) uint64) {
9293
}
9394

9495
func TestFindNewlineDelimiters(t *testing.T) {
96+
if !SupportedCPU() {
97+
t.SkipNow()
98+
}
9599
t.Run("avx2", func(t *testing.T) {
96100
testFindNewlineDelimiters(t, _find_newline_delimiters)
97101
})
@@ -103,7 +107,9 @@ func TestFindNewlineDelimiters(t *testing.T) {
103107
}
104108

105109
func testExcludeNewlineDelimitersWithinQuotes(t *testing.T, f func([]byte, uint64) uint64) {
106-
110+
if !SupportedCPU() {
111+
t.SkipNow()
112+
}
107113
input := []byte(` "-------------------------------------" `)
108114
input[10] = 0x0a // within quoted string, so should be ignored
109115
input[50] = 0x0a // outside quoted string, so should be found
@@ -122,6 +128,9 @@ func testExcludeNewlineDelimitersWithinQuotes(t *testing.T, f func([]byte, uint6
122128
}
123129

124130
func TestExcludeNewlineDelimitersWithinQuotes(t *testing.T) {
131+
if !SupportedCPU() {
132+
t.SkipNow()
133+
}
125134
t.Run("avx2", func(t *testing.T) {
126135
testExcludeNewlineDelimitersWithinQuotes(t, _find_newline_delimiters)
127136
})
@@ -190,6 +199,9 @@ func testFindOddBackslashSequences(t *testing.T, f func([]byte, *uint64) uint64)
190199
}
191200

192201
func TestFindOddBackslashSequences(t *testing.T) {
202+
if !SupportedCPU() {
203+
t.SkipNow()
204+
}
193205
t.Run("avx2", func(t *testing.T) {
194206
testFindOddBackslashSequences(t, find_odd_backslash_sequences)
195207
})
@@ -291,6 +303,9 @@ func testFindQuoteMaskAndBits(t *testing.T, f func([]byte, uint64, *uint64, *uin
291303
}
292304

293305
func TestFindQuoteMaskAndBits(t *testing.T) {
306+
if !SupportedCPU() {
307+
t.SkipNow()
308+
}
294309
t.Run("avx2", func(t *testing.T) {
295310
testFindQuoteMaskAndBits(t, find_quote_mask_and_bits)
296311
})
@@ -346,6 +361,9 @@ func testFindStructuralBits(t *testing.T, f func([]byte, *uint64, *uint64, *uint
346361
}
347362

348363
func TestFindStructuralBits(t *testing.T) {
364+
if !SupportedCPU() {
365+
t.SkipNow()
366+
}
349367
t.Run("avx2", func(t *testing.T) {
350368
testFindStructuralBits(t, find_structural_bits)
351369
})
@@ -404,6 +422,9 @@ func testFindStructuralBitsWhitespacePadding(t *testing.T, f func([]byte, *uint6
404422
}
405423

406424
func TestFindStructuralBitsWhitespacePadding(t *testing.T) {
425+
if !SupportedCPU() {
426+
t.SkipNow()
427+
}
407428
t.Run("avx2", func(t *testing.T) {
408429
testFindStructuralBitsWhitespacePadding(t, find_structural_bits_in_slice)
409430
})
@@ -459,6 +480,9 @@ func testFindStructuralBitsLoop(t *testing.T, f func([]byte, *uint64, *uint64, *
459480
}
460481

461482
func TestFindStructuralBitsLoop(t *testing.T) {
483+
if !SupportedCPU() {
484+
t.SkipNow()
485+
}
462486
t.Run("avx2", func(t *testing.T) {
463487
testFindStructuralBitsLoop(t, find_structural_bits_in_slice)
464488
})
@@ -491,6 +515,9 @@ func benchmarkFindStructuralBits(b *testing.B, f func([]byte, *uint64, *uint64,
491515
}
492516

493517
func BenchmarkFindStructuralBits(b *testing.B) {
518+
if !SupportedCPU() {
519+
b.SkipNow()
520+
}
494521
b.Run("avx2", func(b *testing.B) {
495522
benchmarkFindStructuralBits(b, find_structural_bits)
496523
})
@@ -531,6 +558,9 @@ func benchmarkFindStructuralBitsLoop(b *testing.B, f func([]byte, *uint64, *uint
531558
}
532559

533560
func BenchmarkFindStructuralBitsLoop(b *testing.B) {
561+
if !SupportedCPU() {
562+
b.SkipNow()
563+
}
534564
b.Run("avx2", func(b *testing.B) {
535565
benchmarkFindStructuralBitsLoop(b, find_structural_bits_in_slice)
536566
})
@@ -578,6 +608,10 @@ func benchmarkFindStructuralBitsParallelLoop(b *testing.B, f func([]byte, *uint6
578608
}
579609

580610
func BenchmarkFindStructuralBitsParallelLoop(b *testing.B) {
611+
if !SupportedCPU() {
612+
b.SkipNow()
613+
}
614+
581615
b.Run("avx2", func(b *testing.B) {
582616
benchmarkFindStructuralBitsParallelLoop(b, find_structural_bits_in_slice)
583617
})
@@ -655,6 +689,10 @@ func testFindWhitespaceAndStructurals(t *testing.T, f func([]byte, *uint64, *uin
655689
}
656690

657691
func TestFindWhitespaceAndStructurals(t *testing.T) {
692+
if !SupportedCPU() {
693+
t.SkipNow()
694+
}
695+
658696
t.Run("avx2", func(t *testing.T) {
659697
testFindWhitespaceAndStructurals(t, find_whitespace_and_structurals)
660698
})
@@ -666,6 +704,9 @@ func TestFindWhitespaceAndStructurals(t *testing.T) {
666704
}
667705

668706
func TestFlattenBitsIncremental(t *testing.T) {
707+
if !SupportedCPU() {
708+
t.SkipNow()
709+
}
669710

670711
testCases := []struct {
671712
masks []uint64
@@ -732,6 +773,9 @@ func TestFlattenBitsIncremental(t *testing.T) {
732773
}
733774

734775
func BenchmarkFlattenBits(b *testing.B) {
776+
if !SupportedCPU() {
777+
b.SkipNow()
778+
}
735779

736780
msg := loadCompressed(b, "twitter")
737781

‎ndjson_test.go‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -254,13 +254,16 @@ func TestNdjsonCountWhere(t *testing.T) {
254254
if testing.Short() {
255255
t.Skip("skipping... too long")
256256
}
257-
ndjson := loadFile("testdata/parking-citations-1M.json.zst")
257+
ndjson := loadFile("testdata/parking-citations.json.zst")
258258
pj, err := ParseND(ndjson, nil)
259259
if err != nil {
260260
t.Fatal(err)
261261
}
262-
263-
const want = 110349
262+
ser := NewSerializer()
263+
ser.CompressMode(CompressBest)
264+
b := ser.Serialize(nil, *pj)
265+
t.Log(len(b))
266+
const want = 116
264267
if result := countWhere("Make", "HOND", *pj); result != want {
265268
t.Errorf("TestNdjsonCountWhere: got: %d want: %d", result, want)
266269
}
@@ -306,19 +309,21 @@ func loadFile(filename string) []byte {
306309
break
307310
}
308311
if os.IsNotExist(err) {
309-
fmt.Println("downloading file" + filename)
310-
resp, err := http.DefaultClient.Get("https://files.klauspost.com/compress/" + filepath.Base(filename))
312+
fmt.Println("downloading file", filename)
313+
resp, err := http.DefaultClient.Get("https://dl.minio.io/assets/" + filepath.Base(filename))
311314
if err == nil && resp.StatusCode == http.StatusOK {
312315
b, err := ioutil.ReadAll(resp.Body)
313316
if err == nil {
314317
err = ioutil.WriteFile(filename, b, os.ModePerm)
315318
if err == nil {
316319
continue
317320
}
321+
panic("Failed to write file:" + err.Error())
318322
}
323+
panic("Failed to read file:" + err.Error())
319324
}
325+
panic("Failed to download file:" + err.Error())
320326
}
321-
panic("Failed to (down)load file:" + err.Error())
322327
}
323328
dec, err := zstd.NewReader(f)
324329
if err != nil {

‎parse_json_amd64.go‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
1-
//+build !noasm
2-
//+build !appengine
3-
//+build gc
1+
//go:build !noasm && !appengine && gc
2+
// +build !noasm,!appengine,gc
43

54
/*
65
* MinIO Cloud Storage, (C) 2020 MinIO, Inc.

0 commit comments

Comments
 (0)