Skip to content

Commit e2a7a32

Browse files
swinkelhoferswinkelhofer
committed
Initial commit
0 parents  commit e2a7a32

File tree

13 files changed

+1173
-0
lines changed

13 files changed

+1173
-0
lines changed

‎.github/workflows/go.yml‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
name: Go
2+
3+
on:
4+
push:
5+
tags:
6+
- '*'
7+
8+
jobs:
9+
build:
10+
runs-on: ubuntu-latest
11+
steps:
12+
- uses: actions/checkout@v2
13+
14+
- name: Get the version
15+
id: release_name
16+
run: echo ::set-output name=VERSION::$(echo $GITHUB_REF | cut -d / -f 3)
17+
18+
- name: Set up Go
19+
uses: actions/setup-go@v2
20+
with:
21+
go-version: 1.17
22+
23+
- name: Test
24+
run: go test -v ./...
25+
26+
- name: Create Release
27+
id: create_release
28+
uses: actions/create-release@v1
29+
env:
30+
GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
31+
with:
32+
tag_name: ${{ steps.release_name.outputs.VERSION }}
33+
release_name: Release ${{ steps.release_name.outputs.VERSION }}
34+
draft: false
35+
prerelease: false
36+
37+
- name: Push
38+
run: GOPROXY=proxy.golang.org go list -m github.com/arivum/json2msgpackStreamer@${{ steps.release_name.outputs.VERSION }}

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
*.swp
2+
/cmd

‎README.md‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
# json2msgpackStreamer
2+
3+
This module converts a (ND)JSON stream to a messagepack stream on-the-fly.
4+
5+
## How to use
6+
7+
```go
8+
package main
9+
10+
import (
11+
"bytes"
12+
"encoding/json"
13+
"fmt"
14+
15+
"github.com/arivum/json2msgpackStreamer"
16+
"github.com/vmihailenco/msgpack/v5"
17+
)
18+
19+
func main() {
20+
var rawJSON = map[string]interface{}{
21+
"a": 0,
22+
"b": true,
23+
"c": []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"},
24+
"d": map[string]interface{}{
25+
"d_a": 1,
26+
"d_b": "teststring",
27+
},
28+
"e": float32(1.2),
29+
"f": float64(2),
30+
"g": 0x5a,
31+
"h": nil,
32+
"i": -(1 << 33),
33+
"j": true,
34+
"k": []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p"},
35+
"l": map[string]interface{}{
36+
"d_a": 1,
37+
"d_b": "teststring",
38+
},
39+
"m": float32(1.2),
40+
"n": float64(2),
41+
"o": 0x5a,
42+
"p": nil,
43+
}
44+
var rawJSON2 = map[string]interface{}{
45+
"a": 0,
46+
"b": true,
47+
"c": []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p"},
48+
"d": map[string]interface{}{
49+
"d_a": 1,
50+
"d_b": "teststring",
51+
},
52+
}
53+
54+
// Create the JSON buffer that needs to be converted to a msgpack stream
55+
var jsonObj, _ = json.Marshal(rawJSON)
56+
var jsonObj2, _ = json.Marshal(rawJSON2)
57+
jsonObj = append(jsonObj, jsonObj...)
58+
jsonObj = append(jsonObj, jsonObj2...)
59+
jsonObj = append(jsonObj, jsonObj...)
60+
jsonObj = append(jsonObj, jsonObj2...)
61+
fmt.Println(string(jsonObj))
62+
63+
// Create the converter:
64+
// JSON2MsgPackStreamer is of type io.Reader and can be passed to e.g. "github.com/vmihailenco/msgpack/v5".NewDecoder(r io.Reader)
65+
var conv = json2msgpackStreamer.NewJSON2MsgPackStreamer(bytes.NewBuffer(jsonObj))
66+
67+
// Create the msgpack decoder that reads from the converter
68+
msgpackDec := msgpack.NewDecoder(conv)
69+
70+
i := 0
71+
for {
72+
var entry interface{}
73+
74+
if err := msgpackDec.Decode(&entry); err != nil {
75+
break
76+
}
77+
fmt.Printf("Entry #%d: %+v\n", i, entry)
78+
i++
79+
}
80+
}
81+
```

‎blockbuf.go‎

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
package json2msgpackStreamer
2+
3+
import (
4+
"io"
5+
"sort"
6+
)
7+
8+
func newBlockBuf() *blockbuf {
9+
var first = newBlock()
10+
11+
return &blockbuf{
12+
first: first,
13+
current: first,
14+
}
15+
}
16+
17+
func (b *blockbuf) reset() {
18+
b.first.index = 0
19+
b.first.next = nil
20+
}
21+
22+
func (b *blockbuf) getCurrentPos() *blockBufPos {
23+
return &blockBufPos{
24+
block: b.current,
25+
index: b.current.index,
26+
}
27+
}
28+
29+
func (b *blockbuf) append(data []byte) {
30+
var (
31+
dLen = len(data)
32+
lower = 0
33+
copylen = 0
34+
newblock *block
35+
)
36+
37+
for dLen > 0 {
38+
if b.current.index+dLen < bufsize {
39+
copylen = dLen
40+
} else {
41+
copylen = bufsize - b.current.index
42+
}
43+
copy(b.current.buf[b.current.index:], data[lower:lower+copylen])
44+
b.current.index += copylen
45+
lower += copylen
46+
dLen -= copylen
47+
if b.current.index == bufsize {
48+
newblock = newBlock()
49+
b.current.next = newblock
50+
b.current = newblock
51+
}
52+
}
53+
}
54+
55+
func (b *blockbuf) appendByte(data byte) {
56+
b.current.buf[b.current.index] = data
57+
b.current.index++
58+
if b.current.index == bufsize {
59+
newblock := newBlock()
60+
b.current.next = newblock
61+
b.current = newblock
62+
}
63+
}
64+
65+
func (b *blockbuf) writeToOffset(data []byte, pos *blockBufPos) {
66+
var (
67+
dLen = len(data)
68+
lower = 0
69+
copylen = 0
70+
)
71+
72+
for dLen > 0 {
73+
if pos.index+dLen < bufsize {
74+
copylen = dLen
75+
} else {
76+
copylen = bufsize - pos.index
77+
}
78+
copy(pos.block.buf[pos.index:], data[lower:lower+copylen])
79+
pos.index += copylen
80+
lower += copylen
81+
dLen -= copylen
82+
if pos.index == bufsize {
83+
pos.block = pos.block.next
84+
pos.index = 0
85+
}
86+
}
87+
}
88+
89+
func (b *blockbuf) writeByteToOffset(data byte, pos *blockBufPos) {
90+
pos.block.buf[pos.index] = data
91+
}
92+
93+
func (b *blockbuf) flushToWriter(w io.Writer) error {
94+
var (
95+
iter = b.first
96+
err error
97+
insert *blockInsertion
98+
index int
99+
)
100+
101+
for {
102+
index = 0
103+
for _, insert = range iter.insertions {
104+
if _, err = w.Write(iter.buf[index:insert.pos]); err != nil {
105+
return err
106+
}
107+
if _, err = w.Write(insert.buf); err != nil {
108+
return err
109+
}
110+
index = insert.pos
111+
}
112+
113+
if index < iter.index {
114+
if _, err = w.Write(iter.buf[index:iter.index]); err != nil {
115+
return err
116+
}
117+
}
118+
119+
if iter.next == nil {
120+
break
121+
}
122+
iter = iter.next
123+
}
124+
return nil
125+
}
126+
127+
func (b *blockbuf) insertOnOffset(data []byte, pos *blockBufPos) {
128+
var index = sort.Search(len(pos.block.insertions), func(i int) bool { return pos.block.insertions[i].pos >= pos.index })
129+
130+
if index < len(pos.block.insertions) {
131+
pos.block.insertions = append(pos.block.insertions[:index+1], pos.block.insertions[index:]...)
132+
pos.block.insertions[index] = newInsertion(data, pos.index)
133+
} else {
134+
pos.block.insertions = append(pos.block.insertions, newInsertion(data, pos.index))
135+
}
136+
}
137+
138+
// func (b *blockbuf) getBytes() []byte {
139+
// var (
140+
// iter = b.first
141+
// out = make([]byte, 0)
142+
// )
143+
144+
// for {
145+
// out = append(out, iter.buf[:iter.index]...)
146+
// if iter.next == nil {
147+
// break
148+
// } else {
149+
// iter = iter.next
150+
// }
151+
// }
152+
// return out
153+
// }
154+
155+
// func (b *blockbuf) Print() {
156+
// var iter = b.first
157+
// for {
158+
// fmt.Printf("%+v\n", iter.buf)
159+
// if iter.next == nil {
160+
// break
161+
// } else {
162+
// iter = iter.next
163+
// }
164+
// }
165+
// }
166+
167+
func newBlock() *block {
168+
return &block{
169+
buf: make([]byte, bufsize),
170+
next: nil,
171+
index: 0,
172+
insertions: make([]*blockInsertion, 0),
173+
}
174+
}
175+
176+
func newInsertion(insertion []byte, pos int) *blockInsertion {
177+
var buf = make([]byte, len(insertion))
178+
copy(buf, insertion)
179+
return &blockInsertion{
180+
buf: insertion,
181+
pos: pos,
182+
}
183+
}

‎globals.go‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
package json2msgpackStreamer
2+
3+
var (
4+
valueNil = []byte("null")
5+
valueTrue = []byte("true")
6+
valueFalse = []byte("false")
7+
valueDot = []byte(".")
8+
buf2Bytes = make([]byte, 2)
9+
buf4Bytes = make([]byte, 4)
10+
buf8Bytes = make([]byte, 8)
11+
placeholder32Bit = make([]byte, 4)
12+
lenBuf16 = make([]byte, 2)
13+
lenBuf32 = make([]byte, 4)
14+
)
15+
16+
const (
17+
bufsize = 4 << 10
18+
msgPackFlagFixStr = byte(0xA0)
19+
msgPackFlagStr8 = byte(0xd9)
20+
msgPackFlagStr16 = byte(0xda)
21+
msgPackFlagStr32 = byte(0xdb)
22+
msgPackFlagFixArray = byte(0x90)
23+
msgPackFlagArray16 = byte(0xdc)
24+
msgPackFlagArray32 = byte(0xdd)
25+
msgPackFlagFixMap = byte(0x80)
26+
msgPackFlagMap16 = byte(0xde)
27+
msgPackFlagMap32 = byte(0xdf)
28+
msgPackFlagFloat32 = byte(0xca)
29+
msgPackFlagFloat64 = byte(0xcb)
30+
msgPackFlagPosFixInt = byte(0x00)
31+
msgPackFlagNegFixInt = byte(0xe0)
32+
msgPackFlagInt8 = byte(0xd0)
33+
msgPackFlagInt16 = byte(0xd1)
34+
msgPackFlagInt32 = byte(0xd2)
35+
msgPackFlagInt64 = byte(0xd3)
36+
msgPackFlagUint8 = byte(0xcc)
37+
msgPackFlagUint16 = byte(0xcd)
38+
msgPackFlagUint32 = byte(0xce)
39+
msgPackFlagUint64 = byte(0xcf)
40+
msgPackFlagNil = byte(0xc0)
41+
msgPackFlagBoolFalse = byte(0xc2)
42+
msgPackFlagBoolTrue = byte(0xc3)
43+
max8BitPlusOne = (1 << 8)
44+
max16BitPlusOne = (1 << 16)
45+
max32BitPlusOne = (1 << 32)
46+
max5BitPlusOne = (1 << 5)
47+
max4BitPlusOne = (1 << 4)
48+
)

‎go.mod‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
module github.com/arivum/json2msgpackStreamer
2+
3+
go 1.17
4+
5+
require github.com/vmihailenco/msgpack/v5 v5.3.5
6+
7+
require github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect

‎go.sum‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
2+
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
3+
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
4+
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
5+
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
6+
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
7+
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
8+
github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU=
9+
github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
10+
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
11+
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
12+
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
13+
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
14+
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

0 commit comments

Comments
 (0)