Skip to content

Commit a6d9605

Browse files
taubyte0samyfodil
andauthored
[starlark] optimizations #391
Co-authored-by: samyfodil <fodil.samy@gmail.com>
1 parent e6e8a52 commit a6d9605

File tree

10 files changed

+562
-62
lines changed

10 files changed

+562
-62
lines changed

go.mod

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ require (
8888
github.com/unrolled/secure v1.0.9
8989
github.com/vbauerster/mpb/v8 v8.7.5
9090
github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7
91-
go.starlark.net v0.0.0-20240520160348-046347dcd104
91+
go.starlark.net v0.0.0-20251109183026-be02852a5e1f
9292
golang.org/x/mod v0.26.0
9393
golang.org/x/text v0.27.0
9494
google.golang.org/grpc v1.73.0
@@ -114,6 +114,7 @@ require (
114114
github.com/beorn7/perks v1.0.1 // indirect
115115
github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect
116116
github.com/cespare/xxhash/v2 v2.3.0 // indirect
117+
github.com/chzyer/readline v1.5.1 // indirect
117118
github.com/cloudflare/circl v1.3.3 // indirect
118119
github.com/cockroachdb/crlib v0.0.0-20241015224233-894974b3ad94 // indirect
119120
github.com/cockroachdb/errors v1.11.3 // indirect

go.sum

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,8 +102,12 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf
102102
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
103103
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
104104
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
105+
github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ=
105106
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
107+
github.com/chzyer/readline v1.5.1 h1:upd/6fQk4src78LMRzh5vItIt361/o4uq553V8B5sGI=
108+
github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk=
106109
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
110+
github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8=
107111
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
108112
github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I=
109113
github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs=
@@ -900,6 +904,8 @@ go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz
900904
go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo=
901905
go.starlark.net v0.0.0-20240520160348-046347dcd104 h1:3qhteRISupnJvaWshOmeqEUs2y9oc/+/ePPvDh3Eygg=
902906
go.starlark.net v0.0.0-20240520160348-046347dcd104/go.mod h1:YKMCv9b1WrfWmeqdV5MAuEHWsu5iC+fe6kYl2sQjdI8=
907+
go.starlark.net v0.0.0-20251109183026-be02852a5e1f h1:3KpJSfM1L+ziCR1a3I/Hgen2nwO94GjC7NAyiPArTkA=
908+
go.starlark.net v0.0.0-20251109183026-be02852a5e1f/go.mod h1:YKMCv9b1WrfWmeqdV5MAuEHWsu5iC+fe6kYl2sQjdI8=
903909
go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs=
904910
go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8=
905911
go.uber.org/dig v1.19.0 h1:BACLhebsYdpQ7IROQ1AGPjrXcP5dF80U3gKoFzbaq/4=
@@ -1052,6 +1058,7 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc
10521058
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
10531059
golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
10541060
golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
1061+
golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
10551062
golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
10561063
golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
10571064
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

pkg/starlark/benchmark_test.go

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
package starlark
2+
3+
import (
4+
"testing"
5+
6+
"go.starlark.net/starlark"
7+
)
8+
9+
var (
10+
benchmarkValue starlark.Value
11+
benchmarkNativeValue any
12+
benchmarkVM *vm
13+
)
14+
15+
func BenchmarkContextCall(b *testing.B) {
16+
vm, err := New(testFiles)
17+
if err != nil {
18+
b.Fatalf("failed to create VM: %v", err)
19+
}
20+
21+
ctx, err := vm.File("testdata/fibonacci.star")
22+
if err != nil {
23+
b.Fatalf("failed to load fibonacci.star: %v", err)
24+
}
25+
26+
arg := starlark.MakeInt(10)
27+
28+
b.ReportAllocs()
29+
b.ResetTimer()
30+
31+
for i := 0; i < b.N; i++ {
32+
result, err := ctx.Call("fibonacci", arg)
33+
if err != nil {
34+
b.Fatalf("call failed: %v", err)
35+
}
36+
benchmarkValue = result
37+
}
38+
}
39+
40+
func BenchmarkContextCallWithNative(b *testing.B) {
41+
vm, err := New(testFiles)
42+
if err != nil {
43+
b.Fatalf("failed to create VM: %v", err)
44+
}
45+
46+
if err = vm.Module(new(testModule)); err != nil {
47+
b.Fatalf("failed to register module: %v", err)
48+
}
49+
50+
ctx, err := vm.File("testdata/go.star")
51+
if err != nil {
52+
b.Fatalf("failed to load go.star: %v", err)
53+
}
54+
55+
b.ReportAllocs()
56+
b.ResetTimer()
57+
58+
for i := 0; i < b.N; i++ {
59+
result, err := ctx.CallWithNative("Add2", 5, 3)
60+
if err != nil {
61+
b.Fatalf("call with native failed: %v", err)
62+
}
63+
benchmarkNativeValue = result
64+
}
65+
}
66+
67+
func BenchmarkModuleRegistration(b *testing.B) {
68+
mod := new(testModule)
69+
70+
b.ReportAllocs()
71+
b.ResetTimer()
72+
73+
for i := 0; i < b.N; i++ {
74+
vm := &vm{
75+
builtins: make(map[string]starlark.StringDict),
76+
}
77+
if err := vm.Module(mod); err != nil {
78+
b.Fatalf("module registration failed: %v", err)
79+
}
80+
benchmarkVM = vm
81+
}
82+
}

pkg/starlark/conv_test.go

Lines changed: 246 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,246 @@
1+
package starlark
2+
3+
import (
4+
"reflect"
5+
"testing"
6+
7+
"go.starlark.net/starlark"
8+
"gotest.tools/v3/assert"
9+
)
10+
11+
func TestConvertFromStarlarkBasedOnValue(t *testing.T) {
12+
t.Run("primitives", func(t *testing.T) {
13+
intVal := convertFromStarlarkBasedOnValue(starlark.MakeInt(42))
14+
assert.Equal(t, intVal, int64(42))
15+
16+
floatVal := convertFromStarlarkBasedOnValue(starlark.Float(3.14))
17+
assert.Equal(t, floatVal, 3.14)
18+
19+
stringVal := convertFromStarlarkBasedOnValue(starlark.String("hello"))
20+
assert.Equal(t, stringVal, "hello")
21+
22+
boolVal := convertFromStarlarkBasedOnValue(starlark.Bool(true))
23+
assert.Equal(t, boolVal, true)
24+
})
25+
26+
t.Run("list recursion", func(t *testing.T) {
27+
nestedList := starlark.NewList([]starlark.Value{starlark.Bool(true)})
28+
list := starlark.NewList([]starlark.Value{
29+
starlark.MakeInt(1),
30+
starlark.String("two"),
31+
nestedList,
32+
})
33+
34+
converted, ok := convertFromStarlarkBasedOnValue(list).([]any)
35+
if !ok {
36+
t.Fatalf("expected []any, got %T", converted)
37+
}
38+
39+
assert.Equal(t, converted[0], int64(1))
40+
assert.Equal(t, converted[1], "two")
41+
42+
nested, ok := converted[2].([]any)
43+
if !ok {
44+
t.Fatalf("expected nested []any, got %T", converted[2])
45+
}
46+
assert.Equal(t, nested[0], true)
47+
})
48+
49+
t.Run("dict recursion", func(t *testing.T) {
50+
list := starlark.NewList([]starlark.Value{starlark.MakeInt(3), starlark.String("four")})
51+
52+
dict := starlark.NewDict(0)
53+
assert.NilError(t, dict.SetKey(starlark.String("answer"), starlark.MakeInt(42)))
54+
assert.NilError(t, dict.SetKey(starlark.String("numbers"), list))
55+
assert.NilError(t, dict.SetKey(starlark.MakeInt(1), starlark.String("one")))
56+
57+
converted, ok := convertFromStarlarkBasedOnValue(dict).(map[any]any)
58+
if !ok {
59+
t.Fatalf("expected map[any]any, got %T", converted)
60+
}
61+
62+
assert.Equal(t, converted["answer"], int64(42))
63+
numbers, ok := converted["numbers"].([]any)
64+
if !ok {
65+
t.Fatalf("expected numbers to be []any, got %T", converted["numbers"])
66+
}
67+
assert.DeepEqual(t, numbers, []any{int64(3), "four"})
68+
assert.Equal(t, converted[int64(1)], "one")
69+
})
70+
71+
t.Run("none value", func(t *testing.T) {
72+
assert.Equal(t, convertFromStarlarkBasedOnValue(starlark.None), nil)
73+
})
74+
75+
t.Run("default passthrough", func(t *testing.T) {
76+
tuple := starlark.Tuple{starlark.MakeInt(7)}
77+
out := convertFromStarlarkBasedOnValue(tuple)
78+
79+
resultTuple, ok := out.(starlark.Tuple)
80+
if !ok {
81+
t.Fatalf("expected starlark.Tuple, got %T", out)
82+
}
83+
assert.Equal(t, len(resultTuple), len(tuple))
84+
assert.Equal(t, resultTuple[0].Type(), tuple[0].Type())
85+
assert.Equal(t, resultTuple[0].String(), tuple[0].String())
86+
})
87+
}
88+
89+
func TestConvertToStarlarkSuccess(t *testing.T) {
90+
intVal, err := convertToStarlark(21)
91+
assert.NilError(t, err)
92+
intFromValue, convErr := starlark.AsInt32(intVal)
93+
assert.NilError(t, convErr)
94+
assert.Equal(t, intFromValue, 21)
95+
96+
floatVal, err := convertToStarlark(1.5)
97+
assert.NilError(t, err)
98+
assert.Equal(t, float64(floatVal.(starlark.Float)), 1.5)
99+
100+
stringVal, err := convertToStarlark("hello")
101+
assert.NilError(t, err)
102+
assert.Equal(t, string(stringVal.(starlark.String)), "hello")
103+
104+
boolVal, err := convertToStarlark(true)
105+
assert.NilError(t, err)
106+
assert.Equal(t, bool(boolVal.(starlark.Bool)), true)
107+
108+
listVal, err := convertToStarlark([]interface{}{1, "two", []interface{}{false, nil}})
109+
assert.NilError(t, err)
110+
111+
list, ok := listVal.(*starlark.List)
112+
if !ok {
113+
t.Fatalf("expected *starlark.List, got %T", listVal)
114+
}
115+
assert.Equal(t, list.Len(), 3)
116+
117+
firstInt, _ := starlark.AsInt32(list.Index(0))
118+
assert.Equal(t, int(firstInt), 1)
119+
120+
assert.Equal(t, string(list.Index(1).(starlark.String)), "two")
121+
122+
nestedVal := list.Index(2)
123+
nestedList, ok := nestedVal.(*starlark.List)
124+
if !ok {
125+
t.Fatalf("expected nested *starlark.List, got %T", nestedVal)
126+
}
127+
assert.Equal(t, nestedList.Len(), 2)
128+
assert.Equal(t, bool(nestedList.Index(0).(starlark.Bool)), false)
129+
assert.Equal(t, nestedList.Index(1), starlark.None)
130+
131+
inputMap := map[interface{}]interface{}{
132+
"answer": 42,
133+
"nested": []interface{}{true},
134+
7: "seven",
135+
"nilVal": nil,
136+
"float64": 3.5,
137+
}
138+
139+
dictVal, err := convertToStarlark(inputMap)
140+
assert.NilError(t, err)
141+
142+
dict, ok := dictVal.(*starlark.Dict)
143+
if !ok {
144+
t.Fatalf("expected *starlark.Dict, got %T", dictVal)
145+
}
146+
147+
answer, found, err := dict.Get(starlark.String("answer"))
148+
assert.NilError(t, err)
149+
assert.Assert(t, found)
150+
answerInt, _ := starlark.AsInt32(answer)
151+
assert.Equal(t, int(answerInt), 42)
152+
153+
nested, found, err := dict.Get(starlark.String("nested"))
154+
assert.NilError(t, err)
155+
assert.Assert(t, found)
156+
nestedList, ok = nested.(*starlark.List)
157+
if !ok {
158+
t.Fatalf("expected nested value to be *starlark.List, got %T", nested)
159+
}
160+
assert.Equal(t, nestedList.Len(), 1)
161+
assert.Equal(t, bool(nestedList.Index(0).(starlark.Bool)), true)
162+
163+
intKeyVal, found, err := dict.Get(starlark.MakeInt(7))
164+
assert.NilError(t, err)
165+
assert.Assert(t, found)
166+
assert.Equal(t, string(intKeyVal.(starlark.String)), "seven")
167+
168+
nilVal, found, err := dict.Get(starlark.String("nilVal"))
169+
assert.NilError(t, err)
170+
assert.Assert(t, found)
171+
assert.Equal(t, nilVal, starlark.None)
172+
173+
floatValFromMap, found, err := dict.Get(starlark.String("float64"))
174+
assert.NilError(t, err)
175+
assert.Assert(t, found)
176+
assert.Equal(t, float64(floatValFromMap.(starlark.Float)), 3.5)
177+
}
178+
179+
func TestConvertToStarlarkErrors(t *testing.T) {
180+
_, err := convertToStarlark(struct{}{})
181+
assert.Error(t, err, "unsupported type struct {}")
182+
183+
_, err = convertToStarlark([]interface{}{struct{}{}})
184+
assert.ErrorContains(t, err, "unsupported type struct {}")
185+
186+
_, err = convertToStarlark(map[interface{}]interface{}{struct{}{}: "value"})
187+
assert.ErrorContains(t, err, "unsupported type struct {}")
188+
189+
_, err = convertToStarlark(map[interface{}]interface{}{"badValue": struct{}{}})
190+
assert.ErrorContains(t, err, "unsupported type struct {}")
191+
}
192+
193+
func TestConvertFromStarlarkSuccess(t *testing.T) {
194+
intVal, err := convertFromStarlark(starlark.MakeInt(11), reflect.TypeOf(0))
195+
assert.NilError(t, err)
196+
assert.Equal(t, intVal.(int), 11)
197+
198+
floatVal, err := convertFromStarlark(starlark.Float(4.2), reflect.TypeOf(float64(0)))
199+
assert.NilError(t, err)
200+
assert.Equal(t, floatVal.(float64), 4.2)
201+
202+
stringVal, err := convertFromStarlark(starlark.String("hi"), reflect.TypeOf(""))
203+
assert.NilError(t, err)
204+
assert.Equal(t, stringVal.(string), "hi")
205+
206+
boolVal, err := convertFromStarlark(starlark.Bool(true), reflect.TypeOf(true))
207+
assert.NilError(t, err)
208+
assert.Equal(t, boolVal.(bool), true)
209+
210+
list := starlark.NewList([]starlark.Value{starlark.MakeInt(1), starlark.MakeInt(2)})
211+
sliceVal, err := convertFromStarlark(list, reflect.TypeOf([]int{}))
212+
assert.NilError(t, err)
213+
assert.DeepEqual(t, sliceVal, []int{1, 2})
214+
215+
dict := starlark.NewDict(0)
216+
assert.NilError(t, dict.SetKey(starlark.String("a"), starlark.MakeInt(1)))
217+
assert.NilError(t, dict.SetKey(starlark.String("b"), starlark.MakeInt(2)))
218+
219+
mapVal, err := convertFromStarlark(dict, reflect.TypeOf(map[string]int{}))
220+
assert.NilError(t, err)
221+
assert.DeepEqual(t, mapVal, map[string]int{"a": 1, "b": 2})
222+
}
223+
224+
func TestConvertFromStarlarkErrors(t *testing.T) {
225+
_, err := convertFromStarlark(starlark.String("bad"), reflect.TypeOf(0))
226+
assert.ErrorContains(t, err, "got string")
227+
228+
_, err = convertFromStarlark(starlark.String("bad"), reflect.TypeOf(float64(0)))
229+
assert.Error(t, err, `failed to convert "bad" (type string) to float`)
230+
231+
_, err = convertFromStarlark(starlark.MakeInt(1), reflect.TypeOf(""))
232+
assert.Error(t, err, "failed to convert 1 (type string) to string")
233+
234+
_, err = convertFromStarlark(starlark.String("oops"), reflect.TypeOf(true))
235+
assert.Error(t, err, `failed to convert "oops" (type string) to bool`)
236+
237+
list := starlark.NewList([]starlark.Value{starlark.String("not an int")})
238+
_, err = convertFromStarlark(list, reflect.TypeOf([]int{}))
239+
assert.ErrorContains(t, err, "got string")
240+
241+
dict := starlark.NewDict(0)
242+
assert.NilError(t, dict.SetKey(starlark.String("key"), starlark.String("value")))
243+
244+
_, err = convertFromStarlark(dict, reflect.TypeOf(map[string]int{}))
245+
assert.ErrorContains(t, err, "got string")
246+
}

0 commit comments

Comments
 (0)