Skip to content

runtime/vam: return error on cast of invalid UTF-8 bytes to string #6273

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nwt merged 3 commits into from
Oct 2, 2025
Merged

runtime/vam: return error on cast of invalid UTF-8 bytes to string #6273

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7c4483a
runtime/vam: return error on cast of invalid UTF-8 bytes to string
nwt Oct 1, 2025
5d6b26f
Merge branch 'main' into vector-cast-bytes-to-string-error
nwt Oct 1, 2025
53e8a04
remove runtime/sam/expr/ztests/cast-bytes-string-err.yaml
nwt Oct 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 0 additions & 7 deletions runtime/sam/expr/ztests/cast-bytes-string-err.yaml
View file
Open in desktop

This file was deleted.

8 changes: 4 additions & 4 deletions runtime/vam/expr/cast/bool.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import (
"github.com/brimdata/super/vector/bitvec"
)

func castToBool(vec vector.Any, index []uint32) (vector.Any, []uint32, bool) {
func castToBool(vec vector.Any, index []uint32) (vector.Any, []uint32, string, bool) {
var out *vector.Bool
switch vec := vec.(type) {
case *vector.Int:
Expand All @@ -17,17 +17,17 @@ func castToBool(vec vector.Any, index []uint32) (vector.Any, []uint32, bool) {
out = numberToBool(vec.Values, index)
case *vector.String:
vvec, errs := stringToBool(vec, index)
return vvec, errs, true
return vvec, errs, "", true
default:
return nil, nil, false
return nil, nil, "", false
}
nulls := vector.NullsOf(vec)
if index == nil {
out.Nulls = nulls
} else {
out.Nulls = nulls.Pick(index)
}
return out, nil, true
return out, nil, "", true
}

func numberToBool[E numeric](s []E, index []uint32) *vector.Bool {
Expand Down
6 changes: 3 additions & 3 deletions runtime/vam/expr/cast/bytes.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,14 @@ import (
"github.com/brimdata/super/vector"
)

func castToBytes(vec vector.Any, index []uint32) (vector.Any, []uint32, bool) {
func castToBytes(vec vector.Any, index []uint32) (vector.Any, []uint32, string, bool) {
strVec, ok := vec.(*vector.String)
if !ok {
return nil, nil, false
return nil, nil, "", false
}
out := vector.Any(vector.NewBytes(strVec.Table(), vector.NullsOf(strVec)))
if index != nil {
out = vector.Pick(out, index)
}
return out, nil, true
return out, nil, "", true
}
31 changes: 18 additions & 13 deletions runtime/vam/expr/cast/cast.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ func To(sctx *super.Context, vec vector.Any, typ super.Type) vector.Any {
var c caster
id := typ.ID()
if super.IsNumber(id) {
c = func(vec vector.Any, index []uint32) (vector.Any, []uint32, bool) {
c = func(vec vector.Any, index []uint32) (vector.Any, []uint32, string, bool) {
return castToNumber(vec, typ, index)
}
} else {
Expand All @@ -29,29 +29,30 @@ func To(sctx *super.Context, vec vector.Any, typ super.Type) vector.Any {
case super.IDNet:
c = castToNet
case super.IDType:
c = func(vec vector.Any, index []uint32) (vector.Any, []uint32, bool) {
c = func(vec vector.Any, index []uint32) (vector.Any, []uint32, string, bool) {
return castToType(sctx, vec, index)
}
default:
return errCastFailed(sctx, vec, typ)
return errCastFailed(sctx, vec, typ, "")
}
}
return assemble(sctx, vec, typ, c)
}

type caster func(vector.Any, []uint32) (vector.Any, []uint32, bool)
type caster func(vector.Any, []uint32) (vector.Any, []uint32, string, bool)

func assemble(sctx *super.Context, vec vector.Any, typ super.Type, fn caster) vector.Any {
var out vector.Any
var errs []uint32
var errMsg string
var ok bool
switch vec := vec.(type) {
case *vector.Const:
return castConst(sctx, vec, typ)
case *vector.View:
out, errs, ok = fn(vec.Any, vec.Index)
out, errs, errMsg, ok = fn(vec.Any, vec.Index)
case *vector.Dict:
out, errs, ok = fn(vec.Any, nil)
out, errs, errMsg, ok = fn(vec.Any, nil)
if ok {
if len(errs) > 0 {
index, counts, nulls, nerrs := vec.RebuildDropTags(errs...)
Expand All @@ -62,13 +63,13 @@ func assemble(sctx *super.Context, vec vector.Any, typ super.Type, fn caster) ve
}
}
default:
out, errs, ok = fn(vec, nil)
out, errs, errMsg, ok = fn(vec, nil)
}
if !ok {
return errCastFailed(sctx, vec, typ)
return errCastFailed(sctx, vec, typ, errMsg)
}
if len(errs) > 0 {
return vector.Combine(out, errs, errCastFailed(sctx, vector.Pick(vec, errs), typ))
return vector.Combine(out, errs, errCastFailed(sctx, vector.Pick(vec, errs), typ, errMsg))
}
return out
}
Expand All @@ -88,17 +89,21 @@ func castConst(sctx *super.Context, vec *vector.Const, typ super.Type) vector.An
trueCount++
}
}
err := errCastFailed(sctx, vector.NewConst(vec.Value(), vec.Len()-trueCount, bitvec.Zero), typ)
err := errCastFailed(sctx, vector.NewConst(vec.Value(), vec.Len()-trueCount, bitvec.Zero), typ, "")
nulls := vector.NewConst(super.NewValue(typ, nil), trueCount, bitvec.Zero)
return vector.NewDynamic(index, []vector.Any{err, nulls})
}
return errCastFailed(sctx, vec, typ)
return errCastFailed(sctx, vec, typ, "")
}
return vector.NewConst(val, vec.Len(), vec.Nulls)
}

func errCastFailed(sctx *super.Context, vec vector.Any, typ super.Type) vector.Any {
return vector.NewWrappedError(sctx, "cannot cast to "+sup.FormatType(typ), vec)
func errCastFailed(sctx *super.Context, vec vector.Any, typ super.Type, msgSuffix string) vector.Any {
msg := "cannot cast to " + sup.FormatType(typ)
if msgSuffix != "" {
msg = msg + ": " + msgSuffix
}
return vector.NewWrappedError(sctx, msg, vec)
}

func lengthOf(vec vector.Any, index []uint32) uint32 {
Expand Down
16 changes: 8 additions & 8 deletions runtime/vam/expr/cast/ip.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,10 @@ import (
"github.com/brimdata/super/vector/bitvec"
)

func castToIP(vec vector.Any, index []uint32) (vector.Any, []uint32, bool) {
func castToIP(vec vector.Any, index []uint32) (vector.Any, []uint32, string, bool) {
switch vec := vec.(type) {
case *vector.IP:
return vec, nil, true
return vec, nil, "", true
case *vector.String:
n := lengthOf(vec, index)
var nulls bitvec.Bits
Expand All @@ -37,16 +37,16 @@ func castToIP(vec vector.Any, index []uint32) (vector.Any, []uint32, bool) {
}
ips = append(ips, ip)
}
return vector.NewIP(ips, nulls), errs, true
return vector.NewIP(ips, nulls), errs, "", true
default:
return nil, nil, false
return nil, nil, "", false
}
}

func castToNet(vec vector.Any, index []uint32) (vector.Any, []uint32, bool) {
func castToNet(vec vector.Any, index []uint32) (vector.Any, []uint32, string, bool) {
switch vec := vec.(type) {
case *vector.Net:
return vec, nil, true
return vec, nil, "", true
case *vector.String:
n := lengthOf(vec, index)
var nulls bitvec.Bits
Expand All @@ -72,8 +72,8 @@ func castToNet(vec vector.Any, index []uint32) (vector.Any, []uint32, bool) {
}
nets = append(nets, net)
}
return vector.NewNet(nets, nulls), errs, true
return vector.NewNet(nets, nulls), errs, "", true
default:
return nil, nil, false
return nil, nil, "", false
}
}
12 changes: 6 additions & 6 deletions runtime/vam/expr/cast/number.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,10 @@ type numeric interface {
constraints.Float | constraints.Integer
}

func castToNumber(vec vector.Any, typ super.Type, index []uint32) (vector.Any, []uint32, bool) {
func castToNumber(vec vector.Any, typ super.Type, index []uint32) (vector.Any, []uint32, string, bool) {
if vec.Type().ID() == super.IDString {
out, errs := castStringToNumber(vec, typ, index)
return out, errs, true
return out, errs, "", true
}
nulls := vector.NullsOf(vec)
if index != nil {
Expand All @@ -32,21 +32,21 @@ func castToNumber(vec vector.Any, typ super.Type, index []uint32) (vector.Any, [
if len(errs) > 0 {
nulls = nulls.Pick(inverseIndex(errs, nulls.Len()))
}
return vector.NewInt(typ, vals, nulls), errs, true
return vector.NewInt(typ, vals, nulls), errs, "", true
case super.IsUnsigned(id):
vals, errs := toNumeric[uint64](vec, typ, index)
if len(errs) > 0 {
nulls = nulls.Pick(inverseIndex(errs, nulls.Len()))
}
return vector.NewUint(typ, vals, nulls), errs, true
return vector.NewUint(typ, vals, nulls), errs, "", true
case super.IsFloat(id):
vals, errs := toNumeric[float64](vec, typ, index)
if errs != nil {
nulls = nulls.Pick(inverseIndex(errs, nulls.Len()))
}
return vector.NewFloat(typ, vals, nulls), errs, true
return vector.NewFloat(typ, vals, nulls), errs, "", true
default:
return nil, nil, false
return nil, nil, "", false
}
}

Expand Down
25 changes: 20 additions & 5 deletions runtime/vam/expr/cast/string.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package cast
import (
"strconv"
"time"
"unicode/utf8"

"github.com/brimdata/super"
"github.com/brimdata/super/pkg/nano"
Expand All @@ -11,7 +12,7 @@ import (
"github.com/brimdata/super/vector"
)

func castToString(vec vector.Any, index []uint32) (vector.Any, []uint32, bool) {
func castToString(vec vector.Any, index []uint32) (vector.Any, []uint32, string, bool) {
nulls := vector.NullsOf(vec)
if index != nil {
nulls = nulls.Pick(index)
Expand Down Expand Up @@ -56,21 +57,35 @@ func castToString(vec vector.Any, index []uint32) (vector.Any, []uint32, bool) {
}
case *vector.String:
if index == nil {
return vec, nil, true
return vec, nil, "", true
}
for _, idx := range index {
bytes = append(bytes, vec.Value(idx)...)
offs = append(offs, uint32(len(bytes)))
}
case *vector.Bytes:
var errs []uint32
for i := range n {
idx := i
if index != nil {
idx = index[i]
}
bytes = append(bytes, vec.Value(idx)...)
offs = append(offs, uint32(len(bytes)))
if !utf8.Valid(vec.Value(idx)) {
errs = append(errs, i)
}
}
const errMsg = "invalid UTF-8"
if len(errs) == int(n) {
return nil, nil, errMsg, false
}
out := vector.Any(vector.NewString(vec.Table(), vec.Nulls))
if index != nil {
out = vector.Pick(out, index)
}
if len(errs) > 0 {
out = vector.ReversePick(out, errs)
}
return out, errs, errMsg, true
case *vector.IP:
for i := range n {
idx := i
Expand Down Expand Up @@ -115,7 +130,7 @@ func castToString(vec vector.Any, index []uint32) (vector.Any, []uint32, bool) {
offs = append(offs, uint32(len(bytes)))
}
}
return vector.NewString(vector.NewBytesTable(offs, bytes), nulls), nil, true
return vector.NewString(vector.NewBytesTable(offs, bytes), nulls), nil, "", true
}

func timeToString(vec *vector.Int, index []uint32, n uint32) ([]uint32, []byte) {
Expand Down
8 changes: 4 additions & 4 deletions runtime/vam/expr/cast/type.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@ import (
"github.com/brimdata/super/vector/bitvec"
)

func castToType(sctx *super.Context, vec vector.Any, index []uint32) (vector.Any, []uint32, bool) {
func castToType(sctx *super.Context, vec vector.Any, index []uint32) (vector.Any, []uint32, string, bool) {
switch vec := vec.(type) {
case *vector.TypeValue:
return vec, nil, true
return vec, nil, "", true
case *vector.String:
n := lengthOf(vec, index)
out := vector.NewTypeValueEmpty(0, bitvec.Zero)
Expand All @@ -36,8 +36,8 @@ func castToType(sctx *super.Context, vec vector.Any, index []uint32) (vector.Any
}
out.Append(val.Bytes())
}
return out, errs, true
return out, errs, "", true
default:
return nil, nil, false
return nil, nil, "", false
}
}
4 changes: 4 additions & 0 deletions runtime/ztests/expr/cast/string.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ input: |
1.2.3.4/32
0x68692c20776f726c64
0x666f6f20626172
0x00
0xc328
"hi, world"
"foo bar"
{foo:"bar"}
Expand All @@ -46,6 +48,8 @@ output: |
"1.2.3.4/32"
"hi, world"
"foo bar"
"\u0000"
error({message:"cannot cast to string: invalid UTF-8",on:0xc328})
"hi, world"
"foo bar"
"{foo:\"bar\"}"
Expand Down