Merge branch 'main' into hugo-docs

Author: philrz

lake/branch.go

@@ -16,6 +16,7 @@ import (
 	"github.com/brimdata/super/pkg/plural"
 	"github.com/brimdata/super/pkg/storage"
 	"github.com/brimdata/super/runtime"
+	"github.com/brimdata/super/zbuf"
 	"github.com/brimdata/super/zio"
 	"github.com/brimdata/super/zson"
 	"github.com/segmentio/ksuid"
@@ -154,7 +155,7 @@ func (b *Branch) DeleteWhere(ctx context.Context, c runtime.Compiler, ast *parse
 		if err != nil {
 			return nil, err
 		}
-		err = zio.CopyWithContext(ctx, w, runtime.AsReader(query))
+		err = zbuf.CopyPuller(w, query)
 		if closeErr := w.Close(); err == nil {
 			err = closeErr
 		}

mdtest/mdtest.go

@@ -44,6 +44,23 @@
 //	hello
 //	...
 //	```
+//
+// # SPQ tests
+//
+// A fenced code block with mdtest-spq as the first word of its info string
+// contains an SPQ test.  The content of the block must comprise three sections,
+// each preceeded by one or more "#"-prefixed lines.  The first section contains
+// an SPQ program, the second contains input provided to the program when the
+// test runs, and the third contains the program's expected output.
+//
+//	```mdtest-spq
+//	# spq
+//	yield a
+//	# input
+//	{a:1}
+//	# expected output
+//	1
+//	```
 package mdtest
 
 import (
@@ -53,6 +70,7 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strconv"
 	"strings"
 	"testing"
@@ -131,6 +149,9 @@ func (f *File) Run(t *testing.T) {
 	}
 }
 
+// Matches one or more "#"-prefixed lines.
+var spqSeparatorRE = regexp.MustCompile(`(?m:^#.*\n)+`)
+
 func parseMarkdown(source []byte) (map[string]string, []*Test, error) {
 	var commandFCB *ast.FencedCodeBlock
 	var inputs map[string]string
@@ -197,6 +218,23 @@ func parseMarkdown(source []byte) (map[string]string, []*Test, error) {
 				GoExample: fcbLines(fcb, source),
 				Line:      fcbLineNumber(fcb, source),
 			})
+		case "mdtest-spq":
+			lines := fcbLines(fcb, source)
+			if !strings.HasPrefix(lines, "#") {
+				return ast.WalkStop, fcbError(fcb, source, "mdtest-spq content must begin with '#'")
+			}
+			sections := spqSeparatorRE.Split(lines, -1)
+			// Ignore sections[0].
+			if n := len(sections); n != 4 {
+				msg := fmt.Sprintf("mdtest-spq content has %d '#'-prefixed sections (expected 3)", n-1)
+				return ast.WalkStop, fcbError(fcb, source, msg)
+			}
+			tests = append(tests, &Test{
+				Expected: sections[3],
+				Line:     fcbLineNumber(fcb, source),
+				Input:    sections[2],
+				SPQ:      sections[1],
+			})
 		}
 		return ast.WalkContinue, nil
 	})

mdtest/mdtest_test.go

@@ -162,6 +162,66 @@ block 2
 				{Command: "block 1\n", Expected: "block 2\n...\n", Line: 2},
 			},
 		},
+		{
+			name: "mdtest-spq",
+			markdown: `
+~~~mdtest-spq
+# Multiple '#' lines are allowed.
+#
+spq
+#
+input
+#
+expected
+~~~
+`,
+			tests: []*Test{
+				{Expected: "expected\n", Line: 2, Input: "input\n", SPQ: "spq\n"},
+			},
+		},
+		{
+			name: "mdtest-spq with leading garbage",
+			markdown: `
+~~~mdtest-spq
+garbage
+#
+spq
+#
+input
+#
+expected
+~~~
+	`,
+			strerror: `line 2: mdtest-spq content must begin with '#'`,
+		},
+		{
+			name: "mdtest-spq with too many sections",
+			markdown: `
+~~~mdtest-spq
+#
+spq
+#
+input
+#
+expected
+#
+extra
+~~~
+	`,
+			strerror: `line 2: mdtest-spq content has 4 '#'-prefixed sections (expected 3)`,
+		},
+		{
+			name: "mdtest-spq with too few sections",
+			markdown: `
+~~~mdtest-spq
+#
+spq
+#
+input
+~~~
+	`,
+			strerror: `line 2: mdtest-spq content has 2 '#'-prefixed sections (expected 3)`,
+		},
 	}
 	for _, tc := range tests {
 		tc := tc

mdtest/test.go

@@ -20,16 +20,26 @@ type Test struct {
 	Head      bool
 	Line      int
 	GoExample string
+
+	// For SPQ tests
+	Input string
+	SPQ   string
 }
 
 // Run runs the test, returning nil on success.
 func (t *Test) Run() error {
 	if t.GoExample != "" {
 		return t.vetGoExample()
 	}
-	c := exec.Command("bash", "-e", "-o", "pipefail")
-	c.Dir = t.Dir
-	c.Stdin = strings.NewReader(t.Command)
+	var c *exec.Cmd
+	if t.SPQ != "" {
+		c = exec.Command("super", "-z", "-c", t.SPQ, "-")
+		c.Stdin = strings.NewReader(t.Input)
+	} else {
+		c = exec.Command("bash", "-e", "-o", "pipefail")
+		c.Dir = t.Dir
+		c.Stdin = strings.NewReader(t.Command)
+	}
 	outBytes, err := c.CombinedOutput()
 	out := string(outBytes)
 	if t.Fails {

runtime/compiler.go

@@ -29,10 +29,6 @@ type DeleteQuery interface {
 	DeletionSet() []ksuid.KSUID
 }
 
-func AsReader(q Query) zio.Reader {
-	return zbuf.PullerReader(q)
-}
-
 func CompileQuery(ctx context.Context, zctx *super.Context, c Compiler, ast *parser.AST, readers []zio.Reader) (Query, error) {
 	rctx := NewContext(ctx, zctx)
 	q, err := c.NewQuery(rctx, ast, readers, 0)

runtime/sam/expr/agg/collect.go

@@ -16,12 +16,9 @@ type Collect struct {
 var _ Function = (*Collect)(nil)
 
 func (c *Collect) Consume(val super.Value) {
-	if !val.IsNull() {
-		c.update(val)
+	if val.IsNull() {
+		return
 	}
-}
-
-func (c *Collect) update(val super.Value) {
 	c.values = append(c.values, val.Under().Copy())
 	c.size += len(val.Bytes())
 	for c.size > MaxValueSize {
@@ -75,7 +72,7 @@ func (c *Collect) ConsumeAsPartial(val super.Value) {
 	}
 	typ := arrayType.Type
 	for it := val.Iter(); !it.Done(); {
-		c.update(super.NewValue(typ, it.Next()))
+		c.Consume(super.NewValue(typ, it.Next()))
 	}
 }
 

runtime/sam/expr/agg/ztests/collect-union.yaml

@@ -47,7 +47,11 @@ func (d *DotExpr) Eval(ectx Context, this super.Value) super.Value {
 		if !ok {
 			return d.zctx.Missing()
 		}
-		return super.NewValue(typ.Fields[i].Type, getNthFromContainer(val.Bytes(), i))
+		bytes, ok := getNthFromContainer(val.Bytes(), i)
+		if !ok {
+			return d.zctx.Missing()
+		}
+		return super.NewValue(typ.Fields[i].Type, bytes)
 	case *super.TypeMap:
 		return indexMap(d.zctx, ectx, typ, val.Bytes(), super.NewString(d.field))
 	case *super.TypeOfType:

runtime/sam/expr/dot.go

@@ -601,24 +601,24 @@ func (u *UnaryMinus) Eval(ectx Context, this super.Value) super.Value {
 	return u.zctx.WrapError("type incompatible with unary '-' operator", val)
 }
 
-func getNthFromContainer(container zcode.Bytes, idx int) zcode.Bytes {
+func getNthFromContainer(container zcode.Bytes, idx int) (zcode.Bytes, bool) {
 	if idx < 0 {
 		var length int
 		for it := container.Iter(); !it.Done(); it.Next() {
 			length++
 		}
 		idx = length + idx
 		if idx < 0 || idx >= length {
-			return nil
+			return nil, false
 		}
 	}
 	for i, it := 0, container.Iter(); !it.Done(); i++ {
 		zv := it.Next()
 		if i == idx {
-			return zv
+			return zv, true
 		}
 	}
-	return nil
+	return nil, false
 }
 
 func lookupKey(mapBytes, target zcode.Bytes) (zcode.Bytes, bool) {
@@ -673,11 +673,11 @@ func indexVector(zctx *super.Context, ectx Context, inner super.Type, vector zco
 	} else {
 		idx = int(index.Uint())
 	}
-	zv := getNthFromContainer(vector, idx)
-	if zv == nil {
+	bytes, ok := getNthFromContainer(vector, idx)
+	if !ok {
 		return zctx.Missing()
 	}
-	return deunion(ectx, inner, zv)
+	return deunion(ectx, inner, bytes)
 }
 
 func indexRecord(zctx *super.Context, ectx Context, typ *super.TypeRecord, record zcode.Bytes, index super.Value) super.Value {

runtime/sam/expr/eval.go

@@ -527,7 +527,7 @@ func (s *step) buildRecord(zctx *super.Context, ectx Context, in zcode.Bytes, b
 		// reordering) would be make direct use of a
 		// zcode.Iter along with keeping track of our
 		// position.
-		bytes := getNthFromContainer(in, child.fromIndex)
+		bytes, _ := getNthFromContainer(in, child.fromIndex)
 		typ := child.build(zctx, ectx, bytes, b)
 		if super.TypeUnder(typ) == super.TypeUnder(child.toType) {
 			// Prefer child.toType in case it's a named type.