integrate resolver's lambda unraveling into semantic translator pass

compiler/semantic/analyzer.go

@@ -3,7 +3,6 @@ package semantic
 import (
 	"context"
 	"errors"
-	"strconv"
 	"strings"
 
 	"github.com/brimdata/super"
@@ -40,16 +39,11 @@ func Analyze(ctx context.Context, p *parser.AST, env *exec.Environment, extInput
 			seq.Prepend(&sem.NullScan{})
 		}
 	}
-	resolver := newResolver(t)
-	semSeq, dagFuncs := resolver.resolve(seq)
+	newChecker(t).check(t.reporter, seq)
 	if err := t.Error(); err != nil {
 		return nil, err
 	}
-	newChecker(t, dagFuncs).check(t.reporter, semSeq)
-	if err := t.Error(); err != nil {
-		return nil, err
-	}
-	main := newDagen(t.reporter).assemble(semSeq, dagFuncs)
+	main := newDagen(t.reporter).assemble(seq, t.resolver.funcs)
 	return main, t.Error()
 }
 
@@ -59,26 +53,25 @@ func Analyze(ctx context.Context, p *parser.AST, env *exec.Environment, extInput
 // to dataflow.
 type translator struct {
 	reporter
-	ctx       context.Context
-	opStack   []*ast.OpDecl
-	cteStack  []*ast.SQLCTE
-	env       *exec.Environment
-	scope     *Scope
-	sctx      *super.Context
-	funcs     map[string]*sem.FuncDef
-	funcDecls map[string]*funcDecl
+	ctx      context.Context
+	resolver *resolver
+	opStack  []*ast.OpDecl
+	cteStack []*ast.SQLCTE
+	env      *exec.Environment
+	scope    *Scope
+	sctx     *super.Context
 }
 
 func newTranslator(ctx context.Context, r reporter, env *exec.Environment) *translator {
-	return &translator{
-		reporter:  r,
-		ctx:       ctx,
-		env:       env,
-		scope:     NewScope(nil),
-		sctx:      super.NewContext(),
-		funcs:     make(map[string]*sem.FuncDef),
-		funcDecls: make(map[string]*funcDecl),
+	t := &translator{
+		reporter: r,
+		ctx:      ctx,
+		env:      env,
+		scope:    NewScope(nil),
+		sctx:     super.NewContext(),
 	}
+	t.resolver = newResolver(t)
+	return t
 }
 
 func HasSource(seq sem.Seq) bool {
@@ -111,18 +104,6 @@ func (t *translator) exitScope() {
 	t.scope = t.scope.parent
 }
 
-func (t *translator) newFunc(body ast.Expr, name string, params []string, e sem.Expr) string {
-	tag := strconv.Itoa(len(t.funcs))
-	t.funcs[tag] = &sem.FuncDef{
-		Node:   body,
-		Tag:    tag,
-		Name:   name,
-		Params: params,
-		Body:   e,
-	}
-	return tag
-}
-
 type opDecl struct {
 	ast   *ast.OpDecl
 	scope *Scope // parent scope of op declaration.

compiler/semantic/checker.go

@@ -14,20 +14,14 @@ import (
 
 type checker struct {
 	t       *translator
-	funcs   map[string]*sem.FuncDef
 	checked map[super.Type]super.Type
 	unknown *super.TypeError
 	estack  []errlist
 }
 
-func newChecker(t *translator, funcs []*sem.FuncDef) *checker {
-	funcMap := make(map[string]*sem.FuncDef)
-	for _, f := range funcs {
-		funcMap[f.Tag] = f
-	}
+func newChecker(t *translator) *checker {
 	return &checker{
 		t:       t,
-		funcs:   funcMap,
 		unknown: t.sctx.LookupTypeError(t.sctx.MustLookupTypeRecord(nil)),
 		checked: make(map[super.Type]super.Type),
 	}
@@ -466,15 +460,15 @@ func (c *checker) callBuiltin(call *sem.CallExpr, args []super.Type) super.Type
 }
 
 func (c *checker) callFunc(call *sem.CallExpr, args []super.Type) super.Type {
-	f := c.funcs[call.Tag]
-	if len(args) != len(f.Params) {
+	f := c.t.resolver.funcs[call.Tag]
+	if len(args) != len(f.params) {
 		// The translator has already checked that len(args) is len(params)
 		// but when there's an error, mismatches can still show up here so
 		// we ignore these here.
 		return c.unknown
 	}
 	fields := make([]super.Field, 0, len(args))
-	for k, param := range f.Params {
+	for k, param := range f.params {
 		fields = append(fields, super.Field{Name: param, Type: args[k]})
 	}
 	argsType := c.t.sctx.MustLookupTypeRecord(fields)
@@ -487,7 +481,7 @@ func (c *checker) callFunc(call *sem.CallExpr, args []super.Type) super.Type {
 	// of all recursive functions.  When we add (optional) type signatures to functions,
 	// this problem will (partially) go away.
 	c.checked[argsType] = c.unknown
-	typ := c.expr(argsType, f.Body)
+	typ := c.expr(argsType, f.body)
 	c.checked[argsType] = typ
 	return typ
 }

compiler/semantic/dagen.go

@@ -23,7 +23,7 @@ func newDagen(r reporter) *dagen {
 	}
 }
 
-func (d *dagen) assemble(seq sem.Seq, funcs []*sem.FuncDef) *dag.Main {
+func (d *dagen) assemble(seq sem.Seq, funcs map[string]*funcDef) *dag.Main {
 	dagSeq := d.seq(seq)
 	dagSeq = d.checkOutputs(true, dagSeq)
 	dagFuncs := make([]*dag.FuncDef, 0, len(d.funcs))
@@ -37,7 +37,7 @@ func (d *dagen) assemble(seq sem.Seq, funcs []*sem.FuncDef) *dag.Main {
 	return &dag.Main{Funcs: dagFuncs, Body: dagSeq}
 }
 
-func (d *dagen) assembleExpr(e sem.Expr, funcs []*sem.FuncDef) *dag.MainExpr {
+func (d *dagen) assembleExpr(e sem.Expr, funcs map[string]*funcDef) *dag.MainExpr {
 	dagExpr := d.expr(e)
 	dagFuncs := make([]*dag.FuncDef, 0, len(d.funcs))
 	for _, f := range funcs {
@@ -523,13 +523,13 @@ func (d *dagen) call(c *sem.CallExpr) *dag.CallExpr {
 	}
 }
 
-func (d *dagen) fn(f *sem.FuncDef) *dag.FuncDef {
+func (d *dagen) fn(f *funcDef) *dag.FuncDef {
 	return &dag.FuncDef{
 		Kind:   "FuncDef",
-		Tag:    f.Tag,
-		Name:   f.Name,
-		Params: f.Params,
-		Expr:   d.expr(f.Body),
+		Tag:    f.tag,
+		Name:   f.name,
+		Params: f.params,
+		Expr:   d.expr(f.body),
 	}
 }
 

compiler/semantic/evaluator.go

@@ -14,7 +14,7 @@ import (
 
 type evaluator struct {
 	translator *translator
-	in         map[string]*sem.FuncDef
+	in         map[string]*funcDef
 	errs       errlist
 	constThis  bool
 	bad        bool
@@ -25,7 +25,7 @@ type errloc struct {
 	err error
 }
 
-func newEvaluator(t *translator, funcs map[string]*sem.FuncDef) *evaluator {
+func newEvaluator(t *translator, funcs map[string]*funcDef) *evaluator {
 	return &evaluator{
 		translator: t,
 		in:         funcs,
@@ -47,29 +47,18 @@ func (e *evaluator) maybeEval(sctx *super.Context, expr sem.Expr) (super.Value,
 		}
 		return val, true
 	}
-	// re-enter the semantic analyzer with just this expr by resolving
-	// all needed funcs then traversing the resulting sem tree and seeing
-	// if it will eval as a compile-time constant.  If so, compile it the rest
-	// of the way and invoke rungen to get the result and return it.
-	// If an error is encountered, returns the error.  If the expression
-	// isn't a compile-time const, then errors will accumulate.  Note that
-	// no existing state in the translator is touched nor is the passed-in
-	// sem tree modified at all; instead, the process here creates copies
-	// of any needed sem tree and funcs.
-	r := newResolver(e.translator)
-	resolvedExpr, funcs := r.resolveExpr(expr)
-	e.expr(resolvedExpr)
+	e.expr(expr)
 	if len(e.errs) > 0 || e.bad {
 		return super.Value{}, false
 	}
-	for _, f := range funcs {
+	for _, f := range e.translator.resolver.funcs {
 		e.constThis = true
-		e.expr(f.Body)
+		e.expr(f.body)
 		if len(e.errs) > 0 || e.bad {
 			return super.Value{}, false
 		}
 	}
-	main := newDagen(e.translator.reporter).assembleExpr(resolvedExpr, funcs)
+	main := newDagen(e.translator.reporter).assembleExpr(expr, e.translator.resolver.funcs)
 	val, err := rungen.EvalAtCompileTime(sctx, main)
 	if err != nil {
 		e.errs.error(expr, err)

compiler/semantic/expr.go

@@ -95,15 +95,15 @@ func (t *translator) semExpr(e ast.Expr) sem.Expr {
 		return t.semFString(e)
 	case *ast.FuncNameExpr:
 		// We get here for &refs that are in a call expression. e.g.,
-		// an arg to another function.  These are only built-ins as
-		// user functions should be referenced directly as an ID.
-		tag := e.Name
-		if boundTag, _ := t.scope.lookupFunc(e.Name); boundTag != "" {
-			tag = boundTag
+		// an arg to another function.  It could be a built-in (as in &upper),
+		// or a user function (as in fn foo():... &foo)...
+		id := e.Name
+		if boundID, _ := t.scope.lookupFuncDeclOrParam(id); boundID != "" {
+			id = boundID
 		}
 		return &sem.FuncRef{
 			Node: e,
-			Tag:  tag,
+			ID:   id,
 		}
 	case *ast.GlobExpr:
 		return &sem.RegexpSearchExpr{
@@ -144,10 +144,10 @@ func (t *translator) semExpr(e ast.Expr) sem.Expr {
 		}
 		return out
 	case *ast.LambdaExpr:
-		tag := t.newFunc(e, "lambda", idsAsStrings(e.Params), t.semExpr(e.Expr))
+		funcDecl := t.resolver.newFuncDecl("lambda", e, t.scope)
 		return &sem.FuncRef{
 			Node: e,
-			Tag:  tag,
+			ID:   funcDecl.id,
 		}
 	case *ast.MapExpr:
 		var entries []sem.Entry
@@ -388,7 +388,7 @@ func (t *translator) semID(id *ast.IDExpr, lval bool) sem.Expr {
 	// an error to avoid a rake when such a function is mistakenly passed
 	// without "&" and otherwise turns into a field reference.
 	if entry := t.scope.lookupEntry(id.Name); entry != nil {
-		if _, ok := entry.ref.(*sem.FuncDef); ok && !lval {
+		if _, ok := entry.ref.(*funcDef); ok && !lval {
 			t.error(id, fmt.Errorf("function %q referenced but not called (consider &%s to create a function value)", id.Name, id.Name))
 			return badExpr()
 		}
@@ -653,8 +653,8 @@ func (t *translator) maybeSubquery(n ast.Node, name string) *sem.SubqueryExpr {
 }
 
 func (t *translator) semCallLambda(lambda *ast.LambdaExpr, args []sem.Expr) sem.Expr {
-	tag := t.newFunc(lambda, "lambda", idsAsStrings(lambda.Params), t.semExpr(lambda.Expr))
-	return sem.NewCall(lambda, tag, args)
+	funcDecl := t.resolver.newFuncDecl("lambda", lambda, t.scope)
+	return t.resolver.mustResolveCall(lambda, funcDecl.id, args)
 }
 
 func (t *translator) semCallByName(call *ast.CallExpr, name string, args []sem.Expr) sem.Expr {
@@ -664,23 +664,38 @@ func (t *translator) semCallByName(call *ast.CallExpr, name string, args []sem.E
 	// Check if the name resolves to a symbol in scope.
 	if entry := t.scope.lookupEntry(name); entry != nil {
 		switch ref := entry.ref.(type) {
-		case param:
-			// Called name is a parameter inside of a function.   We create a dummy
-			// CallParam that will be converted to a direct call to the passed-in
-			// function (we don't know it yet and there may be multiple variations
-			// that all land at this call site) in the next pass of semantic analysis.
-			return &sem.CallParam{
-				Node:  call,
-				Param: name,
-				Args:  args,
+		case funcParamValue:
+			t.error(call, fmt.Errorf("function called via parameter %q is bound to a non-function", name))
+			return badExpr()
+		case *funcParamLambda:
+			// Called name is a parameter inside of a function.   We only end up here
+			// when actual values have been bound to the parameter (i.e., we're compiling
+			// a lambda-variant function each time it is called to create each variant),
+			// so we call the resolver here to create a new instance of the function being
+			// called. In the case of recursion, all the lambdas that are further passed
+			// as args are known (in terms of their decl IDs), so the resolver can
+			// look this up in the variants of the decl and stop the recursion even if the body
+			// of the called entity is not completed yet.  We won't know the type but we
+			// can't know the type without function type signatures so when we integrate
+			// type checking here, we will use unknown for this corner case.
+			if isBuiltin(ref.id) {
+				// Check argument count here for builtin functions.
+				if _, err := function.New(super.NewContext(), ref.id, len(args)); err != nil {
+					t.error(call, fmt.Errorf("function %q called via parameter %q: %w", ref.id, ref.param, err))
+					return badExpr()
+				}
+				return sem.NewCall(call, ref.id, args)
 			}
+			return t.resolver.mustResolveCall(call, ref.id, args)
 		case *opDecl:
 			t.error(call, fmt.Errorf("cannot call user operator %q in an expression (consider subquery syntax)", name))
 			return badExpr()
 		case *sem.FuncRef:
-			return sem.NewCall(call, ref.Tag, args)
-		case *sem.FuncDef:
-			return sem.NewCall(call, ref.Tag, args)
+			// FuncRefs are put in the symbol table when passing stuff to user ops, e.g.,
+			// a lambda as a parameter, a &func, or a builtin like &upper.
+			return t.resolver.mustResolveCall(ref, ref.ID, args)
+		case *funcDecl:
+			return t.resolver.mustResolveCall(call, ref.id, args)
 		case *constDecl, *queryDecl:
 			t.error(call, fmt.Errorf("%q is not a function", name))
 			return badExpr()
@@ -691,22 +706,7 @@ func (t *translator) semCallByName(call *ast.CallExpr, name string, args []sem.E
 		}
 		panic(entry.ref)
 	}
-	// Call could be to a user func. Check if we have a matching func in scope.
-	// When the name is a formal argument, the bindings will have been put
-	// in scope and will point to the right entity (a builtin function name or a FuncDef).
-	tag, _ := t.scope.lookupFunc(name)
 	nargs := len(args)
-	// udf should be checked first since a udf can override builtin functions.
-	if f := t.funcs[tag]; f != nil {
-		if len(f.Params) != nargs {
-			t.error(call, fmt.Errorf("call expects %d argument(s)", len(f.Params)))
-			return badExpr()
-		}
-		return sem.NewCall(call, f.Tag, args)
-	}
-	if tag != "" {
-		name = tag
-	}
 	nameLower := strings.ToLower(name)
 	switch {
 	case nameLower == "map":
@@ -779,15 +779,19 @@ func (t *translator) semMapCall(call *ast.CallExpr, args []sem.Expr) sem.Expr {
 		t.error(call, errors.New("map requires two arguments"))
 		return badExpr()
 	}
-	f, ok := args[1].(*sem.FuncRef)
+	ref, ok := args[1].(*sem.FuncRef)
 	if !ok {
 		t.error(call, errors.New("second argument to map must be a function"))
 		return badExpr()
 	}
-	e := &sem.MapCallExpr{
-		Node:   call,
-		Expr:   args[0],
-		Lambda: sem.NewCall(call.Args[1], f.Tag, []sem.Expr{sem.NewThis(call.Args[1], nil)}),
+	mapArgs := []sem.Expr{sem.NewThis(call.Args[1], nil)}
+	e := t.resolver.resolveCall(call.Args[1], ref.ID, mapArgs)
+	if callExpr, ok := e.(*sem.CallExpr); ok {
+		return &sem.MapCallExpr{
+			Node:   call,
+			Expr:   args[0],
+			Lambda: callExpr,
+		}
 	}
 	return e
 }