compiler/semantic: enable type checking for all input files

cli/inputflags/flags.go

@@ -17,6 +17,7 @@ import (
 
 type Flags struct {
 	anyio.ReaderOpts
+	Dynamic  bool
 	ReadMax  auto.Bytes
 	ReadSize auto.Bytes
 	Threads  int
@@ -43,6 +44,7 @@ func (f *Flags) SetFlags(fs *flag.FlagSet, validate bool) {
 	fs.Var(&f.ReadMax, "bsup.readmax", "maximum Super Binary read buffer size in MiB, MB, etc.")
 	f.ReadSize = auto.NewBytes(bsupio.ReadSize)
 	fs.Var(&f.ReadSize, "bsup.readsize", "target Super Binary read buffer size in MiB, MB, etc.")
+	fs.BoolVar(&f.Dynamic, "dynamic", false, "disable static type checking of inputs")
 }
 
 // Init is called after flags have been parsed.

cmd/super/compile/shared.go

@@ -24,6 +24,7 @@ import (
 
 type Shared struct {
 	dag         bool
+	dynamic     bool
 	includes    queryflags.Includes
 	optimize    bool
 	parallel    int
@@ -33,6 +34,7 @@ type Shared struct {
 
 func (s *Shared) SetFlags(fs *flag.FlagSet) {
 	fs.BoolVar(&s.dag, "dag", false, "display output as DAG (implied by -O or -P)")
+	fs.BoolVar(&s.dynamic, "dynamic", false, "disable static type checking of inputs on DAG")
 	fs.Var(&s.includes, "I", "source file containing query text (may be repeated)")
 	fs.BoolVar(&s.optimize, "O", false, "display optimized DAG")
 	fs.IntVar(&s.parallel, "P", 0, "display parallelized DAG")
@@ -80,6 +82,7 @@ func (s *Shared) Run(ctx context.Context, args []string, dbFlags *dbflags.Flags,
 	}
 	rctx := runtime.DefaultContext()
 	env := exec.NewEnvironment(storage.NewLocalEngine(), root)
+	env.Dynamic = s.dynamic
 	dag, err := compiler.Analyze(rctx, ast, env, false)
 	if err != nil {
 		return err

cmd/super/root/command.go

@@ -147,6 +147,7 @@ func (c *Command) Run(args []string) error {
 		ast.PrependFileScan(args)
 	}
 	env := exec.NewEnvironment(storage.NewLocalEngine(), nil)
+	env.Dynamic = c.inputFlags.Dynamic
 	env.IgnoreOpenErrors = !c.stopErr
 	env.ReaderOpts = c.inputFlags.Options()
 	comp := compiler.NewCompilerWithEnv(env)

cmd/super/ztests/from-file-error.yaml

@@ -1,5 +1,5 @@
 script: |
-  ! super -I query.spq
+  ! super -dynamic -I query.spq
 
 inputs:
   - name: query.spq

cmd/super/ztests/stop-on-error-2.yaml

@@ -1,5 +1,5 @@
 script: |
-  super -s -e=false good.sup bad.sup
+  super -dynamic -e=false -s good.sup bad.sup
 
 inputs:
   - name: good.sup

cmd/super/ztests/stop-on-error-3.yaml

@@ -1,6 +1,6 @@
 # Second input has bad middle line (detection succeeds).
 script: |
-  ! super -s -e=false good.sup bad.sup
+  ! super -dynamic -e=false -s good.sup bad.sup
 
 inputs:
   - name: good.sup

compiler/parser/ztests/glob-mul.yaml

@@ -1,7 +1,7 @@
 script: |
-  super -s -c "? a*b" in.sup
+  super -dynamic -s -c "? a*b" in.sup
   echo ===
-  super -s -c "? a*b=s" in.sup
+  super -dynamic -s -c "? a*b=s" in.sup
 
 inputs:
   - name: in.sup

compiler/semantic/op.go

@@ -22,7 +22,7 @@ import (
 	"github.com/brimdata/super/runtime/sam/expr"
 	"github.com/brimdata/super/runtime/sam/expr/function"
 	"github.com/brimdata/super/sio"
-	"github.com/brimdata/super/sio/parquetio"
+	"github.com/brimdata/super/sio/anyio"
 	"github.com/brimdata/super/sup"
 	"github.com/segmentio/ksuid"
 )
@@ -269,8 +269,11 @@ func (t *translator) file(n ast.Node, name string, args []ast.OpArg) sem.Op {
 }
 
 func (t *translator) fileType(path, format string) (super.Type, error) {
+	if t.env.Dynamic {
+		return nil, nil
+	}
 	engine := t.env.Engine()
-	if engine == nil || format != "" && format != "auto" && format != "parquet" {
+	if engine == nil {
 		return nil, nil
 	}
 	uri, err := storage.ParseURI(path)
@@ -282,7 +285,27 @@ func (t *translator) fileType(path, format string) (super.Type, error) {
 		return nil, err
 	}
 	defer r.Close()
-	return parquetio.Type(t.sctx, r), nil
+	var b [1]byte
+	if _, err := r.ReadAt(b[:], 0); err != nil {
+		// r can't seek so it's a fifo or pipe.
+		return nil, nil
+	}
+	f, err := anyio.NewFile(t.sctx, r, path, anyio.ReaderOpts{Format: format})
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	if typer, ok := f.Reader.(interface{ Type() super.Type }); ok {
+		return typer.Type(), nil
+	}
+	fuser := t.checker.newFuser()
+	for {
+		val, err := f.Read()
+		if val == nil || err != nil {
+			return fuser.Type(), err
+		}
+		fuser.fuse(val.Type())
+	}
 }
 
 func (t *translator) fromFileGlob(globLoc ast.Node, pattern string, args []ast.OpArg) sem.Op {

compiler/semantic/ztests/pipe-schema-parquet.yaml

@@ -2,7 +2,8 @@ script: |
   super -o x.parquet -f parquet x.sup
   super -o y.parquet -f parquet y.sup
   super -s -c "select x from x.parquet join y.parquet on x=y"
-  ! super -s -c "select x from x.parquet join y.sup on x=y"
+  touch z.sup
+  ! super -s -c "select x from x.parquet join z.sup on x=y"
 
 inputs:
   - name: x.sup
@@ -20,8 +21,8 @@ outputs:
   - name: stderr
     data: |
       "x": ambiguous column reference at line 1, column 39:
-      select x from x.parquet join y.sup on x=y
+      select x from x.parquet join z.sup on x=y
                                             ~
       "x": ambiguous column reference at line 1, column 8:
-      select x from x.parquet join y.sup on x=y
+      select x from x.parquet join z.sup on x=y
              ~

compiler/sfmt/ztests/decls.yaml

@@ -1,7 +1,7 @@
 script: |
   super compile -C -I test.spq
   echo "==="
-  super compile -dag -C -I test.spq
+  super compile -C -dag -dynamic -I test.spq
 
 inputs:
   - name: test.spq

compiler/sfmt/ztests/join.yaml

@@ -1,11 +1,11 @@
 script: |
   super compile -C "join (from test.sup) on left.x=right.x"
   echo ===
-  super compile -C -dag "join (from test.sup) on right.x=left.x"
+  super compile -C -dag -dynamic "join (from test.sup) on right.x=left.x"
   echo ===
   super compile -C "right join (from test.sup) as {l,r} on r.x=l.x"
   echo ===
-  super compile -C -dag "right join (from test.sup) as {l,r} on r.x=l.x"
+  super compile -C -dag -dynamic "right join (from test.sup) as {l,r} on r.x=l.x"
 
 outputs:
   - name: stdout

compiler/ztests/anycase-funcs.yaml

@@ -1,7 +1,7 @@
 script: |
   super -s -c 'c:=COUNT(),d:=Count(),Collect(LoweR(s)) by key | sort key' in.sup
   echo ===
-  super -s -c 'Grep("G.*", this)' in.sup
+  super -dynamic -s -c 'Grep("G.*", this)' in.sup
   echo ===
   super -s -c 'values {s,match:RegEXP("(f|B).*", s)}' in.sup
   echo ===

compiler/ztests/const-source.yaml

@@ -2,14 +2,14 @@ script: |
   export SUPER_DB=test
   super db init -q
   super db create -q test
-  super db compile -dag -C 'const POOL = "test" from eval(POOL)' | sed -e "s/[a-zA-Z0-9]\{27\}/XXX/"
+  super db compile -C -dag 'const POOL = "test" from eval(POOL)' | sed -e "s/[a-zA-Z0-9]\{27\}/XXX/"
   echo "==="
-  super compile -dag -C 'const FILE = "A.sup" from eval(FILE)'
+  super compile -C -dag -dynamic 'const FILE = "A.sup" from eval(FILE)'
   echo "==="
-  super db compile -dag -C 'const URL = "http://brimdata.io" from eval(URL)'
-  ! super db compile -dag -C 'const POOL = 3.14 from eval(POOL)'
-  ! super db compile -dag -C 'const FILE = 127.0.0.1 from eval(FILE)'
-  ! super db compile -dag -C 'const URL = true from eval(URL)'
+  super db compile -C -dag 'const URL = "http://brimdata.io" from eval(URL)'
+  ! super db compile -C -dag 'const POOL = 3.14 from eval(POOL)'
+  ! super db compile -C -dag 'const FILE = 127.0.0.1 from eval(FILE)'
+  ! super db compile -C -dag 'const URL = true from eval(URL)'
 
 outputs:
   - name: stdout

compiler/ztests/merge-filters.yaml

@@ -1,7 +1,7 @@
 script: |
   super compile -C -O 'from /dev/null | where a | where b'
   echo ===
-  super compile -C -O 'fork ( from /dev/null | where b | where c ) ( from /dev/zero | where e | where f ) | where g'
+  super compile -C -O -dynamic 'fork ( from /dev/null | where b | where c ) ( from /dev/zero | where e | where f ) | where g'
   echo ===
   super compile -C -O 'unnest [a] into ( where b | where c )'
   echo ===

compiler/ztests/par-aggregate-func.yaml

@@ -4,7 +4,7 @@ script: |
   super db create -q -orderby s:asc test
   super db compile -P 2 -C "from test | union(s) by n:=len(s)" | sed -e 's/pool .*/.../'
   echo ===
-  SUPER_VAM=1 super compile -C -P 2 'from test.csup | summarize count(a) by b'
+  SUPER_VAM=1 super compile -C -P 2 -dynamic 'from test.csup | summarize count(a) by b'
 
 outputs:
   - name: stdout

compiler/ztests/par-concurrency.yaml

@@ -1,11 +1,11 @@
 script: |
   export SUPER_VAM=1
   echo === -P 2
-  super compile -C -P 2 'fork ( from a.csup | sort b ) ( from c.csup | sort d )'
+  super compile -C -P 2 -dynamic 'fork ( from a.csup | sort b ) ( from c.csup | sort d )'
   echo === -P 5
-  super compile -C -P 5 'fork ( from a.csup | sort b ) ( from c.csup | sort d )'
+  super compile -C -P 5 -dynamic 'fork ( from a.csup | sort b ) ( from c.csup | sort d )'
   echo === -P 6
-  super compile -C -P 6 'fork ( from a.csup | sort b ) ( from c.csup | sort d )'
+  super compile -C -P 6 -dynamic 'fork ( from a.csup | sort b ) ( from c.csup | sort d )'
 
 outputs:
   - name: stdout

compiler/ztests/par-sort.yaml

@@ -1,5 +1,5 @@
 script: |
-  SUPER_VAM=1 super compile -C -P 2 'from test.csup | sort a, b desc nulls last | values c'
+  SUPER_VAM=1 super compile -C -P 2 -dynamic 'from test.csup | sort a, b desc nulls last | values c'
 
 outputs:
   - name: stdout

compiler/ztests/par-top.yaml

@@ -8,7 +8,7 @@ script: |
   echo ===
   super db compile -C -P 2 'from test | top 3 b desc' | sed -e 's/pool .*/.../'
   echo ===
-  SUPER_VAM=1 super compile -C -P 2 'from test.csup | top 3 a, b desc nulls last | values c'
+  SUPER_VAM=1 super compile -C -P 2 -dynamic 'from test.csup | top 3 a, b desc nulls last | values c'
 
 outputs:
   - name: stdout

compiler/ztests/pruner-in.yaml

@@ -1,9 +1,9 @@
 script: |
   export SUPER_VAM=1
-  super compile -C -O 'from test.csup | x in ["foo","bar"]'
+  super compile -C -O -dynamic 'from test.csup | x in ["foo","bar"]'
   echo // ===
   # Test that we still optimize for a tuple which gets translated into a record.
-  super compile -C -O 'from test.csup | x in ("foo","bar")'
+  super compile -C -O -dynamic 'from test.csup | x in ("foo","bar")'
 
 outputs:
   - name: stdout

compiler/ztests/pruner-regexp.yaml

@@ -1,10 +1,10 @@
 script: |
   export SUPER_VAM=1
-  super compile -C -O 'from test.csup | where x LIKE "cslab%"'
+  super compile -C -O -dynamic 'from test.csup | where x LIKE "cslab%"'
   echo // ===
-  super compile -C -O 'from test.csup | where grep("^csla[bB].*", x)'
+  super compile -C -O -dynamic 'from test.csup | where grep("^csla[bB].*", x)'
   echo // ===
-  super compile -C -O 'from test.csup | where grep("csla[bB].*", x)'
+  super compile -C -O -dynamic 'from test.csup | where grep("csla[bB].*", x)'
   echo // ===
   echo '{x:"a"}' | super -f csup -o x.csup -
   super -s -c "SELECT x FROM x.csup WHERE x LIKE 'a%'"

compiler/ztests/pushdown.yaml

@@ -4,13 +4,13 @@ script: |
   echo === distinct
   super compile -C -O 'from /dev/null | distinct a | values b'
   echo === fork into hash join
-  super compile -C -O 'fork ( from /dev/null ) ( from /dev/zero ) | join on right.a=left.b | values left.c,right.d'
+  super compile -C -O -dynamic 'fork ( from /dev/null ) ( from /dev/zero ) | join on right.a=left.b | values left.c,right.d'
   echo === fork into nested loop join
-  super compile -C -O 'fork ( from /dev/null ) ( from /dev/zero ) | join on right.a>left.b | values left.c,right.d'
+  super compile -C -O -dynamic 'fork ( from /dev/null ) ( from /dev/zero ) | join on right.a>left.b | values left.c,right.d'
   echo === hash join
-  super compile -C -O "from /dev/null | join (from /dev/zero) on left.a=right.b | values left.c,right.d"
+  super compile -C -O -dynamic "from /dev/null | join (from /dev/zero) on left.a=right.b | values left.c,right.d"
   echo === nested loop join
-  super compile -C -O "from /dev/null | join (from /dev/zero) on left.a>right.b | values left.c,right.d"
+  super compile -C -O -dynamic "from /dev/null | join (from /dev/zero) on left.a>right.b | values left.c,right.d"
   echo === switch into hash join
   super compile -C -O 'from /dev/null | switch a case b ( put x:=c ) case d ( put x:=e ) | join on left.f=right.g | values left.h,right.i'
   echo === switch into nested loop join

compiler/ztests/sql/drop.yaml

@@ -1,7 +1,7 @@
 script: |
   super -s -c 'select * from "a.sup" | drop c'
   echo ===
-  super -s -c 'select * from "messy.sup" | drop s,t'
+  super -dynamic -s -c 'select * from "messy.sup" | drop s,t'
   echo ===
   super -s -c 'select * from "b.sup" | drop b'
 

compiler/ztests/sql/scope-errors.yaml

@@ -1,11 +1,11 @@
 script: |
-  ! super -s -c "select * from (select 1 as a, 2 as b) t1 join (select 2 as b) on a=b"
+  ! super -dynamic -s -c "select * from (select 1 as a, 2 as b) t1 join (select 2 as b) on a=b"
   echo === 1>&2
-  ! super -s -c "select * from a.json join b.json on a=b"
+  ! super -dynamic -s -c "select * from a.json join b.json on a=b"
   echo === 1>&2
-  ! super -s -c "select * from (select 2 as x) as a join b.json on x=b.c"
+  ! super -dynamic -s -c "select * from (select 2 as x) as a join b.json on x=b.c"
   echo === 1>&2
-  ! super -s -c "select * from (select 2 as x) a join b.json a on a.x=a.y"
+  ! super -dynamic -s -c "select * from (select 2 as x) a join b.json a on a.x=a.y"
 
 outputs:
   - name: stderr

compiler/ztests/sql/union-error.yaml

@@ -1,7 +1,7 @@
 script: |
   ! super -c 'select 1 x union all select 1 x, 1 y'
   echo // === >&2
-  ! super -c 'select * from a.sup union all select * from a.sup'
+  ! super -dynamic -c 'select * from a.sup union all select * from a.sup'
 
 inputs:
   - name: a.sup

runtime/exec/environment.go

@@ -32,6 +32,7 @@ type Environment struct {
 	db     *db.Root
 	useVAM bool
 
+	Dynamic          bool
 	IgnoreOpenErrors bool
 	ReaderOpts       anyio.ReaderOpts
 }

runtime/ztests/expr/queryexpr-unnest.yaml

@@ -1,5 +1,5 @@
 script: |
-  super -s -c 'from input.sup
+  super -dynamic -s -c 'from input.sup
             | values (unnest {outer: this, inner: (from foo.sup)} | where inner.id=outer.id | values inner.name) +
               "_" +
               (unnest {outer: this, inner: (from bar.sup)} | where inner.id=outer.id | values inner.name)'

runtime/ztests/op/join-empty-inner.yaml

@@ -1,8 +1,8 @@
 script: |
   echo === hash join
-  super -s -c 'left join (from C.sup) on left.a=right.a | values {...left,hit:right.sc} | sort' A.sup
+  super -dynamic -s -c 'left join (from C.sup) on left.a=right.a | values {...left,hit:right.sc} | sort' A.sup
   echo === nested loop join
-  super -s -c 'left join (from C.sup) on left.a<right.a | values {...left,hit:right.sc} | sort' A.sup
+  super -dynamic -s -c 'left join (from C.sup) on left.a<right.a | values {...left,hit:right.sc} | sort' A.sup
 
 vector: true
 

sio/parquetio/reader.go

@@ -56,12 +56,3 @@ func columnIndexes(schema *schema.Schema, fields []field.Path) []int {
 	}
 	return indexes
 }
-
-func Type(sctx *super.Context, r io.Reader) super.Type {
-	if ar, err := NewReader(sctx, r, nil); err == nil {
-		typ := ar.Type()
-		ar.Close()
-		return typ
-	}
-	return nil
-}