compiler: refactor and add tests

This commit finally introduces unit tests for the compiler, to check
whether input Go code is converted to the expected output IR.

To make this necessary, a few refactors were needed. Hopefully these
refactors (to compile a program package by package instead of all at
once) will eventually become standard, so that packages can all be
compiled separate from each other and be cached between compiles.

Author: aykevl

.circleci/config.yml

@@ -114,7 +114,7 @@ commands:
           key: wasi-libc-sysroot-systemclang-v2
           paths:
             - lib/wasi-libc/sysroot
-      - run: go test -v -tags=llvm<<parameters.llvm>> ./cgo ./compileopts ./interp ./transform .
+      - run: go test -v -tags=llvm<<parameters.llvm>> ./cgo ./compileopts ./compiler ./interp ./transform .
       - run: make gen-device -j4
       - run: make smoketest XTENSA=0
       - run: make tinygo-test

Makefile

@@ -163,7 +163,7 @@ tinygo:
 	CGO_CPPFLAGS="$(CGO_CPPFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GO) build -buildmode exe -o build/tinygo$(EXE) -tags byollvm -ldflags="-X main.gitSha1=`git rev-parse --short HEAD`" .
 
 test: wasi-libc
-	CGO_CPPFLAGS="$(CGO_CPPFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GO) test -v -buildmode exe -tags byollvm ./cgo ./compileopts ./interp ./transform .
+	CGO_CPPFLAGS="$(CGO_CPPFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GO) test -v -buildmode exe -tags byollvm ./cgo ./compileopts ./compiler ./interp ./transform .
 
 # Test known-working standard library packages.
 # TODO: do this in one command, parallelize, and only show failing tests (no

builder/build.go

@@ -8,6 +8,7 @@ import (
 	"encoding/binary"
 	"errors"
 	"fmt"
+	"go/types"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -19,6 +20,7 @@ import (
 	"github.com/tinygo-org/tinygo/compiler"
 	"github.com/tinygo-org/tinygo/goenv"
 	"github.com/tinygo-org/tinygo/interp"
+	"github.com/tinygo-org/tinygo/loader"
 	"github.com/tinygo-org/tinygo/stacksize"
 	"github.com/tinygo-org/tinygo/transform"
 	"tinygo.org/x/go-llvm"
@@ -43,16 +45,31 @@ type BuildResult struct {
 // The error value may be of type *MultiError. Callers will likely want to check
 // for this case and print such errors individually.
 func Build(pkgName, outpath string, config *compileopts.Config, action func(BuildResult) error) error {
-	// Compile Go code to IR.
+	// Load the target machine, which is the LLVM object that contains all
+	// details of a target (alignment restrictions, pointer size, default
+	// address spaces, etc).
 	machine, err := compiler.NewTargetMachine(config)
 	if err != nil {
 		return err
 	}
-	buildOutput, errs := compiler.Compile(pkgName, machine, config)
+
+	// Load entire program AST into memory.
+	lprogram, err := loader.Load(config, []string{pkgName}, config.ClangHeaders, types.Config{
+		Sizes: compiler.Sizes(machine),
+	})
+	if err != nil {
+		return err
+	}
+	err = lprogram.Parse()
+	if err != nil {
+		return err
+	}
+
+	// Compile AST to IR.
+	mod, errs := compiler.CompileProgram(pkgName, lprogram, machine, config)
 	if errs != nil {
 		return newMultiError(errs)
 	}
-	mod := buildOutput.Mod
 
 	if config.Options.PrintIR {
 		fmt.Println("; Generated LLVM IR:")
@@ -208,17 +225,21 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 		}
 
 		// Compile C files in packages.
-		for i, file := range buildOutput.ExtraFiles {
-			outpath := filepath.Join(dir, "pkg"+strconv.Itoa(i)+"-"+filepath.Base(file)+".o")
-			err := runCCompiler(config.Target.Compiler, append(config.CFlags(), "-c", "-o", outpath, file)...)
-			if err != nil {
-				return &commandError{"failed to build", file, err}
+		// Gather the list of (C) file paths that should be included in the build.
+		for i, pkg := range lprogram.Sorted() {
+			for j, filename := range pkg.CFiles {
+				file := filepath.Join(pkg.Dir, filename)
+				outpath := filepath.Join(dir, "pkg"+strconv.Itoa(i)+"."+strconv.Itoa(j)+"-"+filepath.Base(file)+".o")
+				err := runCCompiler(config.Target.Compiler, append(config.CFlags(), "-c", "-o", outpath, file)...)
+				if err != nil {
+					return &commandError{"failed to build", file, err}
+				}
+				ldflags = append(ldflags, outpath)
 			}
-			ldflags = append(ldflags, outpath)
 		}
 
-		if len(buildOutput.ExtraLDFlags) > 0 {
-			ldflags = append(ldflags, buildOutput.ExtraLDFlags...)
+		if len(lprogram.LDFlags) > 0 {
+			ldflags = append(ldflags, lprogram.LDFlags...)
 		}
 
 		// Link the object files together.
@@ -304,7 +325,7 @@ func Build(pkgName, outpath string, config *compileopts.Config, action func(Buil
 		}
 		return action(BuildResult{
 			Binary:  tmppath,
-			MainDir: buildOutput.MainDir,
+			MainDir: lprogram.MainPkg().Dir,
 		})
 	}
 }

compiler/compiler.go

@@ -9,6 +9,7 @@ import (
 	"go/token"
 	"go/types"
 	"path/filepath"
+	"sort"
 	"strconv"
 	"strings"
 
@@ -53,6 +54,45 @@ type compilerContext struct {
 	astComments      map[string]*ast.CommentGroup
 }
 
+// newCompilerContext returns a new compiler context ready for use, most
+// importantly with a newly created LLVM context and module.
+func newCompilerContext(moduleName string, machine llvm.TargetMachine, config *compileopts.Config) *compilerContext {
+	c := &compilerContext{
+		Config:     config,
+		difiles:    make(map[string]llvm.Metadata),
+		ditypes:    make(map[types.Type]llvm.Metadata),
+		machine:    machine,
+		targetData: machine.CreateTargetData(),
+	}
+
+	c.ctx = llvm.NewContext()
+	c.mod = c.ctx.NewModule(moduleName)
+	c.mod.SetTarget(config.Triple())
+	c.mod.SetDataLayout(c.targetData.String())
+	if c.Debug() {
+		c.dibuilder = llvm.NewDIBuilder(c.mod)
+	}
+
+	c.uintptrType = c.ctx.IntType(c.targetData.PointerSize() * 8)
+	if c.targetData.PointerSize() <= 4 {
+		// 8, 16, 32 bits targets
+		c.intType = c.ctx.Int32Type()
+	} else if c.targetData.PointerSize() == 8 {
+		// 64 bits target
+		c.intType = c.ctx.Int64Type()
+	} else {
+		panic("unknown pointer size")
+	}
+	c.i8ptrType = llvm.PointerType(c.ctx.Int8Type(), 0)
+
+	dummyFuncType := llvm.FunctionType(c.ctx.VoidType(), nil, false)
+	dummyFunc := llvm.AddFunction(c.mod, "tinygo.dummy", dummyFuncType)
+	c.funcPtrAddrSpace = dummyFunc.Type().PointerAddressSpace()
+	dummyFunc.EraseFromParentAsFunction()
+
+	return c
+}
+
 // builder contains all information relevant to build a single function.
 type builder struct {
 	*compilerContext
@@ -76,6 +116,18 @@ type builder struct {
 	deferBuiltinFuncs map[ssa.Value]deferBuiltin
 }
 
+func newBuilder(c *compilerContext, irbuilder llvm.Builder, f *ir.Function) *builder {
+	return &builder{
+		compilerContext: c,
+		Builder:         irbuilder,
+		fn:              f,
+		locals:          make(map[ssa.Value]llvm.Value),
+		dilocals:        make(map[*types.Var]llvm.Metadata),
+		blockEntries:    make(map[*ssa.BasicBlock]llvm.BasicBlock),
+		blockExits:      make(map[*ssa.BasicBlock]llvm.BasicBlock),
+	}
+}
+
 type deferBuiltin struct {
 	funcName string
 	callback int
@@ -127,94 +179,47 @@ func NewTargetMachine(config *compileopts.Config) (llvm.TargetMachine, error) {
 	return machine, nil
 }
 
-// CompilerOutput is returned from the Compile() call. It contains the compile
-// output and information necessary to continue to compile and link the program.
-type CompilerOutput struct {
-	// The LLVM module that contains the compiled but not optimized LLVM module
-	// for all the Go code in the program.
-	Mod llvm.Module
-
-	// ExtraFiles is a list of C source files included in packages that should
-	// be built and linked together with the main executable to form one
-	// program. They can be used from CGo, for example.
-	ExtraFiles []string
-
-	// ExtraLDFlags are linker flags obtained during CGo processing. These flags
-	// must be passed to the linker which links the entire executable.
-	ExtraLDFlags []string
-
-	// MainDir is the absolute directory path to the directory of the main
-	// package. This is useful for testing: tests must be run in the package
-	// directory that is being tested.
-	MainDir string
-}
+// Sizes returns a types.Sizes appropriate for the given target machine. It
+// includes the correct int size and aligment as is necessary for the Go
+// typechecker.
+func Sizes(machine llvm.TargetMachine) types.Sizes {
+	targetData := machine.CreateTargetData()
+	defer targetData.Dispose()
 
-// Compile the given package path or .go file path. Return an error when this
-// fails (in any stage). If successful it returns the LLVM module and a list of
-// extra C files to be compiled. If not, one or more errors will be returned.
-//
-// The fact that it returns a list of filenames to compile is a layering
-// violation. Eventually, this Compile function should only compile a single
-// package and not the whole program, and loading of the program (including CGo
-// processing) should be moved outside the compiler package.
-func Compile(pkgName string, machine llvm.TargetMachine, config *compileopts.Config) (output CompilerOutput, errors []error) {
-	c := &compilerContext{
-		Config:     config,
-		difiles:    make(map[string]llvm.Metadata),
-		ditypes:    make(map[types.Type]llvm.Metadata),
-		machine:    machine,
-		targetData: machine.CreateTargetData(),
+	intPtrType := targetData.IntPtrType()
+	if intPtrType.IntTypeWidth()/8 <= 32 {
 	}
 
-	c.ctx = llvm.NewContext()
-	c.mod = c.ctx.NewModule(pkgName)
-	c.mod.SetTarget(config.Triple())
-	c.mod.SetDataLayout(c.targetData.String())
-	if c.Debug() {
-		c.dibuilder = llvm.NewDIBuilder(c.mod)
-	}
-	output.Mod = c.mod
-
-	c.uintptrType = c.ctx.IntType(c.targetData.PointerSize() * 8)
-	if c.targetData.PointerSize() <= 4 {
+	var intWidth int
+	if targetData.PointerSize() <= 4 {
 		// 8, 16, 32 bits targets
-		c.intType = c.ctx.Int32Type()
-	} else if c.targetData.PointerSize() == 8 {
+		intWidth = 32
+	} else if targetData.PointerSize() == 8 {
 		// 64 bits target
-		c.intType = c.ctx.Int64Type()
+		intWidth = 64
 	} else {
 		panic("unknown pointer size")
 	}
-	c.i8ptrType = llvm.PointerType(c.ctx.Int8Type(), 0)
 
-	dummyFuncType := llvm.FunctionType(c.ctx.VoidType(), nil, false)
-	dummyFunc := llvm.AddFunction(c.mod, "tinygo.dummy", dummyFuncType)
-	c.funcPtrAddrSpace = dummyFunc.Type().PointerAddressSpace()
-	dummyFunc.EraseFromParentAsFunction()
-
-	lprogram, err := loader.Load(c.Config, []string{pkgName}, c.ClangHeaders, types.Config{
-		Sizes: &stdSizes{
-			IntSize:  int64(c.targetData.TypeAllocSize(c.intType)),
-			PtrSize:  int64(c.targetData.PointerSize()),
-			MaxAlign: int64(c.targetData.PrefTypeAlignment(c.i8ptrType)),
-		}})
-	if err != nil {
-		return output, []error{err}
+	return &stdSizes{
+		IntSize:  int64(intWidth / 8),
+		PtrSize:  int64(targetData.PointerSize()),
+		MaxAlign: int64(targetData.PrefTypeAlignment(intPtrType)),
 	}
+}
 
-	err = lprogram.Parse()
-	if err != nil {
-		return output, []error{err}
-	}
-	output.ExtraLDFlags = lprogram.LDFlags
-	output.MainDir = lprogram.MainPkg().Dir
+// CompileProgram compiles the given package path or .go file path. Return an
+// error when this fails (in any stage). If successful it returns the LLVM
+// module. If not, one or more errors will be returned.
+func CompileProgram(pkgName string, lprogram *loader.Program, machine llvm.TargetMachine, config *compileopts.Config) (llvm.Module, []error) {
+	c := newCompilerContext(pkgName, machine, config)
 
 	c.ir = ir.NewProgram(lprogram)
 
 	// Run a simple dead code elimination pass.
-	err = c.ir.SimpleDCE()
+	err := c.ir.SimpleDCE()
 	if err != nil {
-		return output, []error{err}
+		return llvm.Module{}, []error{err}
 	}
 
 	// Initialize debug information.
@@ -265,15 +270,7 @@ func Compile(pkgName string, machine llvm.TargetMachine, config *compileopts.Con
 		}
 
 		// Create the function definition.
-		b := builder{
-			compilerContext: c,
-			Builder:         irbuilder,
-			fn:              f,
-			locals:          make(map[ssa.Value]llvm.Value),
-			dilocals:        make(map[*types.Var]llvm.Metadata),
-			blockEntries:    make(map[*ssa.BasicBlock]llvm.BasicBlock),
-			blockExits:      make(map[*ssa.BasicBlock]llvm.BasicBlock),
-		}
+		b := newBuilder(c, irbuilder, f)
 		b.createFunctionDefinition()
 	}
 
@@ -352,14 +349,64 @@ func Compile(pkgName string, machine llvm.TargetMachine, config *compileopts.Con
 		c.dibuilder.Finalize()
 	}
 
-	// Gather the list of (C) file paths that should be included in the build.
-	for _, pkg := range c.ir.LoaderProgram.Sorted() {
-		for _, filename := range pkg.CFiles {
-			output.ExtraFiles = append(output.ExtraFiles, filepath.Join(pkg.Dir, filename))
+	return c.mod, c.diagnostics
+}
+
+// CompilePackage compiles a single package to a LLVM module.
+func CompilePackage(moduleName string, pkg *loader.Package, machine llvm.TargetMachine, config *compileopts.Config) (llvm.Module, []error) {
+	c := newCompilerContext(moduleName, machine, config)
+
+	// Build SSA from AST.
+	ssaPkg := pkg.LoadSSA()
+	ssaPkg.Build()
+
+	// Sort by position, so that the order of the functions in the IR matches
+	// the order of functions in the source file. This is useful for testing,
+	// for example.
+	var members []string
+	for name := range ssaPkg.Members {
+		members = append(members, name)
+	}
+	sort.Slice(members, func(i, j int) bool {
+		iPos := ssaPkg.Members[members[i]].Pos()
+		jPos := ssaPkg.Members[members[j]].Pos()
+		if i == j {
+			// Cannot sort by pos, so do it by name.
+			return members[i] < members[j]
+		}
+		return iPos < jPos
+	})
+
+	// Create *ir.Functions objects.
+	var functions []*ir.Function
+	for _, name := range members {
+		member := ssaPkg.Members[name]
+		switch member := member.(type) {
+		case *ssa.Function:
+			functions = append(functions, &ir.Function{
+				Function: member,
+			})
 		}
 	}
 
-	return output, c.diagnostics
+	// Declare all functions.
+	for _, fn := range functions {
+		c.createFunctionDeclaration(fn)
+	}
+
+	// Add definitions to declarations.
+	irbuilder := c.ctx.NewBuilder()
+	defer irbuilder.Dispose()
+	for _, f := range functions {
+		if f.Blocks == nil {
+			continue // external function
+		}
+		// Create the function definition.
+		b := newBuilder(c, irbuilder, f)
+		b.createFunctionDefinition()
+	}
+
+	return c.mod, nil
 }
 
 // getLLVMRuntimeType obtains a named type from the runtime package and returns