reflect: add support for linked lists

Linked lists are usually implemented as follows:

    type linkedList struct {
        next *linkedList
        data int // whatever
    }

This caused a stack overflow while writing out the reflect run-time type
information. This has now been fixed by splitting the allocation of a
named type number from setting the underlying type in the sidetable.

Author: aykevl

compiler/interface.go

@@ -92,7 +92,7 @@ func (c *Compiler) makeStructTypeFields(typ *types.Struct) llvm.Value {
 	for i := 0; i < typ.NumFields(); i++ {
 		fieldGlobalValue := c.getZeroValue(runtimeStructField)
 		fieldGlobalValue = llvm.ConstInsertValue(fieldGlobalValue, c.getTypeCode(typ.Field(i).Type()), []uint32{0})
-		fieldName := c.makeGlobalBytes([]byte(typ.Field(i).Name()), "reflect/types.structFieldName")
+		fieldName := c.makeGlobalArray([]byte(typ.Field(i).Name()), "reflect/types.structFieldName", c.ctx.Int8Type())
 		fieldName.SetLinkage(llvm.PrivateLinkage)
 		fieldName.SetUnnamedAddr(true)
 		fieldName = llvm.ConstGEP(fieldName, []llvm.Value{
@@ -101,7 +101,7 @@ func (c *Compiler) makeStructTypeFields(typ *types.Struct) llvm.Value {
 		})
 		fieldGlobalValue = llvm.ConstInsertValue(fieldGlobalValue, fieldName, []uint32{1})
 		if typ.Tag(i) != "" {
-			fieldTag := c.makeGlobalBytes([]byte(typ.Tag(i)), "reflect/types.structFieldTag")
+			fieldTag := c.makeGlobalArray([]byte(typ.Tag(i)), "reflect/types.structFieldTag", c.ctx.Int8Type())
 			fieldTag = llvm.ConstGEP(fieldTag, []llvm.Value{
 				llvm.ConstInt(llvm.Int32Type(), 0, false),
 				llvm.ConstInt(llvm.Int32Type(), 0, false),

compiler/llvm.go

@@ -1,6 +1,8 @@
 package compiler
 
 import (
+	"reflect"
+
 	"tinygo.org/x/go-llvm"
 )
 
@@ -153,24 +155,26 @@ func (c *Compiler) splitBasicBlock(afterInst llvm.Value, insertAfter llvm.BasicB
 	return newBlock
 }
 
-// makeGlobalBytes creates a new LLVM global with the given name and bytes as
+// makeGlobalArray creates a new LLVM global with the given name and integers as
 // contents, and returns the global.
 // Note that it is left with the default linkage etc., you should set
 // linkage/constant/etc properties yourself.
-func (c *Compiler) makeGlobalBytes(buf []byte, name string) llvm.Value {
-	globalType := llvm.ArrayType(c.ctx.Int8Type(), len(buf))
+func (c *Compiler) makeGlobalArray(bufItf interface{}, name string, elementType llvm.Type) llvm.Value {
+	buf := reflect.ValueOf(bufItf)
+	globalType := llvm.ArrayType(elementType, buf.Len())
 	global := llvm.AddGlobal(c.mod, globalType, name)
 	value := llvm.Undef(globalType)
-	for i, ch := range buf {
-		value = llvm.ConstInsertValue(value, llvm.ConstInt(c.ctx.Int8Type(), uint64(ch), false), []uint32{uint32(i)})
+	for i := 0; i < buf.Len(); i++ {
+		ch := buf.Index(i).Uint()
+		value = llvm.ConstInsertValue(value, llvm.ConstInt(elementType, ch, false), []uint32{uint32(i)})
 	}
 	global.SetInitializer(value)
 	return global
 }
 
-// getGlobalBytes returns the byte slice contained in the i8 array of the
-// provided global. It can recover the bytes originally created using
-// makeGlobalBytes.
+// getGlobalBytes returns the slice contained in the array of the provided
+// global. It can recover the bytes originally created using makeGlobalArray, if
+// makeGlobalArray was given a byte slice.
 func getGlobalBytes(global llvm.Value) []byte {
 	value := global.Initializer()
 	buf := make([]byte, value.Type().ArrayLength())
@@ -180,12 +184,12 @@ func getGlobalBytes(global llvm.Value) []byte {
 	return buf
 }
 
-// replaceGlobalByteWithArray replaces a global i8 in the module with a byte
-// array, using a GEP to make the types match. It is a convenience function used
-// for creating reflection sidetables, for example.
-func (c *Compiler) replaceGlobalByteWithArray(name string, buf []byte) llvm.Value {
-	global := c.makeGlobalBytes(buf, name+".tmp")
+// replaceGlobalByteWithArray replaces a global integer type in the module with
+// an integer array, using a GEP to make the types match. It is a convenience
+// function used for creating reflection sidetables, for example.
+func (c *Compiler) replaceGlobalIntWithArray(name string, buf interface{}) llvm.Value {
 	oldGlobal := c.mod.NamedGlobal(name)
+	global := c.makeGlobalArray(buf, name+".tmp", oldGlobal.Type().ElementType())
 	gep := llvm.ConstGEP(global, []llvm.Value{
 		llvm.ConstInt(c.ctx.Int32Type(), 0, false),
 		llvm.ConstInt(c.ctx.Int32Type(), 0, false),

compiler/reflect.go

@@ -37,7 +37,7 @@ import (
 )
 
 // A list of basic types and their numbers. This list should be kept in sync
-// with the list of Kind constants of type.go in the runtime package.
+// with the list of Kind constants of type.go in the reflect package.
 var basicTypes = map[string]int64{
 	"bool":       1,
 	"int":        2,
@@ -59,6 +59,19 @@ var basicTypes = map[string]int64{
 	"unsafeptr":  18,
 }
 
+// A list of non-basic types. Adding 19 to this number will give the Kind as
+// used in src/reflect/types.go, and it must be kept in sync with that list.
+var nonBasicTypes = map[string]int64{
+	"chan":      0,
+	"interface": 1,
+	"pointer":   2,
+	"slice":     3,
+	"array":     4,
+	"func":      5,
+	"map":       6,
+	"struct":    7,
+}
+
 // typeCodeAssignmentState keeps some global state around for type code
 // assignments, used to assign one unique type code to each Go type.
 type typeCodeAssignmentState struct {
@@ -96,7 +109,7 @@ type typeCodeAssignmentState struct {
 	// Note that this byte buffer is not created when it is not needed
 	// (reflect.namedNonBasicTypesSidetable has no uses), see
 	// needsNamedTypesSidetable.
-	namedNonBasicTypesSidetable []byte
+	namedNonBasicTypesSidetable []uint64
 
 	// This indicates whether namedNonBasicTypesSidetable needs to be created at
 	// all. If it is false, namedNonBasicTypesSidetable will contain simple
@@ -144,17 +157,17 @@ func (c *Compiler) assignTypeCodes(typeSlice typeInfoSlice) {
 
 	// Only create this sidetable when it is necessary.
 	if state.needsNamedNonBasicTypesSidetable {
-		global := c.replaceGlobalByteWithArray("reflect.namedNonBasicTypesSidetable", state.namedNonBasicTypesSidetable)
+		global := c.replaceGlobalIntWithArray("reflect.namedNonBasicTypesSidetable", state.namedNonBasicTypesSidetable)
 		global.SetLinkage(llvm.InternalLinkage)
 		global.SetUnnamedAddr(true)
 	}
 	if state.needsStructTypesSidetable {
-		global := c.replaceGlobalByteWithArray("reflect.structTypesSidetable", state.structTypesSidetable)
+		global := c.replaceGlobalIntWithArray("reflect.structTypesSidetable", state.structTypesSidetable)
 		global.SetLinkage(llvm.InternalLinkage)
 		global.SetUnnamedAddr(true)
 	}
 	if state.needsStructNamesSidetable {
-		global := c.replaceGlobalByteWithArray("reflect.structNamesSidetable", state.structNamesSidetable)
+		global := c.replaceGlobalIntWithArray("reflect.structNamesSidetable", state.structNamesSidetable)
 		global.SetLinkage(llvm.InternalLinkage)
 		global.SetUnnamedAddr(true)
 	}
@@ -194,49 +207,76 @@ func (state *typeCodeAssignmentState) getTypeCodeNum(typecode llvm.Value) *big.I
 		// (channel, interface, pointer, slice) just contain the bits of the
 		// wrapped type. Other types (like struct) need more fields and thus
 		// cannot be encoded as a simple prefix.
-		var num *big.Int
 		var classNumber int64
-		switch class {
-		case "chan":
-			sub := llvm.ConstExtractValue(typecode.Initializer(), []uint32{0})
-			num = state.getTypeCodeNum(sub)
-			classNumber = 0
-		case "interface":
-			num = big.NewInt(int64(state.fallbackIndex))
-			state.fallbackIndex++
-			classNumber = 1
-		case "pointer":
-			sub := llvm.ConstExtractValue(typecode.Initializer(), []uint32{0})
-			num = state.getTypeCodeNum(sub)
-			classNumber = 2
-		case "slice":
-			sub := llvm.ConstExtractValue(typecode.Initializer(), []uint32{0})
-			num = state.getTypeCodeNum(sub)
-			classNumber = 3
-		case "array":
-			num = big.NewInt(int64(state.fallbackIndex))
-			state.fallbackIndex++
-			classNumber = 4
-		case "func":
-			num = big.NewInt(int64(state.fallbackIndex))
-			state.fallbackIndex++
-			classNumber = 5
-		case "map":
-			num = big.NewInt(int64(state.fallbackIndex))
-			state.fallbackIndex++
-			classNumber = 6
-		case "struct":
-			num = big.NewInt(int64(state.getStructTypeNum(typecode)))
-			classNumber = 7
-		default:
+		if n, ok := nonBasicTypes[class]; ok {
+			classNumber = n
+		} else {
 			panic("unknown type kind: " + class)
 		}
+		var num *big.Int
+		lowBits := (classNumber << 1) + 1 // the 5 low bits of the typecode
 		if name == "" {
-			num.Lsh(num, 5).Or(num, big.NewInt((classNumber<<1)+1))
+			num = state.getNonBasicTypeCode(class, typecode)
 		} else {
-			num = big.NewInt(int64(state.getNonBasicNamedTypeNum(name, num))<<1 | 1)
-			num.Lsh(num, 4).Or(num, big.NewInt((classNumber<<1)+1))
+			// We must return a named type here. But first check whether it
+			// has already been defined.
+			if index, ok := state.namedNonBasicTypes[name]; ok {
+				num := big.NewInt(int64(index))
+				num.Lsh(num, 5).Or(num, big.NewInt((classNumber<<1)+1+(1<<4)))
+				return num
+			}
+			lowBits |= 1 << 4 // set the 'n' bit (see above)
+			if !state.needsNamedNonBasicTypesSidetable {
+				// Use simple small integers in this case, to make these numbers
+				// smaller.
+				index := len(state.namedNonBasicTypes) + 1
+				state.namedNonBasicTypes[name] = index
+				num = big.NewInt(int64(index))
+			} else {
+				// We need to store full type information.
+				// First allocate a number in the named non-basic type
+				// sidetable.
+				index := len(state.namedNonBasicTypesSidetable)
+				state.namedNonBasicTypesSidetable = append(state.namedNonBasicTypesSidetable, 0)
+				state.namedNonBasicTypes[name] = index
+				// Get the typecode of the underlying type (which could be the
+				// element type in the case of pointers, for example).
+				num = state.getNonBasicTypeCode(class, typecode)
+				if num.BitLen() > state.uintptrLen || !num.IsUint64() {
+					panic("cannot store value in sidetable")
+				}
+				// Now update the side table with the number we just
+				// determined. We need this multi-step approach to avoid stack
+				// overflow due to adding types recursively in the case of
+				// linked lists (a pointer which points to a struct that
+				// contains that same pointer).
+				state.namedNonBasicTypesSidetable[index] = num.Uint64()
+				num = big.NewInt(int64(index))
+			}
 		}
+		// Concatenate the 'num' and 'lowBits' bitstrings.
+		num.Lsh(num, 5).Or(num, big.NewInt(lowBits))
+		return num
+	}
+}
+
+// getNonBasicTypeCode is used by getTypeCodeNum. It returns the upper bits of
+// the type code used there in the type code.
+func (state *typeCodeAssignmentState) getNonBasicTypeCode(class string, typecode llvm.Value) *big.Int {
+	switch class {
+	case "chan", "pointer", "slice":
+		// Prefix-style type kinds. The upper bits contain the element type.
+		sub := llvm.ConstExtractValue(typecode.Initializer(), []uint32{0})
+		return state.getTypeCodeNum(sub)
+	case "struct":
+		// More complicated type kind. The upper bits contain the index to the
+		// struct type in the struct types sidetable.
+		return big.NewInt(int64(state.getStructTypeNum(typecode)))
+	default:
+		// Type has not yet been implemented, so fall back by using a unique
+		// number.
+		num := big.NewInt(int64(state.fallbackIndex))
+		state.fallbackIndex++
 		return num
 	}
 }
@@ -272,29 +312,6 @@ func (state *typeCodeAssignmentState) getBasicNamedTypeNum(name string) int {
 	return num
 }
 
-// getNonBasicNamedTypeNum returns a number unique for this named type. It tries
-// to return the smallest number possible to make encoding of this type code
-// easier.
-func (state *typeCodeAssignmentState) getNonBasicNamedTypeNum(name string, value *big.Int) int {
-	if num, ok := state.namedNonBasicTypes[name]; ok {
-		return num
-	}
-	if !state.needsNamedNonBasicTypesSidetable {
-		// Use simple small integers in this case, to make these numbers
-		// smaller.
-		num := len(state.namedNonBasicTypes) + 1
-		state.namedNonBasicTypes[name] = num
-		return num
-	}
-	num := len(state.namedNonBasicTypesSidetable)
-	if value.BitLen() > state.uintptrLen || !value.IsUint64() {
-		panic("cannot store value in sidetable")
-	}
-	state.namedNonBasicTypesSidetable = append(state.namedNonBasicTypesSidetable, makeVarint(value.Uint64())...)
-	state.namedNonBasicTypes[name] = num
-	return num
-}
-
 // getStructTypeNum returns the struct type number, which is an index into
 // reflect.structTypesSidetable or an unique number for every struct if this
 // sidetable is not needed in the to-be-compiled program.

src/reflect/sidetables.go

@@ -5,10 +5,10 @@ import (
 )
 
 // This stores a varint for each named type. Named types are identified by their
-// name instead of by their type. The named types stored in this struct are the
-// simpler non-basic types: pointer, struct, and channel.
+// name instead of by their type. The named types stored in this struct are
+// non-basic types: pointer, struct, and channel.
 //go:extern reflect.namedNonBasicTypesSidetable
-var namedNonBasicTypesSidetable byte
+var namedNonBasicTypesSidetable uintptr
 
 //go:extern reflect.structTypesSidetable
 var structTypesSidetable byte

src/reflect/type.go

@@ -163,7 +163,7 @@ func (t Type) stripPrefix() Type {
 	if (t>>4)%2 != 0 {
 		// This is a named type. The data is stored in a sidetable.
 		namedTypeNum := t >> 5
-		n, _ := readVarint(unsafe.Pointer(uintptr(unsafe.Pointer(&namedNonBasicTypesSidetable)) + uintptr(namedTypeNum)))
+		n := *(*uintptr)(unsafe.Pointer(uintptr(unsafe.Pointer(&namedNonBasicTypesSidetable)) + uintptr(namedTypeNum)*unsafe.Sizeof(uintptr(0))))
 		return Type(n)
 	}
 	// Not a named type, so the value is stored directly in the type code.

testdata/reflect.go

@@ -22,6 +22,10 @@ type (
 		buf  []byte
 		Buf  []byte
 	}
+	linkedList struct {
+		next *linkedList `description:"chain"`
+		foo  int
+	}
 )
 
 func main() {
@@ -103,6 +107,9 @@ func main() {
 			c int8
 		}{42, 321, 123},
 		mystruct{5, point{-5, 3}, struct{}{}, []byte{'G', 'o'}, []byte{'X'}},
+		&linkedList{
+			foo: 42,
+		},
 	} {
 		showValue(reflect.ValueOf(v), "")
 	}

testdata/reflect.txt

@@ -293,6 +293,22 @@ reflect type: struct
     indexing: 0
     reflect type: uint8 settable=true
       uint: 88
+reflect type: ptr
+  pointer: true struct
+  nil: false
+  reflect type: struct settable=true
+    struct: 2
+    field: 0 next
+    tag: description:"chain"
+    embedded: false
+    reflect type: ptr
+      pointer: false struct
+      nil: true
+    field: 1 foo
+    tag: 
+    embedded: false
+    reflect type: int
+      int: 42
 
 sizes:
 int8 1 8