Fixed a bug where self-referential types would cause infinite recursion and stack overflows

crates/rustc_codegen_nvvm/src/builder.rs

@@ -303,6 +303,23 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
             if changed {
                 v = transmute_llval(self.llbuilder, self.cx, v, new_ty);
             }
+            // Get the return type.
+            let sig = llvm::LLVMGetElementType(self.val_ty(self.llfn()));
+            let return_ty = llvm::LLVMGetReturnType(sig);
+            // Check if new_ty & return_ty are different pointers.
+            // FIXME: get rid of this nonsense once we are past LLVM 7 and don't have
+            // to suffer from typed pointers.
+            if return_ty != new_ty
+                && llvm::LLVMRustGetTypeKind(return_ty) == llvm::TypeKind::Pointer
+                && llvm::LLVMRustGetTypeKind(new_ty) == llvm::TypeKind::Pointer
+            {
+                v = llvm::LLVMBuildBitCast(
+                    self.llbuilder,
+                    v,
+                    return_ty,
+                    c"return pointer adjust".as_ptr(),
+                );
+            }
             llvm::LLVMBuildRet(self.llbuilder, v);
         }
     }
@@ -923,9 +940,17 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
     }
 
     /* Comparisons */
-    fn icmp(&mut self, op: IntPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
+    fn icmp(&mut self, op: IntPredicate, mut lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
         trace!("Icmp lhs: `{:?}`, rhs: `{:?}`", lhs, rhs);
-
+        // FIXME(FractalFir): Once again, a bunch of nosense to make the LLVM typed pointers happy.
+        // Get rid of this as soon as we move past typed pointers.
+        let lhs_ty = self.val_ty(lhs);
+        let rhs_ty = self.val_ty(rhs);
+        if lhs_ty != rhs_ty {
+            lhs = unsafe {
+                llvm::LLVMBuildBitCast(self.llbuilder, lhs, rhs_ty, c"icmp_cast".as_ptr())
+            };
+        }
         unsafe {
             let op = llvm::IntPredicate::from_generic(op);
             llvm::LLVMBuildICmp(self.llbuilder, op as c_uint, lhs, rhs, unnamed())
@@ -1358,7 +1383,11 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
     // Helper function to check if a value is 128-bit integer
     fn is_i128(&self, val: &'ll Value) -> bool {
         let ty = self.val_ty(val);
-        unsafe { llvm::LLVMGetIntTypeWidth(ty) == 128 }
+        if unsafe { llvm::LLVMRustGetTypeKind(ty) == llvm::TypeKind::Integer } {
+            unsafe { llvm::LLVMGetIntTypeWidth(ty) == 128 }
+        } else {
+            false
+        }
     }
 
     // Helper to split i128 into low and high u64 parts

crates/rustc_codegen_nvvm/src/ty.rs

@@ -611,8 +611,14 @@ fn struct_llfields<'a, 'tcx>(
             assert_eq!(offset.align_to(padding_align) + padding, target_offset);
             result.push(cx.type_padding_filler(padding, padding_align));
         }
+        match field.ty.kind() {
+            // This is a workaround for recursive types.
+            rustc_middle::ty::TyKind::FnPtr(_, _)
+            | rustc_middle::ty::TyKind::RawPtr(_, _)
+            | rustc_middle::ty::TyKind::Ref(_, _, _) => result.push(cx.type_i8p()),
+            _ => result.push(field.llvm_type(cx)),
+        }
 
-        result.push(field.llvm_type(cx));
         offset = target_offset + field.size;
         prev_effective_align = effective_field_align;
     }