AMDGPU: Do not assert on unhandled types when demangling libcalls (#120068)

Author: arsenm

llvm/lib/Target/AMDGPU/AMDGPULibFunc.cpp

@@ -957,8 +957,15 @@ static Type* getIntrinsicParamType(
   case AMDGPULibFunc::EVENT:
     T = PointerType::getUnqual(C);
     break;
-  default:
-    llvm_unreachable("Unhandled param type");
+  case AMDGPULibFunc::B8:
+  case AMDGPULibFunc::B16:
+  case AMDGPULibFunc::B32:
+  case AMDGPULibFunc::B64:
+  case AMDGPULibFunc::SIZE_MASK:
+  case AMDGPULibFunc::FLOAT:
+  case AMDGPULibFunc::INT:
+  case AMDGPULibFunc::UINT:
+  case AMDGPULibFunc::DUMMY:
     return nullptr;
   }
   if (P.VectorSize > 1)
@@ -974,8 +981,13 @@ FunctionType *AMDGPUMangledLibFunc::getFunctionType(const Module &M) const {
   std::vector<Type*> Args;
   ParamIterator I(Leads, manglingRules[FuncId]);
   Param P;
-  while ((P=I.getNextParam()).ArgType != 0)
-    Args.push_back(getIntrinsicParamType(C, P, true));
+  while ((P = I.getNextParam()).ArgType != 0) {
+    Type *ParamTy = getIntrinsicParamType(C, P, true);
+    if (!ParamTy)
+      return nullptr;
+
+    Args.push_back(ParamTy);
+  }
 
   return FunctionType::get(
     getIntrinsicParamType(C, getRetType(FuncId, Leads), true),
@@ -1001,10 +1013,15 @@ bool AMDGPULibFunc::isCompatibleSignature(const Module &M,
                                           const FunctionType *CallTy) const {
   const FunctionType *FuncTy = getFunctionType(M);
 
-  // FIXME: UnmangledFuncInfo does not have any type information other than the
-  // number of arguments.
-  if (!FuncTy)
+  if (!FuncTy) {
+    // Give up on mangled functions with unexpected types.
+    if (AMDGPULibFuncBase::isMangled(getId()))
+      return false;
+
+    // FIXME: UnmangledFuncInfo does not have any type information other than
+    // the number of arguments.
     return getNumArgs() == CallTy->getNumParams();
+  }
 
   // Normally the types should exactly match.
   if (FuncTy == CallTy)

llvm/test/CodeGen/AMDGPU/amdgpu-simplify-libcall-unexpected-types.ll

@@ -0,0 +1,19 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -passes=amdgpu-simplifylib -amdgpu-prelink %s | FileCheck %s
+; Make sure there are no crashes on unexpected types
+
+%struct.vfloat3 = type { float, float, float }
+
+declare hidden %struct.vfloat3 @_Z3mix7vfloat3S_f(float, float, float, float, float, float, float)
+
+define %struct.vfloat3 @_Z8test_mix7vfloat3S_f(float %x.coerce0, float %x.coerce1, float %x.coerce2, float %y.coerce0, float %y.coerce1, float %y.coerce2, float %t) {
+; CHECK-LABEL: define %struct.vfloat3 @_Z8test_mix7vfloat3S_f(
+; CHECK-SAME: float [[X_COERCE0:%.*]], float [[X_COERCE1:%.*]], float [[X_COERCE2:%.*]], float [[Y_COERCE0:%.*]], float [[Y_COERCE1:%.*]], float [[Y_COERCE2:%.*]], float [[T:%.*]]) {
+; CHECK-NEXT:  [[ENTRY:.*:]]
+; CHECK-NEXT:    [[CALL:%.*]] = call [[STRUCT_VFLOAT3:%.*]] @[[_Z3MIX7VFLOAT3S_F:[a-zA-Z0-9_$\"\\.-]*[a-zA-Z_$\"\\.-][a-zA-Z0-9_$\"\\.-]*]](float [[X_COERCE0]], float [[X_COERCE1]], float [[X_COERCE2]], float [[Y_COERCE0]], float [[Y_COERCE1]], float [[Y_COERCE2]], float [[T]])
+; CHECK-NEXT:    ret [[STRUCT_VFLOAT3]] [[CALL]]
+;
+entry:
+  %call = call %struct.vfloat3 @_Z3mix7vfloat3S_f(float %x.coerce0, float %x.coerce1, float %x.coerce2, float %y.coerce0, float %y.coerce1, float %y.coerce2, float %t)
+  ret %struct.vfloat3 %call
+}