[AMDGPU][GlobalISel] Fix / workaround amdgcn.kill/.unreachable lowering

cf. #133907 (comment)

Author: ro-i

llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp

@@ -3088,18 +3088,31 @@ bool IRTranslator::translateCallBr(const User &U,
     return false;
 
   // Retrieve successors.
-  SmallPtrSet<BasicBlock *, 8> Dests = {I.getDefaultDest()};
+  SmallPtrSet<BasicBlock *, 8> Dests;
+  Dests.insert(I.getDefaultDest());
   MachineBasicBlock *Return = &getMBB(*I.getDefaultDest());
 
   // Update successor info.
   addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());
-  // TODO: For most of the cases where there is an intrinsic callbr, we're
-  // having exactly one indirect target, which will be unreachable. As soon as
-  // this changes, we might need to enhance
-  // Target->setIsInlineAsmBrIndirectTarget or add something similar for
-  // intrinsic indirect branches.
+
+  // Add indirect targets as successors. For intrinsic callbr, these represent
+  // implicit control flow (e.g., the "kill" path for amdgcn.kill). We mark them
+  // with setIsInlineAsmBrIndirectTarget so the machine verifier accepts them as
+  // valid successors, even though they're not from inline asm.
+  for (BasicBlock *Dest : I.getIndirectDests()) {
+    MachineBasicBlock *Target = &getMBB(*Dest);
+    Target->setIsInlineAsmBrIndirectTarget();
+    Target->setLabelMustBeEmitted();
+    // Don't add duplicate machine successors.
+    if (Dests.insert(Dest).second)
+      addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero());
+  }
+
   CallBrMBB->normalizeSuccProbs();
 
+  // Drop into default successor.
+  MIRBuilder.buildBr(*Return);
+
   return true;
 }
 

llvm/test/CodeGen/AMDGPU/callbr-intrinsics.ll

@@ -32,14 +32,21 @@ define void @test_kill(ptr %src, ptr %dst, i1 %c) {
 ; GISEL-NEXT:    s_mov_b64 s[4:5], exec
 ; GISEL-NEXT:    s_andn2_b64 s[6:7], exec, vcc
 ; GISEL-NEXT:    s_andn2_b64 s[4:5], s[4:5], s[6:7]
-; GISEL-NEXT:    s_cbranch_scc0 .LBB0_2
+; GISEL-NEXT:    s_cbranch_scc0 .LBB0_4
 ; GISEL-NEXT:  ; %bb.1:
 ; GISEL-NEXT:    s_and_b64 exec, exec, s[4:5]
+; GISEL-NEXT:  ; %bb.2: ; %cont
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GISEL-NEXT:    flat_store_dword v[2:3], v0
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GISEL-NEXT:    s_setpc_b64 s[30:31]
-; GISEL-NEXT:  .LBB0_2:
+; GISEL-NEXT:  .LBB0_3: ; Inline asm indirect target
+; GISEL-NEXT:    ; %kill
+; GISEL-NEXT:    ; Label of block must be emitted
+; GISEL-NEXT:    ; divergent unreachable
+; GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GISEL-NEXT:  .LBB0_4:
 ; GISEL-NEXT:    s_mov_b64 exec, 0
 ; GISEL-NEXT:    s_endpgm
   %a = load i32, ptr %src, align 4
@@ -81,14 +88,21 @@ define void @test_kill_block_order(ptr %src, ptr %dst, i1 %c) {
 ; GISEL-NEXT:    s_mov_b64 s[4:5], exec
 ; GISEL-NEXT:    s_andn2_b64 s[6:7], exec, vcc
 ; GISEL-NEXT:    s_andn2_b64 s[4:5], s[4:5], s[6:7]
-; GISEL-NEXT:    s_cbranch_scc0 .LBB1_2
+; GISEL-NEXT:    s_cbranch_scc0 .LBB1_4
 ; GISEL-NEXT:  ; %bb.1:
 ; GISEL-NEXT:    s_and_b64 exec, exec, s[4:5]
+; GISEL-NEXT:  ; %bb.2: ; %cont
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GISEL-NEXT:    flat_store_dword v[2:3], v0
 ; GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GISEL-NEXT:    s_setpc_b64 s[30:31]
-; GISEL-NEXT:  .LBB1_2:
+; GISEL-NEXT:  .LBB1_3: ; Inline asm indirect target
+; GISEL-NEXT:    ; %kill
+; GISEL-NEXT:    ; Label of block must be emitted
+; GISEL-NEXT:    ; divergent unreachable
+; GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GISEL-NEXT:    s_setpc_b64 s[30:31]
+; GISEL-NEXT:  .LBB1_4:
 ; GISEL-NEXT:    s_mov_b64 exec, 0
 ; GISEL-NEXT:    s_endpgm
   %a = load i32, ptr %src, align 4