[SPIR-V] Add hlsl_private address space for SPIR-V

Keenuts · Keenuts · commit 12524667594d · 2025-04-08T17:46:26.000+02:00
In SPIR-V, private global variables have the Private storage class. This PR adds a new address space which allows frontend to emit variable with this storage class when targeting this backend. This is covered in this proposal: llvm/wg-hlsl@4c9e11a This PR will cause addrspacecast to show up in several cases, like class member functions or assignment. Those will have to be handled in the backend later on, particularly to fixup pointer storage classes in some functions. Before this change, global variable were emitted with the 'Function' storage class, which was wrong.
diff --git a/clang/include/clang/Basic/AddressSpaces.h b/clang/include/clang/Basic/AddressSpaces.h
@@ -59,6 +59,7 @@ enum class LangAS : unsigned {
   // HLSL specific address spaces.
   hlsl_groupshared,
   hlsl_constant,
+  hlsl_private,
 
   // Wasm specific address spaces.
   wasm_funcref,
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
@@ -4328,6 +4328,7 @@ class Sema final : public SemaBase {
   NamedDecl *findLocallyScopedExternCDecl(DeclarationName Name);
 
   void deduceOpenCLAddressSpace(ValueDecl *decl);
+  void deduceHLSLAddressSpace(VarDecl *decl);
 
   /// Adjust the \c DeclContext for a function or variable that might be a
   /// function-local external declaration.
diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp
@@ -94,6 +94,8 @@ bool Qualifiers::isTargetAddressSpaceSupersetOf(LangAS A, LangAS B,
          (A == LangAS::Default &&
           (B == LangAS::cuda_constant || B == LangAS::cuda_device ||
            B == LangAS::cuda_shared)) ||
+         // Default is a superset of HLSL private.
+         (A == LangAS::Default && B == LangAS::hlsl_private) ||
          // Conversions from target specific address spaces may be legal
          // depending on the target information.
          Ctx.getTargetInfo().isAddressSpaceSupersetOf(A, B);
@@ -5168,6 +5170,9 @@ bool Type::isHLSLIntangibleType() const {
   CXXRecordDecl *RD = RT->getAsCXXRecordDecl();
   assert(RD != nullptr &&
          "all HLSL structs and classes should be CXXRecordDecl");
+
+  if (!RD->isCompleteDefinition())
+    return false;
   assert(RD->isCompleteDefinition() && "expecting complete type");
   return RD->isHLSLIntangible();
 }
diff --git a/clang/lib/AST/TypePrinter.cpp b/clang/lib/AST/TypePrinter.cpp
@@ -2581,6 +2581,8 @@ std::string Qualifiers::getAddrSpaceAsString(LangAS AS) {
     return "groupshared";
   case LangAS::hlsl_constant:
     return "hlsl_constant";
+  case LangAS::hlsl_private:
+    return "hlsl_private";
   case LangAS::wasm_funcref:
     return "__funcref";
   default:
diff --git a/clang/lib/Basic/TargetInfo.cpp b/clang/lib/Basic/TargetInfo.cpp
@@ -47,6 +47,8 @@ static const LangASMap FakeAddrSpaceMap = {
     11, // ptr32_uptr
     12, // ptr64
     13, // hlsl_groupshared
+    14, // hlsl_constant
+    15, // hlsl_private
     20, // wasm_funcref
 };
 
diff --git a/clang/lib/Basic/Targets/AArch64.h b/clang/lib/Basic/Targets/AArch64.h
@@ -45,6 +45,7 @@ static const unsigned ARM64AddrSpaceMap[] = {
     static_cast<unsigned>(AArch64AddrSpace::ptr64),
     0, // hlsl_groupshared
     0, // hlsl_constant
+    0, // hlsl_private
     // Wasm address space values for this target are dummy values,
     // as it is only enabled for Wasm targets.
     20, // wasm_funcref
diff --git a/clang/lib/Basic/Targets/AMDGPU.cpp b/clang/lib/Basic/Targets/AMDGPU.cpp
@@ -60,6 +60,9 @@ const LangASMap AMDGPUTargetInfo::AMDGPUDefIsGenMap = {
     llvm::AMDGPUAS::FLAT_ADDRESS,     // ptr64
     llvm::AMDGPUAS::FLAT_ADDRESS,     // hlsl_groupshared
     llvm::AMDGPUAS::CONSTANT_ADDRESS, // hlsl_constant
+    // FIXME(pr/122103): hlsl_private -> PRIVATE is wrong, but at least this
+    // will break loudly.
+    llvm::AMDGPUAS::PRIVATE_ADDRESS, // hlsl_private
 };
 
 const LangASMap AMDGPUTargetInfo::AMDGPUDefIsPrivMap = {
@@ -85,6 +88,7 @@ const LangASMap AMDGPUTargetInfo::AMDGPUDefIsPrivMap = {
     llvm::AMDGPUAS::FLAT_ADDRESS,     // ptr64
     llvm::AMDGPUAS::FLAT_ADDRESS,     // hlsl_groupshared
     llvm::AMDGPUAS::CONSTANT_ADDRESS, // hlsl_constant
+    llvm::AMDGPUAS::PRIVATE_ADDRESS,  // hlsl_private
 };
 } // namespace targets
 } // namespace clang
diff --git a/clang/lib/Basic/Targets/DirectX.h b/clang/lib/Basic/Targets/DirectX.h
@@ -43,6 +43,7 @@ static const unsigned DirectXAddrSpaceMap[] = {
     0, // ptr64
     3, // hlsl_groupshared
     2, // hlsl_constant
+    0, // hlsl_private
     // Wasm address space values for this target are dummy values,
     // as it is only enabled for Wasm targets.
     20, // wasm_funcref
diff --git a/clang/lib/Basic/Targets/NVPTX.h b/clang/lib/Basic/Targets/NVPTX.h
@@ -47,6 +47,7 @@ static const unsigned NVPTXAddrSpaceMap[] = {
     0, // ptr64
     0, // hlsl_groupshared
     0, // hlsl_constant
+    0, // hlsl_private
     // Wasm address space values for this target are dummy values,
     // as it is only enabled for Wasm targets.
     20, // wasm_funcref
diff --git a/clang/lib/Basic/Targets/SPIR.h b/clang/lib/Basic/Targets/SPIR.h
@@ -38,16 +38,17 @@ static const unsigned SPIRDefIsPrivMap[] = {
     0, // cuda_constant
     0, // cuda_shared
     // SYCL address space values for this map are dummy
-    0, // sycl_global
-    0, // sycl_global_device
-    0, // sycl_global_host
-    0, // sycl_local
-    0, // sycl_private
-    0, // ptr32_sptr
-    0, // ptr32_uptr
-    0, // ptr64
-    0, // hlsl_groupshared
-    2, // hlsl_constant
+    0,  // sycl_global
+    0,  // sycl_global_device
+    0,  // sycl_global_host
+    0,  // sycl_local
+    0,  // sycl_private
+    0,  // ptr32_sptr
+    0,  // ptr32_uptr
+    0,  // ptr64
+    0,  // hlsl_groupshared
+    2,  // hlsl_constant
+    10, // hlsl_private
     // Wasm address space values for this target are dummy values,
     // as it is only enabled for Wasm targets.
     20, // wasm_funcref
@@ -70,18 +71,19 @@ static const unsigned SPIRDefIsGenMap[] = {
     // cuda_constant pointer can be casted to default/"flat" pointer, but in
     // SPIR-V casts between constant and generic pointers are not allowed. For
     // this reason cuda_constant is mapped to SPIR-V CrossWorkgroup.
-    1, // cuda_constant
-    3, // cuda_shared
-    1, // sycl_global
-    5, // sycl_global_device
-    6, // sycl_global_host
-    3, // sycl_local
-    0, // sycl_private
-    0, // ptr32_sptr
-    0, // ptr32_uptr
-    0, // ptr64
-    0, // hlsl_groupshared
-    0, // hlsl_constant
+    1,  // cuda_constant
+    3,  // cuda_shared
+    1,  // sycl_global
+    5,  // sycl_global_device
+    6,  // sycl_global_host
+    3,  // sycl_local
+    0,  // sycl_private
+    0,  // ptr32_sptr
+    0,  // ptr32_uptr
+    0,  // ptr64
+    0,  // hlsl_groupshared
+    0,  // hlsl_constant
+    10, // hlsl_private
     // Wasm address space values for this target are dummy values,
     // as it is only enabled for Wasm targets.
     20, // wasm_funcref
@@ -315,7 +317,7 @@ class LLVM_LIBRARY_VISIBILITY SPIRVTargetInfo : public BaseSPIRVTargetInfo {
     // SPIR-V IDs are represented with a single 32-bit word.
     SizeType = TargetInfo::UnsignedInt;
     resetDataLayout("e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-"
-                    "v256:256-v512:512-v1024:1024-n8:16:32:64-G1");
+                    "v256:256-v512:512-v1024:1024-n8:16:32:64-G10");
   }
 
   llvm::SmallVector<Builtin::InfosShard> getTargetBuiltins() const override;
diff --git a/clang/lib/Basic/Targets/SystemZ.h b/clang/lib/Basic/Targets/SystemZ.h
@@ -43,6 +43,7 @@ static const unsigned ZOSAddressMap[] = {
     0, // ptr64
     0, // hlsl_groupshared
     0, // hlsl_constant
+    0, // hlsl_private
     0  // wasm_funcref
 };
 
diff --git a/clang/lib/Basic/Targets/TCE.h b/clang/lib/Basic/Targets/TCE.h
@@ -52,6 +52,7 @@ static const unsigned TCEOpenCLAddrSpaceMap[] = {
     0, // ptr64
     0, // hlsl_groupshared
     0, // hlsl_constant
+    0, // hlsl_private
     // Wasm address space values for this target are dummy values,
     // as it is only enabled for Wasm targets.
     20, // wasm_funcref
diff --git a/clang/lib/Basic/Targets/WebAssembly.h b/clang/lib/Basic/Targets/WebAssembly.h
@@ -43,6 +43,7 @@ static const unsigned WebAssemblyAddrSpaceMap[] = {
     0,  // ptr64
     0,  // hlsl_groupshared
     0,  // hlsl_constant
+    0,  // hlsl_private
     20, // wasm_funcref
 };
 
diff --git a/clang/lib/Basic/Targets/X86.h b/clang/lib/Basic/Targets/X86.h
@@ -47,6 +47,7 @@ static const unsigned X86AddrSpaceMap[] = {
     272, // ptr64
     0,   // hlsl_groupshared
     0,   // hlsl_constant
+    0,   // hlsl_private
     // Wasm address space values for this target are dummy values,
     // as it is only enabled for Wasm targets.
     20, // wasm_funcref
diff --git a/clang/lib/CodeGen/CGDeclCXX.cpp b/clang/lib/CodeGen/CGDeclCXX.cpp
@@ -1157,8 +1157,22 @@ void CodeGenFunction::GenerateCXXGlobalCleanUpFunc(
             CGM.getCXXABI().useSinitAndSterm() &&
             "Arg could not be nullptr unless using sinit and sterm functions.");
         CI = Builder.CreateCall(CalleeTy, Callee);
-      } else
+      } else {
+        // If the object lives in a different address space, the `this` pointer
+        // address space won't match the dtor `this` param. An addrspacecast is
+        // required.
+        assert(Arg->getType()->isPointerTy());
+        assert(CalleeTy->getParamType(0)->isPointerTy());
+        unsigned ActualAddrSpace = Arg->getType()->getPointerAddressSpace();
+        unsigned ExpectedAddrSpace =
+            CalleeTy->getParamType(0)->getPointerAddressSpace();
+        if (ActualAddrSpace != ExpectedAddrSpace) {
+          llvm::PointerType *PTy =
+              llvm::PointerType::get(getLLVMContext(), ExpectedAddrSpace);
+          Arg = llvm::ConstantExpr::getAddrSpaceCast(Arg, PTy);
+        }
         CI = Builder.CreateCall(CalleeTy, Callee, Arg);
+      }
 
       // Make sure the call and the callee agree on calling convention.
       if (llvm::Function *F = dyn_cast<llvm::Function>(Callee))
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -371,12 +371,6 @@ void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
   assert(DeferredDeactivationCleanupStack.empty() &&
          "mismatched activate/deactivate of cleanups!");
 
-  if (CGM.shouldEmitConvergenceTokens()) {
-    ConvergenceTokenStack.pop_back();
-    assert(ConvergenceTokenStack.empty() &&
-           "mismatched push/pop in convergence stack!");
-  }
-
   bool OnlySimpleReturnStmts = NumSimpleReturnExprs > 0
     && NumSimpleReturnExprs == NumReturnExprs
     && ReturnBlock.getBlock()->use_empty();
@@ -564,6 +558,12 @@ void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
       ReturnValue = Address::invalid();
     }
   }
+
+  if (CGM.shouldEmitConvergenceTokens()) {
+    ConvergenceTokenStack.pop_back();
+    assert(ConvergenceTokenStack.empty() &&
+           "mismatched push/pop in convergence stack!");
+  }
 }
 
 /// ShouldInstrumentFunction - Return true if the current function should be
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
@@ -6888,6 +6888,32 @@ static void SetNestedNameSpecifier(Sema &S, DeclaratorDecl *DD, Declarator &D) {
   DD->setQualifierInfo(SS.getWithLocInContext(S.Context));
 }
 
+void Sema::deduceHLSLAddressSpace(VarDecl *Decl) {
+  // The variable already has an address space (groupshared for ex).
+  if (Decl->getType().hasAddressSpace())
+    return;
+
+  if (Decl->getType()->isDependentType())
+    return;
+
+  QualType Type = Decl->getType();
+  if (Type->isSamplerT() || Type->isVoidType())
+    return;
+
+  // Resource handles.
+  if (Type->isHLSLIntangibleType())
+    return;
+
+  // Only static globals belong to the Private address space.
+  // Non-static globals belongs to the cbuffer.
+  if (Decl->getStorageClass() != SC_Static && !Decl->isStaticDataMember())
+    return;
+
+  LangAS ImplAS = LangAS::hlsl_private;
+  Type = Context.getAddrSpaceQualType(Type, ImplAS);
+  Decl->setType(Type);
+}
+
 void Sema::deduceOpenCLAddressSpace(ValueDecl *Decl) {
   if (Decl->getType().hasAddressSpace())
     return;
@@ -7776,6 +7802,9 @@ NamedDecl *Sema::ActOnVariableDeclarator(
       NewVD = VarDecl::Create(Context, DC, D.getBeginLoc(),
                               D.getIdentifierLoc(), II, R, TInfo, SC);
 
+    if (getLangOpts().HLSL)
+      deduceHLSLAddressSpace(NewVD);
+
     // If this is supposed to be a variable template, create it as such.
     if (IsVariableTemplate) {
       NewTemplate =
@@ -7960,6 +7989,9 @@ NamedDecl *Sema::ActOnVariableDeclarator(
     }
   }
 
+  if (getLangOpts().HLSL)
+    deduceHLSLAddressSpace(NewVD);
+
   // WebAssembly tables are always in address space 1 (wasm_var). Don't apply
   // address space if the table has local storage (semantic checks elsewhere
   // will produce an error anyway).
@@ -13131,6 +13163,9 @@ bool Sema::DeduceVariableDeclarationType(VarDecl *VDecl, bool DirectInit,
   if (getLangOpts().OpenCL)
     deduceOpenCLAddressSpace(VDecl);
 
+  if (getLangOpts().HLSL)
+    deduceHLSLAddressSpace(VDecl);
+
   // If this is a redeclaration, check that the type we just deduced matches
   // the previously declared type.
   if (VarDecl *Old = VDecl->getPreviousDecl()) {
diff --git a/clang/test/AST/HLSL/cbuffer.hlsl b/clang/test/AST/HLSL/cbuffer.hlsl
@@ -149,7 +149,7 @@ _Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(TwoFloats, __cblay
 cbuffer CB {
   // CHECK: FunctionDecl {{.*}} f 'void ()'
   void f() {}
-  // CHECK: VarDecl {{.*}} SV 'float' static
+  // CHECK: VarDecl {{.*}} SV 'hlsl_private float' static
   static float SV;
   // CHECK: VarDecl {{.*}} s7 'EmptyStruct' callinit
   EmptyStruct s7;
diff --git a/clang/test/CodeGenHLSL/GlobalConstructors.hlsl b/clang/test/CodeGenHLSL/GlobalConstructors.hlsl
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -x hlsl -emit-llvm -disable-llvm-passes %s -o - | FileCheck %s
+// RUN: %clang_cc1 -triple spirv-unknown-vulkan1.3-compute -x hlsl -emit-llvm -disable-llvm-passes %s -o - | FileCheck %s --check-prefixes=CHECK,SPIRV
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -x hlsl -emit-llvm -disable-llvm-passes %s -o - | FileCheck %s --check-prefixes=CHECK,DXIL
 
 RWBuffer<float> Buffer;
 
@@ -10,7 +11,13 @@ void main(unsigned GI : SV_GroupIndex) {}
 // CHECK-NOT:@llvm.global_dtors
 //CHECK:      define void @main()
 //CHECK-NEXT: entry:
-//CHECK-NEXT:   call void @_GLOBAL__sub_I_GlobalConstructors.hlsl()
-//CHECK-NEXT:   %0 = call i32 @llvm.dx.flattened.thread.id.in.group()
-//CHECK-NEXT:   call void @_Z4mainj(i32 %0)
+
+//SPIRV-NEXT:   %0 = call token @llvm.experimental.convergence.entry()
+//SPIRV-NEXT:   call spir_func void @_GLOBAL__sub_I_GlobalConstructors.hlsl() [ "convergencectrl"(token %0) ]
+//SPIRV-NEXT:   %1 = call i32 @llvm.spv.flattened.thread.id.in.group()
+//SPIRV-NEXT:   call spir_func void @_Z4mainj(i32 %1) [ "convergencectrl"(token %0) ]
+
+//DXIL-NEXT:    call void @_GLOBAL__sub_I_GlobalConstructors.hlsl()
+//DXIL-NEXT:    %0 = call i32 @llvm.dx.flattened.thread.id.in.group()
+//DXIL-NEXT:    call void @_Z4mainj(i32 %0)
 //CHECK-NEXT:   ret void
diff --git a/clang/test/CodeGenHLSL/cbuffer_with_static_global_and_function.hlsl b/clang/test/CodeGenHLSL/cbuffer_with_static_global_and_function.hlsl
@@ -20,6 +20,7 @@ cbuffer B {
 
 // CHECK: define {{.*}} float @_Z3foov() #0 {
 // CHECK: load float, ptr addrspace(2) @a, align 4
+// CHECK: load float, ptr @_ZL1b, align 4
 
 extern float bar() {
   return foo();
diff --git a/clang/test/CodeGenHLSL/out-of-line-static.hlsl b/clang/test/CodeGenHLSL/out-of-line-static.hlsl
@@ -0,0 +1,30 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -std=hlsl202x -emit-llvm -o - -disable-llvm-passes %s | FileCheck %s --check-prefixes=CHECK,DXIL
+// RUN: %clang_cc1 -triple spirv-pc-vulkan1.3-compute -std=hlsl202x -emit-llvm -o - -disable-llvm-passes %s | FileCheck %s --check-prefixes=CHECK,SPIRV
+
+struct S {
+  static int Value;
+};
+
+int S::Value = 1;
+// DXIL: @_ZN1S5ValueE = global i32 1, align 4
+// SPIRV: @_ZN1S5ValueE = addrspace(10) global i32 1, align 4
+
+[shader("compute")]
+[numthreads(1,1,1)]
+void main() {
+  S s;
+  int value1, value2;
+// CHECK:      %s = alloca %struct.S, align 1
+// CHECK: %value1 = alloca i32, align 4
+// CHECK: %value2 = alloca i32, align 4
+
+// DXIL: [[tmp:%.*]] = load i32, ptr @_ZN1S5ValueE, align 4
+// SPIRV: [[tmp:%.*]] = load i32, ptr addrspace(10) @_ZN1S5ValueE, align 4
+// CHECK: store i32 [[tmp]], ptr %value1, align 4
+  value1 = S::Value;
+
+// DXIL: [[tmp:%.*]] = load i32, ptr @_ZN1S5ValueE, align 4
+// SPIRV: [[tmp:%.*]] = load i32, ptr addrspace(10) @_ZN1S5ValueE, align 4
+// CHECK: store i32 [[tmp]], ptr %value2, align 4
+  value2 = s.Value;
+}
diff --git a/clang/test/CodeGenHLSL/static-variable.hlsl b/clang/test/CodeGenHLSL/static-variable.hlsl
@@ -0,0 +1,18 @@
+// RUN: %clang_cc1 -std=hlsl2021 -finclude-default-header -x hlsl -triple \
+// RUN:   spirv-unknown-vulkan1.3-library %s \
+// RUN:   -emit-llvm -disable-llvm-passes -o - | FileCheck %s --check-prefix=SPIRV
+// RUN: %clang_cc1 -std=hlsl2021 -finclude-default-header -x hlsl -triple \
+// RUN:   dxil-pc-shadermodel6.3-library %s \
+// RUN:   -emit-llvm -disable-llvm-passes -o - | FileCheck %s --check-prefix=DXIL
+
+// DXIL:  @_ZL1g = internal global float 0.000000e+00, align 4
+// SPIRV: @_ZL1g = internal addrspace(10) global float 0.000000e+00, align 4
+
+static float g = 0;
+
+[numthreads(8,8,1)]
+void main() {
+// DXIL:  {{.*}} = load float, ptr @_ZL1g, align 4
+// SPIRV: {{.*}} = load float, ptr addrspace(10) @_ZL1g, align 4
+  float l = g;
+}
diff --git a/clang/test/SemaTemplate/address_space-dependent.cpp b/clang/test/SemaTemplate/address_space-dependent.cpp
diff --git a/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp b/llvm/lib/Target/SPIRV/SPIRVTargetMachine.cpp
