[HLSL][SPIR-V] Add hlsl_private address space for SPIR-V

clang/include/clang/Basic/AddressSpaces.h

@@ -59,6 +59,7 @@ enum class LangAS : unsigned {
   // HLSL specific address spaces.
   hlsl_groupshared,
   hlsl_constant,
+  hlsl_private,
 
   // Wasm specific address spaces.
   wasm_funcref,

clang/include/clang/Sema/Sema.h

@@ -4328,6 +4328,7 @@ class Sema final : public SemaBase {
   NamedDecl *findLocallyScopedExternCDecl(DeclarationName Name);
 
   void deduceOpenCLAddressSpace(ValueDecl *decl);
+  void deduceHLSLAddressSpace(VarDecl *decl);
 
   /// Adjust the \c DeclContext for a function or variable that might be a
   /// function-local external declaration.

clang/lib/AST/Type.cpp

@@ -94,6 +94,10 @@ bool Qualifiers::isTargetAddressSpaceSupersetOf(LangAS A, LangAS B,
          (A == LangAS::Default &&
           (B == LangAS::cuda_constant || B == LangAS::cuda_device ||
            B == LangAS::cuda_shared)) ||
+         // `this` overloading depending on address space is not ready,
+         // so this is a hack to allow generating addrspacecasts.
+         // IR legalization will be required when this address space is used.
+         (A == LangAS::Default && B == LangAS::hlsl_private) ||
          // Conversions from target specific address spaces may be legal
          // depending on the target information.
          Ctx.getTargetInfo().isAddressSpaceSupersetOf(A, B);

clang/lib/AST/TypePrinter.cpp

@@ -2581,6 +2581,8 @@ std::string Qualifiers::getAddrSpaceAsString(LangAS AS) {
     return "groupshared";
   case LangAS::hlsl_constant:
     return "hlsl_constant";
+  case LangAS::hlsl_private:
+    return "hlsl_private";
   case LangAS::wasm_funcref:
     return "__funcref";
   default:

clang/lib/Basic/TargetInfo.cpp

@@ -47,6 +47,8 @@ static const LangASMap FakeAddrSpaceMap = {
     11, // ptr32_uptr
     12, // ptr64
     13, // hlsl_groupshared
+    14, // hlsl_constant
+    15, // hlsl_private
     20, // wasm_funcref
 };
 

clang/lib/Basic/Targets/AArch64.h

@@ -45,6 +45,7 @@ static const unsigned ARM64AddrSpaceMap[] = {
     static_cast<unsigned>(AArch64AddrSpace::ptr64),
     0, // hlsl_groupshared
     0, // hlsl_constant
+    0, // hlsl_private
     // Wasm address space values for this target are dummy values,
     // as it is only enabled for Wasm targets.
     20, // wasm_funcref

clang/lib/Basic/Targets/AMDGPU.cpp

@@ -60,6 +60,9 @@ const LangASMap AMDGPUTargetInfo::AMDGPUDefIsGenMap = {
     llvm::AMDGPUAS::FLAT_ADDRESS,     // ptr64
     llvm::AMDGPUAS::FLAT_ADDRESS,     // hlsl_groupshared
     llvm::AMDGPUAS::CONSTANT_ADDRESS, // hlsl_constant
+    // FIXME(pr/122103): hlsl_private -> PRIVATE is wrong, but at least this
+    // will break loudly.
+    llvm::AMDGPUAS::PRIVATE_ADDRESS, // hlsl_private
 };
 
 const LangASMap AMDGPUTargetInfo::AMDGPUDefIsPrivMap = {
@@ -85,6 +88,7 @@ const LangASMap AMDGPUTargetInfo::AMDGPUDefIsPrivMap = {
     llvm::AMDGPUAS::FLAT_ADDRESS,     // ptr64
     llvm::AMDGPUAS::FLAT_ADDRESS,     // hlsl_groupshared
     llvm::AMDGPUAS::CONSTANT_ADDRESS, // hlsl_constant
+    llvm::AMDGPUAS::PRIVATE_ADDRESS,  // hlsl_private
 };
 } // namespace targets
 } // namespace clang

clang/lib/Basic/Targets/DirectX.h

@@ -43,6 +43,7 @@ static const unsigned DirectXAddrSpaceMap[] = {
     0, // ptr64
     3, // hlsl_groupshared
     2, // hlsl_constant
+    0, // hlsl_private
     // Wasm address space values for this target are dummy values,
     // as it is only enabled for Wasm targets.
     20, // wasm_funcref

clang/lib/Basic/Targets/NVPTX.h

@@ -47,6 +47,7 @@ static const unsigned NVPTXAddrSpaceMap[] = {
     0, // ptr64
     0, // hlsl_groupshared
     0, // hlsl_constant
+    0, // hlsl_private
     // Wasm address space values for this target are dummy values,
     // as it is only enabled for Wasm targets.
     20, // wasm_funcref

clang/lib/Basic/Targets/SPIR.h

@@ -38,16 +38,17 @@ static const unsigned SPIRDefIsPrivMap[] = {
     0, // cuda_constant
     0, // cuda_shared
     // SYCL address space values for this map are dummy
-    0, // sycl_global
-    0, // sycl_global_device
-    0, // sycl_global_host
-    0, // sycl_local
-    0, // sycl_private
-    0, // ptr32_sptr
-    0, // ptr32_uptr
-    0, // ptr64
-    0, // hlsl_groupshared
-    2, // hlsl_constant
+    0,  // sycl_global
+    0,  // sycl_global_device
+    0,  // sycl_global_host
+    0,  // sycl_local
+    0,  // sycl_private
+    0,  // ptr32_sptr
+    0,  // ptr32_uptr
+    0,  // ptr64
+    0,  // hlsl_groupshared
+    2,  // hlsl_constant
+    10, // hlsl_private
     // Wasm address space values for this target are dummy values,
     // as it is only enabled for Wasm targets.
     20, // wasm_funcref
@@ -70,18 +71,19 @@ static const unsigned SPIRDefIsGenMap[] = {
     // cuda_constant pointer can be casted to default/"flat" pointer, but in
     // SPIR-V casts between constant and generic pointers are not allowed. For
     // this reason cuda_constant is mapped to SPIR-V CrossWorkgroup.
-    1, // cuda_constant
-    3, // cuda_shared
-    1, // sycl_global
-    5, // sycl_global_device
-    6, // sycl_global_host
-    3, // sycl_local
-    0, // sycl_private
-    0, // ptr32_sptr
-    0, // ptr32_uptr
-    0, // ptr64
-    0, // hlsl_groupshared
-    0, // hlsl_constant
+    1,  // cuda_constant
+    3,  // cuda_shared
+    1,  // sycl_global
+    5,  // sycl_global_device
+    6,  // sycl_global_host
+    3,  // sycl_local
+    0,  // sycl_private
+    0,  // ptr32_sptr
+    0,  // ptr32_uptr
+    0,  // ptr64
+    0,  // hlsl_groupshared
+    0,  // hlsl_constant
+    10, // hlsl_private
     // Wasm address space values for this target are dummy values,
     // as it is only enabled for Wasm targets.
     20, // wasm_funcref
@@ -315,7 +317,7 @@ class LLVM_LIBRARY_VISIBILITY SPIRVTargetInfo : public BaseSPIRVTargetInfo {
     // SPIR-V IDs are represented with a single 32-bit word.
     SizeType = TargetInfo::UnsignedInt;
     resetDataLayout("e-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-"
-                    "v256:256-v512:512-v1024:1024-n8:16:32:64-G1");
+                    "v256:256-v512:512-v1024:1024-n8:16:32:64-G10");
   }
 
   llvm::SmallVector<Builtin::InfosShard> getTargetBuiltins() const override;

clang/lib/Basic/Targets/SystemZ.h

@@ -43,6 +43,7 @@ static const unsigned ZOSAddressMap[] = {
     0, // ptr64
     0, // hlsl_groupshared
     0, // hlsl_constant
+    0, // hlsl_private
     0  // wasm_funcref
 };
 

clang/lib/Basic/Targets/TCE.h

@@ -52,6 +52,7 @@ static const unsigned TCEOpenCLAddrSpaceMap[] = {
     0, // ptr64
     0, // hlsl_groupshared
     0, // hlsl_constant
+    0, // hlsl_private
     // Wasm address space values for this target are dummy values,
     // as it is only enabled for Wasm targets.
     20, // wasm_funcref

clang/lib/Basic/Targets/WebAssembly.h

@@ -43,6 +43,7 @@ static const unsigned WebAssemblyAddrSpaceMap[] = {
     0,  // ptr64
     0,  // hlsl_groupshared
     0,  // hlsl_constant
+    0,  // hlsl_private
     20, // wasm_funcref
 };
 

clang/lib/Basic/Targets/X86.h

@@ -47,6 +47,7 @@ static const unsigned X86AddrSpaceMap[] = {
     272, // ptr64
     0,   // hlsl_groupshared
     0,   // hlsl_constant
+    0,   // hlsl_private
     // Wasm address space values for this target are dummy values,
     // as it is only enabled for Wasm targets.
     20, // wasm_funcref

clang/lib/CodeGen/CGDeclCXX.cpp

@@ -1157,8 +1157,22 @@ void CodeGenFunction::GenerateCXXGlobalCleanUpFunc(
             CGM.getCXXABI().useSinitAndSterm() &&
             "Arg could not be nullptr unless using sinit and sterm functions.");
         CI = Builder.CreateCall(CalleeTy, Callee);
-      } else
+      } else {
+        // If the object lives in a different address space, the `this` pointer
+        // address space won't match the dtor `this` param. An addrspacecast is
+        // required.
+        assert(Arg->getType()->isPointerTy());
+        assert(CalleeTy->getParamType(0)->isPointerTy());
+        unsigned ActualAddrSpace = Arg->getType()->getPointerAddressSpace();
+        unsigned ExpectedAddrSpace =
+            CalleeTy->getParamType(0)->getPointerAddressSpace();
+        if (ActualAddrSpace != ExpectedAddrSpace) {
+          llvm::PointerType *PTy =
+              llvm::PointerType::get(getLLVMContext(), ExpectedAddrSpace);
+          Arg = llvm::ConstantExpr::getAddrSpaceCast(Arg, PTy);
+        }
         CI = Builder.CreateCall(CalleeTy, Callee, Arg);
+      }
 
       // Make sure the call and the callee agree on calling convention.
       if (llvm::Function *F = dyn_cast<llvm::Function>(Callee))

clang/lib/CodeGen/CodeGenFunction.cpp

@@ -371,12 +371,6 @@ void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
   assert(DeferredDeactivationCleanupStack.empty() &&
          "mismatched activate/deactivate of cleanups!");
 
-  if (CGM.shouldEmitConvergenceTokens()) {
-    ConvergenceTokenStack.pop_back();
-    assert(ConvergenceTokenStack.empty() &&
-           "mismatched push/pop in convergence stack!");
-  }
-
   bool OnlySimpleReturnStmts = NumSimpleReturnExprs > 0
     && NumSimpleReturnExprs == NumReturnExprs
     && ReturnBlock.getBlock()->use_empty();
@@ -564,6 +558,12 @@ void CodeGenFunction::FinishFunction(SourceLocation EndLoc) {
       ReturnValue = Address::invalid();
     }
   }
+
+  if (CGM.shouldEmitConvergenceTokens()) {
+    ConvergenceTokenStack.pop_back();
+    assert(ConvergenceTokenStack.empty() &&
+           "mismatched push/pop in convergence stack!");
+  }
 }
 
 /// ShouldInstrumentFunction - Return true if the current function should be

clang/lib/Sema/SemaDecl.cpp

@@ -6888,6 +6888,42 @@ static void SetNestedNameSpecifier(Sema &S, DeclaratorDecl *DD, Declarator &D) {
   DD->setQualifierInfo(SS.getWithLocInContext(S.Context));
 }
 
+void Sema::deduceHLSLAddressSpace(VarDecl *Decl) {
+  // The variable already has an address space (groupshared for ex).
+  if (Decl->getType().hasAddressSpace())
+    return;
+
+  if (Decl->getType()->isDependentType())
+    return;
+
+  QualType Type = Decl->getType();
+  if (Type->isSamplerT() || Type->isVoidType())
+    return;
+
+  // Template instantiations can lack definition. In such case,
+  // we cannot deduce the AS.
+  // FIXME: figure out why RWBuffer<float> yields such declaration.
+  if (const RecordType *RT =
+          dyn_cast<RecordType>(Type->getUnqualifiedDesugaredType())) {
+    CXXRecordDecl *RD = Type->getAsCXXRecordDecl();
+    if (RD && !RD->isCompleteDefinition())
+      return;
+  }
+
+  // Resource handles.
+  if (Type->isHLSLIntangibleType())
+    return;
+
+  // Only static globals belong to the Private address space.
+  // Non-static globals belongs to the cbuffer.
+  if (Decl->getStorageClass() != SC_Static && !Decl->isStaticDataMember())
+    return;
+
+  LangAS ImplAS = LangAS::hlsl_private;
+  Type = Context.getAddrSpaceQualType(Type, ImplAS);
+  Decl->setType(Type);
+}
+
 void Sema::deduceOpenCLAddressSpace(ValueDecl *Decl) {
   if (Decl->getType().hasAddressSpace())
     return;
@@ -7975,8 +8011,11 @@ NamedDecl *Sema::ActOnVariableDeclarator(
   // Handle attributes prior to checking for duplicates in MergeVarDecl
   ProcessDeclAttributes(S, NewVD, D);
 
-  if (getLangOpts().HLSL)
+  if (getLangOpts().HLSL) {
     HLSL().ActOnVariableDeclarator(NewVD);
+    deduceHLSLAddressSpace(NewVD);
+  }
+
   if (getLangOpts().OpenACC)
     OpenACC().ActOnVariableDeclarator(NewVD);
 
@@ -13131,6 +13170,9 @@ bool Sema::DeduceVariableDeclarationType(VarDecl *VDecl, bool DirectInit,
   if (getLangOpts().OpenCL)
     deduceOpenCLAddressSpace(VDecl);
 
+  if (getLangOpts().HLSL)
+    deduceHLSLAddressSpace(VDecl);
+
   // If this is a redeclaration, check that the type we just deduced matches
   // the previously declared type.
   if (VarDecl *Old = VDecl->getPreviousDecl()) {

clang/test/AST/HLSL/cbuffer.hlsl

@@ -149,7 +149,7 @@ _Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(TwoFloats, __cblay
 cbuffer CB {
   // CHECK: FunctionDecl {{.*}} f 'void ()'
   void f() {}
-  // CHECK: VarDecl {{.*}} SV 'float' static
+  // CHECK: VarDecl {{.*}} SV 'hlsl_private float' static
   static float SV;
   // CHECK: VarDecl {{.*}} s7 'EmptyStruct' callinit
   EmptyStruct s7;

clang/test/AST/HLSL/private.hlsl

@@ -0,0 +1,34 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.3-library -ast-dump -o - %s | FileCheck %s
+
+// CHECK: VarDecl {{.*}} global_scalar 'hlsl_private int' static cinit
+static int global_scalar = 0;
+
+// CHECK: VarDecl {{.*}} global_buffer 'RWBuffer<float>':'hlsl::RWBuffer<float>' static callinit
+RWBuffer<float> global_buffer;
+
+class A {
+// CHECK: VarDecl {{.*}} a 'hlsl_private int' static
+  static int a;
+};
+
+class B {
+// CHECK: VarDecl {{.*}} b 'hlsl_private int' static
+  static int b;
+};
+
+// CHECK: VarDecl {{.*}} b 'hlsl_private int' cinit
+int B::b = 0;
+
+export void foo() {
+// CHECK: VarDecl {{.*}} local_buffer 'RWBuffer<float>':'hlsl::RWBuffer<float>' cinit
+  RWBuffer<float> local_buffer = global_buffer;
+
+// CHECK: VarDecl {{.*}} static_local_buffer 'RWBuffer<float>':'hlsl::RWBuffer<float>' static cinit
+  static RWBuffer<float> static_local_buffer = global_buffer;
+
+// CHECK: VarDecl {{.*}} local_scalar 'int' cinit
+  int local_scalar = global_scalar;
+
+// CHECK: VarDecl {{.*}} static_scalar 'hlsl_private int' static cinit
+  static int static_scalar = 0;
+}

clang/test/CodeGenHLSL/GlobalConstructors.hlsl

@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -x hlsl -emit-llvm -disable-llvm-passes %s -o - | FileCheck %s
+// RUN: %clang_cc1 -triple spirv-unknown-vulkan1.3-compute -x hlsl -emit-llvm -disable-llvm-passes %s -o - | FileCheck %s --check-prefixes=CHECK,SPIRV
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-compute -x hlsl -emit-llvm -disable-llvm-passes %s -o - | FileCheck %s --check-prefixes=CHECK,DXIL
 
 RWBuffer<float> Buffer;
 
@@ -10,7 +11,13 @@ void main(unsigned GI : SV_GroupIndex) {}
 // CHECK-NOT:@llvm.global_dtors
 //CHECK:      define void @main()
 //CHECK-NEXT: entry:
-//CHECK-NEXT:   call void @_GLOBAL__sub_I_GlobalConstructors.hlsl()
-//CHECK-NEXT:   %0 = call i32 @llvm.dx.flattened.thread.id.in.group()
-//CHECK-NEXT:   call void @_Z4mainj(i32 %0)
+
+//SPIRV-NEXT:   %0 = call token @llvm.experimental.convergence.entry()
+//SPIRV-NEXT:   call spir_func void @_GLOBAL__sub_I_GlobalConstructors.hlsl() [ "convergencectrl"(token %0) ]
+//SPIRV-NEXT:   %1 = call i32 @llvm.spv.flattened.thread.id.in.group()
+//SPIRV-NEXT:   call spir_func void @_Z4mainj(i32 %1) [ "convergencectrl"(token %0) ]
+
+//DXIL-NEXT:    call void @_GLOBAL__sub_I_GlobalConstructors.hlsl()
+//DXIL-NEXT:    %0 = call i32 @llvm.dx.flattened.thread.id.in.group()
+//DXIL-NEXT:    call void @_Z4mainj(i32 %0)
 //CHECK-NEXT:   ret void

clang/test/CodeGenHLSL/cbuffer_with_static_global_and_function.hlsl

@@ -20,6 +20,7 @@ cbuffer B {
 
 // CHECK: define {{.*}} float @_Z3foov() #0 {
 // CHECK: load float, ptr addrspace(2) @a, align 4
+// CHECK: load float, ptr @_ZL1b, align 4
 
 extern float bar() {
   return foo();