Add GenericCastToPtrOpt pass

In cases where we have no local casts to generics and we allocate
private memory in global space, we can replace GenericCastToPtrExplicit
with simple address space cast.

Author: stefan-il

IGC/AdaptorOCL/UnifyIROCL.cpp

@@ -6,11 +6,8 @@ SPDX-License-Identifier: MIT
 
 ============================= end_copyright_notice ===========================*/
 
-#include <chrono>
-
 #include "common/LLVMWarningsPush.hpp"
 #include <llvm/Support/ScaledNumber.h>
-#include "llvm/ADT/PostOrderIterator.h"
 #include <llvm/IR/LLVMContext.h>
 #include <llvm/IR/Verifier.h>
 #include <llvm/Analysis/CFGPrinter.h>
@@ -41,6 +38,7 @@ SPDX-License-Identifier: MIT
 #include "Compiler/CISACodeGen/ShaderCodeGen.hpp"
 #include "Compiler/CISACodeGen/EstimateFunctionSize.h"
 #include "Compiler/CISACodeGen/FixAddrSpaceCast.h"
+#include "Compiler/Optimizer/OpenCLPasses/GenericCastToPtrOpt/GenericCastToPtrOpt.hpp"
 #include "Compiler/Optimizer/OpenCLPasses/GenericAddressResolution/GASResolving.h"
 #include "Compiler/Optimizer/OpenCLPasses/GenericAddressResolution/GASRetValuePropagator.h"
 #include "Compiler/Optimizer/OpenCLPasses/GenericAddressResolution/StaticGASResolution.h"
@@ -63,35 +61,29 @@ SPDX-License-Identifier: MIT
 #include "Compiler/Optimizer/OpenCLPasses/PrepareInlineSamplerForBindless/PrepareInlineSamplerForBindless.hpp"
 #include "Compiler/Optimizer/OpenCLPasses/ResolveInlineSamplerForBindless/ResolveInlineSamplerForBindless.hpp"
 #include "Compiler/Optimizer/OpenCLPasses/PrivateMemory/PrivateMemoryUsageAnalysis.hpp"
-#include "Compiler/Optimizer/OpenCLPasses/PrivateMemory/PrivateMemoryResolution.hpp"
 #include "Compiler/Optimizer/OpenCLPasses/ProgramScopeConstants/ProgramScopeConstantAnalysis.hpp"
 #include "Compiler/Optimizer/OpenCLPasses/ProgramScopeConstants/ProgramScopeConstantResolution.hpp"
 #include "Compiler/Optimizer/OpenCLPasses/WIFuncs/WIFuncsAnalysis.hpp"
 #include "Compiler/Optimizer/OpenCLPasses/WIFuncs/WIFuncResolution.hpp"
 #include "Compiler/Optimizer/OpenCLPasses/ResourceAllocator/ResourceAllocator.hpp"
 #include "Compiler/Optimizer/OpenCLPasses/BreakConstantExpr/BreakConstantExpr.hpp"
 #include "Compiler/Optimizer/OpenCLPasses/LocalBuffers/InlineLocalsResolution.hpp"
-#include "Compiler/Optimizer/OpenCLPasses/ReplaceUnsupportedIntrinsics/ReplaceUnsupportedIntrinsics.hpp"
 #include "Compiler/Optimizer/OpenCLPasses/Atomics/ResolveOCLAtomics.hpp"
 #include "Compiler/Optimizer/OpenCLPasses/WGFuncs/WGFuncResolution.hpp"
 #include "Compiler/Optimizer/OpenCLPasses/AlignmentAnalysis/AlignmentAnalysis.hpp"
-#include "Compiler/Optimizer/PreCompiledFuncImport.hpp"
 #include "Compiler/Optimizer/OpenCLPasses/OpenCLPrintf/InjectPrintf.hpp"
 #include "Compiler/Optimizer/OpenCLPasses/OpenCLPrintf/OpenCLPrintfAnalysis.hpp"
 #include "Compiler/Optimizer/OpenCLPasses/OpenCLPrintf/OpenCLPrintfResolution.hpp"
 #include "Compiler/Optimizer/OpenCLPasses/AggregateArguments/AggregateArguments.hpp"
 #include "Compiler/Optimizer/OCLBIConverter.h"
 #include "Compiler/Optimizer/OpenCLPasses/SetFastMathFlags/SetFastMathFlags.hpp"
 #include "Compiler/Optimizer/OpenCLPasses/CorrectlyRoundedDivSqrt/CorrectlyRoundedDivSqrt.hpp"
-#include "Compiler/Optimizer/OpenCLPasses/GenericAddressResolution/GenericAddressDynamicResolution.hpp"
-#include "Compiler/Optimizer/OpenCLPasses/AddressSpaceAliasAnalysis/AddressSpaceAliasAnalysis.h"
 #include "Compiler/Optimizer/OpenCLPasses/DeviceEnqueueFuncs/DeviceEnqueue.hpp"
 #include "Compiler/Optimizer/OpenCLPasses/UndefinedReferences/UndefinedReferencesPass.hpp"
 #include "Compiler/Optimizer/OpenCLPasses/SubGroupFuncs/SubGroupFuncsResolution.hpp"
 #include "Compiler/Optimizer/OpenCLPasses/BIFTransforms/BIFTransforms.hpp"
 #include "Compiler/Optimizer/OpenCLPasses/BreakdownIntrinsic/BreakdownIntrinsic.h"
 #include "Compiler/Optimizer/OpenCLPasses/TransformUnmaskedFunctionsPass/TransformUnmaskedFunctionsPass.h"
-#include "Compiler/Optimizer/OpenCLPasses/StatelessToStateful/StatelessToStateful.hpp"
 #include "Compiler/Optimizer/OpenCLPasses/KernelFunctionCloning/KernelFunctionCloning.h"
 #include "Compiler/Optimizer/OpenCLPasses/NontemporalLoadsAndStoresInAssert/NontemporalLoadsAndStoresInAssert.hpp"
 #include "Compiler/Optimizer/OpenCLPasses/HandleDevicelibAssert/HandleDevicelibAssert.hpp"
@@ -114,13 +106,10 @@ SPDX-License-Identifier: MIT
 #include "Compiler/Optimizer/OpenCLPasses/AccuracyDecoratedCallsBiFResolution/AccuracyDecoratedCallsBiFResolution.hpp"
 #include "Compiler/Optimizer/OpenCLPasses/ScalarArgAsPointer/ScalarArgAsPointer.hpp"
 #include "AdaptorCommon/RayTracing/RayTracingPasses.hpp"
-#include "Compiler/MetaDataApi/IGCMetaDataHelper.h"
 #include "Compiler/CodeGenContextWrapper.hpp"
 #include "Compiler/FixResourcePtr.hpp"
-#include "Compiler/InitializePasses.h"
 #include "Compiler/MetaDataApi/SpirMetaDataApi.h"
 #include "Compiler/Optimizer/ReduceOptPass.hpp"
-#include "Compiler/CustomUnsafeOptPass.hpp"
 #include "MoveStaticAllocas.h"
 #include "preprocess_spvir/PreprocessSPVIR.h"
 #include "preprocess_spvir/ConvertUserSemanticDecoratorOnFunctions.h"
@@ -137,20 +126,15 @@ SPDX-License-Identifier: MIT
 
 #include "common/debug/Debug.hpp"
 #include "common/igc_regkeys.hpp"
-#include "common/debug/Dump.hpp"
-#include "common/MemStats.h"
 
 #include <iStdLib/utility.h>
 
 #include "Compiler/CISACodeGen/DebugInfo.hpp"
 #include "Compiler/CISACodeGen/TimeStatsCounter.h"
 #include "Compiler/DebugInfo/ScalarVISAModule.h"
-#include "Compiler/DebugInfo/Utils.h"
 #include "Compiler/Builtins/BIFFlagCtrl/BIFFlagCtrlResolution.hpp"
-#include "DebugInfo/VISADebugEmitter.hpp"
 
 #include <string>
-#include <algorithm>
 
 #include <Metrics/IGCMetric.h>
 
@@ -398,6 +382,10 @@ static void CommonOCLBasedPasses(OpenCLProgramContext *pContext) {
     // mark load and stores inside assert calls as nontemporal to avoid caching.
     mpm.add(new NontemporalLoadsAndStoresInAssert());
 
+    if (IGC_IS_FLAG_ENABLED(EnableGenericCastToPtrOpt)) {
+      mpm.add(new GenericCastToPtrOpt());
+    }
+
     // Report undef references after setting func attribs for import linking
     mpm.add(new UndefinedReferencesPass());
 

IGC/Compiler/InitializePasses.h

@@ -63,6 +63,7 @@ void initializeErrorCheckPass(llvm::PassRegistry &);
 void initializePoisonFP64KernelsPass(llvm::PassRegistry &);
 void initializeReduceOptPassPass(llvm::PassRegistry &);
 void initializeTypesLegalizationPassPass(llvm::PassRegistry &);
+void initializeGenericCastToPtrOptPass(llvm::PassRegistry &);
 void initializeGASResolvingPass(llvm::PassRegistry &);
 void initializeGASRetValuePropagatorPass(llvm::PassRegistry &);
 void initializeStaticGASResolutionPass(llvm::PassRegistry &);

IGC/Compiler/Optimizer/OpenCLPasses/CMakeLists.txt

@@ -23,6 +23,7 @@ add_subdirectory(DpasFuncs)
 add_subdirectory(BfloatFuncs)
 add_subdirectory(ErrorCheckPass)
 add_subdirectory(ExtensionFuncs)
+add_subdirectory(GenericCastToPtrOpt)
 add_subdirectory(GEPLoopStrengthReduction)
 add_subdirectory(GenericAddressResolution)
 add_subdirectory(HandleDevicelibAssert)
@@ -82,6 +83,7 @@ set(IGC_BUILD__SRC__Optimizer_OpenCLPasses_All
     ${IGC_BUILD__SRC__OpenCLPasses_BfloatFuncs}
     ${IGC_BUILD__SRC__OpenCLPasses_ErrorCheckPass}
     ${IGC_BUILD__SRC__OpenCLPasses_ExtensionFuncs}
+    ${IGC_BUILD__SRC__OpenCLPasses_GenericCastToPtrOpt}
     ${IGC_BUILD__SRC__OpenCLPasses_GEPLoopStrengthReduction}
     ${IGC_BUILD__SRC__OpenCLPasses_GenericAddressResolution}
     ${IGC_BUILD__SRC__OpenCLPasses_HandleDevicelibAssert}
@@ -145,6 +147,7 @@ set(IGC_BUILD__HDR__Optimizer_OpenCLPasses_All
     ${IGC_BUILD__HDR__OpenCLPasses_BfloatFuncs}
     ${IGC_BUILD__HDR__OpenCLPasses_ErrorCheckPass}
     ${IGC_BUILD__HDR__OpenCLPasses_ExtensionFuncs}
+    ${IGC_BUILD__HDR__OpenCLPasses_GenericCastToPtrOpt}
     ${IGC_BUILD__HDR__OpenCLPasses_GEPLoopStrengthReduction}
     ${IGC_BUILD__HDR__OpenCLPasses_GenericAddressResolution}
     ${IGC_BUILD__HDR__OpenCLPasses_HandleDevicelibAssert}
@@ -209,6 +212,7 @@ set(IGC_BUILD_Compiler_OpenCLPasses_Groups
     Compiler__OpenCLPasses_BfloatFuncs
     Compiler__OpenCLPasses_ErrorCheckPass
     Compiler__OpenCLPasses_ExtensionFuncs
+    Compiler__OpenCLPasses_GenericCastToPtrOpt
     Compiler__OpenCLPasses_GEPLoopStrengthReduction
     Compiler__OpenCLPasses_GenericAddressResolution
     Compiler__OpenCLPasses_GenericAddressSpaceStaticResolution

IGC/Compiler/Optimizer/OpenCLPasses/GenericCastToPtrOpt/CMakeLists.txt

@@ -0,0 +1,30 @@
+#=========================== begin_copyright_notice ============================
+#
+# Copyright (C) 2025 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+#
+#============================ end_copyright_notice =============================
+
+include_directories("${CMAKE_CURRENT_SOURCE_DIR}")
+
+
+set(IGC_BUILD__SRC__GenericCastToPtrOpt
+    "${CMAKE_CURRENT_SOURCE_DIR}/GenericCastToPtrOpt.cpp"
+  )
+set(IGC_BUILD__SRC__OpenCLPasses_GenericCastToPtrOpt ${IGC_BUILD__SRC__GenericCastToPtrOpt} PARENT_SCOPE)
+
+set(IGC_BUILD__HDR__GenericCastToPtrOpt
+    "${CMAKE_CURRENT_SOURCE_DIR}/GenericCastToPtrOpt.hpp"
+  )
+set(IGC_BUILD__HDR__OpenCLPasses_GenericCastToPtrOpt ${IGC_BUILD__HDR__GenericCastToPtrOpt} PARENT_SCOPE)
+
+
+igc_sg_register(
+    Compiler__OpenCLPasses_GenericCastToPtrOpt
+    "GenericCastToPtrOpt"
+    FILES
+      ${IGC_BUILD__SRC__GenericCastToPtrOpt}
+      ${IGC_BUILD__HDR__GenericCastToPtrOpt}
+  )
+

IGC/Compiler/Optimizer/OpenCLPasses/GenericCastToPtrOpt/GenericCastToPtrOpt.cpp

@@ -0,0 +1,124 @@
+/*========================== begin_copyright_notice ============================
+
+Copyright (C) 2025 Intel Corporation
+
+SPDX-License-Identifier: MIT
+
+============================= end_copyright_notice ===========================*/
+
+#include "Compiler/Optimizer/OpenCLPasses/GenericCastToPtrOpt/GenericCastToPtrOpt.hpp"
+
+#include "common/LLVMWarningsPush.hpp"
+#include <llvm/ADT/SmallVector.h>
+#include <llvm/ADT/StringRef.h>
+#include <llvm/Analysis/CallGraph.h>
+#include <llvm/IR/Function.h>
+#include <llvm/IR/IRBuilder.h>
+#include <llvm/Pass.h>
+#include "common/LLVMWarningsPop.hpp"
+
+#include "CISACodeGen/CastToGASAnalysis.h"
+#include "Probe/Assertion.h"
+#include "Compiler/IGCPassSupport.h"
+#include "llvmWrapper/ADT/Optional.h"
+
+using namespace llvm;
+using namespace IGC;
+
+#define PASS_FLAG "igc-generic-cast-to-ptr-opt"
+#define PASS_DESCRIPTION "Optimize GenericCastToPtrExplicit casts"
+#define PASS_CFG_ONLY false
+#define PASS_ANALYSIS false
+IGC_INITIALIZE_PASS_BEGIN(GenericCastToPtrOpt, PASS_FLAG, PASS_DESCRIPTION,
+                          PASS_CFG_ONLY, PASS_ANALYSIS)
+IGC_INITIALIZE_PASS_DEPENDENCY(CastToGASAnalysis)
+IGC_INITIALIZE_PASS_DEPENDENCY(CallGraphWrapperPass)
+IGC_INITIALIZE_PASS_END(GenericCastToPtrOpt, PASS_FLAG, PASS_DESCRIPTION,
+                        PASS_CFG_ONLY, PASS_ANALYSIS)
+
+char GenericCastToPtrOpt::ID = 0;
+
+constexpr std::string_view GENERIC_CAST_TO_PTR_FN_NAME =
+    "spirv_GenericCastToPtrExplicit_ToGlobal";
+
+static void replaceGenericCastToPtrCall(CallInst *TargetFnCall) {
+  IRBuilder<> Builder(TargetFnCall->getParent());
+  Builder.SetInsertPoint(TargetFnCall);
+  auto *AddrSpaceCast = Builder.CreateAddrSpaceCast(
+      TargetFnCall->getArgOperand(0),
+      TargetFnCall->getCalledFunction()->getReturnType(),
+      "generic_cast_to_ptr");
+  IGC_ASSERT(TargetFnCall->getCalledFunction()->getReturnType()->isPointerTy());
+  IGC_ASSERT(TargetFnCall->getArgOperand(0)->getType()->isPointerTy());
+  TargetFnCall->replaceAllUsesWith(AddrSpaceCast);
+  TargetFnCall->eraseFromParent();
+
+  // Clean up the users of the address space cast
+  auto *NewAddrSpaceCast = dyn_cast<AddrSpaceCastInst>(AddrSpaceCast);
+  SmallVector<AddrSpaceCastInst *, 32> UsersToRemove;
+  for (auto *User : NewAddrSpaceCast->users()) {
+    if (auto *AddrSpaceCastUser = dyn_cast<AddrSpaceCastInst>(User)) {
+      if (NewAddrSpaceCast->getSrcTy() == AddrSpaceCastUser->getDestTy()) {
+        AddrSpaceCastUser->replaceAllUsesWith(NewAddrSpaceCast->getOperand(0));
+        UsersToRemove.push_back(AddrSpaceCastUser);
+      }
+    }
+  }
+  for (auto *User : UsersToRemove) {
+    User->eraseFromParent();
+  }
+}
+
+GenericCastToPtrOpt::GenericCastToPtrOpt() : ModulePass(ID) {
+  initializeGenericCastToPtrOptPass(*PassRegistry::getPassRegistry());
+}
+
+void GenericCastToPtrOpt::getAnalysisUsage(llvm::AnalysisUsage &AU) const {
+  AU.addRequired<CastToGASAnalysis>();
+  AU.addRequired<CallGraphWrapperPass>();
+
+  AU.setPreservesCFG();
+}
+
+bool GenericCastToPtrOpt::runOnModule(Module &M) {
+  if (skipModule(M)) {
+    return false;
+  }
+
+  CallGraph &CG = getAnalysis<CallGraphWrapperPass>().getCallGraph();
+  GASInfo &GI = getAnalysis<CastToGASAnalysis>().getGASInfo();
+  const bool noGenericPtToLocalOrPrivate =
+      GI.isNoLocalToGenericOptionEnabled() &&
+      GI.isPrivateAllocatedInGlobalMemory();
+
+  bool modified = false;
+
+  for (auto &[Fn, FnCallGraph] : CG) {
+    if (!Fn || Fn->hasOptNone()) {
+      continue;
+    }
+
+    const bool noGenericPtToLocalOrPrivateFn =
+        !GI.canGenericPointToLocal(*FnCallGraph->getFunction()) &&
+        !GI.canGenericPointToPrivate(*FnCallGraph->getFunction());
+    if (!noGenericPtToLocalOrPrivate && !noGenericPtToLocalOrPrivateFn) {
+      continue;
+    }
+
+    for (auto &[CallInstVH, CallRecordNode] : *FnCallGraph.get()) {
+      // We check only direct calls
+      auto CallInstVHOptional = IGCLLVM::makeOptional(CallInstVH);
+      if (!CallInstVHOptional.has_value()) {
+        continue;
+      }
+      auto *CallRecordFn = CallRecordNode->getFunction();
+      if (CallRecordFn && CallRecordFn->getName().contains(llvm::StringRef(
+                              GENERIC_CAST_TO_PTR_FN_NAME.data(),
+                              GENERIC_CAST_TO_PTR_FN_NAME.size()))) {
+        replaceGenericCastToPtrCall(cast<CallInst>(CallInstVHOptional.value()));
+        modified = true;
+      }
+    }
+  }
+  return modified;
+}

IGC/Compiler/Optimizer/OpenCLPasses/GenericCastToPtrOpt/GenericCastToPtrOpt.hpp

@@ -0,0 +1,33 @@
+/*========================== begin_copyright_notice ============================
+
+Copyright (C) 2025 Intel Corporation
+
+SPDX-License-Identifier: MIT
+
+============================= end_copyright_notice ===========================*/
+
+#pragma once
+
+#include "common/LLVMWarningsPush.hpp"
+#include <llvm/Pass.h>
+#include "common/LLVMWarningsPop.hpp"
+
+using namespace llvm;
+
+namespace IGC {
+/// @brief  Optimizes GenericCastToPtrExplicit calls by replacing them with
+///         addrspace_cast instructions.
+class GenericCastToPtrOpt : public llvm::ModulePass {
+public:
+  static char ID;
+
+  GenericCastToPtrOpt();
+
+  void getAnalysisUsage(llvm::AnalysisUsage &AU) const override;
+
+  StringRef getPassName() const override { return "GenericCastToPtrOpt"; }
+
+  bool runOnModule(llvm::Module &M) override;
+};
+
+} // namespace IGC

IGC/Compiler/tests/GenericCastToPtrOpt/generic-cast-to-ptr.ll

@@ -0,0 +1,44 @@
+;=========================== begin_copyright_notice ============================
+;
+; Copyright (C) 2025 Intel Corporation
+;
+; SPDX-License-Identifier: MIT
+;
+;============================ end_copyright_notice =============================
+; REQUIRES: regkeys
+; RUN: igc_opt --igc-generic-cast-to-ptr-opt --regkey EnableGenericCastToPtrOpt=1 -S %s --platformpvc | FileCheck %s
+; ------------------------------------------------
+; GenericCastToPtrOpt
+; ------------------------------------------------
+
+; Check that calls to GenericCastToPtrExplicit_ToGlobal are correctly transformed to addrsspacecasts
+
+declare spir_func void @testExternFn(i8 addrspace(1)* noundef)
+
+declare spir_func i8 addrspace(3)* @__builtin_IB_to_local(i8 addrspace(4)* noundef)
+
+declare spir_func i8* @__builtin_IB_to_private(i8 addrspace(4)* noundef)
+
+define internal spir_func i8 addrspace(1)* @_Z41__spirv_GenericCastToPtrExplicit_ToGlobalPU3AS4ci(i8 addrspace(4)* noundef %0, i32 noundef %1) {
+  %3 = tail call spir_func i8 addrspace(3)* @__builtin_IB_to_local(i8 addrspace(4)* noundef %0) #18
+  %4 = icmp eq i8 addrspace(3)* %3, null
+
+  %5 = tail call spir_func i8* @__builtin_IB_to_private(i8 addrspace(4)* noundef %0) #18
+  %6 = icmp eq i8* %5, null
+
+  %7 = and i1 %4, %6
+  %8 = addrspacecast i8 addrspace(4)* %0 to i8 addrspace(1)*
+  %9 = select i1 %7, i8 addrspace(1)* %8, i8 addrspace(1)* null
+  ret i8 addrspace(1)* %9
+}
+
+; CHECK-LABEL: testFn
+; CHECK-NOT: %result = call spir_func i8 addrspace(1)* @_Z41__spirv_GenericCastToPtrExplicit_ToGlobalPU3AS4ci(i8 addrspace(4)* %ptr, i32 0)
+; CHECK: %generic_cast_to_ptr = addrspacecast [[PTR:.*]] addrspace(4){{.*}} %ptr to [[PTR]] addrspace(1){{.*}}
+; CHECK: call spir_func void @testExternFn([[PTR]] addrspace(1){{.*}} %generic_cast_to_ptr)
+define spir_kernel void @testFn() {
+  %ptr = inttoptr i64 0 to i8 addrspace(4)*
+  %result = call spir_func i8 addrspace(1)* @_Z41__spirv_GenericCastToPtrExplicit_ToGlobalPU3AS4ci(i8 addrspace(4)* %ptr, i32 0)
+  call spir_func void @testExternFn(i8 addrspace(1)* %result)
+  ret void
+}

IGC/common/igc_flags.h

@@ -974,6 +974,9 @@ DECLARE_IGC_REGKEY(bool, EnablePreRARematFlag, true, "Enable PreRA Rematerializa
 DECLARE_IGC_REGKEY(bool, EnableGASResolver, true, "Enable GAS Resolver", false)
 DECLARE_IGC_REGKEY(bool, EnableLowerGPCallArg, true, "Enable pass to lower generic pointers in function arguments",
                    false)
+DECLARE_IGC_REGKEY(
+    bool, EnableGenericCastToPtrOpt, true,
+    "Enable simplification of GenericCastToPtrExplicit_ToGlobal calls", false)
 DECLARE_IGC_REGKEY(bool, DisableRecompilation, false, "Disable recompilation, skip retry stage", true)
 DECLARE_IGC_REGKEY(bool, SampleMultiversioning, false,
                    "Create branches aroung samplers which can be redundant with some values", false)