Adding an experimental feature for load coalescing

Adding an experimental feature for load coalescing

Author: mkhoshza

IGC/Compiler/CISACodeGen/MemOpt.cpp

@@ -2716,6 +2716,8 @@ namespace {
             return splitVectorType(V, LdStKind::IS_LOAD);
         }
 
+        void AllowDummyLoadCoalescing(InstAndOffsetPairs Loads);
+
         // GatherCopy:
         //   copy multiple values (arg: Vals) into a single Dst (return value)
         //   (It's a packed copy, thus size(all Vals) = size(Dst).
@@ -2822,7 +2824,7 @@ bool IGC::doLdStCombine(const CodeGenContext* CGC) {
     uint32_t keyval = IGC_GET_FLAG_VALUE(EnableLdStCombine);
     if ((keyval & 0x3) == 1 && !CGC->platform.LSCEnabled())
         return false;
-    return ((keyval & 0x3) != 0);
+    return ((keyval & 0x3) || (keyval & 0x4));
 }
 
 uint32_t IGC::getMaxStoreBytes(const CodeGenContext* CGC) {
@@ -3351,9 +3353,13 @@ void LdStCombine::combineLoads()
     if ((IGC_GET_FLAG_VALUE(EnableLdStCombine) & 0x4) == 0)
         return;
 
-    // Start with OCL, then apply to other APIs.
     if (m_CGC->type != ShaderType::OPENCL_SHADER)
-        return;
+    {
+        if (!m_CGC->getModuleMetaData()->compOpt.EnableLdStCombineforLoad)
+        {
+            return;
+        }
+    }
 
     // All load candidates with addr = common-base + const-offset
     InstAndOffsetPairs Loads;
@@ -3439,6 +3445,14 @@ void LdStCombine::combineLoads()
                 }
             }
 
+            //Experiment: If its the last element of the load and does not fit the DWORD alignment,
+            //It creates a dummy load with the same alignment type as the previous load
+            if (m_CGC->type != ShaderType::OPENCL_SHADER)
+            {
+                if (m_CGC->getModuleMetaData()->compOpt.EnableLdStCombinewithDummyLoad)
+                    AllowDummyLoadCoalescing(Loads);
+            }
+
             //   Note: For now, each load is considered once. For example,
             //     load a
             //       store x : alias to load c
@@ -3802,6 +3816,64 @@ void LdStCombine::createBundles(BasicBlock* BB, InstAndOffsetPairs& LoadStores)
     markVisited(LoadStores);
 }
 
+void LdStCombine::AllowDummyLoadCoalescing(InstAndOffsetPairs Loads)
+{
+    // Currently supports only this pattern.
+    // % 164 = add i32 % 114, 1020
+    // % 165 = and i32 % 164, 1020
+    // % 166 = getelementptr[1024 x half], [1024 x half] addrspace(3) * null, i32 0, i32 % 165
+    // %167 = load half, half addrspace(3) * %166, align 8
+    // % 168 = or i32 % 165, 1
+    // % 169 = getelementptr[1024 x half], [1024 x half] addrspace(3) * null, i32 0, i32 % 168
+    // % 170 = load half, half addrspace(3) * %169, align 2
+    // % 171 = or i32 % 165, 2
+    // % 172 = getelementptr[1024 x half], [1024 x half] addrspace(3) * null, i32 0, i32 % 171
+    // % 173 = load half, half addrspace(3) * %172, align 4
+    // to
+    // % 164 = add i32 % 114, 1020
+    // % 165 = and i32 % 164, 1020
+    // % 166 = getelementptr[1024 x half], [1024 x half] addrspace(3) * null, i32 0, i32 % 165
+    // %167 = load half, half addrspace(3) * %166, align 8
+    // % 168 = or i32 % 165, 1
+    // % 169 = getelementptr[1024 x half], [1024 x half] addrspace(3) * null, i32 0, i32 % 168
+    // % 170 = load half, half addrspace(3) * %169, align 2
+    // % 171 = or i32 % 165, 2
+    // % 172 = getelementptr[1024 x half], [1024 x half] addrspace(3) * null, i32 0, i32 % 171
+    // % 173 = load half, half addrspace(3) * %172, align 4
+    // % 174 = add i32 % 165, 3
+    // % 175 = getelementptr[1024 x half], [1024 x half] addrspace(3) * null, i32 0, i32 % 174
+    // % 176 = load half, half addrspace(3) * %175, align 2
+    int size = Loads.size();
+    LdStInfo LastLoad = Loads[size - 1];
+    uint32_t LastLoadSize = (uint32_t)m_DL->getTypeStoreSize(LastLoad.Inst->getType());
+    uint32_t currLoadSize = LastLoadSize + LastLoad.ByteOffset;
+    if (currLoadSize % 4)
+    {
+        //Replicating the last load to make it DWORD aligned
+        uint32_t newLoadSize = LastLoadSize;
+        if (!((currLoadSize + newLoadSize) % 4))
+        {
+            LoadInst* lead = static_cast<LoadInst*>(LastLoad.Inst);
+            Value* ldPtr = lead->getPointerOperand();
+            if (auto gep = dyn_cast<GetElementPtrInst>(ldPtr))
+            {
+                if ((gep->getNumOperands() == 3) && (isa<ConstantPointerNull>(gep->getPointerOperand())))
+                {
+                    IRBuilder<> irBuilder(LastLoad.Inst);
+                    Value* AddInst = irBuilder.CreateAdd(gep->getOperand(2), irBuilder.getInt32(1));
+                    Value* gepArg[] = { gep->getOperand(1), AddInst };
+                    Value* Addr = irBuilder.CreateInBoundsGEP(gep->getSourceElementType(),
+                        gep->getOperand(0), gepArg);
+                    Instruction* dummyLoad = static_cast<Instruction*>
+                        (irBuilder.CreateLoad(IGCLLVM::getNonOpaquePtrEltTy(Addr->getType()), Addr));
+                    Loads.push_back(LdStInfo(dummyLoad, LastLoad.ByteOffset + newLoadSize));
+                }
+            }
+        }
+    }
+    return;
+}
+
 // A member of layout struct can be a vector type. This function will decide
 // if the vector type or a sequence of its elements' types shall be used as
 // the layout struct's member types. If spliting a vector type into a sequence

IGC/common/MDFrameWork.h

@@ -446,6 +446,8 @@ namespace IGC
         bool DisableConstantCoalescing                  = false;
         bool EnableUndefAlphaOutputAsRed                = true;
         bool WaEnableALTModeVisaWA                      = false;
+        bool EnableLdStCombineforLoad                   = false;
+        bool EnableLdStCombinewithDummyLoad             = false;
         bool NewSpillCostFunction                       = false;
         bool ForceLargeGRFNum4RQ                        = false;
         bool DisableEUFusion                            = false;

IGC/common/igc_flags.h

@@ -481,6 +481,7 @@ DECLARE_IGC_REGKEY(DWORD, MemOptGEPCanon,               2,     "[test] GEP canon
 DECLARE_IGC_REGKEY(bool, DisableMemOpt2,                false, "Disable MemOpt2", false)
 DECLARE_IGC_REGKEY(bool, EnableExplicitCopyForByVal,    true, "Enable generating an explicit copy (alloca + memcpy) in a caller for aggregate argumentes with byval attribute", true)
 DECLARE_IGC_REGKEY(DWORD, EnableLdStCombine,            1,     "Enable load/store combine pass if set to 1 (lsc message only) or 2; bit 3 = 1 [tmp for testing] : enabled load combine (intend to replace memopt)", true)
+DECLARE_IGC_REGKEY(bool, EnableLdStCombinewithDummyLoad, false, "Adds extra load instruction to increase the size of coalesced load", true)
 DECLARE_IGC_REGKEY(DWORD, MaxStoreVectorSizeInBytes,    0,     "[LdStCombine] the max non-uniform vector size for the coalesced store. 0: compiler choice (default, 16(4DW)); others: 4/8/16/32", true)
 DECLARE_IGC_REGKEY(DWORD, MaxLoadVectorSizeInBytes,     0,     "[LdStCombine] the max non-uniform vector size for the coalesced load.  0: compiler choice (default, 16(4DW)); others: 4/8/16/32", true)
 DECLARE_IGC_REGKEY(bool,  DisableMergeStore,            false, "[temp]If EnableLdStCombine is on, disable mergestore (memopt) if this is set. Temp key for testing", true)