|
1 | 1 | /*========================== begin_copyright_notice ============================ |
2 | 2 |
|
3 | | -Copyright (C) 2021-2023 Intel Corporation |
| 3 | +Copyright (C) 2021-2025 Intel Corporation |
4 | 4 |
|
5 | 5 | SPDX-License-Identifier: MIT |
6 | 6 |
|
@@ -55,17 +55,16 @@ struct ExplanationEntry |
55 | 55 | enum InstructionMask : uint32_t |
56 | 56 | { |
57 | 57 | None = 0x0, |
58 | | - AtomicOperation = (1 << 0), |
59 | | - TypedReadOperation = (1 << 1), |
60 | | - TypedWriteOperation = (1 << 2), |
61 | | - OutputUrbReadOperation = (1 << 3), |
62 | | - UrbWriteOperation = (1 << 4), |
63 | | - BufferReadOperation = (1 << 5), |
64 | | - BufferWriteOperation = (1 << 6), |
65 | | - SharedMemoryReadOperation = (1 << 7), |
66 | | - SharedMemoryWriteOperation = (1 << 8), |
67 | | - EndOfThreadOperation = (1 << 9), |
68 | | - |
| 58 | + AtomicOperation = (1 << 0), |
| 59 | + TypedReadOperation = (1 << 1), |
| 60 | + TypedWriteOperation = (1 << 2), |
| 61 | + OutputUrbReadOperation = (1 << 3), |
| 62 | + UrbWriteOperation = (1 << 4), |
| 63 | + BufferReadOperation = (1 << 5), |
| 64 | + BufferWriteOperation = (1 << 6), |
| 65 | + SharedMemoryReadOperation = (1 << 7), |
| 66 | + SharedMemoryWriteOperation = (1 << 8), |
| 67 | + EndOfThreadOperation = (1 << 9), |
69 | 68 | LastMaskPlusOne, |
70 | 69 | }; |
71 | 70 |
|
@@ -284,7 +283,8 @@ class SynchronizationObjectCoalescing : public llvm::FunctionPass |
284 | 283 | ReadSyncAtomic = 0x20, |
285 | 284 | WriteSyncRead = 0x40, |
286 | 285 | AtomicSyncAtomic = 0x80, |
287 | | - WriteSyncRet = 0x100 |
| 286 | + WriteSyncRet = 0x100, |
| 287 | + ExtendedCacheControlSyncRet = 0x200 |
288 | 288 | }; |
289 | 289 |
|
290 | 290 | private: |
@@ -401,19 +401,7 @@ class SynchronizationObjectCoalescing : public llvm::FunctionPass |
401 | 401 | //////////////////////////////////////////////////////////////////////// |
402 | 402 | SynchronizationCaseMask GetSynchronizationMaskForAllResources( |
403 | 403 | InstructionMask localForwardMemoryInstructionMask, |
404 | | - InstructionMask localBackwardMemoryInstructionMask) const;; |
405 | | - |
406 | | - //////////////////////////////////////////////////////////////////////// |
407 | | - static bool IsSyncInstruction(const llvm::Instruction* pInst); |
408 | | - |
409 | | - //////////////////////////////////////////////////////////////////////// |
410 | | - static bool IsMemoryInstruction(const llvm::Instruction* pInst); |
411 | | - |
412 | | - //////////////////////////////////////////////////////////////////////// |
413 | | - static bool IsReadMemoryInstruction(const llvm::Instruction* pInst); |
414 | | - |
415 | | - //////////////////////////////////////////////////////////////////////// |
416 | | - static bool IsWriteMemoryInstruction(const llvm::Instruction* pInst); |
| 404 | + InstructionMask localBackwardMemoryInstructionMask) const; |
417 | 405 |
|
418 | 406 | //////////////////////////////////////////////////////////////////////// |
419 | 407 | static bool IsAtomicOperation(const llvm::Instruction* pInst); |
@@ -983,7 +971,8 @@ InstructionMask SynchronizationObjectCoalescing::GetDefaultWriteMemoryInstructio |
983 | 971 | { |
984 | 972 | case LSC_UGM: // .ugm |
985 | 973 | case LSC_UGML: // .ugml |
986 | | - result |= BufferWriteOperation; |
| 974 | + result |= |
| 975 | + BufferWriteOperation; |
987 | 976 | if (!m_HasIndependentSharedMemoryFenceFunctionality) |
988 | 977 | { |
989 | 978 | result |= SharedMemoryWriteOperation; |
@@ -1088,7 +1077,10 @@ InstructionMask SynchronizationObjectCoalescing::GetDefaultMemoryInstructionMask |
1088 | 1077 | { |
1089 | 1078 | case LSC_UGM: // .ugm |
1090 | 1079 | case LSC_UGML: // .ugml |
1091 | | - result |= AtomicOperation | BufferWriteOperation | BufferReadOperation; |
| 1080 | + result |= |
| 1081 | + AtomicOperation | |
| 1082 | + BufferWriteOperation | |
| 1083 | + BufferReadOperation; |
1092 | 1084 | if (!m_HasIndependentSharedMemoryFenceFunctionality) |
1093 | 1085 | { |
1094 | 1086 | result |= SharedMemoryWriteOperation | SharedMemoryReadOperation; |
@@ -1134,7 +1126,11 @@ InstructionMask SynchronizationObjectCoalescing::GetDefaultMemoryInstructionMask |
1134 | 1126 | IGC_ASSERT(0); |
1135 | 1127 | } |
1136 | 1128 |
|
1137 | | - if (static_cast<uint32_t>(result & (SharedMemoryWriteOperation | BufferWriteOperation | TypedWriteOperation)) != 0) |
| 1129 | + constexpr InstructionMask maskToIncludeEOT = |
| 1130 | + SharedMemoryWriteOperation | |
| 1131 | + BufferWriteOperation | |
| 1132 | + TypedWriteOperation; |
| 1133 | + if (static_cast<uint32_t>(result & maskToIncludeEOT) != 0) |
1138 | 1134 | { |
1139 | 1135 |
|
1140 | 1136 | result = static_cast<InstructionMask>( |
@@ -1911,8 +1907,8 @@ SynchronizationObjectCoalescing::SynchronizationCaseMask SynchronizationObjectCo |
1911 | 1907 | } |
1912 | 1908 |
|
1913 | 1909 | // write -> fence -> ret |
1914 | | - bool requiresFlush = static_cast<uint32_t>(writeBit & (SharedMemoryWriteOperation | BufferWriteOperation | TypedWriteOperation)) != 0; |
1915 | | - bool isWriteSyncRetCase = requiresFlush && ((localBackwardMemoryInstructionMask & writeBit) != 0 && |
| 1910 | + bool requiresFlushWrites = static_cast<uint32_t>(writeBit & (SharedMemoryWriteOperation | BufferWriteOperation | TypedWriteOperation)) != 0; |
| 1911 | + bool isWriteSyncRetCase = requiresFlushWrites && ((localBackwardMemoryInstructionMask & writeBit) != 0 && |
1916 | 1912 | (localForwardMemoryInstructionMask & EndOfThreadOperation) != 0); |
1917 | 1913 | if (isWriteSyncRetCase) |
1918 | 1914 | { |
@@ -2380,43 +2376,6 @@ void SynchronizationObjectCoalescing::InvalidateMembers() |
2380 | 2376 | m_GlobalMemoryInstructionMask = InstructionMask::None; |
2381 | 2377 | } |
2382 | 2378 |
|
2383 | | -//////////////////////////////////////////////////////////////////////// |
2384 | | -bool SynchronizationObjectCoalescing::IsSyncInstruction(const llvm::Instruction* pInst) |
2385 | | -{ |
2386 | | - return IsThreadBarrierOperation(pInst) || |
2387 | | - IsTypedMemoryFenceOperation(pInst) || |
2388 | | - IsUrbFenceOperation(pInst) || |
2389 | | - IsLscFenceOperation(pInst) || |
2390 | | - IsUntypedMemoryFenceOperation(pInst); |
2391 | | -} |
2392 | | - |
2393 | | -//////////////////////////////////////////////////////////////////////// |
2394 | | -bool SynchronizationObjectCoalescing::IsMemoryInstruction(const llvm::Instruction* pInst) |
2395 | | -{ |
2396 | | - return IsReadMemoryInstruction(pInst) || |
2397 | | - IsWriteMemoryInstruction(pInst); |
2398 | | -} |
2399 | | - |
2400 | | -//////////////////////////////////////////////////////////////////////// |
2401 | | -bool SynchronizationObjectCoalescing::IsReadMemoryInstruction(const llvm::Instruction* pInst) |
2402 | | -{ |
2403 | | - return IsAtomicOperation(pInst) || |
2404 | | - IsBufferReadOperation(pInst) || |
2405 | | - IsSharedMemoryReadOperation(pInst) || |
2406 | | - IsTypedReadOperation(pInst) || |
2407 | | - IsOutputUrbReadOperation(pInst); |
2408 | | -} |
2409 | | - |
2410 | | -//////////////////////////////////////////////////////////////////////// |
2411 | | -bool SynchronizationObjectCoalescing::IsWriteMemoryInstruction(const llvm::Instruction* pInst) |
2412 | | -{ |
2413 | | - return IsAtomicOperation(pInst) || |
2414 | | - IsBufferWriteOperation(pInst) || |
2415 | | - IsSharedMemoryWriteOperation(pInst) || |
2416 | | - IsTypedWriteOperation(pInst) || |
2417 | | - IsUrbWriteOperation(pInst); |
2418 | | -} |
2419 | | - |
2420 | 2379 | //////////////////////////////////////////////////////////////////////// |
2421 | 2380 | bool SynchronizationObjectCoalescing::IsAtomicOperation(const llvm::Instruction* pInst) |
2422 | 2381 | { |
|
0 commit comments