diff --git a/src/coreclr/jit/llvm.h b/src/coreclr/jit/llvm.h index 34b62ab29b1..df1f18f9d9a 100644 --- a/src/coreclr/jit/llvm.h +++ b/src/coreclr/jit/llvm.h @@ -626,9 +626,11 @@ class Llvm void emitAlignmentCheckForAddress(GenTree* addr, Value* addrValue, unsigned alignment DEBUGARG(GenTree* indir)); bool isAddressAligned(GenTree* addr, unsigned alignment); - Value* consumeInitVal(GenTree* initVal); + Value* consumeInitVal(GenTree* initVal, uint8_t* pValue); + void consumeInitValAndEmitInitBlk(GenTree* initVal, Value* addrValue, ClassLayout* layout); void storeObjAtAddress(Value* baseAddress, Value* data, StructDesc* structDesc); unsigned buildMemCpy(Value* baseAddress, unsigned startOffset, unsigned endOffset, Value* srcAddress); + void emitMemSet(Value* addr, uint8_t value, unsigned size); void emitJumpToThrowHelper(Value* jumpCondValue, CorInfoHelpFunc helperFunc DEBUGARG(GenTree* nodeThrowing)); Value* emitCheckedArithmeticOperation( @@ -666,6 +668,7 @@ class Llvm Value* gepOrAddr(Value* addr, unsigned offset); Value* gepOrAddrInBounds(Value* addr, unsigned offset); + Value* emitAddLoadStoreOffset(Value* addr, unsigned offset); llvm::Constant* getIntPtrConst(target_size_t value, Type* llvmType = nullptr); Value* getShadowStack(); Value* getShadowStackForCallee(bool isTailCall = false); diff --git a/src/coreclr/jit/llvmcodegen.cpp b/src/coreclr/jit/llvmcodegen.cpp index e809802602f..b4a6df00ba3 100644 --- a/src/coreclr/jit/llvmcodegen.cpp +++ b/src/coreclr/jit/llvmcodegen.cpp @@ -1329,9 +1329,7 @@ void Llvm::buildStoreLocalField(GenTreeLclFld* lclFld) if (lclFld->TypeIs(TYP_STRUCT) && genActualTypeIsInt(data)) { - Value* fillValue = consumeInitVal(data); - Value* sizeValue = _builder.getInt32(layout->GetSize()); - _builder.CreateMemSet(addrValue, fillValue, sizeValue, llvm::MaybeAlign()); + consumeInitValAndEmitInitBlk(data, addrValue, layout); } else { @@ -1931,8 +1929,7 @@ void Llvm::buildStoreBlk(GenTreeBlk* blockOp) // Check for the "initblk" operation ("dataNode" is either INIT_VAL or constant zero). if (blockOp->OperIsInitBlkOp()) { - Value* fillValue = consumeInitVal(dataNode); - _builder.CreateMemSet(addrValue, fillValue, _builder.getInt32(layout->GetSize()), llvm::Align()); + consumeInitValAndEmitInitBlk(dataNode, addrValue, layout); return; } @@ -2467,19 +2464,33 @@ bool Llvm::isAddressAligned(GenTree* addr, unsigned alignment) return alignment == 1; // Any address is aligned to one byte. } -Value* Llvm::consumeInitVal(GenTree* initVal) +Value* Llvm::consumeInitVal(GenTree* initVal, uint8_t* pValue) { assert(initVal->isContained()); if (initVal->IsIntegralConst()) { assert(initVal->IsIntegralConst(0)); - return _builder.getInt8(0); + *pValue = 0; + return nullptr; } assert(initVal->OperIsInitVal()); return consumeValue(initVal->gtGetOp1(), Type::getInt8Ty(m_context->Context)); } +void Llvm::consumeInitValAndEmitInitBlk(GenTree* initVal, Value* addrValue, ClassLayout* layout) +{ + uint8_t constInitValue; + Value* initValue = consumeInitVal(initVal, &constInitValue); + if (initValue != nullptr) + { + _builder.CreateMemSet(addrValue, initValue, layout->GetSize(), llvm::MaybeAlign()); + return; + } + + emitMemSet(addrValue, constInitValue, layout->GetSize()); +} + void Llvm::storeObjAtAddress(Value* baseAddress, Value* data, StructDesc* structDesc) { size_t fieldCount = structDesc->getFieldCount(); @@ -2553,6 +2564,66 @@ unsigned Llvm::buildMemCpy(Value* baseAddress, unsigned startOffset, unsigned en return size; } +void Llvm::emitMemSet(Value* addr, uint8_t value, unsigned size) +{ + static const unsigned LLVM_MAX_UNROLL_SIZE = 64; + + llvm::Align align(1); + if (size > LLVM_MAX_UNROLL_SIZE) + { + _builder.CreateMemSet(addr, _builder.getInt8(value), size, align); + return; + } + + // TODO-LLVM: remove this manual unrolling once https://github.com/llvm/llvm-project/issues/79692 is fixed. + unsigned offset = 0; + Value* int64Value = nullptr; + for (; size - offset >= 8; offset += 8) + { + if (int64Value == nullptr) + { + int64Value = _builder.getInt64(0x0101010101010101ULL * value); + } + Value* addrAtOffset = emitAddLoadStoreOffset(addr, offset); + _builder.CreateAlignedStore(int64Value, addrAtOffset, llvm::commonAlignment(align, offset)); + } + + Value* int32Value = nullptr; + for (; size - offset >= 4; offset += 4) + { + if (int32Value == nullptr) + { + int32Value = _builder.getInt32(0x01010101u * value); + } + Value* addrAtOffset = emitAddLoadStoreOffset(addr, offset); + _builder.CreateAlignedStore(int32Value, addrAtOffset, llvm::commonAlignment(align, offset)); + } + + Value* int16Value = nullptr; + for (; size - offset >= 2; offset += 2) + { + if (int16Value == nullptr) + { + int16Value = _builder.getInt16(0x0101 * value); + } + Value* addrAtOffset = emitAddLoadStoreOffset(addr, offset); + _builder.CreateAlignedStore(int16Value, addrAtOffset, llvm::commonAlignment(align, offset)); + } + + Value* int8Value = nullptr; + for (; size - offset >= 1; offset += 1) + { + if (int8Value == nullptr) + { + int8Value = _builder.getInt8(value); + } + Value* addrAtOffset = emitAddLoadStoreOffset(addr, offset); + _builder.CreateAlignedStore(int8Value, addrAtOffset, llvm::commonAlignment(align, offset)); + } + + assert(offset == size); +} + void Llvm::emitJumpToThrowHelper(Value* jumpCondValue, CorInfoHelpFunc helperFunc DEBUGARG(GenTree* nodeThrowing)) { bool shadowTailCalledThrowHelper = false; @@ -3251,6 +3322,21 @@ Value* Llvm::gepOrAddrInBounds(Value* addr, unsigned offset) return _builder.CreateInBoundsGEP(Type::getInt8Ty(m_context->Context), addr, _builder.getInt32(offset)); } +Value* Llvm::emitAddLoadStoreOffset(Value* addr, unsigned offset) +{ + // TODO-LLVM: replace this with getelementptr 'nusw' once we move to LLVM 20+. + assert(addr->getType()->isPointerTy()); + if (offset == 0) + { + return addr; + } + + addr = _builder.CreatePtrToInt(addr, getIntPtrLlvmType()); + addr = _builder.CreateNUWAdd(addr, getIntPtrConst(offset)); + addr = _builder.CreateIntToPtr(addr, getPtrLlvmType()); + return addr; +} + Value* Llvm::getShadowStack() { if (getCurrentLlvmFunctionIndex() == ROOT_FUNC_IDX) diff --git a/src/tests/nativeaot/SmokeTests/HelloWasm/wasmjit-diff.ps1 b/src/tests/nativeaot/SmokeTests/HelloWasm/wasmjit-diff.ps1 index d44af4e1f03..b00f9ec9c65 100644 --- a/src/tests/nativeaot/SmokeTests/HelloWasm/wasmjit-diff.ps1 +++ b/src/tests/nativeaot/SmokeTests/HelloWasm/wasmjit-diff.ps1 @@ -140,7 +140,7 @@ if ($Analyze -or $Summary) if ($Llvm) { - $LlvmSummaryLineRegex = [Regex]::New('define.*@"?([^"]*)"?\(.*\).*{', "Compiled") + $LlvmSummaryLineRegex = [Regex]::New('define.*@"?([^"]*?)"?\(.*\).*{', "Compiled") function ParseLlvmSummary($ObjDirectory, $SummaryName) { Write-Host -NoNewLine "Analysing ${SummaryName}" @@ -148,6 +148,7 @@ if ($Analyze -or $Summary) # Use the results file to be resilient against stale bitcode files. $SummaryList = [Collections.Generic.Dictionary[string, object]]::new() $BitcodeFiles = Get-Content "$ObjDirectory/$TestProjectName.results.txt" + $BitcodeFiles = $BitcodeFiles | Where-Object { $_.EndsWith(".bc") } # The results file in the 'base' directory will refer to the original ('diff') files. Fix this up. for ($i = 0; $i -lt $BitcodeFiles.Length; $i++)