[MERGE #5861 @nhat-nguyen] Lazy bailout

Merge pull request #5861 from nhat-nguyen:lazybailout

## Overview
+ Eager fixed field checks are no longer generated thanks to lazy bailout
    + Lazy bailout should work for both user JavaScript calls and runtime helper calls
        + Operations that have implicit calls (like getter/setter) are also guarded by lazy bailout
    + Lazy bailout points are inserted in the forward pass and removed if not needed during backward pass
    + For helper calls, we copy the `BailOutInfo` from the bailout path to the call to helper so that the register allocator can fill it correctly
    + Jit'd frames on the stack that depend on fixed fields are invalidated by changing their return addresses to a common lazy bailout thunk per function. The thunk is responsible for cleaning up the stack, loading the right `BailOutRecord` for each bailout point, and finally calls to the bailout code. Other jit'd functions are invalidated normally

## TODO
+ Support x86/ARM/ARM64 as lazy bailout only works on x64 for now
+ `NewScObjArray` helper path needs rework
+ Instead of having `BailOnNotNativeArray` directly on `NewScObjArray`, insert it on a bailout instruction before `NewScObjArray`
+ Support Out-of-proc jit
+ Lower `BailOnImplicitCallPreOp` when we need to attach lazy bailout to instructions that already have preop bailouts
+ Investigate instructions that have helper path but don't go through `ChangeToHelperCall`
+ Stack clean up for x86 in lazy bailout thunk
+ Support `SaveAllRegisterAndBranchBailOut` variant in the thunk
+ `pdf.js` bug

Author: nhat-nguyen

lib/Backend/BackwardPass.cpp

@@ -43,6 +43,7 @@ class BackwardPass
     bool ProcessBailOutInfo(IR::Instr * instr);
     void ProcessBailOutInfo(IR::Instr * instr, BailOutInfo * bailOutInfo);
     IR::Instr* ProcessPendingPreOpBailOutInfo(IR::Instr *const currentInstr);
+    void ClearDstUseForPostOpLazyBailOut(IR::Instr *instr);
     void ProcessBailOutArgObj(BailOutInfo * bailOutInfo, BVSparse<JitArenaAllocator> * byteCodeUpwardExposedUsed);
     void ProcessBailOutConstants(BailOutInfo * bailOutInfo, BVSparse<JitArenaAllocator> * byteCodeUpwardExposedUsed, BVSparse<JitArenaAllocator>* argSymsBv);
     void ProcessBailOutCopyProps(BailOutInfo * bailOutInfo, BVSparse<JitArenaAllocator> * byteCodeUpwardExposedUsed, BVSparse<JitArenaAllocator>* argSymsBv);
@@ -73,7 +74,7 @@ class BackwardPass
     void DumpMarkTemp();
 #endif
 
-    static bool UpdateImplicitCallBailOutKind(IR::Instr *const instr, bool needsBailOutOnImplicitCall);
+    static bool UpdateImplicitCallBailOutKind(IR::Instr *const instr, bool needsBailOutOnImplicitCall, bool needsLazyBailOut);
 
     bool ProcessNoImplicitCallUses(IR::Instr *const instr);
     void ProcessNoImplicitCallDef(IR::Instr *const instr);
@@ -102,9 +103,11 @@ class BackwardPass
 
     void TrackFloatSymEquivalence(IR::Instr *const instr);
 
-    void DeadStoreImplicitCallBailOut(IR::Instr * instr, bool hasLiveFields);
+    bool IsLazyBailOutCurrentlyNeeeded(IR::Instr * instr) const;
+    void DeadStoreImplicitCallBailOut(IR::Instr * instr, bool hasLiveFields, bool needsLazyBailOut);
     void DeadStoreTypeCheckBailOut(IR::Instr * instr);
-    bool IsImplicitCallBailOutCurrentlyNeeded(IR::Instr * instr, bool mayNeedImplicitCallBailOut, bool hasLiveFields);
+    void DeadStoreLazyBailOut(IR::Instr * instr, bool needsLazyBailOut);
+    bool IsImplicitCallBailOutCurrentlyNeeded(IR::Instr * instr, bool mayNeedImplicitCallBailOut, bool needLazyBailOut, bool hasLiveFields);
     bool NeedBailOutOnImplicitCallsForTypedArrayStore(IR::Instr* instr);
     bool TrackNoImplicitCallInlinees(IR::Instr *instr);
     bool ProcessBailOnNoProfile(IR::Instr *instr, BasicBlock *block);

lib/Backend/BackwardPass.h

@@ -13,11 +13,85 @@
 
 extern const IRType RegTypes[RegNumCount];
 
+// In `FillBailOutRecord`, some of the fields of BailOutInfo are modified directly,
+// so simply doing a shallow copy of pointers when duplicating the BailOutInfo to
+// the helper calls for lazy bailouts will mess things up.Make a deep copies of such fields.
+void BailOutInfo::PartialDeepCopyTo(BailOutInfo * const other) const
+{
+    // Primitive types
+    other->wasCloned = this->wasCloned;
+    other->isInvertedBranch = this->isInvertedBranch;
+    other->sharedBailOutKind = this->sharedBailOutKind;
+    other->isLoopTopBailOutInfo = this->isLoopTopBailOutInfo;
+    other->bailOutOffset = this->bailOutOffset;
+    other->polymorphicCacheIndex = this->polymorphicCacheIndex;
+    other->startCallCount = this->startCallCount;
+    other->totalOutParamCount = this->totalOutParamCount;
+    other->stackLiteralBailOutInfoCount = this->stackLiteralBailOutInfoCount;
+#if DBG
+    other->wasCopied = this->wasCopied;
+#endif
+
+    other->bailOutRecord = this->bailOutRecord;
+
+    this->capturedValues->CopyTo(this->bailOutFunc->m_alloc, other->capturedValues);
+    this->usedCapturedValues->CopyTo(this->bailOutFunc->m_alloc, other->usedCapturedValues);
+
+    if (this->byteCodeUpwardExposedUsed != nullptr)
+    {
+        other->byteCodeUpwardExposedUsed = this->byteCodeUpwardExposedUsed->CopyNew(this->bailOutFunc->m_alloc);
+    }
+
+    if (this->liveVarSyms != nullptr)
+    {
+        other->liveVarSyms = this->liveVarSyms->CopyNew(this->bailOutFunc->m_alloc);
+    }
+
+    if (this->liveLosslessInt32Syms != nullptr)
+    {
+        other->liveLosslessInt32Syms = this->liveLosslessInt32Syms->CopyNew(this->bailOutFunc->m_alloc);
+    }
+
+    if (this->liveFloat64Syms != nullptr)
+    {
+        other->liveFloat64Syms = this->liveFloat64Syms->CopyNew(this->bailOutFunc->m_alloc);
+    }
+
+    if (this->outParamInlinedArgSlot != nullptr)
+    {
+        other->outParamInlinedArgSlot = this->outParamInlinedArgSlot->CopyNew(this->bailOutFunc->m_alloc);
+    }
+
+    other->startCallFunc = this->startCallFunc;
+    other->argOutSyms = this->argOutSyms;
+    other->startCallInfo = this->startCallInfo;
+    other->stackLiteralBailOutInfo = this->stackLiteralBailOutInfo;
+    other->outParamOffsets = this->outParamOffsets;
+
+
+#ifdef _M_IX86
+    other->outParamFrameAdjustArgSlot = this->outParamFrameAdjustArgSlot;
+    other->inlinedStartCall = this->inlinedStartCall;
+#endif
+
+    other->bailOutInstr = this->bailOutInstr;
+
+#if ENABLE_DEBUG_CONFIG_OPTIONS
+    other->bailOutOpcode = this->bailOutOpcode;
+#endif
+
+    other->bailOutFunc = this->bailOutFunc;
+    other->branchConditionOpnd = this->branchConditionOpnd;
+}
+
 void
 BailOutInfo::Clear(JitArenaAllocator * allocator)
 {
-    // Currently, we don't have a case where we delete bailout info after we allocated the bailout record
-    Assert(!bailOutRecord);
+    // Previously, we don't have a case where we delete bailout info after we allocated the bailout record.
+    // However, since lazy bailouts can now be attached on helper call instructions, and those instructions
+    // might sometimes be removed in Peeps, we will hit those cases. Make sure that in such cases, lazy bailout
+    // is the only bailout reason we have.
+    Assert(bailOutRecord == nullptr || BailOutInfo::OnlyHasLazyBailOut(bailOutRecord->bailOutKind));
     if (this->capturedValues && this->capturedValues->DecrementRefCount() == 0)
     {
         this->capturedValues->constantValues.Clear(allocator);
@@ -69,6 +143,37 @@ BailOutInfo::Clear(JitArenaAllocator * allocator)
 #endif
 }
 
+// Refer to comments in the header file
+void BailOutInfo::ClearUseOfDst(SymID id)
+{
+    Assert(id != SymID_Invalid);
+    if (this->byteCodeUpwardExposedUsed != nullptr &&
+        this->byteCodeUpwardExposedUsed->Test(id))
+    {
+        this->clearedDstByteCodeUpwardExposedUseId = id;
+        this->byteCodeUpwardExposedUsed->Clear(id);
+    }
+}
+
+void BailOutInfo::RestoreUseOfDst()
+{
+    if (this->byteCodeUpwardExposedUsed != nullptr &&
+        this->NeedsToRestoreUseOfDst())
+    {
+        this->byteCodeUpwardExposedUsed->Set(this->clearedDstByteCodeUpwardExposedUseId);
+    }
+}
+
+bool BailOutInfo::NeedsToRestoreUseOfDst() const
+{
+    return this->clearedDstByteCodeUpwardExposedUseId != SymID_Invalid;
+}
+
+SymID BailOutInfo::GetClearedUseOfDstId() const
+{
+    return this->clearedDstByteCodeUpwardExposedUseId;
+}
+
 #ifdef _M_IX86
 
 uint
@@ -2852,16 +2957,11 @@ SharedBailOutRecord::SharedBailOutRecord(uint32 bailOutOffset, uint bailOutCache
     this->type = BailoutRecordType::Shared;
 }
 
-void LazyBailOutRecord::SetBailOutKind()
-{
-    this->bailoutRecord->SetBailOutKind(IR::BailOutKind::LazyBailOut);
-}
-
 #if DBG
-void LazyBailOutRecord::Dump(Js::FunctionBody* functionBody)
+void LazyBailOutRecord::Dump(Js::FunctionBody* functionBody) const
 {
     OUTPUT_PRINT(functionBody);
-    Output::Print(_u("Bytecode Offset: #%04x opcode: %s"), this->bailoutRecord->GetBailOutOffset(), Js::OpCodeUtil::GetOpCodeName(this->bailoutRecord->GetBailOutOpCode()));
+    Output::Print(_u("Bytecode Offset: #%04x opcode: %s"), this->bailOutRecord->GetBailOutOffset(), Js::OpCodeUtil::GetOpCodeName(this->bailOutRecord->GetBailOutOpCode()));
 }
 #endif
 

lib/Backend/BailOut.cpp

@@ -30,7 +30,8 @@ class BailOutInfo
         totalOutParamCount(0), argOutSyms(nullptr), bailOutRecord(nullptr), wasCloned(false), isInvertedBranch(false), sharedBailOutKind(true), isLoopTopBailOutInfo(false),
         outParamInlinedArgSlot(nullptr), liveVarSyms(nullptr), liveLosslessInt32Syms(nullptr), liveFloat64Syms(nullptr),
         branchConditionOpnd(nullptr),
-        stackLiteralBailOutInfoCount(0), stackLiteralBailOutInfo(nullptr)
+        stackLiteralBailOutInfoCount(0), stackLiteralBailOutInfo(nullptr),
+        clearedDstByteCodeUpwardExposedUseId(SymID_Invalid)
     {
         Assert(bailOutOffset != Js::Constants::NoByteCodeOffset);
 #ifdef _M_IX86
@@ -45,8 +46,23 @@ class BailOutInfo
         this->usedCapturedValues = JitAnew(bailOutFunc->m_alloc, CapturedValues);
         this->usedCapturedValues->argObjSyms = nullptr;
     }
+
+    void PartialDeepCopyTo(BailOutInfo *const bailOutInfo) const;
     void Clear(JitArenaAllocator * allocator);
 
+    // Lazy bailout
+    // 
+    // Workaround for dealing with use of destination register of `call` instructions with postop lazy bailout.
+    // As an example, in globopt, we have s1 = Call and s1 is in byteCodeUpwardExposedUse,
+    // but after lowering, the instructions are: s3 = Call, s1 = s3.
+    // If we add a postop lazy bailout to s3 = call, we will create a use of s1 right at that instructions.
+    // However, s1 at that point is not initialized yet.
+    // As a workaround, we will clear the use of s1 and restore it if we determine that lazy bailout is not needed.
+    void ClearUseOfDst(SymID id);
+    void RestoreUseOfDst();
+    bool NeedsToRestoreUseOfDst() const;
+    SymID GetClearedUseOfDstId() const;
+
     void FinalizeBailOutRecord(Func * func);
 #ifdef MD_GROW_LOCALS_AREA_UP
     void FinalizeOffsets(__in_ecount(count) int * offsets, uint count, Func *func, BVSparse<JitArenaAllocator> *bvInlinedArgSlot);
@@ -66,6 +82,26 @@ class BailOutInfo
             kindMinusBits == IR::BailOutOnImplicitCallsPreOp;
     }
 
+    static bool OnlyHasLazyBailOut(IR::BailOutKind kind)
+    {
+        return kind == IR::LazyBailOut;
+    }
+
+    static bool HasLazyBailOut(IR::BailOutKind kind)
+    {
+        return (kind & IR::LazyBailOut) != 0;
+    }
+
+    static IR::BailOutKind WithoutLazyBailOut(IR::BailOutKind kind)
+    {
+        return kind & ~IR::LazyBailOut;
+    }
+
+    static IR::BailOutKind WithLazyBailOut(IR::BailOutKind kind)
+    {
+        return kind | IR::LazyBailOut;
+    }
+
 #if DBG
     static bool IsBailOutHelper(IR::JnHelperMethod helper);
 #endif
@@ -77,6 +113,7 @@ class BailOutInfo
 #if DBG
     bool wasCopied;
 #endif
+    SymID clearedDstByteCodeUpwardExposedUseId;
     uint32 bailOutOffset;
     BailOutRecord * bailOutRecord;
     CapturedValues * capturedValues;                                      // Values we know about after forward pass

lib/Backend/BailOut.h

@@ -7,42 +7,41 @@
     #error BAIL_OUT_KIND, BAIL_OUT_KIND_VALUE, and BAIL_OUT_KIND_VALUE_LAST must be defined before including this file.
 #endif
                /* kind */                           /* allowed bits */
-BAIL_OUT_KIND(BailOutInvalid,                       IR::BailOutOnResultConditions | IR::BailOutForArrayBits | IR::BailOutForDebuggerBits | IR::BailOutMarkTempObject)
+BAIL_OUT_KIND(BailOutInvalid,                       IR::BailOutOnResultConditions | IR::BailOutForArrayBits | IR::BailOutForDebuggerBits | IR::BailOutMarkTempObject | IR::LazyBailOut)
 BAIL_OUT_KIND(BailOutIntOnly,                       IR::BailOutMarkTempObject)
 BAIL_OUT_KIND(BailOutNumberOnly,                    IR::BailOutMarkTempObject)
 BAIL_OUT_KIND(BailOutPrimitiveButString,            IR::BailOutMarkTempObject)
-BAIL_OUT_KIND(BailOutOnImplicitCalls,               IR::BailOutForArrayBits)
-BAIL_OUT_KIND(BailOutOnImplicitCallsPreOp,          (IR::BailOutOnResultConditions | IR::BailOutForArrayBits | IR::BailOutMarkTempObject) & ~IR::BailOutOnArrayAccessHelperCall )
+BAIL_OUT_KIND(BailOutOnImplicitCalls,               IR::BailOutForArrayBits | IR::LazyBailOut)
+BAIL_OUT_KIND(BailOutOnImplicitCallsPreOp,          (IR::BailOutOnResultConditions | IR::BailOutForArrayBits | IR::BailOutMarkTempObject | IR::LazyBailOut) & ~IR::BailOutOnArrayAccessHelperCall )
 BAIL_OUT_KIND(BailOutOnNotPrimitive,                IR::BailOutMarkTempObject)
 BAIL_OUT_KIND(BailOutOnMemOpError,                  IR::BailOutForArrayBits)
 BAIL_OUT_KIND(BailOutOnInlineFunction,              0)
 BAIL_OUT_KIND(BailOutOnNoProfile,                   0)
 BAIL_OUT_KIND(BailOutOnPolymorphicInlineFunction,   0)
 BAIL_OUT_KIND(BailOutOnFailedPolymorphicInlineTypeCheck,   0)
 BAIL_OUT_KIND(BailOutShared,                        0)
-BAIL_OUT_KIND(BailOutOnNotArray,                    IR::BailOutOnMissingValue)
-BAIL_OUT_KIND(BailOutOnNotNativeArray,              IR::BailOutOnMissingValue)
-BAIL_OUT_KIND(BailOutConventionalTypedArrayAccessOnly, IR::BailOutMarkTempObject)
-BAIL_OUT_KIND(BailOutOnIrregularLength,             IR::BailOutMarkTempObject)
+BAIL_OUT_KIND(BailOutOnNotArray,                    IR::BailOutOnMissingValue | IR::LazyBailOut)
+BAIL_OUT_KIND(BailOutOnNotNativeArray,              IR::BailOutOnMissingValue | IR::LazyBailOut)
+BAIL_OUT_KIND(BailOutConventionalTypedArrayAccessOnly, IR::BailOutMarkTempObject | IR::LazyBailOut)
+BAIL_OUT_KIND(BailOutOnIrregularLength,             IR::BailOutMarkTempObject | IR::LazyBailOut)
 BAIL_OUT_KIND(BailOutCheckThis,                     0)
 BAIL_OUT_KIND(BailOutOnTaggedValue,                 0)
-BAIL_OUT_KIND(BailOutFailedTypeCheck,               IR::BailOutMarkTempObject)
-BAIL_OUT_KIND(BailOutFailedEquivalentTypeCheck,     IR::BailOutMarkTempObject)
+BAIL_OUT_KIND(BailOutFailedTypeCheck,               IR::BailOutMarkTempObject | IR::LazyBailOut)
+BAIL_OUT_KIND(BailOutFailedEquivalentTypeCheck,     IR::BailOutMarkTempObject | IR::LazyBailOut)
 BAIL_OUT_KIND(BailOutInjected,                      0)
 BAIL_OUT_KIND(BailOutExpectingInteger,              0)
 BAIL_OUT_KIND(BailOutExpectingString,               0)
 BAIL_OUT_KIND(BailOutFailedInlineTypeCheck,         IR::BailOutMarkTempObject)
-BAIL_OUT_KIND(BailOutFailedFixedFieldTypeCheck,     IR::BailOutMarkTempObject)
-BAIL_OUT_KIND(BailOutFailedFixedFieldCheck,         0)
-BAIL_OUT_KIND(BailOutFailedEquivalentFixedFieldTypeCheck,     IR::BailOutMarkTempObject)
+BAIL_OUT_KIND(BailOutFailedFixedFieldTypeCheck,     IR::BailOutMarkTempObject | IR::LazyBailOut)
+BAIL_OUT_KIND(BailOutFailedFixedFieldCheck,         IR::LazyBailOut)
+BAIL_OUT_KIND(BailOutFailedEquivalentFixedFieldTypeCheck,     IR::BailOutMarkTempObject | IR::LazyBailOut)
 BAIL_OUT_KIND(BailOutOnFloor,                       0)
 BAIL_OUT_KIND(BailOnModByPowerOf2,                  0)
 BAIL_OUT_KIND(BailOnIntMin,                         0)
 BAIL_OUT_KIND(BailOnDivResultNotInt,                IR::BailOutOnDivByZero | IR::BailOutOnDivOfMinInt | IR::BailOutOnNegativeZero)
 BAIL_OUT_KIND(BailOnSimpleJitToFullJitLoopBody,     0)
 BAIL_OUT_KIND(BailOutFailedCtorGuardCheck,          0)
 BAIL_OUT_KIND(BailOutOnFailedHoistedBoundCheck,     0)
-BAIL_OUT_KIND(LazyBailOut,                          0)
 BAIL_OUT_KIND(BailOutOnFailedHoistedLoopCountBasedBoundCheck, 0)
 BAIL_OUT_KIND(BailOutForGeneratorYield,             0)
 BAIL_OUT_KIND(BailOutOnException,                   0)
@@ -110,9 +109,11 @@ BAIL_OUT_KIND_VALUE(BailOutOnDivSrcConditions, BailOutOnDivByZero | BailOutOnDiv
 
 #define BAIL_OUT_KIND_MISC_BIT_START BAIL_OUT_KIND_DIV_SRC_CONDITIONS_BIT_START + 2
 BAIL_OUT_KIND_VALUE(BailOutMarkTempObject, 1 << (BAIL_OUT_KIND_MISC_BIT_START + 0))
+// this is the most significant bit, must cast it to unsigned int so that the compiler knows we are not using a negative number
+BAIL_OUT_KIND_VALUE(LazyBailOut, (uint) 1 << (BAIL_OUT_KIND_MISC_BIT_START + 1))
+BAIL_OUT_KIND_VALUE(BailOutMisc, BailOutMarkTempObject | LazyBailOut)
 
-
-BAIL_OUT_KIND_VALUE_LAST(BailOutKindBits, BailOutMarkTempObject | BailOutOnDivSrcConditions | BailOutOnResultConditions | BailOutForArrayBits | BailOutForDebuggerBits)
+BAIL_OUT_KIND_VALUE_LAST(BailOutKindBits, BailOutMisc | BailOutOnDivSrcConditions | BailOutOnResultConditions | BailOutForArrayBits | BailOutForDebuggerBits)
 
 // Help caller undefine the macros
 #undef BAIL_OUT_KIND