[MERGE #5854 @rajatd] Inline more .call/.apply targets

Merge pull request #5854 from rajatd:callApplyTarget

Gather profile data about the target of a .call or .apply at the call site instead of the load site of the target.

Today, we can only inline .call/.apply targets loaded via LdFld. This PR attempts to catch many more cases by also collecting profile data at the call site.

Author: rajatd

lib/Backend/FunctionCodeGenJitTimeData.cpp

@@ -16,6 +16,8 @@ namespace Js
 #endif
         next(nullptr),
         ldFldInlinees(nullptr),
+        callbackInlinees(nullptr),
+        callApplyTargetInlinees(nullptr),
         globalThisObject(globalThis),
         profiledIterations(profiledIterations),
         sharedPropertyGuards(nullptr),
@@ -110,6 +112,14 @@ namespace Js
         return callbackInlinees ? callbackInlinees[profiledCallSiteId] : nullptr;
     }
 
+    const FunctionCodeGenJitTimeData * FunctionCodeGenJitTimeData::GetCallApplyTargetInlinee(const ProfileId callApplyCallSiteId) const
+    {
+        Assert(GetFunctionBody());
+        Assert(callApplyCallSiteId < GetFunctionBody()->GetProfiledCallApplyCallSiteCount());
+
+        return callApplyTargetInlinees ? callApplyTargetInlinees[callApplyCallSiteId] : nullptr;
+    }
+
     FunctionCodeGenJitTimeData *FunctionCodeGenJitTimeData::AddInlinee(
         Recycler *const recycler,
         const ProfileId profiledCallSiteId,
@@ -197,6 +207,32 @@ namespace Js
         return inlineeData;
     }
 
+    FunctionCodeGenJitTimeData * FunctionCodeGenJitTimeData::AddCallApplyTargetInlinee(
+        Recycler *const recycler,
+        const ProfileId profiledCallSiteId,
+        const ProfileId callApplyCallSiteId,
+        FunctionInfo *const inlinee)
+    {
+        Assert(recycler != nullptr);
+        FunctionBody * functionBody = GetFunctionBody();
+        Assert(functionBody != nullptr);
+        Assert(profiledCallSiteId < functionBody->GetProfiledCallSiteCount());
+        Assert(callApplyCallSiteId < functionBody->GetProfiledCallApplyCallSiteCount());
+        Assert(inlinee != nullptr);
+
+        if (!callApplyTargetInlinees)
+        {
+            callApplyTargetInlinees = RecyclerNewArrayZ(recycler, Field(FunctionCodeGenJitTimeData *), functionBody->GetProfiledCallApplyCallSiteCount());
+        }
+
+        // Polymorphic call/apply targets are not inlined.
+        Assert(callApplyTargetInlinees[callApplyCallSiteId] == nullptr);
+
+        FunctionCodeGenJitTimeData * inlineeData = FunctionCodeGenJitTimeData::New(recycler, inlinee, nullptr /* entryPoint */, true /*isInlined*/);
+        callApplyTargetInlinees[callApplyCallSiteId] = inlineeData;
+        return inlineeData;
+    }
+
     uint FunctionCodeGenJitTimeData::InlineeCount() const
     {
         return inlineeCount;

lib/Backend/FunctionCodeGenJitTimeData.h

@@ -32,6 +32,8 @@ namespace Js
         Field(Field(FunctionCodeGenJitTimeData*)*) inlinees;
         Field(Field(FunctionCodeGenJitTimeData*)*) ldFldInlinees;
         Field(Field(FunctionCodeGenJitTimeData*)*) callbackInlinees;
+        Field(Field(FunctionCodeGenJitTimeData*)*) callApplyTargetInlinees;
+        
         Field(RecyclerWeakReference<FunctionBody>*) weakFuncRef;
 
         Field(PolymorphicInlineCacheInfoIDL*) inlineeInfo;
@@ -92,6 +94,7 @@ namespace Js
         const FunctionCodeGenJitTimeData *GetInlinee(const ProfileId profiledCallSiteId) const;
         const FunctionCodeGenJitTimeData *GetLdFldInlinee(const InlineCacheIndex inlineCacheIndex) const;
         const FunctionCodeGenJitTimeData * GetCallbackInlinee(const ProfileId profiledCallSiteId) const;
+        const FunctionCodeGenJitTimeData * GetCallApplyTargetInlinee(const ProfileId callApplyCallSiteId) const;
         FunctionCodeGenJitTimeData *AddInlinee(
             Recycler *const recycler,
             const ProfileId profiledCallSiteId,
@@ -125,6 +128,12 @@ namespace Js
             const ProfileId profiledCallSiteId,
             FunctionInfo *const inlinee);
 
+        FunctionCodeGenJitTimeData * AddCallApplyTargetInlinee(
+            Recycler *const recycler,
+            const ProfileId profiledCallSiteId,
+            const ProfileId callApplyCallSiteId,
+            FunctionInfo *const inlinee);
+
         bool IsPolymorphicCallSite(const ProfileId profiledCallSiteId) const;
         // This function walks all the chained jittimedata and returns the one which match the functionInfo.
         // This can return null, if the functionInfo doesn't match.

lib/Backend/FunctionJITTimeInfo.cpp

@@ -94,7 +94,6 @@ FunctionJITTimeInfo::BuildJITTimeData(
             }
         }
 
-        jitData->callbackInlineeCount = jitData->bodyData->profiledCallSiteCount;
         jitData->callbackInlinees = AnewArrayZ(alloc, FunctionJITTimeDataIDL*, jitData->bodyData->profiledCallSiteCount);
 
         for (Js::ProfileId i = 0; i < jitData->bodyData->profiledCallSiteCount; ++i)
@@ -111,6 +110,26 @@ FunctionJITTimeInfo::BuildJITTimeData(
                 BuildJITTimeData(alloc, inlineeJITData, inlineeRuntimeData, jitData->callbackInlinees[i], true, isForegroundJIT);
             }
         }
+
+        jitData->callApplyTargetInlineeCount = jitData->bodyData->profiledCallApplyCallSiteCount;
+        if (jitData->bodyData->profiledCallApplyCallSiteCount > 0)
+        {
+            jitData->callApplyTargetInlinees = AnewArrayZ(alloc, FunctionJITTimeDataIDL*, jitData->bodyData->profiledCallApplyCallSiteCount);
+        }
+        for (Js::ProfileId i = 0; i < jitData->bodyData->profiledCallApplyCallSiteCount; ++i)
+        {
+            const Js::FunctionCodeGenJitTimeData * inlineeJITData = codeGenData->GetCallApplyTargetInlinee(i);
+            if (inlineeJITData != nullptr)
+            {
+                const Js::FunctionCodeGenRuntimeData * inlineeRuntimeData = nullptr;
+                if (inlineeJITData->GetFunctionInfo()->HasBody())
+                {
+                    inlineeRuntimeData = isInlinee ? targetRuntimeData->GetCallApplyTargetInlinee(i) : functionBody->GetCallApplyTargetInlineeCodeGenRuntimeData(i);
+                }
+                jitData->callApplyTargetInlinees[i] = AnewStructZ(alloc, FunctionJITTimeDataIDL);
+                BuildJITTimeData(alloc, inlineeJITData, inlineeRuntimeData, jitData->callApplyTargetInlinees[i], true, isForegroundJIT);
+            }
+        }
     }
     jitData->profiledRuntimeData = AnewStructZ(alloc, FunctionJITRuntimeIDL);
     if (isInlinee && targetRuntimeData->ClonedInlineCaches()->HasInlineCaches())
@@ -293,6 +312,12 @@ FunctionJITTimeInfo::GetInlineeForCallbackInlineeRuntimeData(const Js::ProfileId
     return inlineeData->GetRuntimeInfo();
 }
 
+const FunctionJITRuntimeInfo *
+FunctionJITTimeInfo::GetCallApplyTargetInlineeRuntimeData(const Js::ProfileId callApplyCallSiteId) const
+{
+    return GetCallApplyTargetInlinee(callApplyCallSiteId) ? GetCallApplyTargetInlinee(callApplyCallSiteId)->GetRuntimeInfo() : nullptr;
+}
+
 const FunctionJITRuntimeInfo *
 FunctionJITTimeInfo::GetRuntimeInfo() const
 {
@@ -341,11 +366,23 @@ FunctionJITTimeInfo::GetCallbackInlinee(Js::ProfileId profileId) const
     {
         return nullptr;
     }
-    AssertOrFailFast(profileId < m_data.callbackInlineeCount);
+    AssertOrFailFast(profileId < m_data.inlineeCount);
 
     return reinterpret_cast<const FunctionJITTimeInfo *>(m_data.callbackInlinees[profileId]);
 }
 
+const FunctionJITTimeInfo *
+FunctionJITTimeInfo::GetCallApplyTargetInlinee(Js::ProfileId callApplyCallSiteId) const
+{
+    if (!m_data.callApplyTargetInlinees)
+    {
+        return nullptr;
+    }
+    AssertOrFailFast(callApplyCallSiteId < m_data.bodyData->profiledCallApplyCallSiteCount);
+
+    return reinterpret_cast<const FunctionJITTimeInfo *>(m_data.callApplyTargetInlinees[callApplyCallSiteId]);
+}
+
 const FunctionJITTimeInfo *
 FunctionJITTimeInfo::GetLdFldInlinee(Js::InlineCacheIndex inlineCacheIndex) const
 {

lib/Backend/FunctionJITTimeInfo.h

@@ -21,6 +21,8 @@ class FunctionJITTimeInfo
     bool IsLdFldInlineePresent() const;
 
     const FunctionJITTimeInfo * GetCallbackInlinee(Js::ProfileId profileId) const;
+    const FunctionJITTimeInfo * GetCallApplyTargetInlinee(Js::ProfileId profileId) const;
+    const Js::ProfileId GetCallApplyCallSiteIdForCallSiteId(Js::ProfileId profiledCallSiteId) const;
     const FunctionJITTimeInfo * GetLdFldInlinee(Js::InlineCacheIndex inlineCacheIndex) const;
     const FunctionJITTimeInfo * GetInlinee(Js::ProfileId profileId) const;
     const FunctionJITTimeInfo * GetNext() const;
@@ -45,6 +47,7 @@ class FunctionJITTimeInfo
     const FunctionJITRuntimeInfo *GetLdFldInlineeRuntimeData(const Js::InlineCacheIndex inlineCacheIndex) const;
     const FunctionJITRuntimeInfo * GetCallbackInlineeRuntimeData(const Js::ProfileId profiledCallSiteId) const;
     const FunctionJITRuntimeInfo * GetInlineeForCallbackInlineeRuntimeData(const Js::ProfileId profiledCallSiteId, intptr_t inlineeFuncBodyAddr) const;
+    const FunctionJITRuntimeInfo * GetCallApplyTargetInlineeRuntimeData(const Js::ProfileId callApplyCallSiteId) const;
     bool ForceJITLoopBody() const;
     bool HasSharedPropertyGuards() const;
     bool HasSharedPropertyGuard(Js::PropertyId id) const;

lib/Backend/IR.cpp

@@ -2017,6 +2017,22 @@ Instr::New(Js::OpCode opcode, Func *func)
     return instr;
 }
 
+///----------------------------------------------------------------------------
+///
+/// Instr::New
+///
+///     Create an Instr with a byte code offset.
+///
+///----------------------------------------------------------------------------
+
+Instr *
+Instr::New(Js::OpCode opcode, Func *func, IR::Instr * bytecodeOffsetInstr)
+{
+    Instr * instr = Instr::New(opcode, func);
+    instr->SetByteCodeOffset(bytecodeOffsetInstr);
+    return instr;
+}
+
 ///----------------------------------------------------------------------------
 ///
 /// Instr::New

lib/Backend/IR.h

@@ -177,6 +177,7 @@ class Instr
     }
 public:
     static Instr *  New(Js::OpCode opcode, Func *func);
+    static Instr *  New(Js::OpCode opcode, Func *func, IR::Instr * bytecodeOffsetInstr);
     static Instr *  New(Js::OpCode opcode, Opnd *dstOpnd, Func *func);
     static Instr *  New(Js::OpCode opcode, Opnd *dstOpnd, Opnd *src1Opnd, Func *func);
     static Instr *  New(Js::OpCode opcode, Opnd *dstOpnd, Opnd *src1Opnd, Opnd *src2Opnd, Func *func);