Merge pull request #6093 from rahku/arm64retbuf

Revert Pinvoke ILStub calli signature for desktop clr

Author: Rahul Kumar

src/vm/dllimport.cpp

@@ -4193,19 +4193,40 @@ static void CreateNDirectStubWorker(StubState*         pss,
     UINT nativeStackSize = (SF_IsCOMStub(dwStubFlags) ? sizeof(SLOT) : 0);
     bool fHasCopyCtorArgs = false;
     bool fStubNeedsCOM = SF_IsCOMStub(dwStubFlags);
+    
+    // Normally we would like this to be false so that we use the correct signature 
+    // in the IL_STUB, (i.e if it returns a value class then the signature will use that)
+    // When this bool is true we change the return type to void and explicitly add a
+    // return buffer argument as the first argument.
+    BOOL fMarshalReturnValueFirst = false;
+    
+    // We can only change fMarshalReturnValueFirst to true when we are NOT doing HRESULT-swapping!
+    //
+    if (!SF_IsHRESULTSwapping(dwStubFlags))
+    {
 
 #if defined(_TARGET_X86_) || defined(_TARGET_ARM_)
-    // JIT32 has problems in generating code for pinvoke ILStubs which do a return in return buffer.
-    // Therefore instead we change the signature of calli to return void and make the return buffer as first
-    // argument. This matches the ABI i.e. return buffer is passed as first arg. So native target will get the
-    // return buffer in correct register.
-    // The return structure secret arg comes first, however byvalue return is processed at
-    // the end because it could be the HRESULT-swapped argument which always comes last.
-    bool fMarshalReturnValueFirst = !SF_IsHRESULTSwapping(dwStubFlags) && HasRetBuffArg(&msig);
-#else
-    bool fMarshalReturnValueFirst = false;
+        // JIT32 has problems in generating code for pinvoke ILStubs which do a return in return buffer.
+        // Therefore instead we change the signature of calli to return void and make the return buffer as first
+        // argument. This matches the ABI i.e. return buffer is passed as first arg. So native target will get the
+        // return buffer in correct register.
+        // The return structure secret arg comes first, however byvalue return is processed at
+        // the end because it could be the HRESULT-swapped argument which always comes last.
+        fMarshalReturnValueFirst = HasRetBuffArg(&msig);
 #endif
 
+#if defined(_TARGET_AMD64_) && defined(_WIN64) && !defined(FEATURE_CORECLR)
+        // JIT64 (which is only used on the Windows Desktop CLR) has a problem generating code
+        // for the pinvoke ILStubs which do a return using a struct type.  Therefore, we
+        // change the signature of calli to return void and make the return buffer as first argument. 
+        // This matches the ABI i.e. return buffer is passed as first arg. So native target will get
+        // the return buffer in correct register.
+        // Ideally we only want to set it for JIT64 and not ryujit but currently there isn't a fast way 
+        // to determine that at runtime.
+        fMarshalReturnValueFirst = HasRetBuffArg(&msig);
+#endif
+    }
+    
     if (fMarshalReturnValueFirst)
     {
         marshalType = DoMarshalReturnValue(msig,

src/vm/ilmarshalers.h

@@ -600,7 +600,13 @@ class ILMarshaler
                 nativeSize = wNativeSize;
             }
 
-#if defined(_TARGET_X86_)
+#if defined(_TARGET_X86_) || (defined(_TARGET_AMD64_) && defined(_WIN64) && !defined(FEATURE_CORECLR))
+            // JIT32 and JIT64 (which is only used on the Windows Desktop CLR) has a problem generating
+            // code for the pinvoke ILStubs which do a return using a struct type.  Therefore, we
+            // change the signature of calli to return void and make the return buffer as first argument. 
+
+            // for X86 and AMD64-Windows we bash the return type from struct to U1, U2, U4 or U8
+            // and use byrefNativeReturn for all other structs.
             switch (nativeSize)
             {
                 case 1: typ = ELEMENT_TYPE_U1; break;