Merge pull request #5978 from rahku/arm64retbuf

Fix Pinvoke IL Stubs to emit correct return type in signature of call…

Author: Rahul Kumar

src/vm/dllimport.cpp

@@ -4194,9 +4194,17 @@ static void CreateNDirectStubWorker(StubState*         pss,
     bool fHasCopyCtorArgs = false;
     bool fStubNeedsCOM = SF_IsCOMStub(dwStubFlags);
 
+#if defined(_TARGET_X86_) || defined(_TARGET_ARM_)
+    // JIT32 has problems in generating code for pinvoke ILStubs which do a return in return buffer.
+    // Therefore instead we change the signature of calli to return void and make the return buffer as first
+    // argument. This matches the ABI i.e. return buffer is passed as first arg. So native target will get the
+    // return buffer in correct register.
     // The return structure secret arg comes first, however byvalue return is processed at
     // the end because it could be the HRESULT-swapped argument which always comes last.
     bool fMarshalReturnValueFirst = !SF_IsHRESULTSwapping(dwStubFlags) && HasRetBuffArg(&msig);
+#else
+    bool fMarshalReturnValueFirst = false;
+#endif
 
     if (fMarshalReturnValueFirst)
     {

src/vm/ilmarshalers.h

@@ -600,7 +600,7 @@ class ILMarshaler
                 nativeSize = wNativeSize;
             }
 
-#if !defined(_TARGET_ARM_) && !(defined(UNIX_AMD64_ABI) && defined(FEATURE_UNIX_AMD64_STRUCT_PASSING))
+#if defined(_TARGET_X86_)
             switch (nativeSize)
             {
                 case 1: typ = ELEMENT_TYPE_U1; break;