[release/3.0] Opt COM methods out of the new Windows instance-method handling (#24012)

* Opt COM methods out of the new Windows instance-method handling.

* Remove calls to run the MarshalStructReturn tests since we're rolling back the calling convention support change.

Author: jkoritzinsky

src/vm/dllimport.cpp

@@ -3956,7 +3956,10 @@ static void CreateNDirectStubWorker(StubState*         pss,
     BOOL fMarshalReturnValueFirst = FALSE;
 
     BOOL fReverseWithReturnBufferArg = FALSE;
-    bool isInstanceMethod = fStubNeedsCOM || fThisCall;
+    // Only consider ThisCall methods to be instance methods.
+    // Techinically COM methods are also instance methods, but we don't want to change the behavior of the built-in
+    // COM abi work because there are many users that rely on the current behavior (for example WPF).
+    bool isInstanceMethod = fThisCall;
 
     // We can only change fMarshalReturnValueFirst to true when we are NOT doing HRESULT-swapping!
     // When we are HRESULT-swapping, the managed return type is actually the type of the last parameter and not the return type.

src/vm/ilmarshalers.h

@@ -583,9 +583,7 @@ class ILMarshaler
         bool byrefNativeReturn = false;
         CorElementType typ = ELEMENT_TYPE_VOID;
         UINT32 nativeSize = 0;
-        bool nativeMethodIsMemberFunction = (m_pslNDirect->TargetHasThis() && IsCLRToNative(m_dwMarshalFlags))
-            || (m_pslNDirect->HasThis() && !IsCLRToNative(m_dwMarshalFlags))
-            || ((CorInfoCallConv)m_pslNDirect->GetStubTargetCallingConv() == CORINFO_CALLCONV_THISCALL);
+        bool nativeMethodIsMemberFunction = (CorInfoCallConv)m_pslNDirect->GetStubTargetCallingConv() == CORINFO_CALLCONV_THISCALL;
 
         // we need to convert value type return types to primitives as
         // JIT does not inline P/Invoke calls that return structures

tests/src/Interop/COM/NETClients/Primitives/NumericTests.cs

@@ -38,7 +38,6 @@ public void Run()
             this.Marshal_Float(a / 100f, b / 100f);
             this.Marshal_Double(a / 100.0, b / 100.0);
             this.Marshal_ManyInts();
-            this.Marshal_Struct_Return();
         }
 
         static private bool EqualByBound(float expected, float actual)

tests/src/Interop/COM/NativeClients/Primitives/NumericTests.cpp

@@ -265,5 +265,4 @@ void Run_NumericTests()
     MarshalFloat(numericTesting, (float)a / 100.f, (float)b / 100.f);
     MarshalDouble(numericTesting, (double)a / 100.0, (double)b / 100.0);
     MarshalManyInts(numericTesting);
-    MarshalStructReturn(numericTesting);
 }