[clang][CodeGen][MSVC] Match how MSVC returns vector types from member functions

The MSVC ABI usually returns vector types directly, but on x86 and x86-64, there
seems to be a special case for C++ member functions, which return vector types
indirectly (with the exception of member functions using the `__vectorcall`
calling convention, which return vector types > 64 bits directly).

This is an ABI change and has the potential to cause backward compatibility
issues with previous Clang releases.

Fixes #104.

Author: henrybw

clang/docs/ReleaseNotes.rst

@@ -124,6 +124,12 @@ ABI Changes in This Version
 ---------------------------
 - Fix AArch64 argument passing for C++ empty classes with large explicitly specified alignment.
 
+- Fixed Microsoft calling convention to match how MSVC returns vector types from
+  C++ member functions on x86/x86-64. This change resolves incompatibilities with
+  code compiled by MSVC but will introduce incompatibilities with code compiled
+  by Clang 21 and earlier versions, unless the ``-fclang-abi-compat=21`` option
+  is used. (#GH104)
+
 AST Dumping Potentially Breaking Changes
 ----------------------------------------
 - How nested name specifiers are dumped and printed changes, keeping track of clang AST changes.

clang/include/clang/Basic/ABIVersions.def

@@ -133,6 +133,8 @@ ABI_VER_MAJOR(20)
 ///    compatible with scalar deleting destructors emitted by MSVC for the
 ///    cases when the class whose destructor is being emitted defines
 ///    operator delete.
+///   - Always return vector types directly from member functions on x86 and
+///     x86_64 on Windows, which is not compatible with the MSVC ABI.
 ABI_VER_MAJOR(21)
 
 /// Conform to the underlying platform's C and C++ ABIs as closely as we can.

clang/lib/CodeGen/MicrosoftCXXABI.cpp

@@ -1177,18 +1177,32 @@ static bool isTrivialForMSVC(const CXXRecordDecl *RD, QualType Ty,
 }
 
 bool MicrosoftCXXABI::classifyReturnType(CGFunctionInfo &FI) const {
-  const CXXRecordDecl *RD = FI.getReturnType()->getAsCXXRecordDecl();
-  if (!RD)
-    return false;
-
-  bool isTrivialForABI = RD->canPassInRegisters() &&
-                         isTrivialForMSVC(RD, FI.getReturnType(), CGM);
-
-  // MSVC always returns structs indirectly from C++ instance methods.
-  bool isIndirectReturn = !isTrivialForABI || FI.isInstanceMethod();
+  QualType RetTy = FI.getReturnType();
+  bool isIndirectReturn = false;
+
+  if (const CXXRecordDecl *RD = RetTy->getAsCXXRecordDecl()) {
+    bool isTrivialForABI =
+        RD->canPassInRegisters() && isTrivialForMSVC(RD, RetTy, CGM);
+
+    // MSVC always returns structs indirectly from C++ instance methods.
+    isIndirectReturn = !isTrivialForABI || FI.isInstanceMethod();
+  } else if (isa<VectorType>(RetTy) &&
+             getContext().getLangOpts().getClangABICompat() >
+                 LangOptions::ClangABI::Ver21) {
+    // On x86, MSVC usually returns vector types indirectly from C++ instance
+    // methods. (Clang <= 21.0 always returned vector types directly.)
+    if (CGM.getTarget().getTriple().isX86() && FI.isInstanceMethod()) {
+      // However, MSVC returns vector types > 64 bits directly from vectorcall
+      // instance methods.
+      if (FI.getCallingConvention() == llvm::CallingConv::X86_VectorCall)
+        isIndirectReturn = getContext().getTypeSize(RetTy) == 64;
+      else
+        isIndirectReturn = true;
+    }
+  }
 
   if (isIndirectReturn) {
-    CharUnits Align = CGM.getContext().getTypeAlignInChars(FI.getReturnType());
+    CharUnits Align = CGM.getContext().getTypeAlignInChars(RetTy);
     FI.getReturnInfo() = ABIArgInfo::getIndirect(
         Align, /*AddrSpace=*/CGM.getDataLayout().getAllocaAddrSpace(),
         /*ByVal=*/false);

clang/test/CodeGenCXX/microsoft-abi-vector-types.cpp

@@ -0,0 +1,93 @@
+// RUN: %clang_cc1 -ffreestanding -emit-llvm %s -o - -triple=i686-pc-windows-msvc \
+// RUN:   | FileCheck --check-prefixes=CHECK,X86 %s
+// RUN: %clang_cc1 -ffreestanding -emit-llvm %s -o - -triple=x86_64-pc-windows-msvc \
+// RUN:   | FileCheck --check-prefixes=CHECK,X86 %s
+// RUN: %clang_cc1 -ffreestanding -emit-llvm %s -o - -triple=aarch64-pc-windows-msvc \
+// RUN:   | FileCheck --check-prefixes=CHECK,AARCH64 %s
+// RUN: %clang_cc1 -ffreestanding -emit-llvm %s -o - -triple=i686-pc-windows-msvc \
+// RUN:   -fclang-abi-compat=21 | FileCheck --check-prefixes=CHECK,CLANG21 %s
+// RUN: %clang_cc1 -ffreestanding -emit-llvm %s -o - -triple=x86_64-pc-windows-msvc \
+// RUN:   -fclang-abi-compat=21 | FileCheck --check-prefixes=CHECK,CLANG21 %s
+// RUN: %clang_cc1 -ffreestanding -emit-llvm %s -o - -triple=aarch64-pc-windows-msvc \
+// RUN:   -fclang-abi-compat=21 | FileCheck --check-prefixes=CHECK,CLANG21 %s
+
+// To match the MSVC ABI, vector types are usually returned directly, but on x86
+// and x86-64 they must be returned indirectly from member functions (unless
+// they use the vectorcall calling convention and the vector type is > 64 bits).
+
+#if defined(__i386__) || defined(__x86_64__)
+#include <xmmintrin.h>
+
+#define VECTOR64_TYPE __m64
+#define VECTOR128_TYPE __m128
+
+#define VECTORCALL __vectorcall
+#endif
+
+#ifdef __aarch64__
+#include <arm_neon.h>
+
+// These were chosen such that they lower to the same types that the x86 vector
+// types lower to (e.g. int64x1_t and __m64 both lower to <1 x i64>).
+#define VECTOR64_TYPE int64x1_t
+#define VECTOR128_TYPE float32x4_t
+
+#define VECTORCALL
+#endif
+
+struct Foo {
+  VECTOR64_TYPE method_ret_vec64();
+  VECTOR128_TYPE method_ret_vec128();
+
+  VECTOR64_TYPE VECTORCALL vc_method_ret_vec64();
+  VECTOR128_TYPE VECTORCALL vc_method_ret_vec128();
+};
+
+VECTOR64_TYPE Foo::method_ret_vec64() {
+  return VECTOR64_TYPE{};
+// X86: store <1 x i64>
+// X86: ret void
+// AARCH64: ret <1 x i64>
+// CLANG21: ret <1 x i64>
+}
+
+VECTOR128_TYPE Foo::method_ret_vec128() {
+  return VECTOR128_TYPE{};
+// X86: store <4 x float>
+// X86: ret void
+// AARCH64: ret <4 x float>
+// CLANG21: ret <4 x float>
+}
+
+VECTOR64_TYPE VECTORCALL Foo::vc_method_ret_vec64() {
+  return VECTOR64_TYPE{};
+// X86: store <1 x i64>
+// X86: ret void
+// AARCH64: ret <1 x i64>
+// CLANG21: ret <1 x i64>
+}
+
+VECTOR128_TYPE VECTORCALL Foo::vc_method_ret_vec128() {
+  return VECTOR128_TYPE{};
+// CHECK: ret <4 x float>
+}
+
+VECTOR64_TYPE func_ret_vec64() {
+  return VECTOR64_TYPE{};
+// CHECK: ret <1 x i64>
+}
+
+VECTOR128_TYPE func_ret_vec128() {
+  return VECTOR128_TYPE{};
+// CHECK: ret <4 x float>
+}
+
+VECTOR64_TYPE VECTORCALL vc_func_ret_vec64() {
+  return VECTOR64_TYPE{};
+// CHECK: ret <1 x i64>
+}
+
+VECTOR128_TYPE VECTORCALL vc_func_ret_vec128() {
+  return VECTOR128_TYPE{};
+// CHECK: ret <4 x float>
+}