[clang][CodeGen][MSVC] Match how MSVC returns vector types from member functions

clang/docs/ReleaseNotes.rst

@@ -90,6 +90,12 @@ C++ Specific Potentially Breaking Changes
 ABI Changes in This Version
 ---------------------------
 
+- Fixed Microsoft calling convention to match how MSVC returns vector types from
+  C++ member functions on x86/x86-64. This change resolves incompatibilities with
+  code compiled by MSVC but will introduce incompatibilities with code compiled
+  by Clang 21 and earlier versions, unless the ``-fclang-abi-compat=21`` option
+  is used. (#GH104)
+
 AST Dumping Potentially Breaking Changes
 ----------------------------------------
 - How nested name specifiers are dumped and printed changes, keeping track of clang AST changes.

clang/include/clang/Basic/ABIVersions.def

@@ -127,6 +127,12 @@ ABI_VER_MAJOR(19)
 ///   - Incorrectly return C++ records in AVX registers on x86_64.
 ABI_VER_MAJOR(20)
 
+/// Attempt to be ABI-compatible with code generated by Clang 21.0.x.
+/// This causes clang to:
+///   - Always return vector types directly from member functions on x86 and
+///     x86_64 on Windows, which is not compatible with the MSVC ABI.
+ABI_VER_MAJOR(21)
+
 /// Conform to the underlying platform's C and C++ ABIs as closely as we can.
 ABI_VER_LATEST(Latest)
 

clang/lib/CodeGen/MicrosoftCXXABI.cpp

@@ -1168,18 +1168,32 @@ static bool isTrivialForMSVC(const CXXRecordDecl *RD, QualType Ty,
 }
 
 bool MicrosoftCXXABI::classifyReturnType(CGFunctionInfo &FI) const {
-  const CXXRecordDecl *RD = FI.getReturnType()->getAsCXXRecordDecl();
-  if (!RD)
-    return false;
-
-  bool isTrivialForABI = RD->canPassInRegisters() &&
-                         isTrivialForMSVC(RD, FI.getReturnType(), CGM);
-
-  // MSVC always returns structs indirectly from C++ instance methods.
-  bool isIndirectReturn = !isTrivialForABI || FI.isInstanceMethod();
+  QualType RetTy = FI.getReturnType();
+  bool isIndirectReturn = false;
+
+  if (const CXXRecordDecl *RD = RetTy->getAsCXXRecordDecl()) {
+    bool isTrivialForABI =
+        RD->canPassInRegisters() && isTrivialForMSVC(RD, RetTy, CGM);
+
+    // MSVC always returns structs indirectly from C++ instance methods.
+    isIndirectReturn = !isTrivialForABI || FI.isInstanceMethod();
+  } else if (isa<VectorType>(RetTy) &&
+             getContext().getLangOpts().getClangABICompat() >
+                 LangOptions::ClangABI::Ver21) {
+    // On x86, MSVC usually returns vector types indirectly from C++ instance
+    // methods. (Clang <= 21.0 always returned vector types directly.)
+    if (CGM.getTarget().getTriple().isX86() && FI.isInstanceMethod()) {
+      // However, MSVC returns vector types > 64 bits directly from vectorcall
+      // instance methods.
+      if (FI.getCallingConvention() == llvm::CallingConv::X86_VectorCall)
+        isIndirectReturn = getContext().getTypeSize(RetTy) == 64;
+      else
+        isIndirectReturn = true;
+    }
+  }
 
   if (isIndirectReturn) {
-    CharUnits Align = CGM.getContext().getTypeAlignInChars(FI.getReturnType());
+    CharUnits Align = CGM.getContext().getTypeAlignInChars(RetTy);
     FI.getReturnInfo() = ABIArgInfo::getIndirect(
         Align, /*AddrSpace=*/CGM.getDataLayout().getAllocaAddrSpace(),
         /*ByVal=*/false);

clang/test/CodeGenCXX/microsoft-abi-vector-types.cpp

@@ -0,0 +1,93 @@
+// RUN: %clang_cc1 -ffreestanding -emit-llvm %s -o - -triple=i686-pc-windows-msvc \
+// RUN:   | FileCheck --check-prefixes=CHECK,X86 %s
+// RUN: %clang_cc1 -ffreestanding -emit-llvm %s -o - -triple=x86_64-pc-windows-msvc \
+// RUN:   | FileCheck --check-prefixes=CHECK,X86 %s
+// RUN: %clang_cc1 -ffreestanding -emit-llvm %s -o - -triple=aarch64-pc-windows-msvc \
+// RUN:   | FileCheck --check-prefixes=CHECK,AARCH64 %s
+// RUN: %clang_cc1 -ffreestanding -emit-llvm %s -o - -triple=i686-pc-windows-msvc \
+// RUN:   -fclang-abi-compat=21 | FileCheck --check-prefixes=CHECK,CLANG21 %s
+// RUN: %clang_cc1 -ffreestanding -emit-llvm %s -o - -triple=x86_64-pc-windows-msvc \
+// RUN:   -fclang-abi-compat=21 | FileCheck --check-prefixes=CHECK,CLANG21 %s
+// RUN: %clang_cc1 -ffreestanding -emit-llvm %s -o - -triple=aarch64-pc-windows-msvc \
+// RUN:   -fclang-abi-compat=21 | FileCheck --check-prefixes=CHECK,CLANG21 %s
+
+// To match the MSVC ABI, vector types are usually returned directly, but on x86
+// and x86-64 they must be returned indirectly from member functions (unless
+// they use the vectorcall calling convention and the vector type is > 64 bits).
+
+#if defined(__i386__) || defined(__x86_64__)
+#include <xmmintrin.h>
+
+#define VECTOR64_TYPE __m64
+#define VECTOR128_TYPE __m128
+
+#define VECTORCALL __vectorcall
+#endif
+
+#ifdef __aarch64__
+#include <arm_neon.h>
+
+// These were chosen such that they lower to the same types that the x86 vector
+// types lower to (e.g. int64x1_t and __m64 both lower to <1 x i64>).
+#define VECTOR64_TYPE int64x1_t
+#define VECTOR128_TYPE float32x4_t
+
+#define VECTORCALL
+#endif
+
+struct Foo {
+  VECTOR64_TYPE method_ret_vec64();
+  VECTOR128_TYPE method_ret_vec128();
+
+  VECTOR64_TYPE VECTORCALL vc_method_ret_vec64();
+  VECTOR128_TYPE VECTORCALL vc_method_ret_vec128();
+};
+
+VECTOR64_TYPE Foo::method_ret_vec64() {
+  return VECTOR64_TYPE{};
+// X86: store <1 x i64>
+// X86: ret void
+// AARCH64: ret <1 x i64>
+// CLANG21: ret <1 x i64>
+}
+
+VECTOR128_TYPE Foo::method_ret_vec128() {
+  return VECTOR128_TYPE{};
+// X86: store <4 x float>
+// X86: ret void
+// AARCH64: ret <4 x float>
+// CLANG21: ret <4 x float>
+}
+
+VECTOR64_TYPE VECTORCALL Foo::vc_method_ret_vec64() {
+  return VECTOR64_TYPE{};
+// X86: store <1 x i64>
+// X86: ret void
+// AARCH64: ret <1 x i64>
+// CLANG21: ret <1 x i64>
+}
+
+VECTOR128_TYPE VECTORCALL Foo::vc_method_ret_vec128() {
+  return VECTOR128_TYPE{};
+// CHECK: ret <4 x float>
+}
+
+VECTOR64_TYPE func_ret_vec64() {
+  return VECTOR64_TYPE{};
+// CHECK: ret <1 x i64>
+}
+
+VECTOR128_TYPE func_ret_vec128() {
+  return VECTOR128_TYPE{};
+// CHECK: ret <4 x float>
+}
+
+VECTOR64_TYPE VECTORCALL vc_func_ret_vec64() {
+  return VECTOR64_TYPE{};
+// CHECK: ret <1 x i64>
+}
+
+VECTOR128_TYPE VECTORCALL vc_func_ret_vec128() {
+  return VECTOR128_TYPE{};
+// CHECK: ret <4 x float>
+}