[clang][CodeGen][MSVC] Match how MSVC returns vector types from member functions

clang/docs/ReleaseNotes.rst

@@ -90,6 +90,12 @@ C++ Specific Potentially Breaking Changes
 ABI Changes in This Version
 ---------------------------
 
+- Fixed Microsoft calling convention to return vector types from C++ member
+  functions indirectly. This change resolves incompatibilities with code
+  compiled by MSVC but will introduce incompatibilities with code compiled by
+  Clang 21 and earlier versions, unless the ``-fclang-abi-compat=21`` option is
+  used. (#GH104)
+
 AST Dumping Potentially Breaking Changes
 ----------------------------------------
 - How nested name specifiers are dumped and printed changes, keeping track of clang AST changes.

clang/include/clang/Basic/ABIVersions.def

@@ -127,6 +127,12 @@ ABI_VER_MAJOR(19)
 ///   - Incorrectly return C++ records in AVX registers on x86_64.
 ABI_VER_MAJOR(20)
 
+/// Attempt to be ABI-compatible with code generated by Clang 21.0.x.
+/// This causes clang to:
+///   - Return vector types directly from member functions on x86 and x86_64 on
+///     Windows, which is not compatible with the MSVC ABI.
+ABI_VER_MAJOR(21)
+
 /// Conform to the underlying platform's C and C++ ABIs as closely as we can.
 ABI_VER_LATEST(Latest)
 

clang/lib/CodeGen/MicrosoftCXXABI.cpp

@@ -1168,15 +1168,23 @@ static bool isTrivialForMSVC(const CXXRecordDecl *RD, QualType Ty,
 }
 
 bool MicrosoftCXXABI::classifyReturnType(CGFunctionInfo &FI) const {
-  const CXXRecordDecl *RD = FI.getReturnType()->getAsCXXRecordDecl();
-  if (!RD)
-    return false;
-
-  bool isTrivialForABI = RD->canPassInRegisters() &&
-                         isTrivialForMSVC(RD, FI.getReturnType(), CGM);
-
-  // MSVC always returns structs indirectly from C++ instance methods.
-  bool isIndirectReturn = !isTrivialForABI || FI.isInstanceMethod();
+  bool isIndirectReturn = false;
+  if (const CXXRecordDecl *RD = FI.getReturnType()->getAsCXXRecordDecl()) {
+    bool isTrivialForABI = RD->canPassInRegisters() &&
+                           isTrivialForMSVC(RD, FI.getReturnType(), CGM);
+
+    // MSVC always returns structs indirectly from C++ instance methods.
+    isIndirectReturn = !isTrivialForABI || FI.isInstanceMethod();
+  } else if (isa<VectorType>(FI.getReturnType())) {
+    // On x86, MSVC seems to only return vector types indirectly from non-
+    // vectorcall C++ instance methods.
+    isIndirectReturn =
+        CGM.getTarget().getTriple().isX86() && FI.isInstanceMethod() &&
+        FI.getCallingConvention() != llvm::CallingConv::X86_VectorCall &&
+        // Clang <= 21.0 did not do this.
+        getContext().getLangOpts().getClangABICompat() >
+            LangOptions::ClangABI::Ver21;
+  }
 
   if (isIndirectReturn) {
     CharUnits Align = CGM.getContext().getTypeAlignInChars(FI.getReturnType());

clang/test/CodeGenCXX/microsoft-abi-vector-types.cpp

@@ -0,0 +1,55 @@
+// RUN: %clang_cc1 -ffreestanding -emit-llvm %s -o - -triple=i686-pc-windows-msvc \
+// RUN:   | FileCheck --check-prefixes=CHECK,X86 %s
+// RUN: %clang_cc1 -ffreestanding -emit-llvm %s -o - -triple=x86_64-pc-windows-msvc \
+// RUN:   | FileCheck --check-prefixes=CHECK,X86 %s
+// RUN: %clang_cc1 -ffreestanding -emit-llvm %s -o - -triple=aarch64-pc-windows-msvc \
+// RUN:   | FileCheck --check-prefixes=CHECK,AARCH64 %s
+// RUN: %clang_cc1 -ffreestanding -emit-llvm %s -o - -triple=i686-pc-windows-msvc \
+// RUN:   -fclang-abi-compat=21 | FileCheck --check-prefixes=CHECK,CLANG21 %s
+// RUN: %clang_cc1 -ffreestanding -emit-llvm %s -o - -triple=x86_64-pc-windows-msvc \
+// RUN:   -fclang-abi-compat=21 | FileCheck --check-prefixes=CHECK,CLANG21 %s
+// RUN: %clang_cc1 -ffreestanding -emit-llvm %s -o - -triple=aarch64-pc-windows-msvc \
+// RUN:   -fclang-abi-compat=21 | FileCheck --check-prefixes=CHECK,CLANG21 %s
+
+// To match the MSVC ABI, vector types must be returned indirectly from member
+// functions on x86 and x86-64 (as long as they do not use the vectorcall
+// calling convention), but must be returned directly everywhere else.
+
+#if defined(__i386__) || defined(__x86_64__)
+#include <xmmintrin.h>
+#define VECTOR_TYPE __m128
+#endif
+
+#ifdef __aarch64__
+#include <arm_neon.h>
+#define VECTOR_TYPE float32x4_t
+#endif
+
+struct Foo {
+  VECTOR_TYPE method_ret_vec();
+  VECTOR_TYPE __vectorcall vectorcall_method_ret_vec();
+};
+
+VECTOR_TYPE Foo::method_ret_vec() {
+  return VECTOR_TYPE{};
+// GH104
+// X86: store <4 x float>
+// X86: ret void
+// AARCH64: ret <4 x float>
+// CLANG21: ret <4 x float>
+}
+
+VECTOR_TYPE __vectorcall Foo::vectorcall_method_ret_vec() {
+  return VECTOR_TYPE{};
+// CHECK: ret <4 x float>
+}
+
+VECTOR_TYPE func_ret_vec() {
+  return VECTOR_TYPE{};
+// CHECK: ret <4 x float>
+}
+
+VECTOR_TYPE __vectorcall vectorcall_func_ret_vec() {
+  return VECTOR_TYPE{};
+// CHECK: ret <4 x float>
+}