Suppress returning larger CXX records in mem on PlayStation (#161732)

In commit e8a486e, a change was made so that certain 256-bit and
512-bit CXX records would be returned in memory, fixing a violation of
the x86-64 psABI (where they had been incorrectly returned in AVX
registers). For compatibility reasons, we want to suppress that ABI-fix
on PlayStation. This commit suppresses that change for PlayStation, and
updates the test to include checking the 512-bit case.

Author: wjristow

clang/lib/CodeGen/Targets/X86.cpp

@@ -1343,9 +1343,10 @@ class X86_64ABIInfo : public ABIInfo {
   }
 
   bool returnCXXRecordGreaterThan128InMem() const {
-    // Clang <= 20.0 did not do this.
+    // Clang <= 20.0 did not do this, and PlayStation does not do this.
     if (getContext().getLangOpts().getClangABICompat() <=
-        LangOptions::ClangABI::Ver20)
+            LangOptions::ClangABI::Ver20 ||
+        getTarget().getTriple().isPS())
       return false;
 
     return true;

clang/test/CodeGen/X86/avx-cxx-record.cpp

@@ -1,7 +1,9 @@
 // RUN: %clang_cc1 %s -triple x86_64-unknown-linux-gnu -emit-llvm -O2 -target-cpu x86-64-v3 -o - | FileCheck %s
 // RUN: %clang_cc1 %s -triple x86_64-unknown-linux-gnu -emit-llvm -O2 -target-cpu x86-64-v3 -fclang-abi-compat=20 -o - | FileCheck --check-prefix CLANG-20 %s
+// RUN: %clang_cc1 %s -triple x86_64-sie-ps4 -emit-llvm -O2 -target-cpu x86-64-v3 -o - | FileCheck --check-prefix CLANG-20 %s
 
 using UInt64x2 = unsigned long long __attribute__((__vector_size__(16), may_alias));
+using UInt64x4 = unsigned long long __attribute__((__vector_size__(32), may_alias));
 
 template<int id>
 struct XMM1 {
@@ -23,3 +25,24 @@ XMM2 foo() {
   ((XMM1<1>*)&result)->x = UInt64x2{3, 4};
   return result;
 }
+
+template<int id>
+struct YMM1 {
+    UInt64x4 x;
+};
+
+struct YMM2 : YMM1<0>, YMM1<1> {
+};
+
+// CHECK: define{{.*}} @_Z3barv({{.*}} [[ARG:%.*]]){{.*}}
+// CLANG-20: define{{.*}} <8 x double> @_Z3barv()
+// CHECK: entry:
+// CHECK-NEXT: store {{.*}}, ptr [[ARG]]{{.*}}
+// CHECK-NEXT: [[TMP1:%.*]] = getelementptr {{.*}}, ptr [[ARG]]{{.*}}
+// CHECK-NEXT: store {{.*}}, ptr [[TMP1]]{{.*}}
+YMM2 bar() {
+  YMM2 result;
+  ((YMM1<0>*)&result)->x = UInt64x4{1, 2, 3, 4};
+  ((YMM1<1>*)&result)->x = UInt64x4{5, 6, 7, 8};
+  return result;
+}