Merge branch 'master' into windows-clang

Author: qurious-pixel

.github/workflows/rpcs3.yml

@@ -387,8 +387,9 @@ jobs:
         run: |
           export CCACHE_DIR=$(cygpath -u "$CCACHE_DIR")
           echo "CCACHE_DIR=$CCACHE_DIR"
-          .ci/setup-windows-ci-vars.sh ${{ matrix.arch }} ${{ matrix.compiler }}
           .ci/build-windows-clang.sh
+          .ci/setup-windows-ci-vars.sh ${{ matrix.arch }} ${{ matrix.compiler }}
+          .ci/deploy-windows-${{ matrix.compiler }}.sh
 
       - name: Save build Ccache
         if: github.ref == 'refs/heads/master'

rpcs3/CMakeLists.txt

@@ -193,6 +193,8 @@ if(BUILD_RPCS3_TESTS)
         PRIVATE
             tests/test.cpp
             tests/test_fmt.cpp
+            tests/test_pair.cpp
+            tests/test_tuple.cpp
             tests/test_simple_array.cpp
             tests/test_address_range.cpp
     )

rpcs3/Crypto/key_vault.cpp

@@ -11,10 +11,10 @@ SELF_KEY::SELF_KEY(u64 ver_start, u64 ver_end, u16 rev, u32 type, const std::str
 	version_end   = ver_end;
 	revision      = rev;
 	self_type     = type;
-	hex_to_bytes(erk, e.c_str(), 0);
-	hex_to_bytes(riv, r.c_str(), 0);
-	hex_to_bytes(pub, pb.c_str(), 0);
-	hex_to_bytes(priv, pr.c_str(), 0);
+	hex_to_bytes(erk, e, 0);
+	hex_to_bytes(riv, r, 0);
+	hex_to_bytes(pub, pb, 0);
+	hex_to_bytes(priv, pr, 0);
 	curve_type = ct;
 }
 

rpcs3/Crypto/utils.cpp

@@ -7,9 +7,11 @@
 #include "sha1.h"
 #include "sha256.h"
 #include "key_vault.h"
+#include <charconv>
+#include <cstdlib>
 #include <cstring>
-#include <stdio.h>
-#include <time.h>
+#include <cstdio>
+#include <ctime>
 #include "Utilities/StrUtil.h"
 #include "Utilities/File.h"
 
@@ -21,50 +23,24 @@
 // Auxiliary functions (endian swap, xor).
 
 // Hex string conversion auxiliary functions.
-u64 hex_to_u64(const char* hex_str)
+void hex_to_bytes(unsigned char* data, std::string_view hex_str, unsigned int str_length)
 {
-	auto length = std::strlen(hex_str);
-	u64 tmp = 0;
-	u64 result = 0;
-	char c;
-
-	while (length--)
-	{
-		c = *hex_str++;
-		if((c >= '0') && (c <= '9'))
-			tmp = c - '0';
-		else if((c >= 'a') && (c <= 'f'))
-			tmp = c - 'a' + 10;
-		else if((c >= 'A') && (c <= 'F'))
-			tmp = c - 'A' + 10;
-		else
-			tmp = 0;
-		result |= (tmp << (length * 4));
-	}
-
-	return result;
-}
-
-void hex_to_bytes(unsigned char* data, const char* hex_str, unsigned int str_length)
-{
-	const auto strn_length = (str_length > 0) ? str_length : std::strlen(hex_str);
-	auto data_length = strn_length / 2;
-	char tmp_buf[3] = {0, 0, 0};
+	const auto strn_length = (str_length > 0) ? str_length : hex_str.size();
 
 	// Don't convert if the string length is odd.
 	if ((strn_length % 2) == 0)
 	{
-		while (data_length--)
+		for (size_t i = 0; i < strn_length; i += 2)
 		{
-			tmp_buf[0] = *hex_str++;
-			tmp_buf[1] = *hex_str++;
-
-			*data++ = static_cast<u8>(hex_to_u64(tmp_buf) & 0xFF);
+			const auto [ptr, err] = std::from_chars(hex_str.data() + i, hex_str.data() + i + 2, *data++, 16);
+			if (err != std::errc())
+			{
+				fmt::throw_exception("Failed to read hex string: %s", std::make_error_code(err).message());
+			}
 		}
 	}
 }
 
-
 // Crypto functions (AES128-CBC, AES128-ECB, SHA1-HMAC and AES-CMAC).
 void aescbc128_decrypt(unsigned char *key, unsigned char *iv, unsigned char *in, unsigned char *out, usz len)
 {

rpcs3/Crypto/utils.h

@@ -6,7 +6,8 @@
 
 #include "util/types.hpp"
 
-#include <stdlib.h>
+#include <cstdlib>
+#include <string_view>
 
 enum { CRYPTO_MAX_PATH = 4096 };
 
@@ -15,8 +16,7 @@ char* extract_file_name(const char* file_path, char real_file_name[CRYPTO_MAX_PA
 std::string sha256_get_hash(const char* data, usz size, bool lower_case);
 
 // Hex string conversion auxiliary functions.
-u64 hex_to_u64(const char* hex_str);
-void hex_to_bytes(unsigned char *data, const char *hex_str, unsigned int str_length);
+void hex_to_bytes(unsigned char* data, std::string_view hex_str, unsigned int str_length);
 
 // Crypto functions (AES128-CBC, AES128-ECB, SHA1-HMAC and AES-CMAC).
 void aescbc128_decrypt(unsigned char *key, unsigned char *iv, unsigned char *in, unsigned char *out, usz len);

rpcs3/Emu/Cell/Modules/cellSpurs.cpp

@@ -1230,7 +1230,7 @@ s32 _spurs::initialize(ppu_thread& ppu, vm::ptr<CellSpurs> spurs, u32 revision,
 	if (flags & SAF_UNKNOWN_FLAG_9)          spuTgAttr->type |= 0x0800;
 	if (flags & SAF_SYSTEM_WORKLOAD_ENABLED) spuTgAttr->type |= SYS_SPU_THREAD_GROUP_TYPE_COOPERATE_WITH_SYSTEM;
 
-	if (s32 rc = sys_spu_thread_group_create(ppu, spurs.ptr(&CellSpurs::spuTG), nSpus, spuPriority, spuTgAttr))
+	if (s32 rc = sys_spu_thread_group_create(ppu, spurs.ptr(&CellSpurs::spuTG), nSpus, spuPriority, vm::unsafe_ptr_cast<reduced_sys_spu_thread_group_attribute>(spuTgAttr)))
 	{
 		ppu_execute<&sys_spu_image_close>(ppu, spurs.ptr(&CellSpurs::spuImg));
 		return rollback(), rc;

rpcs3/Emu/Cell/SPUThread.cpp

@@ -303,6 +303,33 @@ static FORCE_INLINE void mov_rdata_avx(__m256i* dst, const __m256i* src)
 }
 #endif
 
+// Check if only a single 16-bytes block has changed
+// Returning its position, or -1 if that is not the situation
+static inline usz scan16_rdata(const decltype(spu_thread::rdata)& _lhs, const decltype(spu_thread::rdata)& _rhs)
+{
+	const auto lhs = reinterpret_cast<const v128*>(_lhs);
+	const auto rhs = reinterpret_cast<const v128*>(_rhs);
+
+	u32 mask = 0;
+
+	for (usz i = 0; i < 8; i += 4)
+	{
+		const u32 a = (lhs[i + 0] != rhs[i + 0]) ? 1 : 0;
+		const u32 b = (lhs[i + 1] != rhs[i + 1]) ? 1 : 0;
+		const u32 c = (lhs[i + 2] != rhs[i + 2]) ? 1 : 0;
+		const u32 d = (lhs[i + 3] != rhs[i + 3]) ? 1 : 0;
+
+		mask |= ((a << 0) + (b << 1) + (c << 2) + (d << 3)) << i;
+	}
+
+	if (mask && (mask & (mask - 1)) == 0)
+	{
+		return std::countr_zero(mask);
+	}
+
+	return umax;
+}
+
 #ifdef _MSC_VER
 __forceinline
 #endif
@@ -3854,6 +3881,11 @@ bool spu_thread::do_putllc(const spu_mfc_cmd& args)
 			return false;
 		}
 
+		static const auto cast_as = [](void* ptr, usz pos){ return reinterpret_cast<u128*>(ptr) + pos; };
+		static const auto cast_as_const = [](const void* ptr, usz pos){ return reinterpret_cast<const u128*>(ptr) + pos; };
+
+		const usz diff16_pos = scan16_rdata(to_write, rdata);
+
 		auto [_oldd, _ok] = res.fetch_op([&](u64& r)
 		{
 			if ((r & -128) != rtime || (r & 127))
@@ -3975,8 +4007,19 @@ bool spu_thread::do_putllc(const spu_mfc_cmd& args)
 
 			if (cmp_rdata(rdata, super_data))
 			{
-				mov_rdata(super_data, to_write);
-				return true;
+				if (diff16_pos != umax)
+				{
+					// Do it with CMPXCHG16B if possible, this allows to improve accuracy whenever "RSX Accurate Reservations" is off 
+					if (atomic_storage<u128>::compare_exchange(*cast_as(super_data, diff16_pos), *cast_as(rdata, diff16_pos), *cast_as_const(to_write, diff16_pos)))
+					{
+						return true;
+					}
+				}
+				else
+				{
+					mov_rdata(super_data, to_write);
+					return true;
+				}
 			}
 
 			return false;

rpcs3/Emu/Cell/lv2/sys_ppu_thread.cpp

@@ -468,11 +468,11 @@ error_code sys_ppu_thread_restart(ppu_thread& ppu)
 	return CELL_OK;
 }
 
-error_code _sys_ppu_thread_create(ppu_thread& ppu, vm::ptr<u64> thread_id, vm::ptr<ppu_thread_param_t> param, u64 arg, u64 unk, s32 prio, u32 _stacksz, u64 flags, vm::cptr<char> threadname)
+error_code _sys_ppu_thread_create(ppu_thread& ppu, vm::ptr<u64> thread_id, vm::ptr<ppu_thread_param_t> param, u64 arg, u64 unk, s32 prio, u64 _stacksz, u64 flags, vm::cptr<char> threadname)
 {
 	ppu.state += cpu_flag::wait;
 
-	sys_ppu_thread.warning("_sys_ppu_thread_create(thread_id=*0x%x, param=*0x%x, arg=0x%llx, unk=0x%llx, prio=%d, stacksize=0x%x, flags=0x%llx, threadname=*0x%x)",
+	sys_ppu_thread.warning("_sys_ppu_thread_create(thread_id=*0x%x, param=*0x%x, arg=0x%llx, unk=0x%llx, prio=%d, stacksize=0x%llx, flags=0x%llx, threadname=*0x%x)",
 		thread_id, param, arg, unk, prio, _stacksz, flags, threadname);
 
 	// thread_id is checked for null in stub -> CELL_ENOMEM
@@ -497,7 +497,8 @@ error_code _sys_ppu_thread_create(ppu_thread& ppu, vm::ptr<u64> thread_id, vm::p
 	const u32 tls = param->tls;
 
 	// Compute actual stack size and allocate
-	const u32 stack_size = utils::align<u32>(std::max<u32>(_stacksz, 4096), 4096);
+	// 0 and UINT64_MAX both convert to 4096
+	const u64 stack_size = FN(x ? x : 4096)(utils::align<u64>(_stacksz, 4096));
 
 	auto& dct = g_fxo->get<lv2_memory_container>();
 
@@ -507,7 +508,7 @@ error_code _sys_ppu_thread_create(ppu_thread& ppu, vm::ptr<u64> thread_id, vm::p
 		return {CELL_ENOMEM, dct.size - dct.used};
 	}
 
-	const vm::addr_t stack_base{vm::alloc(stack_size, vm::stack, 4096)};
+	const vm::addr_t stack_base{vm::alloc(static_cast<u32>(stack_size), vm::stack, 4096)};
 
 	if (!stack_base)
 	{
@@ -532,7 +533,7 @@ error_code _sys_ppu_thread_create(ppu_thread& ppu, vm::ptr<u64> thread_id, vm::p
 	{
 		ppu_thread_params p;
 		p.stack_addr = stack_base;
-		p.stack_size = stack_size;
+		p.stack_size = static_cast<u32>(stack_size);
 		p.tls_addr = tls;
 		p.entry = entry;
 		p.arg0 = arg;

rpcs3/Emu/Cell/lv2/sys_ppu_thread.h

@@ -53,7 +53,7 @@ error_code sys_ppu_thread_get_priority(ppu_thread& ppu, u32 thread_id, vm::ptr<s
 error_code sys_ppu_thread_get_stack_information(ppu_thread& ppu, vm::ptr<sys_ppu_thread_stack_t> sp);
 error_code sys_ppu_thread_stop(ppu_thread& ppu, u32 thread_id);
 error_code sys_ppu_thread_restart(ppu_thread& ppu);
-error_code _sys_ppu_thread_create(ppu_thread& ppu, vm::ptr<u64> thread_id, vm::ptr<ppu_thread_param_t> param, u64 arg, u64 arg4, s32 prio, u32 stacksize, u64 flags, vm::cptr<char> threadname);
+error_code _sys_ppu_thread_create(ppu_thread& ppu, vm::ptr<u64> thread_id, vm::ptr<ppu_thread_param_t> param, u64 arg, u64 arg4, s32 prio, u64 stacksize, u64 flags, vm::cptr<char> threadname);
 error_code sys_ppu_thread_start(ppu_thread& ppu, u32 thread_id);
 error_code sys_ppu_thread_rename(ppu_thread& ppu, u32 thread_id, vm::cptr<char> name);
 error_code sys_ppu_thread_recover_page_fault(ppu_thread& ppu, u32 thread_id);

rpcs3/Emu/Cell/lv2/sys_spu.cpp

@@ -767,7 +767,12 @@ error_code sys_spu_thread_initialize(ppu_thread& ppu, vm::ptr<u32> thread, u32 g
 	}
 
 	// Read thread name
-	const std::string thread_name(attr_data.name.get_ptr(), std::max<u32>(attr_data.name_len, 1) - 1);
+	std::string thread_name;
+
+	if (attr_data.name_len && !vm::read_string(attr_data.name.addr(), attr_data.name_len - 1, thread_name, true))
+	{
+		return { CELL_EFAULT, attr_data.name.addr() };
+	}
 
 	const auto group = idm::get_unlocked<lv2_spu_group>(group_id);
 
@@ -906,21 +911,40 @@ error_code sys_spu_thread_get_exit_status(ppu_thread& ppu, u32 id, vm::ptr<s32>
 	return CELL_ESTAT;
 }
 
-error_code sys_spu_thread_group_create(ppu_thread& ppu, vm::ptr<u32> id, u32 num, s32 prio, vm::ptr<sys_spu_thread_group_attribute> attr)
+error_code sys_spu_thread_group_create(ppu_thread& ppu, vm::ptr<u32> id, u32 num, s32 prio, vm::ptr<reduced_sys_spu_thread_group_attribute> attr)
 {
 	ppu.state += cpu_flag::wait;
 
 	sys_spu.warning("sys_spu_thread_group_create(id=*0x%x, num=%d, prio=%d, attr=*0x%x)", id, num, prio, attr);
 
 	const s32 min_prio = g_ps3_process_info.has_root_perm() ? 0 : 16;
 
-	const sys_spu_thread_group_attribute attr_data = *attr;
+	sys_spu_thread_group_attribute attr_data{};
+	{
+		const reduced_sys_spu_thread_group_attribute attr_reduced = *attr;
+		attr_data.name = attr_reduced.name;
+		attr_data.nsize = attr_reduced.nsize;
+		attr_data.type = attr_reduced.type;
+
+		// Read container-id member at offset 12 bytes conditionally (that's what LV2 does)
+		if (attr_data.type & SYS_SPU_THREAD_GROUP_TYPE_MEMORY_FROM_CONTAINER)
+		{
+			attr_data.ct = vm::unsafe_ptr_cast<sys_spu_thread_group_attribute>(attr)->ct;
+		}
+	}
 
 	if (attr_data.nsize > 0x80 || !num)
 	{
 		return CELL_EINVAL;
 	}
 
+	std::string group_name;
+
+	if (attr_data.nsize && !vm::read_string(attr_data.name.addr(), attr_data.nsize - 1, group_name, true))
+	{
+		return { CELL_EFAULT, attr_data.name.addr() };
+	}
+
 	const s32 type = attr_data.type;
 
 	bool use_scheduler = true;
@@ -1075,7 +1099,7 @@ error_code sys_spu_thread_group_create(ppu_thread& ppu, vm::ptr<u32> id, u32 num
 		return CELL_EBUSY;
 	}
 
-	const auto group = idm::make_ptr<lv2_spu_group>(std::string(attr_data.name.get_ptr(), std::max<u32>(attr_data.nsize, 1) - 1), num, prio, type, ct, use_scheduler, mem_size);
+	const auto group = idm::make_ptr<lv2_spu_group>(std::move(group_name), num, prio, type, ct, use_scheduler, mem_size);
 
 	if (!group)
 	{