SPU: Update CELL Communication Performance

rpcs3/Emu/CPU/CPUThread.cpp

@@ -867,9 +867,9 @@ bool cpu_thread::check_state() noexcept
 				store = true;
 			}
 
-			if (flags & cpu_flag::notify)
+			if (flags & (cpu_flag::notify + cpu_flag::measure))
 			{
-				flags -= cpu_flag::notify;
+				flags -= cpu_flag::notify + cpu_flag::measure;
 				store = true;
 			}
 

rpcs3/Emu/CPU/CPUThread.h

@@ -28,6 +28,7 @@ enum class cpu_flag : u32
 	notify, // Flag meant solely to allow atomic notification on state without changing other flags
 	yield, // Thread is being requested to yield its execution time if it's running
 	preempt, // Thread is being requested to preempt the execution of all CPU threads
+	measure, // Thread must remove the flag as soon as possible, no side effects
 
 	dbg_global_pause, // Emulation paused
 	dbg_pause, // Thread paused

rpcs3/Emu/Cell/Modules/cellCamera.cpp

@@ -1139,8 +1139,10 @@ error_code cellCameraGetBufferInfo(s32 dev_num, vm::ptr<CellCameraInfo> info)
 	return CELL_OK;
 }
 
-error_code cellCameraGetBufferInfoEx(s32 dev_num, vm::ptr<CellCameraInfoEx> info)
+error_code cellCameraGetBufferInfoEx(ppu_thread& ppu, s32 dev_num, vm::ptr<CellCameraInfoEx> info)
 {
+	ppu.state += cpu_flag::wait;
+
 	cellCamera.notice("cellCameraGetBufferInfoEx(dev_num=%d, info=0x%x)", dev_num, info);
 
 	// calls cellCameraGetBufferInfo
@@ -1151,10 +1153,16 @@ error_code cellCameraGetBufferInfoEx(s32 dev_num, vm::ptr<CellCameraInfoEx> info
 	}
 
 	auto& g_camera = g_fxo->get<camera_thread>();
-	std::lock_guard lock(g_camera.mutex);
 
-	*info = g_camera.info;
+	CellCameraInfoEx info_out;
+
+	{
+		std::lock_guard lock(g_camera.mutex);
+
+		info_out = g_camera.info;
+	}
 
+	*info = info_out;
 	return CELL_OK;
 }
 

rpcs3/Emu/Cell/Modules/cellGem.cpp

@@ -2284,6 +2284,8 @@ error_code cellGemClearStatusFlags(u32 gem_num, u64 mask)
 
 error_code cellGemConvertVideoFinish(ppu_thread& ppu)
 {
+	ppu.state += cpu_flag::wait;
+
 	cellGem.warning("cellGemConvertVideoFinish()");
 
 	auto& gem = g_fxo->get<gem_config>();
@@ -2306,8 +2308,10 @@ error_code cellGemConvertVideoFinish(ppu_thread& ppu)
 	return CELL_OK;
 }
 
-error_code cellGemConvertVideoStart(vm::cptr<void> video_frame)
+error_code cellGemConvertVideoStart(ppu_thread& ppu, vm::cptr<void> video_frame)
 {
+	ppu.state += cpu_flag::wait;
+
 	cellGem.warning("cellGemConvertVideoStart(video_frame=*0x%x)", video_frame);
 
 	auto& gem = g_fxo->get<gem_config>();
@@ -2459,6 +2463,8 @@ error_code cellGemEnableMagnetometer2(u32 gem_num, u32 enable)
 
 error_code cellGemEnd(ppu_thread& ppu)
 {
+	ppu.state += cpu_flag::wait;
+
 	cellGem.warning("cellGemEnd()");
 
 	auto& gem = g_fxo->get<gem_config>();
@@ -3263,15 +3269,15 @@ error_code cellGemPrepareCamera(s32 max_exposure, f32 image_quality)
 
 	extern error_code cellCameraGetAttribute(s32 dev_num, s32 attrib, vm::ptr<u32> arg1, vm::ptr<u32> arg2);
 	extern error_code cellCameraSetAttribute(s32 dev_num, s32 attrib, u32 arg1, u32 arg2);
-	extern error_code cellCameraGetBufferInfoEx(s32 dev_num, vm::ptr<CellCameraInfoEx> info);
+	extern error_code cellCameraGetBufferInfoEx(ppu_thread&, s32 dev_num, vm::ptr<CellCameraInfoEx> info);
 
 	vm::var<CellCameraInfoEx> info = vm::make_var<CellCameraInfoEx>({});
 	vm::var<u32> arg1 = vm::make_var<u32>({});
 	vm::var<u32> arg2 = vm::make_var<u32>({});
 
 	cellCameraGetAttribute(0, 0x3e6, arg1, arg2);
 	cellCameraSetAttribute(0, 0x3e6, 0x3e, *arg2 | 0x80);
-	cellCameraGetBufferInfoEx(0, info);
+	cellCameraGetBufferInfoEx(*cpu_thread::get_current<ppu_thread>(), 0, info);
 
 	if (info->width == 640)
 	{
@@ -3603,6 +3609,8 @@ error_code cellGemTrackHues(vm::cptr<u32> req_hues, vm::ptr<u32> res_hues)
 
 error_code cellGemUpdateFinish(ppu_thread& ppu)
 {
+	ppu.state += cpu_flag::wait;
+
 	cellGem.warning("cellGemUpdateFinish()");
 
 	auto& gem = g_fxo->get<gem_config>();

rpcs3/Emu/Cell/SPUThread.cpp

@@ -425,6 +425,28 @@ extern void mov_rdata_nt(spu_rdata_t& _dst, const spu_rdata_t& _src)
 #endif
 }
 
+static inline u32 cmp16_rdata(const decltype(spu_thread::rdata)& rdata, const decltype(spu_thread::rdata)& to_write)
+{
+	u32 diffcnt = 0;
+	u32 pos = 0;
+
+	for (u32 i = 0; i < 8; i++)
+	{
+		if (std::memcmp(rdata + (i * 16), to_write + (i * 16), 16))
+		{
+			diffcnt++;
+			pos = i;
+		}
+	};
+
+	if (diffcnt == 1)
+	{
+		return pos;
+	}
+
+	return -1;
+}
+
 #if defined(_MSC_VER)
 #define mwaitx_func
 #define waitpkg_func
@@ -3969,17 +3991,52 @@ bool spu_thread::do_putllc(const spu_mfc_cmd& args)
 		auto& super_data = *vm::get_super_ptr<spu_rdata_t>(addr);
 		const bool success = [&]()
 		{
-			// Full lock (heavyweight)
-			// TODO: vm::check_addr
-			vm::writer_lock lock(addr, range_lock);
-
-			if (cmp_rdata(rdata, super_data))
+			if (int pos = cmp16_rdata(rdata, to_write); addr != spurs_addr && pos != -1)
 			{
-				mov_rdata(super_data, to_write);
-				return true;
+				auto& bits = *utils::bless<atomic_t<u64>>(vm::g_reservations + ((addr & 0xff80) / 2 + 32));
+				const auto bits_val = +bits;
+
+				// Full lock (heavyweight)
+				// TODO: vm::check_addr
+				const bool halt_ppus = bits_val == umax || bits_val < 0x100000;
+
+				if (vm::writer_lock lock(addr, range_lock, halt_ppus); cmp_rdata(rdata, super_data))
+				{
+					auto cast_as = [](void* ptr, usz pos){ return reinterpret_cast<u128*>(ptr) + pos; };
+					auto cast_as_const = [](const void* ptr, usz pos){ return reinterpret_cast<const u128*>(ptr) + pos; };
+
+					if (halt_ppus)
+					{
+						*cast_as(super_data, pos) = *cast_as_const(to_write, pos);
+						bits.try_inc(u64{umax});
+						return true;
+					}
+					else if (atomic_storage<u128>::compare_exchange(*cast_as(super_data, pos), *cast_as(rdata, pos), *cast_as_const(to_write, pos)))
+					{
+						bits.try_inc(u64{umax});
+						*cast_as(rdata, pos) = *cast_as_const(to_write, pos);
+						ensure(cmp_rdata(rdata, super_data));
+						return true;
+					}
+				}
+
+				bits = u64{umax};
+				return false;
 			}
+			else
+			{
+				// Full lock (heavyweight)
+				// TODO: vm::check_addr
+				vm::writer_lock lock(addr, range_lock);
 
-			return false;
+				if (cmp_rdata(rdata, super_data))
+				{
+					mov_rdata(super_data, to_write);
+					return true;
+				}
+
+				return false;
+			}
 		}();
 
 		res += success ? 64 : 0 - 64;
@@ -4773,7 +4830,7 @@ bool spu_thread::process_mfc_cmd()
 			if (raddr != addr)
 			{
 				// Last check for event before we replace the reservation with a new one
-				if (reservation_check(raddr, rdata))
+				if (~ch_events.load().events & SPU_EVENT_LR && reservation_check(raddr, rdata, addr))
 				{
 					set_events(SPU_EVENT_LR);
 				}
@@ -5483,7 +5540,7 @@ bool spu_thread::process_mfc_cmd()
 		ch_mfc_cmd.cmd, ch_mfc_cmd.lsa, ch_mfc_cmd.eal, ch_mfc_cmd.tag, ch_mfc_cmd.size);
 }
 
-bool spu_thread::reservation_check(u32 addr, const decltype(rdata)& data) const
+bool spu_thread::reservation_check(u32 addr, const decltype(rdata)& data, u32 current_eal) const
 {
 	if (!addr)
 	{
@@ -5502,9 +5559,24 @@ bool spu_thread::reservation_check(u32 addr, const decltype(rdata)& data) const
 		return !cmp_rdata(data, *vm::get_super_ptr<decltype(rdata)>(addr));
 	}
 
+	if ((addr >> 20) == (current_eal >> 20))
+	{
+		if (vm::check_addr(addr, vm::page_1m_size))
+		{
+			// Same random-access-memory page as the current MFC command, assume allocated
+			return !cmp_rdata(data, vm::_ref<decltype(rdata)>(addr));
+		}
+
+		if ((addr >> 16) == (current_eal >> 16) && vm::check_addr(addr, vm::page_64k_size))
+		{
+			// Same random-access-memory page as the current MFC command, assume allocated
+			return !cmp_rdata(data, vm::_ref<decltype(rdata)>(addr));
+		}
+	}
+
 	// Ensure data is allocated (HACK: would raise LR event if not)
 	// Set range_lock first optimistically
-	range_lock->store(u64{128} << 32 | addr);
+	range_lock->store(u64{128} << 32 | addr | vm::range_readable);
 
 	u64 lock_val = *std::prev(std::end(vm::g_range_lock_set));
 	u64 old_lock = 0;
@@ -5580,12 +5652,12 @@ bool spu_thread::reservation_check(u32 addr, u32 hash, atomic_t<u64, 64>* range_
 	if ((addr >> 28) < 2 || (addr >> 28) == 0xd)
 	{
 		// Always-allocated memory does not need strict checking (vm::main or vm::stack)
-		return compute_rdata_hash32(*vm::get_super_ptr<decltype(rdata)>(addr)) == hash;
+		return compute_rdata_hash32(*vm::get_super_ptr<decltype(rdata)>(addr)) != hash;
 	}
 
 	// Ensure data is allocated (HACK: would raise LR event if not)
 	// Set range_lock first optimistically
-	range_lock->store(u64{128} << 32 | addr);
+	range_lock->store(u64{128} << 32 | addr | vm::range_readable);
 
 	u64 lock_val = *std::prev(std::end(vm::g_range_lock_set));
 	u64 old_lock = 0;

rpcs3/Emu/Cell/SPUThread.h

@@ -901,7 +901,8 @@ class spu_thread : public cpu_thread
 
 	// Returns true if reservation existed but was just discovered to be lost
 	// It is safe to use on any address, even if not directly accessed by SPU (so it's slower)
-	bool reservation_check(u32 addr, const decltype(rdata)& data) const;
+	// Optionally pass a known allocated address for internal optimization (the current Effective-Address of the MFC command)
+	bool reservation_check(u32 addr, const decltype(rdata)& data, u32 current_eal = 0) const;
 	static bool reservation_check(u32 addr, u32 hash, atomic_t<u64, 64>* range_lock);
 	usz register_cache_line_waiter(u32 addr);
 	void deregister_cache_line_waiter(usz index);

rpcs3/Emu/Cell/lv2/sys_memory.cpp

@@ -5,6 +5,7 @@
 #include "Emu/CPU/CPUThread.h"
 #include "Emu/Cell/ErrorCodes.h"
 #include "Emu/Cell/SPUThread.h"
+#include "Emu/Cell/PPUThread.h"
 #include "Emu/IdManager.h"
 
 #include "util/asm.hpp"
@@ -213,6 +214,8 @@ error_code sys_memory_allocate_from_container(cpu_thread& cpu, u64 size, u32 cid
 
 			if (alloc_addr)
 			{
+				sys_memory.notice("sys_memory_allocate_from_container(): Allocated 0x%x address (size=0x%x)", addr, size);
+
 				vm::lock_sudo(addr, static_cast<u32>(size));
 				cpu.check_state();
 				*alloc_addr = addr;
@@ -247,17 +250,37 @@ error_code sys_memory_free(cpu_thread& cpu, u32 addr)
 	return CELL_OK;
 }
 
-error_code sys_memory_get_page_attribute(cpu_thread& cpu, u32 addr, vm::ptr<sys_page_attr_t> attr)
+error_code sys_memory_get_page_attribute(ppu_thread& ppu, u32 addr, vm::ptr<sys_page_attr_t> attr)
 {
-	cpu.state += cpu_flag::wait;
+	ppu.state += cpu_flag::wait;
 
 	sys_memory.trace("sys_memory_get_page_attribute(addr=0x%x, attr=*0x%x)", addr, attr);
 
-	vm::writer_lock rlock;
+	if ((addr >> 28) == (ppu.stack_addr >> 28))
+	{
+		// Stack address: fast path
+		if (!(addr >= ppu.stack_addr && addr < ppu.stack_addr + ppu.stack_size) && !vm::check_addr(addr))
+		{
+			return { CELL_EINVAL, addr };
+		}
+
+		if (!vm::check_addr(attr.addr(), vm::page_readable, attr.size()))
+		{
+			return CELL_EFAULT;
+		}
+
+		attr->attribute = 0x40000ull; // SYS_MEMORY_PROT_READ_WRITEs
+		attr->access_right = SYS_MEMORY_ACCESS_RIGHT_PPU_THR;
+		attr->page_size = 4096;
+		attr->pad = 0; // Always write 0
+		return CELL_OK;
+	}
+
+	const auto [ok, vm_flags] = vm::get_addr_flags(addr);
 
-	if (!vm::check_addr(addr) || addr >= SPU_FAKE_BASE_ADDR)
+	if (!ok || addr >= SPU_FAKE_BASE_ADDR)
 	{
-		return CELL_EINVAL;
+		return { CELL_EINVAL, addr };
 	}
 
 	if (!vm::check_addr(attr.addr(), vm::page_readable, attr.size()))
@@ -266,19 +289,20 @@ error_code sys_memory_get_page_attribute(cpu_thread& cpu, u32 addr, vm::ptr<sys_
 	}
 
 	attr->attribute = 0x40000ull; // SYS_MEMORY_PROT_READ_WRITE (TODO)
-	attr->access_right = addr >> 28 == 0xdu ? SYS_MEMORY_ACCESS_RIGHT_PPU_THR : SYS_MEMORY_ACCESS_RIGHT_ANY;// (TODO)
+	attr->access_right = SYS_MEMORY_ACCESS_RIGHT_ANY; // TODO: Report accurately
 
-	if (vm::check_addr(addr, vm::page_1m_size))
+	if (vm_flags & vm::page_1m_size)
 	{
 		attr->page_size = 0x100000;
 	}
-	else if (vm::check_addr(addr, vm::page_64k_size))
+	else if (vm_flags & vm::page_64k_size)
 	{
 		attr->page_size = 0x10000;
 	}
 	else
 	{
-		attr->page_size = 4096;
+		//attr->page_size = 4096;
+		fmt::throw_exception("Unreachable");
 	}
 
 	attr->pad = 0; // Always write 0

rpcs3/Emu/Cell/lv2/sys_memory.h

@@ -4,6 +4,7 @@
 #include "Emu/Cell/ErrorCodes.h"
 
 class cpu_thread;
+class ppu_thread;
 
 enum lv2_mem_container_id : u32
 {
@@ -131,7 +132,7 @@ struct sys_memory_user_memory_stat_t
 error_code sys_memory_allocate(cpu_thread& cpu, u64 size, u64 flags, vm::ptr<u32> alloc_addr);
 error_code sys_memory_allocate_from_container(cpu_thread& cpu, u64 size, u32 cid, u64 flags, vm::ptr<u32> alloc_addr);
 error_code sys_memory_free(cpu_thread& cpu, u32 start_addr);
-error_code sys_memory_get_page_attribute(cpu_thread& cpu, u32 addr, vm::ptr<sys_page_attr_t> attr);
+error_code sys_memory_get_page_attribute(ppu_thread& cpu, u32 addr, vm::ptr<sys_page_attr_t> attr);
 error_code sys_memory_get_user_memory_size(cpu_thread& cpu, vm::ptr<sys_memory_info_t> mem_info);
 error_code sys_memory_get_user_memory_stat(cpu_thread& cpu, vm::ptr<sys_memory_user_memory_stat_t> mem_stat);
 error_code sys_memory_container_create(cpu_thread& cpu, vm::ptr<u32> cid, u64 size);