fix: color write mask and unpackHalf2x16

Author: SamoZ256

src/Cafe/HW/Latte/Core/LatteIndices.cpp

@@ -287,15 +287,15 @@ void LatteIndices_generateAutoLineLoopIndices(void* indexDataOutput, uint32 coun
 template<typename T>
 void LatteIndices_unpackTriangleFanAndConvert(const void* indexDataInput, void* indexDataOutput, uint32 count, uint32& indexMin, uint32& indexMax)
 {
-    debug_printf("TRIANGLE FAN UNPACK\n");
+    debug_printf("TRIANGLE FAN UNPACK %u\n", rand());
 	const betype<T>* src = (betype<T>*)indexDataInput;
 	T* dst = (T*)indexDataOutput;
 	// TODO: check this
 	for (sint32 i = 0; i < count; i++)
 	{
 	    uint32 i0;
 		if (i % 2 == 0)
-		i0 = i / 2;
+		    i0 = i / 2;
         else
             i0 = count - 1 - i / 2;
         T idx = src[i0];
@@ -308,7 +308,7 @@ void LatteIndices_unpackTriangleFanAndConvert(const void* indexDataInput, void*
 template<typename T>
 void LatteIndices_generateAutoTriangleFanIndices(const void* indexDataInput, void* indexDataOutput, uint32 count, uint32& indexMin, uint32& indexMax)
 {
-    debug_printf("TRIANGLE FAN AUTO\n");
+    debug_printf("TRIANGLE FAN AUTO %u\n", rand());
 	const betype<T>* src = (betype<T>*)indexDataInput;
 	T* dst = (T*)indexDataOutput;
 	for (sint32 i = 0; i < count; i++)

src/Cafe/HW/Latte/LegacyShaderDecompiler/LatteDecompilerEmitMSL.cpp

@@ -3734,8 +3734,9 @@ void LatteDecompiler_emitHelperFunctions(LatteDecompilerShaderContext* shaderCon
 
 	// unpackHalf2x16
 	fCStr_shaderSource->add(""
-	"float2 unpackHalf2x16(float x) {\r\n"
-		"return float2(as_type<half>(ushort(as_type<uint>(x) & 0x00FF)), as_type<half>(ushort((as_type<uint>(x) & 0xFF00) >> 16)));\r\n"
+	"template<typename T>\r\n"
+	"float2 unpackHalf2x16(T x) {\r\n"
+		"return float2(as_type<half2>(x));\r\n"
 	"}\r\n");
 
 	// Bit cast

src/Cafe/HW/Latte/Renderer/Metal/LatteToMtl.cpp

@@ -2,6 +2,7 @@
 #include "Common/precompiled.h"
 #include "Metal/MTLDepthStencil.hpp"
 #include "Metal/MTLRenderCommandEncoder.hpp"
+#include "Metal/MTLRenderPipeline.hpp"
 #include "Metal/MTLSampler.hpp"
 
 std::map<Latte::E_GX2SURFFMT, MetalPixelFormatInfo> MTL_COLOR_FORMAT_TABLE = {
@@ -469,3 +470,14 @@ MTL::StencilOperation GetMtlStencilOp(Latte::LATTE_DB_DEPTH_CONTROL::E_STENCILAC
     cemu_assert_debug((uint32)action < std::size(MTL_STENCIL_OPERATIONS));
     return MTL_STENCIL_OPERATIONS[(uint32)action];
 }
+
+MTL::ColorWriteMask GetMtlColorWriteMask(uint8 mask)
+{
+    MTL::ColorWriteMask mtlMask = MTL::ColorWriteMaskNone;
+    if (mask & 0x1) mtlMask |= MTL::ColorWriteMaskRed;
+    if (mask & 0x2) mtlMask |= MTL::ColorWriteMaskGreen;
+    if (mask & 0x4) mtlMask |= MTL::ColorWriteMaskBlue;
+    if (mask & 0x8) mtlMask |= MTL::ColorWriteMaskAlpha;
+
+    return mtlMask;
+}

src/Cafe/HW/Latte/Renderer/Metal/LatteToMtl.h

@@ -7,6 +7,7 @@
 //#include "Cafe/HW/Latte/Core/FetchShader.h"
 #include "Cafe/HW/Latte/Renderer/Renderer.h"
 #include "Metal/MTLDepthStencil.hpp"
+#include "Metal/MTLRenderPipeline.hpp"
 #include "Metal/MTLSampler.hpp"
 #include "Metal/MTLTexture.hpp"
 
@@ -55,3 +56,5 @@ MTL::SamplerAddressMode GetMtlSamplerAddressMode(Latte::LATTE_SQ_TEX_SAMPLER_WOR
 MTL::TextureSwizzle GetMtlTextureSwizzle(uint32 swizzle);
 
 MTL::StencilOperation GetMtlStencilOp(Latte::LATTE_DB_DEPTH_CONTROL::E_STENCILACTION action);
+
+MTL::ColorWriteMask GetMtlColorWriteMask(uint8 mask);

src/Cafe/HW/Latte/Renderer/Metal/MetalPipelineCache.cpp

@@ -92,6 +92,11 @@ MTL::RenderPipelineState* MetalPipelineCache::GetPipelineState(const LatteFetchS
 	desc->setFragmentFunction(mtlPixelShader->GetFunction());
 	// TODO: don't always set the vertex descriptor?
 	desc->setVertexDescriptor(vertexDescriptor);
+
+	// Color attachments
+	const Latte::LATTE_CB_COLOR_CONTROL& colorControlReg = LatteGPUState.contextNew.CB_COLOR_CONTROL;
+	uint32 blendEnableMask = colorControlReg.get_BLEND_MASK();
+	uint32 renderTargetMask = LatteGPUState.contextNew.CB_TARGET_MASK.get_MASK();
 	for (uint8 i = 0; i < 8; i++)
 	{
 	    const auto& colorBuffer = activeFBO->colorBuffer[i];
@@ -102,12 +107,9 @@ MTL::RenderPipelineState* MetalPipelineCache::GetPipelineState(const LatteFetchS
 		}
 		auto colorAttachment = desc->colorAttachments()->object(i);
 		colorAttachment->setPixelFormat(texture->GetRGBAView()->pixelFormat());
+		colorAttachment->setWriteMask(GetMtlColorWriteMask((renderTargetMask >> (i * 4)) & 0xF));
 
 		// Blending
-		const Latte::LATTE_CB_COLOR_CONTROL& colorControlReg = LatteGPUState.contextNew.CB_COLOR_CONTROL;
-		uint32 blendEnableMask = colorControlReg.get_BLEND_MASK();
-		uint32 renderTargetMask = LatteGPUState.contextNew.CB_TARGET_MASK.get_MASK();
-
 		bool blendEnabled = ((blendEnableMask & (1 << i))) != 0;
 		// Only float data type is blendable
 		if (blendEnabled && GetMtlPixelFormatInfo(texture->format, false).dataType == MetalDataType::FLOAT)
@@ -120,7 +122,6 @@ MTL::RenderPipelineState* MetalPipelineCache::GetPipelineState(const LatteFetchS
     		auto srcRgbBlendFactor = GetMtlBlendFactor(blendControlReg.get_COLOR_SRCBLEND());
     		auto dstRgbBlendFactor = GetMtlBlendFactor(blendControlReg.get_COLOR_DSTBLEND());
 
-    		colorAttachment->setWriteMask((renderTargetMask >> (i * 4)) & 0xF);
     		colorAttachment->setRgbBlendOperation(rgbBlendOp);
     		colorAttachment->setSourceRGBBlendFactor(srcRgbBlendFactor);
     		colorAttachment->setDestinationRGBBlendFactor(dstRgbBlendFactor);
@@ -138,6 +139,8 @@ MTL::RenderPipelineState* MetalPipelineCache::GetPipelineState(const LatteFetchS
     		}
 		}
 	}
+
+	// Depth stencil attachment
 	if (activeFBO->depthBuffer.texture)
 	{
 	    auto texture = static_cast<LatteTextureViewMtl*>(activeFBO->depthBuffer.texture);

src/Cafe/HW/Latte/Renderer/Metal/MetalRenderer.cpp

@@ -79,7 +79,7 @@ void MetalRenderer::InitializeLayer(const Vector2i& size, bool mainWindow)
     m_metalLayer->setDevice(m_device);
     // TODO: don't always force sRGB
     // TODO: shouldn't this be handled differently?
-    m_metalLayer->setPixelFormat(MTL::PixelFormatRGBA8Unorm_sRGB);
+    m_metalLayer->setPixelFormat(MTL::PixelFormatRGBA8Unorm/*_sRGB*/);
 
     // Present pipeline
     NS::Error* error = nullptr;