Optimize OnDemand shadow maps rendering

This PR is adressing 3 tickets :
- Wrong sets of culled objects are rendered with OnDemand shadow rendering https://jira.unity3d.com/browse/UUM-60823
- Expensive Culling for Non-Rendered Shadow Cascades https://jira.unity3d.com/browse/UUM-60917
- Use-after-free causing a crash under certain conditions https://jira.unity3d.com/browse/UUM-68625

The first ticket is a bug. Some shadow casters are culled when they shouldn't.

The second ticket is a performance improvement. It allows some culling jobs not to be kicked at all when no shadow map update is needed for a given light. E.g when using OnDemand or OnEnable shadow rendering mode and no update was requested. 
It is also speeding up culling jobs by skipping receiver sphere culling for the shadow cascade which do not need an update.

Implementation keypoints:
- Fix the splitIndex not being passed to ShadowDrawingSettings. Simply store the splitIndex HDShadowCullingSplit and retrieve it when building the ShadowDrawingSettings
- Refactor shadow caching code so that we know at culling time if a shadow cache will need to be updated or not
- Discard culling jobs for the lights which have no shadow maps to update at all (and does not have the draw dynamic checkbox enabled)
- Add support for a new `splitExclusionMask` API which allows SRP code to tell if culling should be skipped for some cascades (e.g. because of shadow caching)
- Use the `splitExclusionMask` value internally to skip receiver sphere culling on C++ side
- Pass the `splitExclusionMask` value to BRG OnPerformCulling callback so that EG, GRD or any BRG user code can beneficiate from it

Author: vincent-breysse

Packages/com.unity.render-pipelines.high-definition/Runtime/Lighting/Light/HDGpuLightsBuilder.LightLoop.cs

@@ -488,7 +488,6 @@ internal unsafe void UpdateShadowRequestsAndCalculateShadowIndices(
                 NativeBitArray shadowRequestValidityArray = visibleLights.shadowRequestValidityArray;
 
                 HDShadowManagerDataForShadowRequestUpateJob shadowManagerData = default;
-                shadowManagerData.cachedShadowManager.cachedDirectionalAngles = m_CachedDirectionalAnglesArray;
                 shadowManager.GetUnmanageDataForShadowRequestJobs(ref shadowManagerData);
 
                 bool usesReversedZBuffer = SystemInfo.usesReversedZBuffer;
@@ -575,8 +574,6 @@ internal unsafe void UpdateShadowRequestsAndCalculateShadowIndices(
 
                 shadowRequestsAndIndicesJob.Run();
 
-                HDCachedShadowManager.instance.SetCachedDirectionalAngles(m_CachedDirectionalAnglesArray.Value);
-
                 ref UnsafeList<ShadowRequestIntermediateUpdateData> cachedDirectionalUpdateInfos = ref *(m_CachedDirectionalUpdateInfos.GetUnsafeList());
                 int cachedDirectionalCount = cachedDirectionalUpdateInfos.Length;
                 ref UnsafeList<ShadowRequestIntermediateUpdateData> dynamicDirectionalUpdateInfos = ref *(m_DynamicDirectionalUpdateInfos.GetUnsafeList());

Packages/com.unity.render-pipelines.high-definition/Runtime/Lighting/Light/HDGpuLightsBuilder.cs

@@ -128,11 +128,6 @@ public void Cleanup()
                 m_DynamicDirectionalUpdateInfos.Dispose();
             }
 
-            if (m_CachedDirectionalAnglesArray.IsCreated)
-            {
-                m_CachedDirectionalAnglesArray.Dispose();
-            }
-
             if (m_ShadowIndicesScratchpadArray.IsCreated)
             {
                 m_ShadowIndicesScratchpadArray.Dispose();
@@ -208,7 +203,6 @@ internal enum GPULightTypeCountSlots
         private NativeList<ShadowRequestIntermediateUpdateData> m_DynamicSpotUpdateInfos = new NativeList<ShadowRequestIntermediateUpdateData>(Allocator.Persistent);
         private NativeList<ShadowRequestIntermediateUpdateData> m_DynamicAreaRectangleUpdateInfos = new NativeList<ShadowRequestIntermediateUpdateData>(Allocator.Persistent);
         private NativeList<ShadowRequestIntermediateUpdateData> m_DynamicDirectionalUpdateInfos = new NativeList<ShadowRequestIntermediateUpdateData>(Allocator.Persistent);
-        private NativeReference<Unity.Mathematics.float3> m_CachedDirectionalAnglesArray = new NativeReference<float3>(Allocator.Persistent);
 
         private void AllocateLightData(int lightCount, int directionalLightCount)
         {
@@ -266,11 +260,6 @@ private void EnsureScratchpadCapacity(int lightCount)
                 m_DynamicAreaRectangleUpdateInfos = new NativeList<ShadowRequestIntermediateUpdateData>(Allocator.Persistent);
                 m_DynamicDirectionalUpdateInfos = new NativeList<ShadowRequestIntermediateUpdateData>(Allocator.Persistent);
             }
-
-            if (!m_CachedDirectionalAnglesArray.IsCreated)
-            {
-                m_CachedDirectionalAnglesArray = new NativeReference<float3>(Allocator.Persistent);
-            }
         }
 
         #endregion

Packages/com.unity.render-pipelines.high-definition/Runtime/Lighting/Light/HDLightRenderDatabase.cs

@@ -133,6 +133,8 @@ public bool updateUponLightMovement
             set => flags[UpdateUponLightMovementFlagsIndex] = value;
         }
 
+        public bool hasCachedComponent => shadowUpdateMode != ShadowUpdateMode.EveryFrame;
+
         public void Set(HDAdditionalLightData additionalLightData)
         {
             shadowNearPlane = additionalLightData.shadowNearPlane;

Packages/com.unity.render-pipelines.high-definition/Runtime/Lighting/Light/HDProcessedVisibleLightsBuilder.cs

@@ -25,11 +25,13 @@ internal struct ShadowIndicesAndVisibleLightData
         public int dataIndex;
         public int lightIndex;
         public HDShadowRequestSetHandle shadowRequestSetHandle;
-        public LightType lightType;
         public int splitCount;
-        public int shadowRequestCount;
         public int sortKeyIndex;
-        public BitArray8 isSplitValidArray;
+        public ShadowMapUpdateType shadowUpdateType;
+        public BitArray8 isSplitValidMask;
+        public BitArray8 needCacheUpdateMask;
+
+        public bool HasShadowCacheUpToDate(int splitIndex) => shadowUpdateType == ShadowMapUpdateType.Cached && !needCacheUpdateMask[(uint)splitIndex];
     }
 
     //Class representing lights in the context of a view.

Packages/com.unity.render-pipelines.high-definition/Runtime/Lighting/Light/HDShadowRequestUpdateJob.cs

@@ -22,12 +22,13 @@ public class HDCachedShadowManager
 
         // Data for cached directional light shadows.
         private const int m_MaxShadowCascades = 4;
-        private BitArray8 directionalShadowPendingUpdate;
-        private BitArray8 directionalShadowHasRendered;
-        private Vector3 m_CachedDirectionalForward;
-        private float3 m_CachedDirectionalAngles;
+        private BitArray8 m_DirectionalShadowHasRendered;
+        private BitArray8 m_DirectionalShadowPendingUpdate;
         private bool m_AllowDirectionalMixedCached = false;
 
+        internal BitArray8 directionalShadowPendingUpdate => m_DirectionalShadowPendingUpdate;
+        internal float3 cachedDirectionalAngles;
+
         internal const int k_MinSlotSize = 64;
 
         // Helper array used to check what has been tmp filled.
@@ -193,7 +194,7 @@ public bool LightHasBeenPlaceAndRenderedAtLeastOnce(HDAdditionalLightData lightD
                 bool hasRendered = true;
                 for (int i = 0; i < numberOfCascades; ++i)
                 {
-                    hasRendered = hasRendered && directionalShadowHasRendered[(uint)i];
+                    hasRendered = hasRendered && m_DirectionalShadowHasRendered[(uint)i];
                 }
                 return !lightData.ShadowIsUpdatedEveryFrame() && hasRendered;
             }
@@ -226,7 +227,7 @@ public bool ShadowHasBeenPlaceAndRenderedAtLeastOnce(HDAdditionalLightData light
             if (lightType == LightType.Directional)
             {
                 Debug.Assert(shadowIndex < m_MaxShadowCascades, "Shadow Index is bigger than the maximum cascades allowed");
-                return !lightData.ShadowIsUpdatedEveryFrame() && directionalShadowHasRendered[(uint)shadowIndex];
+                return !lightData.ShadowIsUpdatedEveryFrame() && m_DirectionalShadowHasRendered[(uint)shadowIndex];
             }
 
             return false;
@@ -238,8 +239,8 @@ private void MarkAllDirectionalShadowsForUpdate()
         {
             for (int i = 0; i < m_MaxShadowCascades; ++i)
             {
-                directionalShadowPendingUpdate[(uint)i] = true;
-                directionalShadowHasRendered[(uint)i] = false;
+                m_DirectionalShadowPendingUpdate[(uint)i] = true;
+                m_DirectionalShadowHasRendered[(uint)i] = false;
             }
         }
 
@@ -289,11 +290,6 @@ internal bool DirectionalHasCachedAtlas()
             return m_AllowDirectionalMixedCached;
         }
 
-        internal void SetCachedDirectionalAngles(float3 angles)
-        {
-            m_CachedDirectionalAngles = angles;
-        }
-
         internal void UpdateDirectionalCacheTexture(RenderGraph renderGraph)
         {
             TextureHandle cacheHandle = directionalLightAtlas.GetOutputTexture(renderGraph);
@@ -357,7 +353,7 @@ internal void RegisterTransformToCache(HDAdditionalLightData lightData)
             if (ShaderConfig.s_AreaLights == 1 && lightType.IsArea())
                 areaShadowAtlas.RegisterTransformCacheSlot(lightData);
             if (lightType == LightType.Directional)
-                m_CachedDirectionalAngles = lightData.transform.eulerAngles;
+                cachedDirectionalAngles = lightData.transform.eulerAngles;
         }
 
         internal void AssignSlotsInAtlases()
@@ -369,8 +365,8 @@ internal void AssignSlotsInAtlases()
 
         internal void MarkDirectionalShadowAsRendered(int shadowIdx)
         {
-            directionalShadowPendingUpdate[(uint)shadowIdx] = false;
-            directionalShadowHasRendered[(uint)shadowIdx] = true;
+            m_DirectionalShadowPendingUpdate[(uint)shadowIdx] = false;
+            m_DirectionalShadowHasRendered[(uint)shadowIdx] = true;
         }
 
         internal void OverrideShadowResolutionRequestWithCachedData(ref HDShadowResolutionRequest request, int shadowIdx, ShadowMapType shadowMapType)
@@ -420,7 +416,7 @@ internal void ScheduleShadowUpdate(HDAdditionalLightData light, int subShadowInd
             if (lightType == LightType.Directional)
             {
                 Debug.Assert(subShadowIndex < m_MaxShadowCascades);
-                directionalShadowPendingUpdate[(uint)subShadowIndex] = true;
+                m_DirectionalShadowPendingUpdate[(uint)subShadowIndex] = true;
             }
         }
 
@@ -437,15 +433,14 @@ internal bool LightIsPendingPlacement(int lightIdxForCachedShadows, ShadowMapTyp
         internal void GetUnmanagedDataForShadowRequestJobs(ref HDCachedShadowManagerDataForShadowRequestUpdateJob dataForShadowRequestUpdateJob)
         {
 
-            dataForShadowRequestUpdateJob.directionalShadowPendingUpdate = directionalShadowPendingUpdate;
+            dataForShadowRequestUpdateJob.directionalShadowPendingUpdate = m_DirectionalShadowPendingUpdate;
             punctualShadowAtlas.GetUnmanageDataForShadowRequestJobs(ref dataForShadowRequestUpdateJob.punctualShadowAtlas);
             if (ShaderConfig.s_AreaLights == 1)
                 areaShadowAtlas.GetUnmanageDataForShadowRequestJobs(ref dataForShadowRequestUpdateJob.areaShadowAtlas);
             else
                 dataForShadowRequestUpdateJob.areaShadowAtlas = emptyAreaShadowAtlasJob;
 
             dataForShadowRequestUpdateJob.directionalLightAtlas.shadowRequests = directionalLightAtlas.m_ShadowRequests;
-            dataForShadowRequestUpdateJob.cachedDirectionalAngles.Value = m_CachedDirectionalAngles;
             dataForShadowRequestUpdateJob.directionalHasCachedAtlas = DirectionalHasCachedAtlas();
         }
 

Packages/com.unity.render-pipelines.high-definition/Runtime/Lighting/Shadow/HDCachedShadowManager.cs

@@ -419,7 +419,6 @@ internal unsafe TextureHandle RenderShadowMaps(RenderGraph renderGraph, CullingR
                         {
                             ref var shadowRequest = ref requestStorageUnsafe.ElementAt(shadowRequestHandle.storageIndexForShadowRequest);
                             var commonState = CommonPerShadowRequestUpdate(ctx.cmd, data, shadowRequest, shadowRequestHandle, ref planesScratchpad, ref frustumPlanesStorageUnsafe);
-
                             if (commonState.shouldSkipRequest)
                                 continue;
 
@@ -439,6 +438,7 @@ internal unsafe TextureHandle RenderShadowMaps(RenderGraph renderGraph, CullingR
                                 CoreUtils.DrawFullScreen(ctx.cmd, data.clearMaterial, null, 0);
 
                             data.shadowDrawSettings.lightIndex = shadowRequest.lightIndex;
+                            data.shadowDrawSettings.splitIndex = shadowRequest.cullingSplit.splitIndex;
 
                             //TODO(ddebaets) as the shadowDrawSettings are modified in this loop, we generate this RL very last minute
                             // We might want to refactor this and create the RL ahead of time (especially if we ever allow AsyncPrepare on them)