Reduce NRP RenderGraph compiler memory usage

This PR fixes a memory usage regression reported in https://jira.unity3d.com/browse/UUM-60476.

The PR that caused the regression was RG Compilation Caching https://github.cds.internal.unity3d.com/unity/unity/pull/40721, which for NRP RG Compiler creates 20 instances of `CompilerContextData` ([see code](https://github.cds.internal.unity3d.com/unity/unity/blob/trunk/Packages/com.unity.render-pipelines.core/Runtime/RenderGraph/RenderGraphCompilationCache.cs#L36-L43)) which in its constructor reserves native list capacity according to worst case, resulting in many megabytes of reserved memory. This multiplied by 20 causes tens of MBs of upfront memory allocation whenever RG Compilation Caching is enabled (which it is by default).

The fix removes the upfront allocation from `ResourcesData` constructor based on a couple of observations:
- I only touched `ResourcesData` because by my count, `ResourcesData.readerData` accounts for ~90% of all memory allocated in `CompilerContextData` constructor. 
- This change should be 100% safe & correct, because  `ResourcesData.Initialize()` is called as the first step of NRP Compilation (by `NativePassCompiler.SetupContextData`), and that function resizes each array to the size it needs to be.
- As a side effect of above, it is not required to estimate the amount of resources, since it is known by the time `Initialize` is called. This reduces unnecessary allocation of unused memory.
- These are native memory allocations and do not generate garbage. When a new graph hash is encountered (meaning we must compile a graph), we take a new `CompilerContextData` from the pool and the resourceData arrays are allocated at that point. This is expected to have some small overhead, but likely negligible compared to the overhead already incurred by having to compile the graph.
- In addition, it's good to be aware that `NativeList` always rounds requested capacity to the next power-of-two (`newCapacity = math.ceilpow2(newCapacity)`). So if you resize to 100000 elements, it will actually reserve memory for 131072 elements.

Note that memory usage can clearly be improved further - I want to make the low-risk simple fix first and improve upon it afterwards.

Author: arttu-peltonen

Packages/com.unity.render-pipelines.core/Runtime/RenderGraph/Compiler/CompilerContextData.cs

@@ -44,7 +44,7 @@ internal static int LastIndex<T>(this ref NativeList<T> list) where T : unmanage
     // Datastructure that contains passes and dependencies and allow you to iterate and reason on them more like a graph
     internal class CompilerContextData : IDisposable, RenderGraph.ICompiledGraph
     {
-        public CompilerContextData(int estimatedNumPasses, int estimatedNumResourcesPerType)
+        public CompilerContextData(int estimatedNumPasses)
         {
             passData = new NativeList<PassData>(estimatedNumPasses, AllocatorManager.Persistent);
             fences = new Dictionary<int, GraphicsFence>();
@@ -53,7 +53,7 @@ public CompilerContextData(int estimatedNumPasses, int estimatedNumResourcesPerT
             outputData = new NativeList<PassOutputData>(estimatedNumPasses * 2, AllocatorManager.Persistent);
             fragmentData = new NativeList<PassFragmentData>(estimatedNumPasses * 4, AllocatorManager.Persistent);
             randomAccessResourceData = new NativeList<PassRandomWriteData>(4, AllocatorManager.Persistent); // We assume not a lot of passes use random write
-            resources = new ResourcesData(estimatedNumResourcesPerType);
+            resources = new ResourcesData();
             nativePassData = new NativeList<NativePassData>(estimatedNumPasses, AllocatorManager.Persistent);// assume nothing gets merged
             nativeSubPassData = new NativeList<SubPassDescriptor>(estimatedNumPasses, AllocatorManager.Persistent);// there should "never" be more subpasses than graph passes
             createData = new NativeList<ResourceHandle>(estimatedNumPasses * 2, AllocatorManager.Persistent);    // assume every pass creates two resources
@@ -183,7 +183,7 @@ public bool AddToFragmentList(TextureAccess access, int listFirstIndex, int numI
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public string GetResourceVersionedName(ResourceHandle h) => GetResourceName(h) + " V" + h.version;
-        
+
         // resources can be added as fragment both as input and output so make sure not to add them twice (return true upon new addition)
         public bool AddToRandomAccessResourceList(ResourceHandle h, int randomWriteSlotIndex, bool preserveCounterValue, int listFirstIndex, int numItems)
         {

Packages/com.unity.render-pipelines.core/Runtime/RenderGraph/Compiler/NativePassCompiler.cs

@@ -26,15 +26,14 @@ internal struct RenderGraphInputInfo
         RenderGraphCompilationCache m_CompilationCache;
 
         internal const int k_EstimatedPassCount = 100;
-        internal const int k_EstimatedResourceCountPerType = 50;
         internal const int k_MaxSubpass = 8; // Needs to match with RenderPassSetup.h
 
         NativeList<AttachmentDescriptor> m_BeginRenderPassAttachments;
 
         public NativePassCompiler(RenderGraphCompilationCache cache)
         {
             m_CompilationCache = cache;
-            defaultContextData = new CompilerContextData(k_EstimatedPassCount, k_EstimatedResourceCountPerType);
+            defaultContextData = new CompilerContextData(k_EstimatedPassCount);
             toVisitPassIds = new Stack<int>(k_EstimatedPassCount);
             m_BeginRenderPassAttachments = new NativeList<AttachmentDescriptor>(FixedAttachmentArray<AttachmentDescriptor>.MaxAttachments, Allocator.Persistent);
         }

Packages/com.unity.render-pipelines.core/Runtime/RenderGraph/Compiler/ResourcesData.cs

@@ -198,7 +198,7 @@ internal class ResourcesData
 
         public DynamicArray<Name>[] resourceNames;
 
-        public ResourcesData(int estimatedNumResourcesPerType)
+        public ResourcesData()
         {
             unversionedData = new NativeList<ResourceUnversionedData>[(int)RenderGraphResourceType.Count];
             versionedData = new NativeList<ResourceVersionedData>[(int)RenderGraphResourceType.Count];
@@ -207,10 +207,12 @@ public ResourcesData(int estimatedNumResourcesPerType)
 
             for (int t = 0; t < (int)RenderGraphResourceType.Count; t++)
             {
-                versionedData[t] = new NativeList<ResourceVersionedData>(MaxVersions * estimatedNumResourcesPerType, AllocatorManager.Persistent);
-                unversionedData[t] = new NativeList<ResourceUnversionedData>(estimatedNumResourcesPerType, AllocatorManager.Persistent);
-                readerData[t] = new NativeList<ResourceReaderData>(MaxVersions * estimatedNumResourcesPerType * MaxReaders, AllocatorManager.Persistent);
-                resourceNames[t] = new DynamicArray<Name>(estimatedNumResourcesPerType); // T in NativeList<T> cannot contain managed types, so the names are stored separately
+                // Note: All these lists are allocated with zero capacity, they will be resized in Initialize when
+                // the amount of resources is known.
+                versionedData[t] = new NativeList<ResourceVersionedData>(0, AllocatorManager.Persistent);
+                unversionedData[t] = new NativeList<ResourceUnversionedData>(0, AllocatorManager.Persistent);
+                readerData[t] = new NativeList<ResourceReaderData>(0, AllocatorManager.Persistent);
+                resourceNames[t] = new DynamicArray<Name>(0); // T in NativeList<T> cannot contain managed types, so the names are stored separately
             }
         }
 

Packages/com.unity.render-pipelines.core/Runtime/RenderGraph/RenderGraphCompilationCache.cs

@@ -38,7 +38,7 @@ public RenderGraphCompilationCache()
         for (int i = 0; i < k_CachedGraphCount; ++i)
         {
             m_CompiledGraphPool.Push(new RenderGraph.CompiledGraph());
-            m_NativeCompiledGraphPool.Push(new CompilerContextData(NativePassCompiler.k_EstimatedPassCount, NativePassCompiler.k_EstimatedResourceCountPerType));
+            m_NativeCompiledGraphPool.Push(new CompilerContextData(NativePassCompiler.k_EstimatedPassCount));
         }
     }