Merge pull request Field-Robotics-Japan#205 from hijimasa/feature/supportNativeRenderGraphCompiler

Fix Unity 23.3+ RenderGraph depth buffer requirement

Author: RyodoTanaka

Packages/UnitySensors/Runtime/Scripts/Sensors/Camera/DepthCamera/DepthCameraSensor.cs

@@ -13,8 +13,51 @@
 using System.Collections;
 using UnitySensors.Utils.Texture;
 
+#if UNITY_6000_0_OR_NEWER
+using UnityEngine.Rendering;
+#endif
+
 namespace UnitySensors.Sensor.Camera
 {
+    // Job for parallel raycast depth calculation
+    public struct ParallelRaycastDepthJob : IJobParallelFor
+    {
+        [ReadOnly] public float3 cameraPosition;
+        [ReadOnly] public float3 forward;
+        [ReadOnly] public float3 right;
+        [ReadOnly] public float3 up;
+        [ReadOnly] public float tanHalfFov;
+        [ReadOnly] public float aspect;
+        [ReadOnly] public int width;
+        [ReadOnly] public int height;
+        [ReadOnly] public float farClipPlane;
+
+        [WriteOnly] public NativeArray<float> depthValues;
+
+        public void Execute(int index)
+        {
+            int x = index % width;
+            int y = index / width;
+
+            float normalizedX = (float)x / (width - 1);
+            float normalizedY = (float)y / (height - 1);
+
+            float ndcX = (2.0f * normalizedX) - 1.0f;
+            float ndcY = (2.0f * normalizedY) - 1.0f;
+
+            float viewX = ndcX * tanHalfFov * aspect;
+            float viewY = ndcY * tanHalfFov;
+
+            float3 rayDirection = math.normalize(forward + right * viewX + up * viewY);
+
+            // Note: Unity.Physics would be needed for burst-compiled raycast
+            // For now, we'll use the fallback value
+            float depth = 1.0f;
+
+            depthValues[index] = depth;
+        }
+    }
+
     [RequireComponent(typeof(UnityEngine.Camera))]
     public class DepthCameraSensor : CameraSensor, IPointCloudInterface<PointXYZ>
     {
@@ -28,7 +71,17 @@ public class DepthCameraSensor : CameraSensor, IPointCloudInterface<PointXYZ>
         private Material _depthCameraMat;
         [SerializeField]
         private bool _convertToPointCloud = false;
+
+        [Header("Performance Settings")]
+        [SerializeField, Range(0.1f, 1.0f)]
+        private float _raycastResolutionScale = 0.5f; // Reduce raycast resolution for better performance
+        [SerializeField]
+        private bool _useAdaptiveQuality = true; // Enable adaptive quality based on frame rate
+
         private TextureLoader _textureLoader;
+        private Texture2D _depthTexture; // Reuse texture to avoid allocations
+        private int _lastRaycastWidth, _lastRaycastHeight;
+        private float _lastFrameTime;
 
         private JobHandle _jobHandle;
 
@@ -52,7 +105,18 @@ protected override void Init()
             _camera.nearClipPlane = _minRange;
             _camera.farClipPlane = _maxRange;
 
+#if UNITY_6000_0_OR_NEWER
+            _rt = new RenderTexture(_resolution.x, _resolution.y, 24, RenderTextureFormat.ARGBFloat);
+            _rt.Create();
+
+            bool isURP = GraphicsSettings.currentRenderPipeline != null;
+            if (isURP)
+            {
+                Debug.Log("DepthCameraSensor: Unity 6000+ URP mode initialized");
+            }
+#else
             _rt = new RenderTexture(_resolution.x, _resolution.y, 0, RenderTextureFormat.ARGBFloat);
+#endif
             _camera.targetTexture = _rt;
 
             _texture = new Texture2D(_resolution.x, _resolution.y, TextureFormat.RGBAFloat, false);
@@ -116,7 +180,21 @@ private void SetupJob()
 
         protected override IEnumerator UpdateSensor()
         {
+#if UNITY_6000_0_OR_NEWER
+            bool isURP = GraphicsSettings.currentRenderPipeline != null;
+
+            if (isURP)
+            {
+                GenerateDepthImageUsingRaycast();
+            }
+            else
+            {
+                _camera.Render();
+            }
+#else
             _camera.Render();
+#endif
+
             yield return _textureLoader.LoadTextureAsync();
 
             if (_textureLoader.success && _convertToPointCloud)
@@ -128,6 +206,111 @@ protected override IEnumerator UpdateSensor()
             }
         }
 
+        private void GenerateDepthImageUsingRaycast()
+        {
+            // Adaptive quality adjustment based on frame rate
+            if (_useAdaptiveQuality)
+            {
+                float currentFrameTime = Time.unscaledDeltaTime;
+                if (_lastFrameTime > 0)
+                {
+                    float currentFPS = 1.0f / currentFrameTime;
+                    float targetFPS = frequency; // Use sensor frequency as target
+                    if (currentFPS < targetFPS * 0.8f) // If FPS drops below 80% of target
+                    {
+                        _raycastResolutionScale = Mathf.Max(0.1f, _raycastResolutionScale - 0.05f);
+                    }
+                    else if (currentFPS > targetFPS * 1.1f) // If FPS is above 110% of target
+                    {
+                        _raycastResolutionScale = Mathf.Min(1.0f, _raycastResolutionScale + 0.02f);
+                    }
+                }
+                _lastFrameTime = currentFrameTime;
+            }
+
+            // Calculate actual raycast resolution
+            int raycastWidth = Mathf.Max(1, Mathf.RoundToInt(_rt.width * _raycastResolutionScale));
+            int raycastHeight = Mathf.Max(1, Mathf.RoundToInt(_rt.height * _raycastResolutionScale));
+
+            // Reuse texture if possible to avoid allocations
+            if (_depthTexture == null || _lastRaycastWidth != raycastWidth || _lastRaycastHeight != raycastHeight)
+            {
+                if (_depthTexture != null)
+                    DestroyImmediate(_depthTexture);
+
+                _depthTexture = new Texture2D(raycastWidth, raycastHeight, TextureFormat.RGBAFloat, false);
+                _lastRaycastWidth = raycastWidth;
+                _lastRaycastHeight = raycastHeight;
+            }
+
+            RenderTexture.active = _rt;
+            GL.Clear(true, true, Color.white);
+            RenderTexture.active = null;
+
+            // Pre-calculate camera parameters
+            float fovRad = _camera.fieldOfView * Mathf.Deg2Rad;
+            float aspect = (float)_rt.width / _rt.height;
+            float tanHalfFov = Mathf.Tan(fovRad * 0.5f);
+
+            Vector3 cameraPos = _camera.transform.position;
+            Vector3 forward = _camera.transform.forward;
+            Vector3 right = _camera.transform.right;
+            Vector3 up = _camera.transform.up;
+
+            // Use Color32 array for better performance
+            Color32[] pixels = new Color32[raycastWidth * raycastHeight];
+
+            // Batch raycast operations
+            for (int y = 0; y < raycastHeight; y++)
+            {
+                for (int x = 0; x < raycastWidth; x++)
+                {
+                    // Map raycast coordinates to full resolution
+                    float normalizedX = (float)x / (raycastWidth - 1);
+                    float normalizedY = (float)y / (raycastHeight - 1);
+
+                    float ndcX = (2.0f * normalizedX) - 1.0f;
+                    float ndcY = (2.0f * normalizedY) - 1.0f;
+
+                    float viewX = ndcX * tanHalfFov * aspect;
+                    float viewY = ndcY * tanHalfFov;
+
+                    Vector3 rayDirection = (forward + right * viewX + up * viewY).normalized;
+                    Ray ray = new Ray(cameraPos, rayDirection);
+
+                    float depth = 1.0f;
+
+                    if (Physics.Raycast(ray, out RaycastHit hit, _camera.farClipPlane))
+                    {
+                        float distance = hit.distance;
+                        depth = Mathf.Clamp01(distance / _camera.farClipPlane);
+                    }
+
+                    byte depthByte = (byte)(depth * 255);
+                    pixels[y * raycastWidth + x] = new Color32(depthByte, depthByte, depthByte, 255);
+                }
+            }
+
+            // Apply pixels and scale to target resolution
+            _depthTexture.SetPixels32(pixels);
+            _depthTexture.Apply();
+
+            // Scale to target resolution if needed
+            if (raycastWidth != _rt.width || raycastHeight != _rt.height)
+            {
+                RenderTexture tempRT = RenderTexture.GetTemporary(_rt.width, _rt.height, 0, RenderTextureFormat.ARGBFloat);
+                Graphics.Blit(_depthTexture, tempRT);
+                Graphics.CopyTexture(tempRT, _rt);
+                RenderTexture.ReleaseTemporary(tempRT);
+            }
+            else
+            {
+                RenderTexture.active = _rt;
+                Graphics.CopyTexture(_depthTexture, _rt);
+                RenderTexture.active = null;
+            }
+        }
+
         protected override void OnSensorDestroy()
         {
             if (_convertToPointCloud)
@@ -137,12 +320,22 @@ protected override void OnSensorDestroy()
                 _noises.Dispose();
                 _directions.Dispose();
             }
+
+            // Clean up depth texture
+            if (_depthTexture != null)
+            {
+                DestroyImmediate(_depthTexture);
+                _depthTexture = null;
+            }
+
             _rt.Release();
         }
 
+#if !UNITY_6000_0_OR_NEWER
         private void OnRenderImage(RenderTexture source, RenderTexture dest)
         {
             Graphics.Blit(null, dest, _depthCameraMat);
         }
+#endif
     }
 }

Packages/UnitySensors/Runtime/Scripts/Sensors/Camera/FisheyeCamera/FisheyeCameraSensor.cs

@@ -33,11 +33,20 @@ public enum CameraModel
         protected override void Init()
         {
             base.Init();
+#if UNITY_6000_0_OR_NEWER
+            // Unity 6000+ requires depth buffer for render textures used with cameras
+            _cubemap = new RenderTexture(_cubemapResolution, _cubemapResolution, 24, RenderTextureFormat.ARGB32)
+            {
+                dimension = TextureDimension.Cube
+            };
+            _rt = new RenderTexture(_resolution.x, _resolution.y, 24, RenderTextureFormat.ARGB32);
+#else
             _cubemap = new RenderTexture(_cubemapResolution, _cubemapResolution, 0, RenderTextureFormat.ARGB32)
             {
                 dimension = TextureDimension.Cube
             };
             _rt = new RenderTexture(_resolution.x, _resolution.y, 0, RenderTextureFormat.ARGB32);
+#endif
             _texture = new Texture2D(_resolution.x, _resolution.y, TextureFormat.RGBA32, false);
             _textureLoader = new TextureLoader
             {

Packages/UnitySensors/Runtime/Scripts/Sensors/Camera/PanoramicCamera/PanoramicCameraSensor.cs

@@ -16,11 +16,20 @@ public class PanoramicCameraSensor : CameraSensor
         protected override void Init()
         {
             base.Init();
+#if UNITY_6000_0_OR_NEWER
+            // Unity 6000+ requires depth buffer for render textures used with cameras
+            _cubemap = new RenderTexture(_cubemapResolution.x, _cubemapResolution.y, 24, RenderTextureFormat.ARGB32)
+            {
+                dimension = TextureDimension.Cube
+            };
+            _rt = new RenderTexture(_resolution.x, _resolution.y, 24, RenderTextureFormat.ARGB32);
+#else
             _cubemap = new RenderTexture(_cubemapResolution.x, _cubemapResolution.y, 0, RenderTextureFormat.ARGB32)
             {
                 dimension = TextureDimension.Cube
             };
             _rt = new RenderTexture(_resolution.x, _resolution.y, 0, RenderTextureFormat.ARGB32);
+#endif
             _texture = new Texture2D(_resolution.x, _resolution.y, TextureFormat.RGBA32, false);
             _textureLoader = new TextureLoader
             {

Packages/UnitySensors/Runtime/Scripts/Sensors/Camera/RGBCamera/RGBCameraSensor.cs

@@ -10,7 +10,12 @@ public class RGBCameraSensor : CameraSensor
         protected override void Init()
         {
             base.Init();
+#if UNITY_6000_0_OR_NEWER
+            // Unity 6000+ requires depth buffer for render textures used with cameras
+            _rt = new RenderTexture(_resolution.x, _resolution.y, 24, RenderTextureFormat.ARGB32);
+#else
             _rt = new RenderTexture(_resolution.x, _resolution.y, 0, RenderTextureFormat.ARGB32);
+#endif
             _camera.targetTexture = _rt;
 
             _texture = new Texture2D(_resolution.x, _resolution.y, TextureFormat.RGBA32, false);