feat(taa): Overhaul TAA implementation for improved quality and upscaling (#9573)

This commit introduces a significant rework of the Temporal Anti-Aliasing (TAA) system, focusing on improving reconstruction quality, robustness, and introducing flexible upscaling.

Core TAA Algorithm improvements:
- Replaced the Catmull-Rom filter with a more efficient 5-tap Lanczos filter for history sampling, which includes deringing to reduce artifacts.
- The input color buffer is now properly "unjittered" using a Lanczos reconstruction filter.
- Improved the history rejection algorithm by skipping the expensive accurate clipping when the history sample is already within the neighborhood's color gamut.
- Added a new `hdr` option to properly handle HDR content by tonemapping colors before blending and untonemapping the result.
- Removed the ineffective `VARIANCE` only history rejection method.
- Added protection against negative numbers in `sqrt()` for increased stability.

TAA Upscaling:
- Replaced the boolean `upscaling` flag with a float factor, allowing for variable upscaling ratios (e.g., 1.5x, 2x).
- Upscaling now correctly adjusts viewport and projection settings.
- The TAA shader now receives viewport and resolution information to correctly handle upscaled rendering.

API and Configuration Changes:
- Deprecated the `filterWidth` TAA option as it no longer has an effect.
- Introduced the `upscaling` float property to `TemporalAntiAliasingOptions`.
- Added the `hdr` boolean property to `TemporalAntiAliasingOptions`.

Other Changes:
- Updated UI elements in the viewer and material sandbox to reflect the new TAA options.
- Updated Javascript bindings and TypeScript definitions for the new TAA settings.
- Refactored shader code for clarity and performance.

Author: pixelflinger

android/filament-android/src/main/java/com/google/android/filament/View.java

@@ -2041,10 +2041,6 @@ public enum BoxType {
              * use an AABB neighborhood
              */
             AABB,
-            /**
-             * use the variance of the neighborhood (not recommended)
-             */
-            VARIANCE,
             /**
              * use both AABB and variance
              */
@@ -2075,7 +2071,7 @@ public enum JitterPattern {
         }
 
         /**
-         * reconstruction filter width typically between 1 (sharper) and 2 (smoother)
+         * @deprecated has no effect.
          */
         public float filterWidth = 1.0f;
         /**
@@ -2095,9 +2091,9 @@ public enum JitterPattern {
          */
         public boolean enabled = false;
         /**
-         * 4x TAA upscaling. Disables Dynamic Resolution. [BETA]
+         * Upscaling factor. Disables Dynamic Resolution. [BETA]
          */
-        public boolean upscaling = false;
+        public float upscaling = 1.0f;
         /**
          * whether to filter the history buffer
          */
@@ -2110,6 +2106,10 @@ public enum JitterPattern {
          * whether to use the YcoCg color-space for history rejection
          */
         public boolean useYCoCg = false;
+        /**
+         * set to true for HDR content
+         */
+        public boolean hdr = true;
         /**
          * type of color gamut box
          */

filament/include/filament/Options.h

@@ -489,16 +489,15 @@ struct MultiSampleAntiAliasingOptions {
  * @see setTemporalAntiAliasingOptions()
  */
 struct TemporalAntiAliasingOptions {
-    float filterWidth = 1.0f;   //!< reconstruction filter width typically between 1 (sharper) and 2 (smoother)
+    float filterWidth = 1.0f;   //!< @deprecated has no effect.
     float feedback = 0.12f;     //!< history feedback, between 0 (maximum temporal AA) and 1 (no temporal AA).
     float lodBias = -1.0f;      //!< texturing lod bias (typically -1 or -2)
     float sharpness = 0.0f;     //!< post-TAA sharpen, especially useful when upscaling is true.
     bool enabled = false;       //!< enables or disables temporal anti-aliasing
-    bool upscaling = false;     //!< 4x TAA upscaling. Disables Dynamic Resolution. [BETA]
+    float upscaling = 1.0f;     //!< Upscaling factor. Disables Dynamic Resolution. [BETA]
 
     enum class BoxType : uint8_t {
         AABB,           //!< use an AABB neighborhood
-        VARIANCE,       //!< use the variance of the neighborhood (not recommended)
         AABB_VARIANCE   //!< use both AABB and variance
     };
 
@@ -519,6 +518,7 @@ struct TemporalAntiAliasingOptions {
     bool filterHistory = true;      //!< whether to filter the history buffer
     bool filterInput = true;        //!< whether to apply the reconstruction filter to the input
     bool useYCoCg = false;          //!< whether to use the YcoCg color-space for history rejection
+    bool hdr = true;                //!< set to true for HDR content
     BoxType boxType = BoxType::AABB;            //!< type of color gamut box
     BoxClipping boxClipping = BoxClipping::ACCURATE;     //!< clipping algorithm
     JitterPattern jitterPattern = JitterPattern::HALTON_23_X16; //! Jitter Pattern

filament/src/PostProcessManager.cpp

@@ -2911,11 +2911,12 @@ void PostProcessManager::configureTemporalAntiAliasingMaterial(
     FMaterial* const ma = getPostProcessMaterial("taa").getMaterial(mEngine);
     bool dirty = false;
 
-    setConstantParameter(ma, "upscaling", taaOptions.upscaling, dirty);
+    setConstantParameter(ma, "upscaling", taaOptions.upscaling > 1.0f, dirty);
     setConstantParameter(ma, "historyReprojection", taaOptions.historyReprojection, dirty);
     setConstantParameter(ma, "filterHistory", taaOptions.filterHistory, dirty);
     setConstantParameter(ma, "filterInput", taaOptions.filterInput, dirty);
     setConstantParameter(ma, "useYCoCg", taaOptions.useYCoCg, dirty);
+    setConstantParameter(ma, "hdr", taaOptions.hdr, dirty);
     setConstantParameter(ma, "preventFlickering", taaOptions.preventFlickering, dirty);
     setConstantParameter(ma, "boxType", int32_t(taaOptions.boxType), dirty);
     setConstantParameter(ma, "boxClipping", int32_t(taaOptions.boxClipping), dirty);
@@ -2984,6 +2985,8 @@ FMaterialInstance* PostProcessManager::configureColorGradingMaterial(
 FrameGraphId<FrameGraphTexture> PostProcessManager::taa(FrameGraph& fg,
         FrameGraphId<FrameGraphTexture> input,
         FrameGraphId<FrameGraphTexture> const depth,
+        filament::Viewport const& xvp,
+        filament::Viewport const& vp,
         FrameHistory& frameHistory,
         FrameHistoryEntry::TemporalAA FrameHistoryEntry::*pTaa,
         TemporalAntiAliasingOptions const& taaOptions,
@@ -3013,9 +3016,9 @@ FrameGraphId<FrameGraphTexture> PostProcessManager::taa(FrameGraph& fg,
     auto const& taaPass = fg.addPass<TAAData>("TAA",
             [&](FrameGraph::Builder& builder, auto& data) {
                 auto desc = fg.getDescriptor(input);
-                if (taaOptions.upscaling) {
-                    desc.width *= 2;
-                    desc.height *= 2;
+                if (taaOptions.upscaling > 1.0f) {
+                    desc.width  *= taaOptions.upscaling;
+                    desc.height *= taaOptions.upscaling;
                 }
                 data.color = builder.sample(input);
                 data.depth = builder.sample(depth);
@@ -3047,55 +3050,15 @@ FrameGraphId<FrameGraphTexture> PostProcessManager::taa(FrameGraph& fg,
                         0, 0, 0,  1
                 }};
 
-                constexpr float2 sampleOffsets[9] = {
-                        { -1.0f, -1.0f }, {  0.0f, -1.0f }, {  1.0f, -1.0f }, { -1.0f,  0.0f },
-                        {  0.0f,  0.0f },
-                        {  1.0f,  0.0f }, { -1.0f,  1.0f }, {  0.0f,  1.0f }, {  1.0f,  1.0f },
-                };
+                auto out = resources.getRenderPassInfo();
 
-                constexpr float2 subSampleOffsets[4] = {
-                        { -0.25f,  0.25f },
-                        {  0.25f,  0.25f },
-                        {  0.25f, -0.25f },
-                        { -0.25f, -0.25f }
-                };
+                auto const color = resources.getTexture(data.color);
+                auto const depth = resources.getTexture(data.depth);
+                auto const history = resources.getTexture(data.history);
 
-                UTILS_UNUSED
-                auto const lanczos = [](float const x, float const a) -> float {
-                    if (x <= std::numeric_limits<float>::epsilon()) {
-                        return 1.0f;
-                    }
-                    if (std::abs(x) <= a) {
-                        return (a * std::sin(f::PI * x) * std::sin(f::PI * x / a))
-                               / ((f::PI * f::PI) * (x * x));
-                    }
-                    return 0.0f;
-                };
+                auto const& colorDesc = resources.getDescriptor(data.color);
+                auto const& historyDesc = resources.getDescriptor(data.history);
 
-                float const filterWidth = std::clamp(taaOptions.filterWidth, 1.0f, 2.0f);
-                float4 sum = 0.0;
-                float4 weights[9];
-
-                // this doesn't get vectorized (probably because of exp()), so don't bother
-                // unrolling it.
-                UTILS_NOUNROLL
-                for (size_t i = 0; i < 9; i++) {
-                    float2 const o = sampleOffsets[i];
-                    for (size_t j = 0; j < 4; j++) {
-                        float2 const subPixelOffset = taaOptions.upscaling ? subSampleOffsets[j] : float2{ 0 };
-                        float2 const d = (o - (current.jitter - subPixelOffset)) / filterWidth;
-                        weights[i][j] = lanczos(length(d), filterWidth);
-                    }
-                    sum += weights[i];
-                }
-                for (auto& w : weights) {
-                    w /= sum;
-                }
-
-                auto out = resources.getRenderPassInfo();
-                auto color = resources.getTexture(data.color);
-                auto depth = resources.getTexture(data.depth);
-                auto history = resources.getTexture(data.history);
                 auto const& material = getPostProcessMaterial("taa");
 
                 PostProcessVariant const variant = colorGradingConfig.translucent ?
@@ -3106,17 +3069,37 @@ FrameGraphId<FrameGraphTexture> PostProcessManager::taa(FrameGraph& fg,
                 FMaterialInstance* mi = getMaterialInstance(ma);
                 mi->setParameter("color",  color, SamplerParams{});  // nearest
                 mi->setParameter("depth",  depth, SamplerParams{});  // nearest
-                mi->setParameter("alpha", taaOptions.feedback);
                 mi->setParameter("history", history, SamplerParams{
                         .filterMag = SamplerMagFilter::LINEAR,
                         .filterMin = SamplerMinFilter::LINEAR
                 });
-                mi->setParameter("filterWeights",  weights, 9);
-                mi->setParameter("jitter",  current.jitter);
+                mi->setParameter("alpha", taaOptions.feedback);
+
+                // jitter is negated here because the material wants the sample jitter offset
+                // not the projection matrix offset
+                mi->setParameter("jitter",  -current.jitter);
+                mi->setParameter("scale",  taaOptions.upscaling);
+
                 mi->setParameter("reprojection",
                         mat4f{ historyProjection * inverse(current.projection) } *
                         normalizedToClip);
 
+                mi->setParameter("colorViewport",
+                        float4{
+                            float(xvp.left)     / colorDesc.width,
+                            float(xvp.bottom)   / colorDesc.height,
+                            float(xvp.width)    / colorDesc.width,
+                            float(xvp.height)   / colorDesc.height,
+                        });
+
+                mi->setParameter("colorResolution",
+                        float4{ colorDesc.width, colorDesc.height,
+                        1.0f / colorDesc.width, 1.0f / colorDesc.height });
+
+                mi->setParameter("historyResolution",
+                        float4{ historyDesc.width, historyDesc.height,
+                        1.0f / historyDesc.width, 1.0f / historyDesc.height });
+
                 mi->commit(driver, getUboManager());
                 mi->use(driver);
 

filament/src/PostProcessManager.h

@@ -235,6 +235,8 @@ class PostProcessManager {
     FrameGraphId<FrameGraphTexture> taa(FrameGraph& fg,
             FrameGraphId<FrameGraphTexture> input,
             FrameGraphId<FrameGraphTexture> depth,
+            Viewport const& xvp,
+            Viewport const& vp,
             FrameHistory& frameHistory,
             FrameHistoryEntry::TemporalAA FrameHistoryEntry::*pTaa,
             TemporalAntiAliasingOptions const& taaOptions,

filament/src/details/Renderer.cpp

@@ -263,7 +263,9 @@ std::pair<float, float2> FRenderer::prepareUpscaler(float2 const scale,
     if (taaOptions.enabled) {
         bias += taaOptions.lodBias;
         if (taaOptions.upscaling) {
-            derivativesScale = 0.5f;
+            if (taaOptions.upscaling > 1.0f) {
+                derivativesScale = 1.0f / taaOptions.upscaling;
+            }
         }
     }
     return { bias, derivativesScale };
@@ -688,13 +690,13 @@ void FRenderer::renderJob(RootArenaScope& rootArenaScope, FView& view) {
         // This configures post-process materials by setting constant parameters
         if (taaOptions.enabled) {
             ppm.configureTemporalAntiAliasingMaterial(taaOptions);
-            if (taaOptions.upscaling) {
+            if (taaOptions.upscaling > 1.0f) {
                 // for now TAA upscaling is incompatible with regular dsr
                 dsrOptions.enabled = false;
                 // also, upscaling doesn't work well with quater-resolution SSAO
                 aoOptions.resolution = 1.0;
                 // Currently we only support a fixed TAA upscaling ratio
-                scale = 0.5f;
+                scale = 1.0f / taaOptions.upscaling;
             }
         }
     }
@@ -781,14 +783,14 @@ void FRenderer::renderJob(RootArenaScope& rootArenaScope, FView& view) {
         //       Without post-processing, we usually draw directly into
         //       the SwapChain, and we might want to keep it this way.
 
-        auto round = [](uint32_t const x) {
+        auto ceil16 = [](uint32_t const x) {
             constexpr uint32_t rounding = 16u;
             return (x + (rounding - 1u)) & ~(rounding - 1u);
         };
 
         // compute the new rendering width and height, multiple of 16.
-        const float width  = float(round(svp.width )) + 2.0f * guardBand;
-        const float height = float(round(svp.height)) + 2.0f * guardBand;
+        const float width  = float(ceil16(svp.width )) + 2.0f * guardBand;
+        const float height = float(ceil16(svp.height)) + 2.0f * guardBand;
 
         // scale the field-of-view up, so it covers exactly the extra pixels
         const float3 clipSpaceScaling{
@@ -1309,16 +1311,26 @@ void FRenderer::renderJob(RootArenaScope& rootArenaScope, FView& view) {
 
     // TAA for color pass
     if (taaOptions.enabled) {
-        input = ppm.taa(fg, input, depth, view.getFrameHistory(), &FrameHistoryEntry::taa,
+        input = ppm.taa(fg, input, depth, xvp, vp,
+                view.getFrameHistory(), &FrameHistoryEntry::taa,
                 taaOptions, colorGradingConfig);
-        if (taaOptions.upscaling) {
+        if (taaOptions.upscaling > 1.0f) {
             scale = 1.0f;
             scaled = false;
+
+            // xvp = vp;
+            // xvp.left = xvp.bottom = 0;
+            // svp = xvp;
+
             UTILS_UNUSED_IN_RELEASE auto const& inputDesc = fg.getDescriptor(input);
             svp.width = inputDesc.width;
             svp.height = inputDesc.height;
-            xvp.width *= 2;
-            xvp.height *= 2;
+            // FIXME: this rounds xvp incorrectly
+
+            xvp.left   *= taaOptions.upscaling;
+            xvp.bottom *= taaOptions.upscaling;
+            xvp.width  *= taaOptions.upscaling;
+            xvp.height *= taaOptions.upscaling;
         }
     }
 

filament/src/details/View.cpp

@@ -320,9 +320,9 @@ float2 FView::updateScale(FEngine& engine,
         // relative scaling ("velocity" control)
         const float scale = mScale.x * mScale.y * command;
 
-        const float w = float(mViewport.width);
-        const float h = float(mViewport.height);
         if (scale < 1.0f && !options.homogeneousScaling) {
+            const float w = float(mViewport.width);
+            const float h = float(mViewport.height);
             // figure out the major and minor axis
             const float major = std::max(w, h);
             const float minor = std::min(w, h);
@@ -351,7 +351,7 @@ float2 FView::updateScale(FEngine& engine,
         const auto s = mScale;
         mScale = clamp(s, options.minScale, options.maxScale);
 
-        // disable the integration term when we're outside the controllable range
+        // Disable the integration term when we're outside the controllable range
         // (i.e. we clamped). This help not to have to wait too long for the Integral term
         // to kick in after a clamping event.
         mPidController.setIntegralInhibitionEnabled(mScale != s);