Fixed issue with blending activation, addressed comments for gating activation. Removed all runtime checks and replaced with asserts.

Author: João Felipe Santos

NAM/gating_activations.h

@@ -14,16 +14,16 @@ namespace gating_activations
 {
 
 // Default linear activation (identity function)
-class LinearActivation : public nam::activations::Activation
+class IdentityActivation : public nam::activations::Activation
 {
 public:
-  LinearActivation() = default;
-  ~LinearActivation() = default;
+  IdentityActivation() = default;
+  ~IdentityActivation() = default;
   // Inherit the default apply methods which do nothing (linear/identity)
 };
 
 // Static instance for default activation
-static LinearActivation default_activation;
+static IdentityActivation default_activation;
 
 class GatingActivation
 {
@@ -40,12 +40,8 @@ class GatingActivation
   : input_activation(input_act ? input_act : &default_activation)
   , gating_activation(gating_act ? gating_act : activations::Activation::get_activation("Sigmoid"))
   , num_input_channels(input_channels)
-  , num_gating_channels(gating_channels)
   {
-    if (num_input_channels <= 0 || num_gating_channels <= 0)
-    {
-      throw std::invalid_argument("GatingActivation: number of channels must be positive");
-    }
+    assert(num_input_channels > 0);
   }
 
   ~GatingActivation() = default;
@@ -57,20 +53,11 @@ class GatingActivation
    */
   void apply(Eigen::MatrixXf& input, Eigen::MatrixXf& output)
   {
-    // Validate input dimensions
-    const int total_channels = num_input_channels + num_gating_channels;
-    if (input.rows() != total_channels)
-    {
-      throw std::invalid_argument("GatingActivation: input matrix must have " + std::to_string(total_channels)
-                                  + " rows");
-    }
-
-    // Validate output dimensions
-    if (output.rows() != num_input_channels || output.cols() != input.cols())
-    {
-      throw std::invalid_argument("GatingActivation: output matrix must have " + std::to_string(num_input_channels)
-                                  + " rows and " + std::to_string(input.cols()) + " columns");
-    }
+    // Validate input dimensions (assert for real-time performance)
+    const int total_channels = 2 * num_input_channels;
+    assert(input.rows() == total_channels);
+    assert(output.rows() == num_input_channels);
+    assert(output.cols() == input.cols());
 
     // Process column-by-column to ensure memory contiguity (important for column-major matrices)
     const int num_samples = input.cols();
@@ -81,20 +68,19 @@ class GatingActivation
       input_activation->apply(input_block);
 
       // Apply activation to gating channels
-      Eigen::MatrixXf gating_block = input.block(num_input_channels, i, num_gating_channels, 1);
+      Eigen::MatrixXf gating_block = input.block(num_input_channels, i, num_input_channels, 1);
       gating_activation->apply(gating_block);
 
       // Element-wise multiplication and store result
       // For wavenet compatibility, we assume one-to-one mapping
-      assert(num_input_channels == num_gating_channels);
       output.block(0, i, num_input_channels, 1) = input_block.array() * gating_block.array();
     }
   }
 
   /**
    * Get the total number of input channels required
    */
-  int get_total_input_channels() const { return num_input_channels + num_gating_channels; }
+  int get_total_input_channels() const { return 2 * num_input_channels; }
 
   /**
    * Get the number of output channels
@@ -105,7 +91,6 @@ class GatingActivation
   activations::Activation* input_activation;
   activations::Activation* gating_activation;
   int num_input_channels;
-  int num_gating_channels;
 };
 
 class BlendingActivation
@@ -115,27 +100,21 @@ class BlendingActivation
    * Constructor for BlendingActivation
    * @param input_act Activation function for input channels
    * @param blend_act Activation function for blending channels
-   * @param alpha_val Blending factor (0.0 to 1.0)
    * @param input_channels Number of input channels
-   * @param blend_channels Number of blending channels
    */
   BlendingActivation(activations::Activation* input_act = nullptr, activations::Activation* blend_act = nullptr,
-                     float alpha_val = 0.5f, int input_channels = 1, int blend_channels = 1)
+                     int input_channels = 1)
   : input_activation(input_act ? input_act : &default_activation)
   , blending_activation(blend_act ? blend_act : &default_activation)
-  , alpha(alpha_val)
   , num_input_channels(input_channels)
-  , num_blend_channels(blend_channels)
   {
-    // Validate alpha is in valid range
-    if (alpha < 0.0f || alpha > 1.0f)
-    {
-      throw std::invalid_argument("BlendingActivation: alpha must be between 0.0 and 1.0");
-    }
-    if (num_input_channels <= 0 || num_blend_channels <= 0)
+    if (num_input_channels <= 0)
     {
-      throw std::invalid_argument("BlendingActivation: number of channels must be positive");
+      throw std::invalid_argument("BlendingActivation: number of input channels must be positive");
     }
+    // Initialize input buffer with correct size
+    // Note: current code copies column-by-column so we only need (num_input_channels, 1)
+    input_buffer.resize(num_input_channels, 1);
   }
 
   ~BlendingActivation() = default;
@@ -147,44 +126,37 @@ class BlendingActivation
    */
   void apply(Eigen::MatrixXf& input, Eigen::MatrixXf& output)
   {
-    // Validate input dimensions
-    const int total_channels = num_input_channels + num_blend_channels;
-    if (input.rows() != total_channels)
-    {
-      throw std::invalid_argument("BlendingActivation: input matrix must have " + std::to_string(total_channels)
-                                  + " rows");
-    }
-
-    // Validate output dimensions
-    if (output.rows() != num_input_channels || output.cols() != input.cols())
-    {
-      throw std::invalid_argument("BlendingActivation: output matrix must have " + std::to_string(num_input_channels)
-                                  + " rows and " + std::to_string(input.cols()) + " columns");
-    }
+    // Validate input dimensions (assert for real-time performance)
+    const int total_channels = num_input_channels * 2; // 2*channels in, channels out
+    assert(input.rows() == total_channels);
+    assert(output.rows() == num_input_channels);
+    assert(output.cols() == input.cols());
 
     // Process column-by-column to ensure memory contiguity
     const int num_samples = input.cols();
     for (int i = 0; i < num_samples; i++)
     {
+      // Store pre-activation input values in buffer
+      input_buffer = input.block(0, i, num_input_channels, 1);
+
       // Apply activation to input channels
       Eigen::MatrixXf input_block = input.block(0, i, num_input_channels, 1);
       input_activation->apply(input_block);
 
-      // Apply activation to blend channels
-      Eigen::MatrixXf blend_block = input.block(num_input_channels, i, num_blend_channels, 1);
+      // Apply activation to blend channels to compute alpha
+      Eigen::MatrixXf blend_block = input.block(num_input_channels, i, num_input_channels, 1);
       blending_activation->apply(blend_block);
 
-      // Weighted blending
-      // For wavenet compatibility, we assume one-to-one mapping
-      assert(num_input_channels == num_blend_channels);
-      output.block(0, i, num_input_channels, 1) = alpha * input_block + (1.0f - alpha) * blend_block;
+      // Weighted blending: alpha * activated_input + (1 - alpha) * pre_activation_input
+      output.block(0, i, num_input_channels, 1) =
+        blend_block.array() * input_block.array() + (1.0f - blend_block.array()) * input_buffer.array();
     }
   }
 
   /**
    * Get the total number of input channels required
    */
-  int get_total_input_channels() const { return num_input_channels + num_blend_channels; }
+  int get_total_input_channels() const { return 2 * num_input_channels; }
 
   /**
    * Get the number of output channels
@@ -194,9 +166,8 @@ class BlendingActivation
 private:
   activations::Activation* input_activation;
   activations::Activation* blending_activation;
-  float alpha;
   int num_input_channels;
-  int num_blend_channels;
+  Eigen::MatrixXf input_buffer;
 };
 
 

tools/run_tests.cpp

@@ -9,6 +9,8 @@
 #include "test/test_fast_lut.cpp"
 #include "test/test_gating_activations.cpp"
 #include "test/test_wavenet_gating_compatibility.cpp"
+#include "test/test_blending_detailed.cpp"
+#include "test/test_input_buffer_verification.cpp"
 
 int main()
 {
@@ -46,18 +48,26 @@ int main()
   test_gating_activations::TestGatingActivation::test_with_custom_activations();
   test_gating_activations::TestGatingActivation::test_error_handling();
 
-  test_gating_activations::TestBlendingActivation::test_basic_functionality();
-  test_gating_activations::TestBlendingActivation::test_different_alpha_values();
-  test_gating_activations::TestBlendingActivation::test_with_custom_activations();
-  test_gating_activations::TestBlendingActivation::test_error_handling();
-  test_gating_activations::TestBlendingActivation::test_edge_cases();
-
   // Wavenet gating compatibility tests
   test_wavenet_gating_compatibility::TestWavenetGatingCompatibility::test_wavenet_style_gating();
   test_wavenet_gating_compatibility::TestWavenetGatingCompatibility::test_column_by_column_processing();
   test_wavenet_gating_compatibility::TestWavenetGatingCompatibility::test_memory_contiguity();
   test_wavenet_gating_compatibility::TestWavenetGatingCompatibility::test_multiple_channels();
 
+  test_gating_activations::TestBlendingActivation::test_basic_functionality();
+  test_gating_activations::TestBlendingActivation::test_blending_behavior();
+  test_gating_activations::TestBlendingActivation::test_with_custom_activations();
+  test_gating_activations::TestBlendingActivation::test_error_handling();
+  test_gating_activations::TestBlendingActivation::test_edge_cases();
+
+  // Detailed blending tests
+  test_blending_detailed::TestBlendingDetailed::test_blending_with_different_activations();
+  test_blending_detailed::TestBlendingDetailed::test_input_buffer_usage();
+
+  // Input buffer verification tests
+  test_input_buffer_verification::TestInputBufferVerification::test_buffer_stores_pre_activation_values();
+  test_input_buffer_verification::TestInputBufferVerification::test_buffer_with_different_activations();
+
   std::cout << "Success!" << std::endl;
   return 0;
 }