Adde support for fuse_linear_bn_eval

Author: NiklasGustafsson

src/TorchSharp/NN/Linear.cs

@@ -17,6 +17,17 @@ public sealed class Linear : torch.nn.Module<Tensor, Tensor>
             const string WeightComponentName = nameof(weight);
             const string BiasComponentName = nameof(bias);
 
+            internal Linear(Parameter weight, Parameter? bias = null) : base(nameof(Linear))
+            {
+                this.in_features = weight.shape[1];
+                this.out_features = weight.shape[0];
+
+                this.weight = weight;
+                if (bias is not null) {
+                    this.bias = bias;
+                }
+            }
+
             internal Linear(long inputSize, long outputSize, bool hasBias = true, Device? device = null, ScalarType? dtype = null) : base(nameof(Linear))
             {
                 this.in_features = inputSize;
@@ -85,7 +96,7 @@ public static partial class torch
         public static partial class nn
         {
             /// <summary>
-            /// Applies a linear transformation to the incoming data.
+            /// Create a Linear module initialized with random weights and bias.
             /// </summary>
             /// <param name="inputSize">Size of each input sample</param>
             /// <param name="outputSize">Size of each output sample</param>
@@ -97,6 +108,16 @@ public static Linear Linear(long inputSize, long outputSize, bool hasBias = true
                 return new Linear(inputSize, outputSize, hasBias, device, dtype);
             }
 
+            /// <summary>
+            /// Create a Linear module with the given weights and bias.
+            /// </summary>
+            /// <param name="weight">The linear weight attribute.</param>
+            /// <param name="bias">The additive linear bias. Optional.</param>
+            public static Linear Linear(Parameter weight, Parameter? bias = null)
+            {
+                return new Linear(weight, bias);
+            }
+
             public static partial class functional
             {
                 /// <summary>

src/TorchSharp/Torch.cs

@@ -493,6 +493,16 @@ public static (Parameter weight, Parameter bias) fuse_linear_bn_weights(
 
                     return scope.MoveToOuter(weight, bias);
                 }
+
+                public static Linear fuse_linear_bn_eval(Linear linear, BatchNorm bn)
+                {
+                    if (linear.training || bn.training)
+                        throw new InvalidOperationException("Fusing operators is valid only for eval mode.");
+
+                    var (weight, bias) = fuse_linear_bn_weights(linear.weight, linear.bias, bn.running_mean!, bn.running_var!, bn.eps, bn.weight, bn.bias!);
+
+                    return Linear(weight, bias);
+                }
             }
         }
 

test/TorchSharpTest/NN.cs

@@ -325,6 +325,29 @@ public void FunctionalBilinearNoBias()
                 Assert.Equal(40, forward.shape[1]);
                 Assert.Equal(device.type, forward.device_type);
             }
+        }       
+        
+        [Fact]
+        public void TestLinearFused()
+        {
+            var lin = Linear(15,15);
+            var bn = BatchNorm1d(15);
+            lin.eval();
+            bn.eval();
+
+            Assert.NotNull(lin);
+            Assert.NotNull(lin.bias);
+
+            var input = torch.rand(8,15);
+            var expected = bn.forward(lin.forward(input));
+
+            var fused = torch.nn.utils.fuse_linear_bn_eval(lin, bn);
+            var output = fused.forward(input);
+
+            var eStr = expected.str();
+            var oStr = output.str();
+
+            Assert.True(expected.allclose(output, rtol: 1e-3, atol: 1e-3));
         }
 
         [Fact]