Update networks.

Author: sseto001

test/ParallelReverseAutoDiff.Test/GraphAttentionPaths/Embedding/EmbeddingNeuralNetwork.cs

@@ -165,7 +165,7 @@ public void AutomaticForwardPropagate(DeepMatrix input)
         /// </summary>
         /// <param name="gradient">The gradient of the loss.</param>
         /// <returns>The gradient.</returns>
-        public async Task<DeepMatrix> AutomaticBackwardPropagate(DeepMatrix gradient)
+        public async Task<Matrix> AutomaticBackwardPropagate(DeepMatrix gradient)
         {
             IOperationBase? backwardStartOperation = null;
             backwardStartOperation = this.computationGraph["vector_concatenate_trans_0_0"];
@@ -179,7 +179,7 @@ public async Task<DeepMatrix> AutomaticBackwardPropagate(DeepMatrix gradient)
             }
 
             IOperationBase? backwardEndOperation = this.computationGraph["batch_embeddings_0_0"];
-            return backwardEndOperation.CalculatedGradient[0] as DeepMatrix ?? throw new InvalidOperationException("Calculated gradient should not be null.");
+            return backwardEndOperation.CalculatedGradient[1] as Matrix ?? throw new InvalidOperationException("Calculated gradient should not be null.");
         }
 
         /// <summary>

test/ParallelReverseAutoDiff.Test/GraphAttentionPaths/Transformer/TransformerNeuralNetwork.cs

@@ -60,6 +60,7 @@ public TransformerNeuralNetwork(int numLayers, int numQueries, int numPaths, int
                     .AddModelElementGroup("VB", new[] { 1, numInputOutputFeatures }, InitializationType.Zeroes);
                 var inputLayer = inputLayerBuilder.Build();
                 this.inputLayers.Add(inputLayer);
+                numInputFeatures = numInputOutputFeatures;
             }
 
             this.nestedLayers = new List<IModelLayer>();
@@ -73,6 +74,7 @@ public TransformerNeuralNetwork(int numLayers, int numQueries, int numPaths, int
                     .AddModelElementGroup("QB", new[] { numQueries, 1, numNestedOutputFeatures }, InitializationType.Zeroes);
                 var nestedLayer = nestedLayerBuilder.Build();
                 this.nestedLayers.Add(nestedLayer);
+                numNestedFeatures = numNestedOutputFeatures;
                 outputFeaturesList.Add(numNestedOutputFeatures * numQueries);
             }
 
@@ -85,18 +87,11 @@ public TransformerNeuralNetwork(int numLayers, int numQueries, int numPaths, int
                     .AddModelElementGroup("F2W", new[] { outputFeaturesList[i], (outputFeaturesList[i] / 2) }, InitializationType.Xavier)
                     .AddModelElementGroup("F2B", new[] { 1, (outputFeaturesList[i] / 2) }, InitializationType.Xavier)
                     .AddModelElementGroup("Beta", new[] { 1, 1 }, InitializationType.He);
-                if (i < (this.NumLayers - 1))
-                {
-                    outputLayerBuilder
-                        .AddModelElementGroup("R", new[] { (outputFeaturesList[i] / 2), this.NumFeatures }, InitializationType.Xavier)
-                        .AddModelElementGroup("RB", new[] { 1, this.NumFeatures }, InitializationType.Zeroes);
-                }
-                else
-                {
+
                     outputLayerBuilder
                         .AddModelElementGroup("R", new[] { (outputFeaturesList[i] / 2), this.NumFeatures * 2 }, InitializationType.Xavier)
                         .AddModelElementGroup("RB", new[] { 1, this.NumFeatures * 2 }, InitializationType.Zeroes);
-                }
+
                 var outputLayer = outputLayerBuilder.Build();
                 this.outputLayers.Add(outputLayer);
             }