Add training dynamics.

Author: ameritusweb

examples/gravnet/ParallelReverseAutoDiff.GravNetExample/GlyphNet.cs

@@ -179,12 +179,12 @@ public void ApplyGradients()
 
             Console.WriteLine($"Max Mag 0: {maxMag0}, Max Mag 1: {maxMag1}");
 
-            SquaredArclengthEuclideanMagnitudeLossOperation arclengthLoss0 = SquaredArclengthEuclideanMagnitudeLossOperation.Instantiate(gatNet);
-            var loss0 = arclengthLoss0.Forward(targetedSum0, (3 * Math.PI) / 4d, maxMag0);
+            SquaredArclengthEuclideanLossOperation arclengthLoss0 = SquaredArclengthEuclideanLossOperation.Instantiate(gatNet);
+            var loss0 = arclengthLoss0.Forward(targetedSum0, (3 * Math.PI) / 4d);
             var gradient0 = arclengthLoss0.Backward();
 
-            SquaredArclengthEuclideanMagnitudeLossOperation arclengthLoss1 = SquaredArclengthEuclideanMagnitudeLossOperation.Instantiate(gatNet);
-            var loss1 = arclengthLoss1.Forward(targetedSum1, (1 * Math.PI) / 4d, maxMag1);
+            SquaredArclengthEuclideanLossOperation arclengthLoss1 = SquaredArclengthEuclideanLossOperation.Instantiate(gatNet);
+            var loss1 = arclengthLoss1.Forward(targetedSum1, (1 * Math.PI) / 4d);
             var gradient1 = arclengthLoss1.Backward();
 
             SquaredArclengthEuclideanMagnitudeLossOperation arclengthLoss = SquaredArclengthEuclideanMagnitudeLossOperation.Instantiate(gatNet);

examples/gravnet/ParallelReverseAutoDiff.GravNetExample/GlyphNetTrainer.cs

@@ -107,6 +107,7 @@ await files.WithRepeatAsync(async (pngFile, token) =>
                         net.SaveWeights();
                     }
 
+                    token.Repeat();
                     //if (token.UsageCount == 0)
                     //{
                     //    token.Repeat(2);

examples/gravnet/ParallelReverseAutoDiff.GravNetExample/GlyphNetwork/GlyphTrainingDynamics.cs

@@ -0,0 +1,44 @@
+namespace ParallelReverseAutoDiff.GravNetExample.GlyphNetwork
+{
+    using ParallelReverseAutoDiff.RMAD;
+
+    public class GlyphTrainingDynamics
+    {
+        private static readonly Lazy<GlyphTrainingDynamics> lazy = new Lazy<GlyphTrainingDynamics>(() => new GlyphTrainingDynamics(), true);
+
+        public static GlyphTrainingDynamics Instance { get { return lazy.Value; } }
+
+        public Matrix[] PreviousTargetedSum { get; set; } = new Matrix[2];
+
+        public Matrix[] LastTargetedSum { get; set; } = new Matrix[2];
+
+        public double PreviousAngleLoss { get; set; }
+
+        public double PreviousEuclideanLoss { get; set; }
+
+        public double PreviousMagnitudeLoss { get; set; }
+
+        public double AngleLoss { get; set; }
+
+        public double EuclideanLoss { get; set; }
+
+        public double MagnitudeLoss { get; set; }
+
+        public double GradAngleLossX { get; set; }
+
+        public double GradAngleLossY { get; set; }
+
+        public double GradEuclideanLossX { get; set; }
+
+        public double GradEuclideanLossY { get; set; }
+
+        public double GradMagnitudeLossX { get; set; }
+
+        public double GradMagnitudeLossY { get; set; }
+
+        private GlyphTrainingDynamics()
+        {
+
+        }
+    }
+}

examples/gravnet/ParallelReverseAutoDiff.GravNetExample/VectorNetwork/RMAD/ElementwiseVectorCartesianTargetedSumOperation.cs

@@ -5,6 +5,8 @@
 //------------------------------------------------------------------------------
 namespace ParallelReverseAutoDiff.RMAD
 {
+    using ParallelReverseAutoDiff.GravNetExample.GlyphNetwork;
+
     /// <summary>
     /// Element-wise cartesian sum operation.
     /// </summary>
@@ -33,6 +35,8 @@ public static IOperation Instantiate(NeuralNetwork net)
         /// <returns>The output of the element-wise vector rotation operation.</returns>
         public Matrix Forward(Matrix inputVectors, Matrix rotationTargets, int target)
         {
+            GlyphTrainingDynamics.Instance.PreviousTargetedSum[target] = GlyphTrainingDynamics.Instance.LastTargetedSum[target];
+            GlyphTrainingDynamics.Instance.LastTargetedSum[target] = inputVectors;
             this.inputVectors = inputVectors;
             this.rotationTargets = rotationTargets;
             this.target = target;

examples/gravnet/ParallelReverseAutoDiff.GravNetExample/VectorNetwork/RMAD/SquaredArclengthEuclideanMagnitudeLossOperation.cs

@@ -5,6 +5,7 @@
 //------------------------------------------------------------------------------
 namespace ParallelReverseAutoDiff.RMAD
 {
+    using ParallelReverseAutoDiff.GravNetExample.GlyphNetwork;
     using System;
 
     /// <summary>
@@ -90,6 +91,13 @@ public Matrix Forward(Matrix predictions, double targetAngle, double maxMagnitud
 
             double arcLength = Math.Pow(radius * theta, 2);
 
+            GlyphTrainingDynamics.Instance.PreviousAngleLoss = GlyphTrainingDynamics.Instance.AngleLoss;
+            GlyphTrainingDynamics.Instance.PreviousEuclideanLoss = GlyphTrainingDynamics.Instance.EuclideanLoss;
+            GlyphTrainingDynamics.Instance.PreviousMagnitudeLoss = GlyphTrainingDynamics.Instance.MagnitudeLoss;
+            GlyphTrainingDynamics.Instance.AngleLoss = arcLength;
+            GlyphTrainingDynamics.Instance.EuclideanLoss = distanceAccum;
+            GlyphTrainingDynamics.Instance.MagnitudeLoss = magnitudeDiscrepancy;
+
             // Compute the squared magnitude of the loss
             double lossMagnitude = (arcLength + distanceAccum + magnitudeDiscrepancy) / 3d;
 
@@ -121,6 +129,13 @@ public Matrix Backward()
             dPredictions[0, 0] = (cX * gradX) + eX + dMagDiscrepancy_dX;
             dPredictions[0, 1] = (cY * gradY) + eY + dMagDiscrepancy_dY;
 
+            GlyphTrainingDynamics.Instance.GradAngleLossX = cX * gradX;
+            GlyphTrainingDynamics.Instance.GradAngleLossY = cY * gradY;
+            GlyphTrainingDynamics.Instance.GradEuclideanLossX = eX;
+            GlyphTrainingDynamics.Instance.GradEuclideanLossY = eY;
+            GlyphTrainingDynamics.Instance.GradMagnitudeLossX = dMagDiscrepancy_dX;
+            GlyphTrainingDynamics.Instance.GradMagnitudeLossY = dMagDiscrepancy_dY;
+
             if (double.IsNaN(dPredictions[0, 0]) || double.IsNaN(dPredictions[0, 1]))
             {