Skip to content

Commit edb3a33

Browse files
amerituswebameritusweb
committed
Update tilednet.
1 parent e14abd4 commit edb3a33

File tree

5 files changed

+84
-48
lines changed

5 files changed

+84
-48
lines changed

‎examples/gravnet/ParallelReverseAutoDiff.GravNetExample/TiledNet.cs‎

Lines changed: 29 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
using ParallelReverseAutoDiff.RMAD;
1+
using ParallelReverseAutoDiff.GravNetExample.Common;
2+
using ParallelReverseAutoDiff.RMAD;
23
using System;
34
using System.Collections.Generic;
45
using System.Linq;
@@ -147,7 +148,7 @@ public void ApplyGradients()
147148
/// Make a forward pass through the computation graph.
148149
/// </summary>
149150
/// <returns>The gradient of the loss wrt the output.</returns>
150-
public (Matrix, Matrix, Matrix) Forward(Matrix input, double[,] percentages)
151+
public (Matrix, (Matrix, Matrix)[,]) Forward(Matrix input, double[,] percentages)
151152
{
152153

153154
var gatNet = this.TiledNetwork;
@@ -157,21 +158,41 @@ public void ApplyGradients()
157158
gatNet.AutomaticForwardPropagate(input);
158159
var output = gatNet.Output;
159160

160-
SquaredArclengthEuclideanLossOperation arclengthLoss = SquaredArclengthEuclideanLossOperation.Instantiate(gatNet);
161-
var loss = arclengthLoss.Forward(output, Math.PI / 4);
162-
var gradient = arclengthLoss.Backward();
161+
var sections = CommonMatrixUtils.BreakIntoSectionsExactly(output, 8);
162+
(Matrix, Matrix)[,] lossAndGradient = new (Matrix, Matrix)[8, 8];
163+
for (int i = 0; i < 8; ++i)
164+
{
165+
for (int j = 0; j < 8; ++j)
166+
{
167+
Matrix oo = sections[i, j];
168+
var perc = percentages[i, j];
169+
if (perc > 0.4d)
170+
{
171+
SquaredArclengthEuclideanLossOperation arclengthLoss = SquaredArclengthEuclideanLossOperation.Instantiate(gatNet);
172+
var loss = arclengthLoss.Forward(oo, 3 * Math.PI / 4);
173+
var gradient = arclengthLoss.Backward();
174+
lossAndGradient[i, j] = (loss, gradient);
175+
} else
176+
{
177+
SquaredArclengthEuclideanLossOperation arclengthLoss = SquaredArclengthEuclideanLossOperation.Instantiate(gatNet);
178+
var loss = arclengthLoss.Forward(oo, Math.PI / 4);
179+
var gradient = arclengthLoss.Backward();
180+
lossAndGradient[i, j] = (loss, gradient);
181+
}
182+
}
183+
}
163184

164-
return (gradient, output, loss);
185+
return (output, lossAndGradient);
165186
}
166187

167188
/// <summary>
168189
/// The backward pass through the computation graph.
169190
/// </summary>
170191
/// <param name="gradientOfLossWrtOutput">The gradient of the loss wrt the output.</param>
171192
/// <returns>A task.</returns>
172-
public async Task<Matrix> Backward(Matrix gradientOfLossWrtOutput)
193+
public async Task<Matrix> Backward((Matrix, Matrix)[,] lossAndGradient)
173194
{
174-
return await this.TiledNetwork.AutomaticBackwardPropagate(gradientOfLossWrtOutput, null, null);
195+
return await this.TiledNetwork.AutomaticBackwardPropagate(lossAndGradient);
175196
}
176197
}
177198
}

‎examples/gravnet/ParallelReverseAutoDiff.GravNetExample/TiledNetTrainer.cs‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -87,9 +87,8 @@ await files.WithRepeatAsync(async (pngFile, token) =>
8787
i++;
8888

8989
var res = net.Forward(matrix, percentages);
90-
var gradient = res.Item1;
91-
var output = res.Item2;
92-
var loss = res.Item3;
90+
var output = res.Item1;
91+
var lossAndGradient = res.Item2;
9392
//var gradient0 = res.Item2;
9493
//var gradient1 = res.Item3;
9594
//var output = res.Item4;
@@ -107,10 +106,10 @@ await files.WithRepeatAsync(async (pngFile, token) =>
107106
//}
108107

109108

110-
Console.WriteLine($"Iteration {i} Output X: {output[0, 0]}, Output Y: {output[0, 1]}, Grad: {gradient[0, 0]}, {gradient[0, 1]}");
111-
Console.WriteLine($"Loss: {loss[0, 0]}, Perc: {perc}");
109+
//Console.WriteLine($"Iteration {i} Output X: {output[0, 0]}, Output Y: {output[0, 1]}, Grad: {gradient[0, 0]}, {gradient[0, 1]}");
110+
//Console.WriteLine($"Loss: {loss[0, 0]}, Perc: {perc}");
112111
//Console.WriteLine($"O1 X: {o1[0, 0]}, O1 Y: {o1[0, 1]}, Loss: {loss[0, 0]}, {loss0[0, 0]}, {loss1[0, 0]}");
113-
await net.Backward(gradient);
112+
await net.Backward(lossAndGradient);
114113
net.ApplyGradients();
115114
//}
116115

‎examples/gravnet/ParallelReverseAutoDiff.GravNetExample/TiledNetwork/TiledNetwork.cs‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -354,15 +354,24 @@ public void AutomaticForwardPropagate(Matrix input)
354354
/// </summary>
355355
/// <param name="gradient">The gradient of the loss.</param>
356356
/// <returns>The gradient.</returns>
357-
public async Task<Matrix> AutomaticBackwardPropagate(Matrix gradient, Matrix gradient0, Matrix gradient1)
357+
public async Task<Matrix> AutomaticBackwardPropagate((Matrix, Matrix)[,] lossAndGradient)
358358
{
359+
Matrix[,] grad = new Matrix[lossAndGradient.GetLength(0), lossAndGradient.GetLength(1)];
360+
for (int i = 0; i < lossAndGradient.GetLength(0); i++)
361+
{
362+
for (int j = 0; j < lossAndGradient.GetLength(1); j++)
363+
{
364+
grad[i, j] = lossAndGradient[i, j].Item2;
365+
}
366+
}
367+
359368
IOperationBase? backwardStartOperation = null;
360369
var output = this.computationGraph["output_0_0"];
361-
Matrix gg = gradient;
370+
Matrix gg = CommonMatrixUtils.PieceTogetherExactly(grad);
362371

363372
backwardStartOperation = output;
364373

365-
if (!CommonMatrixUtils.IsAllZeroes(gradient))
374+
if (!CommonMatrixUtils.IsAllZeroes(gg))
366375
{
367376
backwardStartOperation.BackwardInput = gg;
368377
OperationNeuralNetworkVisitor opVisitor = new OperationNeuralNetworkVisitor(Guid.NewGuid().ToString(), backwardStartOperation, 0);
@@ -386,7 +395,7 @@ public async Task<Matrix> AutomaticBackwardPropagate(Matrix gradient, Matrix gra
386395
IOperationBase? backwardEndOperation = this.computationGraph["weight_vectors_square_0_0"];
387396
if (backwardEndOperation.CalculatedGradient[0] == null)
388397
{
389-
return gradient;
398+
return gg;
390399
}
391400

392401
return backwardEndOperation.CalculatedGradient[0] as Matrix ?? throw new InvalidOperationException("Calculated gradient should not be null.");
@@ -398,7 +407,7 @@ public async Task<Matrix> AutomaticBackwardPropagate(Matrix gradient, Matrix gra
398407
public void InitializeState()
399408
{
400409
// Clear intermediates
401-
var output = new Matrix(CommonMatrixUtils.InitializeZeroMatrix(1, 2).ToArray());
410+
var output = new Matrix(CommonMatrixUtils.InitializeZeroMatrix(8, 16).ToArray());
402411
var targetedSum0 = new Matrix(CommonMatrixUtils.InitializeZeroMatrix(1, 2).ToArray());
403412
var targetedSum1 = new Matrix(CommonMatrixUtils.InitializeZeroMatrix(1, 2).ToArray());
404413
var rotationTargets = new Matrix(CommonMatrixUtils.InitializeZeroMatrix(15, 15).ToArray());

‎examples/gravnet/ParallelReverseAutoDiff.GravNetExample/VectorNetwork/RMAD/ElementwiseVectorCartesianTiledSummationOperation.cs‎

Lines changed: 22 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,10 @@
55
//------------------------------------------------------------------------------
66
namespace ParallelReverseAutoDiff.RMAD
77
{
8-
using ParallelReverseAutoDiff.GravNetExample.Common;
98
using System;
109
using System.Linq;
1110
using System.Threading.Tasks;
11+
using ParallelReverseAutoDiff.GravNetExample.Common;
1212

1313
/// <summary>
1414
/// Element-wise cartesian tiled summation operation.
@@ -51,6 +51,10 @@ public Matrix Forward(Matrix input1, Matrix input2, Matrix weights)
5151
this.calculatedValues = new CalculatedValues[brokenInput1.GetLength(0), brokenInput1.GetLength(1)][,];
5252
this.summationX = new double[brokenInput1.GetLength(0), brokenInput1.GetLength(1)][];
5353
this.summationY = new double[brokenInput1.GetLength(0), brokenInput1.GetLength(1)][];
54+
this.input1 = new Matrix[brokenInput1.GetLength(0), brokenInput1.GetLength(1)];
55+
this.input2 = new Matrix[brokenInput2.GetLength(0), brokenInput2.GetLength(1)];
56+
this.weights = new Matrix[brokenWeights.GetLength(0), brokenWeights.GetLength(1)];
57+
this.output = new Matrix[brokenInput1.GetLength(0), brokenInput1.GetLength(1)];
5458

5559
Parallel.For(0, brokenInput1.GetLength(0), i =>
5660
{
@@ -60,7 +64,7 @@ public Matrix Forward(Matrix input1, Matrix input2, Matrix weights)
6064
}
6165
});
6266

63-
this.Output = CommonMatrixUtils.PieceTogether(this.output);
67+
this.Output = CommonMatrixUtils.PieceTogetherExactly(this.output);
6468

6569
return this.Output;
6670
}
@@ -76,7 +80,7 @@ private void InnerForward(int ii, int jj, Matrix input1, Matrix input2, Matrix w
7680
double[] summationX = new double[input1.Rows];
7781
double[] summationY = new double[input1.Rows];
7882
double[,] resultVectors = new double[input1.Rows * (input1.Cols / 2), 2];
79-
Parallel.For(0, input1.Rows, i =>
83+
for (int i = 0; i < input1.Rows; i++)
8084
{
8185
double sumX = 0.0d;
8286
double sumY = 0.0d;
@@ -169,17 +173,20 @@ private void InnerForward(int ii, int jj, Matrix input1, Matrix input2, Matrix w
169173
calculatedValues.DLocalSumY_DMagnitude = dLocalSumY_dMagnitude;
170174
calculatedValues.DLocalSumY_DWMagnitude = dLocalSumY_dWMagnitude;
171175

176+
this.calculatedValues[ii, jj][i, j] = calculatedValues;
177+
172178
sumX += localSumX;
173179
sumY += localSumY;
174180
}
175181

176182
summationX[i] = sumX;
177183
summationY[i] = sumY;
178-
});
184+
}
179185

180186
this.summationX[ii, jj] = summationX;
181187
this.summationY[ii, jj] = summationY;
182188

189+
this.output[ii, jj] = new Matrix(1, 2);
183190
this.output[ii, jj][0, 0] = this.summationX[ii, jj].Sum();
184191
this.output[ii, jj][0, 1] = this.summationY[ii, jj].Sum();
185192
}
@@ -190,13 +197,13 @@ public override BackwardResult Backward(Matrix dOutput)
190197
this.dInput1 = new Matrix[this.input1.GetLength(0), this.input1.GetLength(1)];
191198
this.dInput2 = new Matrix[this.input2.GetLength(0), this.input2.GetLength(1)];
192199
this.dWeights = new Matrix[this.weights.GetLength(0), this.weights.GetLength(1)];
193-
var dOutputSections = CommonMatrixUtils.BreakIntoSections(dOutput, 8);
200+
var dOutputSections = CommonMatrixUtils.BreakIntoSectionsExactly(dOutput, 8);
194201

195202
Parallel.For(0, this.dInput1.GetLength(0), i =>
196203
{
197204
for (int j = 0; j < this.dInput2.GetLength(1); j++)
198205
{
199-
this.InnerBackward(i, j, this.dInput1[i, j], this.dInput2[i, j], this.dWeights[i, j], dOutputSections[i, j]);
206+
this.InnerBackward(i, j, dOutputSections[i, j]);
200207
}
201208
});
202209

@@ -207,19 +214,19 @@ public override BackwardResult Backward(Matrix dOutput)
207214
.Build();
208215
}
209216

210-
private void InnerBackward(int ii, int jj, Matrix input1, Matrix input2, Matrix weights, Matrix dOutput)
217+
private void InnerBackward(int ii, int jj, Matrix dOutput)
211218
{
212-
Matrix dInput1 = new Matrix(input1.Rows, input1.Cols);
213-
Matrix dInput2 = new Matrix(input2.Rows, input2.Cols);
214-
Matrix dWeights = new Matrix(weights.Rows, weights.Cols);
219+
Matrix dInput1 = new Matrix(this.input1[ii, jj].Rows, this.input1[ii, jj].Cols);
220+
Matrix dInput2 = new Matrix(this.input2[ii, jj].Rows, this.input2[ii, jj].Cols);
221+
Matrix dWeights = new Matrix(this.weights[ii, jj].Rows, this.weights[ii, jj].Cols);
215222

216223
double dSummationXOutput = dOutput[0, 0]; // Gradient of the loss function with respect to the output X
217224
double dSummationYOutput = dOutput[0, 1]; // Gradient of the loss function with respect to the output Y
218225

219226
// Updating gradients with respect to resultMagnitude and resultAngle
220-
Parallel.For(0, input1.Rows, i =>
227+
for (int i = 0; i < this.input1[ii, jj].Rows; i++)
221228
{
222-
for (int j = 0; j < input1.Cols / 2; j++)
229+
for (int j = 0; j < this.input1[ii, jj].Cols / 2; j++)
223230
{
224231
var values = this.calculatedValues[ii, jj][i, j];
225232

@@ -228,12 +235,12 @@ private void InnerBackward(int ii, int jj, Matrix input1, Matrix input2, Matrix
228235

229236
// Apply chain rule to propagate back to dInput1 and dInput2
230237
dInput1[i, j] = dSummationXOutput * values.DLocalSumX_DMagnitude + dSummationYOutput * values.DLocalSumY_DMagnitude;
231-
dInput1[i, j + (input1.Cols / 2)] = dSummationXOutput * values.DLocalSumX_DAngle + dSummationYOutput * values.DLocalSumY_DAngle;
238+
dInput1[i, j + (this.input1[ii, jj].Cols / 2)] = dSummationXOutput * values.DLocalSumX_DAngle + dSummationYOutput * values.DLocalSumY_DAngle;
232239

233240
dInput2[i, j] = dSummationXOutput * values.DLocalSumX_DWMagnitude + dSummationYOutput * values.DLocalSumY_DWMagnitude;
234-
dInput2[i, j + (input2.Cols / 2)] = dSummationXOutput * values.DLocalSumX_DWAngle + dSummationYOutput * values.DLocalSumY_DWAngle;
241+
dInput2[i, j + (this.input2[ii, jj].Cols / 2)] = dSummationXOutput * values.DLocalSumX_DWAngle + dSummationYOutput * values.DLocalSumY_DWAngle;
235242
}
236-
});
243+
}
237244

238245
this.dInput1[ii, jj] = dInput1;
239246
this.dInput2[ii, jj] = dInput2;

‎examples/gravnet/ParallelReverseAutoDiff.GravNetExample/VectorNetwork/RMAD/ElementwiseVectorConstituentTiledMultiplyOperation.cs‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ public override BackwardResult Backward(Matrix dOutput)
7878
{
7979
for (int j = 0; j < this.dInput2.GetLength(1); j++)
8080
{
81-
this.InnerBackward(i, j, this.dInput1[i, j], this.dInput2[i, j], this.dWeights[i, j], dOutputSections[i, j]);
81+
this.InnerBackward(i, j, dOutputSections[i, j]);
8282
}
8383
});
8484

@@ -315,32 +315,32 @@ private void InnerForward(int ii, int jj, Matrix input1, Matrix input2, Matrix w
315315
}
316316
}
317317

318-
private void InnerBackward(int ii, int jj, Matrix input1, Matrix input2, Matrix weights, Matrix dOutput)
318+
private void InnerBackward(int ii, int jj, Matrix dOutput)
319319
{
320320
// Initialize gradient matrices
321-
Matrix dInput1 = new Matrix(input1.Rows, input1.Cols);
322-
Matrix dInput2 = new Matrix(input2.Rows, input2.Cols);
323-
Matrix dWeights = new Matrix(weights.Rows, weights.Cols);
321+
Matrix dInput1 = new Matrix(this.input1[ii, jj].Rows, this.input1[ii, jj].Cols);
322+
Matrix dInput2 = new Matrix(this.input2[ii, jj].Rows, this.input2[ii, jj].Cols);
323+
Matrix dWeights = new Matrix(this.weights[ii, jj].Rows, this.weights[ii, jj].Cols);
324324

325325
// Loop through each element in input1
326-
for (int i = 0; i < input1.Rows; ++i)
326+
for (int i = 0; i < this.input1[ii, jj].Rows; ++i)
327327
{
328328
// Loop through each half of the columns in input1 (representing magnitudes and angles)
329-
for (int j = 0; j < input2.Cols / 2; j++)
329+
for (int j = 0; j < this.input2[ii, jj].Cols / 2; j++)
330330
{
331331
var calculatedValues = this.calculatedValues[ii, jj][i, j];
332332
dInput1[i, j] += dOutput[i, j] * calculatedValues.DInputMag_dOutputMag;
333-
dInput1[i, j] += dOutput[i, j + (input2.Cols / 2)] * calculatedValues.DInputMag_dOutputAngle;
334-
dInput1[i, j + (input1.Cols / 2)] += dOutput[i, j] * calculatedValues.DInputAngle_dOutputMag;
335-
dInput1[i, j + (input1.Cols / 2)] += dOutput[i, j + (input2.Cols / 2)] * calculatedValues.DInputAngle_dOutputAngle;
333+
dInput1[i, j] += dOutput[i, j + (this.input2[ii, jj].Cols / 2)] * calculatedValues.DInputMag_dOutputAngle;
334+
dInput1[i, j + (this.input1[ii, jj].Cols / 2)] += dOutput[i, j] * calculatedValues.DInputAngle_dOutputMag;
335+
dInput1[i, j + (this.input1[ii, jj].Cols / 2)] += dOutput[i, j + (this.input2[ii, jj].Cols / 2)] * calculatedValues.DInputAngle_dOutputAngle;
336336

337337
dInput2[i, j] += dOutput[i, j] * calculatedValues.DInput2Mag_dOutputMag;
338-
dInput2[i, j] += dOutput[i, j + (input2.Cols / 2)] * calculatedValues.DInput2Mag_dOutputAngle;
339-
dInput2[i, j + (input2.Cols / 2)] += dOutput[i, j] * calculatedValues.DInput2Angle_dOutputMag;
340-
dInput2[i, j + (input2.Cols / 2)] += dOutput[i, j + (input2.Cols / 2)] * calculatedValues.DInput2Angle_dOutputAngle;
338+
dInput2[i, j] += dOutput[i, j + (this.input2[ii, jj].Cols / 2)] * calculatedValues.DInput2Mag_dOutputAngle;
339+
dInput2[i, j + (this.input2[ii, jj].Cols / 2)] += dOutput[i, j] * calculatedValues.DInput2Angle_dOutputMag;
340+
dInput2[i, j + (this.input2[ii, jj].Cols / 2)] += dOutput[i, j + (this.input2[ii, jj].Cols / 2)] * calculatedValues.DInput2Angle_dOutputAngle;
341341

342342
dWeights[i, j] += dOutput[i, j] * calculatedValues.DWeight_dOutputMag;
343-
dWeights[i, j] += dOutput[i, j + (input2.Cols / 2)] * calculatedValues.DWeight_dOutputAngle;
343+
dWeights[i, j] += dOutput[i, j + (this.input2[ii, jj].Cols / 2)] * calculatedValues.DWeight_dOutputAngle;
344344
}
345345
}
346346

0 commit comments

Comments
 (0)