Work on the convolutional layers

xEcho1337 · xEcho1337 · commit 3096796635b5 · 2025-03-06T21:13:19.000+01:00
diff --git a/src/main/java/net/echo/brain4j/layer/Layer.java b/src/main/java/net/echo/brain4j/layer/Layer.java
@@ -35,11 +35,12 @@ public abstract class Layer<I, O> {
     protected final Activations activation;
     protected final Activation function;
     protected Layer<?, ?> nextLayer;
+    protected int id;
 
     public Layer(int input, Activations activation) {
-        Parameters.TOTAL_LAYERS++;
         Stream.generate(Neuron::new).limit(input).forEach(neurons::add);
 
+        this.id = Parameters.TOTAL_LAYERS++;
         this.activation = activation;
         this.function = activation.getFunction();
     }
@@ -82,32 +83,32 @@ public void updateWeights(Vector[] synapseMatrixLayer) {
         throw new UnsupportedOperationException("Not implemented for this class.");
     }
 
-    public void applyFunction(StatesCache cacheHolder, Layer<?, ?> previous) {
-        function.apply(cacheHolder, neurons);
+    public void applyFunction(StatesCache cache, Layer<?, ?> previous) {
+        function.apply(cache, neurons);
     }
 
-    public void setInput(StatesCache cacheHolder, Vector input) {
+    public void setInput(StatesCache cache, Vector input) {
         Preconditions.checkState(input.size() == neurons.size(), "Input size does not match!" +
                 " (Input != Expected) " + input.size() + " != " + neurons.size());
 
         for (int i = 0; i < input.size(); i++) {
-            neurons.get(i).setValue(cacheHolder, input.get(i));
+            neurons.get(i).setValue(cache, input.get(i));
         }
     }
 
-    public void propagate(StatesCache cacheHolder, Layer<?, ?> previous, Updater updater, Optimizer optimizer) {
+    public void propagate(StatesCache cache, Layer<?, ?> previous, Updater updater, Optimizer optimizer) {
         int nextLayerSize = nextLayer.getNeurons().size();
 
         for (int i = 0; i < neurons.size(); i++) {
             Neuron neuron = neurons.get(i);
 
-            double value = neuron.getValue(cacheHolder);
+            double value = neuron.getValue(cache);
             double derivative = activation.getFunction().getDerivative(value);
 
             for (int j = 0; j < nextLayerSize; j++) {
                 Synapse synapse = synapses.get(i * nextLayerSize + j);
 
-                float weightChange = calculateGradient(cacheHolder, synapse, derivative);
+                float weightChange = calculateGradient(cache, synapse, derivative);
                 updater.acknowledgeChange(synapse, weightChange);
             }
         }
@@ -148,4 +149,8 @@ public int getTotalParams() {
     public int getTotalNeurons() {
         return neurons.size();
     }
+
+    public int getId() {
+        return id;
+    }
 }
diff --git a/src/main/java/net/echo/brain4j/layer/impl/convolution/ConvLayer.java b/src/main/java/net/echo/brain4j/layer/impl/convolution/ConvLayer.java
@@ -4,6 +4,7 @@
 import net.echo.brain4j.activation.Activations;
 import net.echo.brain4j.convolution.Kernel;
 import net.echo.brain4j.layer.Layer;
+import net.echo.brain4j.structure.cache.Parameters;
 import net.echo.brain4j.structure.cache.StatesCache;
 import net.echo.brain4j.training.optimizers.Optimizer;
 import net.echo.brain4j.training.updater.Updater;
@@ -63,6 +64,7 @@ public ConvLayer(int filters, int kernelWidth, int kernelHeight, int stride, Act
      */
     public ConvLayer(int filters, int kernelWidth, int kernelHeight, int stride, int padding, Activations activation) {
         super(0, activation);
+        this.id = Parameters.TOTAL_CONV_LAYER++;
         this.filters = filters;
         this.kernelWidth = kernelWidth;
         this.kernelHeight = kernelHeight;
@@ -115,7 +117,7 @@ public Kernel forward(StatesCache cache, Layer<?, ?> lastLayer, Kernel input) {
     }
 
     @Override
-    public void propagate(StatesCache cacheHolder, Layer<?, ?> nextLayer, Updater updater, Optimizer optimizer) {
+    public void propagate(StatesCache cache, Layer<?, ?> nextLayer, Updater updater, Optimizer optimizer) {
         throw new UnsupportedOperationException("Not implemented yet.");
     }
 
diff --git a/src/main/java/net/echo/brain4j/layer/impl/convolution/FlattenLayer.java b/src/main/java/net/echo/brain4j/layer/impl/convolution/FlattenLayer.java
@@ -6,6 +6,8 @@
 import net.echo.brain4j.layer.Layer;
 import net.echo.brain4j.layer.impl.DenseLayer;
 import net.echo.brain4j.structure.cache.StatesCache;
+import net.echo.brain4j.training.optimizers.Optimizer;
+import net.echo.brain4j.training.updater.Updater;
 import net.echo.brain4j.utils.Vector;
 
 public class FlattenLayer extends DenseLayer {
@@ -19,6 +21,17 @@ public Vector forward(StatesCache cache, Layer<?, ?> lastLayer, Vector input) {
         return input;
     }
 
+    @Override
+    public void propagate(StatesCache cache, Layer<?, ?> previous, Updater updater, Optimizer optimizer) {
+        super.propagate(cache, previous, updater, optimizer);
+
+        for (int i = 0; i < getTotalNeurons(); i++) {
+            double value = neurons.get(i).getValue(cache);
+
+            System.out.println(i + " has " + value);
+        }
+    }
+
     public Vector flatten(StatesCache cache, Layer<?, ?> layer, Kernel input) {
         Preconditions.checkNotNull(input, "Last convolutional input is null! Missing an input layer?");
 
diff --git a/src/main/java/net/echo/brain4j/layer/impl/recurrent/RecurrentLayer.java b/src/main/java/net/echo/brain4j/layer/impl/recurrent/RecurrentLayer.java
@@ -75,8 +75,6 @@ public Vector forward(StatesCache cache, Layer<?, ?> lastLayer, Vector input) {
             throw new UnsupportedOperationException("Previous layer must be a dense or recurrent layer!");
         }
 
-        int prevSize = lastLayer.getNeurons().size();
-
         Vector hiddenState = previousTimestep.get();
 
         for (int i = 0; i < neurons.size(); i++) {
@@ -108,8 +106,8 @@ public Vector forward(StatesCache cache, Layer<?, ?> lastLayer, Vector input) {
     }
 
     @Override
-    public void propagate(StatesCache cacheHolder, Layer<?, ?> previous, Updater updater, Optimizer optimizer) {
-        super.propagate(cacheHolder, previous, updater, optimizer);
+    public void propagate(StatesCache cache, Layer<?, ?> previous, Updater updater, Optimizer optimizer) {
+        super.propagate(cache, previous, updater, optimizer);
     }
 
     public List<Vector> getRecurrentWeights() {
diff --git a/src/main/java/net/echo/brain4j/structure/cache/Parameters.java b/src/main/java/net/echo/brain4j/structure/cache/Parameters.java
@@ -2,7 +2,8 @@
 
 public class Parameters {
 
-    public static int TOTAL_LAYERS;
-    public static int TOTAL_SYNAPSES;
-    public static int TOTAL_NEURONS;
+    public static int TOTAL_CONV_LAYER = 0;
+    public static int TOTAL_LAYERS = 0;
+    public static int TOTAL_SYNAPSES = 0;
+    public static int TOTAL_NEURONS = 0;
 }
diff --git a/src/main/java/net/echo/brain4j/structure/cache/StatesCache.java b/src/main/java/net/echo/brain4j/structure/cache/StatesCache.java
@@ -1,15 +1,27 @@
 package net.echo.brain4j.structure.cache;
 
+import net.echo.brain4j.convolution.Kernel;
+import net.echo.brain4j.layer.impl.convolution.ConvLayer;
 import net.echo.brain4j.structure.Neuron;
 
 public class StatesCache {
 
+    private final Kernel[] featureMaps;
     private final float[] valuesCache;
     private final float[] deltasCache;
 
     public StatesCache() {
         this.valuesCache = new float[Parameters.TOTAL_NEURONS];
         this.deltasCache = new float[Parameters.TOTAL_NEURONS];
+        this.featureMaps = new Kernel[Parameters.TOTAL_CONV_LAYER];
+    }
+
+    public void setFeatureMap(ConvLayer layer, Kernel output) {
+        featureMaps[layer.getId()] = output;
+    }
+
+    public Kernel getFeatureMap(ConvLayer layer) {
+        return featureMaps[layer.getId()];
     }
 
     public float getValue(Neuron neuron) {
diff --git a/src/test/java/conv/ConvExample.java b/src/test/java/conv/ConvExample.java
@@ -13,6 +13,8 @@
 import net.echo.brain4j.model.initialization.WeightInit;
 import net.echo.brain4j.training.data.DataRow;
 import net.echo.brain4j.training.optimizers.impl.Adam;
+import net.echo.brain4j.training.techniques.SmartTrainer;
+import net.echo.brain4j.training.techniques.TrainListener;
 import net.echo.brain4j.training.updater.impl.StochasticUpdater;
 import net.echo.brain4j.utils.DataSet;
 import net.echo.brain4j.utils.Vector;
@@ -25,24 +27,22 @@
 
 public class ConvExample {
 
-    public static void main(String[] args) {
+    public static void main(String[] args) throws IOException {
         ConvExample example = new ConvExample();
         example.start();
     }
 
-    private void start() {
-        Model model = getModel();
+    private void start() throws IOException {
+        Sequential model = getModel();
         DataSet<DataRow> dataSet = getDataSet();
 
-        double loss = model.evaluate(dataSet);
-        System.out.println("Initial loss: " + loss);
+        model.fit(dataSet);
 
-        for (int i = 0; i < 1000; i++) {
+        double loss = model.evaluate(dataSet);
+        System.out.println("Loss: " + loss);
+        /*for (int i = 0; i < 1000; i++) {
             model.fit(dataSet);
-
-            loss = model.evaluate(dataSet);
-            System.out.println("Final loss: " + loss + " at " + i);
-        }
+        }*/
     }
 
     private Sequential getModel() {
@@ -52,14 +52,14 @@ private Sequential getModel() {
 
                 // #1 convolutional block
                 new ConvLayer(32, 3, 3, Activations.RELU),
-                new PoolingLayer(PoolingType.MAX, 2, 2, 2),
+                // new PoolingLayer(PoolingType.MAX, 2, 2, 2),
 
                 // #2 convolutional block
                 new ConvLayer(64, 5, 5, Activations.RELU),
-                new PoolingLayer(PoolingType.MAX, 2, 2, 2),
+                // new PoolingLayer(PoolingType.MAX, 2, 2, 2),
 
                 // Flattens the feature map to a 1D vector
-                new FlattenLayer(25), // You must find the right size by trial and error
+                new FlattenLayer(484), // You must find the right size by trial and error
 
                 // Classifiers
                 new DenseLayer(32, Activations.RELU),
@@ -69,25 +69,21 @@ private Sequential getModel() {
         return model.compile(WeightInit.HE, LossFunctions.CROSS_ENTROPY, new Adam(0.1), new StochasticUpdater());
     }
 
-    private DataSet<DataRow> getDataSet() {
+    private DataSet<DataRow> getDataSet() throws IOException {
         DataSet<DataRow> set = new DataSet<>();
 
-        try {
-            List<String> lines = FileUtils.readLines(new File("dataset.csv"), "UTF-8");
+        List<String> lines = FileUtils.readLines(new File("dataset.csv"), "UTF-8");
 
-            for (String line : lines) {
-                String[] parts = line.split(",");
-                double[] inputs = Arrays.stream(parts, 1, parts.length).mapToDouble(x -> Double.parseDouble(x) / 255).toArray();
+        for (String line : lines) {
+            String[] parts = line.split(",");
+            double[] inputs = Arrays.stream(parts, 1, parts.length).mapToDouble(x -> Double.parseDouble(x) / 255).toArray();
 
-                Vector output = new Vector(10);
+            Vector output = new Vector(10);
 
-                int value = Integer.parseInt(parts[0]);
-                output.set(value, 1);
+            int value = Integer.parseInt(parts[0]);
+            output.set(value, 1);
 
-                set.getData().add(new DataRow(Vector.of(inputs), output));
-            }
-        } catch (IOException e) {
-            throw new RuntimeException("Error reading dataset: " + e.getMessage(), e);
+            set.getData().add(new DataRow(Vector.of(inputs), output));
         }
 
         return set;
diff --git a/src/test/java/mnist/MNISTClassifier.java b/src/test/java/mnist/MNISTClassifier.java
@@ -138,7 +138,7 @@ public static DataSet<DataRow> getData() {
         return set;
     }
 
-    private static class ExampleListener extends TrainListener {
+    private static class ExampleListener extends TrainListener<DataRow> {
 
         @Override
         public void onEvaluated(DataSet<DataRow> dataSet, int epoch, double loss, long took) {

Original file line number	Diff line number	Diff line change
`@@ -75,8 +75,6 @@ public Vector forward(StatesCache cache, Layer<?, ?> lastLayer, Vector input) {`
`75`	`75`	`throw new UnsupportedOperationException("Previous layer must be a dense or recurrent layer!");`
`76`	`76`	`}`
`77`	`77`
`78`		`- int prevSize = lastLayer.getNeurons().size();`
`79`		`-`
`80`	`78`	`Vector hiddenState = previousTimestep.get();`
`81`	`79`
`82`	`80`	`for (int i = 0; i < neurons.size(); i++) {`
`@@ -108,8 +106,8 @@ public Vector forward(StatesCache cache, Layer<?, ?> lastLayer, Vector input) {`
`108`	`106`	`}`
`109`	`107`
`110`	`108`	`@Override`
`111`		`- public void propagate(StatesCache cacheHolder, Layer<?, ?> previous, Updater updater, Optimizer optimizer) {`
`112`		`- super.propagate(cacheHolder, previous, updater, optimizer);`
	`109`	`+ public void propagate(StatesCache cache, Layer<?, ?> previous, Updater updater, Optimizer optimizer) {`
	`110`	`+ super.propagate(cache, previous, updater, optimizer);`
`113`	`111`	`}`
`114`	`112`
`115`	`113`	`public List<Vector> getRecurrentWeights() {`
Original file line number	Diff line number	Diff line change
`@@ -138,7 +138,7 @@ public static DataSet<DataRow> getData() {`
`138`	`138`	`return set;`
`139`	`139`	`}`
`140`	`140`
`141`		`- private static class ExampleListener extends TrainListener {`
	`141`	`+ private static class ExampleListener extends TrainListener<DataRow> {`
`142`	`142`
`143`	`143`	`@Override`
`144`	`144`	`public void onEvaluated(DataSet<DataRow> dataSet, int epoch, double loss, long took) {`