Added an evaluation method

Author: xEcho1337

src/main/java/net/echo/brain4j/model/Model.java

@@ -18,6 +18,7 @@
 import net.echo.brain4j.structure.Synapse;
 import net.echo.brain4j.structure.cache.Parameters;
 import net.echo.brain4j.structure.cache.StatesCache;
+import net.echo.brain4j.training.evaluation.EvaluationResult;
 import net.echo.brain4j.training.optimizers.Optimizer;
 import net.echo.brain4j.training.optimizers.impl.Adam;
 import net.echo.brain4j.training.optimizers.impl.AdamW;
@@ -114,10 +115,10 @@ public void connect(WeightInit weightInit, boolean update) {
     /**
      * Evaluates the model on the given dataset.
      *
-     * @param set dataset for testing
+     * @param dataSet dataset for testing
      * @return the error of the model
      */
-    public abstract double loss(DataSet<R> set);
+    public abstract double loss(DataSet<R> dataSet);
 
     /**
      * Trains the model for one epoch.
@@ -146,6 +147,13 @@ public void fit(DataSet<R> dataSet, int epoches) {
      */
     public abstract O predict(I input);
 
+    /**
+     * Evaluates the model performance on the given dataset.
+     * @param dataSet dataset to evaluate
+     * @return an evaluation result
+     */
+    public abstract EvaluationResult evaluate(DataSet<R> dataSet);
+
     /**
      * Predicts output for given input.
      *

src/main/java/net/echo/brain4j/model/impl/Sequential.java

@@ -16,14 +16,19 @@
 import net.echo.brain4j.structure.cache.StatesCache;
 import net.echo.brain4j.training.BackPropagation;
 import net.echo.brain4j.training.data.DataRow;
+import net.echo.brain4j.training.evaluation.EvaluationResult;
 import net.echo.brain4j.training.optimizers.Optimizer;
 import net.echo.brain4j.training.updater.Updater;
 import net.echo.brain4j.training.updater.impl.StochasticUpdater;
 import net.echo.brain4j.utils.DataSet;
+import net.echo.brain4j.utils.MLUtils;
 import net.echo.brain4j.utils.Vector;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicReference;
 
 import static net.echo.brain4j.utils.MLUtils.waitAll;
@@ -97,18 +102,61 @@ public Sequential compile(WeightInit weightInit, LossFunctions function, Optimiz
     }
 
     @Override
-    public double loss(DataSet<DataRow> set) {
+    public EvaluationResult evaluate(DataSet<DataRow> dataSet) {
+        int classes = layers.getLast().getNeurons().size();
+
+        Map<Integer, Integer> correctlyClassified = new ConcurrentHashMap<>();
+        Map<Integer, Integer> incorrectlyClassified = new ConcurrentHashMap<>();
+
+        for (int i = 0; i < classes; i++) {
+            correctlyClassified.put(i, 0);
+            incorrectlyClassified.put(i, 0);
+        }
+
+        List<Thread> threads = new ArrayList<>();
+
+        if (!dataSet.isPartitioned()) {
+            dataSet.partition(Math.min(Runtime.getRuntime().availableProcessors(), dataSet.getData().size()));
+        }
+
+        for (List<DataRow> partition : dataSet.getPartitions()) {
+            threads.add(makeEvaluation(partition, correctlyClassified, incorrectlyClassified));
+        }
+
+        waitAll(threads);
+        return new EvaluationResult(classes, correctlyClassified, incorrectlyClassified);
+    }
+
+    private Thread makeEvaluation(List<DataRow> partition, Map<Integer, Integer> correctlyClassified, Map<Integer, Integer> incorrectlyClassified) {
+        return Thread.startVirtualThread(() -> {
+            for (DataRow row : partition) {
+                Vector prediction = predict(row.inputs());
+
+                int predIndex = MLUtils.indexOfMaxValue(prediction);
+                int targetIndex = MLUtils.indexOfMaxValue(row.outputs());
+
+                if (predIndex == targetIndex) {
+                    correctlyClassified.compute(targetIndex, (k, v) -> v == null ? 1 : v + 1);
+                } else {
+                    incorrectlyClassified.compute(targetIndex, (k, v) -> v == null ? 1 : v + 1);
+                }
+            }
+        });
+    }
+
+    @Override
+    public double loss(DataSet<DataRow> dataSet) {
         reloadMatrices();
 
         AtomicReference<Double> totalError = new AtomicReference<>(0.0);
         List<Thread> threads = new ArrayList<>();
 
-        for (List<DataRow> partition : set.getPartitions()) {
+        for (List<DataRow> partition : dataSet.getPartitions()) {
             threads.add(predictPartition(partition, totalError));
         }
 
         waitAll(threads);
-        return totalError.get() / set.size();
+        return totalError.get() / dataSet.size();
     }
 
     @Override

src/main/java/net/echo/brain4j/model/impl/Transformer.java

@@ -5,6 +5,7 @@
 import net.echo.brain4j.model.Model;
 import net.echo.brain4j.model.initialization.WeightInit;
 import net.echo.brain4j.structure.cache.StatesCache;
+import net.echo.brain4j.training.evaluation.EvaluationResult;
 import net.echo.brain4j.training.optimizers.Optimizer;
 import net.echo.brain4j.training.updater.Updater;
 import net.echo.brain4j.training.updater.impl.StochasticUpdater;
@@ -36,13 +37,18 @@ public Transformer compile(WeightInit weightInit, LossFunctions function, Optimi
         return this;
     }
 
+    @Override
+    public EvaluationResult evaluate(DataSet<Object> dataSet) {
+        return null;
+    }
+
     @Override
     public void connect(WeightInit weightInit, boolean update) {
         super.connect(weightInit, update);
     }
 
     @Override
-    public double loss(DataSet<Object> set) {
+    public double loss(DataSet<Object> dataSet) {
         return 0;
     }
 

src/main/java/net/echo/brain4j/training/evaluation/EvaluationResult.java

@@ -0,0 +1,41 @@
+package net.echo.brain4j.training.evaluation;
+
+import java.util.Map;
+
+public record EvaluationResult(int classes, Map<Integer, Integer> correctlyClassified, Map<Integer, Integer> incorrectlyClassified) {
+
+    public String confusionMatrix() {
+        StringBuilder matrix = new StringBuilder();
+        String header = "======================================\n";
+
+        String pattern = "%-10s %-10s %-10s\n";
+        matrix.append(String.format(pattern, "Classes", "Correct", "Incorrect"));
+        matrix.append(header);
+
+        int totalCorrect = 0;
+        int totalIncorrect = 0;
+
+        for (int i = 0; i < correctlyClassified.size(); i++) {
+            totalCorrect += correctlyClassified.get(i);
+            totalIncorrect += incorrectlyClassified.get(i);
+        }
+
+        for (int i = 0; i < classes; i++) {
+            matrix.append(String.format(pattern, i, correctlyClassified.get(i), incorrectlyClassified.get(i)));
+        }
+
+        matrix.append(header);
+
+        double accuracy = totalCorrect / (double) (totalCorrect + totalIncorrect);
+        double precision = totalCorrect / (double) (totalCorrect + incorrectlyClassified.get(0));
+        double recall = totalCorrect / (double) (totalCorrect + correctlyClassified.get(0));
+
+        String secondary = "%-20s %-10s\n";
+        matrix.append(String.format(secondary, "Accuracy:", String.format("%.2f", accuracy * 100), ""));
+        matrix.append(String.format(secondary, "Precision:", String.format("%.2f", precision * 100), ""));
+        matrix.append(String.format(secondary, "Recall: ", String.format("%.2f", recall * 100), ""));
+        matrix.append(header);
+
+        return matrix.toString();
+    }
+}

src/test/java/conv/ConvExample.java

@@ -8,6 +8,7 @@
 import net.echo.brain4j.loss.LossFunctions;
 import net.echo.brain4j.model.impl.Sequential;
 import net.echo.brain4j.training.data.DataRow;
+import net.echo.brain4j.training.evaluation.EvaluationResult;
 import net.echo.brain4j.training.optimizers.impl.Adam;
 import net.echo.brain4j.utils.DataSet;
 import net.echo.brain4j.utils.MLUtils;
@@ -17,6 +18,7 @@
 
 import java.io.FileReader;
 import java.io.IOException;
+import java.util.HashMap;
 import java.util.List;
 
 public class ConvExample {
@@ -32,27 +34,12 @@ private void start() throws IOException {
 
         System.out.println(model.getStats());
 
-        model.fit(dataSet, 10);
-        model.save("mnist-conv.json");
+        // model.fit(dataSet, 10);
+        // model.save("mnist-conv.json");
 
-        int incorrect = 0;
+        EvaluationResult result = model.evaluate(dataSet);
 
-        for (DataRow row : dataSet) {
-            Vector input = row.inputs();
-            Vector output = row.outputs();
-            Vector prediction = model.predict(input);
-
-            int expected = MLUtils.indexOfMaxValue(output) + 1;
-            int predicted = MLUtils.indexOfMaxValue(prediction) + 1;
-
-            if (expected != predicted) {
-                System.out.println("Expected: " + expected + ", Prediction: " + predicted);
-                incorrect++;
-            }
-        }
-
-        int correct = dataSet.size() - incorrect;
-        System.out.println("Correct: " + correct + ", Incorrect: " + incorrect);
+        System.out.println(result.confusionMatrix());
     }
 
     private Sequential getModel() {