Neural Network works!!! TODO remove old code, release!

Author: magicindian

src/aima/learning/neural/FeedForwardNeuralNetwork.java

@@ -5,6 +5,11 @@
 
 public class FeedForwardNeuralNetwork implements FunctionApproximator {
 
+	public static final String UPPER_LIMIT_WEIGHTS = "upper_limit_weights";
+	public static final String LOWER_LIMIT_WEIGHTS = "lower_limit_weights";
+	public static final String NUMBER_OF_OUTPUTS = "number_of_outputs";
+	public static final String NUMBER_OF_HIDDEN_NEURONS = "number_of_hidden_neurons";
+	public static final String NUMBER_OF_INPUTS = "number_of_inputs";
 	private final Layer hiddenLayer;
 	private final Layer outputLayer;
 
@@ -16,16 +21,16 @@ public class FeedForwardNeuralNetwork implements FunctionApproximator {
 	public FeedForwardNeuralNetwork(NNConfig config) {
 
 		int numberOfInputNeurons = config
-				.getParameterAsInteger("number_of_inputs");
+				.getParameterAsInteger(NUMBER_OF_INPUTS);
 		int numberOfHiddenNeurons = config
-				.getParameterAsInteger("number_of_hidden_neurons");
+				.getParameterAsInteger(NUMBER_OF_HIDDEN_NEURONS);
 		int numberOfOutputNeurons = config
-				.getParameterAsInteger("number_of_outputs");
+				.getParameterAsInteger(NUMBER_OF_OUTPUTS);
 
 		double lowerLimitForWeights = config
-				.getParameterAsDouble("lower_limit_weights");
+				.getParameterAsDouble(LOWER_LIMIT_WEIGHTS);
 		double upperLimitForWeights = config
-				.getParameterAsDouble("upper_limit_weights");
+				.getParameterAsDouble(UPPER_LIMIT_WEIGHTS);
 
 		hiddenLayer = new Layer(numberOfHiddenNeurons, numberOfInputNeurons,
 				lowerLimitForWeights, upperLimitForWeights,
@@ -63,6 +68,39 @@ public Vector processInput(Vector input) {
 		return trainingScheme.processInput(this, input);
 	}
 
+	public void trainOn(NNDataSet innds, int numberofEpochs) {
+		for (int i = 0; i < numberofEpochs; i++) {
+			innds.refreshDataset();
+			while (innds.hasMoreExamples()) {
+				NNExample nne = innds.getExampleAtRandom();
+				processInput(nne.getInput());
+				Vector error = getOutputLayer()
+						.errorVectorFrom(nne.getTarget());
+				processError(error);
+			}
+		}
+
+	}
+
+	public Vector predict(NNExample nne) {
+		return processInput(nne.getInput());
+	}
+
+	public int[] testOnDataSet(NNDataSet nnds) {
+		int[] result = new int[] { 0, 0 };
+		nnds.refreshDataset();
+		while (nnds.hasMoreExamples()) {
+			NNExample nne = nnds.getExampleAtRandom();
+			Vector prediction = predict(nne);
+			if (nne.isCorrect(prediction)) {
+				result[0] = result[0] + 1;
+			} else {
+				result[1] = result[1] + 1;
+			}
+		}
+		return result;
+	}
+
 	public void testOn(DataSet ds) {
 		// TODO Auto-generated method stub
 	}

src/aima/learning/neural/Layer.java

@@ -41,7 +41,7 @@ public Layer(int numberOfNeurons, int numberOfInputs,
 			ActivationFunction af) {
 
 		activationFunction = af;
-		this.weightMatrix = new Matrix(numberOfNeurons, numberOfInputs());
+		this.weightMatrix = new Matrix(numberOfNeurons, numberOfInputs);
 		lastWeightUpdateMatrix = new Matrix(weightMatrix.getRowDimension(),
 				weightMatrix.getColumnDimension());
 		penultimateWeightUpdateMatrix = new Matrix(weightMatrix
@@ -104,7 +104,7 @@ public Vector getLastInducedField() {
 	private static void initializeMatrix(Matrix aMatrix, double lowerLimit,
 			double upperLimit) {
 		for (int i = 0; i < aMatrix.getRowDimension(); i++) {
-			for (int j = 0; j < aMatrix.getColumnDimension(); i++) {
+			for (int j = 0; j < aMatrix.getColumnDimension(); j++) {
 				double random = Util.generateRandomDoubleBetween(lowerLimit,
 						upperLimit);
 				aMatrix.set(i, j, random);
@@ -192,4 +192,9 @@ public void acceptNewBiasUpdate(Vector biasUpdate) {
 		setLastBiasUpdateVector(biasUpdate);
 	}
 
+	public Vector errorVectorFrom(Vector target) {
+		return target.minus(getLastActivationValues());
+
+	}
+
 }

src/aima/learning/neural/NNConfig.java

@@ -13,6 +13,10 @@ public NNConfig(Hashtable<String, Object> hash) {
 		this.hash = hash;
 	}
 
+	public NNConfig() {
+		this.hash = new Hashtable<String, Object>();
+	}
+
 	public double getParameterAsDouble(String key) {
 
 		return (Double) hash.get(key);
@@ -23,4 +27,11 @@ public int getParameterAsInteger(String key) {
 		return (Integer) hash.get(key);
 	}
 
+	public void setConfig(String key, Double value) {
+		hash.put(key, value);
+	}
+
+	public void setConfig(String key, int value) {
+		hash.put(key, value);
+	}
 }

src/aima/learning/neural/NNDataSet.java

@@ -6,6 +6,10 @@
 import java.util.Arrays;
 import java.util.List;
 
+import aima.learning.framework.DataSet;
+import aima.learning.framework.Example;
+import aima.learning.statistics.Numerizer;
+import aima.util.Pair;
 import aima.util.Util;
 
 public abstract class NNDataSet {
@@ -21,7 +25,7 @@ public abstract class NNDataSet {
 	/*
 	 * a copy from which examples are drawn.
 	 */
-	private List<NNExample> presentlyProcessed;
+	private List<NNExample> presentlyProcessed = new ArrayList<NNExample>();;
 
 	/*
 	 * list of mean Values for all components of raw data set
@@ -35,7 +39,7 @@ public abstract class NNDataSet {
 	/*
 	 * the normalized data set
 	 */
-	private List<List<Double>> nds;
+	protected List<List<Double>> nds;
 
 	/*
 	 * the column numbers of the "target"
@@ -91,6 +95,21 @@ public void createNormalizedDataFromFile(String filename) throws Exception {
 		nds = normalize(rds);
 	}
 
+	/*
+	 * create a normalized data "table" from the DataSet using numerizer. At
+	 * this stage, the data is *not* split into input pattern and targets TODO
+	 * remove redundancy of recreating the target columns. the numerizer has
+	 * already isolated the targets
+	 */
+
+	public void createNormalizedDataFromDataSet(DataSet ds, Numerizer numerizer)
+			throws Exception {
+
+		List<List<Double>> rds = rawExamplesFromDataSet(ds, numerizer);
+		// normalize raw dataset
+		nds = normalize(rds);
+	}
+
 	private List<List<Double>> normalize(List<List<Double>> rds) {
 		int rawDataLength = rds.get(0).size();
 		List<List<Double>> nds = new ArrayList<List<Double>>();
@@ -140,6 +159,27 @@ private List<Double> exampleFromString(String line, String separator) {
 		return rexample;
 	}
 
+	private List<List<Double>> rawExamplesFromDataSet(DataSet ds,
+			Numerizer numerizer) {
+		// assumes all values for inout and target are doubles
+		List<List<Double>> rds = new ArrayList<List<Double>>();
+		for (int i = 0; i < ds.size(); i++) {
+			List<Double> rexample = new ArrayList<Double>();
+			Example e = ds.getExample(i);
+			Pair<List<Double>, List<Double>> p = numerizer.numerize(e);
+			List<Double> attributes = p.getFirst();
+			for (Double d : attributes) {
+				rexample.add(d);
+			}
+			List<Double> targets = p.getSecond();
+			for (Double d : targets) {
+				rexample.add(d);
+			}
+			rds.add(rexample);
+		}
+		return rds;
+	}
+
 	/*
 	 * Gets (and removes) a random example from the 'presentlyProcessed'
 	 */
@@ -149,6 +189,14 @@ public NNExample getExampleAtRandom() {
 		return presentlyProcessed.remove(i);
 	}
 
+	/*
+	 * Gets (and removes) a random example from the 'presentlyProcessed'
+	 */
+	public NNExample getExample(int index) {
+
+		return presentlyProcessed.remove(index);
+	}
+
 	/*
 	 * check if any more examples remain to be processed
 	 */
@@ -185,6 +233,18 @@ public void createExamplesFromFile(String filename) throws Exception {
 
 	}
 
+	/*
+	 * method called by clients to set up data set and make it ready for
+	 * processing
+	 */
+	public void createExamplesFromDataSet(DataSet ds, Numerizer numerizer)
+			throws Exception {
+		createNormalizedDataFromDataSet(ds, numerizer);
+		setTargetColumns();
+		createExamples();
+
+	}
+
 	public List<List<Double>> getNormalizedData() {
 		return nds;
 	}

src/aima/learning/neural/NNExample.java

@@ -23,4 +23,25 @@ public NNExample copyExample() {
 		return new NNExample(newInput, newTarget);
 	}
 
+	public Vector getInput() {
+		Vector v = new Vector(normalizedInput);
+		return v;
+
+	}
+
+	public Vector getTarget() {
+		Vector v = new Vector(normalizedTarget);
+		return v;
+
+	}
+
+	public boolean isCorrect(Vector prediction) {
+		/*
+		 * compares the index having greatest value in target to indec having
+		 * greatest value in prediction. Ifidentical, correct
+		 */
+		return getTarget().indexHavingMaxValue() == prediction
+				.indexHavingMaxValue();
+	}
+
 }

src/aima/learning/neural/Vector.java

@@ -1,5 +1,7 @@
 package aima.learning.neural;
 
+import java.util.List;
+
 import aima.util.Matrix;
 
 public class Vector extends Matrix {
@@ -9,6 +11,13 @@ public Vector(int size) {
 		super(size, 1);
 	}
 
+	public Vector(List<Double> lst) {
+		super(lst.size(), 1);
+		for (int i = 0; i < lst.size(); i++) {
+			setValue(i, lst.get(i));
+		}
+	}
+
 	public double getValue(int i) {
 		return super.get(i, 0);
 	}
@@ -29,4 +38,25 @@ public int size() {
 		return getRowDimension();
 	}
 
+	public Vector minus(Vector v) {
+		Vector result = new Vector(size());
+		for (int i = 0; i < size(); i++) {
+			result.setValue(i, getValue(i) - v.getValue(i));
+		}
+		return result;
+	}
+
+	public int indexHavingMaxValue() {
+		if (size() <= 0) {
+			throw new RuntimeException("can't perform this op on empty vector");
+		}
+		int res = 0;
+		for (int i = 0; i < size(); i++) {
+			if (getValue(i) > getValue(res)) {
+				res = i;
+			}
+		}
+		return res;
+	}
+
 }

src/aima/test/learningtest/neural/BackPropagationTests.java

@@ -1,9 +1,15 @@
 package aima.test.learningtest.neural;
 
 import junit.framework.TestCase;
+import aima.learning.framework.DataSet;
+import aima.learning.framework.DataSetFactory;
 import aima.learning.neural.BackPropLearning;
 import aima.learning.neural.FeedForwardNeuralNetwork;
+import aima.learning.neural.NNConfig;
+import aima.learning.neural.NNDataSet;
 import aima.learning.neural.Vector;
+import aima.learning.statistics.IrisDataSetNumerizer;
+import aima.learning.statistics.Numerizer;
 import aima.util.Matrix;
 
 public class BackPropagationTests extends TestCase {
@@ -108,4 +114,29 @@ public void testFeedForwardAndBAckLoopWorksWithMomentum() {
 		assertEquals(0.6061, outputLayerBias.getValue(0), 0.001);
 
 	}
+
+	public void testDataSetPopulation() throws Exception {
+		DataSet irisDataSet = DataSetFactory.getIrisDataSet();
+		Numerizer numerizer = new IrisDataSetNumerizer();
+		NNDataSet innds = new IrisNNDataSet();
+
+		innds.createExamplesFromDataSet(irisDataSet, numerizer);
+
+		NNConfig config = new NNConfig();
+		config.setConfig(FeedForwardNeuralNetwork.NUMBER_OF_INPUTS, 4);
+		config.setConfig(FeedForwardNeuralNetwork.NUMBER_OF_OUTPUTS, 3);
+		config.setConfig(FeedForwardNeuralNetwork.NUMBER_OF_HIDDEN_NEURONS, 6);
+		config.setConfig(FeedForwardNeuralNetwork.LOWER_LIMIT_WEIGHTS, -2.0);
+		config.setConfig(FeedForwardNeuralNetwork.UPPER_LIMIT_WEIGHTS, 2.0);
+
+		FeedForwardNeuralNetwork ffnn = new FeedForwardNeuralNetwork(config);
+		ffnn.setTrainingScheme(new BackPropLearning(0.1, 0.9));
+
+		ffnn.trainOn(innds, 10);
+
+		innds.refreshDataset();
+		int[] result = ffnn.testOnDataSet(innds);
+		System.out.println(result[0] + " right, " + result[1] + " wrong");
+
+	}
 }

src/aima/test/learningtest/neural/IrisNNDataSet.java

@@ -0,0 +1,25 @@
+package aima.test.learningtest.neural;
+
+import java.util.ArrayList;
+
+import aima.learning.neural.NNDataSet;
+
+public class IrisNNDataSet extends NNDataSet {
+
+	@Override
+	public void setTargetColumns() {
+		// assumed that data from file has been pre processed
+		// TODO this should be
+		// somewhere else,in the
+		// super class.
+		// Type != class Aargh! I want more
+		// powerful type systems
+		targetColumnNumbers = new ArrayList<Integer>();
+		int size = nds.get(0).size();
+		targetColumnNumbers.add(size - 1); // last column
+		targetColumnNumbers.add(size - 2); // last but one column
+		targetColumnNumbers.add(size - 3); // and the one before that
+
+	}
+
+}