Added ReLU & Leaky ReLU

Author: Guled

Example/MLKit/GameScene.swift

@@ -51,6 +51,8 @@ class GameScene: SKScene, SKPhysicsContactDelegate {
 
     /// Best score (regardless of generation)
     var bestScore: Int = 0
+
+    /// Label that displays the best score (bestScore attribute)
     var bestScoreLabel: SKLabelNode!
 
 
@@ -319,7 +321,7 @@ class GameScene: SKScene, SKPhysicsContactDelegate {
         if generationCounter >= 3 {
 
             // Experiment: Keep some of the last best birds and put them back into the population
-            lastBestGen = (flappyBirdGenerationContainer?.filter({$0.fitness >= 9.0}))!
+            lastBestGen = (flappyBirdGenerationContainer?.filter({$0.fitness >= 7.0}))!
         }
 
         if (currentBird?.fitness)! > maxFitness {
@@ -352,8 +354,8 @@ class GameScene: SKScene, SKPhysicsContactDelegate {
                 var offspring = BiologicalProcessManager.onePointCrossover(crossoverRate: 0.5, parentOneGenotype: parents.0.genotypeRepresentation, parentTwoGenotype: parents.1.genotypeRepresentation)
 
                 // Mutate their genes
-                BiologicalProcessManager.scrambleMutation(mutationRate: 0.7, genotype: &offspring.0)
-                BiologicalProcessManager.scrambleMutation(mutationRate: 0.7, genotype: &offspring.1)
+                BiologicalProcessManager.inverseMutation(mutationRate: 0.7, genotype: &offspring.0)
+                BiologicalProcessManager.inverseMutation(mutationRate: 0.7, genotype: &offspring.1)
 
 
                 // Create a separate neural network for the birds based on their genes

MLKit/Classes/ANN/ActivationFunctionType.swift

@@ -18,6 +18,8 @@ public enum ActivationFunctionType {
     case softsign /// SoftSign Activation Function
     case sinusoid /// Sinusoid Activation Function
     case gaussian /// Sinusoid Activation Function
+    case ReLU /// ReLU Activation Function
+    case LeakyReLU /// Leaky ReLU Activation Function
 
     /**
      The activate method returns an activation function.
@@ -48,6 +50,8 @@ public enum ActivationFunctionType {
 
     /**
      The activate method returns the derivative of the activation function a Layer object is using.
+     Note that a Layer object is instantiated with an ActivationFunctionType. This method simply observes the
+     ActivationFunctionType that the layer is using and returns the derivative for that layers particular ActivationFunctionType.
 
      - returns: A method that taeks in a Float as a parameter and returns a Float.
     */
@@ -101,6 +105,14 @@ public enum ActivationFunctionType {
         return exp(powf((-val), 2))
     }
 
+    private func fncReLU(val: Float) -> Float {
+        return max(0, val)
+    }
+
+    private func fncLeakyReLU(val: Float) -> Float {
+        return max(0.01*val, val)
+    }
+
     // MARK: Derivatives
     private func derivativeOfLinear(val: Float) -> Float {
         return 1.0
@@ -126,6 +138,14 @@ public enum ActivationFunctionType {
         return -2 * val * exp(powf((-val), 2))
     }
 
+    private func derivativeOfReLU(val: Float) -> Float {
+        return (val < 0.0) ? 0.0 : 1.0
+    }
+
+    private func derivativeOfLeakyReLU(val: Float) -> Float {
+        return (val < 0.0) ? 0.01 : 1.0
+    }
+
     /// Simply a method to satisfy the switch statements located in the activate and derivative methods. The method simply returns -1 which indicates that an error has occurred (A non-existant enum was discovered).
     private func error(val: Float) -> Float {
         return -1