Added KL divergence for continuous random variables backed by distributions

Author: mandar2812

dynaml-core/src/main/scala-2.11/io/github/mandar2812/dynaml/probability/package.scala

@@ -43,13 +43,34 @@ package object probability {
   }
 
 
+  /**
+    * Calculate the entropy of a random variable
+    * */
   def entropy[I, Distr <: ContinuousDistr[I]](rv: ContinuousRVWithDistr[I, Distr]): Double = {
     val logp_x: RandomVariable[Double] = MeasurableFunction[I, Double, ContinuousRVWithDistr[I, Distr]](
       rv, DataPipe((sample: I) => -1d*rv.underlyingDist.logPdf(sample)))
     E(logp_x)
   }
 
 
+  /**
+    * KL divergence:
+    * @param p The base random variable
+    * @param q The random variable used to approximate p
+    * @return The Kulback Leibler divergence KL(P||Q)
+    * */
+  def KL[I, Distr <: ContinuousDistr[I]](
+    p: ContinuousRVWithDistr[I, Distr])(
+    q: ContinuousRVWithDistr[I, Distr]): Double = {
+
+
+    val log_q_p: RandomVariable[Double] = MeasurableFunction[I, Double, ContinuousRVWithDistr[I, Distr]](
+      p, DataPipe((sample: I) => p.underlyingDist.logPdf(sample)-q.underlyingDist.logPdf(sample)))
+
+    E(log_q_p)
+  }
+
+
   /**
     * Calculate the (monte carlo approximation to) mean, median, mode, lower and upper confidence interval.
     *

dynaml-core/src/main/scala-2.11/io/github/mandar2812/dynaml/utils/package.scala

@@ -292,6 +292,22 @@ package object utils {
 
   }
 
+  //TODO: Complete tail recursive implementation of b-spline
+  /*
+  def bspline(control: Seq[(Double, Double)])(x: Double): Double = {
+    val (knots, control_points) = control.unzip
+
+    val knot_pairs: Map[Int, (Double, Double)] = knots
+      .sliding(2).toSeq
+      .zipWithIndex
+      .map(p => (p._2, (p._1.head, p._1.last)))
+      .toMap
+
+    def bsplineRec(i: Int, k: Int): Double = 0d
+
+    0.0
+  }*/
+
   /**
     * Calculates the Chebyshev polynomials of the first and second kind,
     * in a tail recursive manner, using their recurrence relations.