Test with colon is running, the result are incorrect though.

Author: sramirez

src/test/scala/org/apache/spark/ml/feature/ITSelectorSuite.scala

@@ -26,7 +26,7 @@ class MDLPDiscretizerSuite extends FunSuite with BeforeAndAfterAll {
 
     val df = readColonData(sqlContext)
     val cols = df.columns
-    val model = getSelectorModel(df, df.columns.drop(1), df.columns.head, 10, 20)
+    val model = getSelectorModel(sqlContext, df, df.columns.drop(1), df.columns.head, 10, 20)
 
     assertResult("764, 1581, 1671, 512, 1670, 1324, 1381, 1971, 1422, 1411") {
       model.selectedFeatures.mkString(", ")

src/test/scala/org/apache/spark/ml/feature/TestHelper.scala

@@ -8,7 +8,10 @@ import org.apache.spark.sql.{DataFrame, Row, SQLContext}
 import org.apache.spark.sql.types._
 import org.joda.time.format.DateTimeFormat
 import org.apache.spark.ml.linalg.Vectors
+import org.apache.spark.ml.linalg.Vector
 import org.apache.spark.ml.linalg.VectorUDT
+import org.apache.spark.sql.Dataset
+import org.apache.spark.ml.util._
 
 /**
   * Loads various test datasets
@@ -28,21 +31,26 @@ object TestHelper {
   /**
     * @return the discretizer fit to the data given the specified features to bin and label use as target.
     */
-  def createSelectorModel(dataframe: DataFrame, inputCols: Array[String],
+  
+  def createSelectorModel(sqlContext: SQLContext, dataframe: Dataset[_], inputCols: Array[String],
                              labelColumn: String,
                              nPartitions: Int = 100,
                              numTopFeatures: Int = 20, 
                              allVectorsDense: Boolean = true): InfoThSelectorModel = {
     val featureAssembler = new VectorAssembler()
       .setInputCols(inputCols)
       .setOutputCol("features")
-    val processedDf = featureAssembler.transform(dataframe)
-    
+    val processedDf = featureAssembler.transform(dataframe).select(labelColumn + INDEX_SUFFIX, "features")
 
-     processedDf.map {
+    /** InfoSelector requires all vectors from the same type (either be sparse or dense) **/
+    val rddData = processedDf.rdd.map {
         case Row(label: Double, features: Vector) =>
-          OldLabeledPoint(label, OldVectors.fromML(features))
+          val standardv = if(allVectorsDense) features.toDense else features.toSparse
+          Row.fromSeq(Seq(label, standardv))
       }
+    
+    val inputData = sqlContext.createDataFrame(rddData, processedDf.schema)
+      
     val selector = new InfoThSelector()
         .setSelectCriterion("mrmr")
         .setNPartitions(nPartitions)
@@ -51,20 +59,20 @@ object TestHelper {
         .setLabelCol(labelColumn + INDEX_SUFFIX)
         .setOutputCol("selectedFeatures")
 
-    selector.fit(processedDf)
+    selector.fit(inputData)
   }
 
 
   /**
     * The label column will have null values replaced with MISSING values in this case.
     * @return the discretizer fit to the data given the specified features to bin and label use as target.
     */
-  def getSelectorModel(dataframe: DataFrame, inputCols: Array[String],
+  def getSelectorModel(sqlContext: SQLContext, dataframe: DataFrame, inputCols: Array[String],
                           labelColumn: String,
                              nPartitions: Int = 100,
                              numTopFeatures: Int = 20): InfoThSelectorModel = {
     val processedDf = cleanLabelCol(dataframe, labelColumn)
-    createSelectorModel(processedDf, inputCols, labelColumn, nPartitions, numTopFeatures)
+    createSelectorModel(sqlContext, processedDf, inputCols, labelColumn, nPartitions, numTopFeatures)
   }