WIP

Author: happytomatoe

core/src/main/scala/com/github/mrpowers/spark/fast/tests/DataFrameComparer.scala

@@ -57,26 +57,24 @@ trait DataFrameComparer extends DatasetComparer {
         )
         val actual = if (ignoreColumnOrder) orderColumns(actualDF, expectedDF) else actualDF
         if (orderedComparison)
-          assertSmallDataFrameEquality(actual, expectedDF, truncate)
+          assertSmallDataFrameEquality(actual, expectedDF)
         else
           assertSmallDataFrameEquality(
             defaultSortDataset(actual),
-            defaultSortDataset(expectedDF),
-            truncate
+            defaultSortDataset(expectedDF)
           )
     }
 
   }
 
   private def assertSmallDataFrameEquality(
       actualDF: DataFrame,
-      expectedDF: DataFrame,
-      truncate: Int
+      expectedDF: DataFrame
   ): Unit = {
     val a = actualDF.collect()
     val e = expectedDF.collect()
     if (!a.toSeq.approximateSameElements(e, (o1: Row, o2: Row) => o1.equals(o2))) {
-      val msg = "Difference\n" ++ DataframeUtil.showDataframeDiff(a, e, actualDF.schema.fieldNames, truncate)
+      val msg = "Difference\n" ++ DataframeUtil.showDataframeDiff(a, e, actualDF.schema.fieldNames)
       throw DatasetContentMismatch(msg)
     }
   }

core/src/main/scala/com/github/mrpowers/spark/fast/tests/DataframeUtil.scala

@@ -12,7 +12,6 @@ object DataframeUtil {
       actual: Array[Row],
       expected: Array[Row],
       fieldNames: Array[String],
-      truncate: Int = 20,
       minColWidth: Int = 3
   ): String = {
     val sb        = new StringBuilder
@@ -31,8 +30,8 @@ object DataframeUtil {
       .zipAll(expected, Row.empty, Row.empty)
       .map { case (actualRow, expectedRow) => (actualRow, expectedRow, actualRow.equals(expectedRow)) }
     val diff = fullJoinWithEquals.map { case (actualRow, expectedRow, rowsAreEqual) =>
-      val paddedActualRow   = pad(actualRow, truncate, rowWidths)
-      val paddedExpectedRow = pad(expectedRow, truncate, rowWidths)
+      val paddedActualRow   = pad(actualRow.toSeq, rowWidths)
+      val paddedExpectedRow = pad(expectedRow.toSeq, rowWidths)
       if (rowsAreEqual) {
         List(DarkGray(paddedActualRow.mkString("|")), DarkGray(paddedActualRow.mkString("|")))
       } else {
@@ -61,12 +60,12 @@ object DataframeUtil {
             val coloredDiff = withEquals.zipWithIndex
               .map {
                 case ((actualRowField, expectedRowField, true), i) =>
-                  val paddedActualRow = padAny(actualRowField, truncate, rowWidths(i))
-                  val paddedExpected  = padAny(expectedRowField, truncate, rowWidths(i))
+                  val paddedActualRow = padAny(actualRowField, rowWidths(i))
+                  val paddedExpected  = padAny(expectedRowField, rowWidths(i))
                   (DarkGray(paddedActualRow), DarkGray(paddedExpected))
                 case ((actualRowField, expectedRowField, false), i) =>
-                  val paddedActualRow = padAny(actualRowField, truncate, rowWidths(i))
-                  val paddedExpected  = padAny(expectedRowField, truncate, rowWidths(i))
+                  val paddedActualRow = padAny(actualRowField, rowWidths(i))
+                  val paddedExpected  = padAny(expectedRowField, rowWidths(i))
                   (Red(paddedActualRow), Green(paddedExpected))
               }
             val start = DarkGray("")
@@ -81,7 +80,7 @@ object DataframeUtil {
       }
     }
 
-    val headerWithLeftPadding = pad(fieldNames, truncate, rowWidths)
+    val headerWithLeftPadding = pad(fieldNames, rowWidths)
     val headerFields          = List(headerWithLeftPadding.mkString("|"))
     val colWidths: Array[Int] = getColWidths(minColWidth, diff)
 
@@ -92,13 +91,10 @@ object DataframeUtil {
 
     sb.append(separatorLine)
 
-    headerFields.zipWithIndex
-      .map { case (cell, i) =>
-        if (truncate > 0) {
-          StringUtils.leftPad(cell, colWidths(i))
-        } else {
-          StringUtils.rightPad(cell, colWidths(i))
-        }
+    headerFields
+      .zip(colWidths)
+      .map { case (cell, colWidth) =>
+        StringUtils.leftPad(cell, colWidth)
       }
       .addString(sb, StringUtils.leftPad("|", largestIndexOffset), "|", "|\n")
     diff.zipWithIndex.map { case (actual :: expected :: Nil, i) =>
@@ -125,34 +121,14 @@ object DataframeUtil {
     sb.append(separatorLine).toString()
   }
 
-  private def pad(row: Row, truncate: Int, colWidths: Array[Int]): Seq[String] =
-    pad(
-      row.toSeq.map { v =>
-        Option(v).map(_.toString).getOrElse("null")
-      },
-      truncate,
-      colWidths
-    )
+  private def pad(items: Seq[Any], colWidths: Array[Int]): Seq[String] =
+    items.zip(colWidths).map { case (v, colWidth) => padAny(v, colWidth) }
 
-  private def padAny(s: Any, truncate: Int, width: Int) = {
+  private def padAny(s: Any, width: Int) = {
     val cell = Option(s).map(_.toString).getOrElse("null")
-    if (truncate > 0) {
-      StringUtils.leftPad(cell, width)
-    } else {
-      StringUtils.rightPad(cell, width)
-    }
+    StringUtils.leftPad(cell, width)
   }
 
-  private def pad(header: Seq[String], truncate: Int, colWidths: Array[Int]) =
-    header.zipWithIndex
-      .map { case (cell, i) =>
-        if (truncate > 0) {
-          StringUtils.leftPad(cell, colWidths(i))
-        } else {
-          StringUtils.rightPad(cell, colWidths(i))
-        }
-      }
-
   private def getColWidths(minColWidth: Int, rows: Array[List[Str]]) = {
     val numCols = rows.map(_.length).max
     // Initialise the width of each column to a minimum value

core/src/test/scala/com/github/mrpowers/spark/fast/tests/DataFrameComparerTest.scala

@@ -553,6 +553,7 @@ class DataFrameComparerTest extends AnyFreeSpec with DataFrameComparer with Spar
       val e = intercept[DatasetContentMismatch] {
         assertSmallDataFrameEquality(sourceDF, expectedDF, outputFormat = DataframeDiffOutputFormat.SeparateLines)
       }
+      println(e.getMessage)
       assert(
         e.getMessage ==
           """Difference
@@ -600,6 +601,7 @@ class DataFrameComparerTest extends AnyFreeSpec with DataFrameComparer with Spar
       val e = intercept[DatasetContentMismatch] {
         assertSmallDataFrameEquality(sourceDF, expectedDF, outputFormat = DataframeDiffOutputFormat.SeparateLines)
       }
+      println(e.getMessage)
       val expected =
         """|Difference
            |  +------+---+-------+
@@ -647,6 +649,7 @@ class DataFrameComparerTest extends AnyFreeSpec with DataFrameComparer with Spar
       val e = intercept[DatasetContentMismatch] {
         assertSmallDataFrameEquality(sourceDF, expectedDF, outputFormat = DataframeDiffOutputFormat.SeparateLines)
       }
+      println(e.getMessage)
 
       assert(e.getMessage == """Difference
                                |  +-----+---+-------+
@@ -689,6 +692,7 @@ class DataFrameComparerTest extends AnyFreeSpec with DataFrameComparer with Spar
       val e = intercept[DatasetContentMismatch] {
         assertSmallDataFrameEquality(sourceDF, expectedDF, outputFormat = DataframeDiffOutputFormat.SeparateLines)
       }
+      println(e.getMessage)
 
       assert(e.getMessage == """Difference
                                |  +------+---+-------+
@@ -726,6 +730,7 @@ class DataFrameComparerTest extends AnyFreeSpec with DataFrameComparer with Spar
       val e = intercept[DatasetContentMismatch] {
         assertSmallDataFrameEquality(actualDF, expectedDF, outputFormat = DataframeDiffOutputFormat.SeparateLines)
       }
+      println(e.getMessage)
 
       assert(e.getMessage == """Difference
                                |  +------+---+-------+
@@ -764,6 +769,7 @@ class DataFrameComparerTest extends AnyFreeSpec with DataFrameComparer with Spar
       val e = intercept[DatasetContentMismatch] {
         assertSmallDataFrameEquality(actualDF, expectedDF, outputFormat = DataframeDiffOutputFormat.SeparateLines)
       }
+      println(e.getMessage)
 
       assert(
         e.getMessage == """Difference