Merge pull request #83 from Quafadas/stats

Stats

Author: Quafadas

.github/workflows/ci.yml

@@ -27,9 +27,13 @@ jobs:
           fetch-depth: 0
           fetch-tags: true
       - uses: coursier/cache-action@v6.4
-      - name: Compile
+      - name: Compile JS
+        run: ./mill scautable.js.compile
+      - name: Compile JVM
+        run: ./mill scautable.jvm.compile
+      - name: Compile All
         run: ./mill __.compile
-      - name: test
+      - name: test All
         run: ./mill scautable.test._
       - name: Publish artifacts
         run: ./mill -i __.publishArtifacts

build.mill

@@ -30,7 +30,7 @@ trait Common extends ScalaModule:
     Config.oslib,
     Config.fansi
   )
-  override def scalacOptions: T[Seq[String]] = super.scalacOptions() ++ Seq("-Xmax-inlines", "128")
+  override def scalacOptions = super.scalacOptions() ++ Seq("-Xmax-inlines", "128")
 end Common
 
 trait CommonJS extends Common with ScalaJSModule:

scautable/package.mill

@@ -15,7 +15,8 @@ object `package` extends Module:
   object jvm extends SharedModule with ScalaModule with build.PublishModule:
     override def mvnDeps = super.mvnDeps() ++ Seq(
       mvn"org.apache.poi:poi:5.4.1",
-      mvn"org.apache.poi:poi-ooxml:5.4.1"
+      mvn"org.apache.poi:poi-ooxml:5.4.1",
+      mvn"com.tdunning:t-digest:3.3"
     )
   end jvm
 

scautable/src-js/stats.scala

@@ -0,0 +1,8 @@
+package io.github.quafadas.scautable
+
+/**
+  * Api compatibility object.
+  */
+object Stats:
+
+end Stats

scautable/src-jvm/stats.scala

@@ -24,60 +24,6 @@ object NamedTupleIteratorExtensions:
 
   extension [K <: Tuple, V <: Tuple](itr: Iterator[NamedTuple[K, V]])
 
-    inline def numericTypeTest: (List[ConversionAcc], Long) =
-      val headers = constValueTuple[K].toList.map(_.toString())
-      val headerAcc = headers.map(_ => ConversionAcc(0, 0, 0))
-
-      itr.foldLeft[List[ConversionAcc] *: Long *: EmptyTuple]((headerAcc, 0L)) { case (acc: (List[ConversionAcc], Long), elem: NamedTuple[K, V]) =>
-        val list = elem.toList
-          .asInstanceOf[List[String]]
-          .zip(acc._1)
-          .map { case (str, acc) =>
-            (
-              ConversionAcc(
-                acc.validInts + str.toIntOption.fold(0)(_ => 1),
-                acc.validDoubles + str.toDoubleOption.fold(0)(_ => 1),
-                acc.validLongs + str.toLongOption.fold(0)(_ => 1)
-              )
-            )
-          }
-
-        (list, acc._2 + 1L)
-
-      }
-    end numericTypeTest
-
-    inline def formatTypeTest: String =
-      val headers = constValueTuple[K].toList.map(_.toString())
-      val (asList, n) = numericTypeTest
-      val intReport = (
-        "int" *: listToTuple(
-          for (acc <- asList) yield (acc.validInts / n.toDouble).formatAsPercentage
-        )
-      )
-      val doubleReported = "doubles" *: listToTuple(
-        for (acc <- asList) yield (acc.validDoubles / n.toDouble).formatAsPercentage
-      )
-      val longReported = "long" *: listToTuple(
-        for (acc <- asList) yield (acc.validLongs / n.toDouble).formatAsPercentage
-      )
-      val recommendation = "recommendation" *: listToTuple(
-        for (acc <- asList) yield recommendConversion(List(acc), n)
-      )
-
-      val ntList = Seq(
-        intReport,
-        doubleReported,
-        longReported,
-        recommendation
-      )
-
-      ConsoleFormat.consoleFormat_(headers = "conversion % to" +: headers, fancy = true, table = ntList)
-    end formatTypeTest
-
-    inline def showTypeTest: Unit =
-      println(formatTypeTest)
-
     inline def sample(frac: Double, inline deterministic: Boolean = false): Iterator[NamedTuple[K, V]] =
       if deterministic then itr.zipWithIndex.filter { case (_, idx) => idx % (1 / frac) == 0 }.map(_._1)
       else itr.filter(_ => rand.nextDouble() < frac)

scautable/src/columnExtensions.scala

@@ -11,7 +11,7 @@ object table:
   export io.github.quafadas.scautable.NamedTupleIteratorExtensions.*
   export io.github.quafadas.scautable.CSVWriterExtensions.*
   export io.github.quafadas.scautable.HeaderOptions.*
-
+  export io.github.quafadas.scautable.Stats.*
 
   /**
    * Enumeration of strategies for inferring column types when reading CSV files.