Allow indexed data to be a correctly-sized array-list.

Author: cnuernber

java/tech/v3/dataset/ByteValidity.java

@@ -4,6 +4,7 @@
 import clojure.lang.IDeref;
 import java.util.function.LongConsumer;
 import ham_fisted.ArrayLists;
+import ham_fisted.IMutList;
 import org.roaringbitmap.RoaringBitmap;
 
 
@@ -29,27 +30,40 @@ public ValidityBase(long nElems, long maxIndexes) {
       idx = 0;
     }
   }
-  public static class ValidityIndexReducer extends ValidityBase {
-    public ValidityIndexReducer(long nElems, long maxIndexes) {
-      super(nElems, maxIndexes);
+  public static class ValidityIndexReducer implements LongConsumer, IDeref {
+    IMutList indexes;
+    public final long maxIdx;
+    int idx;
+    public ValidityIndexReducer(IMutList indexes, long maxIdx) {
+      this.indexes = indexes;
+      this.maxIdx = maxIdx;
+      this.idx = 0;
+    }
+    public int trimIndexes() {
+      int nIndexes = indexes.size();
+      //empty loop intentional
+      for(;nIndexes > 0 && indexes.getLong(nIndexes-1) >= maxIdx; --nIndexes);
+      
+      return nIndexes;
     }
     public void accept(long value) {
       if(value != 0) {
 	int intVal = (int)value;
 	int offset = idx * 8;
-	if( (intVal & 1) == 1) indexes[nIndexes++] = offset;
-	if( (intVal & 2) == 2) indexes[nIndexes++] = offset+1;
-	if( (intVal & 4) == 4) indexes[nIndexes++] = offset+2;
-	if( (intVal & 8) == 8) indexes[nIndexes++] = offset+3;
-	if( (intVal & 16) == 16) indexes[nIndexes++] = offset+4;
-	if( (intVal & 32) == 32) indexes[nIndexes++] = offset+5;
-	if( (intVal & 64) == 64) indexes[nIndexes++] = offset+6;
-	if( (intVal & 128) == 128) indexes[nIndexes++] = offset+7;
+	if( (intVal & 1) == 1) indexes.addLong(offset);
+	if( (intVal & 2) == 2) indexes.addLong(offset+1);
+	if( (intVal & 4) == 4) indexes.addLong(offset+2);
+	if( (intVal & 8) == 8) indexes.addLong(offset+3);
+	if( (intVal & 16) == 16) indexes.addLong(offset+4);
+	if( (intVal & 32) == 32) indexes.addLong(offset+5);
+	if( (intVal & 64) == 64) indexes.addLong(offset+6);
+	if( (intVal & 128) == 128) indexes.addLong(offset+7);
       }
       ++idx;
     }
     public Object deref() {
-      return ArrayLists.toList(indexes).subList(0, trimIndexes(indexes, nIndexes, nElems)); }
+      return indexes.subList(0, trimIndexes());
+    }
   }
   public static class MissingIndexReducer extends ValidityBase {
     public MissingIndexReducer(long nElems, long maxIndexes) {

src/tech/v3/dataset/impl/sparse_column.clj

@@ -8,14 +8,16 @@
             [tech.v3.dataset.protocols :as ds-proto]
             [tech.v3.dataset.impl.column :as col-impl]
             [tech.v3.dataset.string-table :as str-t]
+            [tech.v3.dataset.dynamic-int-list :as int-list]
             [ham-fisted.set :as set]
             [ham-fisted.api :as hamf]
             [ham-fisted.reduce :as hamf-rf]
             [ham-fisted.function :as hamf-fn]
             [ham-fisted.iterator :as hamf-iter])
   (:import [ham_fisted IMutList ArrayLists ITypedReduce ChunkedList]
            [tech.v3.datatype Buffer ObjectReader ElemwiseDatatype LongReader DoubleReader]
-           [java.util Arrays Iterator Map]))
+           [java.util Arrays Iterator Map List]
+           [tech.v3.dataset.string_table StringTable]))
 
 (set! *warn-on-reflection* true)
 (set! *unchecked-math* :warn-on-boxed)
@@ -30,8 +32,8 @@
                 idx (Arrays/binarySearch ^bytes idx-ary (unchecked-byte idx)))
       :int16 (hamf-fn/long-unary-operator
                  idx (Arrays/binarySearch ^shorts idx-ary (unchecked-short idx)))
-      :int32 (hamf-fn/long-unary-operator                 
-                 idx (Arrays/binarySearch ^shorts idx-ary (unchecked-short idx)))
+      :int32 (hamf-fn/long-unary-operator
+                 idx (Arrays/binarySearch ^ints idx-ary (unchecked-int idx)))
       (hamf-fn/long-unary-operator
           idx (Arrays/binarySearch ^longs idx-ary (unchecked-long idx))))))
 
@@ -258,6 +260,11 @@
   ^SparseCol [indexes data ^long rc metadata]
   (SparseCol. indexes data rc metadata nil nil))
 
+(defn- as-string-table
+  ^StringTable [d] (when (and (instance? StringTable d)
+                              (.-str->int ^StringTable d))
+                     d))
+
 (defn ->scol
   ^SparseCol [col]
   (cond
@@ -277,11 +284,21 @@
                            :else
                            (hamf/long-array valid-indexes)))
           col-dt (dt/elemwise-datatype col)
-          buf-rdr (dt/->reader col)
-          data (dt/make-container col-dt (.size valid-indexes))
-          dst (dt/->buffer data)]
-      (reduce (hamf-rf/indexed-long-accum _acc dst-idx src-idx
-                (.writeObject dst dst-idx (.readObject buf-rdr src-idx)))
+          buf-rdr (dt/->reader (ds-proto/column-data col))
+          ^IMutList data (if (identical? :string col-dt)
+                           (if-let [^StringTable strt (as-string-table (ds-proto/column-data col))]
+                             (StringTable. (.-int->str strt)
+                                           (.-str->int strt)
+                                           (int-list/dynamic-int-list (dt/ecount valid-indexes)))
+                             (str-t/make-string-table))
+                           (dt/make-list col-dt (.size valid-indexes)))]
+      (reduce (case (tech.v3.datatype.casting/simple-operation-space col-dt)
+                :int64 (fn [_acc ^long src-idx]
+                         (.addLong data (.readLong buf-rdr src-idx)))
+                :float64 (fn [_acc ^long src-idx]
+                           (.addDouble data (.readDouble buf-rdr src-idx)))
+                (fn [_acc ^long src-idx]
+                  (.add data (.readObject buf-rdr src-idx))))
               nil valid-indexes)
       (SparseCol. valid-indexes data rc (meta col) nil nil))))
 

src/tech/v3/libs/arrow.clj

@@ -1513,7 +1513,16 @@ Dependent block frames are not supported!!")
   (->> validity
        (hamf-rf/reduce-reducer
         (hamf-rf/long-consumer-reducer
-         #(tech.v3.dataset.ByteValidity$ValidityIndexReducer. n-elems (* 8 (dtype/ecount validity)))))))
+         #(tech.v3.dataset.ByteValidity$ValidityIndexReducer. (cond
+                                                                (< n-elems Byte/MAX_VALUE)
+                                                                (hamf/byte-array-list)
+                                                                (< n-elems Short/MAX_VALUE)
+                                                                (hamf/short-array-list)
+                                                                (< n-elems Integer/MAX_VALUE)
+                                                                (hamf/int-array-list)
+                                                                :else
+                                                                (hamf/long-array-list))
+                                                              n-elems)))))
 
 (defn ^:no-doc byte-buffer->bitwise-boolean-buffer
   ^Buffer[bitbuffer ^long n-elems]