More nippy/arrow optimizations.

Author: cnuernber

CHANGELOG.md

@@ -1,4 +1,8 @@
 # Changelog
+# 7.037
+ * Nippy loading is about 2x faster in the case of large string tables.
+ * Arrow read pathways support :text-as-strings? to mirror :strings-as-text? on the write side so you can save out uncompressed data in the fastest-to-read format.
+ 
 # 7.036
  * Major optimization (>9x!) loading of arrow files when large string tables/dictionaries are used.
 

src/tech/v3/dataset/base.clj

@@ -918,17 +918,20 @@
             string-data ^bytes string-data
             offsets (dtype/->buffer offsets)
             n-elems (dec (.lsize offsets))
-            ^IMutList int->str
-            (->> (dtype/make-reader
-                  :string n-elems
-                  (let [start-off (.readLong offsets idx)
-                        end-off (.readLong offsets (inc idx))]
-                    (String. string-data start-off (- end-off start-off))))
-                 (dtype/make-container :list :string))
-            str->int (HashMap. (dtype/ecount int->str))]
-        (dotimes [idx n-elems]
-          (.put str->int (.get int->str idx) idx))
-        (StringTable. int->str str->int int-data))
+            str-rdr (dtype/make-reader
+                     :string n-elems
+                     (let [start-off (.readLong offsets idx)
+                           end-off (.readLong offsets (inc idx))]
+                       (String. string-data start-off (- end-off start-off))))
+            str-ary (hamf/object-array n-elems)]
+        (hamf/pgroups n-elems (fn [^long sidx ^long eidx]
+                                (loop [idx sidx]
+                                  (when (< idx eidx)
+                                    (let [start-off (.readLong offsets idx)
+                                          end-off (.readLong offsets (inc idx))]
+                                      (aset str-ary idx (String. string-data start-off (- end-off start-off))))
+                                    (recur (inc idx))))))
+        (StringTable. (ham_fisted.ArrayLists/toList str-ary) nil int-data))
       (= version 2)
       (let [^List int->str (dtype-list/wrap-container string-table)
             str->int (HashMap. (dtype/ecount int->str))

src/tech/v3/libs/arrow.clj

@@ -1313,7 +1313,7 @@ Dependent block frames are not supported!!")
 
 
 (defn- string-data->column-data
-  [dict-map encoding offset-buf-dtype buffers n-elems]
+  [dict-map encoding offset-buf-dtype buffers n-elems options]
   (if encoding
     (StringTable. (get-in dict-map [(:id encoding) :strings])
                   nil
@@ -1322,11 +1322,13 @@ Dependent block frames are not supported!!")
                        (get-in encoding [:index-type :datatype]))
                       (native-buffer/->jvm-array 0 n-elems)
                       (dyn-int-list/make-from-container)))
-    (let [[offsets varchar-data] buffers]
-      (-> (offsets-data->string-reader (native-buffer/set-native-datatype
-                                        offsets offset-buf-dtype)
-                                       varchar-data n-elems)
-          (string-reader->text-reader)))))
+    (let [[offsets varchar-data] buffers
+          str-rdr (offsets-data->string-reader (native-buffer/set-native-datatype
+                                                offsets offset-buf-dtype)
+                                               varchar-data n-elems)]
+      (if-not (:text-as-strings? options)
+        (string-reader->text-reader)
+        str-rdr))))
 
 
 (defn- int8-buf->missing
@@ -1415,7 +1417,8 @@ Dependent block frames are not supported!!")
       dict-map encoding
       (get-in field [:field-type :offset-buffer-datatype])
       data-buffers
-      (:n-elems node))
+      (:n-elems node)
+      options)
      (field-metadata field)
      (node-buf->missing node validity-buf))))