Added user-controlled sframe and sarray pickling functionality. (#3327)

Added user-controlled SFrame pickling functionality.

Because the SFrame datastructure is  inherently disk-backed,
pickling is enabled with the user specifying a directory in which to save the
serialized artifacts.  Pickling creates a uuid-based sframe in this
directory, which the unpickling references.

Co-authored-by: Hoyt Koepke <[email protected]>

Author: Hoyt Koepke

src/python/turicreate/_cython/cy_sframe.pyx

@@ -32,6 +32,9 @@ from .cy_model cimport create_model_from_proxy
 from .cy_cpp_utils cimport str_to_cpp, cpp_to_str
 from .cy_cpp_utils cimport to_vector_of_strings, from_vector_of_strings
 from .cy_cpp_utils cimport to_nested_vectors_of_strings, dict_to_string_string_map
+        
+from ..data_structures.serialization import _safe_serialization_directory
+import os
 
 cdef create_proxy_wrapper_from_existing_proxy(const unity_sframe_base_ptr& proxy):
     if proxy.get() == NULL:
@@ -53,6 +56,9 @@ cdef pydict_from_gl_error_map(gl_error_map& d):
         inc(it)
     return ret
 
+
+
+
 cdef class UnitySFrameProxy:
 
     def __cinit__(self, do_not_allocate=None):
@@ -375,3 +381,23 @@ cdef class UnitySFrameProxy:
     cpdef delete_on_close(self):
         with nogil:
           self.thisptr.delete_on_close()
+
+
+    def __reduce__(self):
+        import uuid
+
+        save_dir = _safe_serialization_directory()
+        sframe_id = str(uuid.uuid4())
+        
+        self.save(os.path.join(save_dir, sframe_id))
+        return (_UnitySFrame_unpickler, (sframe_id,) )
+
+def _UnitySFrame_unpickler(sframe_id):
+    proxy = UnitySFrameProxy()
+
+    load_dir = _safe_serialization_directory()
+
+    proxy.load_from_sframe_index(os.path.join(load_dir, sframe_id))
+
+    return proxy
+

src/python/turicreate/data_structures/__init__.py

@@ -12,10 +12,12 @@
 from __future__ import division as _
 from __future__ import absolute_import as _
 
-__all__ = ["sframe", "sarray", "sgraph", "sketch", "image"]
+__all__ = ["sframe", "sarray", "sgraph", "sketch", "image", "serialization"]
 
 from . import image
 from . import sframe
 from . import sarray
 from . import sgraph
 from . import sketch
+from . import serialization
+

src/python/turicreate/data_structures/serialization.py

@@ -0,0 +1,67 @@
+
+# -*- coding: utf-8 -*-
+# Copyright © 2020 Apple Inc. All rights reserved.
+#
+# Use of this source code is governed by a BSD-3-clause license that can
+# be found in the LICENSE.txt file or at https://opensource.org/licenses/BSD-3-Clause
+
+# SFrames require a disk-backed file or directory to work with.  This directory 
+# has to be present to allow for serialization or deserialization. 
+__serialization_directory = None 
+
+
+def enable_sframe_serialization(serialization_directory):
+    """
+    Enables pickling of SFrames through the use of a user-set directory to 
+    store the objects.  This directory must be set through his method for 
+    deserialization to work.  It may be a different directory for serialization and 
+    unserialization. 
+     
+    When an SFrame is pickled, a copy of the SFrame is saved in this
+    directory and a reference handle to a randomly generated subdirectory is saved in the 
+    pickle.  As long as that reference handle is present in the set directory, then
+    deserialization should work. 
+    
+    Note that the pickle files themselves do not contain the data -- both the directory contents
+    and the pickle need to be present for deserialization to work. 
+    """
+
+    import os
+
+    global __serialization_directory
+    if serialization_directory is None:
+        __serialization_directory = None
+        return 
+
+    __serialization_directory = os.path.abspath(os.path.expanduser(serialization_directory))
+
+    # Make sure the directory exists.
+    if not os.path.exists(__serialization_directory):
+
+        # Attempt to create it
+        os.makedirs(__serialization_directory)
+
+    # Is it a directory? 
+    elif not os.path.isdir(__serialization_directory):
+        raise ValueError("%s is not a directory." % __serialization_directory)
+
+
+
+def get_serialization_directory():
+    """
+    Returns the current serialization directory if set, or None otherwise. 
+    """
+    global __serialization_directory
+
+    return __serialization_directory
+
+def _safe_serialization_directory():
+    global __serialization_directory
+
+    from pickle import PickleError
+
+    if __serialization_directory is None:
+        raise PickleError("Serialization directory not set to enable pickling or unpickling. "
+            "Set using turicreate.data_structures.serialization.enable_sframe_serialization().")
+        
+    return __serialization_directory

src/python/turicreate/test/test_sframe.py

@@ -642,6 +642,29 @@ def test_save_to_csv(self):
         f.close()
         os.unlink(f.name)
 
+    def test_pickling(self):
+
+        import pickle
+        from ..data_structures import serialization
+
+        X = generate_random_sframe(100, "ncc")
+
+        with util.TempDirectory() as f:
+            
+
+            self.assertRaises(pickle.PickleError, lambda: pickle.dumps(X))
+
+            serialization.enable_sframe_serialization(f)
+             
+            s = pickle.dumps(X)
+
+            Y = pickle.loads(s)
+
+            _assert_sframe_equal(X, Y)
+            
+            serialization.enable_sframe_serialization(None) # Disables it
+
+
     def test_save_to_json(self):
         f = tempfile.NamedTemporaryFile(suffix=".json", delete=False)
         sf = SFrame(data=self.dataframe, format="dataframe")

src/visualization/client/Turi Create Visualization.xcodeproj/project.pbxproj

@@ -651,6 +651,7 @@
 				GCC_WARN_UNUSED_FUNCTION = YES;
 				GCC_WARN_UNUSED_VARIABLE = YES;
 				MTL_ENABLE_DEBUG_INFO = NO;
+				ONLY_ACTIVE_ARCH = YES;
 				SDKROOT = macosx;
 				SWIFT_OPTIMIZATION_LEVEL = "-Owholemodule";
 			};