Merge pull request #55 from i2mint/dev

Dev

Author: thorwhalen

py2store/__init__.py

@@ -4,14 +4,32 @@
 file_sep = os.path.sep
 
 # Imports to be able to easily get started...
-from py2store.base import KvCollection, KvReader, KvPersister, Reader, Persister
-from py2store.stores.local_store import LocalStore, LocalBinaryStore, LocalTextStore, PickleStore
+from py2store.base import Collection, KvReader, KvPersister, Reader, Persister
+
+from py2store.stores.local_store import LocalStore, LocalBinaryStore, LocalTextStore, LocalPickleStore, LocalJsonStore
 from py2store.stores.local_store import QuickStore, QuickBinaryStore, QuickTextStore, QuickJsonStore, QuickPickleStore
+
+from py2store.misc import MiscGetter, MiscGetterAndSetter, misc_objs, misc_objs_get, get_obj, set_obj
 from py2store.base import Store
-from py2store.trans import wrap_kvs
-from py2store.access import user_configs_dict, user_configs, user_defaults_dict, user_defaults
+from py2store.trans import wrap_kvs, disable_delitem, disable_setitem, mk_read_only, kv_wrap, cache_iter, filtered_iter
+from py2store.access import user_configs_dict, user_configs, user_defaults_dict, user_defaults, mystores
+
+from py2store.stores.local_store import PickleStore  # consider deprecating and use LocalPickleStore instead?
+from py2store.persisters.local_files import ZipReader, FilesOfZip, ZipFilesReader
 
 with ModuleNotFoundIgnore():
     from py2store.stores.s3_store import S3BinaryStore, S3TextStore, S3PickleStore
 with ModuleNotFoundIgnore():
     from py2store.stores.mongo_store import MongoStore, MongoTupleKeyStore, MongoAnyKeyStore
+
+
+def kvhead(store):
+    """Get the first item of a kv store"""
+    for k in store:
+        return k, store[k]
+
+
+def ihead(store):
+    """Get the first item of an iterable"""
+    for item in store:
+        return item

py2store/access.py

@@ -1,5 +1,13 @@
 """
-The logic to allow configurations (and defaults) to be parametrized by external environmental variables and files.
+Utils to load stores from store specifications.
+Includes the logic to allow configurations (and defaults) to be parametrized by external environmental
+variables and files.
+
+Every data-sourced problem has it's problem-relevant stores. Once you get your stores right, along with the
+right access credentials, indexing, serialization, caching, filtering etc. you'd like to be able to name, save
+and/or share this specification, and easily get access to it later on.
+
+Here are tools to help you out.
 
 There are two main key-value stores: One for configurations the user wants to reuse, and the other for the user's
 desired defaults. Both have the same structure:
@@ -15,9 +23,18 @@
 respectively.
 """
 import os
+import importlib
 from warnings import warn
+from functools import reduce
 from py2store.util import DictAttr, str_to_var_str
 
+FAK = '$fak'
+
+
+# TODO: Make a config_utils.py module to centralize config tools (configs for access is just one -- serializers another)
+# TODO: Integrate (external because not standard lib) other safer tools for secrets, such as:
+#  https://github.com/SimpleLegal/pocket_protector
+
 
 def getenv(name, default=None):
     """Like os.getenv, but removes a suffix \\r character if present (problem with some env var systems)"""
@@ -28,10 +45,105 @@ def getenv(name, default=None):
         return v
 
 
+def assert_callable(f: callable) -> callable:
+    assert callable(f), f"Is not callable: {f}"
+    return f
+
+
+def dotpath_to_obj(dotpath):
+    """Loads and returns the object referenced by the string DOTPATH_TO_MODULE.OBJ_NAME"""
+    *module_path, obj_name = dotpath.split('.')
+    return getattr(importlib.import_module('.'.join(module_path)), obj_name)
+
+
+def dotpath_to_func(f: (str, callable)) -> callable:
+    """Loads and returns the function referenced by f,
+    which could be a callable or a DOTPATH_TO_MODULE.FUNC_NAME dotpath string to one.
+    """
+
+    if isinstance(f, str):
+        if '.' in f:
+            *module_path, func_name = f.split('.')
+            f = getattr(importlib.import_module('.'.join(module_path)), func_name)
+        else:
+            f = getattr(importlib.import_module('py2store'), f)
+
+    return assert_callable(f)
+
+
+def compose(*functions):
+    """Make a function that is the composition of the input functions"""
+    return reduce(lambda f, g: lambda x: f(g(x)), functions, lambda x: x)
+
+
+def dflt_func_loader(f) -> callable:
+    """Loads and returns the function referenced by f,
+    which could be a callable or a DOTPATH_TO_MODULE.FUNC_NAME dotpath string to one, or a pipeline of these
+    """
+    if isinstance(f, str) or callable(f):
+        return dotpath_to_func(f)
+    else:
+        return compose(*map(dflt_func_loader, f))
+
+
+def _fakit(f: callable, a: (tuple, list), k: dict):
+    return f(*(a or ()), **(k or {}))
+
+
+def fakit_from_dict(d, func_loader=assert_callable):
+    return _fakit(func_loader(d['f']), a=d.get('a', ()), k=d.get('k', {}))
+
+
+def fakit_from_tuple(t: (tuple, list), func_loader: callable = dflt_func_loader):
+    f = func_loader(t[0])
+    a = ()
+    k = {}
+    assert len(t) in {1, 2, 3}, "A tuple fak must be of length 1, 2, or 3. No more, no less."
+    if len(t) > 1:
+        if isinstance(t[1], dict):
+            k = t[1]
+        else:
+            assert isinstance(t[1], (tuple, list)), "argument specs should be dict, tuple, or list"
+            a = t[1]
+        if len(t) > 2:
+            if isinstance(t[2], dict):
+                assert not k, "can only have one kwargs"
+                k = t[2]
+            else:
+                assert isinstance(t[2], (tuple, list)), "argument specs should be dict, tuple, or list"
+                assert not a, "can only have one args"
+                a = t[2]
+    return _fakit(f, a, k)
+
+
+def fakit(fak, func_loader=dflt_func_loader):
+    """Execute a fak with given f, a, k and function loader.
+
+    Essentially returns func_loader(f)(*a, **k)
+
+    Args:
+        fak: A (f, a, k) specification. Could be a tuple or a dict (with 'f', 'a', 'k' keys). All but f are optional.
+        func_loader: A function returning a function. This is where you specify any validation of func specification f,
+            and/or how to get a callable from it.
+
+    Returns: A python object.
+    """
+
+    if isinstance(fak, dict):
+        return fakit_from_dict(fak, func_loader=func_loader)
+    else:
+        assert isinstance(fak, (tuple, list)), "fak should be dict, tuple, or list"
+        return fakit_from_tuple(fak, func_loader=func_loader)
+
+
+fakit.from_dict = fakit_from_dict
+fakit.from_tuple = fakit_from_tuple
+
 user_configs_dict = {}
 user_defaults_dict = {}
 user_configs = None
 user_defaults = None
+mystores = None
 
 try:
     import json
@@ -53,6 +165,44 @@ def directory_json_items():
 
         user_configs = DictAttr(**{k: v for k, v in directory_json_items()})
 
+        from py2store.base import KvStore
+        from py2store.stores.local_store import LocalJsonStore
+        from py2store.trans import wrap_kvs
+
+
+        class MyStores(KvStore):
+            func_loader = staticmethod(dflt_func_loader)
+
+            def _obj_of_data(self, data):
+                if '$fak' in data:
+                    return fakit(data['$fak'], self.func_loader)
+                else:
+                    msg = "Case not handled by MyStores"
+                    if isinstance(data, dict):
+                        raise ValueError(f"{msg}: keys: {list(data.keys())}")
+                    else:
+                        raise ValueError(f"{msg}: type: {type(data)}")
+
+            @property
+            def configs(self):
+                return self.store
+
+
+        def without_json_ext(_id):
+            assert _id.endswith('.json'), "Should end with .json"
+            return _id[:-len('.json')]
+
+
+        def add_json_ext(k):
+            return k + '.json'
+
+
+        ExtLessJsonStore = wrap_kvs(LocalJsonStore, name='ExtLessJsonStore',
+                                    key_of_id=without_json_ext, id_of_key=add_json_ext)
+
+        stores_json_path_format = os.path.join(user_configs_dirpath, 'stores', 'json', '{}.json')
+        mystores = MyStores(ExtLessJsonStore(stores_json_path_format))
+
     else:
         warn(f"The configs directory wasn't found (please make it): {user_configs_dirpath}")
         warn("Configs in a single json is being deprecated")

py2store/base.py

@@ -24,7 +24,8 @@
 the storage methods themselves.
 """
 
-from collections.abc import Collection, Mapping, MutableMapping
+from collections.abc import Collection as CollectionABC
+from collections.abc import Mapping, MutableMapping
 from typing import Any, Iterable, Tuple
 
 Key = Any
@@ -37,20 +38,19 @@
 ItemIter = Iterable[Item]
 
 
-class KvCollection(Collection):
+class Collection(CollectionABC):
 
-    def __contains__(self, k: Key) -> bool:
+    def __contains__(self, x) -> bool:
         """
         Check if collection of keys contains k.
-        Note: This method actually fetches the contents for k, returning False if there's a key error trying to do so
+        Note: This method loops through all contents of collection to see if query element exists.
         Therefore it may not be efficient, and in most cases, a method specific to the case should be used.
         :return: True if k is in the collection, and False if not
         """
-        try:
-            self.__getitem__(k)
-            return True
-        except KeyError:
-            return False
+        for existing_x in self.__iter__():
+            if existing_x == x:
+                return True
+        return False
 
     def __len__(self) -> int:
         """
@@ -59,21 +59,44 @@ def __len__(self) -> int:
         Therefore it is not efficient, and in most cases should be overridden with a more efficient version.
         :return: The number (int) of elements in the collection of keys.
         """
-        # TODO: some other means to more quickly count files?
         # Note: Found that sum(1 for _ in self.__iter__()) was slower for small, slightly faster for big inputs.
         count = 0
         for _ in self.__iter__():
             count += 1
         return count
 
     def head(self):
-        return next(iter(self.items()))
+        if hasattr(self, 'items'):
+            return next(iter(self.items()))
+        else:
+            return next(iter(self))
+
+
+KvCollection = Collection  # alias meant for back-compatibility. Would like to deprecated
 
 
-class KvReader(KvCollection, Mapping):
+# def getitem_based_contains(self, x) -> bool:
+#     """
+#     Check if collection of keys contains k.
+#     Note: This method actually fetches the contents for k, returning False if there's a key error trying to do so
+#     Therefore it may not be efficient, and in most cases, a method specific to the case should be used.
+#     :return: True if k is in the collection, and False if not
+#     """
+#
+#     try:
+#         self.__getitem__(k)
+#         return True
+#     except KeyError:
+#         return False
+
+
+class KvReader(Collection, Mapping):
     """Acts as a Mapping abc, but with default __len__ (implemented by counting keys)
     and head method to get the first (k, v) item of the store"""
-    pass
+
+    def head(self):
+        for k, v in self.items():
+            yield k, v
 
 
 Reader = KvReader  # alias
@@ -117,11 +140,12 @@ class Store(Persister):
     """
     By store we mean key-value store. This could be files in a filesystem, objects in s3, or a database. Where and
     how the content is stored should be specified, but StoreInterface offers a dict-like interface to this.
+    ::
+        __getitem__ calls: _id_of_key			                    _obj_of_data
+        __setitem__ calls: _id_of_key		        _data_of_obj
+        __delitem__ calls: _id_of_key
+        __iter__    calls:	            _key_of_id
 
-    __getitem__ calls: _id_of_key			                    _obj_of_data
-    __setitem__ calls: _id_of_key		        _data_of_obj
-    __delitem__ calls: _id_of_key
-    __iter__    calls:	            _key_of_id
 
     >>> # Default store: no key or value conversion ################################################
     >>> s = Store()
@@ -226,25 +250,56 @@ def __getitem__(self, k: Key) -> Val:
         return self._obj_of_data(self.store.__getitem__(self._id_of_key(k)))
 
     def get(self, k: Key, default=None) -> Val:
-        data = self.store.get(self._id_of_key(k), no_such_item)
-        if data is not no_such_item:
-            return self._obj_of_data(data)
-        else:
-            return default
+        if hasattr(self.store, 'get'):  # if store has a get method, use it
+            data = self.store.get(self._id_of_key(k), no_such_item)
+            if data is not no_such_item:
+                return self._obj_of_data(data)
+            else:
+                return default
+        else:  # if not, do the get function otherwise
+            if k in self:
+                return self._obj_of_data(self[k])
+            else:
+                return default
 
     # Explore ####################################################################
     def __iter__(self) -> KeyIter:
         return map(self._key_of_id, self.store.__iter__())
 
+    # def items(self) -> ItemIter:
+    #     if hasattr(self.store, 'items'):
+    #         yield from ((self._key_of_id(k), self._obj_of_data(v)) for k, v in self.store.items())
+    #     else:
+    #         yield from ((self._key_of_id(k), self._obj_of_data(self.store[k])) for k in self.store.__iter__())
+
     def __len__(self) -> int:
         return self.store.__len__()
 
     def __contains__(self, k) -> bool:
         return self.store.__contains__(self._id_of_key(k))
 
     def head(self) -> Item:
-        for k, v in self.items():
-            return k, v
+        try:
+            for k in self:
+                return k, self[k]
+        except Exception as e:
+
+            from warnings import warn
+            msg = f"Couldn't get data for the key {k}. This could be be...\n"
+            msg += "... because it's not a store (just a collection, that doesn't have a __getitem__)\n"
+            msg += "... because there's a layer transforming outcoming keys that are not the ones the store actually " \
+                   "uses? If you didn't wrap the store with the inverse ingoing keys transformation, " \
+                   "that would happen.\n"
+            msg += "I'll ask the inner-layer what it's head is, but IT MAY NOT REFLECT the reality of your store " \
+                   "if you have some filtering, caching etc."
+            msg += f"The error messages was: \n{e}"
+            warn(msg)
+
+            for _id in self.store:
+                return self._key_of_id(_id), self._obj_of_data(self.store[_id])
+        # NOTE: Old version didn't work when key mapping was asymmetrical
+        # for k, v in self.items():
+        #     return k, v
 
     # Write ####################################################################
     def __setitem__(self, k: Key, v: Val):
@@ -277,6 +332,7 @@ def __repr__(self):
 
 def has_kv_store_interface(o):
     """Check if object has the KvStore interface (that is, has the kv wrapper methods
+
     Args:
         o: object (class or instance)