Merge pull request #63 from i2mint/dev

Dev

Author: thorwhalen

.gitignore

@@ -1,3 +1,5 @@
+.DS_Store
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

py2store/__init__.py

@@ -3,6 +3,8 @@
 
 file_sep = os.path.sep
 
+from py2store.util import lazyprop, groupby, regroupby
+
 # Imports to be able to easily get started...
 from py2store.base import Collection, KvReader, KvPersister, Reader, Persister
 
@@ -11,11 +13,20 @@
 
 from py2store.misc import MiscGetter, MiscGetterAndSetter, misc_objs, misc_objs_get, get_obj, set_obj
 from py2store.base import Store
-from py2store.trans import wrap_kvs, disable_delitem, disable_setitem, mk_read_only, kv_wrap, cache_iter, filtered_iter
-from py2store.access import user_configs_dict, user_configs, user_defaults_dict, user_defaults, mystores
+from py2store.trans import wrap_kvs, disable_delitem, disable_setitem, mk_read_only, kv_wrap, \
+    cached_keys, filtered_iter, add_path_get, insert_aliases
+from py2store.trans import cache_iter  # being deprecated
+from py2store.access import user_configs_dict, user_configs, user_defaults_dict, user_defaults
+from py2store.caching import mk_cached_store, store_cached, store_cached_with_single_key, \
+    ensure_clear_to_kv_store, flush_on_exit, mk_write_cached_store
 
 from py2store.stores.local_store import PickleStore  # consider deprecating and use LocalPickleStore instead?
-from py2store.persisters.local_files import ZipReader, FilesOfZip, ZipFilesReader
+from py2store.slib.s_zipfile import ZipReader, ZipFilesReader, FilesOfZip, FlatZipFilesReader
+
+with ModuleNotFoundIgnore():
+    from py2store.access import myconfigs
+with ModuleNotFoundIgnore():
+    from py2store.access import mystores
 
 with ModuleNotFoundIgnore():
     from py2store.stores.s3_store import S3BinaryStore, S3TextStore, S3PickleStore
@@ -31,5 +42,5 @@ def kvhead(store):
 
 def ihead(store):
     """Get the first item of an iterable"""
-    for item in store:
+    for item in iter(store):
         return item

py2store/access.py

@@ -149,7 +149,11 @@ def fakit(fak, func_loader=dflt_func_loader):
     import json
 
     user_configs_dirpath = os.path.expanduser(getenv('PY2STORE_CONFIGS_DIR', '~/.py2store_configs'))
+    my_configs_dirname = os.path.expanduser(getenv('MY_PY2STORE_DIR_NAME', 'my'))
+    myconfigs_dirpath = os.path.join(user_configs_dirpath, my_configs_dirname)
+
     if os.path.isdir(user_configs_dirpath):
+
         def directory_json_items():
             for f in filter(lambda x: x.endswith('.json'), os.listdir(user_configs_dirpath)):
                 filepath = os.path.join(user_configs_dirpath, f)
@@ -166,8 +170,56 @@ def directory_json_items():
         user_configs = DictAttr(**{k: v for k, v in directory_json_items()})
 
         from py2store.base import KvStore
-        from py2store.stores.local_store import LocalJsonStore
+        from py2store.stores.local_store import LocalJsonStore, LocalBinaryStore
         from py2store.trans import wrap_kvs
+        from py2store.misc import MiscStoreMixin
+        from functools import wraps
+
+        if os.path.isdir(myconfigs_dirpath):
+            class MyConfigs(MiscStoreMixin, LocalBinaryStore):
+                @wraps(LocalBinaryStore)
+                def __init__(self, *args, **kwargs):
+                    LocalBinaryStore.__init__(self, *args, **kwargs)
+                    self._init_args_kwargs = (args, kwargs)
+                    if len(args) > 0:
+                        self.dirpath = args[0]
+                    elif len(kwargs) > 0:
+                        self.dirpath = kwargs[next(iter(kwargs))]
+
+                def refresh(self):
+                    """Sometimes you add a file, and you want to make sure your myconfigs sees it.
+                    refresh() does that for you. It reinitializes the reader."""
+                    args, kwargs = self._init_args_kwargs
+                    self.__init__(*args, **kwargs)
+
+                def get(self, k):
+                    v = super().get(k, None)
+                    if v is None:
+                        from warnings import warn
+                        warn(f"""You don't have the config file I'm expecting, so you'll have to enter it manually.
+                        I'm expecting this filepath:\n\t {os.path.join(self.dirpath, k)}
+                        """)
+                    return v
+
+                def get_config_value(self, k, path=None):
+                    v = self.get(k)
+                    if path is None:
+                        return v
+                    else:
+                        if path in v:
+                            return v[path]
+                        else:
+                            raise KeyError(f"I don't see any {path} in {k}")
+
+
+            myconfigs = MyConfigs(myconfigs_dirpath)
+            myconfigs.dirpath = myconfigs_dirpath
+        else:
+            warn(f"""The py2store-myconfigs directory wasn't found: {myconfigs_dirpath}
+            If you want to have all the cool functionality of `myconfigs`, you should make this directory, 
+            and put stuff in it. Here's to make it easy for you to do it. Go to a terminal and run this:
+                mkdir {myconfigs_dirpath}
+            """)
 
 
         class MyStores(KvStore):
@@ -202,10 +254,10 @@ def add_json_ext(k):
 
         stores_json_path_format = os.path.join(user_configs_dirpath, 'stores', 'json', '{}.json')
         mystores = MyStores(ExtLessJsonStore(stores_json_path_format))
+        mystores.dirpath = user_configs_dirpath
 
     else:
         warn(f"The configs directory wasn't found (please make it): {user_configs_dirpath}")
-        warn("Configs in a single json is being deprecated")
         user_configs_filepath = os.path.expanduser(getenv('PY2STORE_CONFIGS_JSON_FILEPATH', '~/.py2store_configs.json'))
         if os.path.isfile(user_configs_filepath):
             user_configs_dict = json.load(open(user_configs_filepath))

py2store/base.py

@@ -38,7 +38,20 @@
 ItemIter = Iterable[Item]
 
 
+class AttrNames:
+    CollectionABC = {'__len__', '__iter__', '__contains__'}
+    Mapping = CollectionABC | {'keys', 'get', 'items', '__reversed__', 'values', '__getitem__'}
+    MutableMapping = Mapping | {'setdefault', 'pop', 'popitem', 'clear', 'update', '__delitem__', '__setitem__'}
+
+    Collection = CollectionABC | {'head'}
+    KvReader = (Mapping | {'head'}) - {'__reversed__'}
+    KvPersister = (MutableMapping | {'head'}) - {'__reversed__'} - {'clear'}
+
+
 class Collection(CollectionABC):
+    """The same as collections.abc.Collection, with some modifications:
+    - Addition of a ``head``
+    """
 
     def __contains__(self, x) -> bool:
         """
@@ -96,28 +109,86 @@ class KvReader(Collection, Mapping):
 
     def head(self):
         for k, v in self.items():
-            yield k, v
+            return k, v
+
+    def __reversed__(self):
+        """The __reversed__ is disabled at the base, but can be re-defined in subclasses.
+        Rationale: KvReader is meant to wrap a variety of storage backends or key-value perspectives thereof.
+        Not all of these would have a natural or intuitive order nor do we want to maintain one systematically.
+
+        If you need a reversed list, here's one way to do it, but note that it
+        depends on how self iterates, which is not even assured to be consistent at every call:
+        ```
+        reversed = list(self)[::-1]
+        ```
+
+        If the keys are comparable, therefore sortable, another natural option would be:
+        ```
+        reversed = sorted(self)[::-1]
+        ```
+        """
+        raise NotImplementedError(__doc__)
 
 
 Reader = KvReader  # alias
 
 
+# TODO: Should we really be using MutableMapping if we're disabling so many of it's methods?
 # TODO: Wishful thinking: Define store type so the type is defined by it's methods, not by subclassing.
-class Persister(Reader, MutableMapping):
-    """ Acts as a MutableMapping abc, but disabling the clear method, and computing __len__ by counting keys"""
+class KvPersister(KvReader, MutableMapping):
+    """ Acts as a MutableMapping abc, but disabling the clear and __reversed__ method,
+    and computing __len__ by iterating over all keys, and counting them.
+
+    Note that KvPersister is a MutableMapping, and as such, is dict-like.
+    But that doesn't mean it's a dict.
+
+    For instance, consider the following code:
+    ```
+        s = SomeKvPersister()
+        s['a']['b'] = 3
+    ```
+    If `s` is a dict, this would have the effect of adding a ('b', 3) item under 'a'.
+    But in the general case, this might
+    - fail, because the `s['a']` doesn't support sub-scripting (doesn't have a `__getitem__`)
+    - or, worse, will pass silently but not actually persist the write as expected (e.g. LocalFileStore)
+
+    Another example: `s.popitem()` will pop a `(k, v)` pair off of the `s` store.
+    That is, retrieve the `v` for `k`, delete the entry for `k`, and return a `(k, v)`.
+    Note that unlike modern dicts which will return the last item that was stored
+     -- that is, LIFO (last-in, first-out) order -- for KvPersisters,
+     there's no assurance as to what item will be, since it will depend on the backend storage system
+     and/or how the persister was implemented.
+
+    """
 
     def clear(self):
-        raise NotImplementedError('''
-        The clear method was overridden to make dangerous difficult.
-        If you really want to delete all your data, you can do so by doing:
-            try:
-                while True:
-                    self.popitem()
-            except KeyError:
-                pass''')
+        """The clear method is disabled to make dangerous difficult.
+        You don't want to delete your whole DB
+        If you really want to delete all your data, you can do so by doing something like this:
+            ```
+            for k in self:
+                try:
+                    del self[k]
+                except KeyError:
+                    pass
+            ```
+        """
+        raise NotImplementedError(__doc__)
 
+    # # TODO: Tests and documentation demos needed.
+    # def popitem(self):
+    #     """pop a (k, v) pair off of the store.
+    #     That is, retrieve the v for k, delete the entry for k, and return a (k, v)
+    #     Note that unlike modern dicts which will return the last item that was stored
+    #      -- that is, LIFO (last-in, first-out) order -- for KvPersisters,
+    #      there's no assurance as to what item will be, since it will depend on the backend storage system
+    #      and/or how the persister was implemented.
+    #     :return:
+    #     """
+    #     return super(KvPersister, self).popitem()
 
-KvPersister = Persister  # alias with explict name
+
+Persister = KvPersister  # alias for back-compatibility
 
 
 # TODO: Make identity_func "identifiable". If we use the following one, we can use == to detect it's use,
@@ -136,7 +207,7 @@ class NoSuchItem():
 no_such_item = NoSuchItem()
 
 
-class Store(Persister):
+class Store(KvPersister):
     """
     By store we mean key-value store. This could be files in a filesystem, objects in s3, or a database. Where and
     how the content is stored should be specified, but StoreInterface offers a dict-like interface to this.
@@ -245,9 +316,12 @@ def __init__(self, store=dict):
     _data_of_obj = static_identity_method
     _obj_of_data = static_identity_method
 
+    _max_repr_size = None
+
     # Read ####################################################################
     def __getitem__(self, k: Key) -> Val:
-        return self._obj_of_data(self.store.__getitem__(self._id_of_key(k)))
+        return self._obj_of_data(self.store[self._id_of_key(k)])
+        # return self._obj_of_data(self.store.__getitem__(self._id_of_key(k)))
 
     def get(self, k: Key, default=None) -> Val:
         if hasattr(self.store, 'get'):  # if store has a get method, use it
@@ -262,9 +336,20 @@ def get(self, k: Key, default=None) -> Val:
             else:
                 return default
 
+    # def update(self, other=(), /, **kwds):
+    #     """
+    #     update(self, other=(), /, **kwds)
+    # D.update([E, ]**F) -> None.  Update D from mapping/iterable E and F.
+    # If E present and has a .keys() method, does:     for k in E: D[k] = E[k]
+    # If E present and lacks .keys() method, does:     for (k, v) in E: D[k] = v
+    # In either case, this is followed by: for k, v in F.items(): D[k] = v
+    #     :return:
+    #     """
+
     # Explore ####################################################################
     def __iter__(self) -> KeyIter:
-        return map(self._key_of_id, self.store.__iter__())
+        yield from (self._key_of_id(k) for k in self.store)
+        # return map(self._key_of_id, self.store.__iter__())
 
     # def items(self) -> ItemIter:
     #     if hasattr(self.store, 'items'):
@@ -273,27 +358,33 @@ def __iter__(self) -> KeyIter:
     #         yield from ((self._key_of_id(k), self._obj_of_data(self.store[k])) for k in self.store.__iter__())
 
     def __len__(self) -> int:
-        return self.store.__len__()
+        return len(self.store)
+        # return self.store.__len__()
 
     def __contains__(self, k) -> bool:
-        return self.store.__contains__(self._id_of_key(k))
+        return self._id_of_key(k) in self.store
+        # return self.store.__contains__(self._id_of_key(k))
 
     def head(self) -> Item:
+        k = None
         try:
             for k in self:
                 return k, self[k]
         except Exception as e:
 
             from warnings import warn
-            msg = f"Couldn't get data for the key {k}. This could be be...\n"
-            msg += "... because it's not a store (just a collection, that doesn't have a __getitem__)\n"
-            msg += "... because there's a layer transforming outcoming keys that are not the ones the store actually " \
-                   "uses? If you didn't wrap the store with the inverse ingoing keys transformation, " \
-                   "that would happen.\n"
-            msg += "I'll ask the inner-layer what it's head is, but IT MAY NOT REFLECT the reality of your store " \
-                   "if you have some filtering, caching etc."
-            msg += f"The error messages was: \n{e}"
-            warn(msg)
+            if k is None:
+                raise
+            else:
+                msg = f"Couldn't get data for the key {k}. This could be be...\n"
+                msg += "... because it's not a store (just a collection, that doesn't have a __getitem__)\n"
+                msg += "... because there's a layer transforming outcoming keys that are not the ones the store actually " \
+                       "uses? If you didn't wrap the store with the inverse ingoing keys transformation, " \
+                       "that would happen.\n"
+                msg += "I'll ask the inner-layer what it's head is, but IT MAY NOT REFLECT the reality of your store " \
+                       "if you have some filtering, caching etc."
+                msg += f"The error messages was: \n{e}"
+                warn(msg)
 
             for _id in self.store:
                 return self._key_of_id(_id), self._obj_of_data(self.store[_id])
@@ -312,21 +403,28 @@ def __setitem__(self, k: Key, v: Val):
     def __delitem__(self, k: Key):
         return self.store.__delitem__(self._id_of_key(k))
 
-    def clear(self):
-        raise NotImplementedError('''
-        The clear method was overridden to make dangerous difficult.
-        If you really want to delete all your data, you can do so by doing:
-            try:
-                while True:
-                    self.popitem()
-            except KeyError:
-                pass''')
+    # def clear(self):
+    #     raise NotImplementedError('''
+    #     The clear method was overridden to make dangerous difficult.
+    #     If you really want to delete all your data, you can do so by doing:
+    #         try:
+    #             while True:
+    #                 self.popitem()
+    #         except KeyError:
+    #             pass''')
 
     # Misc ####################################################################
     def __repr__(self):
-        return self.store.__repr__()
+        x = repr(self.store)
+        if isinstance(self._max_repr_size, int):
+            half = int(self._max_repr_size)
+            if len(x) > self._max_repr_size:
+                x = x[:half] + '  ...  ' + x[-half:]
+        return x
+        # return self.store.__repr__()
 
 
+# Store.register(dict)  # TODO: Would this be a good idea? To make isinstance({}, Store) be True (though missing head())
 KvStore = Store  # alias with explict name