Merge branch 'dev'

# Conflicts:
#	README.md

Author: twhalen

py2store/__init__.py

@@ -1,8 +1,17 @@
 import os
+from py2store.util import ModuleNotFoundIgnore
+
+file_sep = os.path.sep
 
 # Imports to be able to easily get started...
-from py2store.stores.local_store import LocalStore
-from py2store.stores.local_store import QuickStore
+from py2store.base import KvCollection, KvReader, KvPersister, Reader, Persister
+from py2store.stores.local_store import LocalStore, LocalBinaryStore, LocalTextStore, PickleStore
+from py2store.stores.local_store import QuickStore, QuickBinaryStore, QuickTextStore, QuickJsonStore, QuickPickleStore
 from py2store.base import Store
+from py2store.trans import wrap_kvs
+from py2store.access import user_configs_dict, user_configs, user_defaults_dict, user_defaults
 
-file_sep = os.path.sep
+with ModuleNotFoundIgnore():
+    from py2store.stores.s3_store import S3BinaryStore, S3TextStore, S3PickleStore
+with ModuleNotFoundIgnore():
+    from py2store.stores.mongo_store import MongoStore, MongoTupleKeyStore, MongoAnyKeyStore

py2store/access.py

@@ -0,0 +1,70 @@
+"""
+The logic to allow configurations (and defaults) to be parametrized by external environmental variables and files.
+
+There are two main key-value stores: One for configurations the user wants to reuse, and the other for the user's
+desired defaults. Both have the same structure:
+    * first level key: Name of the resource (should be a valid python variable name)
+    * The reminder is more or less free form (until the day we lay out some schemas for this)
+
+The system will look for the specification of user_configs and user_defaults in a json file.
+The filepath to this json file can specified in environment variables
+    PY2STORE_CONFIGS_JSON_FILEPATH and PY2STORE_DEFAULTS_JSON_FILEPATH
+respectively.
+By default, they are:
+    ~/.py2store_configs.json and ~/.py2store_defaults.json
+respectively.
+"""
+import os
+from warnings import warn
+from py2store.util import DictAttr, str_to_var_str
+
+
+def getenv(name, default=None):
+    """Like os.getenv, but removes a suffix \\r character if present (problem with some env var systems)"""
+    v = os.getenv(name, default)
+    if v.endswith('\r'):
+        return v[:-1]
+    else:
+        return v
+
+
+user_configs_dict = {}
+user_defaults_dict = {}
+user_configs = None
+user_defaults = None
+
+try:
+    import json
+
+    user_configs_dirpath = os.path.expanduser(getenv('PY2STORE_CONFIGS_DIR', '~/.py2store_configs'))
+    if os.path.isdir(user_configs_dirpath):
+        def directory_json_items():
+            for f in filter(lambda x: x.endswith('.json'), os.listdir(user_configs_dirpath)):
+                filepath = os.path.join(user_configs_dirpath, f)
+                name, _ = os.path.splitext(f)
+                try:
+                    d = json.load(open(filepath))
+                    yield str_to_var_str(name), d
+                except json.JSONDecodeError:
+                    warn(f"This json file couldn't be json-decoded: {filepath}")
+                except Exception:
+                    warn(f"Unknown error when trying to json.load this file: {filepath}")
+
+
+        user_configs = DictAttr(**{k: v for k, v in directory_json_items()})
+
+    else:
+        warn(f"The configs directory wasn't found (please make it): {user_configs_dirpath}")
+        warn("Configs in a single json is being deprecated")
+        user_configs_filepath = os.path.expanduser(getenv('PY2STORE_CONFIGS_JSON_FILEPATH', '~/.py2store_configs.json'))
+        if os.path.isfile(user_configs_filepath):
+            user_configs_dict = json.load(open(user_configs_filepath))
+            user_configs = DictAttr(**{str_to_var_str(k): v for k, v in user_configs_dict.items()})
+
+    user_defaults_filepath = os.path.expanduser(getenv('PY2STORE_DEFAULTS_JSON_FILEPATH', '~/.py2store_defaults.json'))
+    if os.path.isfile(user_defaults_filepath):
+        user_defaults_dict = json.load(open(user_defaults_filepath))
+        user_defaults = DictAttr(**{str_to_var_str(k): v for k, v in user_defaults_dict.items()})
+
+except Exception as e:
+    warn(f"There was an exception when trying to get configs and defaults: {e}")

py2store/base.py

@@ -24,7 +24,7 @@
 the storage methods themselves.
 """
 
-from collections.abc import MutableMapping
+from collections.abc import Collection, Mapping, MutableMapping
 from typing import Any, Iterable, Tuple
 
 Key = Any
@@ -37,16 +37,52 @@
 ItemIter = Iterable[Item]
 
 
-# TODO: Wishful thinking: Define store type so the type is defined by it's methods, not by subclassing.
-class Persister(MutableMapping):
-    """ Acts as a MutableMapping abc, but disabling the clear method, and computing __len__ by counting keys"""
+class KvCollection(Collection):
+
+    def __contains__(self, k: Key) -> bool:
+        """
+        Check if collection of keys contains k.
+        Note: This method actually fetches the contents for k, returning False if there's a key error trying to do so
+        Therefore it may not be efficient, and in most cases, a method specific to the case should be used.
+        :return: True if k is in the collection, and False if not
+        """
+        try:
+            self.__getitem__(k)
+            return True
+        except KeyError:
+            return False
 
-    def __len__(self):
+    def __len__(self) -> int:
+        """
+        Number of elements in collection of keys.
+        Note: This method iterates over all elements of the collection and counts them.
+        Therefore it is not efficient, and in most cases should be overridden with a more efficient version.
+        :return: The number (int) of elements in the collection of keys.
+        """
+        # TODO: some other means to more quickly count files?
+        # Note: Found that sum(1 for _ in self.__iter__()) was slower for small, slightly faster for big inputs.
         count = 0
         for _ in self.__iter__():
             count += 1
         return count
 
+    def head(self):
+        return next(iter(self.items()))
+
+
+class KvReader(KvCollection, Mapping):
+    """Acts as a Mapping abc, but with default __len__ (implemented by counting keys)
+    and head method to get the first (k, v) item of the store"""
+    pass
+
+
+Reader = KvReader  # alias
+
+
+# TODO: Wishful thinking: Define store type so the type is defined by it's methods, not by subclassing.
+class Persister(Reader, MutableMapping):
+    """ Acts as a MutableMapping abc, but disabling the clear method, and computing __len__ by counting keys"""
+
     def clear(self):
         raise NotImplementedError('''
         The clear method was overridden to make dangerous difficult.
@@ -58,6 +94,9 @@ def clear(self):
                 pass''')
 
 
+KvPersister = Persister  # alias with explict name
+
+
 # TODO: Make identity_func "identifiable". If we use the following one, we can use == to detect it's use,
 # TODO: ... but there may be a way to annotate, register, or type any identity function so it can be detected.
 def identity_func(x):
@@ -211,6 +250,9 @@ def head(self) -> Item:
     def __setitem__(self, k: Key, v: Val):
         return self.store.__setitem__(self._id_of_key(k), self._data_of_obj(v))
 
+    # def update(self, *args, **kwargs):
+    #     return self.store.update(*args, **kwargs)
+
     # Delete ####################################################################
     def __delitem__(self, k: Key):
         return self.store.__delitem__(self._id_of_key(k))
@@ -230,6 +272,21 @@ def __repr__(self):
         return self.store.__repr__()
 
 
+KvStore = Store  # alias with explict name
+
+
+def has_kv_store_interface(o):
+    """Check if object has the KvStore interface (that is, has the kv wrapper methods
+    Args:
+        o: object (class or instance)
+
+    Returns: True if kv has the four key (in/out) and value (in/out) transformation methods
+
+    """
+    return hasattr(o, '_id_of_key') and hasattr(o, '_key_of_id') \
+           and hasattr(o, '_data_of_obj') and hasattr(o, '_obj_of_data')
+
+
 from abc import ABCMeta, abstractmethod
 from py2store.errors import KeyValidationError