changed access to talk to a config directory instead of a single json

added a MongoAnyKeyStore

Author: twhalen

py2store/__init__.py

@@ -1,10 +1,17 @@
 import os
+from py2store.util import ModuleNotFoundIgnore
+
+file_sep = os.path.sep
 
 # Imports to be able to easily get started...
-from py2store.stores.local_store import LocalStore, LocalBinaryStore, LocalTextStore
-from py2store.stores.local_store import QuickStore, QuickBinaryStore, QuickTextStore
+from py2store.base import KvCollection, KvReader, KvPersister, Reader, Persister
+from py2store.stores.local_store import LocalStore, LocalBinaryStore, LocalTextStore, PickleStore
+from py2store.stores.local_store import QuickStore, QuickBinaryStore, QuickTextStore, QuickJsonStore, QuickPickleStore
 from py2store.base import Store
 from py2store.trans import wrap_kvs
 from py2store.access import user_configs_dict, user_configs, user_defaults_dict, user_defaults
 
-file_sep = os.path.sep
+with ModuleNotFoundIgnore():
+    from py2store.stores.s3_store import S3BinaryStore, S3TextStore, S3PickleStore
+with ModuleNotFoundIgnore():
+    from py2store.stores.mongo_store import MongoStore, MongoTupleKeyStore, MongoAnyKeyStore

py2store/access.py

@@ -15,6 +15,7 @@
 respectively.
 """
 import os
+from warnings import warn
 from py2store.util import DictAttr, str_to_var_str
 
 
@@ -35,17 +36,35 @@ def getenv(name, default=None):
 try:
     import json
 
-    user_configs_filepath = os.path.expanduser(getenv('PY2STORE_CONFIGS_JSON_FILEPATH', '~/.py2store_configs.json'))
-    if os.path.isfile(user_configs_filepath):
-        user_configs_dict = json.load(open(user_configs_filepath))
-        user_configs = DictAttr(**{str_to_var_str(k): v for k, v in user_configs_dict.items()})
+    user_configs_dirpath = os.path.expanduser(getenv('PY2STORE_CONFIGS_DIR', '~/.py2store_configs'))
+    if os.path.isdir(user_configs_dirpath):
+        def directory_json_items():
+            for f in filter(lambda x: x.endswith('.json'), os.listdir(user_configs_dirpath)):
+                filepath = os.path.join(user_configs_dirpath, f)
+                name, _ = os.path.splitext(f)
+                try:
+                    d = json.load(open(filepath))
+                    yield str_to_var_str(name), d
+                except json.JSONDecodeError:
+                    warn(f"This json file couldn't be json-decoded: {filepath}")
+                except Exception:
+                    warn(f"Unknown error when trying to json.load this file: {filepath}")
+
+
+        user_configs = DictAttr(**{k: v for k, v in directory_json_items()})
+
+    else:
+        warn(f"The configs directory wasn't found (please make it): {user_configs_dirpath}")
+        warn("Configs in a single json is being deprecated")
+        user_configs_filepath = os.path.expanduser(getenv('PY2STORE_CONFIGS_JSON_FILEPATH', '~/.py2store_configs.json'))
+        if os.path.isfile(user_configs_filepath):
+            user_configs_dict = json.load(open(user_configs_filepath))
+            user_configs = DictAttr(**{str_to_var_str(k): v for k, v in user_configs_dict.items()})
 
     user_defaults_filepath = os.path.expanduser(getenv('PY2STORE_DEFAULTS_JSON_FILEPATH', '~/.py2store_defaults.json'))
     if os.path.isfile(user_defaults_filepath):
         user_defaults_dict = json.load(open(user_defaults_filepath))
         user_defaults = DictAttr(**{str_to_var_str(k): v for k, v in user_defaults_dict.items()})
 
 except Exception as e:
-    from warnings import warn
-
     warn(f"There was an exception when trying to get configs and defaults: {e}")

py2store/stores/mongo_store.py

@@ -17,7 +17,8 @@ class MongoTupleKeyStore(MongoStore):
     """
     MongoStore using tuple keys.
 
-    >>> s = MongoTupleKeyStore(collection_name='tmp', key_fields=('_id', 'user'))
+    >>> s = MongoTupleKeyStore(db_name='py2store_tests', collection_name='tmp', key_fields=('_id', 'user'))
+    >>> for k in s: del s[k]
     >>> k = (1234, 'user')
     >>> v = {'name': 'bob', 'age': 42}
     >>> if k in s:  # deleting all docs in tmp
@@ -46,6 +47,44 @@ def _key_of_id(self, _id):
         return tuple(_id[x] for x in self._key_fields)
 
 
+# TODO: Finish
+class MongoAnyKeyStore(MongoStore):
+    """
+    MongoStore using tuple keys.
+
+    >>> s = MongoAnyKeyStore(db_name='py2store_tests', collection_name='tmp', )
+    >>> for k in s: del s[k]
+    >>> s['foo'] = {'must': 'be', 'a': 'dict'}
+    >>> assert s['foo']
+    {'must': 'be', 'a': 'dict'}
+    >>> ss = MongoStore()
+    >>> ss[{'_id': 'foo'}]  # see that 'foo' was actually put in the _id field
+    >>>
+    """
+
+    @wraps(MongoStore.__init__)
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        assert isinstance(self._key_fields, tuple), "key_fields should be a tuple or a string"
+        assert len(self._key_fields) == 1, "key_fields must have one and only one element (a string)"
+        self._key_field = self._key_fields[0]
+
+    @lazyprop
+    def _key_fields(self):
+        return self.store._key_fields
+
+    def _id_of_key(self, k):
+        return {self._key_field: k}
+
+    def _key_of_id(self, _id):
+        return _id[self._key_field]
+
+    def __setitem__(self, k, v):
+        if k in self:
+            del self[k]
+        super().__setitem__(k, v)
+
+
 def test_mongo_store(s=MongoStore(), k=None, v=None):
     if k is None:
         k = {'_id': 'foo'}

py2store/util.py

@@ -296,17 +296,27 @@ def __enter__(self):
     def __exit__(self, exc_type, exc_val, exc_tb):
         if exc_type is ModuleNotFoundError:
             warn(self.msg)
-#             if exc_val is not None and getattr(exc_val, 'name', None) is not None:
-#                 warn(f"""
-# It seems you don't have required `{exc_val.name}` package for this Store.
-# This is just a warning: The process goes on...
-# (But, hey, if you really need that package, try installing it by running:
-#
-#     pip install {exc_val.name}
-#
-# in your terminal.
-# For more information: https://pypi.org/project/{exc_val.name}, or google around...
-#                 """)
-#             else:
-#                 print("It seems you don't have a required package")
+            #             if exc_val is not None and getattr(exc_val, 'name', None) is not None:
+            #                 warn(f"""
+            # It seems you don't have required `{exc_val.name}` package for this Store.
+            # This is just a warning: The process goes on...
+            # (But, hey, if you really need that package, try installing it by running:
+            #
+            #     pip install {exc_val.name}
+            #
+            # in your terminal.
+            # For more information: https://pypi.org/project/{exc_val.name}, or google around...
+            #                 """)
+            #             else:
+            #                 print("It seems you don't have a required package")
             return True
+
+
+class ModuleNotFoundIgnore:
+    def __enter__(self):
+        pass
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if exc_type is ModuleNotFoundError:
+            pass
+        return True

py2store/utils/uri_parsing.py

@@ -0,0 +1,41 @@
+from urllib.parse import urlsplit
+
+
+def parse_uri(uri):
+    """
+    Parses DB URI string into a dict of params.
+    :param uri: string formatted as: "scheme://username:password@host:port/database"
+    :return: a dict with these params parsed.
+    """
+    splitted_uri = urlsplit(uri)
+
+    if splitted_uri.path.startswith('/'):
+        path = splitted_uri.path[1:]
+    else:
+        path = ''
+
+    return {
+        'scheme': splitted_uri.scheme,
+        'database': path,
+        'username': splitted_uri.username,
+        'password': splitted_uri.password,
+        'hostname': splitted_uri.hostname,
+        'port': splitted_uri.port,
+    }
+
+
+def build_uri(
+        scheme,
+        database='',  # TODO: Change name: Not always a database
+        username=None,
+        password=None,
+        host='localhost',
+        port=None,
+):
+    """
+    Reverse of `parse_uri` function.
+    Builds a URI string from provided params.
+    """
+    port_ = f':{port}' if port else ''
+    uri = f'{scheme}://{username}:{password}@{host}{port_}/{database}'
+    return uri