Automatic model handling #59

@entity decorator collects all types, Store gets them and syncs the model.
Also, search for the model JSON file at the callers module path.

Author: greenrobot

example/ollama/llamas.py

@@ -3,6 +3,8 @@
 
 import ollama
 import objectbox
+from objectbox.model import *
+from objectbox.model.properties import *
 
 documents = [
   "Llamas are members of the camelid family meaning they're pretty closely related to vicuñas and camels",
@@ -13,12 +15,6 @@
   "Llamas live to be about 20 years old, though some only live for 15 years and others live to be 30 years old",
 ]
 
-
-from objectbox.model import *
-from objectbox.model.idsync import sync_model
-from objectbox.model.properties import *
-import numpy as np
-
 # Have fresh data for each start
 objectbox.Store.remove_db_files("objectbox")
 
@@ -31,11 +27,7 @@ class DocumentEmbedding:
         distance_type=VectorDistanceType.COSINE
     ))
 
-model = Model()
-model.entity(DocumentEmbedding)
-sync_model(model, os.path.join(os.path.dirname(__file__),"objectbox-model.json") )
-
-store = objectbox.Store(model=model)
+store = objectbox.Store()
 box = store.box(DocumentEmbedding)
 
 print("Documents to embed: ", len(documents))

example/tasks/__main__.py

@@ -1,7 +1,7 @@
 from cmd import Cmd
 import objectbox
 import time
-from .model import *
+from model import *
 
 
 # objectbox expects date timestamp in milliseconds since UNIX epoch
@@ -15,7 +15,7 @@ def format_date(timestamp_ms: int) -> str:
 
 class TasklistCmd(Cmd):
     prompt = "> "
-    _store = objectbox.Store(model=get_objectbox_model(), directory="tasklist-db")
+    _store = objectbox.Store(directory="tasklist-db")
     _box = _store.box(Task)
 
     def do_ls(self, _):

example/tasks/model.py

@@ -10,9 +10,3 @@ class Task:
     date_created = Date(py_type=int)
     date_finished = Date(py_type=int)
 
-
-def get_objectbox_model():
-    m = Model()
-    m.entity(Task)
-    sync_model(m, os.path.join(os.path.dirname(__file__),"objectbox-model.json") )
-    return m

example/vectorsearch-cities/__main__.py

@@ -1,7 +1,7 @@
 from cmd import Cmd
 import objectbox
 import time
-from .model import *
+from model import *
 import csv
 import os
 
@@ -23,7 +23,7 @@ def __init__(self, *args):
         Cmd.__init__(self, *args)
         dbdir = "cities-db"
         new_db = not os.path.exists(dbdir)
-        self._store = objectbox.Store(model=get_objectbox_model(),directory=dbdir)
+        self._store = objectbox.Store(directory=dbdir)
         self._box = self._store.box(City)
         if new_db: 
             with open(os.path.join(os.path.dirname(__file__), 'cities.csv')) as f:

example/vectorsearch-cities/model.py

@@ -14,8 +14,3 @@ class City:
         distance_type=VectorDistanceType.EUCLIDEAN
     ))
 
-def get_objectbox_model():
-    m = Model()
-    m.entity(City)
-    sync_model(m, os.path.join(os.path.dirname(__file__),"objectbox-model.json") )
-    return m

objectbox/model/entity.py

@@ -12,20 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 import flatbuffers
 import flatbuffers.flexbuffers
-from typing import Generic
 import numpy as np
 from datetime import datetime, timezone
+import logging
 from objectbox.c import *
+from objectbox.model.iduid import IdUid
 from objectbox.model.properties import Property
 from objectbox.utils import date_value_to_int
 import threading
-from objectbox.c import *
-from objectbox.model.iduid import IdUid
-from objectbox.model.properties import Property
-
 
 
 # _Entity class holds model information as well as conversions between python objects and FlatBuffers (ObjectBox data)
@@ -276,11 +272,29 @@ def _unmarshal(self, data: bytes):
             setattr(obj, prop.name, val)
         return obj
 
+# Dictionary of entity types (metadata) collected by the Entity decorator
+obx_models_by_name: Dict[str, Set[_Entity]] = {}
+
 
-def Entity(uid: int = 0) -> Callable[[Type], _Entity]:
+def Entity(uid: int = 0, model: str = "default") -> Callable[[Type], _Entity]:
     """ Entity decorator that wraps _Entity to allow @Entity(id=, uid=); i.e. no class arguments. """
 
     def wrapper(class_):
-        return _Entity(class_, uid)
+        metadata_set = obx_models_by_name.get(model)
+        if metadata_set is None:
+            metadata_set = set()
+            obx_models_by_name[model] = metadata_set
+
+        metadata = _Entity(class_, uid)
+        for existing in metadata_set:
+            if existing.name == metadata.name:
+                # OK for tests, where multiple models are created with the same entity name
+                logging.warning(f"Model \"{model}\" already contains an entity \"{metadata.name}\"; replacing it.")
+                metadata_set.remove(existing)
+                break
+
+        obx_models_by_name[model].add(metadata)
+        logging.info(f"Entity {metadata.name} added to model {model}")
+        return metadata
 
     return wrapper

objectbox/store.py

@@ -11,20 +11,28 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
+import inspect
+import logging
+import os
+import sys
+from types import ModuleType
 
 import objectbox.c as c
 import objectbox.transaction
+from objectbox.model.idsync import sync_model
 from objectbox.store_options import StoreOptions
 import objectbox
 from objectbox.model.entity import _Entity
+from objectbox.model.model import Model
 from typing import *
 
+
 class Store:
-    def __init__(self, 
-                 model : Optional[objectbox.model.Model] = None, 
-                 directory : Optional[str] = None, 
-                 max_db_size_in_kb : Optional[int] = None,
+    def __init__(self,
+                 model: Optional[Union[Model, str]] = "default",
+                 model_json_file: Optional[str] = None,
+                 directory: Optional[str] = None,
+                 max_db_size_in_kb: Optional[int] = None,
                  max_data_size_in_kb: Optional[int] = None,
                  file_mode: Optional[int] = None,
                  max_readers: Optional[int] = None,
@@ -46,8 +54,8 @@ def __init__(self,
                  async_minor_refill_max_count: Optional[int] = None,
                  async_object_bytes_max_cache_size: Optional[int] = None,
                  async_object_bytes_max_size_to_cache: Optional[int] = None,
-                 c_store : Optional[c.OBX_store_p] = None):
-        
+                 c_store: Optional[c.OBX_store_p] = None):
+
         """Opens an ObjectBox database Store
 
         :param model:
@@ -107,13 +115,14 @@ def __init__(self,
             Maximum size for an object to be cached.
         :param c_store:
             Internal parameter for deprecated ObjectBox interface. Do not use it; other options would be ignored if passed.
-        """ 
-        
+        """
+
         self._c_store = None
         if not c_store:
             options = StoreOptions()
             try:
                 if model is not None:
+                    model = Store._sync_model(model, model_json_file)
                     options.model(model)
                 if directory is not None:
                     options.directory(directory)
@@ -160,18 +169,77 @@ def __init__(self,
                 if async_object_bytes_max_cache_size is not None:
                     options.async_object_bytes_max_cache_size(async_object_bytes_max_cache_size)
                 if async_object_bytes_max_size_to_cache is not None:
-                    options.async_object_bytes_max_size_to_cache(async_object_bytes_max_size_to_cache)    
-                
+                    options.async_object_bytes_max_size_to_cache(async_object_bytes_max_size_to_cache)
+
             except c.CoreException:
                 options._free()
                 raise
-            self._c_store = c.obx_store_open(options._c_handle)                 
+            self._c_store = c.obx_store_open(options._c_handle)
         else:
             self._c_store = c_store
 
+    @staticmethod
+    def _sync_model(model: Optional[Union[Model, str]],
+                    model_json_file: Optional[str]) -> Model:
+        if isinstance(model, str):  # Model name provided; get entities collected via @Entity
+            metadata_set = objectbox.model.entity.obx_models_by_name.get(model)
+            if metadata_set is None:
+                raise ValueError(
+                    f"Model \"{model}\" not found; ensure to set the name attribute on the model class.")
+            model = Model()
+            for metadata in metadata_set:
+                model.entity(metadata)
+        elif not isinstance(model, Model):
+            raise ValueError("Model must be a Model object or a string.")
+
+        if not model_json_file:
+            model_json_file = Store._locate_model_json_file()
+
+        sync_model(model, model_json_file)
+
+        return model
+
+    @staticmethod
+    def _locate_model_json_file():
+        def get_module_path(module: Optional[ModuleType]) -> Optional[str]:
+            if module and hasattr(module, "__file__"):
+                return os.path.dirname(os.path.realpath(module.__file__))
+            return None
+
+        def json_file_inside_module_path(module: Optional[ModuleType]) -> Optional[str]:
+            module_path = get_module_path(module)
+            if module_path:
+                logging.info("Using module path to locate objectbox-model.json: ", module_path)
+                return os.path.join(module_path, "objectbox-model.json")
+            return None
+
+        # The (direct) calling module seems like a good first choice
+        this_module = sys.modules[__name__]
+        this_module_path = get_module_path(this_module)
+        stack = inspect.stack()
+        calling_module: Optional[ModuleType] = None
+        for stack_element in stack:
+            module = inspect.getmodule(stack_element[0])
+            if module is not this_module:
+                path = get_module_path(module)
+                if not path:  # Cannot get the direct caller's path, so do not try further
+                    break
+                if path != this_module_path:  # Not inside the objectbox package
+                    calling_module = module
+                    break
+        model_json_file = json_file_inside_module_path(calling_module)
+
+        if not model_json_file:
+            # Note: the main module seems less reliable,
+            #       e.g. it resulted in a some pycharm dir when running tests from PyCharm.
+            model_json_file = json_file_inside_module_path(sys.modules.get('__main__'))
+            if not model_json_file:
+                model_json_file = "objectbox-model.json"
+        return model_json_file
+
     def __del__(self):
         self.close()
-    
+
     def box(self, entity: _Entity) -> 'objectbox.Box':
         """
         Open a box for an entity.

tests/common.py

@@ -1,20 +1,26 @@
-from os import path
+import os
 import pytest
 import objectbox
 from objectbox.logger import logger
 from objectbox.store import Store
-from objectbox.model.idsync import sync_model
 from tests.model import *
 import numpy as np
 from datetime import timezone
 
+
+def remove_json_model_file():
+    path = os.path.dirname(os.path.realpath(__file__))
+    json_file = os.path.join(path, "objectbox-model.json")
+    if os.path.exists(json_file):
+        os.remove(json_file)
+
+
 def create_default_model():
     model = objectbox.Model()
     model.entity(TestEntity)
     model.entity(TestEntityDatetime)
     model.entity(TestEntityFlex)
     model.entity(VectorEntity)
-    sync_model(model)  # Assign IDs/UIDs
     return model
 
 
@@ -26,6 +32,7 @@ def create_test_store(db_path: str = "testdata", clear_db: bool = True) -> objec
 
     if clear_db:
         Store.remove_db_files(db_path)
+        remove_json_model_file()
     return objectbox.Store(model=create_default_model(), directory=db_path)
 
 

tests/test_deprecated.py

@@ -1,13 +1,17 @@
-import pytest
 import tests.common
-import objectbox
 from objectbox import ObjectBox
 from objectbox.c import *
+from objectbox.model.idsync import sync_model
 from objectbox.store_options import StoreOptions
 from tests.common import *
 
 def test_deprecated_ObjectBox():
+    Store.remove_db_files("testdata")
+    remove_json_model_file()
+
     model = tests.common.create_default_model()
+    sync_model(model)  # It expects IDs to be already assigned
+
     options = StoreOptions()
     options.model(model)
     options.directory("testdata")
@@ -19,6 +23,9 @@ def test_deprecated_ObjectBox():
 
 
 def test_deprecated_Builder():
+    Store.remove_db_files("testdata")
+    remove_json_model_file()
+
     model = tests.common.create_default_model()
     with pytest.deprecated_call():
         ob = objectbox.Builder().model(model).directory("testdata").build()

tests/test_store_options.py

@@ -1,7 +1,7 @@
+from objectbox import Store
 from objectbox.c import *  # TODO ideally we wouldn't have to import c.py
 from objectbox.store_options import StoreOptions
-import objectbox
-import tests.common
+from tests.common import *
 
 def test_set_options():
     """ Test setting dummy values for each option.
@@ -50,8 +50,11 @@ def test_set_options():
     del options
 
 def test_store_with_options():
-    store = objectbox.Store(
-        model=tests.common.create_default_model(),
+    Store.remove_db_files("testdata")
+    remove_json_model_file()
+
+    store = Store(
+        model=create_default_model(),
         directory="testdata",
         max_db_size_in_kb=1<<20,
         max_data_size_in_kb=(1<<20)-(1<<10),
@@ -73,5 +76,5 @@ def test_store_with_options():
         async_minor_refill_max_count=100,
         async_object_bytes_max_cache_size=1<<20,
         async_object_bytes_max_size_to_cache=100<<10
-    ) 
+    )
     del store