implemented substores

Ref: annotation/stam#29

Author: proycon

src/annotationstore.rs

@@ -12,6 +12,7 @@ use crate::error::PyStamError;
 use crate::query::*;
 use crate::resources::PyTextResource;
 use crate::selector::PySelector;
+use crate::substore::PyAnnotationSubStore;
 use crate::textselection::PyTextSelection;
 use stam::*;
 use stamtools::split::{split, SplitMode};
@@ -255,6 +256,30 @@ impl PyAnnotationStore {
         })
     }
 
+    /// Load an existing annotation store as a dependency to this one
+    fn add_substore(&mut self, filename: &str) -> PyResult<PyAnnotationSubStore> {
+        let store_clone = self.store.clone();
+        self.map_mut(|store| {
+            let handle = store.add_substore(filename)?;
+            Ok(PyAnnotationSubStore {
+                handle,
+                store: store_clone,
+            })
+        })
+    }
+
+    /// Create a new annotation store as a dependency of this one
+    fn add_new_substore(&mut self, id: &str, filename: &str) -> PyResult<PyAnnotationSubStore> {
+        let store_clone = self.store.clone();
+        self.map_mut(|store| {
+            let handle = store.add_new_substore(id, filename)?;
+            Ok(PyAnnotationSubStore {
+                handle,
+                store: store_clone,
+            })
+        })
+    }
+
     /// Adds an annotation. Returns an :obj:`Annotation` instance pointing to the added annotation.
     ///
     /// Args:
@@ -336,6 +361,14 @@ impl PyAnnotationStore {
         })
     }
 
+    /// Returns a generator over all substores in this store
+    fn substores(&self) -> PyResult<PySubStoreIter> {
+        Ok(PySubStoreIter {
+            store: self.store.clone(),
+            index: 0,
+        })
+    }
+
     /// Returns the number of annotations in the store (not substracting deletions)
     fn annotations_len(&self) -> PyResult<usize> {
         self.map(|store| Ok(store.annotations_len()))
@@ -351,6 +384,11 @@ impl PyAnnotationStore {
         self.map(|store| Ok(store.datasets_len()))
     }
 
+    /// Returns the number of substores in the store (not substracting deletions)
+    fn substores_len(&self) -> PyResult<usize> {
+        self.map(|store| Ok(store.substores_len()))
+    }
+
     fn shrink_to_fit(&mut self) -> PyResult<()> {
         self.map_mut(|store| Ok(store.shrink_to_fit(true)))
     }
@@ -804,3 +842,55 @@ impl PyResourceIter {
         }
     }
 }
+
+#[pyclass(name = "SubStoreIter")]
+struct PySubStoreIter {
+    pub(crate) store: Arc<RwLock<AnnotationStore>>,
+    pub(crate) index: usize,
+}
+
+#[pymethods]
+impl PySubStoreIter {
+    fn __iter__(pyself: PyRef<'_, Self>) -> PyRef<'_, Self> {
+        pyself
+    }
+
+    fn __next__(mut pyself: PyRefMut<'_, Self>) -> Option<PyAnnotationSubStore> {
+        pyself.index += 1; //increment first (prevent exclusive mutability issues)
+        let result = pyself.map(|store| {
+            let handle: AnnotationSubStoreHandle = AnnotationSubStoreHandle::new(pyself.index - 1);
+            if let Ok(substore) = store.get(handle) {
+                //index is one ahead, prevents exclusive lock issues
+                let handle = substore.handle().expect("annotation must have an ID");
+                Some(PyAnnotationSubStore {
+                    handle,
+                    store: pyself.store.clone(),
+                })
+            } else {
+                None
+            }
+        });
+        if result.is_some() {
+            result
+        } else {
+            if pyself.index >= pyself.map(|store| Some(store.annotations_len())).unwrap() {
+                None
+            } else {
+                Self::__next__(pyself)
+            }
+        }
+    }
+}
+
+impl PySubStoreIter {
+    fn map<T, F>(&self, f: F) -> Option<T>
+    where
+        F: FnOnce(&AnnotationStore) -> Option<T>,
+    {
+        if let Ok(store) = self.store.read() {
+            f(&store)
+        } else {
+            None //should never happen here
+        }
+    }
+}

src/lib.rs

@@ -10,6 +10,7 @@ mod error;
 mod query;
 mod resources;
 mod selector;
+mod substore;
 mod textselection;
 
 use crate::annotation::{PyAnnotation, PyAnnotations};
@@ -19,6 +20,7 @@ use crate::annotationstore::PyAnnotationStore;
 use crate::error::PyStamError;
 use crate::resources::{PyCursor, PyOffset, PyTextResource};
 use crate::selector::{PySelector, PySelectorKind};
+use crate::substore::PyAnnotationSubStore;
 use crate::textselection::{PyTextSelection, PyTextSelectionOperator, PyTextSelections};
 
 const VERSION: &'static str = env!("CARGO_PKG_VERSION");
@@ -43,5 +45,6 @@ fn stam(py: Python<'_>, m: &PyModule) -> PyResult<()> {
     m.add_class::<PyAnnotations>()?;
     m.add_class::<PyData>()?;
     m.add_class::<PyTextSelections>()?;
+    m.add_class::<PyAnnotationSubStore>()?;
     Ok(())
 }

src/substore.rs

@@ -0,0 +1,101 @@
+use pyo3::exceptions::{PyRuntimeError, PyValueError};
+use pyo3::prelude::*;
+use pyo3::pyclass::CompareOp;
+use pyo3::types::*;
+use std::borrow::Cow;
+use std::ops::FnOnce;
+use std::sync::{Arc, RwLock};
+
+use crate::annotation::PyAnnotations;
+use crate::error::PyStamError;
+use stam::*;
+
+#[pyclass(dict, module = "stam", name = "AnnotationSubStore")]
+/// This holds an annotation store that is included as a depenency into another one
+///
+/// The text *SHOULD* be in
+/// [Unicode Normalization Form C (NFC)](https://www.unicode.org/reports/tr15/) but
+/// *MAY* be in another unicode normalization forms.
+pub(crate) struct PyAnnotationSubStore {
+    pub(crate) handle: AnnotationSubStoreHandle,
+    pub(crate) store: Arc<RwLock<AnnotationStore>>,
+}
+
+impl PyAnnotationSubStore {
+    pub(crate) fn new(
+        handle: AnnotationSubStoreHandle,
+        store: Arc<RwLock<AnnotationStore>>,
+    ) -> Self {
+        Self { handle, store }
+    }
+
+    pub(crate) fn new_py<'py>(
+        handle: AnnotationSubStoreHandle,
+        store: Arc<RwLock<AnnotationStore>>,
+        py: Python<'py>,
+    ) -> &'py PyAny {
+        Self::new(handle, store).into_py(py).into_ref(py)
+    }
+}
+
+#[pymethods]
+impl PyAnnotationSubStore {
+    /// Returns the public ID (by value, aka a copy)
+    /// Don't use this for ID comparisons, use has_id() instead
+    fn id(&self) -> PyResult<Option<String>> {
+        self.map(|substore| Ok(substore.id().map(|x| x.to_owned())))
+    }
+
+    fn filename(&self) -> PyResult<Option<String>> {
+        self.map(|s| {
+            Ok(s.as_ref()
+                .filename()
+                .map(|s| s.to_string_lossy().into_owned()))
+        })
+    }
+
+    fn has_id(&self, other: &str) -> PyResult<bool> {
+        self.map(|substore| Ok(substore.id() == Some(other)))
+    }
+
+    fn has_filename(&self, filename: &str) -> PyResult<bool> {
+        self.map(|substore| {
+            Ok(substore.as_ref().filename().map(|s| s.to_string_lossy())
+                == Some(Cow::Borrowed(filename)))
+        })
+    }
+
+    fn __richcmp__(&self, other: PyRef<Self>, op: CompareOp) -> bool {
+        match op {
+            CompareOp::Eq => self.handle == other.handle,
+            CompareOp::Ne => self.handle != other.handle,
+            CompareOp::Lt => self.handle < other.handle,
+            CompareOp::Gt => self.handle > other.handle,
+            CompareOp::Le => self.handle <= other.handle,
+            CompareOp::Ge => self.handle >= other.handle,
+        }
+    }
+
+    fn __hash__(&self) -> usize {
+        self.handle.as_usize()
+    }
+}
+
+impl PyAnnotationSubStore {
+    /// Map function to act on the actual underlying store, helps reduce boilerplate
+    pub(crate) fn map<T, F>(&self, f: F) -> Result<T, PyErr>
+    where
+        F: FnOnce(ResultItem<AnnotationSubStore>) -> Result<T, StamError>,
+    {
+        if let Ok(store) = self.store.read() {
+            let substore = store
+                .substore(self.handle)
+                .ok_or_else(|| PyRuntimeError::new_err("Failed to resolve substore"))?;
+            f(substore).map_err(|err| PyStamError::new_err(format!("{}", err)))
+        } else {
+            Err(PyRuntimeError::new_err(
+                "Unable to obtain store (should never happen)",
+            ))
+        }
+    }
+}

stam.pyi

@@ -133,6 +133,19 @@ class AnnotationStore:
         If you want to store the dataset as a stand-off JSON file, you can specify a filename. The dataset will be loaded from file if it exists. Make sure to set `use_include = True` in the Annotation Store's configuration then.
         """
 
+    def add_substore(self, filename: str) -> AnnotationSubStore:
+        """
+        Load an existing annotation store as a dependency to this one. It will be store in an stand-off JSON file and included using the @include mechanism.
+        Returns the added substore.
+        """
+
+    def add_new_substore(self, id: str, filename: str) -> AnnotationSubStore:
+        """
+        Add a new empty annotation store as a dependency to this one.
+        It will be stored in an stand-off JSON file and included using the @include mechanism.
+        Returns the added substore.
+        """
+
     def set_filename(self, filename: str) -> None:
         """Set the filename for the annotationstore, the format is derived from the extension, can be `.json` or `csv`"""
 
@@ -214,6 +227,9 @@ class AnnotationStore:
     def resources(self) -> Iterator[TextResource]:
         """Returns an iterator over all text resources (:class:`TextResource`) in this store"""
 
+    def substores(self) -> Iterator[AnnotationSubStore]:
+        """Returns an iterator over all substores (:class:`AnnotationSubStore`) in this store, i.e. stores that are included by this one as dependencies"""
+
     def annotations_len(self) -> int:
         """Returns the number of annotations in the store (not substracting deletions)"""
 
@@ -223,6 +239,9 @@ class AnnotationStore:
     def resources_len(self) -> int:
         """Returns the number of text resources in the store (not substracting deletions)"""
 
+    def substores_len(self) -> int:
+        """Returns the number of substores in the store"""
+
     def shrink_to_fit(self):
         """Reallocates internal data structures to tight fits to conserve memory space (if necessary). You can use this after having added lots of annotations to possibly reduce the memory consumption."""
 
@@ -1604,7 +1623,7 @@ class TextResource:
     def filename(self, filename: str) -> Optional[str]:
         """Returns the filename for the stand-off file specified using @include (if any). This allocates a copy, use has_filename() for checking."""
 
-    def has_filename(self, filename: str) -> Optional[str]:
+    def has_filename(self, filename: str) -> bool:
         """Tests the filename for the stand-off file specified using @include (if any)."""
 
     def id(self) -> Optional[str]:
@@ -2157,5 +2176,21 @@ class TextSelectionOperator:
             Inverses the operator (turns it into a negation).
         """
 
+class AnnotationSubStore:
+    """
+    A substore is a sub-collection of annotations that is serialised as an independent AnnotationStore,
+    In STAM JSON it is included using the @include mechanism.
+    """
+
+    def id(self) -> Optional[str]:
+        """Returns the public identifier (by value, aka a copy)"""
+
+    def filename(self, filename: str) -> Optional[str]:
+        """Returns the filename for the stand-off annotation store. This allocates a copy, use has_filename() for checking."""
+
+    def has_filename(self, filename: str) -> bool:
+        """Tests the filename for the stand-off file specified using @include (if any)."""
+
+
 class StamError(Exception):
     """STAM Error"""