added caching

Author: Jhsmit

pyhdx/web/apps.py

@@ -1,4 +1,3 @@
-
 from pathlib import Path
 
 import panel as pn
@@ -7,14 +6,18 @@
 from pyhdx import VERSION_STRING
 from pyhdx.web.constructor import AppConstructor
 from pyhdx.web.log import logger
+from pyhdx.web.cache import MemoryCache, HybridHDFCache
+
+cache = MemoryCache(max_items=2000)
 
+#cache = HybridHDFCache(file_path ='test123.h5')
 
 @logger('pyhdx')
 def main_app():
     cwd = Path(__file__).parent.resolve()
     yaml_dict = yaml.safe_load((cwd / 'pyhdx_app.yaml').read_text(encoding='utf-8'))
 
-    ctr = AppConstructor(loggers={'pyhdx': main_app.logger})
+    ctr = AppConstructor(loggers={'pyhdx': main_app.logger}, cache=cache)
 
     ctrl = ctr.parse(yaml_dict)
 

pyhdx/web/cache.py

@@ -1,16 +1,95 @@
 import param
+import pandas as pd
 
 
 class Cache(param.Parameterized):
-    pass
 
+    def __getitem__(self, item):
+        return None
+
+    def __setitem__(self, key, value):
+        pass
+
+    def __contains__(self, item):
+        return False
 
-class MemoryCache(param.Parameterized):
+
+class MemoryCache(Cache):
 
     _cache = param.Dict(default={})
 
+    max_items = param.Integer(
+        None,
+        doc='Maximum number of items allowed in the cache'
+    )
+
     def __getitem__(self, item):
-        return self._cache.get(item)
+        return self._cache.__getitem__(item)
 
     def __setitem__(self, key, value):
+        if self.max_items is not None and len(self._cache) >= self.max_items:
+            self._cache.popitem()
+
         self._cache[key] = value
+
+    def __contains__(self, item):
+        return item in self._cache
+
+
+class HybridHDFCache(Cache):
+    """
+
+    Hybrid HDFStore / Memory cache
+
+    Sometimes there are errors depending on the dtypes of dataframes stored
+
+    """
+    file_path = param.String()
+
+    _store = param.ClassSelector(class_=pd.HDFStore)
+
+    _cache = param.Dict(default={})
+
+    bytes_threshold = param.Integer(default=int(1e8))
+
+    def __init__(self, **params):
+        super().__init__(**params)
+        if self.file_path is not None:
+            self._store = pd.HDFStore(self.file_path)
+
+    def __getitem__(self, item):
+        key = str(item)
+        try:
+            return self._cache.__getitem__(key)
+        except KeyError:
+            return self._store.__getitem__(key)
+
+    def _store_put(self, key, value):
+        try:
+            self._store[key] = value
+
+            # Check if reading back the dataframe works
+            try:
+                _value = self._store[key]
+            except AttributeError:
+                del self._store[key]
+                self._cache[key] = value
+
+        except (NotImplementedError, TypeError):  # pytables does not support categorical dtypes
+            self._cache[key] = value
+
+    def __setitem__(self, key, value):
+        key = str(key)
+        if isinstance(value, pd.DataFrame) and value.memory_usage().sum() > self.bytes_threshold:
+            self._store_put(key, value)
+        elif isinstance(value, pd.Series) and value.memory_usage() > self.bytes_threshold:
+            self._store_put(key, value)
+        else:
+            self._cache[str(key)] = value
+
+    def __contains__(self, item):
+        return str(item) in self._cache.keys() | self._store.keys()
+
+    # todo with statement for creating caches?
+    # def __exit__(self):
+    #     pass

pyhdx/web/constructor.py

@@ -11,9 +11,11 @@
 from pyhdx.web.tools import supported_tools
 from pyhdx.web.transforms import *
 from pyhdx.web.views import View
+from pyhdx.web.cache import Cache
 
 element_count = 0
 
+
 class AppConstructor(param.Parameterized):
 
     sources = param.Dict(default={})
@@ -32,6 +34,8 @@ class AppConstructor(param.Parameterized):
 
     client = param.ClassSelector(default=None, class_=Client)
 
+    cache = param.ClassSelector(default=Cache(), class_=Cache)
+
     def __init__(self, **params):
         super().__init__(**params)
         self.classes = self.find_classes()
@@ -105,12 +109,21 @@ def _parse_sections(self, yaml_dict):
                 obj = self.create_element(name, element, **spec)
                 element_dict[name] = obj
 
-    def create_element(self, name, element, **spec):
+    def create_element(self, name: str, element: str, **spec):
+        """
+
+        :param name:
+        :param element: eiter source, filter, opt, view, tool
+        :param spec:
+        :return:
+        """
         global element_count
 
         _type = spec.pop('type')
         kwargs = self._resolve_kwargs(**spec)
         class_ = self._resolve_class(_type, element)
+        if element == 'transform':
+            kwargs['_cache'] = self.cache
         obj = class_(name=name, **kwargs)
         element_count += 1
 

pyhdx/web/controllers.py

@@ -76,12 +76,17 @@ def _action_debug(self):
         print('break')
 
     def _action_test(self):
-        trs = self.transforms['table_1_select']
-        print(trs.widgets)
+        trs = self.transforms['peptide_select']
+        cache = trs._cache
+        print(cache._cache.keys())
+        print(cache)
+        print(cache._store.keys())
+
+        for item in cache._store.keys():
+            print(item)
+            print(cache[item])
+
 
-        view = self.views['graph_1']
-        df = view.get_data()
-        print(df)
 
     @property
     def _layout(self):

pyhdx/web/transforms.py

@@ -27,15 +27,11 @@ class Transform(param.Parameterized):
 
     _hash = param.Integer(doc='Hash of current transform state')
 
-    _cache = param.ClassSelector(default=None, class_=Cache)
+    _cache = param.ClassSelector(default=Cache(), class_=Cache)
 
     def __init__(self, **params):
         super().__init__(**params)
 
-    def get(self):
-        """method called to get the dataframe"""
-        return None
-
     # perhaps htey should all be private to prevent namespace collision with filter options
     @property
     def source_hash(self):
@@ -113,9 +109,18 @@ class AppTransform(Transform):
 
     source = param.ClassSelector(class_=Transform)
 
+    def transform(self):
+        """get source data, apply transform, return result"""
+        return self.source.get()
+
     def get(self):
-        df = self.source.get()
-        return df
+        """method called to get the dataframe"""
+        if self.hash in self._cache:
+            return self._cache[self.hash]
+        else:
+            data = self.transform()
+            self._cache[self.hash] = data
+            return data
 
     @param.depends('source.updated', watch=True)
     def update(self):
@@ -197,20 +202,15 @@ def redraw(self):
 
         self.redrawn = True
 
-    #todo cache df?
-    def get(self):
+    def transform(self):
         df = self.source.get()
         if df is None:
             return df
-        else:
-            kwargs = self.pd_kwargs
-            # drop level bugged? https://github.com/pandas-dev/pandas/issues/6507
-            df = df.xs(**kwargs)
-            # if self.drop_level and self.axis == 1 and df.columns.nlevels > 1:
-            #     df.columns = df.columns.droplevel()
-            # elif self.drop_level and self.axis == 0:
-            #     df.index = df.index.droplevel()
-            return df
+
+        kwargs = self.pd_kwargs
+        # drop level bugged? https://github.com/pandas-dev/pandas/issues/6507
+        df = df.xs(**kwargs)
+        return df
 
     def _selector_changed(self, *events):
         #this sends multiple updated events as it triggers changes in other selectors
@@ -332,7 +332,7 @@ class RectangleLayoutTransform(AppTransform):
     step = param.Integer(5, bounds=(1, None), doc="Step size used for finding 'wrap' when its not specified")
     margin = param.Integer(4, doc="Margin space to keep between peptides when finding 'wrap'")
 
-    def get(self):
+    def transform(self):
         df = self.source.get()
         if df is None:
             return None
@@ -378,7 +378,7 @@ def __init__(self, **params):
         self.kwargs = {k: v for k, v in params.items() if k not in self.param}
         super().__init__(**{k: v for k, v in params.items() if k in self.param})
 
-    def get(self):
+    def transform(self):
         df = self.source.get()
         if df is None:
             return df
@@ -408,7 +408,7 @@ def __init__(self, **params):
         self.kwargs = {k: v for k, v in params.items() if k not in self.param}
         super().__init__(**params)
 
-    def get(self):
+    def transform(self):
         df = self.source.get()
         if df is None:
             return None
@@ -440,7 +440,7 @@ class RescaleTransform(GenericTransform):
 
     scale_factor = param.Number(1.)
 
-    def get(self):  # todo perhaps some kind of decorator that returns nonealwasy?
+    def transform(self):  # todo perhaps some kind of decorator that returns nonealwasy?
         df = self.source.get()
         if df is None:
             return None
@@ -506,7 +506,7 @@ class SampleTransform(AppTransform):
 
     axis = param.Number(0, inclusive_bounds=(0, 1))
 
-    def get(self):
+    def transform(self):
         df = self.source.get()
         if df is None:
             return None
@@ -549,7 +549,7 @@ class PipeTransform(AppTransform):
 
     pipe = param.List()  # list of dicts
 
-    def get(self):
+    def transform(self):
         df = self.source.get()
         if df is None:
             return None