Merge pull request #242 from Jhsmit/cache

Cache

Author: Jhsmit

pyhdx/support.py

@@ -13,6 +13,26 @@
 from dask.distributed import Client
 
 
+def make_tuple(item):
+    if isinstance(item, list):
+        return tuple(make_tuple(i) for i in item)
+    elif isinstance(item, dict):
+        return tuple((key, make_tuple(value)) for key, value in item.items())
+    else:
+        return item
+
+
+def hash_dataframe(df):
+    try:
+        tup = (*pd.util.hash_pandas_object(df, index=True).values, *df.columns, *df.columns.names, df.index.name)
+
+    except TypeError:
+        print(df)
+        print('hoi')
+
+    return hash(tup)
+
+
 def multiindex_apply_function(
     index: pd.MultiIndex,
     level: int,

pyhdx/web/apps.py

@@ -1,4 +1,3 @@
-
 from pathlib import Path
 
 import panel as pn
@@ -7,14 +6,18 @@
 from pyhdx import VERSION_STRING
 from pyhdx.web.constructor import AppConstructor
 from pyhdx.web.log import logger
+from pyhdx.web.cache import MemoryCache, HybridHDFCache
+
+cache = MemoryCache(max_items=2000)
 
+#cache = HybridHDFCache(file_path ='test123.h5')
 
 @logger('pyhdx')
 def main_app():
     cwd = Path(__file__).parent.resolve()
     yaml_dict = yaml.safe_load((cwd / 'pyhdx_app.yaml').read_text(encoding='utf-8'))
 
-    ctr = AppConstructor(loggers={'pyhdx': main_app.logger})
+    ctr = AppConstructor(loggers={'pyhdx': main_app.logger}, cache=cache)
 
     ctrl = ctr.parse(yaml_dict)
 

pyhdx/web/cache.py

@@ -0,0 +1,95 @@
+import param
+import pandas as pd
+
+
+class Cache(param.Parameterized):
+
+    def __getitem__(self, item):
+        return None
+
+    def __setitem__(self, key, value):
+        pass
+
+    def __contains__(self, item):
+        return False
+
+
+class MemoryCache(Cache):
+
+    _cache = param.Dict(default={})
+
+    max_items = param.Integer(
+        None,
+        doc='Maximum number of items allowed in the cache'
+    )
+
+    def __getitem__(self, item):
+        return self._cache.__getitem__(item)
+
+    def __setitem__(self, key, value):
+        if self.max_items is not None and len(self._cache) >= self.max_items:
+            self._cache.popitem()
+
+        self._cache[key] = value
+
+    def __contains__(self, item):
+        return item in self._cache
+
+
+class HybridHDFCache(Cache):
+    """
+
+    Hybrid HDFStore / Memory cache
+
+    Sometimes there are errors depending on the dtypes of dataframes stored
+
+    """
+    file_path = param.String()
+
+    _store = param.ClassSelector(class_=pd.HDFStore)
+
+    _cache = param.Dict(default={})
+
+    bytes_threshold = param.Integer(default=int(1e8))
+
+    def __init__(self, **params):
+        super().__init__(**params)
+        if self.file_path is not None:
+            self._store = pd.HDFStore(self.file_path)
+
+    def __getitem__(self, item):
+        key = str(item)
+        try:
+            return self._cache.__getitem__(key)
+        except KeyError:
+            return self._store.__getitem__(key)
+
+    def _store_put(self, key, value):
+        try:
+            self._store[key] = value
+
+            # Check if reading back the dataframe works
+            try:
+                _value = self._store[key]
+            except AttributeError:
+                del self._store[key]
+                self._cache[key] = value
+
+        except (NotImplementedError, TypeError):  # pytables does not support categorical dtypes
+            self._cache[key] = value
+
+    def __setitem__(self, key, value):
+        key = str(key)
+        if isinstance(value, pd.DataFrame) and value.memory_usage().sum() > self.bytes_threshold:
+            self._store_put(key, value)
+        elif isinstance(value, pd.Series) and value.memory_usage() > self.bytes_threshold:
+            self._store_put(key, value)
+        else:
+            self._cache[str(key)] = value
+
+    def __contains__(self, item):
+        return str(item) in self._cache.keys() | self._store.keys()
+
+    # todo with statement for creating caches?
+    # def __exit__(self):
+    #     pass

pyhdx/web/constructor.py

@@ -11,9 +11,11 @@
 from pyhdx.web.tools import supported_tools
 from pyhdx.web.transforms import *
 from pyhdx.web.views import View
+from pyhdx.web.cache import Cache
 
 element_count = 0
 
+
 class AppConstructor(param.Parameterized):
 
     sources = param.Dict(default={})
@@ -32,6 +34,8 @@ class AppConstructor(param.Parameterized):
 
     client = param.ClassSelector(default=None, class_=Client)
 
+    cache = param.ClassSelector(default=Cache(), class_=Cache)
+
     def __init__(self, **params):
         super().__init__(**params)
         self.classes = self.find_classes()
@@ -105,12 +109,21 @@ def _parse_sections(self, yaml_dict):
                 obj = self.create_element(name, element, **spec)
                 element_dict[name] = obj
 
-    def create_element(self, name, element, **spec):
+    def create_element(self, name: str, element: str, **spec):
+        """
+
+        :param name:
+        :param element: eiter source, filter, opt, view, tool
+        :param spec:
+        :return:
+        """
         global element_count
 
         _type = spec.pop('type')
         kwargs = self._resolve_kwargs(**spec)
         class_ = self._resolve_class(_type, element)
+        if element == 'transform':
+            kwargs['_cache'] = self.cache
         obj = class_(name=name, **kwargs)
         element_count += 1
 

pyhdx/web/controllers.py

@@ -76,12 +76,17 @@ def _action_debug(self):
         print('break')
 
     def _action_test(self):
-        trs = self.transforms['table_1_select']
-        print(trs.widgets)
+        trs = self.transforms['peptide_select']
+        cache = trs._cache
+        print(cache._cache.keys())
+        print(cache)
+        print(cache._store.keys())
+
+        for item in cache._store.keys():
+            print(item)
+            print(cache[item])
+
 
-        view = self.views['graph_1']
-        df = view.get_data()
-        print(df)
 
     @property
     def _layout(self):
@@ -772,14 +777,17 @@ def _action_add_comparison(self):
 
         combined = pd.concat([ddG, cov], axis=1)
 
+        #todo use _add_table method on source
         if current_df is not None:
             new_df = pd.concat([current_df, combined], axis=1)
         else:
             new_df = combined
 
-        self.parent.sources['main'].tables['ddG_comparison'] = new_df
-        self.parent.sources['main'].param.trigger('tables')  #todo check/remove tables trigger
-        self.parent.sources['main'].updated = True
+        #self.parent.sources['main'].tables['ddG_comparison'] = new_df
+        self.src.add_table('ddG_comparison', new_df)
+
+        #self.parent.sources['main'].param.trigger('tables')  #todo check/remove tables trigger
+        self.src.updated = True
 
 
 class ColorTransformControl(PyHDXControlPanel):

pyhdx/web/sources.py

@@ -7,7 +7,7 @@
 from pyhdx import TorchFitResult
 from pyhdx.fitting import RatesFitResult
 from pyhdx.models import HDXMeasurement, HDXMeasurementSet
-from pyhdx.support import multiindex_astype, multiindex_set_categories
+from pyhdx.support import multiindex_astype, multiindex_set_categories, hash_dataframe
 
 
 class Source(param.Parameterized):
@@ -23,7 +23,13 @@ def get(self):
 
 class TableSource(Source):
 
-    tables = param.Dict({})
+    tables = param.Dict(
+        default={},
+        doc="Dictionary of tables (pd.DataFrames)")
+
+    hashes = param.Dict(
+        default={},
+        doc="Dictionary of table hashes")
 
     _type = 'table'
 
@@ -33,6 +39,13 @@ def get(self):
         else:
             raise ValueError("TableSource has multiple tables, use `get_table`")
 
+    def add_table(self, table: str, df: pd.DataFrame):
+        table_hash = hash_dataframe(df)
+        self.hashes[table] = table_hash
+        self.tables[table] = df
+
+        #todo self.updated = True?
+
     def get_table(self, table):
         df = self.tables.get(table, None)
 
@@ -193,7 +206,8 @@ def _add_table(self, df, table, categorical=True):
         if categorical:
             new.columns = multiindex_astype(new.columns, 0, 'category')
             new.columns = multiindex_set_categories(new.columns, 0, categories, ordered=True)
-        self.tables[table] = new
+
+        self.add_table(table, new)
 
 
 class PDBSource(Source):
@@ -202,6 +216,8 @@ class PDBSource(Source):
 
     pdb_files = param.Dict({}, doc='Dictionary with id: pdb_string pdb file entries')
 
+    hashes = param.Dict({})
+
     max_entries = param.Number(
         1,
         doc='set maximum size for pdb files. set to none for infinite size. set to one for single pdb mode')
@@ -213,11 +229,13 @@ def add_from_pdb(self, pdb_id):
             pdb_string = response.read().decode()
 
         self.pdb_files[pdb_id] = pdb_string
+        self.hashes[pdb_id] = hash(pdb_string)
         self.updated = True
 
     def add_from_string(self, pdb_string, pdb_id):
         self._make_room()
         self.pdb_files[pdb_id] = pdb_string
+        self.hashes[pdb_id] = hash(pdb_string)
         self.updated = True
 
     def _make_room(self):
@@ -227,9 +245,10 @@ def _make_room(self):
         elif len(self.pdb_files) == self.max_entries:
             key = next(iter(self.pdb_files))
             del self.pdb_files[key]
+            del self.hashes[key]
 
     def get(self):
-        """returns the first entry in the """
+        """returns the first entry in the pdb source"""
         return next(iter(self.pdb_files.values()))
 
     def get_pdb(self, pdb_id):