Umprove support of big datasets

Author: smalinin

bn_lightweight_charts/abstract.py

@@ -13,7 +13,7 @@
 from .util import (
     BulkRunScript, Pane, Events, IDGen, as_enum, jbool, js_json, TIME, NUM, FLOAT,
     LINE_STYLE, MARKER_POSITION, MARKER_SHAPE, CROSSHAIR_MODE,
-    PRICE_SCALE_MODE, marker_position, marker_shape, js_data,
+    PRICE_SCALE_MODE, marker_position, marker_shape, js_data, js_zipdata
 )
 
 current_dir = os.path.dirname(os.path.abspath(__file__))
@@ -56,7 +56,7 @@ def on_js_load(self):
         self.scripts.extend(self.final_scripts)
         for script in self.scripts:
             initial_script += f'\n{script}'
-        self.script_func(initial_script)
+        self.script_func(f'(async ()=> {{ {initial_script} }})();')
 
     def run_script(self, script: str, run_last: bool = False):
         """
@@ -217,6 +217,21 @@ def _single_datetime_format(self, arg) -> float:
         arg = self._interval * (arg.timestamp() // self._interval)+self.offset
         return arg
 
+    def set_zipped(self, df: Optional[pd.DataFrame] = None, format_cols: bool = True):
+        if df is None or df.empty:
+            self.run_script(f'{self.id}.series.setData([])')
+            self.data = pd.DataFrame()
+            return
+        if format_cols:
+            df = self._df_datetime_format(df, exclude_lowercase=self.name)
+        if self.name:
+            if self.name not in df:
+                raise NameError(f'No column named "{self.name}".')
+            df = df.rename(columns={self.name: 'value'})
+        self.data = df.copy()
+        self._last_bar = df.iloc[-1]
+        self.run_script(f'{self.id}.series.setData(await decodeGzJSON("{js_zipdata(df)}")); ')
+
     def set(self, df: Optional[pd.DataFrame] = None, format_cols: bool = True):
         if df is None or df.empty:
             self.run_script(f'{self.id}.series.setData([])')
@@ -594,6 +609,47 @@ def set(self, df: Optional[pd.DataFrame] = None, keep_drawings=False):
         else:
             self.run_script(f"{self._chart.id}.toolBox?.clearDrawings()")
 
+    def set_zipped(self, df: Optional[pd.DataFrame] = None, keep_drawings=False):
+        """
+        Sets the initial data for the chart.\n
+        :param df: columns: date/time, open, high, low, close, volume (if volume enabled).
+        :param keep_drawings: keeps any drawings made through the toolbox. Otherwise, they will be deleted.
+        """
+        if df is None or df.empty:
+            self.run_script(f'{self.id}.series.setData([])')
+            self.run_script(f'{self.id}.volumeSeries.setData([])')
+            self.candle_data = pd.DataFrame()
+            return
+        df = self._df_datetime_format(df)
+        df_copy = df.copy()
+        self.candle_data = df_copy[['time', 'open', 'high', 'low', 'close']]
+        self._last_bar = df.iloc[-1]
+        candle_js_data = js_zipdata(df[['time', 'open', 'high', 'low', 'close']])
+        self.run_script(f'{self.id}.series.setData(await decodeGzJSON("{candle_js_data}"))')
+
+        if 'volume' not in df:
+            return
+        volume = df[['time', 'volume']].rename(columns={'volume': 'value'})
+        volume['color'] = self._volume_down_color
+        volume.loc[df['close'] > df['open'], 'color'] = self._volume_up_color
+        volume_js_data = js_zipdata(volume)
+        self.run_script(f'{self.id}.volumeSeries.setData(await decodeGzJSON("{volume_js_data}"))')
+
+        for line in self._lines:
+            if line.name not in df.columns:
+                continue
+            line.set(df[['time', line.name]], format_cols=False)
+        # set autoScale to true in case the user has dragged the price scale
+        self.run_script(f'''
+            if (!{self.id}.chart.priceScale("right").options.autoScale)
+                {self.id}.chart.priceScale("right").applyOptions({{autoScale: true}})
+        ''')
+        # TODO keep drawings doesn't work consistenly w
+        if keep_drawings:
+            self.run_script(f'{self._chart.id}.toolBox?._drawingTool.repositionOnTime()')
+        else:
+            self.run_script(f"{self._chart.id}.toolBox?.clearDrawings()")
+
     def update(self, series: pd.Series, _from_tick=False):
         """
         Updates the data from a bar;

bn_lightweight_charts/js/index.html

@@ -20,6 +20,34 @@
 <body>
     <div id="container"></div>
     <script src="./bundle.js"></script>
+    <script>
+        async function decodeGzipBase64(b64) {
+          // base64 -> Uint8Array
+          const bytes = Uint8Array.from(
+            atob(b64),
+            c => c.charCodeAt(0)
+          );
+
+          // gunzip
+          const stream = new DecompressionStream("gzip");
+          const decompressedStream = new Blob([bytes])
+            .stream()
+            .pipeThrough(stream);
+
+          // Uint8Array -> string
+          const decompressed = await new Response(decompressedStream).arrayBuffer();
+          return new TextDecoder("utf-8").decode(decompressed);
+        }
+
+        async function decodeGzJSON(b64) {
+          let s = await decodeGzipBase64(b64)
+          try {
+            return JSON.parse(s)
+          } catch {
+            return []
+          }
+        }
+    </script>
 </body>
 </html>
 

bn_lightweight_charts/js/index_bn.html

@@ -138,6 +138,34 @@ <h6>Performance Report for [ <span id="strategy-title"></span> ] </h6>
     </div>
 
     <script src="./bundle.js"></script>
+    <script>
+        async function decodeGzipBase64(b64) {
+          // base64 -> Uint8Array
+          const bytes = Uint8Array.from(
+            atob(b64),
+            c => c.charCodeAt(0)
+          );
+
+          // gunzip
+          const stream = new DecompressionStream("gzip");
+          const decompressedStream = new Blob([bytes])
+            .stream()
+            .pipeThrough(stream);
+
+          // Uint8Array -> string
+          const decompressed = await new Response(decompressedStream).arrayBuffer();
+          return new TextDecoder("utf-8").decode(decompressed);
+        }
+
+        async function decodeGzJSON(b64) {
+          let s = await decodeGzipBase64(b64)
+          try {
+            return JSON.parse(s)
+          } catch {
+            return []
+          }
+        }
+    </script>
 </body>
 </html>
 

bn_lightweight_charts/util.py

@@ -1,5 +1,7 @@
 import asyncio
 import json
+import gzip
+import base64
 from datetime import datetime
 from zoneinfo import ZoneInfo
 from tzlocal import get_localzone_name
@@ -79,6 +81,18 @@ def js_data(data: Union[pd.DataFrame, pd.Series]):
     return json.dumps(filtered_records)
 
 
+def js_zipdata(data: Union[pd.DataFrame, pd.Series]):
+    if isinstance(data, pd.DataFrame):
+        d = data.to_dict(orient='records')
+        filtered_records = [{k: v for k, v in record.items() if v is not None and not pd.isna(v)} for record in d]
+    else:
+        d = data.to_dict()
+        filtered_records = {k: v for k, v in d.items()}
+    raw = json.dumps(filtered_records, ensure_ascii=False).encode("utf-8")
+    compressed = gzip.compress(raw)
+    return base64.b64encode(compressed).decode("ascii")
+
+
 def snake_to_camel(s: str):
     components = s.split('_')
     return components[0] + ''.join(x.title() for x in components[1:])

bn_lightweight_charts/widgets.py

@@ -177,7 +177,7 @@ def __init__(self, width=None, height=None, inner_width=1, inner_height=1, scale
     def _load(self):
         if sthtml is None:
             raise ModuleNotFoundError('streamlit.components.v1.html was not found, and must be installed to use StreamlitChart.')
-        sthtml(f'{self._html_init} {self._html}</script></body></html>', width=self.width, height=self.height)
+        sthtml(f'{self._html_init}  (async ()=> {{\n{self._html}\n}})();\n </script></body></html>', width=self.width, height=self.height)
 
 
 class JupyterChart(StaticLWC):
@@ -188,7 +188,7 @@ def __init__(self, width: int = 800, height=350, inner_width=1, inner_height=1,
     def _load(self):
         if HTML is None:
             raise ModuleNotFoundError('IPython.display.HTML was not found, and must be installed to use JupyterChart.')
-        html_code = html.escape(f"{self._html_init} {self._html}</script></body></html>")
+        html_code = html.escape(f"{self._html_init}  (async ()=> {{\n{self._html}\n}})();\n </script></body></html>")
         iframe = f'<iframe width="{self.width}" height="{self.height}" frameBorder="0" srcdoc="{html_code}"></iframe>'
         display(HTML(iframe))
 
@@ -200,13 +200,13 @@ def __init__(self, width: int = 800, height=350, inner_width=1, inner_height=1,
         self.filename = filename
 
     def _load(self):
-        html_code = f"{self._html_init} {self._html}</script></body></html>"
+        html_code = f"{self._html_init}  (async ()=> {{\n {self._html}\n}})();\n </script></body></html>"
         with open(self.filename, 'w') as file:
             file.write(html_code)
 
 
 class HTMLChart_BN(StaticLWC):
-    def __init__(self, width: int = 800, height=350, inner_width=1, inner_height=1, 
+    def __init__(self, width: int = 800, height=350, inner_width=1, inner_height=1,
                 scale_candles_only: bool = False, toolbox: bool = False, filename = "bn_charts.html"):
         super().__init__(width=width, height=height, inner_width=inner_width, inner_height=inner_height,
                         scale_candles_only=scale_candles_only, toolbox=toolbox, autosize=True,
@@ -237,7 +237,7 @@ def _prepare_html(self):
         const perf_metrics = {json.dumps(self.performance)};
         const strategy_titles = {json.dumps(self.strat_titles)};
 
-        function updateChart(id){{
+        async function updateChart(id){{
             document.querySelector('#nav-home-tab')?.click();
             {func_code}
         }}