🚚 restructure and add large data

Author: byron

tests/integration/store/conftest.py

@@ -1,6 +1,8 @@
+import sys
 import json
 import pytest
 import uuid
+import mimesis
 import dash
 from dash.dependencies import Input, Output, State
 from dash.exceptions import PreventUpdate
@@ -62,3 +64,24 @@ def on_click(n_clicks):
         return ({"n_clicks": n_clicks},) * 3
 
     yield app
+
+
+@pytest.fixture(scope="module")
+def fake_data():
+    buf = ""
+    chunk = ""
+    limit = 5 * 1024 * 1024
+    while sys.getsizeof(buf) <= limit:
+        g = mimesis.Generic()
+        chunk = "\n".join(
+            (
+                "{},{}".format(g.person.full_name(), g.person.email())
+                for _ in range(10000)
+            )
+        )
+        buf += chunk
+
+    with open("/tmp/x.csv", "w") as fp:
+        fp.write(buf[len(chunk):limit])
+
+    yield buf[len(chunk):limit]

tests/integration/store/test_clear_data.py

@@ -0,0 +1,69 @@
+import json
+import time
+from pytest import approx
+import dash
+from dash.dependencies import Input, Output, State
+from dash.exceptions import PreventUpdate
+import dash_core_components as dcc
+import dash_html_components as html
+
+
+def test_stcp100_clear_data_on_all_types(store_app, dash_duo):
+    dash_duo.start_server(store_app)
+
+    assert dash_duo.wait_for_contains_text("#output", store_app.uuid)
+
+    dash_duo.multiple_click("#btn", 3)
+    assert dash_duo.get_local_storage() == {"n_clicks": 3}
+
+    # button click sets clear_data=True on all type of stores
+    dash_duo.find_element("#clear-btn").click()
+
+    assert (
+        not dash_duo.find_element("#output").text
+        and not dash_duo.get_local_storage()
+        and not dash_duo.get_session_storage()
+    ), "set clear_data=True should clear all data in three storage types"
+
+
+def test_stcp200_modified_ts(store_app, dash_duo):
+    app = dash.Dash(__name__)
+    app.layout = html.Div(
+        [
+            dcc.Store(id="initial-storage", storage_type="session"),
+            html.Button("set-init-storage", id="set-init-storage"),
+            html.Div(id="init-output"),
+        ]
+    )
+
+    @app.callback(
+        Output("initial-storage", "data"),
+        [Input("set-init-storage", "n_clicks")],
+    )
+    def on_init(n_clicks):
+        if n_clicks is None:
+            raise PreventUpdate
+        return "initialized"
+
+    @app.callback(
+        Output("init-output", "children"),
+        [Input("initial-storage", "modified_timestamp")],
+        [State("initial-storage", "data")],
+    )
+    def init_output(ts, data):
+        return json.dumps({"data": data, "ts": ts})
+
+    dash_duo.start_server(app)
+
+    dash_duo.find_element("#set-init-storage").click()
+    # the python ts ends at seconds while javascript one ends at ms
+    ts = float(time.time() * 1000)
+
+    output_data = json.loads(dash_duo.find_element("#init-output").text)
+
+    assert (
+        output_data.get("data") == "initialized"
+    ), "the data should be the text set in on_init"
+    assert ts == approx(
+        output_data.get("ts"), abs=10
+    ), "the modified_timestamp should be updated right after the click action"

tests/integration/store/test_component_props.py

@@ -1,110 +1,68 @@
 import json
-import time
+import hashlib
 import itertools
-from pytest import approx
 import dash
 from dash.dependencies import Input, Output, State
 from dash.exceptions import PreventUpdate
 import dash_core_components as dcc
 import dash_html_components as html
 
 
-def test_stco001_storage_component_smoke(store_app, dash_duo):
-
+def test_stda001_data_types(dash_duo):
     app = dash.Dash(__name__)
+
+    types = [
+        ("str", "hello"),
+        ("number", 1),
+        ("dict", {"data": [2, 3, None]}),
+        ("list", [5, -6, 700000, 1e-12]),
+        ("null", None),
+        ("bool", True),
+        ("bool", False),
+        ("empty-dict", {}),
+    ]
+    data_types = list(
+        itertools.chain(*itertools.combinations(types, 2))
+    ) + [  # No combinations as it add much test time.
+        ("list-dict-1", [1, 2, {"data": [55, 66, 77], "dummy": "dum"}]),
+        ("list-dict-2", [1, 2, {"data": [111, 99, 88]}]),
+        ("dict-3", {"a": 1, "c": 1}),
+        ("dict-2", {"a": 1, "b": None}),
+    ]
+
     app.layout = html.Div(
         [
-            dcc.Store(id="memory", storage_type="memory"),
-            dcc.Store(id="storage", storage_type="local"),
-            dcc.Store(id="session", storage_type="session", data=dummy_data),
-            dcc.Store(id="initial-storage", storage_type="session"),
-            html.Button("click me", id="btn"),
-            html.Button("clear", id="clear-btn"),
-            html.Button("set-init-storage", id="set-init-storage"),
-            html.Div(id="memory-output"),
-            html.Div(id="init-output"),
+            html.Div(id="output"),
+            html.Button("click", id="click"),
+            dcc.Store(id="store"),
         ]
     )
 
     @app.callback(
-        Output("storage", "data"),
-        [Input("btn", "n_clicks")],
-        [State("storage", "data")],
-    )
-    def on_click(n_clicks, storage):
-        if n_clicks is None:
-            return
-        storage = storage or {}
-        return {"clicked": storage.get("clicked", 0) + 1}
-
-    @app.callback(
-        Output("storage", "clear_data"), [Input("clear-btn", "n_clicks")]
-    )
-    def on_clear(n_clicks):
-        if n_clicks is None:
-            return
-        return True
-
-    @app.callback(Output("memory", "data"), [Input("storage", "data")])
-    def on_memory(data):
-        return data
-
-    @app.callback(
-        Output("memory-output", "children"), [Input("memory", "data")]
+        Output("output", "children"),
+        [Input("store", "modified_timestamp")],
+        [State("store", "data")],
     )
-    def on_memory_json(data):
-        if data is None:
-            return ""
+    def on_data(ts, data):
+        if ts is None:
+            raise PreventUpdate
         return json.dumps(data)
 
-    @app.callback(
-        Output("initial-storage", "data"),
-        [Input("set-init-storage", "n_clicks")],
-    )
-    def on_init(n_clicks):
+    @app.callback(Output("store", "data"), [Input("click", "n_clicks")])
+    def on_click(n_clicks):
         if n_clicks is None:
             raise PreventUpdate
-
-        return "initialized"
-
-    @app.callback(
-        Output("init-output", "children"),
-        [Input("initial-storage", "modified_timestamp")],
-        [State("initial-storage", "data")],
-    )
-    def init_output(ts, data):
-        return json.dumps({"data": data, "ts": ts})
+        return data_types[n_clicks - 1][1]
 
     dash_duo.start_server(app)
 
-    getter = 'return JSON.parse(window.{}.getItem("{}"));'
-    clicked_getter = getter.format("localStorage", "storage")
-
-    session = dash_duo.driver.execute_script(
-        getter.format("sessionStorage", "session")
-    )
-    assert dummy_data == session
-
-    for i in range(1, 11):
-        dash_duo.find_element("#btn").click()
-        click_data = dash_duo.driver.execute_script(clicked_getter)
-        assert i == click_data.get("clicked")
-        mem = dash_duo.wait_for_element("#memory-output")
-        assert i == int(json.loads(mem.text).get("clicked"))
-
-
-    # Test initial timestamp output
-    dash_duo.find_element("#set-init-storage").click()
-    # the python ts ends at seconds while javascript one ends at ms
-    ts = float(time.time() * 1000)
-    dash_duo.driver.refresh()
-    init = json.loads(dash_duo.wait_for_element("#init-output").text)
-
-    assert ts == approx(init.get("ts"), abs=10)
-    assert init.get("data") == "initialized"
+    button = dash_duo.wait_for_element("#click")
+    for data_type in data_types:
+        button.click()
+        dash_duo.wait_for_text_to_equal("#output", json.dumps(data_type[1]))
 
 
-def test_store_nested_data(dash_duo):
+def test_stda002_nested_data(dash_duo):
     app = dash.Dash(__name__)
 
     nested = {"nested": {"nest": "much"}}
@@ -148,68 +106,75 @@ def on_ts(ts, data):
 
     dash_duo.start_server(app)
 
-    obj_btn = dash_duo.wait_for_element_by_css_selector("#obj-btn")
-    list_btn = dash_duo.wait_for_element_by_css_selector("#list-btn")
+    obj_btn = dash_duo.wait_for_element("#obj-btn")
+    list_btn = dash_duo.find_element("#list-btn")
 
     obj_btn.click()
-    time.sleep(1)
     dash_duo.wait_for_text_to_equal("#output", json.dumps(nested))
     # it would of crashed the app before adding the recursive check.
 
     list_btn.click()
-    time.sleep(1)
     dash_duo.wait_for_text_to_equal("#output", json.dumps(nested_list))
 
 
-def test_stco003_data_type_updates(dash_duo):
-    app = dash.Dash(__name__)
-
-    types = [
-        ("str", "hello"),
-        ("number", 1),
-        ("dict", {"data": [2, 3, None]}),
-        ("list", [5, -6, 700000, 1e-12]),
-        ("null", None),
-        ("bool", True),
-        ("bool", False),
-        ("empty-dict", {}),
-    ]
-    types_changes = list(
-        itertools.chain(*itertools.combinations(types, 2))
-    ) + [  # No combinations as it add much test time.
-        ("list-dict-1", [1, 2, {"data": [55, 66, 77], "dummy": "dum"}]),
-        ("list-dict-2", [1, 2, {"data": [111, 99, 88]}]),
-        ("dict-3", {"a": 1, "c": 1}),
-        ("dict-2", {"a": 1, "b": None}),
-    ]
+def test_stda003_data_size_limit(fake_data, dash_duo):
+    def fingerprint(data):
+        return hashlib.sha1(data.encode("utf-8")).hexdigest()
 
+    app = dash.Dash(__name__)
     app.layout = html.Div(
         [
-            html.Div(id="output"),
-            html.Button("click", id="click"),
-            dcc.Store(id="store"),
+            dcc.Store(id="memory", storage_type="memory"),
+            dcc.Store(id="local", storage_type="local"),
+            dcc.Store(id="session", storage_type="session"),
+            html.Button("big data", id="btn"),
+            html.Div(id="mout"),
+            html.Div(id="sout"),
+            html.Div(id="lout"),
         ]
     )
 
     @app.callback(
-        Output("output", "children"),
-        [Input("store", "modified_timestamp")],
-        [State("store", "data")],
+        [
+            Output("mout", "children"),
+            Output("sout", "children"),
+            Output("lout", "children"),
+        ],
+        [
+            Input("memory", "modified_timestamp"),
+            Input("session", "modified_timestamp"),
+            Input("local", "modified_timestamp"),
+        ],
+        [
+            State("memory", "data"),
+            State("session", "data"),
+            State("local", "data"),
+        ],
     )
-    def on_data(ts, data):
-        if ts is None:
-            raise PreventUpdate
-        return json.dumps(data)
+    def update_output(mts, sts, lts, mdata, sdata, ldata):
+        if None in {mdata, sdata, ldata}:
+            return ("nil",) * 3
+        return [fingerprint(data) for data in (mdata, sdata, ldata)]
 
-    @app.callback(Output("store", "data"), [Input("click", "n_clicks")])
+    @app.callback(
+        [
+            Output("memory", "data"),
+            Output("local", "data"),
+            Output("session", "data"),
+        ],
+        [Input("btn", "n_clicks")],
+    )
     def on_click(n_clicks):
         if n_clicks is None:
             raise PreventUpdate
-        return types_changes[n_clicks - 1][1]
+        return (fake_data,) * 3
 
     dash_duo.start_server(app)
+    outputs = ('#mout', '#lout', '#sout')
+    for output in outputs:
+        assert dash_duo.find_element(output).text == 'nil'
+
+    dash_duo.find_element('#btn').click()
+    for output in outputs:
+        dash_duo.wait_for_text_to_equal(output, fingerprint(fake_data))
 
-    button = dash_duo.wait_for_element("#click")
-    for type_change in types_changes:
-        button.click()
-        dash_duo.wait_for_text_to_equal("#output", json.dumps(type_change[1]))