ObjStateCC introduced, pin is implemented, basic viz works, need to restore popups

Author: asmirnov69

examples/notebooks/conditional-join-w-cmp.ipynb

@@ -17,7 +17,6 @@
 
     with pyjviz.CB("top") as top:
         with pyjviz.CB("c1") as c1:
-            c1.set_method_call_opts('conditional_join', {'obj-state-output-type': 'plot'})
             res1 = df1.conditional_join(df2,
                                         ('id', 'id', "<"),
                                         df_columns = {'id':'df_id'},
@@ -30,12 +29,11 @@
 
     with top:
         with pyjviz.CB("c2") as c2:
-            c2.set_method_call_opts('conditional_join', {'obj-state-output-type': 'plot'})            
             res2 = df1.select_columns('value_1').conditional_join(
                 df2.select_columns('val*'),
                 ('value_1', 'value_2A', '>'),
                 ('value_1', 'value_2B', '<'),
-            )
+            ).pin()
 
     print(res2)
     res2.describe()

examples/scripts/conditional_join-w-plots.py

@@ -8,62 +8,78 @@
 from . import wb_stack
 
 random_id = 0
-def dump_obj_state(obj, pyjviz_opts = None):
+def dump_obj_state(obj):
     caller_stack_entry = wb_stack.wb_stack.get_parent_of_current_entry()
     stack_entry = wb_stack.wb_stack.get_top()    
-    t_obj, _ = obj_tracking.tracking_store.get_tracking_obj(obj)
+    t_obj, obj_found = obj_tracking.tracking_store.get_tracking_obj(obj)
 
     global random_id
     obj_state_uri = f"<ObjState#{random_id}>"; random_id += 1
-
     fstriplestore.triple_store.dump_triple(obj_state_uri, "rdf:type", "<ObjState>")
     fstriplestore.triple_store.dump_triple(obj_state_uri, "<obj>", t_obj.uri)
     fstriplestore.triple_store.dump_triple(obj_state_uri, "<part-of>", caller_stack_entry.uri)    
     fstriplestore.triple_store.dump_triple(obj_state_uri, "<version>", f'"{t_obj.last_version_num}"')
+    t_obj.last_obj_state_uri = obj_state_uri
     t_obj.last_version_num += 1
 
-    output_type = 'head'
-    if pyjviz_opts:
-        key_s = 'obj-state-output-type'
-        output_type = pyjviz_opts.get(key_s) if key_s in pyjviz_opts else 'head'
-        
+    dump_obj_state_cc(obj_state_uri, obj, output_type = 'head')
+
+    return t_obj
+    
+def dump_obj_state_cc(obj_state_uri, obj, output_type = 'head'):        
     if isinstance(obj, pd.DataFrame):
-        dump_DataFrame_obj_state(obj_state_uri, obj, output_type)
+        dump_DataFrame_obj_state_cc(obj_state_uri, obj, output_type)
     elif isinstance(obj, pd.Series):
-        dump_Series_obj_state(obj_state_uri, obj, output_type)
+        dump_Series_obj_state_cc(obj_state_uri, obj, output_type)
     else:
         raise Exception(f"unknown obj type at {obj_state_uri}")
 
-    t_obj.last_obj_state_uri = obj_state_uri
-    return t_obj
+def dump_DataFrame_obj_state_cc(obj_state_uri, df, output_type):
+    ts = fstriplestore.triple_store
+    global random_id
+    obj_state_cc_uri = f"<ObjStateCC#{random_id}>"; random_id += 1
+
+    ts.dump_triple(obj_state_cc_uri, "rdf:type", "<ObjStateCC>")
+    ts.dump_triple(obj_state_cc_uri, "<obj-state>", obj_state_uri)
 
-def dump_DataFrame_obj_state(obj_state_uri, df, output_type):
-    fstriplestore.triple_store.dump_triple(obj_state_uri, "<df-shape>", f'"{df.shape}"')
     if output_type == 'head':
+        ts.dump_triple(obj_state_cc_uri, "rdf:type", "<CCGlance>")
+        ts.dump_triple(obj_state_cc_uri, "<shape>", f'"{df.shape}"')
         df_head_html = df.head(10).applymap(lambda x: textwrap.shorten(str(x), 50)).to_html().replace("<", "&lt;").replace(">", "&gt;").replace('"', "&quot;").replace("\n", "&#10;")
-        fstriplestore.triple_store.dump_triple(obj_state_uri, "<df-head>", '"' + df_head_html + '"')
+        ts.dump_triple(obj_state_cc_uri, "<df-head>", '"' + df_head_html + '"')
     elif output_type == 'plot':
+        ts.dump_triple(obj_state_cc_uri, "rdf:type", "<CCBasicPlot>")
+        ts.dump_triple(obj_state_cc_uri, "<df-shape>", f'"{df.shape}"')
         out_fd = io.BytesIO()
         fig = df.plot().get_figure()
         fig.savefig(out_fd)
         #ipdb.set_trace()
         im_s = base64.b64encode(out_fd.getvalue()).decode('ascii')
-        fstriplestore.triple_store.dump_triple(obj_state_uri, '<df-plot>', '"' + im_s + '"')
+        ts.dump_triple(obj_state_cc_uri, '<plot-im>', '"' + im_s + '"')
     else:
         raise Exception(f"unknown output_type: {output_type}")
 
 
-def dump_Series_obj_state(obj_state_uri, s, output_type):
-    fstriplestore.triple_store.dump_triple(obj_state_uri, "<df-shape>", f'"{s.shape}"')        
+def dump_Series_obj_state_cc(obj_state_uri, s, output_type):
+    ts = fstriplestore.triple_store
+    global random_id
+    obj_state_cc_uri = f"<ObjStateCC#{random_id}>"; random_id += 1
+
+    ts.dump_triple(obj_state_cc_uri, "rdf:type", "<ObjStateCC>")
+    ts.dump_triple(obj_state_cc_uri, "<obj-state>", obj_state_uri)
+
     if output_type == 'head':
-        pass
+        ts.dump_triple(obj_state_cc_uri, "rdf:type", "<CCGlance>")
+        ts.dump_triple(obj_state_cc_uri, "<shape>", f"{len(s)}")
     elif output_type == 'plot':
+        ts.dump_triple(obj_state_cc_uri, "rdf:type", "<CCBasicPlot>")        
+        ts.dump_triple(obj_state_cc_uri, "<s-size>", f"{len(s)}")
         out_fd = io.BytesIO()
         fig = s.plot().get_figure()
         fig.savefig(out_fd)
         #ipdb.set_trace()
         im_s = base64.b64encode(out_fd.getvalue()).decode('ascii')
-        fstriplestore.triple_store.dump_triple(obj_state_uri, '<df-plot>', '"' + im_s + '"')
+        ts.dump_triple(obj_state_cc_uri, '<plot-im>', '"' + im_s + '"')
     else:
         raise Exception(f"unknown output_type: {output_type}")
 

pyjviz/obj_utils.py

@@ -8,6 +8,7 @@
 from contextlib import nullcontext
 
 from . import obj_tracking
+from . import obj_utils
 from . import wb_stack
 from . import wb_stack_entries
 
@@ -47,9 +48,7 @@ def create(method_name, method_args, method_kwargs):
 
         latest_method_call = wb_stack.wb_stack.get_latest_method_call()
         if latest_method_call is None:
-            method_opts = wb_stack.wb_stack.get_top().method_opts
-            pyjviz_opts = method_opts.get(method_name) if method_name in method_opts else {}
-            ret = wb_stack_entries.MethodCall(method_name, pyjviz_opts)
+            ret = wb_stack_entries.MethodCall(method_name)
         else:
             ret = nullcontext()
 
@@ -76,10 +75,19 @@ def aux_init(func, *x, **y):
     if 1:
         old_to_datetime = pd.to_datetime
         pd.to_datetime = lambda *x, **y: DataFrameFunc("to_datetime", old_to_datetime)(*x, **y)
-        
+
+    if 1:
+        # we need those registration of pin method to make sure method call syntax work
+        # actual pin implementation is in MethodCall handle_* methods
+        @pf.register_series_method
+        def pin(s: pd.Series, output_type = 'head'): return s
+
+        @pf.register_dataframe_method
+        def pin(df: pd.DataFrame, output_type = 'head'): return df
+    
     old_describe = pd.DataFrame.describe
     #del pd.DataFrame.describe
-
+    
     @pf.register_dataframe_method
     def describe(df: pd.DataFrame) -> pd.DataFrame:
         print("override describe")

pyjviz/pf_pandas.py

@@ -32,55 +32,37 @@ def dump_subgraph(g, cc_uri, out_fd):
         dump_subgraph(g, subgraph, out_fd)
 
         rq = """
-        select ?obj_state ?version ?obj_type ?obj_uuid ?df_shape ?df_head ?df_im { 
+        select ?obj_state ?version ?obj_type ?obj_uuid { 
           ?obj_state rdf:type <ObjState>; <obj> ?obj.
           ?obj rdf:type <Obj>; <obj-type> ?obj_type; <obj-uuid> ?obj_uuid.
           ?obj_state <part-of>+ ?sg; <version> ?version .
-          ?obj_state <df-shape> ?df_shape .
-          optional {?obj_state <df-head> ?df_head}
-          optional {?obj_state <df-plot> ?df_im}
         }
         """
-        for obj_state, version, obj_type, obj_uudi, df_shape, df_head, df_im in g.query(rq, base = fstriplestore.base_uri, initBindings = {'sg': subgraph}):
-            if fstriplestore.triple_store.output_dir:
-                temp_dir = os.path.join(fstriplestore.triple_store.output_dir, "tmp")
-                with tempfile.NamedTemporaryFile(dir = temp_dir, suffix = '.html', delete = False) as temp_fp:
-                    if df_head:
-                        node_bgcolor = "#88000022"
-                        popup_size = (800, 200)
-                        temp_fp.write(df_head.toPython().replace("&lt;", "<").replace("&gt;", ">").replace("&quot;", "'").replace("&#10;", "\n").encode('ascii'))
-                    elif df_im:
-                        node_bgcolor = "#44056022"
-                        popup_size = (900, 500)
-                        temp_fp.write(("<img src='data:image/png;base64," + df_im.toPython() + "'></img>").encode('ascii'))
-                    else:
-                        node_bgcolor = "#88000022"
-                        popup_size = (800, 200)
-                        temp_fp.write('NONE'.encode('ascii'))
+        for obj_state, version, obj_type, obj_uudi in g.query(rq, base = fstriplestore.base_uri, initBindings = {'sg': subgraph}):
+            obj_state_cc_rq = """
+            select ?shape {
+            ?obj_state_cc <obj-state> ?obj_state.
+            ?obj_state_cc rdf:type <CCGlance>;
+                          <shape> ?shape
+            }
+            """
 
-                    href = f"""href="javascript:
-                    /* alert(location.pathname.match(/.*\//) + '\n' + '{temp_fp.name}' + '\n' + '{fstriplestore.triple_store.output_dir}'); */
-                    {{ window.open(location.pathname.match(/.*\//) + 'tmp/' + '{os.path.basename(temp_fp.name)}', '_blank', 'width={popup_size[0]},height={popup_size[1]}'); }}
-                    "
-                    """
-            else:
-                href = ""
-                node_bgcolor = "#88000022"
-                popup_size = (800, 200)
-                
-            #ipdb.set_trace()
-            print(f"""
-            node_{uri_to_dot_id(obj_state)} [
+            node_bgcolor = "#88000022"
+            for shape in g.query(obj_state_cc_rq, base = fstriplestore.base_uri, initBindings = {'obj_state': obj_state}):
+                #ipdb.set_trace()
+                shape = shape[0]
+                print(f"""
+                node_{uri_to_dot_id(obj_state)} [
                 color="{node_bgcolor}"
                 shape = rect
                 label = <<table border="0" cellborder="0" cellspacing="0" cellpadding="4">
-                         <tr> <td> <b>{obj_state.split('/')[-1]}</b><br/>{obj_type} {df_shape} {version}</td> </tr>
-                         </table>>
-                {href}
+                         <tr> <td> <b>{obj_state.split('/')[-1]}</b><br/>{obj_type} {shape.toPython()} {version}</td> </tr>
+                         </table>>                
                 ];
 
             """, file = out_fd)
 
+
         rq = """
         select ?method_call_obj ?method_name ?method_display ?method_count ?method_stack_depth ?method_stack_trace { 
           ?method_call_obj rdf:type <MethodCall>; 

pyjviz/viz.py

@@ -1,4 +1,4 @@
-#import ipdb
+import ipdb
 import threading
 import pandas as pd
 import base64
@@ -15,22 +15,27 @@
 class CodeBlock(wb_stack.WithBlock):
     def __init__(self, label = None, rdf_type = "CodeBlock"):
         super().__init__(label = label, rdf_type = rdf_type)
-        self.method_opts = {}
-
-    def set_method_call_opts(self, method_name, pyjviz_opts):
-        self.method_opts[method_name] = pyjviz_opts
 
 CB = CodeBlock
 
 method_counter = 0 # NB: should be better way to cout method calls
 class MethodCall(wb_stack.WithBlock):
     def __init__(self, method_name, pyjviz_opts = {}):
         super().__init__(label = method_name, rdf_type = "MethodCall")
+        self.method_name = method_name
         self.method_bound_args = None
         self.nested_call_args = []
         self.pyjviz_opts = pyjviz_opts
 
-    def handle_start_method_call(self, method_name, method_signature, method_args, method_kwargs):        
+    def handle_start_method_call(self, method_name, method_signature, method_args, method_kwargs):
+        if method_name == 'pin':
+            arg0_obj =  method_args[0]
+            t_obj, obj_found = obj_tracking.tracking_store.get_tracking_obj(arg0_obj)
+            if not obj_found:
+                t_obj = obj_utils.dump_obj_state(arg0_obj)
+            obj_utils.dump_obj_state_cc(t_obj.last_obj_state_uri, arg0_obj, 'plot')
+            return method_args, method_kwargs
+        
         all_args = method_args
         self.method_signature = method_signature
         #ipdb.set_trace()
@@ -73,11 +78,14 @@ def handle_start_method_call(self, method_name, method_signature, method_args, m
         return new_args, new_kwargs
 
     def handle_end_method_call(self, ret):
+        if self.method_name == 'pin':
+            return
+        
         ts = fstriplestore.triple_store
 
         ret_obj = ret if not ret is None else obj
 
-        ret_t_obj = obj_utils.dump_obj_state(ret_obj, self.pyjviz_opts)
+        ret_t_obj = obj_utils.dump_obj_state(ret_obj)
         ts.dump_triple(self.uri, "<method-call-return>", ret_t_obj.last_obj_state_uri)
 
         # catching nested calls values returned after method call executed