⚡ improve performance and 💄 show loading in GUI

Author: Joseph Hirsch

demonstrator/app.py

@@ -62,7 +62,6 @@ def render_tabs_content(tab):
     return sites[int(tab)].layout()
 
 
-data_manager.register_callbacks(app)
 controls.register_callbacks(app)
 
 for site in sites:

demonstrator/assets/style.css

@@ -86,7 +86,7 @@ body {
 }
 .main_page_container {
     width: 100%;
-    min-height: 700px;
+    min-height: 850px;
     overflow: none;
     display: flex;
     flex-flow: column;

demonstrator/data_manager.py

@@ -4,17 +4,19 @@
 import plotly.express as px
 from plotly import graph_objects as go
 
-from data_manager import FilterOptions, get_frames_angle
+from data_manager import FilterOptions, get_frames_angle, get_headings_hist
 
 
 
 
 def layout():
 
 
-    return html.Div(className="two_columns", children=[
-        html.Div(className="graphContainer",id="img_direction"),
-        html.Div(className="nodeInfo", id="frameList")
+    return dcc.Loading([
+        html.Div(className="two_columns", children=[
+            html.Div(className="graphContainer",id="img_direction"),
+            html.Div(className="nodeInfo", id="frameList")
+        ])
     ])
 
 def title():
@@ -25,11 +27,10 @@ def register_callback(app):
     g_to_inspection = []
     @app.callback(
         Output(component_id='frameList', component_property='children'),
-        State(component_id='data_store', component_property='data'),
         State(component_id='filter_store', component_property= 'data'),
         Input(component_id={'type':'heading_graph', 'index': ALL}, component_property='clickData'),
     )
-    def update_nodes_table(data_store, filter_store, click_data):
+    def update_nodes_table(filter_store, click_data):
         filter_options = FilterOptions(**filter_store)
         if not hasattr(update_nodes_table, 'old_data'):
             update_nodes_table.old_data = click_data
@@ -41,21 +42,22 @@ def update_nodes_table(data_store, filter_store, click_data):
                 changed = g_to_inspection[i]
                 changed_angle = d['points'][0]['theta']
 
-        s = ""
         if changed is not None:
            frames = get_frames_angle(changed, (changed_angle + 360) % 360, filter_options) 
            return [html.Img(src=f"/assets/imgs/{'mosaics' if f['path'].startswith('m') else 'frames'}/{f['path']}", style={'width':'100%'}) for f in frames]
 
         update_nodes_table.old_data = click_data
-        return "Select sector to show frames" if changed is None else s
+        return "Select sector to show frames"
 
     @app.callback(
         Output(component_id='img_direction', component_property='children'),
-        Input(component_id='data_store', component_property='data'),
+        Input(component_id='filter_store', component_property='data'),
     )
-    def update_image_table(data):
+    def update_image_table(filter_options):
+        filter_options = FilterOptions(**filter_options)
+        heading_hist = get_headings_hist(filter_options)
         graphs = []
-        for inspection_id, heading_hist in data['heading_hist'].items():
+        for inspection_id, heading_hist in heading_hist.items():
             g_to_inspection.append(inspection_id)
             title=f"{heading_hist['ship_name']} on {heading_hist['date']}"
             fig = px.bar_polar(heading_hist['data'], r="count", theta="heading", width=800, height=800, title=title)

demonstrator/views/bar_charts.py

@@ -3,7 +3,7 @@
 from dash import dcc, Input, Output, State, html, dash_table
 import dash_cytoscape as cyto
 import plotly.express as px
-from data_manager import get_frames_cluster
+from data_manager import FilterOptions, get_cluster_table, get_frames_cluster
 
 
 
@@ -35,20 +35,24 @@ def layout():
         'sort_action':"native",
         'sort_by': [{'column_id': 'quality', 'direction': 'desc'}]
     }
-    return html.Div(className="two_column", children=
+    return html.Div(className="two_column", children=[dcc.Loading(
         [html.Div(className="graphContainer", id="clusters_table_container", children=[dash_table.DataTable(id='clusters_table', columns=columns, **table_options)]),
-        html.Div(className="nodeInfo", style={'width':'25%'}, id="cluster_info", children=["yes its me"])])
+        html.Div(className="nodeInfo", style={'width':'25%'}, id="cluster_info", children=[])
+        ])
+        ])
 
 def title():
     return 'Clusters'
 
 def register_callback(app):
     @app.callback(
         Output(component_id='clusters_table', component_property='data'),
-        Input(component_id='data_store', component_property='data'),
+        Input(component_id='filter_store', component_property='data'),
     )
-    def update_image_table(data):
-        return data['clusters_table']
+    def update_image_table(filter_options):
+        filter_options = FilterOptions(**filter_options)
+        cluster_table = get_cluster_table(filter_options)
+        return cluster_table
 
     @app.callback(
         Output(component_id='cluster_info', component_property='children'),

demonstrator/views/clusters.py

@@ -190,6 +190,8 @@
 
 import networkx as nx
 
+from data_manager import FilterOptions, get_graph_stuff
+
 finding = ['defect', 'corrosion', 'marine_growth', 'paint_peel', 'anode', 'over_board_valve', 'propeller', 'sea_chest_grating', 'bilge_keel']
 
 empty_elements = {
@@ -310,20 +312,19 @@ def title():
 def register_callback(app):
     @app.callback(
         Output(component_id='gr', component_property='elements'),
-        Input(component_id='data_store', component_property='data'),
-        State(component_id='filter_store', component_property='data'),
+        Input(component_id='filter_store', component_property='data'),
     )
-    def update_graph(data, filter_options):
+    def update_graph(filter_options):
+        filter_options = FilterOptions(**filter_options)
+
+        nodes, similar_to, visually_similar_to, in_inspection, in_mosaic, in_cluster, shows_part, part_of_ship = get_graph_stuff(filter_options)
 
-        print('updating graph...')
         graph = nx.Graph()
-        nodes = data['nodes']
         if len(nodes) == 0:
             return empty_elements
 
 
         node_tuples = []
-        #node_tuples = random.sample([(key, value) for key, value in nodes.items()], k=min(len(nodes.items()), 100))
         node_tuples = [(key, value) for key, value in nodes.items()]
         node_keys = {k for k,_ in node_tuples}
 
@@ -339,34 +340,34 @@ def update_graph(data, filter_options):
         PARTOFS_EDGE_WEIGHT = 0.1
         SHOWSPR_EDGE_WEIGHT = 0.1
 
-        for n1, n2 in data['in_inspection']:
+        for n1, n2 in in_inspection:
             if not n1 in node_keys or not n2 in node_keys: continue
             relcount += 1
             graph.add_edge(n1, n2, classes="inspection", weight=ININSPE_EDGE_WEIGHT)
-        for n1, n2 in data['similar_to']:
+        for n1, n2 in similar_to:
             if not n1 in node_keys or not n2 in node_keys: continue
             relcount += 1
             graph.add_edge(n1, n2, classes="dashed", weight=SIMILAR_EDGE_WEIGHT)
-        for n1, n2 in data['visually_similar_to']:
+        for n1, n2 in visually_similar_to:
             if not n1 in node_keys or not n2 in node_keys: continue
             relcount += 1
             graph.add_edge(n1, n2, classes="dotted", weight=SIMILAR_EDGE_WEIGHT)
-        for n1, n2 in data['in_mosaic']:
+        for n1, n2 in in_mosaic:
             if not n1 in node_keys or not n2 in node_keys: continue
             relcount += 1
             replace_with.add((n1, n2))
             graph.add_edge(n1, n2)
 
-        for n1, n2 in data['in_cluster']:
+        for n1, n2 in in_cluster:
             if not n1 in node_keys or not n2 in node_keys: continue
             relcount += 1
             graph.add_edge(n1, n2, weight=CLUSTER_EDGE_WEIGHT)
 
-        for n1, n2 in data['part_of_ship']:
+        for n1, n2 in part_of_ship:
             if not n1 in node_keys or not n2 in node_keys: continue
             relcount += 1
             graph.add_edge(n1, n2, weight=PARTOFS_EDGE_WEIGHT)
-        for n1, n2 in data['shows_part']:
+        for n1, n2 in shows_part:
             if not n1 in node_keys or not n2 in node_keys: continue
             relcount += 1
             graph.add_edge(n1, n2, weight=SHOWSPR_EDGE_WEIGHT, classes="part")
@@ -378,21 +379,8 @@ def update_graph(data, filter_options):
                 graph.add_edge(n1, n2, **data)
             graph.remove_node(image)
 
-
-        positions_by_pca = {
-            key: (float(tup['tsne'].split(',')[0]),float(tup['tsne'].split(',')[1]))
-            
-            for key, tup in node_tuples if 'tsne' in tup} 
-         
-        fixed_positions = [k for k in positions_by_pca.keys()]
-        print(f"loaded result, got {len(node_tuples)} nodes and {relcount} relationships, layouting...")
-
-        #pos = nx.spring_layout(graph, fixed = fixed_positions, pos = positions_by_pca, k=1/math.sqrt(graph.order() + 1), iterations=100)
-
         pos = nx.spring_layout(graph, scale=1000, k=10/math.sqrt(graph.order() + 1), iterations=300, seed=84365)
-
         elmts = redact_json_data(nx.cytoscape_data(graph), pos)['elements']
-        print(f"done")
 
         return elmts
 

demonstrator/views/graph.py

@@ -2,95 +2,39 @@
 import dash_cytoscape as cyto
 import plotly.express as px
 
+from data_manager import FilterOptions, get_histogram_data
+
 
 
 def layout():
-    return html.Div([
-        dcc.Graph(id="q_u_hist"),
-        dcc.Graph(id="d_hist"),
-        dcc.Graph(id="mg_hist"),
-        dcc.Graph(id="sim_hist"),
-        dcc.Graph(id="vsim_hist")
-    ], className="histogramContainer")
+    return dcc.Loading([
+        html.Div(id = 'histogram_container', children=[], className="histogramContainer")
+    ])
 
 def title():
     return 'Histograms'
 
 def register_callback(app):
     @app.callback(
-        Output(component_id='q_u_hist', component_property='figure'),
-        Input(component_id='data_store', component_property='data'),
-    )
-    def update_histograms(data):
-        fig = px.histogram(data['q_hist'], range_x=(0,16), labels=["Image Quality (UCIQE)"], color_discrete_sequence=["darkgray"])
-        fig.update_layout(
-            title_text='Image Quality (UCIQE)', # title of plot
-            xaxis_title_text='Image Quality (UCIQE)', # xaxis label
-            yaxis_title_text='#Frames', # yaxis label
-            bargap=0.1, # gap between bars of adjacent location coordinates
-            showlegend=False,
-            hovermode=False
-        )
-        return fig
-
-    @app.callback(
-        Output(component_id='mg_hist', component_property='figure'),
-        Input(component_id='data_store', component_property='data'),
-    )
-    def update_histograms(data):
-        fig = px.histogram(data['mg_hist'], labels=["Marine Growth Percentage"], color_discrete_sequence=["darkgray"])
-        fig.update_layout(
-            title_text='Marine Growth Percentage', # title of plot
-            xaxis_title_text='Marine Growth Percentage', # xaxis label
-            yaxis_title_text='#Mosaics', # yaxis label
-            bargap=0.1, # gap between bars of adjacent location coordinates
-            showlegend=False,
-            hovermode=False
-        )
-        return fig
-
-    @app.callback(
-        Output(component_id='vsim_hist', component_property='figure'),
-        Input(component_id='data_store', component_property='data'),
-    )
-    def update_histograms(data):
-        fig = px.histogram(data['vsim_hist'], labels=['Visual Similarities'], color_discrete_sequence=["darkgray"])
-        fig.update_layout(
-            title_text='Visual Similarities', # title of plot
-            xaxis_title_text='Visual Similarities', # xaxis label
-            yaxis_title_text='#Frames', # yaxis label
-            bargap=0.1, # gap between bars of adjacent location coordinates
-            showlegend=False,
-            hovermode=False
-        )
-        return fig
-    @app.callback(
-        Output(component_id='sim_hist', component_property='figure'),
-        Input(component_id='data_store', component_property='data'),
-    )
-    def update_histograms(data):
-        fig = px.histogram(data['sim_hist'], labels=['Similarities'], color_discrete_sequence=["darkgray"])
-        fig.update_layout(
-            title_text='Similarities', # title of plot
-            xaxis_title_text='Similarities', # xaxis label
-            yaxis_title_text='#Similarities', # yaxis label
-            bargap=0.1, # gap between bars of adjacent location coordinates
-            showlegend=False,
-            hovermode=False
-        )
-        return fig
-    @app.callback(
-        Output(component_id='d_hist', component_property='figure'),
-        Input(component_id='data_store', component_property='data'),
+        Output(component_id='histogram_container', component_property='children'),
+        Input(component_id='filter_store', component_property='data'),
     )
-    def update_histograms(data):
-        fig = px.histogram(data['d_hist'], range_x=(-10,0), labels=['Depth'], color_discrete_sequence=["darkgray"])
-        fig.update_layout(
-            title_text='Depth', # title of plot
-            xaxis_title_text='Depth', # xaxis label
-            yaxis_title_text='#Frames', # yaxis label
-            bargap=0.1, # gap between bars of adjacent location coordinates
-            showlegend=False,
-            hovermode=False
-        )
-        return fig
+    def update_histograms(filter_options):
+        filter_options = FilterOptions(**filter_options)
+        data = get_histogram_data(filter_options)
+        labels = ['Image Quality (UCIQE)', 'Marine growth percentage', 'Depth', 'Similarities', 'Visual similarities']
+        x_axes = ['Image Quality (UCIQE)', 'Marine growth percentage', 'Depth', 'Similarities', 'Visual similarities']
+        y_axes = ['#Frames', '#Mosaics', '#Frames', '#Frames', '#Frames']
+        histograms = []
+        for data, label, y_axis, x_axis in zip(data, labels, y_axes, x_axes):
+            fig = px.histogram(data, labels=[label], color_discrete_sequence=["darkgray"])
+            fig.update_layout(
+                title_text=label, # title of plot
+                xaxis_title_text=x_axis, # xaxis label
+                yaxis_title_text=y_axis, # yaxis label
+                bargap=0.1, # gap between bars of adjacent location coordinates
+                showlegend=False,
+                hovermode=False
+            )
+            histograms.append(dcc.Graph(figure=fig))
+        return histograms