Merge pull request #471 from CodeForPhilly/PhilMiller/model-performance

Speed up by 20x by never copying arrays and only constructing a dataframe after the model has run

Author: jlubken

src/app.py

@@ -36,7 +36,7 @@
 st.markdown("Projected number of **daily** COVID-19 admissions. \n\n _NOTE: Now including estimates of prior admissions for comparison._")
 admits_chart = build_admits_chart(alt=alt, admits_floor_df=m.admits_floor_df, max_y_axis=p.max_y_axis)
 st.altair_chart(admits_chart, use_container_width=True)
-st.markdown(build_descriptions(chart=admits_chart, labels=p.labels, suffix=" Admissions"))
+st.markdown(build_descriptions(chart=admits_chart, labels=p.labels, prefix="admits_", suffix=" Admissions"))
 display_download_link(
     st,
     filename=f"{p.current_date}_projected_admits.csv",
@@ -58,7 +58,7 @@
 st.markdown("Projected **census** of COVID-19 patients, accounting for arrivals and discharges \n\n _NOTE: Now including estimates of prior census for comparison._")
 census_chart = build_census_chart(alt=alt, census_floor_df=m.census_floor_df, max_y_axis=p.max_y_axis)
 st.altair_chart(census_chart, use_container_width=True)
-st.markdown(build_descriptions(chart=census_chart, labels=p.labels, suffix=" Census"))
+st.markdown(build_descriptions(chart=census_chart, labels=p.labels, prefix="census_", suffix=" Census"))
 display_download_link(
     st,
     filename=f"{p.current_date}_projected_census.csv",

src/chime_dash/app/services/callbacks.py

@@ -38,10 +38,14 @@ def handle_model_change(i, sidebar_data):
         if sidebar_data:
             pars = parameters_deserializer(sidebar_data["parameters"])
             model = SimSirModel(pars)
+            vis = i.components.get("visualizations", None) if i else None
+            vis_content = vis.content if vis else None
+
             viz_kwargs = dict(
                 labels=pars.labels,
                 table_mod=7,
                 max_y_axis=pars.max_y_axis,
+                content=vis_content
             )
         result.extend(i.components["intro"].build(model, pars))
         for df_key in ["admits_df", "census_df", "sim_sir_w_date_df"]:

src/chime_dash/app/services/plotting.py

@@ -7,9 +7,17 @@
 from pandas import DataFrame
 
 
-def plot_dataframe(dataframe: DataFrame, max_y_axis: int = None,) -> Dict[str, Any]:
-    """
+def plot_dataframe(
+    dataframe: DataFrame,
+    max_y_axis: int = None,
+) -> Dict[str, Any]:
+    """Returns dictionary used for plotly graphs
+
+    Arguments:
+        dataframe: The dataframe to plot. Plots all columns as y, index is x.
+        max_y_axis: Maximal value on y-axis.
     """
+
     if max_y_axis is None:
         yaxis = {}
     else:

src/chime_dash/app/templates/en/visualizations.yml

@@ -16,5 +16,20 @@ hosp: Hospitalized
 icu: ICU
 vent: Ventalized
 
+# Admits df column translations
+admits_hospitalized: Hospitalized
+admits_icu: ICU
+admits_ventilated: Ventilated
+
+# Admits df column translations
+census_hospitalized: Hospitalized
+census_icu: ICU
+census_ventilated: Ventilated
+
+# SIR df column translations
+susceptible: Susceptible
+infected: Infected
+recovered: Recovered
+
 # Date Localization
 date-format: "%m%d%Y"

src/chime_dash/app/utils/__init__.py

@@ -116,18 +116,50 @@ def get_n_switch_values(input_value, elements_to_update) -> List[bool]:
 
 def prepare_visualization_group(df: DataFrame = None, **kwargs) -> List[Any]:
     """Creates plot, table and download link for data frame.
+
+    Arguments:
+        df: The Dataframe to plot
+        content: Dict[str, str]
+            Mapping for translating columns and index.
+        max_y_axis:  int
+            Maximal value on y-axis
+        labels: List[str]
+            Columns to display
+        table_mod: int
+            Displays only each `table_mod` row in table
+
     """
     result = [{}, None, None]
     if df is not None and isinstance(df, DataFrame):
+
+        date_column = "date"
+        day_column = "day"
+
+        # Translate column and index if specified
+        content = kwargs.get("content", None)
+        if content:
+            columns = {col: content[col] for col in df.columns if col in content}
+            index = (
+                {df.index.name: content[df.index.name]}
+                if df.index.name and df.index.name in content
+                else None
+            )
+            df = df.rename(columns=columns, index=index)
+            date_column = content.get(date_column, date_column)
+            day_column = content.get(day_column, day_column)
+
         plot_data = plot_dataframe(
-            df.dropna().set_index("date").drop(columns=["day"]),
+            df.dropna().set_index(date_column).drop(columns=[day_column]),
             max_y_axis=kwargs.get("max_y_axis", None),
         )
 
+
+        # translate back for backwards compability of build_table
+        column_map = {day_column: "day", date_column: "date"}
         table = (
             df_to_html_table(
                 build_table(
-                    df=df,
+                    df=df.rename(columns=column_map),
                     labels=kwargs.get("labels", df.columns),
                     modulo=kwargs.get("table_mod", 7),
                 ),
@@ -140,7 +172,9 @@ def prepare_visualization_group(df: DataFrame = None, **kwargs) -> List[Any]:
             # else None
         )
 
-        csv = build_csv_download(df)
+        # Convert columnnames to lowercase
+        column_map = {col: col.lower() for col in df.columns}
+        csv = build_csv_download(df.rename(columns=column_map))
         result = [plot_data, table, csv]
 
     return result
@@ -153,4 +187,5 @@ def get_instance(*args, **kwargs):
         if class_ not in instances:
             instances[class_] = class_(*args, **kwargs)
         return instances[class_]
+
     return get_instance

src/penn_chime/charts.py

@@ -26,7 +26,7 @@ def build_admits_chart(
     # TODO fix the fold to allow any number of dispositions
     points = (
         alt.Chart()
-        .transform_fold(fold=["hospitalized", "icu", "ventilated"])
+        .transform_fold(fold=["admits_hospitalized", "admits_icu", "admits_ventilated"])
         .encode(x=alt.X(**x), y=alt.Y(**y), color=color, tooltip=tooltip)
         .mark_line(point=True)
         .encode(
@@ -65,7 +65,7 @@ def build_census_chart(
     # TODO fix the fold to allow any number of dispositions
     points = (
         alt.Chart()
-        .transform_fold(fold=["hospitalized", "icu", "ventilated"])
+        .transform_fold(fold=["census_hospitalized", "census_icu", "census_ventilated"])
         .encode(x=alt.X(**x), y=alt.Y(**y), color=color, tooltip=tooltip)
         .mark_line(point=True)
         .encode(
@@ -128,7 +128,11 @@ def build_sim_sir_w_date_chart(
 
 
 def build_descriptions(
-    *, chart: Chart, labels: Dict[str, str], suffix: str = ""
+    *,
+    chart: Chart,
+    labels: Dict[str, str],
+    prefix: str = "",
+    suffix: str = ""
 ) -> str:
     """
 
@@ -145,17 +149,17 @@ def build_descriptions(
     day = "date" if "date" in chart.data.columns else "day"
 
     for col in cols:
-        if chart.data[col].idxmax() + 1 == len(chart.data):
+        if chart.data[prefix+col].idxmax() + 1 == len(chart.data):
             asterisk = True
 
         # todo: bring this to an optional arg / i18n
-        on = datetime.strftime(chart.data[day][chart.data[col].idxmax()], "%b %d")
+        on = datetime.strftime(chart.data[day][chart.data[prefix+col].idxmax()], "%b %d")
 
         messages.append(
             "{}{} peaks at {:,} on {}{}".format(
                 labels[col],
                 suffix,
-                ceil(chart.data[col].max()),
+                ceil(chart.data[prefix+col].max()),
                 on,
                 "*" if asterisk else "",
             )