Implement feature to load components (plots, dataframes, and md files) from URLs provided in the config in streamlit and quarto reports

Author: sayalaruano

report_config_micw2graph.yaml

@@ -20,21 +20,21 @@ sections:
           - title: "Top 5 species by biome (plotly)"
             component_type: "plot"
             plot_type: "plotly"
-            file_path: "example_data/MicW2Graph/top_species_plot_biome.json"
+            file_path: "https://raw.githubusercontent.com/JosefineTM/DemProt_report/main/example_data/MicW2Graph/top_species_plot_biome.json"
             caption: "Optional caption"
           - title: "Multiline plot (altair)"
             component_type: "plot"
             plot_type: "altair"
             file_path: "example_data/altair_multilineplot.json"
           - title: "Abundance data for all studies (csv)"
             component_type: "dataframe"
-            file_path: "example_data/MicW2Graph/abundance_data_allbiomes.csv"
+            file_path: "https://raw.githubusercontent.com/JosefineTM/DemProt_report/main/example_data/MicW2Graph/abundance_data_allbiomes.csv"
             file_format: "csv"
             delimiter: ","
-          - title: "Abundance data for all studies (excel)"
+          - title: "Abundance data for all studies (xls)"
             component_type: "dataframe"
             file_path: "example_data/MicW2Graph/abundance_data_allbiomes.xls"
-            file_format: "excel"
+            file_format: "xls"
             delimiter: ","
       - title: "Sample data"
         components:
@@ -58,7 +58,7 @@ sections:
         components:
           - title: "Markdown example"
             component_type: "markdown"
-            file_path: "example_data/test_md.md"
+            file_path: "https://raw.githubusercontent.com/JosefineTM/DemProt_report/main/example_data/test_md.md"
   - title: "Microbial Association Networks"
     subsections:
       - title: "Network Visualization1"
@@ -76,10 +76,10 @@ sections:
             file_path: "example_data/MicW2Graph/man_example.csv"
       - title: "Network Visualization3"
         components:
-          - title: "Network3 (cyjs)"
+          - title: "Network3 (remote html)"
             component_type: "plot"
             plot_type: "interactive_network"
-            file_path: "example_data/MicW2Graph/man_example.cyjs"
+            file_path: "https://rawcdn.githack.com/JosefineTM/DemProt_report/3fca83b18e8b0666dff8b03fcdcf3d05d0d2541a/example_data/network1.html"
       - title: "Network Visualization4"
         components:
           - title: "Network4 (html)"

vuegen/quarto_reportview.py

@@ -3,7 +3,8 @@
 import report as r
 from typing import List
 import networkx as nx
-from utils import create_folder
+import pandas as pd
+from utils import create_folder, is_url
 
 class QuartoReportView(r.ReportView):
     """
@@ -329,23 +330,39 @@ def _generate_plot_code(self, plot, output_file = "") -> str:
         str
             The generated plot code as a string.
         """
-        # Start with the common data loading code
+        # Initialize plot code with common structure
         plot_code = f"""```{{python}}
 #| label: '{plot.title}'
 #| fig-cap: ""
+"""
+        # If the file path is a URL, generate code to fetch content via requests
+        if is_url(plot.file_path): 
+            plot_code += f"""
+response = requests.get('{plot.file_path}')
+response.raise_for_status()
+plot_json = response.text\n"""
+        else:  # If it's a local file
+            plot_code += f"""
 with open('{os.path.join("..", plot.file_path)}', 'r') as plot_file:
-    plot_data = plot_file.read()
-    """
+    plot_json = plot_file.read()\n""" 
         # Add specific code for each visualization tool
         if plot.plot_type == r.PlotType.PLOTLY:
-            plot_code += """fig_plotly = pio.from_json(plot_data)
-fig_plotly.update_layout(width=950, height=500)
-    """
+            plot_code += """
+fig_plotly = pio.from_json(plot_json)
+fig_plotly.update_layout(width=950, height=500)\n"""
         elif plot.plot_type == r.PlotType.ALTAIR:
-            plot_code += """fig_altair = alt.Chart.from_json(plot_data).properties(width=900, height=400)"""
+            plot_code += """fig_altair = alt.Chart.from_json(plot_json).properties(width=900, height=400)"""
         elif plot.plot_type == r.PlotType.INTERACTIVE_NETWORK:
-            plot_code = f"""<div style="text-align: center;">
-<iframe src="{os.path.join("..", output_file)}" alt="{plot.title} plot" width="800px" height="630px"></iframe>
+            # Generate the HTML embedding for interactive networks
+            if is_url(plot.file_path) and plot.file_path.endswith('.html'):
+                iframe_src = output_file
+            else:
+                iframe_src = os.path.join("..", output_file)
+
+            # Embed the HTML file in an iframe
+            plot_code = f"""
+<div style="text-align: center;">
+<iframe src="{iframe_src}" alt="{plot.title} plot" width="800px" height="630px"></iframe>
 </div>\n"""
         return plot_code
 
@@ -365,46 +382,47 @@ def _generate_dataframe_content(self, dataframe, is_report_static) -> List[str]:
         list : List[str]
             The list of content lines for the DataFrame.
         """
-        datframe_content = []
+        dataframe_content = []
         # Add title
-        datframe_content.append(f'### {dataframe.title}')
+        dataframe_content.append(f'### {dataframe.title}')
 
         # Append header for DataFrame loading
-        datframe_content.append(f"""```{{python}}
+        dataframe_content.append(f"""```{{python}}
 #| label: '{dataframe.title}'
 #| fig-cap: ""
 """)
+        # Mapping of file extensions to read functions
+        read_function_mapping = {
+            r.DataFrameFormat.CSV.value_with_dot: pd.read_csv,
+            r.DataFrameFormat.PARQUET.value_with_dot: pd.read_parquet,
+            r.DataFrameFormat.TXT.value_with_dot: pd.read_table,
+            r.DataFrameFormat.XLS.value_with_dot: pd.read_excel
+        }
         try:
-            if dataframe.file_format == r.DataFrameFormat.CSV:
-                if dataframe.delimiter:
-                    datframe_content.append(f"""df = pd.read_csv('{os.path.join("..", dataframe.file_path)}', delimiter='{dataframe.delimiter}')""")
-                    datframe_content.extend(self._show_dataframe(dataframe, is_report_static))
-                else:
-                    datframe_content.append(f"""df = pd.read_csv('{os.path.join("..", dataframe.file_path)}')""")
-                    datframe_content.extend(self._show_dataframe(dataframe, is_report_static))
-            elif dataframe.file_format == r.DataFrameFormat.PARQUET:
-                datframe_content.append(f"""df = pd.read_parquet('{os.path.join("..", dataframe.file_path)}')""")
-                datframe_content.extend(self._show_dataframe(dataframe, is_report_static))
-            elif dataframe.file_format == r.DataFrameFormat.TXT:
-                datframe_content.append(f"""df = pd.read_csv('{os.path.join("..", dataframe.file_path)}', sep='\\t')""")
-                datframe_content.extend(self._show_dataframe(dataframe, is_report_static))
-            elif dataframe.file_format == r.DataFrameFormat.EXCEL:
-                datframe_content.append(f"""df = pd.read_excel('{os.path.join("..", dataframe.file_path)}')""")
-                datframe_content.extend(self._show_dataframe(dataframe, is_report_static))
-            else:
-                self.report.logger.error(f"Unsupported DataFrame file format: {dataframe.file_format}")
-                raise ValueError(f"Unsupported DataFrame file format: {dataframe.file_format}")
+            # Check if the file extension matches any DataFrameFormat value
+            file_extension = os.path.splitext(dataframe.file_path)[1].lower()
+            if not any(file_extension == fmt.value_with_dot for fmt in r.DataFrameFormat):
+                self.report.logger.error(f"Unsupported file extension: {file_extension}. Supported extensions are: {', '.join(fmt.value for fmt in r.DataFrameFormat)}.")
+            
+            # Build the file path (URL or local file)
+            file_path = dataframe.file_path if is_url(dataframe.file_path) else os.path.join("..", dataframe.file_path)
+
+            # Load the DataFrame using the correct function
+            read_function = read_function_mapping[file_extension]
+            dataframe_content.append(f"""df = pd.{read_function.__name__}('{file_path}')""")
+
+            # Display the dataframe
+            dataframe_content.extend(self._show_dataframe(dataframe, is_report_static))
 
         except Exception as e:
             self.report.logger.error(f"Error generating content for DataFrame: {dataframe.title}. Error: {str(e)}")
             raise
-
         # Add caption if available
         if dataframe.caption:
-            datframe_content.append(f'>{dataframe.caption}\n')
+            dataframe_content.append(f'>{dataframe.caption}\n')
 
         self.report.logger.info(f"Successfully generated content for DataFrame: '{dataframe.title}'")
-        return datframe_content
+        return dataframe_content
 
     def _generate_markdown_content(self, markdown) -> List[str]:
         """
@@ -425,13 +443,25 @@ def _generate_markdown_content(self, markdown) -> List[str]:
         markdown_content.append(f'### {markdown.title}')
 
         try:
-            markdown_content.append(f"""```{{python}}
+            # Initialize md code with common structure
+            markdown_content.append(f"""
+```{{python}}
 #| label: '{markdown.title}'
-#| fig-cap: ""
+#| fig-cap: ""\n""")
+            # If the file path is a URL, generate code to fetch content via requests
+            if is_url(markdown.file_path): 
+                markdown_content.append(f"""
+response = requests.get('{markdown.file_path}')
+response.raise_for_status()
+markdown_content = response.text\n""")
+            else: #If it's a local file
+                markdown_content.append(f"""
 with open('{os.path.join("..", markdown.file_path)}', 'r') as markdown_file:
-    markdown_content = markdown_file.read()
-display.Markdown(markdown_content)
-```\n""")
+    markdown_content = markdown_file.read()\n""")
+                
+            # Code to display md content
+            markdown_content.append(f"""display.Markdown(markdown_content)\n```\n""")
+
         except Exception as e:
             self.report.logger.error(f"Error generating content for Markdown: {markdown.title}. Error: {str(e)}")
             raise
@@ -450,7 +480,7 @@ def _generate_image_content(self, image_path: str, alt_text: str = "", width: in
         Parameters
         ----------
         image_path : str
-            Path to the image file.
+            Path to the image file or a URL to the image.
         width : int, optional
             Width of the image in pixels (default is 650).
         height : int, optional
@@ -463,8 +493,11 @@ def _generate_image_content(self, image_path: str, alt_text: str = "", width: in
         str
             The formatted image content.
         """
-        return f"""
-![{alt_text}]({os.path.join('..', image_path)}){{ width={width}px height={height}px fig-align="center"}}\n"""
+        # Check if the image path is a URL or a local file path
+        if is_url(image_path):
+            return f"""![{alt_text}]({image_path}){{ width={width}px height={height}px fig-align="center"}}\n"""
+        else:
+            return f"""![{alt_text}]({os.path.join('..', image_path)}){{ width={width}px height={height}px fig-align="center"}}\n"""
 
     def _show_dataframe(self, dataframe, is_report_static, static_dir: str = STATIC_FILES_DIR) -> List[str]:
         """
@@ -517,11 +550,11 @@ def _generate_component_imports(self, component: r.Component) -> List[str]:
         # Dictionary to hold the imports for each component type
         components_imports = {
             'plot': {
-                r.PlotType.ALTAIR: ['import altair as alt'],
-                r.PlotType.PLOTLY: ['import plotly.io as pio']
+                r.PlotType.ALTAIR: ['import altair as alt', 'import requests'],
+                r.PlotType.PLOTLY: ['import plotly.io as pio', 'import requests']
             },
             'dataframe': ['import pandas as pd', 'from itables import show', 'import dataframe_image as dfi'],
-            'markdown': ['import IPython.display as display']
+            'markdown': ['import IPython.display as display', 'import requests']
         }
 
         # Iterate over sections and subsections to determine needed imports 

vuegen/report.py

@@ -11,7 +11,7 @@
 import json
 import matplotlib.pyplot as plt
 from pyvis.network import Network
-from utils import cyjs_to_networkx, pyvishtml_to_networkx
+from utils import cyjs_to_networkx, pyvishtml_to_networkx, fetch_file_stream
 
 class ReportType(StrEnum):
     STREAMLIT = auto()
@@ -58,7 +58,12 @@ class DataFrameFormat(StrEnum):
     CSV = auto()
     TXT = auto()
     PARQUET = auto()
-    EXCEL = auto()
+    XLS = auto()
+
+    @property
+    def value_with_dot(self):
+        """Return the file extension with the dot."""
+        return f".{self.name.lower()}"
 
 @dataclass
 class Component():
@@ -134,52 +139,55 @@ def read_network(self) -> nx.Graph:
         -------
         G : networkx.Graph
             A NetworkX graph object created from the specified network file.
+
+        Raises
+        ------
+        ValueError
+            If the file format is unsupported.
+        FileNotFoundError
+            If the file cannot be accessed or is missing.
+        RuntimeError
+            If there is an error while reading the network file.
         """
-        # Mapping of file extensions to NetworkX loading functions
+        # Mapping of file extensions to NetworkX and custom loading functions
         file_extension_map = {
             NetworkFormat.GML.value_with_dot: nx.read_gml,
             NetworkFormat.GRAPHML.value_with_dot: nx.read_graphml,
             NetworkFormat.GEXF.value_with_dot: nx.read_gexf,
             NetworkFormat.CYJS.value_with_dot: cyjs_to_networkx
         }
 
-        # Check if the file exists
-        if not os.path.isfile(self.file_path):
-            self.logger.error(f"File not found or cannot be accessed: {self.file_path}.")
-            raise FileNotFoundError(f"The file at {self.file_path} was not found or cannot be accessed.")
-
-        # Determine the file extension and check if it is supported
-        file_extension = os.path.splitext(self.file_path)[-1].lower()
-
-        # Check if the file extension matches any Enum value
-        if not any(file_extension == fmt.value_with_dot for fmt in NetworkFormat):
-            self.logger.error(f"Unsupported file extension: {file_extension}.")
-            raise ValueError(
-                f"Unsupported file extension: {file_extension}. Supported extensions are: "
-                f"{', '.join(fmt.value for fmt in NetworkFormat)}."
-            )
-
         # Handle .csv and .txt files with custom delimiters based on the text format (edgelist or adjlist)
         try:
-            # Handle HTML files (for pyvis interactive networks)
+            # Fetch the file stream (local or URL) using fetch_file_stream
+            file_stream = fetch_file_stream(self.file_path)
+            
+            # Determine the file extension and check if it is supported
+            file_extension = os.path.splitext(self.file_path)[-1].lower()
+
+            # Check if the file extension matches any Enum value
+            if not any(file_extension == fmt.value_with_dot for fmt in NetworkFormat):
+                self.logger.error(f"Unsupported file extension: {file_extension}. Supported extensions are: {', '.join(fmt.value for fmt in NetworkFormat)}.")
+            
+            # Handle HTML files for pyvis interactive networks
             if file_extension == NetworkFormat.HTML.value_with_dot:
-                G = pyvishtml_to_networkx(self.file_path)
+                G = pyvishtml_to_networkx(file_stream)
                 return (G, self.file_path)
 
-            # Handle .csv and .txt files with custom delimiters based on the text format (edgelist or adjlist)
+            # Handle CSV and TXT files with custom delimiters based on the text format (edgelist or adjlist)
             if file_extension in [NetworkFormat.CSV.value_with_dot, NetworkFormat.TXT.value_with_dot] and self.csv_network_format:
                 delimiter = ',' if file_extension == '.csv' else '\\t'
                 try:
-                    df_net = pd.read_csv(self.file_path, delimiter=delimiter)
+                    df_net = pd.read_csv(file_stream, delimiter=delimiter)
                 except pd.errors.ParserError:
                     self.logger.error(f"Error parsing CSV/TXT file {self.file_path}. Please check the file format or delimiter.")
-                    raise ValueError(f"Error parsing the file {self.file_path}. Please check the file format or delimiter.")
 
                 if self.csv_network_format == CSVNetworkFormat.EDGELIST:
                     # Assert that "source" and "target" columns are present in the DataFrame
                     required_columns = {"source", "target"}
                     if not required_columns.issubset(df_net.columns):
-                        self.logger.warning(f"CSV network file must contain columns named 'source' and 'target'. Missing columns: {', '.join(required_columns.difference(df_net.columns))}.")
+                        missing_cols = ", ".join(required_columns.difference(df_net.columns))
+                        self.logger.error(f"CSV network file must contain 'source' and 'target' columns. Missing columns: {missing_cols}.")
 
                     # Use additional columns as edge attributes, excluding "source" and "target"
                     edge_attributes = [col for col in df_net.columns if col not in required_columns]
@@ -194,10 +202,9 @@ def read_network(self) -> nx.Graph:
                     return G
                 else:
                     self.logger.error(f"Unsupported format for CSV/TXT file: {self.csv_network_format}.")
-                    raise ValueError(f"Unsupported format for CSV/TXT file: {self.csv_network_format}")
 
             # Handle other formats using the mapping and return the NetworkX graph object from the specified network file
-            G = file_extension_map[file_extension](self.file_path)
+            G = file_extension_map[file_extension](file_stream)
             G = self._add_size_attribute(G)
             self.logger.info(f"Successfully read network from file: {self.file_path}.")
             return G