From 50c53a70b4ccf43df8ed8d5634e30c54aa5e81f2 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Tue, 10 Dec 2024 15:37:53 +0100 Subject: [PATCH 1/8] :sparkles: vuegen pkg - package can be imported - relative import except in main script (will be renamed __main__ and added as an entry point) --- README.md | 14 +++++++++++--- pyproject.toml | 3 ++- vuegen/__init__.py | 3 +++ vuegen/config_manager.py | 7 ++++--- vuegen/main.py | 4 ++-- vuegen/quarto_reportview.py | 7 +++++-- vuegen/report.py | 14 ++++++++------ vuegen/report_generator.py | 14 ++++++++------ vuegen/streamlit_reportview.py | 9 ++++++--- 9 files changed, 49 insertions(+), 26 deletions(-) create mode 100644 vuegen/__init__.py diff --git a/README.md b/README.md index f3bbb80..89ebdf2 100644 --- a/README.md +++ b/README.md @@ -34,16 +34,24 @@ Also, the class diagram for the project is presented below to illustrate the arc

## Installation -...... + +You can install the package for development from this repository by running the following command: + +```bash +pip install -e path/to/vuegen # specify location +pip install -e . # in case you pwd is in the vuegen directory +``` ## Execution -``` shell + +```bash python vuegen/main.py --config report_config_micw2graph.yaml --report_type streamlit ``` + The current report types are streamlit, html, pdf, docx, odt, revealjs, pptx, and jupyter. ## Contact If you have comments or suggestions about this project, you can [open an issue][issues] in this repository. [issues]: https://github.com/Multiomics-Analytics-Group/vuegen/issues/new -[streamlit]: https://streamlit.io/ \ No newline at end of file +[streamlit]: https://streamlit.io/ diff --git a/pyproject.toml b/pyproject.toml index d6233a4..3178b24 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,11 @@ [tool.poetry] -name = "report-generator" +name = "vuegen" version = "0.1.0" description = "Module to generate automatic web interface reports with visualizations" authors = ["MoNA group"] license = "MIT" readme = "README.md" +repository = "https://github.com/Multiomics-Analytics-Group/vuegen" [tool.poetry.dependencies] python = ">=3.9,<3.9.7 || >3.9.7,<4.0" diff --git a/vuegen/__init__.py b/vuegen/__init__.py new file mode 100644 index 0000000..5986200 --- /dev/null +++ b/vuegen/__init__.py @@ -0,0 +1,3 @@ +from importlib.metadata import version + +__version__ = version("vuegen") diff --git a/vuegen/config_manager.py b/vuegen/config_manager.py index d06f9f5..4ad12ac 100644 --- a/vuegen/config_manager.py +++ b/vuegen/config_manager.py @@ -1,5 +1,6 @@ -import report as r -from utils import get_logger, assert_enum_value +from . import report as r +from .utils import assert_enum_value, get_logger + class ConfigManager: """ @@ -260,4 +261,4 @@ def _create_chatbot_component(self, component_data: dict) -> r.ChatBot: caption = component_data.get('caption'), headers = component_data.get('headers'), params = component_data.get('params') - ) \ No newline at end of file + ) diff --git a/vuegen/main.py b/vuegen/main.py index b163954..9d1af50 100644 --- a/vuegen/main.py +++ b/vuegen/main.py @@ -1,5 +1,5 @@ -import report_generator -from utils import get_logger, load_yaml_config, get_args +from vuegen import report_generator +from vuegen.utils import get_args, get_logger, load_yaml_config if __name__ == '__main__': # Parse command-line arguments diff --git a/vuegen/quarto_reportview.py b/vuegen/quarto_reportview.py index 954e9e2..218463b 100644 --- a/vuegen/quarto_reportview.py +++ b/vuegen/quarto_reportview.py @@ -1,10 +1,13 @@ import os import subprocess -import report as r from typing import List + import networkx as nx import pandas as pd -from utils import create_folder, is_url + +from . import report as r +from .utils import create_folder, is_url + class QuartoReportView(r.ReportView): """ diff --git a/vuegen/report.py b/vuegen/report.py index a85830b..645e06a 100644 --- a/vuegen/report.py +++ b/vuegen/report.py @@ -1,17 +1,19 @@ +import json +import logging import os from abc import ABC, abstractmethod from dataclasses import dataclass, field -from typing import ClassVar from enum import StrEnum, auto -from typing import List, Optional +from typing import ClassVar, List, Optional + +import matplotlib.pyplot as plt import networkx as nx import pandas as pd -import logging import requests -import json -import matplotlib.pyplot as plt from pyvis.network import Network -from utils import cyjs_to_networkx, pyvishtml_to_networkx, fetch_file_stream + +from .utils import cyjs_to_networkx, fetch_file_stream, pyvishtml_to_networkx + class ReportType(StrEnum): STREAMLIT = auto() diff --git a/vuegen/report_generator.py b/vuegen/report_generator.py index 80c11c9..9975d99 100644 --- a/vuegen/report_generator.py +++ b/vuegen/report_generator.py @@ -1,10 +1,12 @@ -from streamlit_reportview import StreamlitReportView -from quarto_reportview import QuartoReportView -from config_manager import ConfigManager -from utils import assert_enum_value -from report import ReportType import logging +from .config_manager import ConfigManager +from .quarto_reportview import QuartoReportView +from .report import ReportType +from .streamlit_reportview import StreamlitReportView +from .utils import assert_enum_value + + def get_report(config: dict, report_type: str, logger: logging.Logger) -> None: """ Generate and run a report based on the specified engine. @@ -45,4 +47,4 @@ def get_report(config: dict, report_type: str, logger: logging.Logger) -> None: report_type=report_type ) quarto_report.generate_report() - quarto_report.run_report() \ No newline at end of file + quarto_report.run_report() diff --git a/vuegen/streamlit_reportview.py b/vuegen/streamlit_reportview.py index a64b178..76f54be 100644 --- a/vuegen/streamlit_reportview.py +++ b/vuegen/streamlit_reportview.py @@ -1,9 +1,12 @@ -import report as r import os import subprocess from typing import List + import pandas as pd -from utils import create_folder, is_url + +from . import report as r +from .utils import create_folder, is_url + class StreamlitReportView(r.WebAppReportView): """ @@ -658,4 +661,4 @@ def _generate_component_imports(self, component: r.Component) -> List[str]: component_imports.extend(components_imports['chatbot']) # Return the list of import statements - return component_imports \ No newline at end of file + return component_imports From 402bc01c9400316a5fa77e9070b6873afcc7a6ec Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Tue, 10 Dec 2024 15:39:10 +0100 Subject: [PATCH 2/8] :bug: make pkg Python 3.9 compatible - import future anntotations and use external pkg to make StrEnum availble for python below 3.11 --- pyproject.toml | 2 +- vuegen/report.py | 10 ++++++++-- vuegen/utils.py | 25 +++++++++++++++++-------- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3178b24..9b4e4f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ itables = "^2.2.2" kaleido = "0.2.0" vl-convert-python = "^1.7.0" dataframe-image = "^0.2.6" - +strenum = { version = "^0.4.15", python = "<3.11" } [tool.poetry.group.dev.dependencies] ipykernel = "^6.29.5" diff --git a/vuegen/report.py b/vuegen/report.py index 645e06a..7c8224d 100644 --- a/vuegen/report.py +++ b/vuegen/report.py @@ -3,7 +3,13 @@ import os from abc import ABC, abstractmethod from dataclasses import dataclass, field -from enum import StrEnum, auto +from enum import auto + +try: + from enum import StrEnum +except ImportError: + from strenum import StrEnum + from typing import ClassVar, List, Optional import matplotlib.pyplot as plt @@ -738,4 +744,4 @@ def _generate_subsection(self, subsection: Subsection) -> tuple[List[str], List[ - list of subsection content lines (List[str]) - list of imports for the subsection (List[str]) """ - pass \ No newline at end of file + pass diff --git a/vuegen/utils.py b/vuegen/utils.py index 69429ed..27e1de6 100644 --- a/vuegen/utils.py +++ b/vuegen/utils.py @@ -1,18 +1,27 @@ +from __future__ import annotations + +import argparse +import json +import logging import os import sys -import yaml from datetime import datetime -import logging -import argparse -import requests -import networkx as nx -import json + +try: + from enum import StrEnum +except ImportError: + from strenum import StrEnum + from io import StringIO -from enum import StrEnum from typing import Type -from bs4 import BeautifulSoup from urllib.parse import urlparse +import networkx as nx +import requests +import yaml +from bs4 import BeautifulSoup + + ## CHECKS def check_path(filepath: str) -> bool: """ From 36bcfdfb9084d5f81a5f7cca937e4127fae6f5d4 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Tue, 10 Dec 2024 15:43:33 +0100 Subject: [PATCH 3/8] :bug: specify relative link to config file from root - same as for main.py - update config accordingly --- README.md | 2 +- .../MicW2Graph/report_config_micw2graph.yaml | 26 +++++++++---------- pyproject.toml | 7 +++++ 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 89ebdf2..51ca827 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ pip install -e . # in case you pwd is in the vuegen directory ## Execution ```bash -python vuegen/main.py --config report_config_micw2graph.yaml --report_type streamlit +python vuegen/main.py --config example_data/MicW2Graph/report_config_micw2graph.yaml --report_type streamlit ``` The current report types are streamlit, html, pdf, docx, odt, revealjs, pptx, and jupyter. diff --git a/example_data/MicW2Graph/report_config_micw2graph.yaml b/example_data/MicW2Graph/report_config_micw2graph.yaml index e9af6b4..e4aa322 100644 --- a/example_data/MicW2Graph/report_config_micw2graph.yaml +++ b/example_data/MicW2Graph/report_config_micw2graph.yaml @@ -9,8 +9,8 @@ report: infer potential interactions among microorganisms through microbial association networks (MANs). MicW2Graph enables the investigation of research questions related to WWT, focusing on aspects such as microbial connections, community memberships, and potential ecological functions. - graphical_abstract: /Users/asaru/Documents/DTU/MoNA/vuegen/example_data/MicW2Graph/Methods_MicW2Graph.png - logo: /Users/asaru/Documents/DTU/MoNA/vuegen/docs/images/vuegen_logo.png + graphical_abstract: example_data/MicW2Graph/Methods_MicW2Graph.png + logo: docs/images/vuegen_logo.png sections: - title: Exploratory Data Analysis subsections: @@ -25,7 +25,7 @@ sections: - title: Multiline plot (altair) component_type: plot plot_type: altair - file_path: /Users/asaru/Documents/DTU/MoNA/vuegen/example_data/MicW2Graph/1_Exploratory_Data_Analysis/1_Abundance_data/2_multilineplot_altair.json + file_path: example_data/MicW2Graph/1_Exploratory_Data_Analysis/1_Abundance_data/2_multilineplot_altair.json - title: Abundance data for all studies (csv) component_type: dataframe file_path: https://raw.githubusercontent.com/sayalaruano/report_generator_st_cloud_test/main/example_data/MicW2Graph/abundance_data_allbiomes.csv @@ -33,7 +33,7 @@ sections: delimiter: ',' - title: Abundance data for all studies (xls) component_type: dataframe - file_path: /Users/asaru/Documents/DTU/MoNA/vuegen/example_data/MicW2Graph/1_Exploratory_Data_Analysis/1_Abundance_data/4_abundance_data_allbiomes.xls + file_path: example_data/MicW2Graph/1_Exploratory_Data_Analysis/1_Abundance_data/4_abundance_data_allbiomes.xls file_format: xls delimiter: ',' - title: Sample data @@ -41,18 +41,18 @@ sections: - title: Number of samples per study (png) component_type: plot plot_type: static - file_path: /Users/asaru/Documents/DTU/MoNA/vuegen/example_data/MicW2Graph/1_Exploratory_Data_Analysis/2_Sample_data/3_number_samples_per_study.png + file_path: example_data/MicW2Graph/1_Exploratory_Data_Analysis/2_Sample_data/3_number_samples_per_study.png - title: Sampling countries for all studies (plotly) component_type: plot plot_type: plotly - file_path: /Users/asaru/Documents/DTU/MoNA/vuegen/example_data/MicW2Graph/1_Exploratory_Data_Analysis/2_Sample_data/1_pie_plot_countries_plotly.json + file_path: example_data/MicW2Graph/1_Exploratory_Data_Analysis/2_Sample_data/1_pie_plot_countries_plotly.json - title: Sample data for all studies (parquet) component_type: dataframe - file_path: /Users/asaru/Documents/DTU/MoNA/vuegen/example_data/MicW2Graph/1_Exploratory_Data_Analysis/2_Sample_data/4_sample_info_allbiomes.parquet + file_path: example_data/MicW2Graph/1_Exploratory_Data_Analysis/2_Sample_data/4_sample_info_allbiomes.parquet file_format: parquet - title: Sample data for all studies (txt) component_type: dataframe - file_path: /Users/asaru/Documents/DTU/MoNA/vuegen/example_data/MicW2Graph/1_Exploratory_Data_Analysis/2_Sample_data/5_sample_info_allbiomes.txt + file_path: example_data/MicW2Graph/1_Exploratory_Data_Analysis/2_Sample_data/5_sample_info_allbiomes.txt file_format: txt - title: Extra information components: @@ -66,17 +66,17 @@ sections: - title: Network1 (graphml) component_type: plot plot_type: interactive_network - file_path: /Users/asaru/Documents/DTU/MoNA/vuegen/example_data/MicW2Graph/2_Microbial_Association_Networks/1_Network_visualization1/1_man_example.graphml + file_path: example_data/MicW2Graph/2_Microbial_Association_Networks/1_Network_visualization1/1_man_example.graphml - title: Network Visualization2 components: - title: Network2 (edge list csv) component_type: plot plot_type: interactive_network csv_network_format: edgelist - file_path: /Users/asaru/Documents/DTU/MoNA/vuegen/example_data/MicW2Graph/2_Microbial_Association_Networks/2_Network_visualization2/1_man_example.csv + file_path: example_data/MicW2Graph/2_Microbial_Association_Networks/2_Network_visualization2/1_man_example.csv - title: Edge list (csv) component_type: dataframe - file_path: /Users/asaru/Documents/DTU/MoNA/vuegen/example_data/MicW2Graph/2_Microbial_Association_Networks/2_Network_visualization2/1_man_example.csv + file_path: example_data/MicW2Graph/2_Microbial_Association_Networks/2_Network_visualization2/1_man_example.csv file_format: csv delimiter: ',' caption: This is the edge list of the network @@ -85,13 +85,13 @@ sections: - title: Network3 (cyjs) component_type: plot plot_type: interactive_network - file_path: /Users/asaru/Documents/DTU/MoNA/vuegen/example_data/MicW2Graph/2_Microbial_Association_Networks/3_Network_visualization3/1_man_example.cyjs + file_path: example_data/MicW2Graph/2_Microbial_Association_Networks/3_Network_visualization3/1_man_example.cyjs - title: Network Visualization4 components: - title: Network4 (html) component_type: plot plot_type: interactive_network - file_path: /Users/asaru/Documents/DTU/MoNA/vuegen/example_data/MicW2Graph/2_Microbial_Association_Networks/4_Network_visualization4/1_ckg_network.html + file_path: example_data/MicW2Graph/2_Microbial_Association_Networks/4_Network_visualization4/1_ckg_network.html - title: Network Visualization5 components: - title: Network5 (remote html) diff --git a/pyproject.toml b/pyproject.toml index 9b4e4f2..e732bb4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,3 +34,10 @@ ipykernel = "^6.29.5" [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" + +# https://stackoverflow.com/a/60990574/9684872 +[tool.poetry.extras] +streamlit = ["streamlit"] + +# [project.scripts] +# my-script = "vuegen.main:main" From dee89d8b76440ff29f96807244d3164391cb1fdc Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Tue, 10 Dec 2024 15:53:04 +0100 Subject: [PATCH 4/8] :sparkles: basic workflow to test installation --- .github/workflows/cdci.yml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/cdci.yml diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml new file mode 100644 index 0000000..b72efee --- /dev/null +++ b/.github/workflows/cdci.yml @@ -0,0 +1,33 @@ +name: Python package + +on: + push: + branches: [main] + pull_request: + branches: [main] + release: + types: [published] + +jobs: + test: + name: Test + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' # caching pip dependencies + cache-dependency-path: '**/pyproject.toml' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pytest + pip install -e . + # - name: Run tests + # run: python -m pytest tests + From b1e2e2be978fbb3c3260fb4e7ed89254bc4f8532 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Tue, 10 Dec 2024 16:25:22 +0100 Subject: [PATCH 5/8] :bug: show that quatro is not installed --- vuegen/quarto_reportview.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vuegen/quarto_reportview.py b/vuegen/quarto_reportview.py index 218463b..3cf829a 100644 --- a/vuegen/quarto_reportview.py +++ b/vuegen/quarto_reportview.py @@ -120,6 +120,9 @@ def run_report(self, output_dir: str = BASE_DIR) -> None: except subprocess.CalledProcessError as e: self.report.logger.error(f"Error running '{self.report.title}' {self.report_type} report: {str(e)}") raise + except FileNotFoundError as e: + self.report.logger.error(f"Quarto is not installed. Please install Quarto to run the report: {str(e)}") + raise def _create_yaml_header(self) -> str: """ From f6fab1eb7227e6fb8857b26c015bf813ff416ef6 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Tue, 10 Dec 2024 16:37:55 +0100 Subject: [PATCH 6/8] :white_check_mark: add basic module test --- .github/workflows/cdci.yml | 4 ++-- tests/test_module_imports.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 tests/test_module_imports.py diff --git a/.github/workflows/cdci.yml b/.github/workflows/cdci.yml index b72efee..1a9b233 100644 --- a/.github/workflows/cdci.yml +++ b/.github/workflows/cdci.yml @@ -28,6 +28,6 @@ jobs: python -m pip install --upgrade pip pip install pytest pip install -e . - # - name: Run tests - # run: python -m pytest tests + - name: Run tests + run: python -m pytest tests diff --git a/tests/test_module_imports.py b/tests/test_module_imports.py new file mode 100644 index 0000000..64657e4 --- /dev/null +++ b/tests/test_module_imports.py @@ -0,0 +1,12 @@ + +def test_imports(): + import vuegen + import vuegen.main + import vuegen.quarto_reportview + import vuegen.report + import vuegen.report_generator + import vuegen.streamlit_reportview + import vuegen.utils + + assert vuegen.__version__ + From 6de760d1a54480d1944fa20eecd2d2c50e9cf47e Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Wed, 11 Dec 2024 13:10:38 +0100 Subject: [PATCH 7/8] :memo: add hints on quarto --- README.md | 22 ++++++++++++++++++++++ pyproject.toml | 2 ++ 2 files changed, 24 insertions(+) diff --git a/README.md b/README.md index 51ca827..de1ac85 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,28 @@ pip install -e path/to/vuegen # specify location pip install -e . # in case you pwd is in the vuegen directory ``` +### Quatro installation + +Install quatro with vueguen if you don't have it globally installed. + +```bash +pip install -e ".[quarto]" # specify location +``` + +if you use conda a conda environement you can install quatro from the conda-forge channel +in case it did not work. + +```bash +conda install -c conda-forge quarto +``` + +Test your quarto installation by running the following command: + +```bash +quarto check +``` + + ## Execution ```bash diff --git a/pyproject.toml b/pyproject.toml index e732bb4..484fd2c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ strenum = { version = "^0.4.15", python = "<3.11" } [tool.poetry.group.dev.dependencies] ipykernel = "^6.29.5" + [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" @@ -38,6 +39,7 @@ build-backend = "poetry.core.masonry.api" # https://stackoverflow.com/a/60990574/9684872 [tool.poetry.extras] streamlit = ["streamlit"] +quarto = ["quarto", "ipykernel"] # [project.scripts] # my-script = "vuegen.main:main" From 71e0ab7d74f7b61706fea20ff3eae19c80ae1733 Mon Sep 17 00:00:00 2001 From: Henry Webel Date: Wed, 11 Dec 2024 13:11:23 +0100 Subject: [PATCH 8/8] :bug: set a valid default, not None alternatively the parameter could have no default. --- vuegen/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vuegen/utils.py b/vuegen/utils.py index 27e1de6..6a6423e 100644 --- a/vuegen/utils.py +++ b/vuegen/utils.py @@ -185,7 +185,7 @@ def get_args(prog_name: str, others: dict = {}) -> argparse.Namespace: "-rt", "--report_type", type=str, - default=None, + default='streamlit', # this is not a valid default help="Type of the report to generate (streamlit, html, pdf, docx, odt, revealjs, pptx, or jupyter)." )