diff --git a/README.md b/README.md index 4f9a3a6..7edb1d8 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ By default, it's under `GreenAlgorithms4HPC/outputs/`, but this can be changed. ## Full list of options ``` -usage: __init__.py [-h] [-S STARTDAY] [-E ENDDAY] [-o OUTPUT] [--outputDir OUTPUTDIR] [--filterCWD] [--filterJobIDs FILTERJOBIDS] [--filterAccount FILTERACCOUNT] [--customSuccessStates CUSTOMSUCCESSSTATES] +usage: myCarbonFootprint [-h] [-S STARTDAY] [-E ENDDAY] [-o OUTPUT] [--outputDir OUTPUTDIR] [--filterCWD] [--filterJobIDs FILTERJOBIDS] [--filterAccount FILTERACCOUNT] [--customSuccessStates CUSTOMSUCCESSSTATES] [--reportBug | --reportBugHere] [--useCustomLogs USECUSTOMLOGS] Calculate your carbon footprint on the server. @@ -105,12 +105,12 @@ optional arguments: 2. Edit `myCarbonFootprint.sh` line 20 to create the virtual environment with Python 3.8 or later. The default line is: ```bash - /usr/bin/python3.8 -m venv GA_env + /usr/bin/python3.8 -m venv .venv ``` But it may be something else on your server, for example: ```bash module load python/3.11.7 - python -m venv GA_env + python -m venv .venv ``` 3. Make the bash script executable: diff --git a/example_files/Screenshot HPC 2024-08-20.png b/examples/Screenshot HPC 2024-08-20.png similarity index 100% rename from example_files/Screenshot HPC 2024-08-20.png rename to examples/Screenshot HPC 2024-08-20.png diff --git a/example_files/example_output_workloadManager.tsv b/examples/example_output_workloadManager.tsv similarity index 100% rename from example_files/example_output_workloadManager.tsv rename to examples/example_output_workloadManager.tsv diff --git a/example_files/example_sacctOutput_raw.txt b/examples/example_sacctOutput_raw.txt similarity index 100% rename from example_files/example_sacctOutput_raw.txt rename to examples/example_sacctOutput_raw.txt diff --git a/example_files/example_sacctOutput_raw_asDF.tsv b/examples/example_sacctOutput_raw_asDF.tsv similarity index 100% rename from example_files/example_sacctOutput_raw_asDF.tsv rename to examples/example_sacctOutput_raw_asDF.tsv diff --git a/__init__.py b/greenalgorithms4HPC/__init__.py similarity index 92% rename from __init__.py rename to greenalgorithms4HPC/__init__.py index ff4e03d..1ca3c9b 100644 --- a/__init__.py +++ b/greenalgorithms4HPC/__init__.py @@ -3,9 +3,10 @@ import datetime import os -from backend import main_backend -from frontend import main_frontend +from greenalgorithms4HPC.backend import main_backend +from greenalgorithms4HPC.frontend import main_frontend +SYSTEM_CONF="/etc/GreenAlgorithms4HPC.conf" def create_arguments(): """ Command line arguments for the tool. @@ -108,17 +109,21 @@ def all(self, args): self._validate_dates(args) self._validate_output(args) -if __name__ == "__main__": +def main(): # print("Working dir0: ", os.getcwd()) # DEBUGONLY - args = create_arguments() ## Decide which infrastructure info to use - if args.useOtherInfrastuctureInfo != '': - args.path_infrastucture_info = args.useOtherInfrastuctureInfo - print(f"Overriding infrastructure info with: {args.path_infrastucture_info}") - else: - args.path_infrastucture_info = 'data' + # Load cluster specific info + cluster_config = os.environ.get("GA4HPC_CONFDIR", None) + + if not cluster_config: + if os.path.exists(SYSTEM_CONF): + cluster_config = SYSTEM_CONF + else: + cluster_config = os.path.join(os.path.dirname(__file__), 'data') + + args.path_infrastucture_info = cluster_config ## Organise the unique output directory (used for output report and logs export for debugging) ## creating a uniquely named subdirectory in whatever @@ -149,5 +154,5 @@ def all(self, args): ### Run backend to get data extracted_data = main_backend(args) + main_frontend(extracted_data, args) - main_frontend(extracted_data, args) \ No newline at end of file diff --git a/backend/__init__.py b/greenalgorithms4HPC/backend/__init__.py similarity index 97% rename from backend/__init__.py rename to greenalgorithms4HPC/backend/__init__.py index feb6dcd..c03da31 100644 --- a/backend/__init__.py +++ b/greenalgorithms4HPC/backend/__init__.py @@ -4,8 +4,8 @@ import pandas as pd import numpy as np -from backend.helpers import check_empty_results, simulate_mock_jobs -from backend.slurm_extract import WorkloadManager +from greenalgorithms4HPC.backend.helpers import check_empty_results, simulate_mock_jobs +from greenalgorithms4HPC.backend.slurm_extract import WorkloadManager # print("Working dir1: ", os.getcwd()) # DEBUGONLY @@ -220,7 +220,7 @@ def main_backend(args): print(exc) ### Load fixed parameters - with open("data/fixed_parameters.yaml", "r") as stream: + with open(os.path.join(args.path_infrastucture_info,"fixed_parameters.yaml"), "r") as stream: try: fParams = yaml.safe_load(stream) except yaml.YAMLError as exc: diff --git a/backend/helpers.py b/greenalgorithms4HPC/backend/helpers.py similarity index 100% rename from backend/helpers.py rename to greenalgorithms4HPC/backend/helpers.py diff --git a/backend/slurm_extract.py b/greenalgorithms4HPC/backend/slurm_extract.py similarity index 100% rename from backend/slurm_extract.py rename to greenalgorithms4HPC/backend/slurm_extract.py diff --git a/data/cluster_info.yaml b/greenalgorithms4HPC/data/cluster_info.yaml similarity index 100% rename from data/cluster_info.yaml rename to greenalgorithms4HPC/data/cluster_info.yaml diff --git a/data/fixed_parameters.yaml b/greenalgorithms4HPC/data/fixed_parameters.yaml similarity index 100% rename from data/fixed_parameters.yaml rename to greenalgorithms4HPC/data/fixed_parameters.yaml diff --git a/frontend/__init__.py b/greenalgorithms4HPC/frontend/__init__.py similarity index 93% rename from frontend/__init__.py rename to greenalgorithms4HPC/frontend/__init__.py index ab81868..72ecfe3 100644 --- a/frontend/__init__.py +++ b/greenalgorithms4HPC/frontend/__init__.py @@ -2,8 +2,8 @@ import yaml import os -from frontend.terminal_output import generate_terminal_view -from frontend.dashboard_output import dashboard_html +from greenalgorithms4HPC.frontend.terminal_output import generate_terminal_view +from greenalgorithms4HPC.frontend.dashboard_output import dashboard_html def main_frontend(dict_stats, args): ### Load cluster specific info diff --git a/frontend/dashboard_output.py b/greenalgorithms4HPC/frontend/dashboard_output.py similarity index 98% rename from frontend/dashboard_output.py rename to greenalgorithms4HPC/frontend/dashboard_output.py index 0e2f966..f7984d2 100644 --- a/frontend/dashboard_output.py +++ b/greenalgorithms4HPC/frontend/dashboard_output.py @@ -8,7 +8,7 @@ import numpy as np import plotly.express as px -from frontend.helpers import formatText_footprint, formatText_treemonths, formatText_flying +from greenalgorithms4HPC.frontend.helpers import formatText_footprint, formatText_treemonths, formatText_flying # class SilentUndefined(Undefined): # DEBUGONLY # def _fail_with_undefined_error(self, *args, **kwargs): diff --git a/frontend/helpers.py b/greenalgorithms4HPC/frontend/helpers.py similarity index 100% rename from frontend/helpers.py rename to greenalgorithms4HPC/frontend/helpers.py diff --git a/frontend/templates/_user.html b/greenalgorithms4HPC/frontend/templates/_user.html similarity index 100% rename from frontend/templates/_user.html rename to greenalgorithms4HPC/frontend/templates/_user.html diff --git a/frontend/templates/report_blank.html b/greenalgorithms4HPC/frontend/templates/report_blank.html similarity index 100% rename from frontend/templates/report_blank.html rename to greenalgorithms4HPC/frontend/templates/report_blank.html diff --git a/frontend/templates/styles.css b/greenalgorithms4HPC/frontend/templates/styles.css similarity index 100% rename from frontend/templates/styles.css rename to greenalgorithms4HPC/frontend/templates/styles.css diff --git a/frontend/terminal_output.py b/greenalgorithms4HPC/frontend/terminal_output.py similarity index 98% rename from frontend/terminal_output.py rename to greenalgorithms4HPC/frontend/terminal_output.py index e798673..92c5f22 100644 --- a/frontend/terminal_output.py +++ b/greenalgorithms4HPC/frontend/terminal_output.py @@ -1,6 +1,6 @@ import math -from frontend.helpers import formatText_footprint, formatText_treemonths, formatText_flying +from greenalgorithms4HPC.frontend.helpers import formatText_footprint, formatText_treemonths, formatText_flying import pandas as pd import os diff --git a/myCarbonFootprint.sh b/myCarbonFootprint.sh index bc2e45f..37c461d 100755 --- a/myCarbonFootprint.sh +++ b/myCarbonFootprint.sh @@ -11,16 +11,16 @@ userCWD="$(pwd)" # Cd into the directory where the GA files are located parent_path=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P ) cd "$parent_path" - +export GA4HPC_CONFDIR=$parent_path/greenalgorithms4HPC/data # Test if the virtualenv GA_env already exists, and if not, creates it. Download python 3.8 or higher for better results. -if [ ! -f GA_env/bin/activate ]; then +if [ ! -f .venv/bin/activate ]; then echo "Need to create virtualenv" - /usr/bin/python3.8 -m venv GA_env # this line needs updating to load python on your server - source GA_env/bin/activate - pip3 install -r requirements.txt + /usr/bin/python3.8 -m venv .venv # this line needs updating to load python on your server + source .venv/bin/activate + pip3 install . else echo "Virtualenv: OK" - source GA_env/bin/activate + source .venv/bin/activate fi # Test if the python version is at least 3.8 @@ -40,4 +40,4 @@ echo "Python versions: OK" # Run the python code and pass on the arguments #userCWD="/home/ll582/ with space" # DEBUGONLY -python __init__.py "$@" --userCWD "$userCWD" +myCarbonFootprint "$@" --userCWD "$userCWD" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..2284f3e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,24 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "greenalgorithms4HPC" +version = "0.4" +dependencies = [ + "numpy","pandas>=2.0", "PyYAML", "jinja2", "plotly" +] +requires-python = ">=3.6" +authors = [ + {name = " Loïc Lannelongue", email = "loic.lannelongue@posteo.net"}, +] +maintainers = [ + {name = " Loïc Lannelongue", email = "loic.lannelongue@posteo.net"}, +] +description = "Green Algorithms for High Performance Computing" + +[project.scripts] +myCarbonFootprint = "greenalgorithms4HPC:main" + +[tool.setuptools.package-data] +"*" = ["*.yaml"] \ No newline at end of file diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 3a9207f..0000000 --- a/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -numpy==1.24 -pandas==2.0 -PyYAML==6.0 -jinja2==3.1 -plotly==5.18 \ No newline at end of file