32 upgrade as many packages to latest#2 (#46)

DrJStrudwick · web-flow · commit 4e1825a36f24 · 2025-08-01T11:33:12.000+01:00
* Changed: dockerfile to install only main

* Removed: unneeded packages

* Changed: updated non-root requirements

* fix: fixes for updating non-root requirements

* changed: streamlined imports

* added: tests for `variance_removal`

* security: package updates

* Changed: updated pydantic

* changed: updated bio packages

* fix: path printing error

* Changed: loosened limits on package version already at max

* Changed: updated lightgbm

* fixed: changelog version

* change: changelog update
diff --git a/.env b/.env
@@ -0,0 +1 @@
+PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION='python'
diff --git a/.gitignore b/.gitignore
@@ -12,7 +12,7 @@ configs
 experiments
 data
 .vscode/settings.json
-playground.ipynb
+playground*.ipynb
 coverage.xml
 .coverage
 ao-env
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -18,7 +18,29 @@
 
 Change log for the codebase. Initialised from the developments following version `V0.11.3`
 
-## [unreleased] - 2025-05-09
+## [version] - 2025-08-01
+
+### Added
+
+- Added: test case for variance threshold
+- Added: `.env' file to set some required env vars
+
+### Changed
+
+- changed: dockerfile to only install main dependencies
+- changed: streamlined imports
+- changed: updated as many packages as possible
+
+### Fixed
+
+- fix: omic path parsing bug
+- fix: plotting bug arising from api change
+
+### Security
+
+- security: updated packages to resolve dependabot alerts
+
+## [v1.2.0] - 2025-05-09
 
 ### Changed
 
@@ -186,6 +208,7 @@ Change log for the codebase. Initialised from the developments following version
 - Detect secrets added
 - Upgraded python base image from `3.9.14` to `3.9.18` for additional security fixes
 
+[V1.2.0]: https://github.com/IBM/AutoXAI4Omics/releases/tag/V1.2.0
 [V1.1.1]: https://github.com/IBM/AutoXAI4Omics/releases/tag/V1.1.1
 [V1.1.0]: https://github.com/IBM/AutoXAI4Omics/releases/tag/V1.1.0
 [V1.0.1]: https://github.com/IBM/AutoXAI4Omics/releases/tag/V1.0.1
diff --git a/DEV_MANUAL.md b/DEV_MANUAL.md
@@ -30,7 +30,7 @@ Please use blacks & ruff to format any code contributions, we have a pre-commit
 
 To create the virtual enviroment for AutoXAI4Omics using an enviroment manager of your choice, like conda for example, using `python3.9` as your starting point. Then proceed to install the contents of `pyproject.toml` for both the main and dev dependencies. Note that you may also need to set `PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python` within your enviroment.
 
-*Note:* Since May '25 we have switched to using [Poetry](https://python-poetry.org/)  for managing our env and dependencies which we also recommend to use. There is also an associated `poetry.lock` file to ensure consistent versions between developers.
+*Note:* Since May '25 we have switched to using [Poetry](https://python-poetry.org/)  for managing our env and dependencies which we also recommend to use. There is also an associated `poetry.lock` file to ensure consistent versions between developers. If you also the poetry shell extension it will automatically load the `.env` file setting the env var mentioned above.
 
 ## Testing
 
diff --git a/Dockerfile b/Dockerfile
@@ -35,7 +35,7 @@ COPY --chown=omicsuser:0 poetry.lock pyproject.toml ./
 COPY --chown=omicsuser:0 autoxai4omics .
 
 RUN poetry env use system
-RUN poetry install --no-root
+RUN poetry install --no-root --only main
 USER omicsuser 
 
 CMD ["$@"]
diff --git a/autoxai4omics/models/tabauto/keras_model.py b/autoxai4omics/models/tabauto/keras_model.py
@@ -19,7 +19,7 @@
 from tensorflow.keras.layers import Dense, Flatten, Conv1D
 from tensorflow.keras import losses
 
-import tensorflow.keras.optimizers.legacy
+from tensorflow.keras.optimizers.legacy import Adam
 
 from tensorflow.keras.callbacks import (
     LearningRateScheduler,
@@ -95,7 +95,7 @@ def __init_ak__(self, input_dim, output_dim, dataset_type):
                 objective=["val_loss"],
                 tuner=self.tuner,
                 seed=self.random_state,
-                optimizer=tensorflow.keras.optimizers.legacy.Adam(),
+                optimizer=Adam(),
             )
 
         else:
@@ -117,7 +117,7 @@ def __init_ak__(self, input_dim, output_dim, dataset_type):
                 objective=["accuracy"],
                 tuner=self.tuner,
                 seed=self.random_state,
-                optimizer=tensorflow.keras.optimizers.legacy.Adam(),
+                optimizer=Adam(),
             )
 
         self.model = model
@@ -242,7 +242,7 @@ def __init_fx__(self, input_dim, output_dim, dataset_type):
                     )
 
         # choose optimizer
-        opt = tensorflow.keras.optimizers.legacy.Adam()
+        opt = Adam()
 
         # choose loss function
         if self.dataset_type == REGRESSION:
diff --git a/autoxai4omics/omics/geneExp.py b/autoxai4omics/omics/geneExp.py
@@ -16,6 +16,7 @@
 from omics import R_replacement as rrep
 import joblib
 import pandas as pd
+from pathlib import Path
 
 
 def get_data_gene_expression(
@@ -43,15 +44,19 @@ def get_data_gene_expression(
 
     # add the output file name from config_dict that is required
     if config_dict["gene_expression"]["output_file_ge"] is not None:
-        output_file = config_dict["gene_expression"]["output_file_ge"]
+        output_file = str(
+            config_dict["gene_expression"]["output_file_ge"].with_suffix("")
+        )
         print("Output file: " + output_file)
     else:
         output_file = "processed_gene_expression_data"
     output_file += "_holdout" if holdout else ""
 
     # add metadata output file from config_dict that is required
     if config_dict["gene_expression"]["output_metadata"] is not None:
-        metout_file = config_dict["gene_expression"]["output_metadata"]
+        metout_file = str(
+            config_dict["gene_expression"]["output_metadata"].with_suffix("")
+        )
     else:
         metout_file = "processed_gene_expression_metadata"
     metout_file += "_holdout" if holdout else ""
diff --git a/autoxai4omics/omics/metabolomic.py b/autoxai4omics/omics/metabolomic.py
@@ -16,6 +16,7 @@
 from omics import R_replacement as rrep
 import joblib
 import pandas as pd
+from pathlib import Path
 
 
 def get_data_metabolomic(
@@ -43,15 +44,15 @@ def get_data_metabolomic(
 
     # add the output file name from config_dict that is required
     if config_dict["metabolomic"]["output_file_met"] is not None:
-        output_file = config_dict["metabolomic"]["output_file_met"]
+        output_file = str(config_dict["metabolomic"]["output_file_met"].with_suffix(""))
         print("Output file: " + output_file)
     else:
         output_file = "processed_metabolomic_data"
     output_file += "_holdout" if holdout else ""
 
     # add metadata output file from config_dict that is required
     if config_dict["metabolomic"]["output_metadata"] is not None:
-        metout_file = config_dict["metabolomic"]["output_metadata"]
+        metout_file = str(config_dict["metabolomic"]["output_metadata"].with_suffix(""))
     else:
         metout_file = "processed_metabolomic_metadata"
     metout_file += "_holdout" if holdout else ""
diff --git a/autoxai4omics/omics/tabular.py b/autoxai4omics/omics/tabular.py
@@ -16,6 +16,7 @@
 from omics import R_replacement as rrep
 import joblib
 import pandas as pd
+from pathlib import Path
 
 
 def get_data_tabular(
@@ -43,7 +44,7 @@ def get_data_tabular(
 
     # add the output file name from config_dict that is required
     if config_dict["tabular"]["output_file_tab"] is not None:
-        output_file = config_dict["tabular"]["output_file_tab"]
+        output_file = str(config_dict["tabular"]["output_file_tab"].with_suffix(""))
         print("Output file: " + output_file)
     else:
         output_file = "processed_tabular_data"
@@ -52,7 +53,7 @@ def get_data_tabular(
 
     # add metadata output file from config_dict that is required
     if config_dict["tabular"]["output_metadata"] is not None:
-        metout_file = config_dict["tabular"]["output_metadata"]
+        metout_file = str(config_dict["tabular"]["output_metadata"].with_suffix(""))
     else:
         metout_file = "processed_tabular_metadata"
 
diff --git a/autoxai4omics/plotting/importance/perm_imp.py b/autoxai4omics/plotting/importance/perm_imp.py
@@ -125,7 +125,10 @@ def permut_importance(
             )
 
         # Make a horizontal boxplot ordered by the magnitude
-        ax = sns.boxplot(x=top_values, y=top_features, orient="h", ax=ax)
+
+        ax = sns.boxplot(
+            pd.DataFrame(top_values, index=top_features).T, orient="h", ax=ax
+        )
         if problem_type == CLASSIFICATION:
             ax.set_xlabel(f"{pretty_names(fit_scorer, 'score')} Decrease")
         else:
diff --git a/autoxai4omics/plotting/plots_both.py b/autoxai4omics/plotting/plots_both.py
@@ -258,7 +258,7 @@ def boxplot_scorer_cv_groupby(
     ]
 
     # Make the boxplot
-    sns.boxplot(x=pretty_model_names, y=all_scores, ax=ax, width=0.4)
+    sns.boxplot(pd.DataFrame(all_scores, index=pretty_model_names).T, ax=ax, width=0.4)
     # Format the graph
     ax.set_xlabel("ML Methods")
 
@@ -362,7 +362,7 @@ def boxplot_scorer_cv(
     pretty_model_names = [pretty_names(name, "model") for name in model_list]
 
     # Make the boxplot
-    sns.boxplot(x=pretty_model_names, y=all_scores, ax=ax, width=0.4)
+    sns.boxplot(pd.DataFrame(all_scores, index=pretty_model_names).T, ax=ax, width=0.4)
     # Format the graph
     ax.set_xlabel("ML Methods")
     plt.xticks(rotation=90)
diff --git a/autoxai4omics/utils/ml/feature_selection.py b/autoxai4omics/utils/ml/feature_selection.py
@@ -22,13 +22,16 @@
 import logging
 import math
 import numpy as np
+from numpy.typing import ArrayLike
 import pandas as pd
 from plotting.plots_both import feat_acc_plot, opt_k_plot
 
 omicLogger = logging.getLogger("OmicLogger")
 
 
-def variance_removal(x, threshold=0):
+def variance_removal(
+    x: ArrayLike, threshold: float = 0
+) -> tuple[ArrayLike, VarianceThreshold]:
     omicLogger.debug(
         f"Applying varience threshold, using {threshold} as the threshold..."
     )
diff --git a/autoxai4omics/utils/ml/tests/test_feature_selection.py b/autoxai4omics/utils/ml/tests/test_feature_selection.py
@@ -0,0 +1,35 @@
+# Copyright (c) 2025 IBM Corp.
+#
+# This software is released under the MIT License.
+# https://opensource.org/licenses/MIT
+
+import pytest
+import numpy as np
+from .. import feature_selection as fs
+from sklearn.feature_selection import VarianceThreshold
+
+
+np.random.seed(1234)
+
+VAR = 0.5
+SAMPLES = 30
+FEATS = 10
+FIXED = np.random.normal(size=(SAMPLES, FEATS))
+SMALLER = np.random.normal(scale=(VAR**0.5), size=(SAMPLES, 1))
+ZERO = np.random.normal(scale=0, size=(SAMPLES, 1))
+
+X = np.concatenate((FIXED, SMALLER, ZERO), axis=1)
+
+
+class Test_variance_removal:
+    def test_zero_var_removal(self):
+        x_trans, selector = fs.variance_removal(X)
+
+        assert x_trans.shape == (SAMPLES, FEATS + 1)
+        assert isinstance(selector, VarianceThreshold)
+
+    def test_given_var_removal(self, given_var: float = VAR):
+        x_trans, selector = fs.variance_removal(X, threshold=given_var)
+
+        assert x_trans.shape == (SAMPLES, FEATS)
+        assert isinstance(selector, VarianceThreshold)
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION='python'`