Merge pull request #132 from daisybio/development

New version

Author: JudithBernett

.gitignore

@@ -6,6 +6,8 @@ data/GDSC1
 data/GDSC2
 data/CCLE
 data/Toy_Data
+data/CTRPv1
+data/CTRPv2
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

create_report.py

@@ -267,8 +267,10 @@ def draw_per_grouping_algorithm_plots(
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Generate reports from evaluation results")
     parser.add_argument("--run_id", required=True, help="Run ID for the current execution")
+    parser.add_argument("--dataset", required=True, help="Dataset name for which to render the result file")
     args = parser.parse_args()
     run_id = args.run_id
+    dataset = args.dataset
 
     # assert that the run_id folder exists
     if not os.path.exists(f"results/{run_id}"):
@@ -280,7 +282,7 @@ def draw_per_grouping_algorithm_plots(
         evaluation_results_per_drug,
         evaluation_results_per_cell_line,
         true_vs_pred,
-    ) = parse_results(path_to_results=f"results/{run_id}")
+    ) = parse_results(path_to_results=f"results/{run_id}", dataset=dataset)
 
     # part of pipeline: EVALUATE_FINAL, COLLECT_RESULTS
     (

docs/requirements.txt

@@ -1,4 +1,4 @@
 sphinx-autobuild==2024.10.3 ; python_version >= "3.11" and python_version < "3.13"
-sphinx-autodoc-typehints==3.0.1 ; python_version >= "3.11" and python_version < "3.13"
+sphinx-autodoc-typehints==3.1.0 ; python_version >= "3.11" and python_version < "3.13"
 sphinx-click==6.0.0 ; python_version >= "3.11" and python_version < "3.13"
 sphinx-rtd-theme==3.0.2 ; python_version >= "3.11" and python_version < "3.13"

drevalpy/datasets/dataset.py

@@ -55,7 +55,7 @@ def from_csv(
         - response:         the drug response values as floating point values
         - cell_line_ids:    a string identifier for cell lines
         - drug_ids:         a string identifier for drugs
-        - predictions:      an optional column containing a predicted value TODO what exactly?
+        - predictions:      an optional column containing drug response predictions
 
         :param input_file: Path to the csv file containing the data to be loaded
         :param dataset_name: Optional name to associate the dataset with, default = "unknown"
@@ -64,6 +64,8 @@ def from_csv(
         :returns: DrugResponseDataset object containing data from provided csv file.
         """
         data = pd.read_csv(input_file)
+        data["drug_id"] = data["drug_id"].astype(str)
+
         if "predictions" in data.columns:
             predictions = data["predictions"].values
         else:
@@ -152,9 +154,9 @@ def __init__(
         """
         super().__init__()
         if len(response) != len(cell_line_ids):
-            raise AssertionError("Response and cell_line_ids have different lengths.")
+            raise AssertionError("Response and cell line identifiers have different lengths.")
         if len(response) != len(drug_ids):
-            raise AssertionError("Response and drug_ids have different lengths.")
+            raise AssertionError("Response and drug identifiers have different lengths.")
         if predictions is not None and len(response) != len(predictions):
             raise AssertionError("Response and predictions have different lengths.")
         self._response = response

drevalpy/datasets/loader.py

@@ -9,13 +9,13 @@
 from ..pipeline_function import pipeline_function
 from .curvecurator import fit_curves
 from .dataset import DrugResponseDataset
-from .utils import download_dataset
+from .utils import CELL_LINE_IDENTIFIER, DRUG_IDENTIFIER, download_dataset
 
 
 def load_gdsc1(
     path_data: str = "data",
-    measure: str = "LN_IC50",
-    file_name: str = "response_GDSC1.csv",
+    measure: str = "LN_IC50_curvecurator",
+    file_name: str = "GDSC1.csv",
     dataset_name: str = "GDSC1",
 ) -> DrugResponseDataset:
     """
@@ -32,18 +32,18 @@ def load_gdsc1(
     if not os.path.exists(path):
         download_dataset(dataset_name, path_data, redownload=True)
 
-    response_data = pd.read_csv(path)
-    response_data["DRUG_NAME"] = response_data["DRUG_NAME"].str.replace(",", "")
+    response_data = pd.read_csv(path, dtype={"pubchem_id": str})
+    response_data[DRUG_IDENTIFIER] = response_data[DRUG_IDENTIFIER].str.replace(",", "")
 
     return DrugResponseDataset(
         response=response_data[measure].values,
-        cell_line_ids=response_data["CELL_LINE_NAME"].values,
-        drug_ids=response_data["DRUG_NAME"].values,
+        cell_line_ids=response_data[CELL_LINE_IDENTIFIER].values,
+        drug_ids=response_data[DRUG_IDENTIFIER].values,
         dataset_name=dataset_name,
     )
 
 
-def load_gdsc2(path_data: str = "data", measure: str = "LN_IC50", file_name: str = "response_GDSC2.csv"):
+def load_gdsc2(path_data: str = "data", measure: str = "LN_IC50_curvecurator", file_name: str = "GDSC2.csv"):
     """
     Loads the GDSC2 dataset.
 
@@ -57,7 +57,7 @@ def load_gdsc2(path_data: str = "data", measure: str = "LN_IC50", file_name: str
 
 
 def load_ccle(
-    path_data: str = "data", measure: str = "LN_IC50", file_name: str = "response_CCLE.csv"
+    path_data: str = "data", measure: str = "LN_IC50_curvecurator", file_name: str = "CCLE.csv"
 ) -> DrugResponseDataset:
     """
     Loads the CCLE dataset.
@@ -73,18 +73,18 @@ def load_ccle(
     if not os.path.exists(path):
         download_dataset(dataset_name, path_data, redownload=True)
 
-    response_data = pd.read_csv(path)
-    response_data["DRUG_NAME"] = response_data["DRUG_NAME"].str.replace(",", "")
+    response_data = pd.read_csv(path, dtype={"pubchem_id": str})
+    response_data[DRUG_IDENTIFIER] = response_data[DRUG_IDENTIFIER].str.replace(",", "")
 
     return DrugResponseDataset(
         response=response_data[measure].values,
-        cell_line_ids=response_data["CELL_LINE_NAME"].values,
-        drug_ids=response_data["DRUG_NAME"].values,
+        cell_line_ids=response_data[CELL_LINE_IDENTIFIER].values,
+        drug_ids=response_data[DRUG_IDENTIFIER].values,
         dataset_name=dataset_name,
     )
 
 
-def load_toy(path_data: str = "data", measure: str = "response") -> DrugResponseDataset:
+def load_toy(path_data: str = "data", measure: str = "LN_IC50_curvecurator") -> DrugResponseDataset:
     """
     Loads small Toy dataset, subsampled from GDSC1.
 
@@ -94,20 +94,67 @@ def load_toy(path_data: str = "data", measure: str = "response") -> DrugResponse
     :return: DrugResponseDataset containing response, cell line IDs, and drug IDs.
     """
     dataset_name = "Toy_Data"
-    measure = "response"  # overwrite this explicitly to avoid problems, should be changed in the future
     path = os.path.join(path_data, dataset_name, "toy_data.csv")
     if not os.path.exists(path):
         download_dataset(dataset_name, path_data, redownload=True)
-    response_data = pd.read_csv(path)
+    response_data = pd.read_csv(path, dtype={"pubchem_id": str})
 
     return DrugResponseDataset(
         response=response_data[measure].values,
-        cell_line_ids=response_data["cell_line_id"].values,
-        drug_ids=response_data["drug_id"].values,
+        cell_line_ids=response_data[CELL_LINE_IDENTIFIER].values,
+        drug_ids=response_data[DRUG_IDENTIFIER].values,
         dataset_name=dataset_name,
     )
 
 
+def _load_ctrpv(version: str, path_data: str = "data", measure: str = "LN_IC50_curvecurator") -> DrugResponseDataset:
+    """
+    Load CTRPv1 dataset.
+
+    :param version: The version of the CTRP dataset to load.
+    :param path_data: Path to location of CTRPv1 dataset
+    :param measure: The name of the column containing the measure to predict, default = "response"
+
+    :return: DrugResponseDataset containing response, cell line IDs, and drug IDs
+    """
+    dataset_name = "CTRPv" + version
+    path = os.path.join(path_data, dataset_name, f"{dataset_name}.csv")
+    if not os.path.exists(path):
+        download_dataset(dataset_name, path_data, redownload=True)
+    response_data = pd.read_csv(path, dtype={"pubchem_id": str})
+
+    return DrugResponseDataset(
+        response=response_data[measure].values,
+        cell_line_ids=response_data[CELL_LINE_IDENTIFIER].values,
+        drug_ids=response_data[DRUG_IDENTIFIER].values,
+        dataset_name=dataset_name,
+    )
+
+
+def load_ctrpv1(path_data: str = "data", measure: str = "LN_IC50_curvecurator") -> DrugResponseDataset:
+    """
+    Load CTRPv2 dataset.
+
+    :param path_data: Path to location of CTRPv2 dataset
+    :param measure: The name of the column containing the measure to predict, default = "LN_IC50_curvecurator"
+
+    :return: DrugResponseDataset containing response, cell line IDs, and drug IDs
+    """
+    return _load_ctrpv("1", path_data, measure)
+
+
+def load_ctrpv2(path_data: str = "data", measure: str = "LN_IC50_curvecurator") -> DrugResponseDataset:
+    """
+    Load CTRPv2 dataset.
+
+    :param path_data: Path to location of CTRPv2 dataset
+    :param measure: The name of the column containing the measure to predict, default: LN_IC50_curvecurator
+
+    :return: DrugResponseDataset containing response, cell line IDs, and drug IDs
+    """
+    return _load_ctrpv("2", path_data, measure)
+
+
 def load_custom(path_data: str | Path, measure: str = "response") -> DrugResponseDataset:
     """
     Load custom dataset.
@@ -125,6 +172,8 @@ def load_custom(path_data: str | Path, measure: str = "response") -> DrugRespons
     "GDSC2": load_gdsc2,
     "CCLE": load_ccle,
     "Toy_Data": load_toy,
+    "CTRPv1": load_ctrpv1,
+    "CTRPv2": load_ctrpv2,
 }
 
 

drevalpy/datasets/utils.py

@@ -9,6 +9,9 @@
 import numpy as np
 import requests
 
+DRUG_IDENTIFIER = "pubchem_id"
+CELL_LINE_IDENTIFIER = "cell_line_name"
+
 
 def download_dataset(
     dataset_name: str,
@@ -26,18 +29,18 @@ def download_dataset(
     file_name = f"{dataset_name}.zip"
     file_path = Path(data_path) / file_name
     extracted_folder_path = file_path.with_suffix("")
-
+    timeout = 120
     # Check if the extracted data exists and skip download if not redownloading
     if extracted_folder_path.exists() and not redownload:
         print(f"{dataset_name} is already extracted, skipping download.")
     else:
         url = "https://zenodo.org/doi/10.5281/zenodo.12633909"
         # Fetch the latest record
-        response = requests.get(url, timeout=60)
+        response = requests.get(url, timeout=timeout)
         if response.status_code != 200:
             raise requests.exceptions.HTTPError(f"Error fetching record: {response.status_code}")
         latest_url = response.links["linkset"]["url"]
-        response = requests.get(latest_url, timeout=60)
+        response = requests.get(latest_url, timeout=timeout)
         if response.status_code != 200:
             raise requests.exceptions.HTTPError(f"Error fetching record: {response.status_code}")
         data = response.json()
@@ -50,7 +53,7 @@ def download_dataset(
         file_url = name_to_url[file_name]
         # Download the file
         print(f"Downloading {dataset_name} from {file_url}...")
-        response = requests.get(file_url, timeout=60)
+        response = requests.get(file_url, timeout=timeout)
         if response.status_code != 200:
             raise requests.exceptions.HTTPError(f"Error downloading file {dataset_name}: " f"{response.status_code}")
 
@@ -61,7 +64,7 @@ def download_dataset(
         with zipfile.ZipFile(file_path, "r") as z:
             for member in z.infolist():
                 if not member.filename.startswith("__MACOSX/"):
-                    z.extract(member, os.path.join(data_path, dataset_name))
+                    z.extract(member, os.path.join(data_path))
         file_path.unlink()  # Remove zip file after extraction
 
         print(f"{dataset_name} data downloaded and extracted to {data_path}")

drevalpy/models/SimpleNeuralNetwork/hyperparameters.yaml

@@ -22,6 +22,8 @@ SimpleNeuralNetwork:
       - 128
       - 64
       - 16
+  max_epochs:
+    - 100
 
 MultiOmicsNeuralNetwork:
   dropout_prob:
@@ -44,3 +46,5 @@ MultiOmicsNeuralNetwork:
       - 32
   methylation_pca_components:
     - 100
+  max_epochs:
+    - 100

drevalpy/models/SimpleNeuralNetwork/multiomics_neural_network.py

@@ -110,6 +110,10 @@ def train(
                 cell_line_views=self.cell_line_views,
                 drug_views=self.drug_views,
                 output_earlystopping=output_earlystopping,
+                trainer_params={
+                    "max_epochs": self.hyperparameters.get("max_epochs", 100),
+                    "progress_bar_refresh_rate": 500,
+                },
                 batch_size=16,
                 patience=5,
                 num_workers=1,

drevalpy/models/SimpleNeuralNetwork/simple_neural_network.py

@@ -105,6 +105,10 @@ def train(
                 cell_line_views=self.cell_line_views,
                 drug_views=self.drug_views,
                 output_earlystopping=output_earlystopping,
+                trainer_params={
+                    "max_epochs": self.hyperparameters.get("max_epochs", 100),
+                    "progress_bar_refresh_rate": 500,
+                },
                 batch_size=16,
                 patience=5,
                 num_workers=1 if platform.system() == "Windows" else 8,

drevalpy/models/SimpleNeuralNetwork/utils.py

@@ -180,16 +180,15 @@ def fit(
         :param model_checkpoint_dir: directory to save the model checkpoints
         :raises ValueError: if drug_input is missing
         """
-        if drug_input is None:
-            raise ValueError(
-                "Drug input (fingerprints) are required for SimpleNeuralNetwork and " "MultiOMICsNeuralNetwork."
-            )
-
         if trainer_params is None:
             trainer_params = {
+                "max_epochs": 100,
                 "progress_bar_refresh_rate": 500,
-                "max_epochs": 70,
             }
+        if drug_input is None:
+            raise ValueError(
+                "Drug input (fingerprints) are required for SimpleNeuralNetwork and " "MultiOMICsNeuralNetwork."
+            )
 
         train_dataset = RegressionDataset(
             output=output_train,