scripts to evaluate models

Author: Alexander Hillsley

src/ops_model/eval/eval_classification.py

@@ -0,0 +1,99 @@
+import yaml
+
+import pandas as pd
+import torch
+
+from ops_model.data import data_loader
+
+NONFEATURE_COLUMNS = [
+    "label_str",
+    "label_int",
+    "sgRNA",
+    "well",
+    "experiment",
+    "x_position",
+    "y_position",
+]
+
+
+def eval_classification_accuracy(
+    scores_df: pd.DataFrame, labels_df: pd.DataFrame, label_column: str = None
+) -> dict:
+    """
+    Evaluate classification accuracy given prediction scores and true labels.
+
+    Args:
+        scores_df: DataFrame with classification scores, shape (n_samples, n_classes).
+        labels_df: DataFrame containing the true integer labels.
+        label_column: Name of the column in labels_df containing the labels.
+
+    Returns:
+        dict: Dictionary containing accuracy metrics.
+    """
+    # Convert to tensors
+    scores = torch.from_numpy(scores_df.values)
+
+    # Extract true labels
+    if label_column is not None:
+        labels = torch.from_numpy(labels_df[label_column].values)
+    elif len(labels_df.columns) == 1:
+        labels = torch.from_numpy(labels_df.iloc[:, 0].values)
+    else:
+        labels = torch.from_numpy(labels_df.values.flatten())
+
+    # Get predictions and calculate accuracy
+    predictions = torch.argmax(scores, dim=1)
+    accuracy = (predictions == labels).float().mean().item()
+
+    return {
+        "accuracy": accuracy,
+        "correct": (predictions == labels).sum().item(),
+        "total": len(labels),
+        "predictions": predictions.numpy(),
+    }
+
+    return
+
+
+def cnn_inference(config_path: str):
+    with open(config_path, "r") as f:
+        config = yaml.safe_load(f)
+
+    experiment_dict = {
+        "ops0031_20250424": ["A/1/0", "A/2/0", "A/3/0"],
+        # "ops0053_20250709": ["A/1/0", "A/2/0", "A/3/0"],
+        # "ops0079_20250916": ["A/1/0", "A/2/0", "A/3/0"],
+        # "ops0064_20250811": ["A/1/0", "A/2/0", "A/3/0"],
+        # "ops0065_20250812": ["A/1/0", "A/2/0", "A/3/0"],
+    }
+    run_name = config["run_name"]
+
+    dm = data_loader.OpsDataManager(
+        experiments=experiment_dict,
+        batch_size=config["data_manager"]["batch_size"],
+        data_split=tuple(config["data_manager"]["data_split"]),
+        out_channels=config["data_manager"]["out_channels"],
+        initial_yx_patch_size=tuple(config["data_manager"]["initial_yx_patch_size"]),
+        final_yx_patch_size=tuple(config["data_manager"]["final_yx_patch_size"]),
+        verbose=False,
+    )
+
+    # Construct dataloaders first (without sampler) to get the train indices
+    dm.construct_dataloaders(
+        num_workers=config["data_manager"]["num_workers"],
+        dataset_type=config["dataset_type"],
+        basic_kwargs=config["data_manager"].get("basic_kwargs"),
+        balanced_sampling=config["data_manager"].get("balanced_sampling", False),
+    )
+    return
+
+
+if __name__ == "__main__":
+    save_path = "/hpc/projects/intracellular_dashboard/ops/ops0031_20250424/3-assembly/cytoself_features/cytoself_classification_scores.csv"
+    df = pd.read_csv(save_path)
+    labels_df = df[["label_int"]]
+    classification_scores = df.drop(columns=NONFEATURE_COLUMNS)
+
+    a = eval_classification_accuracy(
+        scores_df=classification_scores, labels_df=labels_df, label_column="label_int"
+    )

src/ops_model/eval/eval_cytoself.py

@@ -0,0 +1,123 @@
+import yaml
+from pathlib import Path
+
+import pandas as pd
+import torch
+import lightning as L
+
+from ops_model.data import data_loader
+from ops_model.models import cytoself_model
+
+torch.multiprocessing.set_sharing_strategy("file_system")
+
+
+def run_inference(
+    config_path: str,
+    checkpoint_path: str,
+    output_path: str,
+):
+    output_path = Path(output_path)
+
+    with open(config_path, "r") as f:
+        config = yaml.safe_load(f)
+
+    model_type = config["model_type"]
+    dataset_type = config["dataset_type"]
+
+    data_manager = data_loader.OpsDataManager(
+        experiments=config["data_manager"]["experiments"],
+        batch_size=config["data_manager"]["batch_size"],
+        data_split=(0, 0, 1),
+        out_channels=config["data_manager"]["out_channels"],
+        initial_yx_patch_size=tuple(config["data_manager"]["initial_yx_patch_size"]),
+        final_yx_patch_size=tuple(config["data_manager"]["final_yx_patch_size"]),
+    )
+    data_manager.construct_dataloaders(
+        num_workers=config["data_manager"]["num_workers"],
+        dataset_type=dataset_type,
+        basic_kwargs=config["data_manager"].get("basic_kwargs"),
+        triplet_kwargs=config["data_manager"].get("triplet_kwargs"),
+    )
+
+    test_loader = data_manager.test_loader
+
+    torch.set_float32_matmul_precision("medium")  # huge boost in speed
+
+    mc = config["model"]
+    lit_model = cytoself_model.LitCytoSelf.load_from_checkpoint(
+        checkpoint_path,
+        emb_shapes=(
+            tuple(mc["embedding_shapes"][0]),
+            tuple(mc["embedding_shapes"][1]),
+        ),
+        vq_args=mc["vq_args"],
+        num_class=mc["num_classes"],
+        input_shape=tuple(mc["input_shape"]),
+        output_shape=tuple(mc["input_shape"]),
+        fc_input_type=mc["fc_input_type"],
+        fc_output_idx=[mc["fc_output_index"]],
+    )
+    pred_writer = cytoself_model.CytoselfPredictionWriter
+
+    trainer = L.Trainer(
+        devices=1,
+        accelerator="gpu",
+        callbacks=[
+            pred_writer(
+                output_dir=output_path,
+                write_interval="batch",
+                int_label_lut=data_manager.int_label_lut,
+            )
+        ],
+        # limit_predict_batches=2
+    )
+    predictions = trainer.predict(lit_model, dataloaders=test_loader)
+
+    aggregate_csvs(
+        chunk_subdir=output_path / "emb_2_chunks",
+        final_csv_name="cytoself_local_features.csv",
+    )
+    aggregate_csvs(
+        chunk_subdir=output_path / "classification_scores",
+        final_csv_name="cytoself_classification_scores.csv",
+    )
+    aggregate_csvs(
+        chunk_subdir=output_path / "global_emb_metadata",
+        final_csv_name="cytoself_global_metadata.csv",
+    )
+
+    return
+
+
+def aggregate_csvs(
+    chunk_subdir: Path,
+    final_csv_name: str,
+):
+    print(f"\nLoading and concatenating chunks from {chunk_subdir.name}...")
+    csv_files = sorted(chunk_subdir.glob("*.csv"))
+
+    if not csv_files:
+        print("No feature files found!")
+        return None
+
+    df_list = [pd.read_csv(csv_file) for csv_file in csv_files]
+    final_df = pd.concat(df_list, ignore_index=True)
+
+    # Save the final concatenated dataframe
+    final_path = chunk_subdir.parent / final_csv_name
+    final_df.to_csv(final_path, index=False)
+    print(f"Saved final concatenated features to {final_path}")
+    print(f"Final dataframe shape: {final_df.shape}")
+
+    return
+
+
+if __name__ == "__main__":
+    checkpoint_path = "/hpc/projects/intracellular_dashboard/ops/models/logs/cytoself/cytoself_20251202_2/cytoself_20251202_2-2025-12-04-global_step=0.000000-val/total_loss=330.02.ckpt"
+    config_path = "/hpc/mydata/alexander.hillsley/ops/ops_model/configs/cytoself/cytoself_20251204.yml"
+    output_path = "/hpc/projects/intracellular_dashboard/ops/ops0031_20250424/3-assembly/cytoself_features"
+    run_inference(
+        config_path=config_path,
+        checkpoint_path=checkpoint_path,
+        output_path=output_path,
+    )