|
| 1 | +""" |
| 2 | +This script is related to a patent that has been filed. |
| 3 | +
|
| 4 | +Please contact the EPFL Technology Transfer Office (https://tto.epfl.ch/, info.tto@epfl.ch) for licensing inquiries. |
| 5 | +
|
| 6 | +---- |
| 7 | +
|
| 8 | +These script computes ROC curves for lensless authentication. |
| 9 | +
|
| 10 | +For this script, install: |
| 11 | +``` |
| 12 | +pip install scikit-learn seaborn |
| 13 | +``` |
| 14 | +ROC curve docs: https://scikit-learn.org/stable/modules/generated/sklearn.metrics.roc_curve.html |
| 15 | +
|
| 16 | +""" |
| 17 | + |
| 18 | +import numpy as np |
| 19 | +from sklearn import metrics |
| 20 | +import matplotlib.pyplot as plt |
| 21 | +import json |
| 22 | +import pandas as pd |
| 23 | +import seaborn as sn |
| 24 | + |
| 25 | +font_scale = 2.3 |
| 26 | +plt.rcParams.update({"font.size": 30}) |
| 27 | +lw = 5 # linewidth |
| 28 | +linestyles = ["--", "-.", ":"] |
| 29 | + |
| 30 | +# scores_paths = { |
| 31 | +# "ADMM10": "/root/LenslessPiCam/outputs/2024-03-25/23-36-06/scores_10_grayscaleTrue_down1_nfiles10000.json", |
| 32 | +# "ADMM25": "/root/LenslessPiCam/outputs/2024-03-26/17-52-49/scores_25_grayscaleTrue_down1_nfiles10000.json", |
| 33 | +# "ADMM50": "/root/LenslessPiCam/outputs/2024-03-27/10-49-08/scores_50_grayscaleTrue_down1_nfiles10000.json", |
| 34 | +# } |
| 35 | + |
| 36 | +# scores_paths = { |
| 37 | +# "Data fid.": { |
| 38 | +# # "path": "/root/LenslessPiCam/outputs/2024-12-07/20-26-06/scores_Unet4M+U5+Unet4M_wave_psfNN_down1_nfiles3750_metricrecon.txt", |
| 39 | +# "path": "/root/LenslessPiCam/authenticate_learned/data_fid/scores_Unet4M+U5+Unet4M_wave_psfNN_down1_nfiles3750_metricrecon.txt", |
| 40 | +# "invert": True, # if lower score is True |
| 41 | +# }, |
| 42 | +# # "MSE": { |
| 43 | +# # # "path": "/root/LenslessPiCam/outputs/2024-12-07/22-12-52/scores_Unet4M+U5+Unet4M_wave_psfNN_down1_nfiles3750_metricmse.txt", |
| 44 | +# # "path": "/root/LenslessPiCam/authenticate_learned/mse/scores_Unet4M+U5+Unet4M_wave_psfNN_down1_nfiles3750_metricmse.txt", |
| 45 | +# # "invert": True, # if lower score is True |
| 46 | +# # }, |
| 47 | +# "LPIPS": { |
| 48 | +# # "path": "/root/LenslessPiCam/outputs/2024-12-07/18-23-12/scores_Unet4M+U5+Unet4M_wave_psfNN_down1_nfiles3750_metriclpips.txt", |
| 49 | +# "path": "/root/LenslessPiCam/authenticate_learned/lpips/scores_Unet4M+U5+Unet4M_wave_psfNN_down1_nfiles3750_metriclpips.txt", |
| 50 | +# "invert": True, # if lower score is True |
| 51 | +# }, |
| 52 | +# } |
| 53 | +scores_paths = { |
| 54 | + "Data fid.": { |
| 55 | + # "path": "/root/LenslessPiCam/outputs/2024-12-08/07-17-49/scores_admm100_down1_nfiles3750_metricrecon.txt", |
| 56 | + "path": "/root/LenslessPiCam/authenticate_admm/recon/scores_admm100_down1_nfiles3750_metricrecon.txt", |
| 57 | + "invert": True, # if lower score is True |
| 58 | + }, |
| 59 | + # "MSE": { |
| 60 | + # "path": "/root/LenslessPiCam/outputs/2024-12-08/19-53-17/scores_admm100_down1_nfiles3750_metricmse.txt", |
| 61 | + # "invert": True, # if lower score is True |
| 62 | + # }, |
| 63 | + "LPIPS": { |
| 64 | + # "path": "/root/LenslessPiCam/outputs/2024-12-07/18-26-43/scores_admm100_down1_nfiles3750_metriclpips.txt", |
| 65 | + "path": "/root/LenslessPiCam/authenticate_admm/lpips/scores_admm100_down1_nfiles3750_metriclpips.txt", |
| 66 | + "invert": True, # if lower score is True |
| 67 | + }, |
| 68 | +} |
| 69 | + |
| 70 | +print_incorrect = False |
| 71 | + |
| 72 | +# TODO way to get this without loading dataset? |
| 73 | +n_files_per_mask = 250 |
| 74 | +mask_labels = list(np.arange(15)) * n_files_per_mask |
| 75 | +mask_labels = np.array(mask_labels) |
| 76 | + |
| 77 | +# initialize figure |
| 78 | +fig, ax = plt.subplots() |
| 79 | +for method, scores_dict in scores_paths.items(): |
| 80 | + print(f"--- Processing {method}...") |
| 81 | + scores_fp = scores_dict["path"] |
| 82 | + invert = scores_dict["invert"] |
| 83 | + |
| 84 | + scores = [] |
| 85 | + with open(scores_fp, "r") as f: |
| 86 | + for line in f: |
| 87 | + scores.append(json.loads(line)) |
| 88 | + scores = np.array(scores) |
| 89 | + n_psf = len(scores) |
| 90 | + n_files = len(scores[0]) |
| 91 | + |
| 92 | + # compute and plot confusion matrix |
| 93 | + confusion_matrix = np.zeros((n_psf, n_psf)) |
| 94 | + accuracy = np.zeros(n_psf) |
| 95 | + incorrect = dict() |
| 96 | + n_incorrect = 0 |
| 97 | + y_true = [] # for ROC curve |
| 98 | + y_score = [] # for ROC curve |
| 99 | + for psf_idx in range(n_psf): |
| 100 | + |
| 101 | + source_psf_mask = mask_labels == psf_idx |
| 102 | + confusion_matrix[psf_idx] = np.mean(np.array(scores[:, source_psf_mask]), axis=1) |
| 103 | + |
| 104 | + # for ROC curve |
| 105 | + y_true += list(source_psf_mask) |
| 106 | + y_score += list(scores[psf_idx]) |
| 107 | + |
| 108 | + # compute accuracy for each PSF |
| 109 | + detected_mask = np.argmin(scores[:, source_psf_mask], axis=0) |
| 110 | + if print_incorrect: |
| 111 | + print(f"PSF {psf_idx} detected as: ", detected_mask) |
| 112 | + accuracy[int(psf_idx)] = np.mean(detected_mask == int(psf_idx)) |
| 113 | + if accuracy[int(psf_idx)] < 1: |
| 114 | + incorrect_idx = np.where(detected_mask != int(psf_idx))[0] |
| 115 | + |
| 116 | + # reconvert idx back to original idx |
| 117 | + incorrect_idx = np.array([np.where(source_psf_mask)[0][i] for i in incorrect_idx]) |
| 118 | + incorrect[int(psf_idx)] = [int(i) for i in incorrect_idx] |
| 119 | + n_incorrect += len(incorrect_idx) |
| 120 | + |
| 121 | + total_accuracy = np.mean(accuracy) |
| 122 | + print("Total accuracy: ", total_accuracy) |
| 123 | + print("Number of incorrect detections: ", n_incorrect) |
| 124 | + |
| 125 | + #### FOR OLD ADMM SCORES |
| 126 | + # # load scores |
| 127 | + # with open(scores_fp, "r") as f: |
| 128 | + # scores = json.load(f) |
| 129 | + # |
| 130 | + # # prepare scores |
| 131 | + # y_true = [] |
| 132 | + # y_score = [] |
| 133 | + # n_psf = len(scores) |
| 134 | + # accuracy = np.zeros(n_psf) |
| 135 | + # confusion_matrix = np.zeros((n_psf, n_psf)) |
| 136 | + # for psf_idx in scores: |
| 137 | + # y_true_idx = np.ones(n_psf) |
| 138 | + # y_true_idx[int(psf_idx)] = 0 |
| 139 | + # for score in scores[psf_idx]: |
| 140 | + # y_true += list(y_true_idx) |
| 141 | + # y_score += list(score) |
| 142 | + |
| 143 | + # # confusion matrix |
| 144 | + # confusion_matrix[int(psf_idx)] = np.mean(np.array(scores[psf_idx]), axis=0) |
| 145 | + |
| 146 | + # # compute accuracy for each PSF |
| 147 | + # detected_mask = np.argmin(scores[psf_idx], axis=1) |
| 148 | + # accuracy[int(psf_idx)] = np.mean(detected_mask == int(psf_idx)) |
| 149 | + |
| 150 | + # total_accuracy = np.mean(accuracy) |
| 151 | + # print(f"Total accuracy ({method}): {total_accuracy:.2f}") |
| 152 | + |
| 153 | + # compute and plot confusion matrix |
| 154 | + df_cm = pd.DataFrame( |
| 155 | + confusion_matrix, index=[i for i in range(n_psf)], columns=[i for i in range(n_psf)] |
| 156 | + ) |
| 157 | + plt.figure(figsize=(10, 7)) |
| 158 | + # set font scale |
| 159 | + sn.set(font_scale=font_scale) |
| 160 | + sn.heatmap(df_cm, annot=False, cbar=True, xticklabels=5, yticklabels=5) |
| 161 | + confusion_fn = f"confusion_matrix_{method}.png" |
| 162 | + plt.savefig(confusion_fn, bbox_inches="tight") |
| 163 | + print(f"Confusion matrix saved as {confusion_fn}") |
| 164 | + |
| 165 | + # compute the ROC curve |
| 166 | + y_true = np.array(y_true).astype(bool) |
| 167 | + y_score = np.array(y_score) |
| 168 | + if invert: |
| 169 | + y_score = -1 * y_score |
| 170 | + fpr, tpr, thresholds = metrics.roc_curve(y_true, y_score) |
| 171 | + auc = metrics.roc_auc_score(y_true, y_score) |
| 172 | + |
| 173 | + # create ROC curve |
| 174 | + ax.plot(fpr, tpr, label=f"{method}, AUC={auc:.2f}", linewidth=lw, linestyle=linestyles.pop()) |
| 175 | + |
| 176 | + |
| 177 | +# set axis font size |
| 178 | +ax.set_ylabel("True Positive Rate") |
| 179 | +ax.set_xlabel("False Positive Rate") |
| 180 | +ax.legend() |
| 181 | +ax.grid() |
| 182 | + |
| 183 | + |
| 184 | +# save ROC curve |
| 185 | +plt.tight_layout() |
| 186 | +fig.savefig("roc_curve.png", bbox_inches="tight") |
0 commit comments