|
2 | 2 | "cells": [ |
3 | 3 | { |
4 | 4 | "cell_type": "markdown", |
5 | | - "metadata": {}, |
| 5 | + "metadata": { |
| 6 | + "id": "8ma4uIoIZjC1" |
| 7 | + }, |
| 8 | + "source": [ |
| 9 | + "# benchmark: medpy vs. mikan-rs 🍊" |
| 10 | + ] |
| 11 | + }, |
| 12 | + { |
| 13 | + "cell_type": "markdown", |
| 14 | + "metadata": { |
| 15 | + "id": "L5D9-i91eYUY" |
| 16 | + }, |
6 | 17 | "source": [ |
7 | | - "## Benchmark" |
| 18 | + "Please note that **Colab only provides dual-core CPUs**, so the speedup is limited. You can test on a server with more CPUs to observe mikan's blazingly fast performance." |
8 | 19 | ] |
9 | 20 | }, |
10 | 21 | { |
11 | 22 | "cell_type": "code", |
12 | 23 | "execution_count": 1, |
13 | | - "metadata": {}, |
| 24 | + "metadata": { |
| 25 | + "colab": { |
| 26 | + "base_uri": "https://localhost:8080/" |
| 27 | + }, |
| 28 | + "id": "y_qn133LaML8", |
| 29 | + "outputId": "618ac8ca-5e1f-4519-ee9f-343ca1a64ccc" |
| 30 | + }, |
| 31 | + "outputs": [ |
| 32 | + { |
| 33 | + "name": "stdout", |
| 34 | + "output_type": "stream", |
| 35 | + "text": [ |
| 36 | + "Requirement already satisfied: SimpleITK in /usr/local/lib/python3.11/dist-packages (2.4.1)\n", |
| 37 | + "Requirement already satisfied: medpy in /usr/local/lib/python3.11/dist-packages (0.5.2)\n", |
| 38 | + "Requirement already satisfied: mikan-rs in /usr/local/lib/python3.11/dist-packages (0.1.2)\n", |
| 39 | + "Requirement already satisfied: scipy>=1.10 in /usr/local/lib/python3.11/dist-packages (from medpy) (1.13.1)\n", |
| 40 | + "Requirement already satisfied: numpy>=1.24 in /usr/local/lib/python3.11/dist-packages (from medpy) (1.26.4)\n" |
| 41 | + ] |
| 42 | + } |
| 43 | + ], |
| 44 | + "source": [ |
| 45 | + "!pip install SimpleITK medpy mikan-rs" |
| 46 | + ] |
| 47 | + }, |
| 48 | + { |
| 49 | + "cell_type": "code", |
| 50 | + "execution_count": 2, |
| 51 | + "metadata": { |
| 52 | + "colab": { |
| 53 | + "base_uri": "https://localhost:8080/", |
| 54 | + "height": 181 |
| 55 | + }, |
| 56 | + "id": "a_RYW5sPX6mY", |
| 57 | + "outputId": "2a3fec85-5b1c-4b4b-82d7-d4cb961d7354" |
| 58 | + }, |
| 59 | + "outputs": [ |
| 60 | + { |
| 61 | + "name": "stderr", |
| 62 | + "output_type": "stream", |
| 63 | + "text": [ |
| 64 | + "Downloading...\n", |
| 65 | + "From: https://drive.google.com/uc?id=1R6rph1_Wc2HfLhzvGkcNt7hgbQiHWZBY\n", |
| 66 | + "To: /content/patients_26_ground_truth.nii.gz\n", |
| 67 | + "100%|██████████| 1.17M/1.17M [00:00<00:00, 70.3MB/s]\n", |
| 68 | + "Downloading...\n", |
| 69 | + "From: https://drive.google.com/uc?id=1cShgX96WgK_j4EbfR4wN2RLgcyVR4ca0\n", |
| 70 | + "To: /content/patients_26_segmentation.nii.gz\n", |
| 71 | + "100%|██████████| 5.05M/5.05M [00:00<00:00, 28.3MB/s]\n" |
| 72 | + ] |
| 73 | + }, |
| 74 | + { |
| 75 | + "data": { |
| 76 | + "application/vnd.google.colaboratory.intrinsic+json": { |
| 77 | + "type": "string" |
| 78 | + }, |
| 79 | + "text/plain": [ |
| 80 | + "'patients_26_segmentation.nii.gz'" |
| 81 | + ] |
| 82 | + }, |
| 83 | + "execution_count": 2, |
| 84 | + "metadata": {}, |
| 85 | + "output_type": "execute_result" |
| 86 | + } |
| 87 | + ], |
| 88 | + "source": [ |
| 89 | + "import gdown\n", |
| 90 | + "\n", |
| 91 | + "# We use datasets from seg_metrics\n", |
| 92 | + "\n", |
| 93 | + "gdth_url_id = \"1R6rph1_Wc2HfLhzvGkcNt7hgbQiHWZBY\"\n", |
| 94 | + "pred_url_id = \"1cShgX96WgK_j4EbfR4wN2RLgcyVR4ca0\"\n", |
| 95 | + "gdth_fpath = \"patients_26_ground_truth.nii.gz\"\n", |
| 96 | + "pred_fpath = \"patients_26_segmentation.nii.gz\"\n", |
| 97 | + "\n", |
| 98 | + "gdown.download(id=gdth_url_id, output=gdth_fpath, quiet=False)\n", |
| 99 | + "gdown.download(id=pred_url_id, output=pred_fpath, quiet=False)" |
| 100 | + ] |
| 101 | + }, |
| 102 | + { |
| 103 | + "cell_type": "code", |
| 104 | + "execution_count": 3, |
| 105 | + "metadata": { |
| 106 | + "id": "cxVnoqqEYd6F" |
| 107 | + }, |
14 | 108 | "outputs": [], |
15 | 109 | "source": [ |
| 110 | + "import time\n", |
16 | 111 | "import mikan\n", |
17 | 112 | "import numpy as np\n", |
18 | 113 | "import SimpleITK as sitk\n", |
19 | | - "import time\n", |
20 | 114 | "from medpy.metric import dc, hd, hd95, assd" |
21 | 115 | ] |
22 | 116 | }, |
23 | 117 | { |
24 | 118 | "cell_type": "markdown", |
25 | | - "metadata": {}, |
| 119 | + "metadata": { |
| 120 | + "id": "Lmzfs9DaZyaD" |
| 121 | + }, |
26 | 122 | "source": [ |
27 | | - "## Load Data" |
| 123 | + "## Load datas" |
28 | 124 | ] |
29 | 125 | }, |
30 | 126 | { |
31 | 127 | "cell_type": "code", |
32 | | - "execution_count": null, |
33 | | - "metadata": {}, |
| 128 | + "execution_count": 4, |
| 129 | + "metadata": { |
| 130 | + "id": "E3lZFPFNYiSK" |
| 131 | + }, |
34 | 132 | "outputs": [], |
35 | 133 | "source": [ |
36 | | - "gt = sitk.ReadImage(rf\"..\\data\\patients_26_ground_truth.nii.gz\", sitk.sitkUInt8)\n", |
37 | | - "pred = sitk.ReadImage(rf\"..\\data\\patients_26_segmentation.nii.gz\", sitk.sitkUInt8)\n", |
| 134 | + "gt = sitk.ReadImage(\"patients_26_ground_truth.nii.gz\", sitk.sitkUInt8)\n", |
| 135 | + "pred = sitk.ReadImage(\"patients_26_segmentation.nii.gz\", sitk.sitkUInt8)\n", |
38 | 136 | "\n", |
39 | 137 | "gt_arr = sitk.GetArrayFromImage(gt)\n", |
40 | 138 | "pred_arr = sitk.GetArrayFromImage(pred)\n", |
41 | 139 | "\n", |
42 | 140 | "# Downsample for faster\n", |
43 | | - "# If you're patient, you can comment out here and wait for the medpy to run for 30 minutes 😆\n", |
| 141 | + "# If you're patient, you can comment out here and wait for medpy to run for 30 minutes 😆\n", |
44 | 142 | "gt_arr = np.array(gt_arr[::2, ::2, ::2])\n", |
45 | | - "pred_arr = np.array(pred_arr[::2, ::2, ::2])" |
| 143 | + "pred_arr = np.array(pred_arr[::2, ::2, ::2])" |
46 | 144 | ] |
47 | 145 | }, |
48 | 146 | { |
49 | 147 | "cell_type": "markdown", |
50 | | - "metadata": {}, |
| 148 | + "metadata": { |
| 149 | + "id": "m9vGRSY8Z2v9" |
| 150 | + }, |
51 | 151 | "source": [ |
52 | | - "### DSC" |
| 152 | + "## Dice" |
53 | 153 | ] |
54 | 154 | }, |
55 | 155 | { |
56 | 156 | "cell_type": "code", |
57 | | - "execution_count": 3, |
58 | | - "metadata": {}, |
| 157 | + "execution_count": 8, |
| 158 | + "metadata": { |
| 159 | + "colab": { |
| 160 | + "base_uri": "https://localhost:8080/" |
| 161 | + }, |
| 162 | + "id": "nMxlyYl3YpDO", |
| 163 | + "outputId": "c87b5310-88ba-4381-8115-13a7187c3af6" |
| 164 | + }, |
59 | 165 | "outputs": [ |
60 | 166 | { |
61 | 167 | "name": "stdout", |
62 | 168 | "output_type": "stream", |
63 | 169 | "text": [ |
64 | | - "DSC: 15.09x faster\n" |
| 170 | + "Mikan cost 0.12 s.\n", |
| 171 | + "medpy costs 0.19 s.\n", |
| 172 | + "DSC: 1.54x faster\n" |
65 | 173 | ] |
66 | 174 | } |
67 | 175 | ], |
|
71 | 179 | "evaluator = mikan.ArrayEvaluator(gt_arr, pred_arr, spacing=gt.GetSpacing())\n", |
72 | 180 | "dsc = evaluator.labels([1,2,3,4,5]).metrics(\"dsc\")\n", |
73 | 181 | "mikan_costs = time.time() - t\n", |
| 182 | + "print(f\"Mikan cost {mikan_costs:.2f} s.\")\n", |
74 | 183 | "\n", |
75 | 184 | "# medpy: DSC\n", |
76 | 185 | "t = time.time()\n", |
77 | 186 | "for i in (1,2,3,4,5):\n", |
78 | 187 | " dsc = dc(pred_arr == i, gt_arr == i)\n", |
79 | 188 | "medpy_costs = time.time() - t\n", |
80 | | - "\n", |
| 189 | + "print(f\"medpy costs {time.time() - t:.2f} s.\")\n", |
81 | 190 | "print(f\"DSC: {medpy_costs / mikan_costs :.2f}x faster\")\n" |
82 | 191 | ] |
83 | 192 | }, |
84 | 193 | { |
85 | 194 | "cell_type": "markdown", |
86 | | - "metadata": {}, |
| 195 | + "metadata": { |
| 196 | + "id": "YX0b4t2XZ4z8" |
| 197 | + }, |
87 | 198 | "source": [ |
88 | | - "### HD" |
| 199 | + "## HD" |
89 | 200 | ] |
90 | 201 | }, |
91 | 202 | { |
92 | 203 | "cell_type": "code", |
93 | | - "execution_count": 4, |
94 | | - "metadata": {}, |
| 204 | + "execution_count": 9, |
| 205 | + "metadata": { |
| 206 | + "colab": { |
| 207 | + "base_uri": "https://localhost:8080/" |
| 208 | + }, |
| 209 | + "id": "hrU3grdDdrCn", |
| 210 | + "outputId": "bc289aa3-2001-4455-b7b6-2f6242640b7e" |
| 211 | + }, |
95 | 212 | "outputs": [ |
96 | 213 | { |
97 | 214 | "name": "stdout", |
98 | 215 | "output_type": "stream", |
99 | 216 | "text": [ |
100 | | - "Mikan has calculated Hausdorff distance and cost 0.78 s.\n", |
| 217 | + "Mikan has calculated Hausdorff distance and cost 4.47 s.\n", |
101 | 218 | "Let's waiting for medpy, be patient for a while...\n", |
102 | | - "HD: 68.78x faster\n" |
| 219 | + "HD: 11.14x faster\n" |
103 | 220 | ] |
104 | 221 | } |
105 | 222 | ], |
|
123 | 240 | }, |
124 | 241 | { |
125 | 242 | "cell_type": "markdown", |
126 | | - "metadata": {}, |
| 243 | + "metadata": { |
| 244 | + "id": "vC0KgPIlZ6LU" |
| 245 | + }, |
127 | 246 | "source": [ |
128 | | - "### HD/HD95/ASSD" |
| 247 | + "## All Distances" |
129 | 248 | ] |
130 | 249 | }, |
131 | 250 | { |
132 | 251 | "cell_type": "code", |
133 | | - "execution_count": 5, |
134 | | - "metadata": {}, |
| 252 | + "execution_count": 10, |
| 253 | + "metadata": { |
| 254 | + "colab": { |
| 255 | + "base_uri": "https://localhost:8080/" |
| 256 | + }, |
| 257 | + "id": "3weKKOcMZeLC", |
| 258 | + "outputId": "4c87051f-9d4c-4e79-a192-ae8a26ccb68a" |
| 259 | + }, |
135 | 260 | "outputs": [ |
136 | 261 | { |
137 | 262 | "name": "stdout", |
138 | 263 | "output_type": "stream", |
139 | 264 | "text": [ |
140 | 265 | "{'1': {'hd': 3.8066790103912354, 'hd95': 0.7410011291503906, 'assd': 0.24823972582817078}, '2': {'hd': 5.875893592834473, 'hd95': 0.8939142823219299, 'assd': 0.30189621448516846}, '3': {'hd': 7.1895952224731445, 'hd95': 0.7410049438476562, 'assd': 0.29092279076576233}, '4': {'hd': 55.0378303527832, 'hd95': 0.7410011291503906, 'assd': 0.2536499500274658}, '5': {'hd': 46.238975524902344, 'hd95': 0.7410125732421875, 'assd': 0.27448806166648865}}\n", |
141 | | - "Mikan has calculated distance and cost 0.85 s.\n", |
| 266 | + "Mikan has calculated distance and cost 3.80 s.\n", |
142 | 267 | "Let's waiting for medpy, be patient for a while...\n", |
143 | 268 | "{1: {'hd': 3.8066796490554484, 'hd95': 0.7409999966621399, 'assd': 0.24827345569665876}, 2: {'hd': 5.875891921349948, 'hd95': 0.893913303991663, 'assd': 0.3018870383121832}, 3: {'hd': 7.18959419428273, 'hd95': 0.7409999966621399, 'assd': 0.2909637348388582}, 4: {'hd': 55.03783156368531, 'hd95': 0.7409999966621399, 'assd': 0.25367831066385055}, 5: {'hd': 46.23897571237574, 'hd95': 0.7409999966621399, 'assd': 0.2745253765803193}}\n", |
144 | | - "Distances: 190.71x faster\n" |
| 269 | + "Distances: 39.67x faster\n" |
145 | 270 | ] |
146 | 271 | } |
147 | 272 | ], |
|
174 | 299 | } |
175 | 300 | ], |
176 | 301 | "metadata": { |
| 302 | + "colab": { |
| 303 | + "provenance": [] |
| 304 | + }, |
177 | 305 | "kernelspec": { |
178 | 306 | "display_name": "Python 3", |
179 | | - "language": "python", |
180 | 307 | "name": "python3" |
181 | 308 | }, |
182 | 309 | "language_info": { |
183 | | - "codemirror_mode": { |
184 | | - "name": "ipython", |
185 | | - "version": 3 |
186 | | - }, |
187 | | - "file_extension": ".py", |
188 | | - "mimetype": "text/x-python", |
189 | 310 | "name": "python", |
190 | | - "nbconvert_exporter": "python", |
191 | | - "pygments_lexer": "ipython3", |
192 | 311 | "version": "3.12.3" |
193 | 312 | } |
194 | 313 | }, |
195 | 314 | "nbformat": 4, |
196 | | - "nbformat_minor": 2 |
| 315 | + "nbformat_minor": 0 |
197 | 316 | } |
0 commit comments