Skip to content

Commit ecf4317

Browse files
author
Jaime Céspedes Sisniega
authored
Merge pull request #309 from IFCA-Advanced-Computing/feature-p-values-callback
Add methods to compute p-values in PermutationTestDistanceBased callback
2 parents df4ea5d + c118df3 commit ecf4317

File tree

8 files changed

+491
-156
lines changed

8 files changed

+491
-156
lines changed

‎docs/source/examples/data_drift/MMD_advance.ipynb‎

Lines changed: 67 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,8 @@
2828
"metadata": {
2929
"collapsed": false,
3030
"ExecuteTime": {
31-
"end_time": "2024-02-16T16:12:58.157477481Z",
32-
"start_time": "2024-02-16T16:12:53.942261427Z"
31+
"end_time": "2024-02-17T19:17:48.453674110Z",
32+
"start_time": "2024-02-17T19:17:44.731749562Z"
3333
}
3434
}
3535
},
@@ -42,8 +42,8 @@
4242
"metadata": {
4343
"collapsed": false,
4444
"ExecuteTime": {
45-
"end_time": "2024-02-16T16:12:58.177626709Z",
46-
"start_time": "2024-02-16T16:12:58.156820964Z"
45+
"end_time": "2024-02-17T19:17:48.456838140Z",
46+
"start_time": "2024-02-17T19:17:48.443627264Z"
4747
}
4848
},
4949
"execution_count": 2
@@ -90,8 +90,8 @@
9090
"metadata": {
9191
"collapsed": false,
9292
"ExecuteTime": {
93-
"end_time": "2024-02-16T16:12:58.231182651Z",
94-
"start_time": "2024-02-16T16:12:58.166843644Z"
93+
"end_time": "2024-02-17T19:17:48.508728679Z",
94+
"start_time": "2024-02-17T19:17:48.456036506Z"
9595
}
9696
},
9797
"execution_count": 3
@@ -117,8 +117,8 @@
117117
"metadata": {
118118
"collapsed": false,
119119
"ExecuteTime": {
120-
"end_time": "2024-02-16T16:12:58.233172037Z",
121-
"start_time": "2024-02-16T16:12:58.211752497Z"
120+
"end_time": "2024-02-17T19:17:48.527449687Z",
121+
"start_time": "2024-02-17T19:17:48.507163830Z"
122122
}
123123
}
124124
},
@@ -244,8 +244,8 @@
244244
"metadata": {
245245
"collapsed": false,
246246
"ExecuteTime": {
247-
"end_time": "2024-02-16T16:12:58.325761642Z",
248-
"start_time": "2024-02-16T16:12:58.220690716Z"
247+
"end_time": "2024-02-17T19:17:48.618274285Z",
248+
"start_time": "2024-02-17T19:17:48.523267778Z"
249249
}
250250
}
251251
},
@@ -292,8 +292,8 @@
292292
"metadata": {
293293
"collapsed": false,
294294
"ExecuteTime": {
295-
"end_time": "2024-02-16T16:12:59.014362881Z",
296-
"start_time": "2024-02-16T16:12:58.330106479Z"
295+
"end_time": "2024-02-17T19:17:49.664911615Z",
296+
"start_time": "2024-02-17T19:17:48.621951276Z"
297297
}
298298
}
299299
},
@@ -343,8 +343,8 @@
343343
"metadata": {
344344
"collapsed": false,
345345
"ExecuteTime": {
346-
"end_time": "2024-02-16T16:12:59.025376511Z",
347-
"start_time": "2024-02-16T16:12:59.020884199Z"
346+
"end_time": "2024-02-17T19:17:49.678392697Z",
347+
"start_time": "2024-02-17T19:17:49.671817507Z"
348348
}
349349
},
350350
"execution_count": 7
@@ -480,8 +480,8 @@
480480
"metadata": {
481481
"collapsed": false,
482482
"ExecuteTime": {
483-
"end_time": "2024-02-16T16:17:46.577275393Z",
484-
"start_time": "2024-02-16T16:12:59.030173823Z"
483+
"end_time": "2024-02-17T19:22:44.701253430Z",
484+
"start_time": "2024-02-17T19:17:49.685187012Z"
485485
}
486486
},
487487
"execution_count": 8
@@ -545,8 +545,8 @@
545545
"metadata": {
546546
"collapsed": false,
547547
"ExecuteTime": {
548-
"end_time": "2024-02-16T16:17:46.595250026Z",
549-
"start_time": "2024-02-16T16:17:46.577060977Z"
548+
"end_time": "2024-02-17T19:22:44.760266237Z",
549+
"start_time": "2024-02-17T19:22:44.746259927Z"
550550
}
551551
}
552552
},
@@ -593,8 +593,8 @@
593593
"metadata": {
594594
"collapsed": false,
595595
"ExecuteTime": {
596-
"end_time": "2024-02-16T16:17:46.659949145Z",
597-
"start_time": "2024-02-16T16:17:46.593867783Z"
596+
"end_time": "2024-02-17T19:22:44.785296531Z",
597+
"start_time": "2024-02-17T19:22:44.747126194Z"
598598
}
599599
}
600600
},
@@ -622,8 +622,8 @@
622622
"metadata": {
623623
"collapsed": false,
624624
"ExecuteTime": {
625-
"end_time": "2024-02-16T16:17:46.662754862Z",
626-
"start_time": "2024-02-16T16:17:46.602440160Z"
625+
"end_time": "2024-02-17T19:22:44.787800781Z",
626+
"start_time": "2024-02-17T19:22:44.747623265Z"
627627
}
628628
}
629629
},
@@ -664,8 +664,8 @@
664664
"metadata": {
665665
"collapsed": false,
666666
"ExecuteTime": {
667-
"end_time": "2024-02-16T16:17:46.678825618Z",
668-
"start_time": "2024-02-16T16:17:46.610583289Z"
667+
"end_time": "2024-02-17T19:22:44.802033948Z",
668+
"start_time": "2024-02-17T19:22:44.748160859Z"
669669
}
670670
},
671671
"execution_count": 12
@@ -825,8 +825,8 @@
825825
"metadata": {
826826
"collapsed": false,
827827
"ExecuteTime": {
828-
"end_time": "2024-02-16T16:19:32.434187179Z",
829-
"start_time": "2024-02-16T16:17:46.624947418Z"
828+
"end_time": "2024-02-17T19:24:33.579552459Z",
829+
"start_time": "2024-02-17T19:22:44.748570864Z"
830830
}
831831
}
832832
},
@@ -879,8 +879,8 @@
879879
"metadata": {
880880
"collapsed": false,
881881
"ExecuteTime": {
882-
"end_time": "2024-02-16T16:19:33.007540003Z",
883-
"start_time": "2024-02-16T16:19:32.439104213Z"
882+
"end_time": "2024-02-17T19:24:34.275867733Z",
883+
"start_time": "2024-02-17T19:24:33.585003875Z"
884884
}
885885
},
886886
"execution_count": 14
@@ -923,8 +923,8 @@
923923
"metadata": {
924924
"collapsed": false,
925925
"ExecuteTime": {
926-
"end_time": "2024-02-16T16:19:33.011524066Z",
927-
"start_time": "2024-02-16T16:19:33.006432926Z"
926+
"end_time": "2024-02-17T19:24:34.282442755Z",
927+
"start_time": "2024-02-17T19:24:34.278278230Z"
928928
}
929929
}
930930
},
@@ -973,8 +973,8 @@
973973
"metadata": {
974974
"collapsed": false,
975975
"ExecuteTime": {
976-
"end_time": "2024-02-16T16:19:33.104789676Z",
977-
"start_time": "2024-02-16T16:19:33.009110935Z"
976+
"end_time": "2024-02-17T19:24:34.300226242Z",
977+
"start_time": "2024-02-17T19:24:34.282242198Z"
978978
}
979979
}
980980
},
@@ -1032,8 +1032,8 @@
10321032
"metadata": {
10331033
"collapsed": false,
10341034
"ExecuteTime": {
1035-
"end_time": "2024-02-16T16:19:33.466987384Z",
1036-
"start_time": "2024-02-16T16:19:33.050152210Z"
1035+
"end_time": "2024-02-17T19:24:34.779159458Z",
1036+
"start_time": "2024-02-17T19:24:34.297080808Z"
10371037
}
10381038
}
10391039
},
@@ -1072,8 +1072,8 @@
10721072
"metadata": {
10731073
"collapsed": false,
10741074
"ExecuteTime": {
1075-
"end_time": "2024-02-16T16:19:33.486259158Z",
1076-
"start_time": "2024-02-16T16:19:33.472288854Z"
1075+
"end_time": "2024-02-17T19:24:34.795749904Z",
1076+
"start_time": "2024-02-17T19:24:34.779515645Z"
10771077
}
10781078
},
10791079
"execution_count": 18
@@ -1104,8 +1104,8 @@
11041104
"metadata": {
11051105
"collapsed": false,
11061106
"ExecuteTime": {
1107-
"end_time": "2024-02-16T16:19:33.623751778Z",
1108-
"start_time": "2024-02-16T16:19:33.480473726Z"
1107+
"end_time": "2024-02-17T19:24:34.855686275Z",
1108+
"start_time": "2024-02-17T19:24:34.785007832Z"
11091109
}
11101110
},
11111111
"execution_count": 19
@@ -1144,8 +1144,8 @@
11441144
"metadata": {
11451145
"collapsed": false,
11461146
"ExecuteTime": {
1147-
"end_time": "2024-02-16T16:20:11.126831863Z",
1148-
"start_time": "2024-02-16T16:19:33.490875845Z"
1147+
"end_time": "2024-02-17T19:25:12.637290740Z",
1148+
"start_time": "2024-02-17T19:24:34.793373179Z"
11491149
}
11501150
},
11511151
"execution_count": 20
@@ -1188,7 +1188,7 @@
11881188
"application/vnd.jupyter.widget-view+json": {
11891189
"version_major": 2,
11901190
"version_minor": 0,
1191-
"model_id": "a4e345ef8f6e4652a565d59fed3886d7"
1191+
"model_id": "7fededc9957d450d9df83a331e6c4482"
11921192
}
11931193
},
11941194
"metadata": {},
@@ -1200,7 +1200,7 @@
12001200
"application/vnd.jupyter.widget-view+json": {
12011201
"version_major": 2,
12021202
"version_minor": 0,
1203-
"model_id": "5f150463f92c4f05b49eeea4a3b69448"
1203+
"model_id": "d838eb10e39442a59dbdd471a4df1e66"
12041204
}
12051205
},
12061206
"metadata": {},
@@ -1212,7 +1212,7 @@
12121212
"application/vnd.jupyter.widget-view+json": {
12131213
"version_major": 2,
12141214
"version_minor": 0,
1215-
"model_id": "21554996ba2642b8a197637c07969c08"
1215+
"model_id": "f185a3e5f92f4dd893371efede2b0861"
12161216
}
12171217
},
12181218
"metadata": {},
@@ -1224,7 +1224,7 @@
12241224
"application/vnd.jupyter.widget-view+json": {
12251225
"version_major": 2,
12261226
"version_minor": 0,
1227-
"model_id": "342a329a4ec8482681191407865d0795"
1227+
"model_id": "71b3e47714b342ccb7c34992e707490d"
12281228
}
12291229
},
12301230
"metadata": {},
@@ -1242,8 +1242,8 @@
12421242
"metadata": {
12431243
"collapsed": false,
12441244
"ExecuteTime": {
1245-
"end_time": "2024-02-16T16:20:14.682462555Z",
1246-
"start_time": "2024-02-16T16:20:11.130324408Z"
1245+
"end_time": "2024-02-17T19:25:16.349855710Z",
1246+
"start_time": "2024-02-17T19:25:12.640768365Z"
12471247
}
12481248
},
12491249
"execution_count": 21
@@ -1273,8 +1273,8 @@
12731273
"metadata": {
12741274
"collapsed": false,
12751275
"ExecuteTime": {
1276-
"end_time": "2024-02-16T16:20:14.735987652Z",
1277-
"start_time": "2024-02-16T16:20:14.682151132Z"
1276+
"end_time": "2024-02-17T19:25:16.399374408Z",
1277+
"start_time": "2024-02-17T19:25:16.349624925Z"
12781278
}
12791279
}
12801280
},
@@ -1342,8 +1342,8 @@
13421342
"metadata": {
13431343
"collapsed": false,
13441344
"ExecuteTime": {
1345-
"end_time": "2024-02-16T16:20:15.963870971Z",
1346-
"start_time": "2024-02-16T16:20:14.729968761Z"
1345+
"end_time": "2024-02-17T19:25:17.774299178Z",
1346+
"start_time": "2024-02-17T19:25:16.397465080Z"
13471347
}
13481348
},
13491349
"execution_count": 23
@@ -1376,8 +1376,8 @@
13761376
"metadata": {
13771377
"collapsed": false,
13781378
"ExecuteTime": {
1379-
"end_time": "2024-02-16T16:20:15.965661922Z",
1380-
"start_time": "2024-02-16T16:20:15.963526707Z"
1379+
"end_time": "2024-02-17T19:25:17.776333552Z",
1380+
"start_time": "2024-02-17T19:25:17.774063310Z"
13811381
}
13821382
}
13831383
},
@@ -1417,8 +1417,8 @@
14171417
"metadata": {
14181418
"collapsed": false,
14191419
"ExecuteTime": {
1420-
"end_time": "2024-02-16T16:20:15.978381396Z",
1421-
"start_time": "2024-02-16T16:20:15.963740999Z"
1420+
"end_time": "2024-02-17T19:25:17.799793047Z",
1421+
"start_time": "2024-02-17T19:25:17.774429027Z"
14221422
}
14231423
},
14241424
"execution_count": 25
@@ -1440,8 +1440,8 @@
14401440
" num_permutations=num_permutations,\n",
14411441
" random_state=seed,\n",
14421442
" num_jobs=-1,\n",
1443+
" method=\"exact\",\n",
14431444
" name=\"permutation_test\",\n",
1444-
" conservative=True,\n",
14451445
" verbose=False,\n",
14461446
" ),\n",
14471447
" ],\n",
@@ -1451,8 +1451,8 @@
14511451
"metadata": {
14521452
"collapsed": false,
14531453
"ExecuteTime": {
1454-
"end_time": "2024-02-16T16:20:16.181632069Z",
1455-
"start_time": "2024-02-16T16:20:15.978662597Z"
1454+
"end_time": "2024-02-17T19:25:18.003211081Z",
1455+
"start_time": "2024-02-17T19:25:17.792743610Z"
14561456
}
14571457
}
14581458
},
@@ -1505,8 +1505,8 @@
15051505
"metadata": {
15061506
"collapsed": false,
15071507
"ExecuteTime": {
1508-
"end_time": "2024-02-16T16:20:16.402059920Z",
1509-
"start_time": "2024-02-16T16:20:16.022002212Z"
1508+
"end_time": "2024-02-17T19:25:18.284100029Z",
1509+
"start_time": "2024-02-17T19:25:17.833430365Z"
15101510
}
15111511
},
15121512
"execution_count": 27
@@ -1529,11 +1529,11 @@
15291529
"output_type": "stream",
15301530
"text": [
15311531
"Original (unmodified):\n",
1532-
"\tMMD statistic=-0.0003, p-value=0.7219, drift=False\n",
1532+
"\tMMD statistic=-0.0003, p-value=0.72185513, drift=False\n",
15331533
"Gaussian Blur:\n",
1534-
"\tMMD statistic=0.0083, p-value=0.0002, drift=True\n",
1534+
"\tMMD statistic=0.0083, p-value=0.00019946, drift=True\n",
15351535
"Elastic Transform:\n",
1536-
"\tMMD statistic=0.0075, p-value=0.0002, drift=True\n"
1536+
"\tMMD statistic=0.0075, p-value=0.00019946, drift=True\n"
15371537
]
15381538
}
15391539
],
@@ -1555,13 +1555,13 @@
15551555
" mmd, callbacks_logs = detector.compare(X=sample)\n",
15561556
" permutation_test_logs[type_] = copy.copy(callbacks_logs[\"permutation_test\"])\n",
15571557
" mmd, p_value = mmd.distance, callbacks_logs[\"permutation_test\"][\"p_value\"]\n",
1558-
" print(f\"{type_}:\\n\\tMMD statistic={round(mmd, 4)}, p-value={round(p_value, 4)}, drift={p_value <= alpha}\")"
1558+
" print(f\"{type_}:\\n\\tMMD statistic={round(mmd, 4)}, p-value={round(p_value, 8)}, drift={p_value <= alpha}\")"
15591559
],
15601560
"metadata": {
15611561
"collapsed": false,
15621562
"ExecuteTime": {
1563-
"end_time": "2024-02-16T16:22:17.370932094Z",
1564-
"start_time": "2024-02-16T16:20:16.406203620Z"
1563+
"end_time": "2024-02-17T19:27:38.781406175Z",
1564+
"start_time": "2024-02-17T19:25:18.291176981Z"
15651565
}
15661566
}
15671567
},
@@ -1648,8 +1648,8 @@
16481648
"metadata": {
16491649
"collapsed": false,
16501650
"ExecuteTime": {
1651-
"end_time": "2024-02-16T16:22:19.458873310Z",
1652-
"start_time": "2024-02-16T16:22:17.379199703Z"
1651+
"end_time": "2024-02-17T19:27:40.819480169Z",
1652+
"start_time": "2024-02-17T19:27:38.800648045Z"
16531653
}
16541654
}
16551655
},

0 commit comments

Comments
 (0)