Skip to content

Commit 6d541fe

Browse files
dpuenteramirezdpuenteramirez
committed
Updated stats #218 
1 parent 4f43226 commit 6d541fe

File tree

1 file changed

+189
-56
lines changed

1 file changed

+189
-56
lines changed

‎experimentation/rank_statistics.ipynb‎

Lines changed: 189 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -15,54 +15,74 @@
1515
},
1616
{
1717
"cell_type": "code",
18-
"execution_count": 209,
18+
"execution_count": 9,
19+
"metadata": {
20+
"collapsed": false,
21+
"pycharm": {
22+
"name": "#%%\n"
23+
}
24+
},
1925
"outputs": [],
2026
"source": [
2127
"import os.path\n",
2228
"import scipy.stats\n",
2329
"from itertools import product\n",
2430
"import pandas as pd\n",
2531
"import numpy as np"
26-
],
32+
]
33+
},
34+
{
35+
"cell_type": "code",
36+
"execution_count": 42,
37+
"metadata": {},
38+
"outputs": [],
39+
"source": [
40+
"np.set_printoptions(formatter={'float_kind': \"{:6.3f}\".format})\n",
41+
"#np.set_printoptions(precision=4)\n",
42+
"RED='\\033[0;31m'\n",
43+
"GRN='\\033[0;32m'\n",
44+
"NC='\\033[0m'\n",
45+
"BLD='\\033[1m'"
46+
]
47+
},
48+
{
49+
"cell_type": "code",
50+
"execution_count": 10,
2751
"metadata": {
2852
"collapsed": false,
2953
"pycharm": {
3054
"name": "#%%\n"
3155
}
32-
}
33-
},
34-
{
35-
"cell_type": "code",
36-
"execution_count": 210,
56+
},
3757
"outputs": [],
3858
"source": [
3959
"metric = 'f1-score'\n",
4060
"results = os.path.join('ranks', 'results.csv')"
41-
],
61+
]
62+
},
63+
{
64+
"cell_type": "code",
65+
"execution_count": 11,
4266
"metadata": {
4367
"collapsed": false,
4468
"pycharm": {
4569
"name": "#%%\n"
4670
}
47-
}
48-
},
49-
{
50-
"cell_type": "code",
51-
"execution_count": 211,
71+
},
5272
"outputs": [],
5373
"source": [
5474
"results_df = pd.read_csv(results)"
55-
],
75+
]
76+
},
77+
{
78+
"cell_type": "code",
79+
"execution_count": 12,
5680
"metadata": {
5781
"collapsed": false,
5882
"pycharm": {
5983
"name": "#%%\n"
6084
}
61-
}
62-
},
63-
{
64-
"cell_type": "code",
65-
"execution_count": 212,
85+
},
6686
"outputs": [
6787
{
6888
"name": "stdout",
@@ -86,17 +106,17 @@
86106
"print(f\"Filters used: {filters}\")\n",
87107
"print(f\"Percents labeled used: {percents_labeled}\")\n",
88108
"print(f\"# Datasets used: {len(datasets)}\")\n"
89-
],
109+
]
110+
},
111+
{
112+
"cell_type": "code",
113+
"execution_count": 13,
90114
"metadata": {
91115
"collapsed": false,
92116
"pycharm": {
93117
"name": "#%%\n"
94118
}
95-
}
96-
},
97-
{
98-
"cell_type": "code",
99-
"execution_count": 213,
119+
},
100120
"outputs": [
101121
{
102122
"name": "stdout",
@@ -127,17 +147,17 @@
127147
"\n",
128148
"bases_filters = np.array(list(product(base_estimators, filters)))\n",
129149
"print(\"Pairs of base estimator with filters:\\n\", bases_filters)"
130-
],
150+
]
151+
},
152+
{
153+
"cell_type": "code",
154+
"execution_count": 14,
131155
"metadata": {
132156
"collapsed": false,
133157
"pycharm": {
134158
"name": "#%%\n"
135159
}
136-
}
137-
},
138-
{
139-
"cell_type": "code",
140-
"execution_count": 214,
160+
},
141161
"outputs": [
142162
{
143163
"name": "stdout",
@@ -175,17 +195,17 @@
175195
],
176196
"source": [
177197
"print(grouped_df)"
178-
],
198+
]
199+
},
200+
{
201+
"cell_type": "code",
202+
"execution_count": 15,
179203
"metadata": {
180204
"collapsed": false,
181205
"pycharm": {
182206
"name": "#%%\n"
183207
}
184-
}
185-
},
186-
{
187-
"cell_type": "code",
188-
"execution_count": 215,
208+
},
189209
"outputs": [],
190210
"source": [
191211
"def base_filter_values():\n",
@@ -208,17 +228,26 @@
208228
"def split_onto_base_estimators():\n",
209229
" for base_ in base_estimators:\n",
210230
" base_dfs.append(curr_vals.filter(regex=base_))"
211-
],
231+
]
232+
},
233+
{
234+
"cell_type": "code",
235+
"execution_count": 24,
236+
"metadata": {},
237+
"outputs": [],
238+
"source": [
239+
"data = []"
240+
]
241+
},
242+
{
243+
"cell_type": "code",
244+
"execution_count": 40,
212245
"metadata": {
213246
"collapsed": false,
214247
"pycharm": {
215248
"name": "#%%\n"
216249
}
217-
}
218-
},
219-
{
220-
"cell_type": "code",
221-
"execution_count": 216,
250+
},
222251
"outputs": [
223252
{
224253
"name": "stdout",
@@ -368,6 +397,7 @@
368397
}
369398
],
370399
"source": [
400+
"data = []\n",
371401
"for percent in percents_labeled:\n",
372402
" curr_vals = base_filter_values()\n",
373403
"\n",
@@ -392,36 +422,139 @@
392422
" value in zip(df.columns,\n",
393423
" average_ranks)}\n",
394424
" print()\n",
395-
" for k, v in average_rank_for.items():\n",
396-
" print(k, f'{v:.2f}')\n"
397-
],
398-
"metadata": {
399-
"collapsed": false,
400-
"pycharm": {
401-
"name": "#%%\n"
425+
" for key, value in average_rank_for.items():\n",
426+
" print(key, f'{value:.2f}')\n",
427+
" \n",
428+
" \n",
429+
" part0 = (12*N)/(k*(k+1))\n",
430+
" part1 = sum([Rj**2 for Rj in average_ranks])\n",
431+
" part2 = (k*(k+1)**2)/4 \n",
432+
" 𝛘2_F = part0*(part1-part2)\n",
433+
" F_F = (N-1)*𝛘2_F/(N*(k-1)-𝛘2_F)\n",
434+
" \n",
435+
" data.append([percent, F_F, average_rank_for, N, k]) "
436+
]
437+
},
438+
{
439+
"cell_type": "markdown",
440+
"metadata": {},
441+
"source": [
442+
"# Ivan And Davenport \n",
443+
"F_F = (N-1)*𝛘2_F/(N*(k-1)-𝛘2_F)"
444+
]
445+
},
446+
{
447+
"cell_type": "markdown",
448+
"metadata": {},
449+
"source": [
450+
"https://www.statology.org/f-distribution-calculator/\n",
451+
"\n",
452+
"With 3 classifiers, 3 filters and the base one, and 18 data sets, $F_F$ is distributed according to the $F$ distribution with\n",
453+
"$7-1=6$ and $(7-1)×(18−1)=102$ degrees of freedom. The critical value of $F(6,102)$ for $\\alpha=0.05$\n",
454+
"is $2.00002$, so we reject the null-hypothesis"
455+
]
456+
},
457+
{
458+
"cell_type": "code",
459+
"execution_count": null,
460+
"metadata": {},
461+
"outputs": [],
462+
"source": []
463+
},
464+
{
465+
"cell_type": "code",
466+
"execution_count": 43,
467+
"metadata": {},
468+
"outputs": [
469+
{
470+
"name": "stdout",
471+
"output_type": "stream",
472+
"text": [
473+
"\u001b[0;31mWe CAN NOT reject the null-hypothesis, as the F_F value (0.359) is less than the critical value (2.79)\n",
474+
"\u001b[1mThat means that there are not statistically differences between classifiers\u001b[0m\n",
475+
"\u001b[0;31mWe CAN NOT reject the null-hypothesis, as the F_F value (1.84) is less than the critical value (2.79)\n",
476+
"\u001b[1mThat means that there are not statistically differences between classifiers\u001b[0m\n",
477+
"\u001b[0;32mWe reject the null-hypothesis, as the F_F value (3.15) is greater than the critical value (2.79)\n",
478+
"\u001b[1mThat means that there are statistically differences between classifiers\u001b[0m\n",
479+
"\u001b[0;31mWe CAN NOT reject the null-hypothesis, as the F_F value (1.12) is less than the critical value (2.79)\n",
480+
"\u001b[1mThat means that there are not statistically differences between classifiers\u001b[0m\n",
481+
"\u001b[0;31mWe CAN NOT reject the null-hypothesis, as the F_F value (0.277) is less than the critical value (2.79)\n",
482+
"\u001b[1mThat means that there are not statistically differences between classifiers\u001b[0m\n",
483+
"\u001b[0;32mWe reject the null-hypothesis, as the F_F value (3.4) is greater than the critical value (2.79)\n",
484+
"\u001b[1mThat means that there are statistically differences between classifiers\u001b[0m\n",
485+
"\u001b[0;31mWe CAN NOT reject the null-hypothesis, as the F_F value (0.452) is less than the critical value (2.79)\n",
486+
"\u001b[1mThat means that there are not statistically differences between classifiers\u001b[0m\n",
487+
"\u001b[0;31mWe CAN NOT reject the null-hypothesis, as the F_F value (0.475) is less than the critical value (2.79)\n",
488+
"\u001b[1mThat means that there are not statistically differences between classifiers\u001b[0m\n",
489+
"\u001b[0;31mWe CAN NOT reject the null-hypothesis, as the F_F value (0.106) is less than the critical value (2.79)\n",
490+
"\u001b[1mThat means that there are not statistically differences between classifiers\u001b[0m\n",
491+
"\u001b[0;31mWe CAN NOT reject the null-hypothesis, as the F_F value (0.814) is less than the critical value (2.79)\n",
492+
"\u001b[1mThat means that there are not statistically differences between classifiers\u001b[0m\n",
493+
"\u001b[0;31mWe CAN NOT reject the null-hypothesis, as the F_F value (2.56) is less than the critical value (2.79)\n",
494+
"\u001b[1mThat means that there are not statistically differences between classifiers\u001b[0m\n",
495+
"\u001b[0;31mWe CAN NOT reject the null-hypothesis, as the F_F value (0.277) is less than the critical value (2.79)\n",
496+
"\u001b[1mThat means that there are not statistically differences between classifiers\u001b[0m\n",
497+
"\u001b[0;31mWe CAN NOT reject the null-hypothesis, as the F_F value (0.321) is less than the critical value (2.79)\n",
498+
"\u001b[1mThat means that there are not statistically differences between classifiers\u001b[0m\n",
499+
"\u001b[0;31mWe CAN NOT reject the null-hypothesis, as the F_F value (1.19) is less than the critical value (2.79)\n",
500+
"\u001b[1mThat means that there are not statistically differences between classifiers\u001b[0m\n",
501+
"\u001b[0;31mWe CAN NOT reject the null-hypothesis, as the F_F value (0.906) is less than the critical value (2.79)\n",
502+
"\u001b[1mThat means that there are not statistically differences between classifiers\u001b[0m\n",
503+
"\u001b[0;31mWe CAN NOT reject the null-hypothesis, as the F_F value (0.452) is less than the critical value (2.79)\n",
504+
"\u001b[1mThat means that there are not statistically differences between classifiers\u001b[0m\n",
505+
"\u001b[0;31mWe CAN NOT reject the null-hypothesis, as the F_F value (1.19) is less than the critical value (2.79)\n",
506+
"\u001b[1mThat means that there are not statistically differences between classifiers\u001b[0m\n",
507+
"\u001b[0;31mWe CAN NOT reject the null-hypothesis, as the F_F value (1.71) is less than the critical value (2.79)\n",
508+
"\u001b[1mThat means that there are not statistically differences between classifiers\u001b[0m\n",
509+
"\u001b[0;31mWe CAN NOT reject the null-hypothesis, as the F_F value (0.25) is less than the critical value (2.79)\n",
510+
"\u001b[1mThat means that there are not statistically differences between classifiers\u001b[0m\n",
511+
"\u001b[0;31mWe CAN NOT reject the null-hypothesis, as the F_F value (0.364) is less than the critical value (2.79)\n",
512+
"\u001b[1mThat means that there are not statistically differences between classifiers\u001b[0m\n",
513+
"\u001b[0;31mWe CAN NOT reject the null-hypothesis, as the F_F value (2.21) is less than the critical value (2.79)\n",
514+
"\u001b[1mThat means that there are not statistically differences between classifiers\u001b[0m\n"
515+
]
402516
}
403-
}
517+
],
518+
"source": [
519+
"# Using critical values\n",
520+
"# FROM: https://stackoverflow.com/questions/39813470/f-test-with-python-finding-the-critical-value\n",
521+
"# SEE ALSO: https://www.statology.org/f-critical-value-python/\n",
522+
"alpha = 0.05\n",
523+
"for percent, F_F, average_rank_for, N, k in data:\n",
524+
" critical_value = scipy.stats.f.ppf(q=1-alpha, dfn=(k-1), dfd=(k-1)*(N-1))\n",
525+
"\n",
526+
" if F_F > critical_value:\n",
527+
" print(f\"{GRN}We reject the null-hypothesis, as the F_F value ({F_F:.3}) is greater than the critical value ({critical_value:.3})\")\n",
528+
" print(f\"{BLD}That means that there are statistically differences between classifiers{NC}\")\n",
529+
" else:\n",
530+
" print(f\"{RED}We CAN NOT reject the null-hypothesis, as the F_F value ({F_F:.3}) is less than the critical value ({critical_value:.3})\")\n",
531+
" print(f\"{BLD}That means that there are not statistically differences between classifiers{NC}\")\n",
532+
" #print(f\"Or that the test is not powerfull enough to detect the differences\")\n"
533+
]
404534
}
405535
],
406536
"metadata": {
537+
"interpreter": {
538+
"hash": "ddbf58beb0ceb3f28487c8a8d7192b043471fe4d33849d20361912ddb46861bb"
539+
},
407540
"kernelspec": {
408-
"display_name": "Python 3",
541+
"display_name": "Python 3.9.12 ('IS-SSL')",
409542
"language": "python",
410543
"name": "python3"
411544
},
412545
"language_info": {
413546
"codemirror_mode": {
414547
"name": "ipython",
415-
"version": 2
548+
"version": 3
416549
},
417550
"file_extension": ".py",
418551
"mimetype": "text/x-python",
419552
"name": "python",
420553
"nbconvert_exporter": "python",
421-
"pygments_lexer": "ipython2",
422-
"version": "2.7.6"
554+
"pygments_lexer": "ipython3",
555+
"version": "3.9.12"
423556
}
424557
},
425558
"nbformat": 4,
426559
"nbformat_minor": 0
427-
}
560+
}

0 commit comments

Comments
 (0)