Skip to content

Commit 9e28ed6

Browse files
emmanueljordyemmanueljordy
authored
Merge pull request #12 from NGO-Algorithm-Audit/main
update data processing
2 parents 70e6d01 + d9ef00f commit 9e28ed6

File tree

6 files changed

+1675
-244
lines changed

6 files changed

+1675
-244
lines changed

‎example_notebooks/00_readme.ipynb‎

Lines changed: 61 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -388,46 +388,46 @@
388388
" <tbody>\n",
389389
" <tr>\n",
390390
" <th>0</th>\n",
391-
" <td>0</td>\n",
392-
" <td>26.155148</td>\n",
391+
" <td>1</td>\n",
392+
" <td>27.000861</td>\n",
393+
" <td>3</td>\n",
394+
" <td>5940.743483</td>\n",
393395
" <td>4</td>\n",
394-
" <td>-8.000000</td>\n",
395-
" <td>2</td>\n",
396396
" <td>0</td>\n",
397397
" </tr>\n",
398398
" <tr>\n",
399399
" <th>1</th>\n",
400400
" <td>1</td>\n",
401-
" <td>37.983907</td>\n",
401+
" <td>33.685012</td>\n",
402402
" <td>3</td>\n",
403-
" <td>1584.939480</td>\n",
403+
" <td>1747.087586</td>\n",
404404
" <td>4</td>\n",
405405
" <td>1</td>\n",
406406
" </tr>\n",
407407
" <tr>\n",
408408
" <th>2</th>\n",
409409
" <td>0</td>\n",
410-
" <td>67.653086</td>\n",
410+
" <td>76.212963</td>\n",
411411
" <td>5</td>\n",
412-
" <td>1067.917131</td>\n",
412+
" <td>955.997763</td>\n",
413413
" <td>2</td>\n",
414414
" <td>0</td>\n",
415415
" </tr>\n",
416416
" <tr>\n",
417417
" <th>3</th>\n",
418-
" <td>0</td>\n",
419-
" <td>48.066803</td>\n",
418+
" <td>1</td>\n",
419+
" <td>36.493914</td>\n",
420420
" <td>3</td>\n",
421-
" <td>1892.519382</td>\n",
421+
" <td>1783.896317</td>\n",
422422
" <td>4</td>\n",
423-
" <td>0</td>\n",
423+
" <td>1</td>\n",
424424
" </tr>\n",
425425
" <tr>\n",
426426
" <th>4</th>\n",
427427
" <td>0</td>\n",
428-
" <td>53.574285</td>\n",
429-
" <td>5</td>\n",
430-
" <td>1114.231039</td>\n",
428+
" <td>54.019538</td>\n",
429+
" <td>3</td>\n",
430+
" <td>791.422965</td>\n",
431431
" <td>4</td>\n",
432432
" <td>0</td>\n",
433433
" </tr>\n",
@@ -437,11 +437,11 @@
437437
],
438438
"text/plain": [
439439
" sex age marital income ls smoke\n",
440-
"0 0 26.155148 4 -8.000000 2 0\n",
441-
"1 1 37.983907 3 1584.939480 4 1\n",
442-
"2 0 67.653086 5 1067.917131 2 0\n",
443-
"3 0 48.066803 3 1892.519382 4 0\n",
444-
"4 0 53.574285 5 1114.231039 4 0"
440+
"0 1 27.000861 3 5940.743483 4 0\n",
441+
"1 1 33.685012 3 1747.087586 4 1\n",
442+
"2 0 76.212963 5 955.997763 2 0\n",
443+
"3 1 36.493914 3 1783.896317 4 1\n",
444+
"4 0 54.019538 3 791.422965 4 0"
445445
]
446446
},
447447
"metadata": {},
@@ -499,46 +499,46 @@
499499
" <tbody>\n",
500500
" <tr>\n",
501501
" <th>0</th>\n",
502-
" <td>FEMALE</td>\n",
503-
" <td>26.155148</td>\n",
504-
" <td>SINGLE</td>\n",
505-
" <td>-8.000000</td>\n",
506-
" <td>MOSTLY DISSATISFIED</td>\n",
502+
" <td>MALE</td>\n",
503+
" <td>27.000861</td>\n",
504+
" <td>MARRIED</td>\n",
505+
" <td>5940.743483</td>\n",
506+
" <td>PLEASED</td>\n",
507507
" <td>NO</td>\n",
508508
" </tr>\n",
509509
" <tr>\n",
510510
" <th>1</th>\n",
511511
" <td>MALE</td>\n",
512-
" <td>37.983907</td>\n",
512+
" <td>33.685012</td>\n",
513513
" <td>MARRIED</td>\n",
514-
" <td>1584.939480</td>\n",
514+
" <td>1747.087586</td>\n",
515515
" <td>PLEASED</td>\n",
516516
" <td>YES</td>\n",
517517
" </tr>\n",
518518
" <tr>\n",
519519
" <th>2</th>\n",
520520
" <td>FEMALE</td>\n",
521-
" <td>67.653086</td>\n",
521+
" <td>76.212963</td>\n",
522522
" <td>WIDOWED</td>\n",
523-
" <td>1067.917131</td>\n",
523+
" <td>955.997763</td>\n",
524524
" <td>MOSTLY DISSATISFIED</td>\n",
525525
" <td>NO</td>\n",
526526
" </tr>\n",
527527
" <tr>\n",
528528
" <th>3</th>\n",
529-
" <td>FEMALE</td>\n",
530-
" <td>48.066803</td>\n",
529+
" <td>MALE</td>\n",
530+
" <td>36.493914</td>\n",
531531
" <td>MARRIED</td>\n",
532-
" <td>1892.519382</td>\n",
532+
" <td>1783.896317</td>\n",
533533
" <td>PLEASED</td>\n",
534-
" <td>NO</td>\n",
534+
" <td>YES</td>\n",
535535
" </tr>\n",
536536
" <tr>\n",
537537
" <th>4</th>\n",
538538
" <td>FEMALE</td>\n",
539-
" <td>53.574285</td>\n",
540-
" <td>WIDOWED</td>\n",
541-
" <td>1114.231039</td>\n",
539+
" <td>54.019538</td>\n",
540+
" <td>MARRIED</td>\n",
541+
" <td>791.422965</td>\n",
542542
" <td>PLEASED</td>\n",
543543
" <td>NO</td>\n",
544544
" </tr>\n",
@@ -548,11 +548,11 @@
548548
],
549549
"text/plain": [
550550
" sex age marital income ls smoke\n",
551-
"0 FEMALE 26.155148 SINGLE -8.000000 MOSTLY DISSATISFIED NO\n",
552-
"1 MALE 37.983907 MARRIED 1584.939480 PLEASED YES\n",
553-
"2 FEMALE 67.653086 WIDOWED 1067.917131 MOSTLY DISSATISFIED NO\n",
554-
"3 FEMALE 48.066803 MARRIED 1892.519382 PLEASED NO\n",
555-
"4 FEMALE 53.574285 WIDOWED 1114.231039 PLEASED NO"
551+
"0 MALE 27.000861 MARRIED 5940.743483 PLEASED NO\n",
552+
"1 MALE 33.685012 MARRIED 1747.087586 PLEASED YES\n",
553+
"2 FEMALE 76.212963 WIDOWED 955.997763 MOSTLY DISSATISFIED NO\n",
554+
"3 MALE 36.493914 MARRIED 1783.896317 PLEASED YES\n",
555+
"4 FEMALE 54.019538 MARRIED 791.422965 PLEASED NO"
556556
]
557557
},
558558
"metadata": {},
@@ -633,7 +633,7 @@
633633
" <td>N/A</td>\n",
634634
" <td>N/A</td>\n",
635635
" <td>N/A</td>\n",
636-
" <td>0.9364</td>\n",
636+
" <td>0.9964</td>\n",
637637
" <td>N/A</td>\n",
638638
" <td>1.0</td>\n",
639639
" <td>1.0</td>\n",
@@ -643,11 +643,11 @@
643643
" <td>age</td>\n",
644644
" <td>numerical</td>\n",
645645
" <td>1.0</td>\n",
646-
" <td>0.934821</td>\n",
646+
" <td>0.885944</td>\n",
647647
" <td>1.0</td>\n",
648-
" <td>0.9368</td>\n",
648+
" <td>0.9342</td>\n",
649649
" <td>N/A</td>\n",
650-
" <td>0.969487</td>\n",
650+
" <td>0.966491</td>\n",
651651
" <td>N/A</td>\n",
652652
" <td>N/A</td>\n",
653653
" </tr>\n",
@@ -659,7 +659,7 @@
659659
" <td>N/A</td>\n",
660660
" <td>N/A</td>\n",
661661
" <td>N/A</td>\n",
662-
" <td>0.9574</td>\n",
662+
" <td>0.975</td>\n",
663663
" <td>N/A</td>\n",
664664
" <td>0.833333</td>\n",
665665
" <td>1.0</td>\n",
@@ -669,11 +669,11 @@
669669
" <td>income</td>\n",
670670
" <td>numerical</td>\n",
671671
" <td>1.0</td>\n",
672-
" <td>0.936471</td>\n",
672+
" <td>0.498955</td>\n",
673673
" <td>1.0</td>\n",
674-
" <td>0.9036</td>\n",
674+
" <td>0.8468</td>\n",
675675
" <td>N/A</td>\n",
676-
" <td>0.755353</td>\n",
676+
" <td>0.912639</td>\n",
677677
" <td>N/A</td>\n",
678678
" <td>N/A</td>\n",
679679
" </tr>\n",
@@ -685,7 +685,7 @@
685685
" <td>N/A</td>\n",
686686
" <td>N/A</td>\n",
687687
" <td>N/A</td>\n",
688-
" <td>0.9278</td>\n",
688+
" <td>0.9176</td>\n",
689689
" <td>N/A</td>\n",
690690
" <td>0.857143</td>\n",
691691
" <td>1.0</td>\n",
@@ -698,7 +698,7 @@
698698
" <td>N/A</td>\n",
699699
" <td>N/A</td>\n",
700700
" <td>N/A</td>\n",
701-
" <td>0.9854</td>\n",
701+
" <td>0.9746</td>\n",
702702
" <td>N/A</td>\n",
703703
" <td>1.0</td>\n",
704704
" <td>1.0</td>\n",
@@ -710,19 +710,19 @@
710710
"text/plain": [
711711
" column type missing_value_similarity range_coverage \\\n",
712712
"0 sex categorical 1.0 N/A \n",
713-
"1 age numerical 1.0 0.934821 \n",
713+
"1 age numerical 1.0 0.885944 \n",
714714
"2 marital categorical 1.0 N/A \n",
715-
"3 income numerical 1.0 0.936471 \n",
715+
"3 income numerical 1.0 0.498955 \n",
716716
"4 ls categorical 1.0 N/A \n",
717717
"5 smoke categorical 1.0 N/A \n",
718718
"\n",
719719
" boundary_adherence ks_complement tv_complement statistic_similarity \\\n",
720-
"0 N/A N/A 0.9364 N/A \n",
721-
"1 1.0 0.9368 N/A 0.969487 \n",
722-
"2 N/A N/A 0.9574 N/A \n",
723-
"3 1.0 0.9036 N/A 0.755353 \n",
724-
"4 N/A N/A 0.9278 N/A \n",
725-
"5 N/A N/A 0.9854 N/A \n",
720+
"0 N/A N/A 0.9964 N/A \n",
721+
"1 1.0 0.9342 N/A 0.966491 \n",
722+
"2 N/A N/A 0.975 N/A \n",
723+
"3 1.0 0.8468 N/A 0.912639 \n",
724+
"4 N/A N/A 0.9176 N/A \n",
725+
"5 N/A N/A 0.9746 N/A \n",
726726
"\n",
727727
" category_coverage category_adherence \n",
728728
"0 1.0 1.0 \n",
@@ -757,7 +757,7 @@
757757
"output_type": "stream",
758758
"text": [
759759
"=== Regression Efficacy Metrics ===\n",
760-
"{'mse': 1669726.6979087007, 'mae': 904.2202005090558, 'r2': -0.19619130295207743}\n"
760+
"{'mse': 1455890.3266689673, 'mae': 810.365268015888, 'r2': -0.04299904229518581}\n"
761761
]
762762
}
763763
],
@@ -782,7 +782,7 @@
782782
"text": [
783783
"\n",
784784
"=== Classification Efficacy Metrics ===\n",
785-
"{'accuracy': 0.6058, 'f1_score': 0.6184739077074358}\n"
785+
"{'accuracy': 0.64, 'f1_score': 0.6461333333333333}\n"
786786
]
787787
}
788788
],

0 commit comments

Comments
 (0)