Skip to content

Commit 5eabe2a

Browse files
committed
Add APVD text
1 parent 9f8a784 commit 5eabe2a

File tree

1 file changed

+34
-47
lines changed

1 file changed

+34
-47
lines changed

examples/Group fairness metrics.ipynb

Lines changed: 34 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,15 @@
55
"id": "62e70a2a",
66
"metadata": {},
77
"source": [
8-
"# Data"
8+
"# Data\n",
9+
"\n",
10+
"For the illustration of the group fairness metrics in TrustyAI, two synthetic datasets were created with the same input features and outcome types. \n",
11+
"The outcome is whether a certain invidual reaches a 50k income threshold by using age, race and gender as categorical inputs and both datasets consist of $N=10000$ data points.\n",
12+
"The gender values are allocated with a proportion of 20% to `gender=0` and 80% to `gender=1`.\n",
13+
"\n",
14+
"Both datasets have an increasing likelihood (with uniform probability) of having a positive outcome with age, regardless of race or gender.\n",
15+
"The first dataset, deemed _unbiased_, simply allocates the income value with an uniform random value, regardless of race or gender.\n",
16+
"The second dataset, deemed _biased_, allocates a positive outcome to `gender=0` with a lower probability than `gender=1`."
917
]
1018
},
1119
{
@@ -202,7 +210,7 @@
202210
"where $\\hat{y}=1$ is the favorable outcome and $\\mathcal{D}_u$, $\\mathcal{D}_p$ are respectively the privileged and unpriviledge group data.\n",
203211
"\n",
204212
"\n",
205-
"### Not so biased dataset"
213+
"### Unbiased dataset"
206214
]
207215
},
208216
{
@@ -265,7 +273,7 @@
265273
},
266274
{
267275
"cell_type": "code",
268-
"execution_count": 7,
276+
"execution_count": 24,
269277
"id": "2b2c678a",
270278
"metadata": {},
271279
"outputs": [],
@@ -283,7 +291,7 @@
283291
},
284292
{
285293
"cell_type": "code",
286-
"execution_count": 8,
294+
"execution_count": 7,
287295
"id": "9e548018",
288296
"metadata": {},
289297
"outputs": [
@@ -317,7 +325,7 @@
317325
},
318326
{
319327
"cell_type": "code",
320-
"execution_count": 9,
328+
"execution_count": 8,
321329
"id": "63b953c9",
322330
"metadata": {},
323331
"outputs": [
@@ -332,7 +340,7 @@
332340
"Name: income, dtype: int64"
333341
]
334342
},
335-
"execution_count": 9,
343+
"execution_count": 8,
336344
"metadata": {},
337345
"output_type": "execute_result"
338346
}
@@ -344,7 +352,7 @@
344352
},
345353
{
346354
"cell_type": "code",
347-
"execution_count": 10,
355+
"execution_count": 9,
348356
"id": "aed61b77",
349357
"metadata": {},
350358
"outputs": [
@@ -354,7 +362,7 @@
354362
"<AxesSubplot:xlabel='gender'>"
355363
]
356364
},
357-
"execution_count": 10,
365+
"execution_count": 9,
358366
"metadata": {},
359367
"output_type": "execute_result"
360368
},
@@ -375,7 +383,7 @@
375383
},
376384
{
377385
"cell_type": "code",
378-
"execution_count": 11,
386+
"execution_count": 10,
379387
"id": "901e5720",
380388
"metadata": {},
381389
"outputs": [],
@@ -390,7 +398,7 @@
390398
},
391399
{
392400
"cell_type": "code",
393-
"execution_count": 12,
401+
"execution_count": 11,
394402
"id": "7be544a7",
395403
"metadata": {},
396404
"outputs": [
@@ -438,7 +446,7 @@
438446
},
439447
{
440448
"cell_type": "code",
441-
"execution_count": 13,
449+
"execution_count": 12,
442450
"id": "949fae2f",
443451
"metadata": {},
444452
"outputs": [],
@@ -452,7 +460,7 @@
452460
},
453461
{
454462
"cell_type": "code",
455-
"execution_count": 14,
463+
"execution_count": 13,
456464
"id": "2e601762",
457465
"metadata": {},
458466
"outputs": [
@@ -478,7 +486,7 @@
478486
},
479487
{
480488
"cell_type": "code",
481-
"execution_count": 15,
489+
"execution_count": 14,
482490
"id": "3231326d",
483491
"metadata": {},
484492
"outputs": [],
@@ -490,7 +498,7 @@
490498
},
491499
{
492500
"cell_type": "code",
493-
"execution_count": 16,
501+
"execution_count": 15,
494502
"id": "4b88eec8",
495503
"metadata": {},
496504
"outputs": [
@@ -536,7 +544,7 @@
536544
},
537545
{
538546
"cell_type": "code",
539-
"execution_count": 22,
547+
"execution_count": 16,
540548
"id": "50814c63",
541549
"metadata": {},
542550
"outputs": [],
@@ -552,7 +560,7 @@
552560
},
553561
{
554562
"cell_type": "code",
555-
"execution_count": 23,
563+
"execution_count": 17,
556564
"id": "cedbd492",
557565
"metadata": {},
558566
"outputs": [
@@ -581,12 +589,14 @@
581589
"id": "74cf44c7",
582590
"metadata": {},
583591
"source": [
584-
"# Average Predictive Value Difference"
592+
"## Average Predictive Value Difference\n",
593+
"\n",
594+
"The _Average Predictive Value Difference (APVD)_ measures the difference in the average accuracy of predicted values between the privileged and unprivileged groups in a dataset. \n"
585595
]
586596
},
587597
{
588598
"cell_type": "code",
589-
"execution_count": 21,
599+
"execution_count": 18,
590600
"id": "ca591646",
591601
"metadata": {},
592602
"outputs": [
@@ -603,35 +613,12 @@
603613
"\n",
604614
"score = average_predictive_value_difference(test=bias,\n",
605615
" truth=nobias,\n",
606-
" privilege_columns=[2],\n",
616+
" privilege_columns=[\"gender\"],\n",
607617
" privilege_values=[1],\n",
608618
" positive_class=[1])\n",
609619
"print(score)"
610620
]
611621
},
612-
{
613-
"cell_type": "code",
614-
"execution_count": 22,
615-
"id": "611a0a6f",
616-
"metadata": {},
617-
"outputs": [
618-
{
619-
"name": "stdout",
620-
"output_type": "stream",
621-
"text": [
622-
"-0.052866818520044656\n"
623-
]
624-
}
625-
],
626-
"source": [
627-
"score = average_predictive_value_difference(test=bias,\n",
628-
" truth=nobias,\n",
629-
" privilege_columns=[2],\n",
630-
" privilege_values=[0],\n",
631-
" positive_class=[1])\n",
632-
"print(score)"
633-
]
634-
},
635622
{
636623
"cell_type": "markdown",
637624
"id": "b85694b4",
@@ -650,7 +637,7 @@
650637
},
651638
{
652639
"cell_type": "code",
653-
"execution_count": 23,
640+
"execution_count": 19,
654641
"id": "e1a03db7",
655642
"metadata": {},
656643
"outputs": [],
@@ -660,7 +647,7 @@
660647
},
661648
{
662649
"cell_type": "code",
663-
"execution_count": 24,
650+
"execution_count": 20,
664651
"id": "da1c2775",
665652
"metadata": {},
666653
"outputs": [],
@@ -692,7 +679,7 @@
692679
"model = Model(xgb.predict, dataframe_input=True, output_names=[\"approved\"])\n",
693680
"score = statistical_parity_difference_model(samples=X,\n",
694681
" model=model,\n",
695-
" privilege_columns=[2],\n",
682+
" privilege_columns=[\"gender\"],\n",
696683
" privilege_values=[1],\n",
697684
" favorable=[favorable])\n",
698685
"print(score)"
@@ -725,7 +712,7 @@
725712
"\n",
726713
"score = disparate_impact_ratio_model(samples=X,\n",
727714
" model=model,\n",
728-
" privilege_columns=[2],\n",
715+
" privilege_columns=[\"gender\"],\n",
729716
" privilege_values=[1],\n",
730717
" favorable=[favorable])\n",
731718
"print(score)"
@@ -758,7 +745,7 @@
758745
"\n",
759746
"score = average_odds_difference_model(samples=X,\n",
760747
" model=model,\n",
761-
" privilege_columns=[2],\n",
748+
" privilege_columns=[\"gender\"],\n",
762749
" privilege_values=[1],\n",
763750
" positive_class=[1])\n",
764751
"print(score)"

0 commit comments

Comments
 (0)