|
5 | 5 | "id": "62e70a2a",
|
6 | 6 | "metadata": {},
|
7 | 7 | "source": [
|
8 |
| - "# Data" |
| 8 | + "# Data\n", |
| 9 | + "\n", |
| 10 | + "For the illustration of the group fairness metrics in TrustyAI, two synthetic datasets were created with the same input features and outcome types. \n", |
| 11 | + "The outcome is whether a certain invidual reaches a 50k income threshold by using age, race and gender as categorical inputs and both datasets consist of $N=10000$ data points.\n", |
| 12 | + "The gender values are allocated with a proportion of 20% to `gender=0` and 80% to `gender=1`.\n", |
| 13 | + "\n", |
| 14 | + "Both datasets have an increasing likelihood (with uniform probability) of having a positive outcome with age, regardless of race or gender.\n", |
| 15 | + "The first dataset, deemed _unbiased_, simply allocates the income value with an uniform random value, regardless of race or gender.\n", |
| 16 | + "The second dataset, deemed _biased_, allocates a positive outcome to `gender=0` with a lower probability than `gender=1`." |
9 | 17 | ]
|
10 | 18 | },
|
11 | 19 | {
|
|
202 | 210 | "where $\\hat{y}=1$ is the favorable outcome and $\\mathcal{D}_u$, $\\mathcal{D}_p$ are respectively the privileged and unpriviledge group data.\n",
|
203 | 211 | "\n",
|
204 | 212 | "\n",
|
205 |
| - "### Not so biased dataset" |
| 213 | + "### Unbiased dataset" |
206 | 214 | ]
|
207 | 215 | },
|
208 | 216 | {
|
|
265 | 273 | },
|
266 | 274 | {
|
267 | 275 | "cell_type": "code",
|
268 |
| - "execution_count": 7, |
| 276 | + "execution_count": 24, |
269 | 277 | "id": "2b2c678a",
|
270 | 278 | "metadata": {},
|
271 | 279 | "outputs": [],
|
|
283 | 291 | },
|
284 | 292 | {
|
285 | 293 | "cell_type": "code",
|
286 |
| - "execution_count": 8, |
| 294 | + "execution_count": 7, |
287 | 295 | "id": "9e548018",
|
288 | 296 | "metadata": {},
|
289 | 297 | "outputs": [
|
|
317 | 325 | },
|
318 | 326 | {
|
319 | 327 | "cell_type": "code",
|
320 |
| - "execution_count": 9, |
| 328 | + "execution_count": 8, |
321 | 329 | "id": "63b953c9",
|
322 | 330 | "metadata": {},
|
323 | 331 | "outputs": [
|
|
332 | 340 | "Name: income, dtype: int64"
|
333 | 341 | ]
|
334 | 342 | },
|
335 |
| - "execution_count": 9, |
| 343 | + "execution_count": 8, |
336 | 344 | "metadata": {},
|
337 | 345 | "output_type": "execute_result"
|
338 | 346 | }
|
|
344 | 352 | },
|
345 | 353 | {
|
346 | 354 | "cell_type": "code",
|
347 |
| - "execution_count": 10, |
| 355 | + "execution_count": 9, |
348 | 356 | "id": "aed61b77",
|
349 | 357 | "metadata": {},
|
350 | 358 | "outputs": [
|
|
354 | 362 | "<AxesSubplot:xlabel='gender'>"
|
355 | 363 | ]
|
356 | 364 | },
|
357 |
| - "execution_count": 10, |
| 365 | + "execution_count": 9, |
358 | 366 | "metadata": {},
|
359 | 367 | "output_type": "execute_result"
|
360 | 368 | },
|
|
375 | 383 | },
|
376 | 384 | {
|
377 | 385 | "cell_type": "code",
|
378 |
| - "execution_count": 11, |
| 386 | + "execution_count": 10, |
379 | 387 | "id": "901e5720",
|
380 | 388 | "metadata": {},
|
381 | 389 | "outputs": [],
|
|
390 | 398 | },
|
391 | 399 | {
|
392 | 400 | "cell_type": "code",
|
393 |
| - "execution_count": 12, |
| 401 | + "execution_count": 11, |
394 | 402 | "id": "7be544a7",
|
395 | 403 | "metadata": {},
|
396 | 404 | "outputs": [
|
|
438 | 446 | },
|
439 | 447 | {
|
440 | 448 | "cell_type": "code",
|
441 |
| - "execution_count": 13, |
| 449 | + "execution_count": 12, |
442 | 450 | "id": "949fae2f",
|
443 | 451 | "metadata": {},
|
444 | 452 | "outputs": [],
|
|
452 | 460 | },
|
453 | 461 | {
|
454 | 462 | "cell_type": "code",
|
455 |
| - "execution_count": 14, |
| 463 | + "execution_count": 13, |
456 | 464 | "id": "2e601762",
|
457 | 465 | "metadata": {},
|
458 | 466 | "outputs": [
|
|
478 | 486 | },
|
479 | 487 | {
|
480 | 488 | "cell_type": "code",
|
481 |
| - "execution_count": 15, |
| 489 | + "execution_count": 14, |
482 | 490 | "id": "3231326d",
|
483 | 491 | "metadata": {},
|
484 | 492 | "outputs": [],
|
|
490 | 498 | },
|
491 | 499 | {
|
492 | 500 | "cell_type": "code",
|
493 |
| - "execution_count": 16, |
| 501 | + "execution_count": 15, |
494 | 502 | "id": "4b88eec8",
|
495 | 503 | "metadata": {},
|
496 | 504 | "outputs": [
|
|
536 | 544 | },
|
537 | 545 | {
|
538 | 546 | "cell_type": "code",
|
539 |
| - "execution_count": 22, |
| 547 | + "execution_count": 16, |
540 | 548 | "id": "50814c63",
|
541 | 549 | "metadata": {},
|
542 | 550 | "outputs": [],
|
|
552 | 560 | },
|
553 | 561 | {
|
554 | 562 | "cell_type": "code",
|
555 |
| - "execution_count": 23, |
| 563 | + "execution_count": 17, |
556 | 564 | "id": "cedbd492",
|
557 | 565 | "metadata": {},
|
558 | 566 | "outputs": [
|
|
581 | 589 | "id": "74cf44c7",
|
582 | 590 | "metadata": {},
|
583 | 591 | "source": [
|
584 |
| - "# Average Predictive Value Difference" |
| 592 | + "## Average Predictive Value Difference\n", |
| 593 | + "\n", |
| 594 | + "The _Average Predictive Value Difference (APVD)_ measures the difference in the average accuracy of predicted values between the privileged and unprivileged groups in a dataset. \n" |
585 | 595 | ]
|
586 | 596 | },
|
587 | 597 | {
|
588 | 598 | "cell_type": "code",
|
589 |
| - "execution_count": 21, |
| 599 | + "execution_count": 18, |
590 | 600 | "id": "ca591646",
|
591 | 601 | "metadata": {},
|
592 | 602 | "outputs": [
|
|
603 | 613 | "\n",
|
604 | 614 | "score = average_predictive_value_difference(test=bias,\n",
|
605 | 615 | " truth=nobias,\n",
|
606 |
| - " privilege_columns=[2],\n", |
| 616 | + " privilege_columns=[\"gender\"],\n", |
607 | 617 | " privilege_values=[1],\n",
|
608 | 618 | " positive_class=[1])\n",
|
609 | 619 | "print(score)"
|
610 | 620 | ]
|
611 | 621 | },
|
612 |
| - { |
613 |
| - "cell_type": "code", |
614 |
| - "execution_count": 22, |
615 |
| - "id": "611a0a6f", |
616 |
| - "metadata": {}, |
617 |
| - "outputs": [ |
618 |
| - { |
619 |
| - "name": "stdout", |
620 |
| - "output_type": "stream", |
621 |
| - "text": [ |
622 |
| - "-0.052866818520044656\n" |
623 |
| - ] |
624 |
| - } |
625 |
| - ], |
626 |
| - "source": [ |
627 |
| - "score = average_predictive_value_difference(test=bias,\n", |
628 |
| - " truth=nobias,\n", |
629 |
| - " privilege_columns=[2],\n", |
630 |
| - " privilege_values=[0],\n", |
631 |
| - " positive_class=[1])\n", |
632 |
| - "print(score)" |
633 |
| - ] |
634 |
| - }, |
635 | 622 | {
|
636 | 623 | "cell_type": "markdown",
|
637 | 624 | "id": "b85694b4",
|
|
650 | 637 | },
|
651 | 638 | {
|
652 | 639 | "cell_type": "code",
|
653 |
| - "execution_count": 23, |
| 640 | + "execution_count": 19, |
654 | 641 | "id": "e1a03db7",
|
655 | 642 | "metadata": {},
|
656 | 643 | "outputs": [],
|
|
660 | 647 | },
|
661 | 648 | {
|
662 | 649 | "cell_type": "code",
|
663 |
| - "execution_count": 24, |
| 650 | + "execution_count": 20, |
664 | 651 | "id": "da1c2775",
|
665 | 652 | "metadata": {},
|
666 | 653 | "outputs": [],
|
|
692 | 679 | "model = Model(xgb.predict, dataframe_input=True, output_names=[\"approved\"])\n",
|
693 | 680 | "score = statistical_parity_difference_model(samples=X,\n",
|
694 | 681 | " model=model,\n",
|
695 |
| - " privilege_columns=[2],\n", |
| 682 | + " privilege_columns=[\"gender\"],\n", |
696 | 683 | " privilege_values=[1],\n",
|
697 | 684 | " favorable=[favorable])\n",
|
698 | 685 | "print(score)"
|
|
725 | 712 | "\n",
|
726 | 713 | "score = disparate_impact_ratio_model(samples=X,\n",
|
727 | 714 | " model=model,\n",
|
728 |
| - " privilege_columns=[2],\n", |
| 715 | + " privilege_columns=[\"gender\"],\n", |
729 | 716 | " privilege_values=[1],\n",
|
730 | 717 | " favorable=[favorable])\n",
|
731 | 718 | "print(score)"
|
|
758 | 745 | "\n",
|
759 | 746 | "score = average_odds_difference_model(samples=X,\n",
|
760 | 747 | " model=model,\n",
|
761 |
| - " privilege_columns=[2],\n", |
| 748 | + " privilege_columns=[\"gender\"],\n", |
762 | 749 | " privilege_values=[1],\n",
|
763 | 750 | " positive_class=[1])\n",
|
764 | 751 | "print(score)"
|
|
0 commit comments