Merge pull request #1310 from abigailgold/dev_1.8.0_notebook_fix

Fix error in membership attack notebook

Author: beat-buesser

notebooks/attack_membership_inference.ipynb

@@ -30,7 +30,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -52,14 +52,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Base model accuracy:  0.9731398579808583\n"
+      "Base model accuracy:  0.9739117011423278\n"
      ]
     }
    ],
@@ -86,16 +86,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
       "1.0\n",
-      "0.026860142019141664\n",
-      "0.5134300710095708\n"
+      "0.026088298857672165\n",
+      "0.513044149428836\n"
      ]
     }
    ],
@@ -127,14 +127,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "(0.5068064465654827, 1.0)\n"
+      "(0.50660827402831, 1.0)\n"
      ]
     }
    ],
@@ -179,7 +179,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -205,16 +205,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
-   "metadata": {},
+   "execution_count": 6,
+   "metadata": {
+    "scrolled": true
+   },
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "0.7313985798085829\n",
-      "0.5541833899351651\n",
-      "0.642790984871874\n"
+      "0.6736647113306576\n",
+      "0.6375424513738808\n",
+      "0.6556035813522693\n"
      ]
     }
    ],
@@ -235,19 +237,19 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Acheives slightly better results than the rule-based attack."
+    "Acheives much better results than the rule-based attack."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "(0.6212955677943877, 0.7313985798085829)\n"
+      "(0.6501787842669845, 0.6736647113306576)\n"
      ]
     }
    ],
@@ -266,14 +268,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Base model accuracy:  0.9668107440568077\n"
+      "Base model accuracy:  0.926\n"
      ]
     }
    ],
@@ -284,6 +286,15 @@
     "from torch.utils.data.dataset import Dataset\n",
     "from art.estimators.classification.pytorch import PyTorchClassifier\n",
     "\n",
+    "# reduce size of training set to make attack slightly better\n",
+    "train_set_size = 500\n",
+    "x_train = x_train[:train_set_size]\n",
+    "y_train = y_train[:train_set_size]\n",
+    "x_test = x_test[:train_set_size]\n",
+    "y_test = y_test[:train_set_size]\n",
+    "attack_train_size = int(len(x_train) * attack_train_ratio)\n",
+    "attack_test_size = int(len(x_test) * attack_train_ratio)\n",
+    "\n",
     "class ModelToAttack(nn.Module):\n",
     "\n",
     "    def __init__(self, num_classes, num_features):\n",
@@ -297,18 +308,28 @@
     "                nn.Linear(1024, 512),\n",
     "                nn.Tanh(), )\n",
     "\n",
-    "        self.classifier = nn.Linear(512, num_classes)\n",
-    "        # self.softmax = nn.Softmax(dim=1)\n",
+    "        self.fc3 = nn.Sequential(\n",
+    "            nn.Linear(512, 256),\n",
+    "            nn.Tanh(), )\n",
+    "        \n",
+    "        self.fc4 = nn.Sequential(\n",
+    "            nn.Linear(256, 128),\n",
+    "            nn.Tanh(),\n",
+    "        )\n",
+    "\n",
+    "        self.classifier = nn.Linear(128, num_classes)\n",
     "\n",
     "    def forward(self, x):\n",
     "        out = self.fc1(x)\n",
     "        out = self.fc2(out)\n",
+    "        out = self.fc3(out)\n",
+    "        out = self.fc4(out)\n",
     "        return self.classifier(out)\n",
     "\n",
     "mlp_model = ModelToAttack(4, 24)\n",
     "mlp_model = torch.nn.DataParallel(mlp_model)\n",
     "criterion = nn.CrossEntropyLoss()\n",
-    "optimizer = optim.Adam(mlp_model.parameters(), lr=0.0001)\n",
+    "optimizer = optim.Adam(mlp_model.parameters(), lr=0.01)\n",
     "\n",
     "class NurseryDataset(Dataset):\n",
     "    def __init__(self, x, y=None):\n",
@@ -358,17 +379,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 18,
+   "execution_count": 12,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "0.9786971287434393\n",
-      "0.03318925594319233\n",
-      "0.5059431923433159\n",
-      "(0.5030548282155043, 0.9786971287434393)\n"
+      "0.998\n",
+      "0.07399999999999995\n",
+      "0.536\n",
+      "(0.5187110187110187, 0.998)\n"
      ]
     }
    ],
@@ -400,17 +421,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 19,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "0.7488422352577956\n",
-      "0.7472985489348565\n",
-      "0.748070392096326\n",
-      "(0.7476880394574599, 0.7488422352577956)\n"
+      "0.608\n",
+      "0.5680000000000001\n",
+      "0.588\n",
+      "(0.5846153846153846, 0.608)\n"
      ]
     }
    ],
@@ -422,8 +443,8 @@
     "              x_test[:attack_test_size].astype(np.float32), y_test[:attack_test_size])\n",
     "\n",
     "# infer \n",
-    "mlp_inferred_train_bb = mlp_attack_bb.infer(x_train.astype(np.float32), y_train)\n",
-    "mlp_inferred_test_bb = mlp_attack_bb.infer(x_test.astype(np.float32), y_test)\n",
+    "mlp_inferred_train_bb = mlp_attack_bb.infer(x_train[attack_train_size:].astype(np.float32), y_train[attack_train_size:])\n",
+    "mlp_inferred_test_bb = mlp_attack_bb.infer(x_test[attack_test_size:].astype(np.float32), y_test[attack_test_size:])\n",
     "\n",
     "# check accuracy\n",
     "mlp_train_acc_bb = np.sum(mlp_inferred_train_bb) / len(mlp_inferred_train_bb)\n",
@@ -441,7 +462,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Using a random forest as the attack model we were able to acheive better performance than the rule-based attack, both in terms of accuracy and precision."
+    "For the pytorch target model we were able to acheive slightly better than random attack performance, but not as good as for the random forest model."
    ]
   }
  ],
@@ -461,7 +482,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.1"
+   "version": "3.8.3"
   }
  },
  "nbformat": 4,