Merge pull request #1097 from Trusted-AI/development_iris_162

Fix failing download of Iris dataset in tests

Author: beat-buesser

art/utils.py

@@ -844,22 +844,14 @@ def load_iris(raw: bool = False, test_set: float = 0.3) -> DATASET_TYPE:
     :param test_set: Proportion of the data to use as validation split. The value should be between 0 and 1.
     :return: Entire dataset and labels.
     """
-    # Download data if needed
-    path = get_file(
-        "iris.data",
-        path=config.ART_DATA_PATH,
-        extract=False,
-        url="https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data",
-    )
-
-    data = np.loadtxt(path, delimiter=",", usecols=(0, 1, 2, 3), dtype=config.ART_NUMPY_DTYPE)
-    labels = np.loadtxt(path, delimiter=",", usecols=4, dtype=str)
+    from sklearn.datasets import load_iris as load_iris_sk
 
-    # Preprocess
+    iris = load_iris_sk()
+    data = iris.data
     if not raw:
-        label_map = {"Iris-setosa": 0, "Iris-versicolor": 1, "Iris-virginica": 2}
-        labels = np.array([label_map[labels[i]] for i in range(labels.size)], dtype=np.int32)
-        data, labels = preprocess(data, labels, nb_classes=3)
+        data /= np.amax(data)
+    labels = to_categorical(iris.target, 3)
+
     min_, max_ = np.amin(data), np.amax(data)
 
     # Split training and test sets
@@ -883,7 +875,7 @@ def load_iris(raw: bool = False, test_set: float = 0.3) -> DATASET_TYPE:
                 data[50 + split_index : 100],
                 data[100 + split_index :],
             )
-        )
+        ).astype(np.float32)
         y_test = np.vstack(
             (
                 labels[split_index:50],
@@ -899,7 +891,7 @@ def load_iris(raw: bool = False, test_set: float = 0.3) -> DATASET_TYPE:
 
     # Shuffle training set
     random_indices = np.random.permutation(len(y_train))
-    x_train, y_train = x_train[random_indices], y_train[random_indices]
+    x_train, y_train = x_train[random_indices].astype(np.float32), y_train[random_indices]
 
     return (x_train, y_train), (x_test, y_test), min_, max_
 

tests/attacks/inference/attribute_inference/test_black_box.py

@@ -81,8 +81,8 @@ def transform_feature(x):
         # check accuracy
         train_acc = np.sum(inferred_train == x_train_feature.reshape(1, -1)) / len(inferred_train)
         test_acc = np.sum(inferred_test == x_test_feature.reshape(1, -1)) / len(inferred_test)
-        assert train_acc == pytest.approx(0.8285, abs=0.09)
-        assert test_acc == pytest.approx(0.8888, abs=0.09)
+        assert train_acc == pytest.approx(0.8285, abs=0.12)
+        assert test_acc == pytest.approx(0.8888, abs=0.12)
 
     except ARTTestException as e:
         art_warning(e)
@@ -202,8 +202,8 @@ def transform_feature(x):
         # check accuracy
         train_acc = np.sum(np.all(inferred_train == train_one_hot, axis=1)) / len(inferred_train)
         test_acc = np.sum(np.all(inferred_test == test_one_hot, axis=1)) / len(inferred_test)
-        assert pytest.approx(0.9145, abs=0.03) == train_acc
-        assert pytest.approx(0.9333, abs=0.03) == test_acc
+        assert pytest.approx(0.8666, abs=0.03) == train_acc
+        assert pytest.approx(0.8888, abs=0.03) == test_acc
 
     except ARTTestException as e:
         art_warning(e)
@@ -270,7 +270,7 @@ def transform_feature(x):
         # train attack model
         attack.fit(x_train)
         # infer attacked feature
-        values = [[-0.6324555, 1.5811388], [-0.4395245, 2.2751858], [-1.1108746, 0.9001915]]
+        values = [[-0.559017, 1.7888544], [-0.47003216, 2.127514], [-1.1774395, 0.84930056]]
         inferred_train = attack.infer(x_train_for_attack, x_train_predictions, values=values)
         inferred_test = attack.infer(x_test_for_attack, x_test_predictions, values=values)
         # check accuracy
@@ -280,8 +280,8 @@ def transform_feature(x):
         test_acc = np.sum(
             np.all(np.around(inferred_test, decimals=3) == np.around(test_one_hot, decimals=3), axis=1)
         ) / len(inferred_test)
-        assert pytest.approx(0.9145, abs=0.03) == train_acc
-        assert pytest.approx(0.9333, abs=0.03) == test_acc
+        assert pytest.approx(0.8666, abs=0.05) == train_acc
+        assert pytest.approx(0.8666, abs=0.05) == test_acc
 
     except ARTTestException as e:
         art_warning(e)

tests/attacks/test_elastic_net.py

@@ -422,7 +422,7 @@ def test_8_keras_iris_clipped(self):
         classifier = get_tabular_classifier_kr()
         attack = ElasticNet(classifier, targeted=False, max_iter=10, verbose=False)
         x_test_adv = attack.generate(self.x_test_iris)
-        expected_x_test_adv = np.asarray([0.85931635, 0.44633555, 0.65658355, 0.23840423])
+        expected_x_test_adv = np.asarray([0.860373, 0.455002, 0.654925, 0.240258])
         np.testing.assert_array_almost_equal(x_test_adv[0, :], expected_x_test_adv, decimal=6)
         self.assertLessEqual(np.amax(x_test_adv), 1.0)
         self.assertGreaterEqual(np.amin(x_test_adv), 0.0)
@@ -470,7 +470,7 @@ def test_8_keras_iris_clipped(self):
                     1,
                     1,
                     2,
-                    0,
+                    2,
                     2,
                     2,
                     1,
@@ -491,7 +491,7 @@ def test_9_keras_iris_unbounded(self):
         classifier = KerasClassifier(model=classifier._model, use_logits=False, channels_first=True)
         attack = ElasticNet(classifier, targeted=False, max_iter=10, verbose=False)
         x_test_adv = attack.generate(self.x_test_iris)
-        expected_x_test_adv = np.asarray([0.85931635, 0.44633555, 0.65658355, 0.23840423])
+        expected_x_test_adv = np.asarray([0.860373, 0.455002, 0.654925, 0.240258])
         np.testing.assert_array_almost_equal(x_test_adv[0, :], expected_x_test_adv, decimal=6)
         predictions_adv = np.argmax(classifier.predict(x_test_adv), axis=1)
         np.testing.assert_array_equal(
@@ -537,7 +537,7 @@ def test_9_keras_iris_unbounded(self):
                     1,
                     1,
                     2,
-                    0,
+                    2,
                     2,
                     2,
                     1,
@@ -557,7 +557,7 @@ def test_3_tensorflow_iris(self):
         # Test untargeted attack
         attack = ElasticNet(classifier, targeted=False, max_iter=10, verbose=False)
         x_test_adv = attack.generate(self.x_test_iris)
-        expected_x_test_adv = np.asarray([0.8479195, 0.42525578, 0.70166135, 0.28664514])
+        expected_x_test_adv = np.asarray([0.852286, 0.434626, 0.703376, 0.293738])
         np.testing.assert_array_almost_equal(x_test_adv[0, :], expected_x_test_adv, decimal=6)
         self.assertLessEqual(np.amax(x_test_adv), 1.0)
         self.assertGreaterEqual(np.amin(x_test_adv), 0.0)
@@ -596,13 +596,13 @@ def test_3_tensorflow_iris(self):
                     2,
                     2,
                     2,
-                    2,
+                    1,
                     2,
                     1,
                     2,
                     1,
                     0,
-                    2,
+                    1,
                     2,
                     1,
                     2,
@@ -622,7 +622,7 @@ def test_3_tensorflow_iris(self):
         targets = random_targets(self.y_test_iris, nb_classes=3)
         attack = ElasticNet(classifier, targeted=True, max_iter=10, verbose=False)
         x_test_adv = attack.generate(self.x_test_iris, **{"y": targets})
-        expected_x_test_adv = np.asarray([0.885649, 0.51815695, 0.5026782, 0.0558902])
+        expected_x_test_adv = np.asarray([0.892806, 0.531875, 0.501707, 0.059951])
         np.testing.assert_array_almost_equal(x_test_adv[0, :], expected_x_test_adv, decimal=6)
         self.assertLessEqual(np.amax(x_test_adv), 1.0)
         self.assertGreaterEqual(np.amin(x_test_adv), 0.0)
@@ -692,7 +692,7 @@ def test_5_pytorch_iris(self):
         classifier = get_tabular_classifier_pt()
         attack = ElasticNet(classifier, targeted=False, max_iter=10, verbose=False)
         x_test_adv = attack.generate(self.x_test_iris.astype(np.float32))
-        expected_x_test_adv = np.asarray([0.8479194, 0.42525578, 0.70166135, 0.28664517])
+        expected_x_test_adv = np.asarray([0.852286, 0.434626, 0.703376, 0.293738])
         np.testing.assert_array_almost_equal(x_test_adv[0, :], expected_x_test_adv, decimal=6)
         self.assertLessEqual(np.amax(x_test_adv), 1.0)
         self.assertGreaterEqual(np.amin(x_test_adv), 0.0)
@@ -731,13 +731,13 @@ def test_5_pytorch_iris(self):
                     2,
                     2,
                     2,
-                    2,
+                    1,
                     2,
                     1,
                     2,
                     1,
                     0,
-                    2,
+                    1,
                     2,
                     1,
                     2,

tests/estimators/classification/test_scikitlearn.py

@@ -203,16 +203,16 @@ def test_type(self):
 
     def test_predict(self):
         y_predicted = self.classifier.predict(self.x_test_iris[0:1])
-        y_expected = np.asarray([[0.07809449, 0.36258262, 0.55932295]])
+        y_expected = np.asarray([[0.07997696, 0.36272544, 0.5572976]])
         np.testing.assert_array_almost_equal(y_predicted, y_expected, decimal=4)
 
     def test_class_gradient_none_1(self):
         grad_predicted = self.classifier.class_gradient(self.x_test_iris[0:1], label=None)
         grad_expected = [
             [
-                [-1.97934151, 1.36346793, -6.29719639, -2.61386204],
-                [-0.56940532, -0.71100581, -1.00625587, -0.68006182],
-                [0.64548057, 0.27053964, 1.5315429, 0.80580771],
+                [-1.98016214, 1.35561633, -6.28256321, -2.60129547],
+                [-0.56317347, -0.70493793, -0.98908591, -0.67106223],
+                [0.65072, 0.2642768, 1.54536295, 0.81007898],
             ]
         ]
         np.testing.assert_array_almost_equal(grad_predicted, grad_expected, decimal=4)
@@ -221,45 +221,46 @@ def test_class_gradient_none_2(self):
         grad_predicted = self.classifier.class_gradient(self.x_test_iris[0:2], label=None)
         grad_expected = [
             [
-                [-1.97934151, 1.36346793, -6.29719639, -2.61386204],
-                [-0.56940532, -0.71100581, -1.00625587, -0.68006182],
-                [0.64548057, 0.27053964, 1.5315429, 0.80580771],
+                [-1.98016214, 1.35561633, -6.28256369, -2.60129547],
+                [-0.56317353, -0.70493793, -0.98908603, -0.67106229],
+                [0.65071994, 0.2642768, 1.54536283, 0.81007892],
             ],
             [
-                [-1.92147708, 1.3512013, -6.13324356, -2.53924561],
-                [-0.51154077, -0.72327244, -0.84230322, -0.60544527],
-                [0.70334512, 0.25827295, 1.69549561, 0.88042426],
+                [-1.92221594, 1.34292829, -6.11845303, -2.5268743],
+                [-0.50522733, -0.71762598, -0.82497525, -0.59664112],
+                [0.70866615, 0.25158882, 1.70947361, 0.88450009],
             ],
         ]
         np.testing.assert_array_almost_equal(grad_predicted, grad_expected, decimal=4)
 
     def test_class_gradient_int_1(self):
         grad_predicted = self.classifier.class_gradient(self.x_test_iris[0:1], label=1)
-        grad_expected = [[[-0.56940532, -0.71100581, -1.00625587, -0.68006182]]]
+        grad_expected = [[[-0.56317347, -0.70493793, -0.98908591, -0.67106223]]]
 
         for i_shape in range(4):
             self.assertAlmostEqual(grad_predicted[0, 0, i_shape], grad_expected[0][0][i_shape], 3)
 
     def test_class_gradient_int_2(self):
         grad_predicted = self.classifier.class_gradient(self.x_test_iris[0:2], label=1)
         grad_expected = [
-            [[-0.56940532, -0.71100581, -1.00625587, -0.68006182]],
-            [[-0.51154077, -0.72327244, -0.84230322, -0.60544527]],
+            [[-0.56317353, -0.70493793, -0.98908603, -0.67106229]],
+            [[-0.50522733, -0.71762598, -0.82497525, -0.59664112]],
         ]
         np.testing.assert_array_almost_equal(grad_predicted, grad_expected, decimal=4)
 
     def test_class_gradient_list_1(self):
         grad_predicted = self.classifier.class_gradient(self.x_test_iris[0:1], label=[1])
-        grad_expected = [[[-0.56940532, -0.71100581, -1.00625587, -0.68006182]]]
+        grad_expected = [[[-0.56317347, -0.70493793, -0.98908591, -0.67106223]]]
+        print(grad_predicted)
 
         for i_shape in range(4):
             self.assertAlmostEqual(grad_predicted[0, 0, i_shape], grad_expected[0][0][i_shape], 3)
 
     def test_class_gradient_list_2(self):
         grad_predicted = self.classifier.class_gradient(self.x_test_iris[0:2], label=[1, 2])
         grad_expected = [
-            [[-0.56940532, -0.71100581, -1.00625587, -0.68006182]],
-            [[0.70334512, 0.25827295, 1.69549561, 0.88042426]],
+            [[-0.56317353, -0.70493793, -0.98908603, -0.67106229]],
+            [[0.70866615, 0.25158882, 1.70947361, 0.88450009]],
         ]
         np.testing.assert_array_almost_equal(grad_predicted, grad_expected, decimal=4)
 
@@ -274,7 +275,7 @@ def test_class_gradient_label_wrong_type(self):
 
     def test_loss_gradient(self):
         grad_predicted = self.classifier.loss_gradient(self.x_test_iris[0:1], self.y_test_iris[0:1])
-        grad_expected = np.asarray([[-0.21516019, -0.09017988, -0.51051431, -0.26860258]])
+        grad_expected = np.asarray([[-0.21690667, -0.08809228, -0.51512096, -0.27002633]])
         np.testing.assert_array_almost_equal(grad_predicted, grad_expected, decimal=4)
 
 
@@ -304,7 +305,7 @@ def test_type(self):
     def test_class_gradient(self):
         grad_predicted = self.classifier.class_gradient(self.x_test_iris[0:1], label=None)
         grad_expected = np.asarray(
-            [[[-0.1428355, 0.12111039, -0.45059183, -0.17579888], [0.1428355, -0.12111039, 0.45059183, 0.17579888]]]
+            [[[-0.14551339, 0.12298754, -0.45839342, -0.17835225], [0.14551339, -0.12298754, 0.45839342, 0.17835225]]]
         )
         np.testing.assert_array_almost_equal(grad_predicted, grad_expected, decimal=3)
 
@@ -318,7 +319,7 @@ def test_loss_gradient(self):
         ][:, [0, 1]]
 
         grad_predicted = self.classifier.loss_gradient(x_test_binary[0:1], y_test_binary[0:1])
-        grad_expected = np.asarray([[-0.37461641, 0.31763777, -1.18177287, -0.4610699]])
+        grad_expected = np.asarray([[-0.3771413, 0.31875887, -1.18806318, -0.46225301]])
         np.testing.assert_array_almost_equal(grad_predicted, grad_expected, decimal=4)
 
 
@@ -342,16 +343,16 @@ def test_predict(self):
 
     def test_loss_gradient(self):
         grad_predicted = self.classifier.loss_gradient(self.x_test_iris[0:1], self.y_test_iris[0:1])
-        grad_expected = np.asarray([[-2.8753524, 0.31140438, -7.889445, -3.8314016]])
+        grad_expected = np.asarray([[-2.9100819, 0.3048792, -7.935282, -3.840562]])
         np.testing.assert_array_almost_equal(grad_predicted, grad_expected, decimal=4)
 
     def test_class_gradient_none_1(self):
         grad_predicted = self.classifier.class_gradient(self.x_test_iris[0:1], label=None)
         grad_expected = [
             [
-                [-1.52398277, 1.6984953, -6.05832438, -2.45788848],
-                [-0.43530558, -1.38692786, 0.41607214, -0.15109791],
-                [1.95928835, -0.31156744, 5.64225224, 2.6089864],
+                [-1.5939425, 1.67301144, -6.15095666, -2.4862934],
+                [-0.40469415, -1.37572607, 0.46867108, -0.13317975],
+                [1.99863665, -0.29728537, 5.68228559, 2.61947315],
             ]
         ]
         np.testing.assert_array_almost_equal(grad_predicted, grad_expected, decimal=4)
@@ -360,41 +361,41 @@ def test_class_gradient_none_2(self):
         grad_predicted = self.classifier.class_gradient(self.x_test_iris[0:2], label=None)
         grad_expected = [
             [
-                [-1.52398277, 1.6984953, -6.05832438, -2.45788848],
-                [-0.43530558, -1.38692786, 0.41607214, -0.15109791],
-                [1.95928835, -0.31156744, 5.64225224, 2.6089864],
+                [-1.5939425, 1.67301144, -6.15095666, -2.4862934],
+                [-0.40469415, -1.37572607, 0.46867108, -0.13317975],
+                [1.99863665, -0.29728537, 5.68228559, 2.61947315],
             ],
             [
-                [-1.52592969, 1.67535916, -6.2138464, -2.57887417],
-                [-0.43875292, -1.38381583, 0.48486185, -0.1011285],
-                [1.96468261, -0.29154334, 5.72898455, 2.68000267],
+                [-1.5962279, 1.64964639, -6.30453897, -2.60572715],
+                [-0.40788449, -1.37232544, 0.53680777, -0.08368929],
+                [2.00411239, -0.27732096, 5.7677312, 2.68941644],
             ],
         ]
         np.testing.assert_array_almost_equal(grad_predicted, grad_expected, decimal=4)
 
     def test_class_gradient_int_1(self):
         grad_predicted = self.classifier.class_gradient(self.x_test_iris[0:1], label=1)
-        grad_expected = [[[-0.43530558, -1.38692786, 0.41607214, -0.15109791]]]
+        grad_expected = [[[-0.40469415, -1.37572607, 0.46867108, -0.13317975]]]
         np.testing.assert_array_almost_equal(grad_predicted, grad_expected, decimal=4)
 
     def test_class_gradient_int_2(self):
         grad_predicted = self.classifier.class_gradient(self.x_test_iris[0:2], label=1)
         grad_expected = [
-            [[-0.43530558, -1.38692786, 0.41607214, -0.15109791]],
-            [[-0.43875292, -1.38381583, 0.48486185, -0.1011285]],
+            [[-0.40469415, -1.37572607, 0.46867108, -0.13317975]],
+            [[-0.40788449, -1.37232544, 0.53680777, -0.08368929]],
         ]
         np.testing.assert_array_almost_equal(grad_predicted, grad_expected, decimal=4)
 
     def test_class_gradient_list_1(self):
         grad_predicted = self.classifier.class_gradient(self.x_test_iris[0:1], label=[1])
-        grad_expected = [[[-0.43530558, -1.38692786, 0.41607214, -0.15109791]]]
+        grad_expected = [[[-0.40469415, -1.37572607, 0.46867108, -0.13317975]]]
         np.testing.assert_array_almost_equal(grad_predicted, grad_expected, decimal=4)
 
     def test_class_gradient_list_2(self):
         grad_predicted = self.classifier.class_gradient(self.x_test_iris[0:2], label=[1, 2])
         grad_expected = [
-            [[-0.43530558, -1.38692786, 0.41607214, -0.15109791]],
-            [[1.96468261, -0.29154334, 5.72898455, 2.68000267]],
+            [[-0.40469415, -1.37572607, 0.46867108, -0.13317975]],
+            [[2.00411239, -0.27732096, 5.7677312, 2.68941644]],
         ]
         np.testing.assert_array_almost_equal(grad_predicted, grad_expected, decimal=4)
 
@@ -427,16 +428,16 @@ def test_predict(self):
 
     def test_loss_gradient(self):
         grad_predicted = self.classifier.loss_gradient(self.x_test_iris[0:1], self.y_test_iris[0:1])
-        grad_expected = np.asarray([[0.38537693, 0.5659405, -3.600912, -2.338979]])
+        grad_expected = np.asarray([[0.38021886, 0.57562107, -3.599666, -2.3177252]])
         np.testing.assert_array_almost_equal(grad_predicted, grad_expected, decimal=4)
 
     def test_class_gradient(self):
         grad_predicted = self.classifier.class_gradient(self.x_test_iris[0:1], label=None)
         grad_expected = [
             [
-                [-0.34997019, 1.61489704, -3.49002061, -1.46298544],
-                [-0.11249995, -2.52947052, 0.7052329, -0.44872424],
-                [-0.3853818, -0.5659519, 3.60090744, 2.33898192],
+                [-0.34648966, 1.63777444, -3.51845999, -1.4609451],
+                [-0.11198028, -2.51565392, 0.71538245, -0.44830889],
+                [-0.38021886, -0.57562105, 3.59966607, 2.31772514],
             ]
         ]
         np.testing.assert_array_almost_equal(grad_predicted, grad_expected, decimal=4)

tests/test_utils.py

@@ -327,8 +327,8 @@ def test_preprocess(self):
     def test_iris(self):
         (x_train, y_train), (x_test, y_test), min_, max_ = load_iris()
 
-        self.assertTrue((min_ == 0).all())
-        self.assertTrue((max_ == 1).all())
+        self.assertAlmostEqual(min_, 0.012658227848101266, places=6)
+        self.assertEqual(max_, 1.0)
         self.assertEqual(x_train.shape[0], y_train.shape[0])
         self.assertEqual(x_test.shape[0], y_test.shape[0])
         train_labels = np.argmax(y_train, axis=1)