Skip to content

Commit 6aa55c9

Browse files
atheo89atheo89
committed
Update trustyAI test_notebook to include the tests from minimal and datascience due to python package inconsistencies
1 parent 47fd91b commit 6aa55c9

File tree

3 files changed

+443
-19
lines changed

3 files changed

+443
-19
lines changed

‎Makefile

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -408,14 +408,12 @@ test-%: bin/kubectl
408408
elif echo "$(FULL_NOTEBOOK_NAME)" | grep -q "intel-ml-ubi9"; then \
409409
$(call test_with_papermill,intel/ml,ubi9,python-3.9) \
410410
elif echo "$(FULL_NOTEBOOK_NAME)" | grep -q "trustyai-ubi9"; then \
411-
$(MAKE) validate-ubi9-datascience -e FULL_NOTEBOOK_NAME=$(FULL_NOTEBOOK_NAME); \
412411
$(call test_with_papermill,trustyai,ubi9,python-3.9) \
413412
elif echo "$(FULL_NOTEBOOK_NAME)" | grep -q "minimal-ubi8"; then \
414413
$(call test_with_papermill,minimal,ubi8,python-3.8) \
415414
elif echo "$(FULL_NOTEBOOK_NAME)" | grep -q "datascience-ubi8"; then \
416415
$(MAKE) validate-ubi8-datascience -e FULL_NOTEBOOK_NAME=$(FULL_NOTEBOOK_NAME); \
417416
elif echo "$(FULL_NOTEBOOK_NAME)" | grep -q "trustyai-ubi8"; then \
418-
$(MAKE) validate-ubi8-datascience -e FULL_NOTEBOOK_NAME=$(FULL_NOTEBOOK_NAME); \
419417
$(call test_with_papermill,trustyai,ubi8,python-3.8) \
420418
elif echo "$(FULL_NOTEBOOK_NAME)" | grep -q "anaconda"; then \
421419
echo "There is no test notebook implemented yet for Anaconda Notebook...." \

‎jupyter/trustyai/ubi8-python-3.8/test/test_notebook.ipynb

Lines changed: 223 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -8,24 +8,239 @@
88
"outputs": [],
99
"source": [
1010
"import unittest\n",
11+
"from unittest import mock\n",
1112
"import pandas as pd\n",
1213
"import trustyai as trustyai\n",
1314
"from platform import python_version\n",
1415
"from trustyai.metrics.fairness.group import statistical_parity_difference\n",
1516
"from trustyai.model import output\n",
17+
"from pandas.testing import assert_frame_equal\n",
18+
"import numpy as np\n",
19+
"import scipy\n",
20+
"import sklearn\n",
21+
"from scipy import special\n",
22+
"from scipy import integrate\n",
23+
"from sklearn import datasets\n",
24+
"from sklearn.model_selection import train_test_split\n",
25+
"import matplotlib\n",
26+
"import matplotlib.pyplot as plt\n",
27+
"import kafka\n",
28+
"from kafka import KafkaConsumer, KafkaProducer, TopicPartition\n",
29+
"from kafka.producer.buffer import SimpleBufferPool\n",
30+
"from kafka import KafkaConsumer\n",
31+
"from kafka.errors import KafkaConfigurationError\n",
32+
"import boto3\n",
33+
"import jupyterlab as jp\n",
34+
"import nbdime\n",
35+
"import nbgitpuller\n",
1636
"\n",
17-
"class TestTrustyaiNotebook(unittest.TestCase):\n",
1837
"\n",
19-
" def test_python_version(self):\n",
20-
" expected_major_minor = '3.8' # Set the expected version (x.y) \n",
21-
" actual_major_minor = '.'.join(python_version().split('.')[:2])\n",
38+
"def get_major_minor(s):\n",
39+
" return '.'.join(s.split('.')[:2])\n",
40+
"\n",
41+
"class TestPythonVersion(unittest.TestCase):\n",
42+
" def test_version(self):\n",
43+
" expected_major_minor = '3.8'\n",
44+
" actual_major_minor = get_major_minor(python_version())\n",
45+
" self.assertEqual(actual_major_minor, expected_major_minor, \"incorrect version\")\n",
46+
"\n",
47+
"class TestDependenciesVersions(unittest.TestCase):\n",
48+
" def test_jupyter_version(self):\n",
49+
" expected_major_minor = '3.6'\n",
50+
" actual_major_minor = get_major_minor(jp.__version__)\n",
51+
" self.assertEqual(actual_major_minor, expected_major_minor, \"incorrect version\")\n",
52+
"\n",
53+
" def test_nbgitpuller_version(self):\n",
54+
" expected_major_minor = '1.2'\n",
55+
" actual_major_minor = get_major_minor(nbgitpuller.__version__)\n",
56+
" self.assertEqual(actual_major_minor, expected_major_minor, \"incorrect version\")\n",
57+
"\n",
58+
" def test_nbdime_version(self):\n",
59+
" expected_major_minor = '3.2'\n",
60+
" actual_major_minor = get_major_minor(nbdime.__version__)\n",
61+
" self.assertEqual(actual_major_minor, expected_major_minor, \"incorrect version\")\n",
62+
"\n",
63+
"class TestPandas(unittest.TestCase):\n",
64+
" def test_version(self):\n",
65+
" expected_major_minor = '1.5'\n",
66+
" actual_major_minor = get_major_minor(pd.__version__)\n",
67+
" self.assertEqual(actual_major_minor, expected_major_minor, \"incorrect version\")\n",
68+
"\n",
69+
" def test_dataframe_creation(self):\n",
70+
" sample_df = pd.DataFrame({'a': [1, 2], 'b': [3, 4]})\n",
71+
" self.assertIsInstance(sample_df, pd.core.frame.DataFrame)\n",
72+
"\n",
73+
" def test_equal_dataframes(self):\n",
74+
" df1 = pd.DataFrame({'a': [1, 2], 'b': [3, 4]})\n",
75+
" df2 = pd.DataFrame({'a': [1, 2], 'b': [3.0, 4.0]})\n",
76+
" self.assertIsNone(assert_frame_equal(df1, df2, check_dtype=False), \"Dataframes provided are unequal\")\n",
77+
"\n",
78+
" def test_unequal_dataframes(self):\n",
79+
" df1 = pd.DataFrame({'a': [1, 2], 'b': [3, 4]})\n",
80+
" df2 = pd.DataFrame({'a': [1, 2], 'b': [3.0, 5.0]})\n",
81+
" with self.assertRaises(AssertionError):\n",
82+
" assert_frame_equal(df1, df2, check_dtype=False)\n",
83+
"\n",
84+
" def test_dataframe_shape(self):\n",
85+
" random_data = {\n",
86+
" 'apples': [3, 2, 0, 1], \n",
87+
" 'oranges': [0, 3, 7, 2]\n",
88+
" }\n",
89+
" sample_df = pd.DataFrame(random_data)\n",
90+
" self.assertEqual(sample_df.shape, (4,2))\n",
91+
"\n",
92+
" def test_index_out_of_bounds(self):\n",
93+
" random_data = {\n",
94+
" 'apples': [3, 2, 0, 1], \n",
95+
" 'oranges': [0, 3, 7, 2]\n",
96+
" }\n",
97+
" sample_df = pd.DataFrame(random_data)\n",
98+
" with self.assertRaises(IndexError):\n",
99+
" print(sample_df.iat[0,3])\n",
100+
"\n",
101+
" def test_sampling(self):\n",
102+
" random_data = {\n",
103+
" 'apples': [3, 2, 0, 1], \n",
104+
" 'oranges': [0, 3, 7, 2]\n",
105+
" }\n",
106+
" sample_df = pd.DataFrame(random_data)\n",
107+
" half_sampled_df = sample_df.sample(frac = 0.5)\n",
108+
" self.assertEqual(len(half_sampled_df), 2)\n",
109+
"\n",
110+
" def test_drop(self):\n",
111+
" random_data = {\n",
112+
" 'apples': [3, 2, 0, 1], \n",
113+
" 'oranges': [0, 3, 7, 2]\n",
114+
" }\n",
115+
" sample_df = pd.DataFrame(random_data)\n",
116+
" self.assertEqual(sample_df.drop(columns=['apples']).shape, (4, 1))\n",
117+
"\n",
118+
"class TestNumpy(unittest.TestCase):\n",
119+
" def test_version(self):\n",
120+
" expected_major_minor = '1.24'\n",
121+
" actual_major_minor = get_major_minor(np.__version__)\n",
122+
" self.assertEqual(actual_major_minor, expected_major_minor, \"incorrect version\")\n",
123+
"\n",
124+
" def test_array_creation(self):\n",
125+
" arr = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9])\n",
126+
" self.assertIsInstance(arr, np.ndarray)\n",
127+
" \n",
128+
" def test_array_opeartions(self):\n",
129+
" arr = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9])\n",
130+
"\n",
131+
" self.assertEqual(arr.sum(), 45)\n",
132+
" self.assertEqual(len(arr), 9)\n",
133+
" self.assertEqual(arr.max(), 9)\n",
134+
" self.assertEqual(arr.min(), 1)\n",
135+
"\n",
136+
" def test_array_statistical_functions(self):\n",
137+
" arr = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9])\n",
138+
"\n",
139+
" self.assertEqual(np.median(arr), 5)\n",
140+
" self.assertEqual(np.mean(arr), 5)\n",
141+
" self.assertEqual(np.std(arr), 2.581988897471611)\n",
142+
"\n",
143+
"class TestScipy(unittest.TestCase):\n",
144+
" def test_version(self):\n",
145+
" expected_major_minor = '1.10'\n",
146+
" actual_major_minor = get_major_minor(scipy.__version__)\n",
147+
" self.assertEqual(actual_major_minor, expected_major_minor, \"incorrect version\")\n",
148+
"\n",
149+
" def test_scipy_special(self):\n",
150+
" self.assertEqual(special.exp10(3), 1000.0)\n",
151+
" self.assertEqual(special.exp2(10), 1024.0)\n",
152+
" self.assertEqual(special.sindg(90), 1)\n",
153+
" self.assertEqual(special.cosdg(0), 1)\n",
154+
"\n",
155+
" def test_scipy_integrate(self):\n",
156+
" a= lambda x:special.exp10(x)\n",
157+
" b = integrate.quad(a, 0, 1)\n",
158+
" self.assertEqual(b, (3.9086503371292665, 4.3394735994897923e-14))\n",
159+
"\n",
160+
"class TestSKLearn(unittest.TestCase):\n",
161+
" def test_version(self):\n",
162+
" expected_major_minor = '1.3'\n",
163+
" actual_major_minor = get_major_minor(sklearn.__version__)\n",
164+
" self.assertEqual(actual_major_minor, expected_major_minor, \"incorrect version\")\n",
165+
" \n",
166+
" def test_sklearn_dataset(self):\n",
167+
" data_set = datasets.load_iris()\n",
168+
" self.assertIsInstance(data_set, sklearn.utils._bunch.Bunch)\n",
169+
"\n",
170+
" def test_sklearn_train_test_split(self):\n",
171+
" my_iris = datasets.load_iris()\n",
172+
" X = my_iris.data\n",
173+
" Y = my_iris.target\n",
174+
"\n",
175+
" X_traindata, X_testdata, Y_traindata, Y_testdata = train_test_split(\n",
176+
" X, Y, test_size = 0.3, random_state = 1)\n",
177+
" \n",
178+
" self.assertEqual(X_traindata.shape, (105, 4))\n",
179+
" self.assertEqual(X_testdata.shape, (45, 4))\n",
180+
" self.assertEqual(Y_traindata.shape, (105,))\n",
181+
" self.assertEqual(Y_testdata.shape, (45,))\n",
182+
"\n",
183+
"class TestMatplotlib(unittest.TestCase):\n",
184+
"\n",
185+
" def test_version(self):\n",
186+
" expected_major_minor = '3.6'\n",
187+
" actual_major_minor = get_major_minor(matplotlib.__version__)\n",
22188
" self.assertEqual(actual_major_minor, expected_major_minor, \"incorrect version\")\n",
23189
"\n",
24-
" def test_trustyai_version(self):\n",
25-
" expected_major_minor = '0.6' # Set the expected version (x.y) \\n\n",
26-
" actual_major_minor = '.'.join(trustyai.__version__.split('.')[:2]) \n",
190+
" def test_matplotlib_figure_creation(self):\n",
191+
" self.assertIsInstance(plt.figure(figsize=(8,5)), matplotlib.figure.Figure)\n",
192+
"\n",
193+
"class TestKafkaPython(unittest.TestCase):\n",
194+
"\n",
195+
" def test_version(self):\n",
196+
" expected_major_minor = '2.0'\n",
197+
" actual_major_minor = get_major_minor(kafka.__version__)\n",
27198
" self.assertEqual(actual_major_minor, expected_major_minor, \"incorrect version\")\n",
199+
"\n",
200+
" def test_buffer_pool(self):\n",
201+
" pool = SimpleBufferPool(1000, 1000)\n",
202+
" \n",
203+
" buf1 = pool.allocate(1000, 1000)\n",
204+
" message = ''.join(map(str, range(100)))\n",
205+
" buf1.write(message.encode('utf-8'))\n",
206+
" pool.deallocate(buf1)\n",
28207
" \n",
208+
" buf2 = pool.allocate(1000, 1000)\n",
209+
" self.assertEqual(buf2.read(), b'')\n",
210+
"\n",
211+
" def test_session_timeout_larger_than_request_timeout_raises(self):\n",
212+
" with self.assertRaises(KafkaConfigurationError):\n",
213+
" KafkaConsumer(bootstrap_servers='localhost:9092', api_version=(0, 9), group_id='foo', session_timeout_ms=50000, request_timeout_ms=40000)\n",
214+
"\n",
215+
"class TestBoto3(unittest.TestCase):\n",
216+
"\n",
217+
" def test_version(self):\n",
218+
" expected_major_minor = '1.34'\n",
219+
" actual_major_minor = get_major_minor(boto3.__version__)\n",
220+
" self.assertEqual(actual_major_minor, expected_major_minor, \"incorrect version\")\n",
221+
"\n",
222+
" def setUp(self):\n",
223+
" self.session_patch = mock.patch('boto3.Session', autospec=True)\n",
224+
" self.Session = self.session_patch.start()\n",
225+
"\n",
226+
" def tearDown(self):\n",
227+
" boto3.DEFAULT_SESSION = None\n",
228+
" self.session_patch.stop()\n",
229+
"\n",
230+
" def test_create_default_session(self):\n",
231+
" session = self.Session.return_value\n",
232+
"\n",
233+
" boto3.setup_default_session()\n",
234+
"\n",
235+
" self.assertEqual(boto3.DEFAULT_SESSION, session)\n",
236+
"\n",
237+
"class TestTrustyaiNotebook(unittest.TestCase):\n",
238+
"\n",
239+
" def test_trustyai_version(self): \n",
240+
" expected_major_minor = '0.6' # Set the expected version (x.y) \n",
241+
" actual_major_minor = get_major_minor(trustyai.__version__)\n",
242+
" self.assertEqual(actual_major_minor, expected_major_minor, \"incorrect version\")\n",
243+
"\n",
29244
" def test_fairnessmetrics(self):\n",
30245
" url_unbiased = \"https://raw.githubusercontent.com/opendatahub-io/notebooks/main/jupyter/trustyai/ubi8-python-3.8/test/income-unbiased.csv\"\n",
31246
" nobias = pd.read_csv(url_unbiased, index_col=False)\n",
@@ -61,9 +276,7 @@
61276
" self.assertTrue(score <= -0.15670061634672994)\n",
62277
" print(\"On the test_bias_metrics test case the statistical_parity_difference score for this dataset, as expected, is outside the threshold [-0.1,0.1], which classifies the model as unfair.\")\n",
63278
" \n",
64-
"\n",
65-
"suite = unittest.TestLoader().loadTestsFromTestCase(TestTrustyaiNotebook)\n",
66-
"unittest.TextTestRunner().run(suite)"
279+
"unittest.main(argv=[''], verbosity=2, exit=False)"
67280
]
68281
}
69282
],

0 commit comments

Comments
 (0)