Skip to content

[FIX] Normalize: Fix crash with nan column #4125

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
janezd merged 1 commit into from
Oct 24, 2019
Merged

[FIX] Normalize: Fix crash with nan column #4125

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion Orange/preprocess/normalize.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import numpy as np

from Orange.data import ContinuousVariable, Domain
from Orange.statistics import distribution
from Orange.util import Reprable
Expand Down Expand Up @@ -57,7 +59,7 @@ def normalize_by_sd(self, dist, var):
)

def normalize_by_span(self, dist, var):
dma, dmi = dist.max(), dist.min()
dma, dmi = (dist.max(), dist.min()) if dist.shape[1] else (np.nan, np.nan)
diff = dma - dmi
if diff < 1e-15:
diff = 1
Expand Down
12 changes: 6 additions & 6 deletions Orange/tests/datasets/test10.tab
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
c1 c2 d1 d2 n1 n2 c3 d3 cl1 cl2
c t d d c d c d d c
class class
1 1995-01-21 a a ? a 2 a a 2
1 2003-07-23 a b 1 ? 0 b b 0
1 1967-03-12 a b 2 b -2 c c 1
c1 c2 d1 d2 n1 n2 c3 d3 c4 cl1 cl2
c t d d c d c d c d c
class class
1 1995-01-21 a a ? a 2 a a 2
1 2003-07-23 a b 1 ? 0 b b 0
1 1967-03-12 a b 2 b -2 c c 1
12 changes: 6 additions & 6 deletions Orange/tests/datasets/test5.tab
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
c1 c2 d1 d2 n1 n2 c3 d3 cl1 cl2
c c d d c d c d d c
class class
1 2 a a ? a 2 a a 2
1 0 a b 1 ? 0 b b 0
1 1 a b 2 b -2 c c 1
c1 c2 d1 d2 n1 n2 c3 d3 c4 cl1 cl2
c c d d c d c d c d c
class class
1 2 a a ? a 2 a a 2
1 0 a b 1 ? 0 b b 0
1 1 a b 2 b -2 c c 1
4 changes: 2 additions & 2 deletions Orange/tests/test_distances.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -970,8 +970,8 @@ def test_preprocess_multiclass(self):
new_table = _preprocess(table)
np.testing.assert_equal(new_table.Y, table.Y)
self.assertEqual([a.name for a in new_table.domain.attributes],
[a.name for a in table.domain.attributes
if a.is_continuous])
[a.name for a in table.domain.attributes if
a.is_continuous and not all(np.isnan(table[:, a].X))])
self.assertEqual(new_table.domain.class_vars, table.domain.class_vars)

def test_preprocess_impute(self):
Expand Down
50 changes: 25 additions & 25 deletions Orange/tests/test_normalize.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ def compare_tables(self, dataNorm, solution):
else:
self.assertEqual(dataNorm[i, j], solution[i][j])
self.assertEqual([attr.name for attr in dataNorm.domain.attributes],
["c1", "c2", "d1", "d2", "n1", "n2", "c3", "d3"])
["c1", "c2", "d1", "d2", "n1", "n2", "c3", "d3", "c4"])
self.assertEqual([attr.name for attr in dataNorm.domain.class_vars],
["cl1", "cl2"])
@classmethod
Expand All @@ -30,69 +30,69 @@ def setUpClass(cls):
def test_normalize_default(self):
normalizer = Normalize()
data_norm = normalizer(self.data)
solution = [[0., 1.225, 'a', 'a', '?', 'a', 1.225, 'a', 'a', 2],
[0., -1.225, 'a', 'b', -1., '?', 0., 'b', 'b', 0],
[0., 0., 'a', 'b', 1., 'b', -1.225, 'c', 'c', 1]]
solution = [[0., 1.225, 'a', 'a', '?', 'a', 1.225, 'a', '?', 'a', 2],
[0., -1.225, 'a', 'b', -1., '?', 0., 'b', '?', 'b', 0],
[0., 0., 'a', 'b', 1., 'b', -1.225, 'c', '?', 'c', 1]]
self.compare_tables(data_norm, solution)

def test_normalize_transform_by_sd(self):
normalizer = Normalize(zero_based=False,
norm_type=Normalize.NormalizeBySD,
transform_class=False)
data_norm = normalizer(self.data)
solution = [[0., 1.225, 'a', 'a', '?', 'a', 1.225, 'a', 'a', 2],
[0., -1.225, 'a', 'b', -1., '?', 0., 'b', 'b', 0],
[0., 0., 'a', 'b', 1., 'b', -1.225, 'c', 'c', 1]]
solution = [[0., 1.225, 'a', 'a', '?', 'a', 1.225, 'a', '?', 'a', 2],
[0., -1.225, 'a', 'b', -1., '?', 0., 'b', '?', 'b', 0],
[0., 0., 'a', 'b', 1., 'b', -1.225, 'c', '?', 'c', 1]]
self.compare_tables(data_norm, solution)

def test_normalize_transform_class(self):
normalizer = Normalize(zero_based=True,
norm_type=Normalize.NormalizeBySD,
transform_class=True)
data_norm = normalizer(self.data)
solution = [[0., 1.225, 'a', 'a', '?', 'a', 1.225, 'a', 'a', 1.225],
[0., -1.225, 'a', 'b', -1., '?', 0., 'b', 'b', -1.225],
[0., 0., 'a', 'b', 1., 'b', -1.225, 'c', 'c', 0.]]
solution = [[0., 1.225, 'a', 'a', '?', 'a', 1.225, 'a', '?', 'a', 1.225],
[0., -1.225, 'a', 'b', -1., '?', 0., 'b', '?', 'b', -1.225],
[0., 0., 'a', 'b', 1., 'b', -1.225, 'c', '?', 'c', 0.]]
self.compare_tables(data_norm, solution)

def test_normalize_transform_by_span(self):
normalizer = Normalize(zero_based=False,
norm_type=Normalize.NormalizeBySpan,
transform_class=False)
data_norm = normalizer(self.data)
solution = [[0., 1., 'a', 'a', '?', 'a', 1., 'a', 'a', 2.],
[0., -1., 'a', 'b', -1., '?', 0., 'b', 'b', 0.],
[0., 0., 'a', 'b', 1., 'b', -1., 'c', 'c', 1.]]
solution = [[0., 1., 'a', 'a', '?', 'a', 1., 'a', '?', 'a', 2.],
[0., -1., 'a', 'b', -1., '?', 0., 'b', '?', 'b', 0.],
[0., 0., 'a', 'b', 1., 'b', -1., 'c', '?', 'c', 1.]]
self.compare_tables(data_norm, solution)

def test_normalize_transform_by_span_zero(self):
normalizer = Normalize(zero_based=True,
norm_type=Normalize.NormalizeBySpan,
transform_class=False)
data_norm = normalizer(self.data)
solution = [[0., 1., 'a', 'a', '?', 'a', 1., 'a', 'a', 2.],
[0., 0., 'a', 'b', 0., '?', 0.5, 'b', 'b', 0.],
[0., 0.5, 'a', 'b', 1., 'b', 0., 'c', 'c', 1.]]
solution = [[0., 1., 'a', 'a', '?', 'a', 1., 'a', '?', 'a', 2.],
[0., 0., 'a', 'b', 0., '?', 0.5, 'b', '?', 'b', 0.],
[0., 0.5, 'a', 'b', 1., 'b', 0., 'c', '?', 'c', 1.]]
self.compare_tables(data_norm, solution)

def test_normalize_transform_by_span_class(self):
normalizer = Normalize(zero_based=False,
norm_type=Normalize.NormalizeBySpan,
transform_class=True)
data_norm = normalizer(self.data)
solution = [[0., 1., 'a', 'a', '?', 'a', 1., 'a', 'a', 1.],
[0., -1., 'a', 'b', -1., '?', 0., 'b', 'b', -1.],
[0., 0., 'a', 'b', 1., 'b', -1., 'c', 'c', 0.]]
solution = [[0., 1., 'a', 'a', '?', 'a', 1., 'a', '?', 'a', 1.],
[0., -1., 'a', 'b', -1., '?', 0., 'b', '?', 'b', -1.],
[0., 0., 'a', 'b', 1., 'b', -1., 'c', '?', 'c', 0.]]
self.compare_tables(data_norm, solution)

def test_normalize_transform_by_span_zero_class(self):
normalizer = Normalize(zero_based=True,
norm_type=Normalize.NormalizeBySpan,
transform_class=True)
data_norm = normalizer(self.data)
solution = [[0., 1., 'a', 'a', '?', 'a', 1., 'a', 'a', 1.],
[0., 0., 'a', 'b', 0., '?', 0.5, 'b', 'b', 0.],
[0., 0.5, 'a', 'b', 1., 'b', 0., 'c', 'c', 0.5]]
solution = [[0., 1., 'a', 'a', '?', 'a', 1., 'a', '?', 'a', 1.],
[0., 0., 'a', 'b', 0., '?', 0.5, 'b', '?', 'b', 0.],
[0., 0.5, 'a', 'b', 1., 'b', 0., 'c', '?', 'c', 0.5]]
self.compare_tables(data_norm, solution)

def test_normalize_sparse(self):
Expand Down Expand Up @@ -138,7 +138,7 @@ def test_datetime_normalization(self):
norm_type=Normalize.NormalizeBySD,
transform_class=False)
data_norm = normalizer(data)
solution = [[0., '1995-01-21', 'a', 'a', '?', 'a', 1.225, 'a', 'a', 2],
[0., '2003-07-23', 'a', 'b', -1., '?', 0., 'b', 'b', 0],
[0., '1967-03-12', 'a', 'b', 1., 'b', -1.225, 'c', 'c', 1]]
solution = [[0., '1995-01-21', 'a', 'a', '?', 'a', 1.225, 'a', '?', 'a', 2],
[0., '2003-07-23', 'a', 'b', -1., '?', 0., 'b', '?', 'b', 0],
[0., '1967-03-12', 'a', 'b', 1., 'b', -1.225, 'c', '?', 'c', 1]]
self.compare_tables(data_norm, solution)