Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 21 additions & 7 deletions Orange/widgets/data/oweditdomain.py
Original file line number Diff line number Diff line change
Expand Up @@ -2527,8 +2527,16 @@ def mapper(arr, out=None, dtype=dtype, **kwargs):
def time_parse(values: Sequence[str], name="__"):
tvar = Orange.data.TimeVariable(name)
parse_time = ftry(tvar.parse, ValueError, np.nan)
values = [parse_time(v) for v in values]
return tvar, values
_values = [parse_time(v) for v in values]
if np.all(np.isnan(_values)):
# try parsing it with pandas (like in transform)
dti = pd.to_datetime(values, errors="coerce")
_values = datetime_to_epoch(dti)
date_only = getattr(dti, "_is_dates_only", False)
if np.all(dti != pd.NaT):
tvar.have_date = True
tvar.have_time = not date_only
return tvar, _values


as_string = np.frompyfunc(str, 1, 1)
Expand Down Expand Up @@ -2734,17 +2742,23 @@ def transform(self, c):
raise TypeError


def datetime_to_epoch(dti: pd.DatetimeIndex) -> np.ndarray:
"""Convert datetime to epoch"""
data = dti.values.astype("M8[us]")
mask = np.isnat(data)
data = data.astype(float) / 1e6
data[mask] = np.nan
return data


class ReparseTimeTransform(Transformation):
"""
Re-parse the column's string repr as datetime.
"""
def transform(self, c):
c = column_str_repr(self.variable, c)
c = pd.to_datetime(c, errors="coerce").values.astype("M8[us]")
mask = np.isnat(c)
orangecol = c.astype(float) / 1e6
orangecol[mask] = np.nan
return orangecol
c = pd.to_datetime(c, errors="coerce")
return datetime_to_epoch(c)


class LookupMappingTransform(Transformation):
Expand Down
16 changes: 15 additions & 1 deletion Orange/widgets/data/tests/test_oweditdomain.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

import numpy as np
from numpy.testing import assert_array_equal
import pandas as pd

from AnyQt.QtCore import QItemSelectionModel, Qt, QItemSelection
from AnyQt.QtWidgets import QAction, QComboBox, QLineEdit, \
Expand All @@ -33,7 +34,7 @@
table_column_data, ReinterpretVariableEditor, CategoricalVector,
VariableEditDelegate, TransformRole,
RealVector, TimeVector, StringVector, make_dict_mapper, DictMissingConst,
LookupMappingTransform, as_float_or_nan, column_str_repr,
LookupMappingTransform, as_float_or_nan, column_str_repr, time_parse,
GroupItemsDialog)
from Orange.widgets.data.owcolor import OWColor, ColorRole
from Orange.widgets.tests.base import WidgetTest, GuiTest
Expand Down Expand Up @@ -917,6 +918,19 @@ def test_column_str_repr(self):
d = column_str_repr(v, np.array([0., np.nan, 1.0]))
assert_array_equal(d, ["00:00:00", "?", "00:00:01"])

def test_time_parse(self):
"""parsing additional datetimes by pandas"""
date = ["1/22/20", "1/23/20", "1/24/20"]
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How about when dates are 1/1/20? How do you know, which is day and which is month? M/D/Y is not universal.

Copy link
Collaborator Author

@robertcv robertcv Apr 1, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If data is not in iso8601 format then month first is assumed. Trying to parse dates without specifying the format is always tricky. Perhaps an additional field to specify d-m-y order would be the solution (to_datetime has dayfirst and yearfirst parameters). However, that would probably be in a new pr. I would really like to merge this since I need it to parse such dates.

# we use privet method, check if still exists
assert hasattr(pd.DatetimeIndex, '_is_dates_only')

tval, values = time_parse(date)

self.assertTrue(tval.have_date)
self.assertFalse(tval.have_time)
self.assertListEqual(list(values),
[1579651200.0, 1579737600.0, 1579824000.0])


class TestLookupMappingTransform(TestCase):
def setUp(self) -> None:
Expand Down