diff --git a/pandas/core/panel.py b/pandas/core/panel.py index fa469242fa4e3..424074de2ca7c 100644 --- a/pandas/core/panel.py +++ b/pandas/core/panel.py @@ -96,35 +96,14 @@ def _arith_method(func, name): def f(self, other): if not np.isscalar(other): - raise ValueError('Simple arithmetic with Panel can only be ' - 'done with scalar values') + raise ValueError('Simple arithmetic with %s can only be ' + 'done with scalar values' % self._constructor.__name__) return self._combine(other, func) f.__name__ = name return f -def _panel_arith_method(op, name): - @Substitution(op) - def f(self, other, axis = 0): - """ - Wrapper method for %s - - Parameters - ---------- - other : DataFrame or Panel class - axis : {'items', 'major', 'minor'} - Axis to broadcast over - - Returns - ------- - Panel - """ - return self._combine(other, op, axis=axis) - - f.__name__ = name - return f - def _comp_method(func, name): def na_op(x, y): @@ -164,27 +143,6 @@ def f(self, other): return f -_agg_doc = """ -Return %(desc)s over requested axis - -Parameters ----------- -axis : {'items', 'major', 'minor'} or {0, 1, 2} -skipna : boolean, default True - Exclude NA/null values. If an entire row/column is NA, the result - will be NA - -Returns -------- -%(outname)s : DataFrame -""" - -_na_info = """ - -NA/null values are %s. -If all values are NA, result will be NA""" - - class Panel(NDFrame): _AXIS_ORDERS = ['items','major_axis','minor_axis'] _AXIS_NUMBERS = dict([ (a,i) for i, a in enumerate(_AXIS_ORDERS) ]) @@ -217,13 +175,24 @@ def _constructor(self): # return the type of the slice constructor _constructor_sliced = DataFrame - def _construct_axes_dict(self, axes = None): + def _construct_axes_dict(self, axes = None, **kwargs): """ return an axes dictionary for myself """ - return dict([ (a,getattr(self,a)) for a in (axes or self._AXIS_ORDERS) ]) + d = dict([ (a,getattr(self,a)) for a in (axes or self._AXIS_ORDERS) ]) + d.update(kwargs) + return d - def _construct_axes_dict_for_slice(self, axes = None): + @staticmethod + def _construct_axes_dict_from(self, axes, **kwargs): + """ return an axes dictionary for the passed axes """ + d = dict([ (a,ax) for a,ax in zip(self._AXIS_ORDERS,axes) ]) + d.update(kwargs) + return d + + def _construct_axes_dict_for_slice(self, axes = None, **kwargs): """ return an axes dictionary for myself """ - return dict([ (self._AXIS_SLICEMAP[a],getattr(self,a)) for a in (axes or self._AXIS_ORDERS) ]) + d = dict([ (self._AXIS_SLICEMAP[a],getattr(self,a)) for a in (axes or self._AXIS_ORDERS) ]) + d.update(kwargs) + return d __add__ = _arith_method(operator.add, '__add__') __sub__ = _arith_method(operator.sub, '__sub__') @@ -296,8 +265,7 @@ def _from_axes(cls, data, axes): if isinstance(data, BlockManager): return cls(data) else: - d = dict([ (i, a) for i, a in zip(cls._AXIS_ORDERS,axes) ]) - d['copy'] = False + d = cls._construct_axes_dict_from(cls, axes, copy = False) return cls(data, **d) def _init_dict(self, data, axes, dtype=None): @@ -436,8 +404,7 @@ def __array__(self, dtype=None): return self.values def __array_wrap__(self, result): - d = self._construct_axes_dict(self._AXIS_ORDERS) - d['copy'] = False + d = self._construct_axes_dict(self._AXIS_ORDERS, copy = False) return self._constructor(result, **d) #---------------------------------------------------------------------- @@ -455,8 +422,7 @@ def _compare_constructor(self, other, func): for col in getattr(self,self._info_axis): new_data[col] = func(self[col], other[col]) - d = self._construct_axes_dict() - d['copy'] = False + d = self._construct_axes_dict(copy = False) return self._constructor(data=new_data, **d) # boolean operators @@ -552,9 +518,7 @@ def iteritems(self): iterkv = iteritems def _get_plane_axes(self, axis): - """ - - """ + """ get my plane axes: these are already (as compared with higher level planes), as we are returning a DataFrame axes """ axis = self._get_axis_name(axis) if axis == 'major_axis': @@ -580,8 +544,7 @@ def ix(self): return self._ix def _wrap_array(self, arr, axes, copy=False): - d = dict([ (a,ax) for a,ax in zip(self._AXIS_ORDERS,axes) ]) - d['copy'] = False + d = self._construct_axes_dict_from(self, axes, copy = copy) return self._constructor(arr, **d) fromDict = from_dict @@ -627,7 +590,7 @@ def to_excel(self, path, na_rep=''): # TODO: needed? def keys(self): - return list(self.items) + return list(getattr(self,self._info_axis)) def _get_values(self): self._consolidate_inplace() @@ -684,9 +647,8 @@ def set_value(self, *args): frame.set_value(*args[1:]) return self except KeyError: - axes = self._expand_axes(args) - d = dict([ (a,ax) for a,ax in zip(self._AXIS_ORDERS,axes) ]) - d['copy'] = False + axes = self._expand_axes(args) + d = self._construct_axes_dict_from(self, axes, copy = False) result = self.reindex(**d) likely_dtype = com._infer_dtype(args[-1]) @@ -916,8 +878,7 @@ def reindex_like(self, other, method=None): ------- reindexed : Panel """ - d = other._construct_axes_dict() - d['method'] = method + d = other._construct_axes_dict(method = method) return self.reindex(**d) def dropna(self, axis=0, how='any'): @@ -1046,16 +1007,6 @@ def bfill(self): return self.fillna(method='bfill') - add = _panel_arith_method(operator.add, 'add') - subtract = sub = _panel_arith_method(operator.sub, 'subtract') - multiply = mul = _panel_arith_method(operator.mul, 'multiply') - - try: - divide = div = _panel_arith_method(operator.div, 'divide') - except AttributeError: # pragma: no cover - # Python 3 - divide = div = _panel_arith_method(operator.truediv, 'divide') - def major_xs(self, key, copy=True): """ Return slice of panel along major axis @@ -1203,7 +1154,7 @@ def transpose(self, *args, **kwargs): if len(axes) != len(set(axes)): raise ValueError('Must specify %s unique axes' % self._AXIS_LEN) - new_axes = dict([ (a,self._get_axis(x)) for a, x in zip(self._AXIS_ORDERS,axes)]) + new_axes = self._construct_axes_dict_from(self, [ self._get_axis(x) for x in axes]) new_values = self.values.transpose(tuple(axes)) if kwargs.get('copy') or (len(args) and args[-1]): new_values = new_values.copy() @@ -1337,56 +1288,6 @@ def count(self, axis='major'): return self._wrap_result(result, axis) - @Substitution(desc='sum', outname='sum') - @Appender(_agg_doc) - def sum(self, axis='major', skipna=True): - return self._reduce(nanops.nansum, axis=axis, skipna=skipna) - - @Substitution(desc='mean', outname='mean') - @Appender(_agg_doc) - def mean(self, axis='major', skipna=True): - return self._reduce(nanops.nanmean, axis=axis, skipna=skipna) - - @Substitution(desc='unbiased variance', outname='variance') - @Appender(_agg_doc) - def var(self, axis='major', skipna=True): - return self._reduce(nanops.nanvar, axis=axis, skipna=skipna) - - @Substitution(desc='unbiased standard deviation', outname='stdev') - @Appender(_agg_doc) - def std(self, axis='major', skipna=True): - return self.var(axis=axis, skipna=skipna).apply(np.sqrt) - - @Substitution(desc='unbiased skewness', outname='skew') - @Appender(_agg_doc) - def skew(self, axis='major', skipna=True): - return self._reduce(nanops.nanskew, axis=axis, skipna=skipna) - - @Substitution(desc='product', outname='prod') - @Appender(_agg_doc) - def prod(self, axis='major', skipna=True): - return self._reduce(nanops.nanprod, axis=axis, skipna=skipna) - - @Substitution(desc='compounded percentage', outname='compounded') - @Appender(_agg_doc) - def compound(self, axis='major', skipna=True): - return (1 + self).prod(axis=axis, skipna=skipna) - 1 - - @Substitution(desc='median', outname='median') - @Appender(_agg_doc) - def median(self, axis='major', skipna=True): - return self._reduce(nanops.nanmedian, axis=axis, skipna=skipna) - - @Substitution(desc='maximum', outname='maximum') - @Appender(_agg_doc) - def max(self, axis='major', skipna=True): - return self._reduce(nanops.nanmax, axis=axis, skipna=skipna) - - @Substitution(desc='minimum', outname='minimum') - @Appender(_agg_doc) - def min(self, axis='major', skipna=True): - return self._reduce(nanops.nanmin, axis=axis, skipna=skipna) - def shift(self, lags, axis='major'): """ Shift major or minor axis by specified number of leads/lags. Drops @@ -1524,9 +1425,11 @@ def update(self, other, join='left', overwrite=True, filter_func=None, if not isinstance(other, self._constructor): other = self._constructor(other) - other = other.reindex(items=self.items) + axis = self._info_axis + axis_values = getattr(self,axis) + other = other.reindex(**{ axis : axis_values }) - for frame in self.items: + for frame in axis_values: self[frame].update(other[frame], join, overwrite, filter_func, raise_conflict) @@ -1645,6 +1548,124 @@ def _extract_axis(self, data, axis=0, intersect=False): return _ensure_index(index) + @classmethod + def _add_aggregate_operations(cls): + """ add the operations to the cls; evaluate the doc strings again """ + + # doc strings substitors + _agg_doc = """ +Wrapper method for %s + +Parameters +---------- +other : """ + "%s or %s" % (cls._constructor_sliced.__name__,cls.__name__) + """ +axis : {""" + ', '.join(cls._AXIS_ORDERS) + "}" + """ +Axis to broadcast over + +Returns +------- +""" + cls.__name__ + "\n" + + def _panel_arith_method(op, name): + @Substitution(op) + @Appender(_agg_doc) + def f(self, other, axis = 0): + return self._combine(other, op, axis=axis) + f.__name__ = name + return f + + cls.add = _panel_arith_method(operator.add, 'add') + cls.subtract = cls.sub = _panel_arith_method(operator.sub, 'subtract') + cls.multiply = cls.mul = _panel_arith_method(operator.mul, 'multiply') + + try: + cls.divide = cls.div = _panel_arith_method(operator.div, 'divide') + except AttributeError: # pragma: no cover + # Python 3 + cls.divide = cls.div = _panel_arith_method(operator.truediv, 'divide') + + + _agg_doc = """ +Return %(desc)s over requested axis + +Parameters +---------- +axis : {""" + ', '.join(cls._AXIS_ORDERS) + "} or {" + ', '.join([ str(i) for i in range(cls._AXIS_LEN) ]) + """} +skipna : boolean, default True + Exclude NA/null values. If an entire row/column is NA, the result + will be NA + +Returns +------- +%(outname)s : """ + cls._constructor_sliced.__name__ + "\n" + + _na_info = """ + +NA/null values are %s. +If all values are NA, result will be NA""" + + @Substitution(desc='sum', outname='sum') + @Appender(_agg_doc) + def sum(self, axis='major', skipna=True): + return self._reduce(nanops.nansum, axis=axis, skipna=skipna) + cls.sum = sum + + @Substitution(desc='mean', outname='mean') + @Appender(_agg_doc) + def mean(self, axis='major', skipna=True): + return self._reduce(nanops.nanmean, axis=axis, skipna=skipna) + cls.mean = mean + + @Substitution(desc='unbiased variance', outname='variance') + @Appender(_agg_doc) + def var(self, axis='major', skipna=True): + return self._reduce(nanops.nanvar, axis=axis, skipna=skipna) + cls.var = var + + @Substitution(desc='unbiased standard deviation', outname='stdev') + @Appender(_agg_doc) + def std(self, axis='major', skipna=True): + return self.var(axis=axis, skipna=skipna).apply(np.sqrt) + cls.std = std + + @Substitution(desc='unbiased skewness', outname='skew') + @Appender(_agg_doc) + def skew(self, axis='major', skipna=True): + return self._reduce(nanops.nanskew, axis=axis, skipna=skipna) + cls.skew = skew + + @Substitution(desc='product', outname='prod') + @Appender(_agg_doc) + def prod(self, axis='major', skipna=True): + return self._reduce(nanops.nanprod, axis=axis, skipna=skipna) + cls.prod = prod + + @Substitution(desc='compounded percentage', outname='compounded') + @Appender(_agg_doc) + def compound(self, axis='major', skipna=True): + return (1 + self).prod(axis=axis, skipna=skipna) - 1 + cls.compound = compound + + @Substitution(desc='median', outname='median') + @Appender(_agg_doc) + def median(self, axis='major', skipna=True): + return self._reduce(nanops.nanmedian, axis=axis, skipna=skipna) + cls.median = median + + @Substitution(desc='maximum', outname='maximum') + @Appender(_agg_doc) + def max(self, axis='major', skipna=True): + return self._reduce(nanops.nanmax, axis=axis, skipna=skipna) + cls.max = max + + @Substitution(desc='minimum', outname='minimum') + @Appender(_agg_doc) + def min(self, axis='major', skipna=True): + return self._reduce(nanops.nanmin, axis=axis, skipna=skipna) + cls.min = min + +Panel._add_aggregate_operations() + WidePanel = Panel LongPanel = DataFrame @@ -1660,7 +1681,7 @@ def install_ipython_completers(): # pragma: no cover @complete_object.when_type(Panel) def complete_dataframe(obj, prev_completions): - return prev_completions + [c for c in obj.items \ + return prev_completions + [c for c in obj.keys() \ if isinstance(c, basestring) and py3compat.isidentifier(c)] # Importing IPython brings in about 200 modules, so we want to avoid it unless diff --git a/pandas/core/panel4d.py b/pandas/core/panel4d.py index 649739f3f60ce..71e3e7d68e252 100644 --- a/pandas/core/panel4d.py +++ b/pandas/core/panel4d.py @@ -1,9 +1,9 @@ """ Panel4D: a 4-d dict like collection of panels """ +from pandas.core.panelnd import create_nd_panel_factory from pandas.core.panel import Panel -from pandas.core import panelnd -Panel4D = panelnd.create_nd_panel_factory( +Panel4D = create_nd_panel_factory( klass_name = 'Panel4D', axis_orders = [ 'labels','items','major_axis','minor_axis'], axis_slices = { 'labels' : 'labels', 'items' : 'items', diff --git a/pandas/core/panelnd.py b/pandas/core/panelnd.py index 22f6dac6b640c..b7d2e29a3c79d 100644 --- a/pandas/core/panelnd.py +++ b/pandas/core/panelnd.py @@ -1,7 +1,5 @@ """ Factory methods to create N-D panels """ -import pandas -from pandas.core.panel import Panel import pandas.lib as lib def create_nd_panel_factory(klass_name, axis_orders, axis_slices, slicer, axis_aliases = None, stat_axis = 2): @@ -27,6 +25,14 @@ def create_nd_panel_factory(klass_name, axis_orders, axis_slices, slicer, axis_a """ + # if slicer is a name, get the object + if isinstance(slicer,basestring): + import pandas + try: + slicer = getattr(pandas,slicer) + except: + raise Exception("cannot create this slicer [%s]" % slicer) + # build the klass klass = type(klass_name, (slicer,),{}) @@ -40,6 +46,7 @@ def create_nd_panel_factory(klass_name, axis_orders, axis_slices, slicer, axis_a klass._default_stat_axis = stat_axis klass._het_axis = 0 klass._info_axis = axis_orders[klass._het_axis] + klass._constructor_sliced = slicer # add the axes @@ -96,49 +103,13 @@ def _combine_with_constructor(self, other, func): klass._combine_with_constructor = _combine_with_constructor # set as NonImplemented operations which we don't support - for f in ['to_frame','to_excel','to_sparse','groupby','join','_get_join_index']: + for f in ['to_frame','to_excel','to_sparse','groupby','join','filter','dropna','shift','take']: def func(self, *args, **kwargs): raise NotImplementedError setattr(klass,f,func) - return klass - - -if __name__ == '__main__': - - # create a sample - from pandas.util import testing - print pandas.__version__ + # add the aggregate operations + klass._add_aggregate_operations() - # create a 4D - Panel4DNew = create_nd_panel_factory( - klass_name = 'Panel4DNew', - axis_orders = ['labels1','items1','major_axis','minor_axis'], - axis_slices = { 'items1' : 'items', 'major_axis' : 'major_axis', 'minor_axis' : 'minor_axis' }, - slicer = Panel, - axis_aliases = { 'major' : 'major_axis', 'minor' : 'minor_axis' }, - stat_axis = 2) - - p4dn = Panel4DNew(dict(L1 = testing.makePanel(), L2 = testing.makePanel())) - print "creating a 4-D Panel" - print p4dn, "\n" - - # create a 5D - Panel5DNew = create_nd_panel_factory( - klass_name = 'Panel5DNew', - axis_orders = [ 'cool1', 'labels1','items1','major_axis','minor_axis'], - axis_slices = { 'labels1' : 'labels1', 'items1' : 'items', 'major_axis' : 'major_axis', 'minor_axis' : 'minor_axis' }, - slicer = Panel4DNew, - axis_aliases = { 'major' : 'major_axis', 'minor' : 'minor_axis' }, - stat_axis = 2) - - p5dn = Panel5DNew(dict(C1 = p4dn)) - - print "creating a 5-D Panel" - print p5dn, "\n" - - print "Slicing p5dn" - print p5dn.ix['C1',:,:,0:3,:], "\n" + return klass - print "Transposing p5dn" - print p5dn.transpose(1,2,3,4,0), "\n" diff --git a/pandas/tests/test_panel4d.py b/pandas/tests/test_panel4d.py index bdfc4933f31ab..39d7d5fd08245 100644 --- a/pandas/tests/test_panel4d.py +++ b/pandas/tests/test_panel4d.py @@ -890,11 +890,38 @@ def test_to_frame_mixed(self): # self.assertEqual(wp['bool'].values.dtype, np.bool_) # assert_frame_equal(wp['bool'], panel['bool']) + def test_update(self): + + p4d = Panel4D([[[[1.5, np.nan, 3.], + [1.5, np.nan, 3.], + [1.5, np.nan, 3.], + [1.5, np.nan, 3.]], + [[1.5, np.nan, 3.], + [1.5, np.nan, 3.], + [1.5, np.nan, 3.], + [1.5, np.nan, 3.]]]]) + + other = Panel4D([[[[3.6, 2., np.nan]], + [[np.nan, np.nan, 7]]]]) + + p4d.update(other) + + expected = Panel4D([[[[3.6, 2, 3.], + [1.5, np.nan, 3.], + [1.5, np.nan, 3.], + [1.5, np.nan, 3.]], + [[1.5, np.nan, 7], + [1.5, np.nan, 3.], + [1.5, np.nan, 3.], + [1.5, np.nan, 3.]]]]) + + assert_panel4d_equal(p4d, expected) + def test_filter(self): - pass + raise nose.SkipTest def test_apply(self): - pass + raise nose.SkipTest def test_compound(self): raise nose.SkipTest diff --git a/pandas/tests/test_panelnd.py b/pandas/tests/test_panelnd.py index 0d8a8c2023014..6cf0afe11c7e2 100644 --- a/pandas/tests/test_panelnd.py +++ b/pandas/tests/test_panelnd.py @@ -36,6 +36,32 @@ def test_4d_construction(self): p4d = Panel4D(dict(L1 = tm.makePanel(), L2 = tm.makePanel())) + def test_4d_construction_alt(self): + + # create a 4D + Panel4D = panelnd.create_nd_panel_factory( + klass_name = 'Panel4D', + axis_orders = ['labels','items','major_axis','minor_axis'], + axis_slices = { 'items' : 'items', 'major_axis' : 'major_axis', 'minor_axis' : 'minor_axis' }, + slicer = 'Panel', + axis_aliases = { 'major' : 'major_axis', 'minor' : 'minor_axis' }, + stat_axis = 2) + + p4d = Panel4D(dict(L1 = tm.makePanel(), L2 = tm.makePanel())) + + def test_4d_construction_error(self): + + # create a 4D + self.assertRaises(Exception, + panelnd.create_nd_panel_factory, + klass_name = 'Panel4D', + axis_orders = ['labels','items','major_axis','minor_axis'], + axis_slices = { 'items' : 'items', 'major_axis' : 'major_axis', 'minor_axis' : 'minor_axis' }, + slicer = 'foo', + axis_aliases = { 'major' : 'major_axis', 'minor' : 'minor_axis' }, + stat_axis = 2) + + def test_5d_construction(self): # create a 4D