diff --git a/doc/source/whatsnew/v0.16.2.txt b/doc/source/whatsnew/v0.16.2.txt index b4a1bc72ed386..15a1842c8c266 100644 --- a/doc/source/whatsnew/v0.16.2.txt +++ b/doc/source/whatsnew/v0.16.2.txt @@ -168,4 +168,8 @@ Bug Fixes - Bug where MySQL interface could not handle numeric table/column names (:issue:`10255`) -- Bug in ``read_csv`` with a ``date_parser`` that returned a ``datetime64`` array of other time resolution than ``[ns]`` (:issue:`10245`) + +- Bug in ``read_csv`` with a ``date_parser`` that returned a ``datetime64`` array of other time resolution than ``[ns]`` (:issue:`10245`). + +- Bug in ``read_hdf`` where ``auto_close`` could not be passed (:issue:`9327`). +- Bug in ``read_hdf`` where open stores could not be used (:issue:`10330`). diff --git a/pandas/io/pytables.py b/pandas/io/pytables.py index 8948592358636..eca855a38d725 100644 --- a/pandas/io/pytables.py +++ b/pandas/io/pytables.py @@ -290,8 +290,6 @@ def read_hdf(path_or_buf, key, **kwargs): return columns iterator : optional, boolean, return an iterator, default False chunksize : optional, nrows to include in iteration, return an iterator - auto_close : optional, boolean, should automatically close the store - when finished, default is False Returns ------- @@ -303,9 +301,6 @@ def read_hdf(path_or_buf, key, **kwargs): if 'where' in kwargs: kwargs['where'] = _ensure_term(kwargs['where'], scope_level=1) - f = lambda store, auto_close: store.select( - key, auto_close=auto_close, **kwargs) - if isinstance(path_or_buf, string_types): try: @@ -321,20 +316,28 @@ def read_hdf(path_or_buf, key, **kwargs): # can't auto open/close if we are using an iterator # so delegate to the iterator store = HDFStore(path_or_buf, **kwargs) - try: - return f(store, True) - except: + auto_close = True - # if there is an error, close the store - try: - store.close() - except: - pass + elif isinstance(path_or_buf, HDFStore): + if not path_or_buf.is_open: + raise IOError('The HDFStore must be open for reading.') - raise + store = path_or_buf + auto_close = False + else: + raise NotImplementedError('Support for generic buffers has not been ' + 'implemented.') + + try: + return store.select(key, auto_close=auto_close, **kwargs) + except: + # if there is an error, close the store + try: + store.close() + except: + pass - # a passed store; user controls open/close - f(path_or_buf, False) + raise class HDFStore(StringMixin): diff --git a/pandas/io/tests/test_pytables.py b/pandas/io/tests/test_pytables.py index f671e61e90084..720ebc1db1466 100644 --- a/pandas/io/tests/test_pytables.py +++ b/pandas/io/tests/test_pytables.py @@ -13,7 +13,7 @@ from pandas import (Series, DataFrame, Panel, MultiIndex, Categorical, bdate_range, date_range, timedelta_range, Index, DatetimeIndex, TimedeltaIndex, isnull) -from pandas.io.pytables import _tables +from pandas.io.pytables import _tables, TableIterator try: _tables() except ImportError as e: @@ -4670,6 +4670,53 @@ def test_to_hdf_with_object_column_names(self): assert(len(result)) + def test_read_hdf_open_store(self): + # GH10330 + # No check for non-string path_or-buf, and no test of open store + df = DataFrame(np.random.rand(4, 5), + index=list('abcd'), + columns=list('ABCDE')) + df.index.name = 'letters' + df = df.set_index(keys='E', append=True) + + with ensure_clean_path(self.path) as path: + df.to_hdf(path, 'df', mode='w') + direct = read_hdf(path, 'df') + store = HDFStore(path, mode='r') + indirect = read_hdf(store, 'df') + tm.assert_frame_equal(direct, indirect) + self.assertTrue(store.is_open) + store.close() + + def test_read_hdf_iterator(self): + df = DataFrame(np.random.rand(4, 5), + index=list('abcd'), + columns=list('ABCDE')) + df.index.name = 'letters' + df = df.set_index(keys='E', append=True) + + with ensure_clean_path(self.path) as path: + df.to_hdf(path, 'df', mode='w', format='t') + direct = read_hdf(path, 'df') + iterator = read_hdf(path, 'df', iterator=True) + self.assertTrue(isinstance(iterator, TableIterator)) + indirect = next(iterator.__iter__()) + tm.assert_frame_equal(direct, indirect) + + def test_read_hdf_errors(self): + df = DataFrame(np.random.rand(4, 5), + index=list('abcd'), + columns=list('ABCDE')) + + with ensure_clean_path(self.path) as path: + self.assertRaises(IOError, read_hdf, path, 'key') + df.to_hdf(path, 'df') + store = HDFStore(path, mode='r') + store.close() + self.assertRaises(IOError, read_hdf, store, 'df') + with open(path, mode='r') as store: + self.assertRaises(NotImplementedError, read_hdf, store, 'df') + def _test_sort(obj): if isinstance(obj, DataFrame): return obj.reindex(sorted(obj.index))