Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion doc/source/whatsnew/v0.16.2.txt
Original file line number Diff line number Diff line change
Expand Up @@ -168,4 +168,8 @@ Bug Fixes

- Bug where MySQL interface could not handle numeric table/column names (:issue:`10255`)

- Bug in ``read_csv`` with a ``date_parser`` that returned a ``datetime64`` array of other time resolution than ``[ns]`` (:issue:`10245`)

- Bug in ``read_csv`` with a ``date_parser`` that returned a ``datetime64`` array of other time resolution than ``[ns]`` (:issue:`10245`).

- Bug in ``read_hdf`` where ``auto_close`` could not be passed (:issue:`9327`).
- Bug in ``read_hdf`` where open stores could not be used (:issue:`10330`).
35 changes: 19 additions & 16 deletions pandas/io/pytables.py
Original file line number Diff line number Diff line change
Expand Up @@ -290,8 +290,6 @@ def read_hdf(path_or_buf, key, **kwargs):
return columns
iterator : optional, boolean, return an iterator, default False
chunksize : optional, nrows to include in iteration, return an iterator
auto_close : optional, boolean, should automatically close the store
when finished, default is False

Returns
-------
Expand All @@ -303,9 +301,6 @@ def read_hdf(path_or_buf, key, **kwargs):
if 'where' in kwargs:
kwargs['where'] = _ensure_term(kwargs['where'], scope_level=1)

f = lambda store, auto_close: store.select(
key, auto_close=auto_close, **kwargs)

if isinstance(path_or_buf, string_types):

try:
Expand All @@ -321,20 +316,28 @@ def read_hdf(path_or_buf, key, **kwargs):
# can't auto open/close if we are using an iterator
# so delegate to the iterator
store = HDFStore(path_or_buf, **kwargs)
try:
return f(store, True)
except:
auto_close = True

# if there is an error, close the store
try:
store.close()
except:
pass
elif isinstance(path_or_buf, HDFStore):
if not path_or_buf.is_open:
raise IOError('The HDFStore must be open for reading.')

raise
store = path_or_buf
auto_close = False
else:
raise NotImplementedError('Support for generic buffers has not been '
'implemented.')

try:
return store.select(key, auto_close=auto_close, **kwargs)
except:
# if there is an error, close the store
try:
store.close()
except:
pass

# a passed store; user controls open/close
f(path_or_buf, False)
raise


class HDFStore(StringMixin):
Expand Down
49 changes: 48 additions & 1 deletion pandas/io/tests/test_pytables.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
from pandas import (Series, DataFrame, Panel, MultiIndex, Categorical, bdate_range,
date_range, timedelta_range, Index, DatetimeIndex, TimedeltaIndex, isnull)

from pandas.io.pytables import _tables
from pandas.io.pytables import _tables, TableIterator
try:
_tables()
except ImportError as e:
Expand Down Expand Up @@ -4670,6 +4670,53 @@ def test_to_hdf_with_object_column_names(self):
assert(len(result))


def test_read_hdf_open_store(self):
# GH10330
# No check for non-string path_or-buf, and no test of open store
df = DataFrame(np.random.rand(4, 5),
index=list('abcd'),
columns=list('ABCDE'))
df.index.name = 'letters'
df = df.set_index(keys='E', append=True)

with ensure_clean_path(self.path) as path:
df.to_hdf(path, 'df', mode='w')
direct = read_hdf(path, 'df')
store = HDFStore(path, mode='r')
indirect = read_hdf(store, 'df')
tm.assert_frame_equal(direct, indirect)
self.assertTrue(store.is_open)
store.close()

def test_read_hdf_iterator(self):
df = DataFrame(np.random.rand(4, 5),
index=list('abcd'),
columns=list('ABCDE'))
df.index.name = 'letters'
df = df.set_index(keys='E', append=True)

with ensure_clean_path(self.path) as path:
df.to_hdf(path, 'df', mode='w', format='t')
direct = read_hdf(path, 'df')
iterator = read_hdf(path, 'df', iterator=True)
self.assertTrue(isinstance(iterator, TableIterator))
indirect = next(iterator.__iter__())
tm.assert_frame_equal(direct, indirect)

def test_read_hdf_errors(self):
df = DataFrame(np.random.rand(4, 5),
index=list('abcd'),
columns=list('ABCDE'))

with ensure_clean_path(self.path) as path:
self.assertRaises(IOError, read_hdf, path, 'key')
df.to_hdf(path, 'df')
store = HDFStore(path, mode='r')
store.close()
self.assertRaises(IOError, read_hdf, store, 'df')
with open(path, mode='r') as store:
self.assertRaises(NotImplementedError, read_hdf, store, 'df')

def _test_sort(obj):
if isinstance(obj, DataFrame):
return obj.reindex(sorted(obj.index))
Expand Down