append xarray dataset to an existing netcdf file

[gewitterblitz]

Hi,

I have been unsuccessful at adding an xarray dataset to an existing netcdf file (previously created using xarray). Sample code and error message below:

# open hdf files to be added to existing netcdf file
ds = xr.open_mfdataset(filelist,engine='pynio',parallel=True,)

# rename dimensions in the dataset to match with dimensions in netcdf file
ds = ds.rename_dims({'fakeDim0':'zc','fakeDim1':'yc','fakeDim2':'xc'})

# assign same coordinates as dataset stored as netcdf file
ds1 = ds.assign_coords(coords=tmp.coords)

# append dataset to existing file
ds1.to_netcdf('existing_netcdf.nc',mode='a')

Error message:

---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
~/.conda/envs/cent7/2020.02-py37/arps/lib/python3.8/site-packages/xarray/backends/file_manager.py in _acquire_with_cache_info(self, needs_lock)
    198             try:
--> 199                 file = self._cache[self._key]
    200             except KeyError:

~/.conda/envs/cent7/2020.02-py37/arps/lib/python3.8/site-packages/xarray/backends/lru_cache.py in __getitem__(self, key)
     52         with self._lock:
---> 53             value = self._cache[key]
     54             self._cache.move_to_end(key)

KeyError: [<class 'netCDF4._netCDF4.Dataset'>, ('/depot/rtanama/users/sharm261/VORTEX_SE_cases/vortexse_analysis/notebooks/20160331/ARPS_analysis/existing_netcdf_file.nc',), 'a', (('clobber', True), ('diskless', False), ('format', 'NETCDF4'), ('persist', False))]

During handling of the above exception, another exception occurred:

OSError                                   Traceback (most recent call last)
<ipython-input-18-a06ca9a743da> in <module>
----> 1 ds1.to_netcdf('existing_netcdf_file.nc',mode='a')

~/.conda/envs/cent7/2020.02-py37/arps/lib/python3.8/site-packages/xarray/core/dataset.py in to_netcdf(self, path, mode, format, group, engine, encoding, unlimited_dims, compute, invalid_netcdf)
   1900         from ..backends.api import to_netcdf
   1901 
-> 1902         return to_netcdf(
   1903             self,
   1904             path,

~/.conda/envs/cent7/2020.02-py37/arps/lib/python3.8/site-packages/xarray/backends/api.py in to_netcdf(dataset, path_or_file, mode, format, group, engine, encoding, unlimited_dims, compute, multifile, invalid_netcdf)
   1053                 f"unrecognized option 'invalid_netcdf' for engine {engine}"
   1054             )
-> 1055     store = store_open(target, mode, format, group, **kwargs)
   1056 
   1057     if unlimited_dims is None:

~/.conda/envs/cent7/2020.02-py37/arps/lib/python3.8/site-packages/xarray/backends/netCDF4_.py in open(cls, filename, mode, format, group, clobber, diskless, persist, lock, lock_maker, autoclose)
    377             netCDF4.Dataset, filename, mode=mode, kwargs=kwargs
    378         )
--> 379         return cls(manager, group=group, mode=mode, lock=lock, autoclose=autoclose)
    380 
    381     def _acquire(self, needs_lock=True):

~/.conda/envs/cent7/2020.02-py37/arps/lib/python3.8/site-packages/xarray/backends/netCDF4_.py in __init__(self, manager, group, mode, lock, autoclose)
    325         self._group = group
    326         self._mode = mode
--> 327         self.format = self.ds.data_model
    328         self._filename = self.ds.filepath()
    329         self.is_remote = is_remote_uri(self._filename)

~/.conda/envs/cent7/2020.02-py37/arps/lib/python3.8/site-packages/xarray/backends/netCDF4_.py in ds(self)
    386     @property
    387     def ds(self):
--> 388         return self._acquire()
    389 
    390     def open_store_variable(self, name, var):

~/.conda/envs/cent7/2020.02-py37/arps/lib/python3.8/site-packages/xarray/backends/netCDF4_.py in _acquire(self, needs_lock)
    380 
    381     def _acquire(self, needs_lock=True):
--> 382         with self._manager.acquire_context(needs_lock) as root:
    383             ds = _nc4_require_group(root, self._group, self._mode)
    384         return ds

~/.conda/envs/cent7/2020.02-py37/arps/lib/python3.8/contextlib.py in __enter__(self)
    111         del self.args, self.kwds, self.func
    112         try:
--> 113             return next(self.gen)
    114         except StopIteration:
    115             raise RuntimeError("generator didn't yield") from None

~/.conda/envs/cent7/2020.02-py37/arps/lib/python3.8/site-packages/xarray/backends/file_manager.py in acquire_context(self, needs_lock)
    185     def acquire_context(self, needs_lock=True):
    186         """Context manager for acquiring a file."""
--> 187         file, cached = self._acquire_with_cache_info(needs_lock)
    188         try:
    189             yield file

~/.conda/envs/cent7/2020.02-py37/arps/lib/python3.8/site-packages/xarray/backends/file_manager.py in _acquire_with_cache_info(self, needs_lock)
    203                     kwargs = kwargs.copy()
    204                     kwargs["mode"] = self._mode
--> 205                 file = self._opener(*self._args, **kwargs)
    206                 if self._mode == "w":
    207                     # ensure file doesn't get overriden when opened again

netCDF4/_netCDF4.pyx in netCDF4._netCDF4.Dataset.__init__()

netCDF4/_netCDF4.pyx in netCDF4._netCDF4._ensure_nc_success()

OSError: [Errno -101] NetCDF: HDF error: b'/depot/rtanama/users/sharm261/VORTEX_SE_cases/vortexse_analysis/notebooks/20160331/ARPS_analysis/existing_netcdf_file.nc'

What am I doing wrong?

[dcherian]

OSError and HDFError usually mean a corrupt file. Can you read the existing netcdf?

[gewitterblitz]

Yes, I was able to read the file. Also tried changing the read/write permissions of the directory containing this file but it did not work.

Although, when I used .to_netcdf to write the same dataset to a new file file_new.nc, I was able to append new dataset to this newly created netcdf file. I have no clue what that happened.

[lassiterdc]

Were you ever able to resolve this without having to first re-write the netcdf to a new file? I'm running into a similar issue - I have a netcdf that I can open just fine, but I run into the same error when I call to_netcdf() with mode='a'