diff --git a/returnn/datasets/meta.py b/returnn/datasets/meta.py
index 901fbec8d..0871baba7 100644
--- a/returnn/datasets/meta.py
+++ b/returnn/datasets/meta.py
@@ -1348,7 +1348,11 @@ def _get_data(self, dataset_key, dataset_seq_idx, data_key):
         if dataset_data_key is not None:
             return dataset.get_data(dataset_seq_idx, dataset_data_key)
         else:
-            return numpy.array([], self.data_dtypes[data_key])
+            data_dim = self.data_dims[data_key]
+            if data_dim[1] == 1:
+                # sparse data
+                return numpy.zeros((0,), dtype=self.data_dtypes[data_key])
+            return numpy.zeros((0, data_dim[0]), dtype=self.data_dtypes[data_key])
 
     def _collect_single_seq(self, seq_idx):
         """