fix: cleanup old to_record_batch_reader (#4368)

Confusing and incomplete API, instead we just expose via the
`VortexFile::scan` method

---------

Signed-off-by: Andrew Duffy <[email protected]>

Author: a10y

docs/quickstart/python.rst

@@ -28,30 +28,14 @@ Vortex array:
    >>> vtx.nbytes
    141024
 
-Compress
---------
-
-Use :func:`~vortex.compress` to compress the Vortex array and check the relative size:
-
-.. doctest::
-
-   >>> cvtx = vx.compress(vtx)
-   >>> cvtx.nbytes
-   14298
-   >>> cvtx.nbytes / vtx.nbytes
-   0.10...
-
-Vortex uses nearly ten times fewer bytes than Arrow. Fewer bytes means more of your data fits in
-cache and RAM.
-
 Write
 -----
 
 Use :func:`~vortex.io.write` to write the Vortex array to disk:
 
 .. doctest::
 
-   >>> vortex.io.write(cvtx, "example.vortex")
+   >>> vortex.io.write(cvtx, "example.vortex") # doctest: +SKIP
 
 Small Vortex files (this one is just 71KiB) currently have substantial overhead relative to their
 size. This will be addressed shortly. On files with at least tens of megabytes of data, Vortex is
@@ -71,3 +55,17 @@ Use :func:`~vortex.open` to open and read the Vortex array from disk:
 .. doctest::
 
    >>> cvtx = vortex.open("example.vortex").scan().read_all()
+
+
+Vortex is architected to achieve fast random access, in many cases hundreds of times faster
+than what can be achieved with Parquet.
+
+If you have an external index that gives you specific rows to pull out of the Vortex file, you can skip a lot more
+IO and decoding and read just the data that is relevant to you:
+
+.. doctest::
+
+    >>> vf = vortex.open("example.vortex")
+    >>> # row indices must be ordered and unique
+    >>> result = vf.scan(indices=vortex.array([1, 2, 10])).read_all()
+    >>> assert len(result) == 3

vortex-python/src/dataset.rs

@@ -127,26 +127,20 @@ impl PyVortexDataset {
         Ok(PyArrayRef::from(array))
     }
 
-    #[pyo3(signature = (*, columns = None, row_filter = None, indices = None, split_by = None))]
+    #[pyo3(signature = (*, columns = None, row_filter = None, split_by = None))]
     pub fn to_record_batch_reader(
         self_: PyRef<Self>,
         columns: Option<Vec<Bound<'_, PyAny>>>,
         row_filter: Option<&Bound<'_, PyExpr>>,
-        indices: Option<PyArrayRef>,
         split_by: Option<usize>,
     ) -> PyResult<PyObject> {
-        let mut scan = self_
+        let scan = self_
             .vxf
             .scan()?
             .with_projection(projection_from_python(columns)?)
             .with_some_filter(filter_from_python(row_filter))
             .with_split_by(split_by.map(SplitBy::RowCount).unwrap_or(SplitBy::Layout));
 
-        if let Some(indices) = indices.map(|i| i.inner().clone()) {
-            let indices = indices.to_primitive()?.into_buffer();
-            scan = scan.with_row_indices(indices);
-        }
-
         // TODO(ngates): should we use multi-threaded read or not?
         let schema = Arc::new(scan.dtype()?.to_arrow_schema()?);
         let reader: Box<dyn RecordBatchReader + Send> =

vortex-python/test/test_file.py

@@ -45,6 +45,13 @@ def test_scan(vxf: VortexFile):
         pass
 
 
+def test_scan_with_indices(vxf: VortexFile):
+    total_rows = 0
+    for rb in vxf.scan(indices=vx.array([1, 10, 1_000, 999_999])):
+        total_rows += len(rb)
+    assert total_rows == 4
+
+
 def test_to_arrow_batch_size(vxf: VortexFile):
     assert len(list(vxf.to_arrow(batch_size=1_000_000))) == 1, "batch_size=1_000_000"
     assert len(list(vxf.to_arrow(batch_size=1_000))) == 1_000, "batch_size=1_000"