fix: FFI example fixed up, hooked into CI (#3684)

Author: a10y

.github/workflows/ci.yml

@@ -421,6 +421,15 @@ jobs:
         with:
           command: check ${{ matrix.checks }}
 
+  ffi-test:
+    name: "FFI example test"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup-rust
+      - run: cargo build -p vortex-ffi
+      - run: cargo run -p vortex-ffi --example hello_vortex
+
   wasm-integration:
     name: "wasm-integration"
     runs-on: ubuntu-latest

Cargo.lock

@@ -17,7 +17,6 @@ rust-version = { workspace = true }
 categories = { workspace = true }
 
 [dependencies]
-futures = { workspace = true }
 itertools = { workspace = true }
 log = { workspace = true }
 mimalloc = { workspace = true, optional = true }
@@ -31,6 +30,9 @@ tokio-stream = { workspace = true }
 url = { workspace = true, features = [] }
 vortex = { workspace = true, features = ["object_store", "proto"] }
 
+[dev-dependencies]
+tempfile = { workspace = true }
+
 [features]
 default = ["mimalloc"]
 mimalloc = ["dep:mimalloc"]

vortex-ffi/Cargo.toml

@@ -3,40 +3,65 @@
 #include "vortex.h"
 
 int main(int argc, char *argv[]) {
-    // enable logging
-    vortex_init_logging(LOG_LEVEL_INFO);
-
     if (argc < 2) {
         printf("Usage: %s <VORTEX_FILE_URI>\n", argv[0]);
         return 1;
     }
 
+    vx_error *error = NULL;
+    vx_session *session = vx_session_new();
+    if (session == NULL) {
+        fprintf(stderr, "vx_session_new return NULL\n");
+        return -1;
+    }
+
     // Open the file
     char *path = argv[1];
-    FileOpenOptions open_opts = {
+
+    printf("Opening file %s\n", path);
+
+    vx_file_open_options open_opts = {
       .uri = path,
       .property_keys = NULL,
       .property_vals = NULL,
       .property_len = 0,
     };
     printf("Scanning file: %s\n", path);
-    File *file = File_open(&open_opts);
+
+    const vx_file *file = vx_file_open_reader(&open_opts, session, &error);
+    if (error != NULL) {
+        fprintf(stderr, "error opening file\n");
+
+        return -1;
+    }
 
     // Start scanning, read new rows.
-    ArrayStream *stream = File_scan(file, NULL);
+    vx_array_iterator *scan = vx_file_scan(file, NULL, &error);
+
     int chunk = 0;
-    while (FFIArrayStream_next(stream)) {
-        Array *array = FFIArrayStream_current(stream);
-        int len = FFIArray_len(array);
-        printf("Chunk %d: %d\n", chunk++, len);
-        FFIArray_free(array);
+    const vx_array *batch = vx_array_iterator_next(scan, &error);
+    while (batch != NULL) {
+        size_t len = vx_array_len(batch);
+        printf("chunk %d has length %ld\n", chunk++, len);
+
+        // free the batch
+        vx_array_free(batch);
+        // grab the next batch.
+        batch = vx_array_iterator_next(scan, &error);
     }
 
-    printf("Scanning complete\n");
+    vx_array_iterator_free(scan);
+    vx_file_free(file);
+
+    if (error != NULL) {
+        fprintf(stderr, "failed in scan operation\n");
+        vx_session_free(session);
+        vx_error_free(error);
+        return -1;
+    }
 
-    // Cleanup resources.
-    FFIArrayStream_free(stream);
-    File_free(file);
+    printf("Scanning completed successfully\n");
 
+    vx_session_free(session);
     return 0;
 }

vortex-ffi/examples/hello-vortex.c

@@ -0,0 +1,108 @@
+#![allow(clippy::unwrap_used, clippy::expect_used)]
+//! This example shows usage of the Vortex C FFI to read a Vortex file written by a Rust client.
+//!
+//! You can invoke this example from a checkout by running
+//!
+//! ```ignore
+//!cargo run -p vortex-ffi --example hello_vortex
+//! ```
+
+use std::env;
+use std::path::{Path, PathBuf};
+use std::process::Command;
+use std::sync::LazyLock;
+
+use tempfile::NamedTempFile;
+use tokio::fs::File as TokioFile;
+use tokio::runtime::Runtime;
+use vortex::arrays::{ChunkedArray, StructArray};
+use vortex::buffer::Buffer;
+use vortex::error::VortexResult;
+use vortex::file::VortexWriteOptions;
+use vortex::{Array, ArrayRef, IntoArray};
+
+static RUNTIME: LazyLock<Runtime> = LazyLock::new(|| Runtime::new().unwrap());
+
+const BIN_NAME: &str = "hello_vortex";
+
+pub fn main() -> VortexResult<()> {
+    let bin_path = PathBuf::new()
+        .join(env!("CARGO_MANIFEST_DIR"))
+        .join("examples")
+        .join(BIN_NAME);
+
+    let exit = Command::new("cc")
+        .arg("-o")
+        .arg(&bin_path)
+        .arg("-I")
+        .arg(
+            PathBuf::new()
+                .join(env!("CARGO_MANIFEST_DIR"))
+                .join("cinclude")
+                .display()
+                .to_string(),
+        )
+        .arg(
+            PathBuf::new()
+                .join(env!("CARGO_MANIFEST_DIR"))
+                .join("examples")
+                .join("hello-vortex.c")
+                .display()
+                .to_string(),
+        )
+        .arg("-L")
+        .arg(
+            PathBuf::new()
+                .join(env!("CARGO_MANIFEST_DIR"))
+                .join("..")
+                .join("target")
+                .join("debug"),
+        )
+        .arg("-l")
+        .arg("vortex_ffi")
+        .status()?;
+
+    assert!(exit.success());
+
+    let file = NamedTempFile::with_suffix(".vortex")?.into_temp_path();
+
+    // Write the test data
+    RUNTIME.block_on(write_vortex_file(&file))?;
+
+    let uri = format!("file://{}", file.display());
+
+    // Invoke the binary we just created
+    let exit = Command::new(&bin_path).arg(uri).status()?;
+    assert!(exit.success());
+
+    Ok(())
+}
+
+async fn write_vortex_file(path: impl AsRef<Path>) -> VortexResult<()> {
+    let file = TokioFile::create(path).await?;
+
+    let chunk1 = chunk((0..1000).collect(), (0..1000).map(|x| x as f32).collect());
+    let chunk2 = chunk(
+        (1000..2000).collect(),
+        (1000..2000).map(|x| x as f32).collect(),
+    );
+    let chunk3 = chunk(
+        (2000..3000).collect(),
+        (2000..3000).map(|x| x as f32).collect(),
+    );
+    let dtype = chunk1.dtype().clone();
+
+    let test_data = ChunkedArray::try_new(vec![chunk1, chunk2, chunk3], dtype)?;
+
+    VortexWriteOptions::default()
+        .write(file, test_data.to_array_stream())
+        .await?;
+
+    Ok(())
+}
+
+fn chunk(nums: Buffer<i32>, floats: Buffer<f32>) -> ArrayRef {
+    StructArray::try_from_iter([("nums", nums.into_array()), ("floats", floats.into_array())])
+        .unwrap()
+        .into_array()
+}