Addition of Filestore API (#1070)

Author: nguyenv

HISTORY.md

@@ -1,3 +1,7 @@
+# In Progress
+
+## API Changes
+* Addition of Filestore API [#1070](https://github.com/TileDB-Inc/TileDB-Py/pull/1070)
 # TileDB-Py 0.15.6 Release Notes
 
 ## TileDB Embedded updates:

doc/source/python-api.rst

@@ -211,6 +211,14 @@ VFS
 
    .. automethod:: __len__
 
+Filestore
+---------
+
+.. autoclass:: tiledb.Filestore
+   :members:
+
+   .. automethod:: __len__
+
 Version
 -------
 

setup.py

@@ -23,7 +23,7 @@
 ### DO NOT USE ON CI
 
 # Target branch: Note that this should be set to the current core release, not `dev`
-TILEDB_VERSION = "2.9.5"
+TILEDB_VERSION = "2.10.0"
 # allow overriding w/ environment variable
 TILEDB_VERSION = os.environ.get("TILEDB_VERSION") or TILEDB_VERSION
 

tiledb/__init__.py

@@ -78,6 +78,8 @@
     ChecksumSHA256Filter,
 )
 
+from .filestore import Filestore
+
 from .fragment import (
     FragmentInfoList,
     FragmentInfo,

tiledb/cc/enum.cc

@@ -81,6 +81,10 @@ void init_enums(py::module &m) {
       .value("INVALID", Object::Type::Invalid)
       .export_values();
 
+#define DMENUM(x) .value(#x, TILEDB_MIME_##x)
+  py::enum_<tiledb_mime_type_t>(m, "MIMEType") DMENUM(AUTODETECT) DMENUM(TIFF)
+      DMENUM(PDF);
+
   // test helpers to check enum name against typed value
   m.def("_enum_string", &tiledb::impl::type_to_str);
   m.def("_enum_string",

tiledb/cc/filestore.cc

@@ -0,0 +1,97 @@
+#include <tiledb/tiledb>
+#include <tiledb/tiledb_experimental>
+
+#include <pybind11/numpy.h>
+#include <pybind11/pybind11.h>
+#include <pybind11/pytypes.h>
+#include <pybind11/stl.h>
+
+#include "common.h"
+
+namespace libtiledbcpp {
+
+using namespace tiledb;
+using namespace tiledbpy::common;
+namespace py = pybind11;
+
+class Filestore {
+public:
+  // TODO this works, but isn't actually in use at the moment.
+  // we are still using tiledb.libtiledb.ArraySchema. when we switch to using
+  // tiledb.cc.ArraySchema, use this function instead.
+  static ArraySchema schema_create(const Context &ctx, const char *uri) {
+    tiledb_array_schema_t *schema;
+    tiledb_filestore_schema_create(ctx.ptr().get(), uri, &schema);
+    return ArraySchema(ctx, py::capsule(schema));
+  }
+
+  static void uri_import(const Context &ctx, const char *filestore_array_uri,
+                         const char *file_uri, tiledb_mime_type_t mime_type) {
+    ctx.handle_error(tiledb_filestore_uri_import(
+        ctx.ptr().get(), filestore_array_uri, file_uri, mime_type));
+  }
+
+  static void uri_export(const Context &ctx, const char *filestore_array_uri,
+                         const char *file_uri) {
+    ctx.handle_error(tiledb_filestore_uri_export(ctx.ptr().get(), file_uri,
+                                                 filestore_array_uri));
+  }
+
+  static void buffer_import(const Context &ctx, const char *filestore_array_uri,
+                            py::buffer buf, tiledb_mime_type_t mime_type) {
+
+    py::buffer_info buffer = buf.request();
+    ctx.handle_error(
+        tiledb_filestore_buffer_import(ctx.ptr().get(), filestore_array_uri,
+                                       buffer.ptr, py::len(buf), mime_type));
+  }
+
+  static py::bytes buffer_export(const Context &ctx,
+                                 const char *filestore_array_uri, size_t offset,
+                                 size_t size) {
+    py::array data = py::array(py::dtype::of<std::byte>(), size);
+    py::buffer_info buffer = data.request();
+
+    ctx.handle_error(tiledb_filestore_buffer_export(
+        ctx.ptr().get(), filestore_array_uri, offset, buffer.ptr, size));
+
+    auto np = py::module::import("numpy");
+    auto to_bytes = np.attr("ndarray").attr("tobytes");
+
+    return to_bytes(data);
+  }
+
+  static size_t size(const Context &ctx, const char *filestore_array_uri) {
+    size_t size;
+    ctx.handle_error(
+        tiledb_filestore_size(ctx.ptr().get(), filestore_array_uri, &size));
+    return size;
+  }
+
+  static const char *mime_type_to_str(tiledb_mime_type_t mime_type) {
+    const char *str;
+    tiledb_mime_type_to_str(mime_type, &str);
+    return str;
+  }
+
+  static tiledb_mime_type_t mime_type_from_str(const char *str) {
+    tiledb_mime_type_t mime_type;
+    tiledb_mime_type_from_str(str, &mime_type);
+    return mime_type;
+  }
+};
+
+void init_filestore(py::module &m) {
+  py::class_<Filestore>(m, "Filestore")
+      .def_static("_schema_create", &Filestore::schema_create)
+      .def_static("_uri_import", &Filestore::uri_import)
+      .def_static("_uri_export", &Filestore::uri_export)
+      .def_static("_buffer_import", &Filestore::buffer_import)
+      .def_static("_buffer_export", &Filestore::buffer_export)
+      .def_static("_size", &Filestore::size)
+      .def_static("_mime_type_to_str", &Filestore::mime_type_to_str)
+      .def_static("_mime_type_from_str", &Filestore::mime_type_from_str);
+  ;
+};
+
+}; // namespace libtiledbcpp

tiledb/cc/schema.cc

@@ -12,15 +12,12 @@ namespace py = pybind11;
 
 void init_schema(py::module &m) {
   py::class_<tiledb::ArraySchema>(m, "ArraySchema")
-      .def(py::init<Context &, tiledb_array_type_t>(),
-           py::keep_alive<1, 2>() /* ArraySchema keeps Context alive */)
-      .def(py::init<Context &, std::string &>(),
-           py::keep_alive<1, 2>() /* ArraySchema keeps Context alive */)
+      .def(py::init<Context &, tiledb_array_type_t>(), py::keep_alive<1, 2>())
+      .def(py::init<Context &, std::string &>(), py::keep_alive<1, 2>())
       .def(py::init<Context &, std::string &, tiledb_encryption_type_t,
                     std::string &>(),
-           py::keep_alive<1, 2>() /* ArraySchema keeps Context alive */)
-      // TODO .def(py::init<Context, py::capsule>) // tiledb_array_schema_t*
-      // signature
+           py::keep_alive<1, 2>())
+      .def(py::init<Context &, py::capsule>(), py::keep_alive<1, 2>())
 
       .def("dump", &ArraySchema::dump) // TODO add FILE* signature support?
       .def("dump",

tiledb/cc/tests/test_filestore.py

@@ -0,0 +1,74 @@
+import os
+import pytest
+
+import tiledb
+from tiledb import cc as lt
+
+
+@pytest.fixture
+def text_fname(tmp_path):
+    path = os.path.join(tmp_path, "text_fname")
+    vfs = tiledb.VFS()
+    vfs.touch(path)
+    with vfs.open(path, "wb") as fio:
+        fio.write(b"Simple text file.\n")
+        fio.write(b"With two lines.")
+    return path
+
+
+def test_lt_schema_create(text_fname):
+    ctx = lt.Context()
+    schema = lt.Filestore._schema_create(ctx, text_fname)
+    assert type(schema) == lt.ArraySchema
+
+
+def test_libtiledb_schema_create_buffer(tmp_path, text_fname):
+    ctx = lt.Context()
+    path = os.path.join(tmp_path, "test_libtiledb_schema_create_buffer")
+    schema = tiledb.ArraySchema.from_file(text_fname)
+    tiledb.Array.create(path, schema)
+
+    data = b"buffer"
+    lt.Filestore._buffer_import(ctx, path, data, lt.MIMEType.AUTODETECT)
+    assert bytes(data) == lt.Filestore._buffer_export(ctx, path, 0, len(data))
+    assert lt.Filestore._size(ctx, path) == len(data)
+
+    output_file = os.path.join(tmp_path, "output_file")
+    vfs = tiledb.VFS()
+    vfs.touch(output_file)
+    lt.Filestore._uri_export(ctx, path, output_file)
+    with vfs.open(output_file, "rb") as fio:
+        assert fio.read() == data
+
+
+def test_libtiledb_schema_create_uri(tmp_path, text_fname):
+    ctx = lt.Context()
+    path = os.path.join(tmp_path, "test_libtiledb_schema_create_uri")
+    schema = tiledb.ArraySchema.from_file(text_fname)
+    tiledb.Array.create(path, schema)
+
+    lt.Filestore._uri_import(ctx, path, text_fname, lt.MIMEType.AUTODETECT)
+    with open(text_fname, "rb") as text:
+        data = text.read()
+        assert data == lt.Filestore._buffer_export(ctx, path, 0, len(data))
+        assert lt.Filestore._size(ctx, path) == len(data)
+
+
+def test_mime_type():
+    to_str = {
+        lt.MIMEType.AUTODETECT: "AUTODETECT",
+        lt.MIMEType.TIFF: "image/tiff",
+        lt.MIMEType.PDF: "application/pdf",
+    }
+
+    for k in to_str:
+        assert lt.Filestore._mime_type_to_str(k) == to_str[k]
+
+    from_str = {
+        "AUTODETECT": lt.MIMEType.AUTODETECT,
+        "image/tiff": lt.MIMEType.TIFF,
+        "application/pdf": lt.MIMEType.PDF,
+    }
+
+    for k in from_str:
+        assert lt.Filestore._mime_type_from_str(k) == from_str[k]

tiledb/cc/tiledbcpp.cc

@@ -20,6 +20,7 @@ void init_config(py::module &);
 void init_enums(py::module &);
 void init_domain(py::module &m);
 void init_file_handle(py::module &);
+void init_filestore(py::module &m);
 void init_filter(py::module &);
 void init_group(py::module &);
 void init_object(py::module &m);
@@ -36,6 +37,7 @@ PYBIND11_MODULE(cc, m) {
   init_domain(m);
   init_enums(m);
   init_file_handle(m);
+  init_filestore(m);
   init_filter(m);
   init_group(m);
   init_object(m);