Enable Quarto rendering for docs and reference (#51)

* Enable Quarto rendering for docs and reference

This enabled quarto for reference docs and in-repo documentation and
examples

* Default c++ unit tests to disabled for python setup

* Add missing pyyaml package

* Add missing click package

* WIP: Temp allow pushing docs on this branch

* Update doc strings

* Formatting

* Add index

* Adjust README and repo rename

* Initial installation and building docs

* Minor updates

* Re-enable limiting to main

* Clean up, remove duplication

Author: Shelnutt2

.github/workflows/quarto-render.yml

@@ -0,0 +1,55 @@
+# Cloned from https://github.com/TileDB-Inc/tiledb-quarto-template
+
+name: Render and deploy Quarto files
+on: 
+  push:
+  pull_request:
+
+jobs:
+  quarto-render-and-deploy:
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.9"]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Install OpenBLAS
+        run: sudo apt install libopenblas-dev
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Print Python version
+        run: |
+          which python
+          which pip
+          python --version
+      - name: Install pybind11
+        run: pip install "pybind11[global]"
+      - uses: actions/checkout@v3
+      - name: Install
+        run: cd apis/python && pip install .
+
+      - name: "Install Quarto"
+        uses: quarto-dev/quarto-actions/setup@v2
+
+      - name: "Quarto render"
+        shell: bash
+        run: |
+          pip install quartodoc PyYAML click
+          # create a symlink to the tiledbvcf python package, so it doesn't have to be installed
+          #ln -s apis/python/src/tiledb/vector_search
+          quartodoc build
+          quarto render --fail-if-warnings
+          # https://github.com/quarto-dev/quarto-cli/issues/493
+
+      - name: "Deploy to gh-pages"
+        uses: peaceiris/actions-gh-pages@v3
+        # Change to the name of your repo's primary branch name:
+        if: github.ref == 'refs/heads/main'
+        with:
+          # This is GitHub Actions magic; no secrets for us to manage; and this works first-time
+          # without any extra configs other than visiting Settings -> Pages in your GitHub repo.
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: docs
+          destination_dir: docs

.gitignore

@@ -94,3 +94,10 @@ apis/python/.mypy_cache/
 .idea
 apis/python/src/tiledb/vector_search/version.py
 _skbuild
+
+# Quarto dev-cache
+/.quarto/
+docs/.quarto
+
+documentation/reference/*
+documentation/reference

README.md

@@ -1 +1,35 @@
-TileDB vector similarity search
+<a href="https://tiledb.com"><img src="https://github.com/TileDB-Inc/TileDB/raw/dev/doc/source/_static/[email protected]" alt="TileDB logo" width="400"></a>
+
+
+
+# TileDB Vector Search
+
+*TileDB-Vector-Search* is a C++ library and Python interface to Vector Search built onto of the [TileDB Storage Engine](https://github.com/TileDB-Inc/TileDB).
+
+# Quick Links
+
+* [Installation](https://tiledb-inc.github.io/tiledb-vector-search/documentation/Installation.html)
+* [Build Instructions](https://tiledb-inc.github.io/TileDB-Vector-Search/documentation/Building.html)
+* [TileDB Documentation](https://tiledb-inc.github.io/TileDB-Vector-Search/)
+* [Python API reference](https://tiledb-inc.github.io/TileDB-Vector-Search/documentation/reference/)
+
+# Quick Installation
+
+TileDB-Vector-Search is available from either [PyPI](https://pypi.org/project/tiledb-vector-search/) with ``pip``:
+
+```
+pip install tiledb-vector-search
+```
+
+or from [conda-forge](https://anaconda.org/conda-forge/tiledb-vector-searcg) with
+[conda](https://conda.io/docs/) or [mamba](https://github.com/mamba-org/mamba#installation):
+
+```
+conda install -c tiledb tiledb-vector-search
+```
+
+# Contributing
+
+We welcome contributions, please see [`CONTRIBUTING.md`](CONTRIBUTING.md) for suggestions and
+development-build instructions. For larger features, please open an issue to discuss goals and
+approach in order to ensure a smooth PR integration and review process.

_quarto.yml

@@ -0,0 +1,83 @@
+# Cloned from https://github.com/TileDB-Inc/tiledb-quarto-template
+project:
+  type: website
+  output-dir: docs
+  render:
+    # The first item will be the landing page
+    - "documentation/index.md"
+    - "documentation"
+    - "documentation/reference"
+    - "examples/README.md"
+    - "examples"
+
+format:
+  html:
+    # https://quarto.org/docs/output-formats/page-layout.html#grid-customization
+    grid:
+      sidebar-width: 350px
+      body-width: 1000px
+      margin-width: 350px
+      gutter-width: 1rem
+    toc: true
+    theme:
+      light: [flatly, "quarto-materials/tiledb.scss"]
+    mainfont: Helvetica
+    fontsize: 1rem
+    linkcolor: "#4d9fff"
+    code-copy: true
+    code-overflow: wrap
+    css: "quarto-materials/tiledb.css"
+
+quartodoc:
+  title: "Python"
+  package: tiledb.vector_search
+  dir: "documentation/reference"
+  sections:
+    - title: "tiledb.vector_search"
+      desc: ""
+      contents:
+          - index.Index
+          - index.FlatIndex
+          - index.IVFFlatIndex
+          - ingestion
+
+website:
+  favicon: "documentation/assets/tiledb.ico"
+  site-url: https://tiledb-inc.github.io/TileDB-Vector-Search/
+  repo-url: https://github.com/TileDB-Inc/TileDB-Vector-Search
+  # You may want one or both of these, or neither:
+  repo-actions: [edit, issue]
+  page-navigation: true
+  navbar:
+    background: light
+    logo: "quarto-materials/tiledb-logo.png"
+    collapse-below: lg
+    left:
+      - text: "Home page"
+        href: "https://tiledb.com"
+      - text: "Login"
+        href: "https://cloud.tiledb.com/auth/login"
+      - text: "Contact us"
+        href: "https://tiledb.com/contact"
+      - text: "Repo"
+        href: "https://github.com/TileDB-Inc/TileDB-Vector-Search"
+
+  sidebar:
+    - style: "floating"
+      collapse-level: 2
+      align: left
+      contents:
+        - href: "documentation/index.md"
+        - href: "documentation/Building.md"
+
+        # - section: "Examples"
+        #   contents:
+        #     # - href: "examples/overview.md"
+
+        # - section: "Ingestion"
+        #   contents:
+        #     # - href: "documentation/ingestion/overview.md"
+
+        - section: "API Reference"
+          contents:
+            - href: "documentation/reference/index.qmd"

apis/python/pyproject.toml

@@ -28,7 +28,7 @@ test = ["pytest"]
 
 [project.urls]
 homepage = "https://tiledb.com"
-repository = "https://github.com/TileDB-Inc/feature-vector-prototype"
+repository = "https://github.com/TileDB-Inc/tiledb-vector-search"
 
 [build-system]
 requires = [

apis/python/setup.py

@@ -1,9 +1,9 @@
+import os
+
 from skbuild import setup  # This line replaces 'from setuptools import setup'
 
-import os
 
 def get_cmake_overrides():
-    import sys
 
     conf = list()
 
@@ -23,9 +23,8 @@ def get_cmake_overrides():
         conf.append("-DCMAKE_BUILD_TYPE={}".format(val))
 
     key = "BUILD_TESTS"
-    val = os.environ.get(key, default=None)
-    if val:
-        conf.append("-DBUILD_TESTS={}".format(val))
+    val = os.environ.get(key, default="OFF")
+    conf.append("-DBUILD_TESTS={}".format(val))
 
     key = "USE_MKL_CBLAS"
     val = os.environ.get(key, default=None)
@@ -34,6 +33,7 @@ def get_cmake_overrides():
 
     return conf
 
+
 cmake_args = ["-DTILEDB_VS_PYTHON=ON"]
 
 cmake_args += get_cmake_overrides()

apis/python/src/tiledb/vector_search/__init__.py

@@ -1,5 +1,14 @@
 from .index import FlatIndex
 from .ingestion import ingest
-from .module import load_as_array, load_as_matrix, query_vq
+from .module import load_as_array
+from .module import load_as_matrix
+from .module import query_vq
 
-__all__ = ["FlatIndex", "load_as_array", "load_as_matrix", "ingest", "query_vq", "query_kmeans"]
+__all__ = [
+    "FlatIndex",
+    "load_as_array",
+    "load_as_matrix",
+    "ingest",
+    "query_vq",
+    "query_kmeans",
+]

apis/python/src/tiledb/vector_search/index.py

@@ -1,6 +1,7 @@
 import os
 
 import numpy as np
+
 from tiledb.vector_search.module import *
 
 
@@ -10,17 +11,50 @@ def query(self, targets: np.ndarray, k=10, nqueries=10, nthreads=8, nprobe=1):
 
 
 class FlatIndex(Index):
-    def __init__(self,
-                 uri,
-                 dtype: np.dtype,
-                 parts_name="parts.tdb"):
+    """
+    Open a flat index
+
+    Parameters
+    ----------
+    uri: str
+        URI of datataset
+    dtype: numpy.dtype
+        datatype float32 or uint8
+    parts_name: str
+        Optional name of partitions
+    """
+
+    def __init__(self, uri: str, dtype: np.dtype, parts_name: str = "parts.tdb"):
         self.uri = uri
         self.dtype = dtype
         self._index = None
 
         self._db = load_as_matrix(os.path.join(uri, parts_name))
 
-    def query(self, targets: np.ndarray, k=10, nqueries=10, nthreads=8, nprobe=1):
+    def query(
+        self,
+        targets: np.ndarray,
+        k: int = 10,
+        nqueries: int = 10,
+        nthreads: int = 8,
+        nprobe: int = 1,
+    ):
+        """
+        Open a flat index
+
+        Parameters
+        ----------
+        targets: numpy.ndarray
+            ND Array of query targets
+        k: int
+            Number of top results to return per target
+        nqueries: int
+            Number of queries
+        nthreads: int
+            Number of threads to use for queyr
+        nprobe: int
+            number of probes
+        """
         # TODO:
         # - typecheck targets
         # - don't copy the array
@@ -39,9 +73,18 @@ def query(self, targets: np.ndarray, k=10, nqueries=10, nthreads=8, nprobe=1):
 
 
 class IVFFlatIndex(Index):
-    def __init__(self,
-                 uri,
-                 dtype: np.dtype):
+    """
+    Open a IVF Flat index
+
+    Parameters
+    ----------
+    uri: str
+        URI of datataset
+    dtype: numpy.dtype
+        datatype float32 or uint8
+    """
+
+    def __init__(self, uri, dtype: np.dtype):
         self.parts_db_uri = os.path.join(uri, "parts.tdb")
         self.centroids_uri = os.path.join(uri, "centroids.tdb")
         self.index_uri = os.path.join(uri, "index.tdb")
@@ -55,6 +98,22 @@ def __init__(self,
         # self._ids = load_as_matrix(self.ids_uri)
 
     def query(self, targets: np.ndarray, k=10, nqueries=10, nthreads=8, nprobe=1):
+        """
+        Open a flat index
+
+        Parameters
+        ----------
+        targets: numpy.ndarray
+            ND Array of query targets
+        k: int
+            Number of top results to return per target
+        nqueries: int
+            Number of queries
+        nthreads: int
+            Number of threads to use for queyr
+        nprobe: int
+            number of probes
+        """
         assert targets.dtype == np.float32
 
         # TODO: use Matrix constructor from py::array