refactor: make cocoindex crate a sub crate under the workspace

Author: georgeh0

Cargo.toml

@@ -1,26 +1,8 @@
-[package]
-name = "cocoindex"
-# Version used for local development is always higher than others to take precedence.
-# Will be overridden for specific release versions.
-version = "999.0.0"
-edition = "2024"
-rust-version = "1.89"
-license = "Apache-2.0"
+[workspace]
+members = ["rust/cocoindex"]
+resolver = "2"
 
-[profile.release]
-codegen-units = 1
-strip = "symbols"
-lto = true
-
-[lib]
-name = "cocoindex_engine"
-crate-type = ["cdylib"]
-
-[features]
-default = ["legacy-states-v0"]
-legacy-states-v0 = []
-
-[dependencies]
+[workspace.dependencies]
 pyo3 = { version = "0.25.1", features = [
     "abi3-py311",
     "auto-initialize",
@@ -157,3 +139,8 @@ redis = { version = "0.31.0", features = ["tokio-comp", "connection-manager"] }
 expect-test = "1.5.0"
 encoding_rs = "0.8.35"
 tokio-util = { version = "0.7.16", features = ["rt"] }
+
+[profile.release]
+codegen-units = 1
+strip = "symbols"
+lto = true

pyproject.toml

@@ -60,6 +60,8 @@ python-source = "python"
 module-name = "cocoindex._engine"
 features = ["pyo3/extension-module"]
 include = ["THIRD_PARTY_NOTICES.html"]
+# Point to the crate within the workspace
+manifest-path = "rust/cocoindex/Cargo.toml"
 
 [project.optional-dependencies]
 dev = ["pytest", "pytest-asyncio", "ruff", "mypy", "pre-commit"]

rust/cocoindex/Cargo.toml

@@ -0,0 +1,123 @@
+[package]
+name = "cocoindex"
+# Version used for local development is always higher than others to take precedence.
+# Will be overridden for specific release versions.
+version = "999.0.0"
+edition = "2024"
+rust-version = "1.89"
+license = "Apache-2.0"
+
+[lib]
+name = "cocoindex_engine"
+crate-type = ["cdylib"]
+
+[features]
+default = ["legacy-states-v0"]
+legacy-states-v0 = []
+
+[dependencies]
+pyo3 = { workspace = true }
+pythonize = { workspace = true }
+pyo3-async-runtimes = { workspace = true }
+
+anyhow = { workspace = true }
+async-trait = { workspace = true }
+axum = { workspace = true }
+axum-extra = { workspace = true }
+base64 = { workspace = true }
+chrono = { workspace = true }
+config = { workspace = true }
+const_format = { workspace = true }
+futures = { workspace = true }
+log = { workspace = true }
+regex = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+sqlx = { workspace = true }
+tokio = { workspace = true }
+tower = { workspace = true }
+tower-http = { workspace = true }
+indexmap = { workspace = true }
+blake2 = { workspace = true }
+pgvector = { workspace = true }
+phf = { workspace = true }
+indenter = { workspace = true }
+indicatif = { workspace = true }
+itertools = { workspace = true }
+derivative = { workspace = true }
+hex = { workspace = true }
+schemars = { workspace = true }
+env_logger = { workspace = true }
+reqwest = { workspace = true }
+async-openai = { workspace = true }
+
+tree-sitter = { workspace = true }
+tree-sitter-language = { workspace = true }
+# Per language tree-sitter parsers
+tree-sitter-c = { workspace = true }
+tree-sitter-cpp = { workspace = true }
+tree-sitter-c-sharp = { workspace = true }
+tree-sitter-css = { workspace = true }
+tree-sitter-fortran = { workspace = true }
+tree-sitter-go = { workspace = true }
+tree-sitter-html = { workspace = true }
+tree-sitter-java = { workspace = true }
+tree-sitter-javascript = { workspace = true }
+tree-sitter-json = { workspace = true }
+# The other more popular crate tree-sitter-kotlin requires tree-sitter < 0.23 for now
+tree-sitter-kotlin-ng = { workspace = true }
+tree-sitter-md = { workspace = true }
+tree-sitter-pascal = { workspace = true }
+tree-sitter-php = { workspace = true }
+tree-sitter-python = { workspace = true }
+tree-sitter-r = { workspace = true }
+tree-sitter-ruby = { workspace = true }
+tree-sitter-rust = { workspace = true }
+tree-sitter-scala = { workspace = true }
+tree-sitter-sequel = { workspace = true }
+tree-sitter-swift = { workspace = true }
+tree-sitter-toml-ng = { workspace = true }
+tree-sitter-typescript = { workspace = true }
+tree-sitter-xml = { workspace = true }
+tree-sitter-yaml = { workspace = true }
+tree-sitter-solidity = { workspace = true }
+
+globset = { workspace = true }
+unicase = { workspace = true }
+google-drive3 = { workspace = true }
+hyper-util = { workspace = true }
+hyper-rustls = { workspace = true }
+yup-oauth2 = { workspace = true }
+rustls = { workspace = true }
+http-body-util = { workspace = true }
+yaml-rust2 = { workspace = true }
+urlencoding = { workspace = true }
+qdrant-client = { workspace = true }
+uuid = { workspace = true }
+tokio-stream = { workspace = true }
+async-stream = { workspace = true }
+neo4rs = { workspace = true }
+bytes = { workspace = true }
+rand = { workspace = true }
+indoc = { workspace = true }
+owo-colors = { workspace = true }
+json5 = { workspace = true }
+aws-config = { workspace = true }
+aws-sdk-s3 = { workspace = true }
+aws-sdk-sqs = { workspace = true }
+time = { workspace = true }
+numpy = { workspace = true }
+infer = { workspace = true }
+serde_with = { workspace = true }
+google-cloud-aiplatform-v1 = { workspace = true }
+google-cloud-gax = { workspace = true }
+
+azure_identity = { workspace = true }
+azure_core = { workspace = true }
+azure_storage = { workspace = true }
+azure_storage_blobs = { workspace = true }
+serde_path_to_error = { workspace = true }
+redis = { workspace = true }
+expect-test = { workspace = true }
+encoding_rs = { workspace = true }
+tokio-util = { workspace = true }