python3Packages.chromadb: 0.6.3 -> 1.0.12

release notes: https://github.com/chroma-core/chroma/releases/tag/1.0.12

Author: sarahec

pkgs/development/python-modules/chromadb/default.nix

@@ -3,17 +3,15 @@
   stdenv,
   buildPythonPackage,
   fetchFromGitHub,
-
-  # build-system
-  setuptools-scm,
-  setuptools,
+  fetchurl,
 
   # build inputs
   cargo,
   pkg-config,
   protobuf,
   rustc,
   rustPlatform,
+  pkgs, # zstd hidden by python3Packages.zstd
   openssl,
 
   # dependencies
@@ -23,6 +21,7 @@
   grpcio,
   httpx,
   importlib-resources,
+  jsonschema,
   kubernetes,
   mmh3,
   numpy,
@@ -45,16 +44,20 @@
   typer,
   typing-extensions,
   uvicorn,
-  zstd,
 
   # optional dependencies
   chroma-hnswlib,
 
   # tests
+  hnswlib,
   hypothesis,
+  pandas,
   psutil,
   pytest-asyncio,
+  pytest-xdist,
   pytestCheckHook,
+  sqlite,
+  starlette,
 
   # passthru
   nixosTests,
@@ -63,31 +66,45 @@
 
 buildPythonPackage rec {
   pname = "chromadb";
-  version = "0.6.3";
+  version = "1.0.12";
   pyproject = true;
 
   src = fetchFromGitHub {
     owner = "chroma-core";
     repo = "chroma";
     tag = version;
-    hash = "sha256-yvAX8buETsdPvMQmRK5+WFz4fVaGIdNlfhSadtHwU5U=";
+    hash = "sha256-Q4PhJTRNzJeVx6DIPWirnI9KksNb8vfOtqb/q9tSK3c=";
   };
 
   cargoDeps = rustPlatform.fetchCargoVendor {
     inherit src;
-    name = "${pname}-${version}";
-    hash = "sha256-lHRBXJa/OFNf4x7afEJw9XcuDveTBIy3XpQ3+19JXn4=";
+    name = "${pname}-${version}-vendor";
+    hash = "sha256-+Ea2aRrsBGfVCLdOF41jeMehJhMurc8d0UKrpR6ndag=";
+  };
+
+  # Can't use fetchFromGitHub as the build expects a zipfile
+  swagger-ui = fetchurl {
+    url = "https://github.com/swagger-api/swagger-ui/archive/refs/tags/v5.22.0.zip";
+    hash = "sha256-H+kXxA/6rKzYA19v7Zlx2HbIg/DGicD5FDIs0noVGSk=";
   };
 
+  patches = [
+    # The fastapi servers can't set up their networking in the test environment, so disable for testing
+    ./disable-fastapi-fixtures.patch
+  ];
+
+  postPatch = ''
+    # Nixpkgs is taking the version from `chromadb_rust_bindings` which is versioned independently
+    substituteInPlace pyproject.toml \
+      --replace-fail "dynamic = [\"version\"]" "version = \"${version}\""
+  '';
+
   pythonRelaxDeps = [
-    "chroma-hnswlib"
-    "orjson"
-    "tokenizers"
+    "fastapi"
   ];
 
   build-system = [
-    setuptools
-    setuptools-scm
+    rustPlatform.maturinBuildHook
   ];
 
   nativeBuildInputs = [
@@ -100,17 +117,17 @@ buildPythonPackage rec {
 
   buildInputs = [
     openssl
-    zstd
+    pkgs.zstd
   ];
 
   dependencies = [
     bcrypt
     build
-    chroma-hnswlib
     fastapi
     grpcio
     httpx
     importlib-resources
+    jsonschema
     kubernetes
     mmh3
     numpy
@@ -135,11 +152,21 @@ buildPythonPackage rec {
     uvicorn
   ];
 
+  optional-dependencies = {
+    dev = [ chroma-hnswlib ];
+  };
+
   nativeCheckInputs = [
+    chroma-hnswlib
+    hnswlib
     hypothesis
+    pandas
     psutil
     pytest-asyncio
+    pytest-xdist
     pytestCheckHook
+    sqlite
+    starlette
   ];
 
   # Disable on aarch64-linux due to broken onnxruntime
@@ -151,11 +178,16 @@ buildPythonPackage rec {
 
   env = {
     ZSTD_SYS_USE_PKG_CONFIG = true;
+    SWAGGER_UI_DOWNLOAD_URL = "file://${swagger-ui}";
   };
 
   pytestFlagsArray = [
     "-x" # these are slow tests, so stop on the first failure
     "-v"
+    "-W"
+    "ignore:DeprecationWarning"
+    "-W"
+    "ignore:PytestCollectionWarning"
   ];
 
   preCheck = ''
@@ -173,28 +205,43 @@ buildPythonPackage rec {
 
     # httpx ReadError
     "test_not_existing_collection_delete"
+
+    # Tests launch a server and try to connect to it
+    # These either have https connection errors or name resolution errors
+    "test_collection_query_with_invalid_collection_throws"
+    "test_collection_update_with_invalid_collection_throws"
+    "test_default_embedding"
+    "test_invalid_index_params"
+    "test_peek"
+    "test_persist_index_loading"
+    "test_query_id_filtering_e2e"
+    "test_query_id_filtering_medium_dataset"
+    "test_query_id_filtering_small_dataset"
+    "test_ssl_self_signed_without_ssl_verify"
+    "test_ssl_self_signed"
+
+    # Apparent race condition with sqlite
+    # See https://github.com/chroma-core/chroma/issues/4661
+    "test_multithreaded_get_or_create"
   ];
 
   disabledTestPaths = [
     # Tests require network access
-    "chromadb/test/auth/test_simple_rbac_authz.py"
-    "chromadb/test/db/test_system.py"
+    "bin/rust_python_compat_test.py"
+    "chromadb/test/configurations/test_collection_configuration.py"
     "chromadb/test/ef/test_default_ef.py"
+    "chromadb/test/ef/test_onnx_mini_lm_l6_v2.py"
+    "chromadb/test/ef/test_voyageai_ef.py"
     "chromadb/test/property/"
     "chromadb/test/property/test_cross_version_persist.py"
     "chromadb/test/stress/"
     "chromadb/test/test_api.py"
 
-    # httpx failures
-    "chromadb/test/api/test_delete_database.py"
+    # Tests time out (waiting for server)
+    "chromadb/test/test_cli.py"
 
-    # Cannot be loaded by pytest without path hacks (fixed in 1.0.0)
-    "chromadb/test/test_logservice.py"
-    "chromadb/test/proto/test_utils.py"
-    "chromadb/test/segment/distributed/test_protobuf_translation.py"
-
-    # Hypothesis FailedHealthCheck due to nested @given tests
-    "chromadb/test/cache/test_cache.py"
+    # Cannot find protobuf file while loading test
+    "chromadb/test/distributed/test_log_failover.py"
   ];
 
   __darwinAllowLocalNetworking = true;
@@ -212,17 +259,17 @@ buildPythonPackage rec {
         "([0-9].+)"
       ];
     };
+  };
 
-    meta = {
-      description = "AI-native open-source embedding database";
-      homepage = "https://github.com/chroma-core/chroma";
-      changelog = "https://github.com/chroma-core/chroma/releases/tag/${version}";
-      license = lib.licenses.asl20;
-      maintainers = with lib.maintainers; [
-        fab
-        sarahec
-      ];
-      mainProgram = "chroma";
-    };
+  meta = {
+    description = "AI-native open-source embedding database";
+    homepage = "https://github.com/chroma-core/chroma";
+    changelog = "https://github.com/chroma-core/chroma/releases/tag/${version}";
+    license = lib.licenses.asl20;
+    maintainers = with lib.maintainers; [
+      fab
+      sarahec
+    ];
+    mainProgram = "chroma";
   };
 }

pkgs/development/python-modules/chromadb/disable-fastapi-fixtures.patch

@@ -0,0 +1,14 @@
+diff --git a/chromadb/test/conftest.py b/chromadb/test/conftest.py
+index efde1c382..163f55c57 100644
+--- a/chromadb/test/conftest.py
++++ b/chromadb/test/conftest.py
+@@ -678,9 +678,6 @@ def sqlite_persistent(request: pytest.FixtureRequest) -> Generator[System, None,
+ 
+ def system_fixtures() -> List[Callable[[], Generator[System, None, None]]]:
+     fixtures = [
+-        fastapi,
+-        async_fastapi,
+-        fastapi_persistent,
+         sqlite_fixture,
+         sqlite_persistent_fixture,
+     ]