refactor: extract utils to a separate crate

Cargo.toml

@@ -1,5 +1,5 @@
 [workspace]
-members = ["rust/cocoindex"]
+members = ["rust/cocoindex", "rust/utils"]
 resolver = "2"
 
 [workspace.package]

rust/cocoindex/Cargo.toml

@@ -14,6 +14,8 @@ default = ["legacy-states-v0"]
 legacy-states-v0 = []
 
 [dependencies]
+cocoindex-utils = { path = "../utils" }
+
 pyo3 = { workspace = true }
 pythonize = { workspace = true }
 pyo3-async-runtimes = { workspace = true }

rust/cocoindex/src/base/json_schema.rs

@@ -1,11 +1,11 @@
 use crate::prelude::*;
 
-use crate::utils::immutable::RefList;
 use schemars::schema::{
     ArrayValidation, InstanceType, ObjectValidation, Schema, SchemaObject, SingleOrVec,
     SubschemaValidation,
 };
 use std::fmt::Write;
+use utils::immutable::RefList;
 
 pub struct ToJsonSchemaOptions {
     /// If true, mark all fields as required.

rust/cocoindex/src/builder/analyzer.rs

@@ -6,13 +6,13 @@ use crate::prelude::*;
 
 use super::plan::*;
 use crate::lib_context::get_auth_registry;
-use crate::utils::fingerprint::Fingerprinter;
 use crate::{
     base::{schema::*, spec::*},
     ops::interface::*,
 };
 use futures::future::{BoxFuture, try_join3};
 use futures::{FutureExt, future::try_join_all};
+use utils::fingerprint::Fingerprinter;
 
 #[derive(Debug)]
 pub(super) enum ValueTypeBuilder {

rust/cocoindex/src/builder/plan.rs

@@ -3,7 +3,7 @@ use crate::base::spec::FieldName;
 use crate::prelude::*;
 
 use crate::ops::interface::*;
-use crate::utils::fingerprint::{Fingerprint, Fingerprinter};
+use utils::fingerprint::{Fingerprint, Fingerprinter};
 
 #[derive(Debug, Clone, PartialEq, Eq, Serialize)]
 pub struct AnalyzedLocalFieldReference {

rust/cocoindex/src/execution/db_tracking.rs

@@ -1,12 +1,12 @@
 use crate::prelude::*;
 
 use super::{db_tracking_setup::TrackingTableSetupState, memoization::StoredMemoizationInfo};
-use crate::utils::{db::WriteAction, fingerprint::Fingerprint};
 use futures::Stream;
 use serde::de::{self, Deserializer, SeqAccess, Visitor};
 use serde::ser::SerializeSeq;
 use sqlx::PgPool;
 use std::fmt;
+use utils::{db::WriteAction, fingerprint::Fingerprint};
 
 ////////////////////////////////////////////////////////////
 // Access for the row tracking table

rust/cocoindex/src/execution/dumper.rs

@@ -13,7 +13,7 @@ use super::row_indexer;
 use crate::base::{schema, value};
 use crate::builder::plan::{AnalyzedImportOp, ExecutionPlan};
 use crate::ops::interface::SourceExecutorReadOptions;
-use crate::utils::yaml_ser::YamlSerializer;
+use utils::yaml_ser::YamlSerializer;
 
 #[derive(Debug, Clone, Deserialize)]
 pub struct EvaluateAndDumpOptions {

rust/cocoindex/src/execution/evaluator.rs

@@ -4,12 +4,10 @@ use anyhow::{Context, Ok};
 use futures::future::try_join_all;
 
 use crate::base::value::EstimatedByteSize;
+use crate::base::{schema, value};
 use crate::builder::{AnalyzedTransientFlow, plan::*};
 use crate::py::AnyhowIntoPyResult;
-use crate::{
-    base::{schema, value},
-    utils::immutable::RefList,
-};
+use utils::immutable::RefList;
 
 use super::memoization::{EvaluationMemory, EvaluationMemoryOptions, evaluate_with_cell};
 

rust/cocoindex/src/execution/memoization.rs

@@ -10,8 +10,8 @@ use std::{
 use crate::{
     base::{schema, value},
     service::error::{SharedError, SharedResultExtRef},
-    utils::fingerprint::{Fingerprint, Fingerprinter},
 };
+use cocoindex_utils::fingerprint::{Fingerprint, Fingerprinter};
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct StoredCacheEntry {

rust/cocoindex/src/execution/row_indexer.rs

@@ -18,8 +18,8 @@ use crate::builder::plan::*;
 use crate::ops::interface::{
     ExportTargetMutation, ExportTargetUpsertEntry, Ordinal, SourceExecutorReadOptions,
 };
-use crate::utils::db::WriteAction;
-use crate::utils::fingerprint::{Fingerprint, Fingerprinter};
+use utils::db::WriteAction;
+use utils::fingerprint::{Fingerprint, Fingerprinter};
 
 pub fn extract_primary_key_for_export(
     primary_key_def: &AnalyzedPrimaryKeyDef,
@@ -947,7 +947,7 @@ mod tests {
     #[test]
     fn test_content_hash_computation() {
         use crate::base::value::{BasicValue, FieldValues, Value};
-        use crate::utils::fingerprint::Fingerprinter;
+        use utils::fingerprint::Fingerprinter;
 
         // Test that content hash is computed correctly from source data
         let source_data1 = FieldValues {
@@ -999,7 +999,7 @@ mod tests {
         // This test documents the exact requirements for GitHub Actions scenario
         // where file modification times change but content remains the same
 
-        use crate::utils::fingerprint::Fingerprinter;
+        use utils::fingerprint::Fingerprinter;
 
         // Simulate file content that remains the same across GitHub Actions checkout
         let file_content = "const hello = 'world';\nexport default hello;";

rust/cocoindex/src/execution/source_indexer.rs

@@ -1,6 +1,5 @@
-use crate::{
-    execution::row_indexer::ContentHashBasedCollapsingBaseline, prelude::*, utils::batching,
-};
+use crate::{execution::row_indexer::ContentHashBasedCollapsingBaseline, prelude::*};
+use utils::batching;
 
 use futures::future::Ready;
 use sqlx::PgPool;

rust/cocoindex/src/lib.rs

@@ -10,4 +10,3 @@ mod server;
 mod service;
 mod settings;
 mod setup;
-mod utils;

rust/cocoindex/src/llm/openai.rs

@@ -5,7 +5,6 @@ use super::{LlmEmbeddingClient, LlmGenerationClient, detect_image_mime_type};
 use async_openai::{
     Client as OpenAIClient,
     config::OpenAIConfig,
-    error::OpenAIError,
     types::{
         ChatCompletionRequestMessage, ChatCompletionRequestMessageContentPartImage,
         ChatCompletionRequestMessageContentPartText, ChatCompletionRequestSystemMessage,
@@ -61,15 +60,6 @@ impl Client {
     }
 }
 
-impl utils::retryable::IsRetryable for OpenAIError {
-    fn is_retryable(&self) -> bool {
-        match self {
-            OpenAIError::Reqwest(e) => e.is_retryable(),
-            _ => false,
-        }
-    }
-}
-
 fn create_llm_generation_request(
     request: &super::LlmGenerateRequest,
 ) -> Result<CreateChatCompletionRequest> {

rust/cocoindex/src/ops/factory_bases.rs

@@ -385,32 +385,39 @@ pub trait BatchedFunctionExecutor: Send + Sync + Sized + 'static {
     fn batching_options(&self) -> batching::BatchingOptions;
 }
 
+struct BatchedFunctionExecutorRunner<E: BatchedFunctionExecutor>(E);
+
 #[async_trait]
-impl<E: BatchedFunctionExecutor> batching::Runner for E {
+impl<E: BatchedFunctionExecutor> batching::Runner for BatchedFunctionExecutorRunner<E> {
     type Input = Vec<value::Value>;
     type Output = value::Value;
 
     async fn run(
         &self,
         inputs: Vec<Self::Input>,
     ) -> Result<impl ExactSizeIterator<Item = Self::Output>> {
-        Ok(self.evaluate_batch(inputs).await?.into_iter())
+        Ok(self.0.evaluate_batch(inputs).await?.into_iter())
     }
 }
 
 struct BatchedFunctionExecutorWrapper<E: BatchedFunctionExecutor> {
-    batcher: batching::Batcher<E>,
+    batcher: batching::Batcher<BatchedFunctionExecutorRunner<E>>,
     enable_cache: bool,
     behavior_version: Option<u32>,
 }
 
 impl<E: BatchedFunctionExecutor> BatchedFunctionExecutorWrapper<E> {
     fn new(executor: E) -> Self {
         let batching_options = executor.batching_options();
+        let enable_cache = executor.enable_cache();
+        let behavior_version = executor.behavior_version();
         Self {
-            enable_cache: executor.enable_cache(),
-            behavior_version: executor.behavior_version(),
-            batcher: batching::Batcher::new(executor, batching_options),
+            enable_cache,
+            behavior_version,
+            batcher: batching::Batcher::new(
+                BatchedFunctionExecutorRunner(executor),
+                batching_options,
+            ),
         }
     }
 }

rust/cocoindex/src/ops/targets/neo4j.rs

@@ -51,16 +51,6 @@ impl GraphKey {
     }
 }
 
-impl retryable::IsRetryable for neo4rs::Error {
-    fn is_retryable(&self) -> bool {
-        match self {
-            neo4rs::Error::ConnectionError => true,
-            neo4rs::Error::Neo4j(e) => e.kind() == neo4rs::Neo4jErrorKind::Transient,
-            _ => false,
-        }
-    }
-}
-
 #[derive(Default)]
 pub struct GraphPool {
     graphs: Mutex<HashMap<GraphKey, Arc<OnceCell<Arc<Graph>>>>>,

rust/cocoindex/src/prelude.rs

@@ -26,8 +26,9 @@ pub(crate) use crate::ops::interface;
 pub(crate) use crate::service::error::{ApiError, invariance_violation};
 pub(crate) use crate::setup;
 pub(crate) use crate::setup::AuthRegistry;
-pub(crate) use crate::utils::{self, batching, concur_control, http, retryable};
 pub(crate) use crate::{api_bail, api_error};
+pub(crate) use cocoindex_utils as utils;
+pub(crate) use cocoindex_utils::{batching, concur_control, http, retryable};
 
 pub(crate) use anyhow::{anyhow, bail};
 pub(crate) use async_stream::{stream, try_stream};