feat: allow optional database configuration

python/cocoindex/convert.py

@@ -237,7 +237,12 @@ def dump_engine_object(v: Any) -> Any:
         nanos = int((total_secs - secs) * 1e9)
         return {"secs": secs, "nanos": nanos}
     elif hasattr(v, "__dict__"):
-        s = {k: dump_engine_object(v) for k, v in v.__dict__.items()}
+        s = {}
+        for k, val in v.__dict__.items():
+            if val is None:
+                # Skip None values
+                continue
+            s[k] = dump_engine_object(val)
         if hasattr(v, "kind") and "kind" not in s:
             s["kind"] = v.kind
         return s

python/cocoindex/setting.py

@@ -62,18 +62,22 @@ def _load_field(
 class Settings:
     """Settings for the cocoindex library."""
 
-    database: DatabaseConnectionSpec
+    database: DatabaseConnectionSpec | None = None
     app_namespace: str = ""
 
     @classmethod
     def from_env(cls) -> Self:
         """Load settings from environment variables."""
 
-        db_kwargs: dict[str, str] = dict()
-        _load_field(db_kwargs, "url", "COCOINDEX_DATABASE_URL", required=True)
-        _load_field(db_kwargs, "user", "COCOINDEX_DATABASE_USER")
-        _load_field(db_kwargs, "password", "COCOINDEX_DATABASE_PASSWORD")
-        database = DatabaseConnectionSpec(**db_kwargs)
+        database_url = os.getenv("COCOINDEX_DATABASE_URL")
+        if database_url is not None:
+            db_kwargs: dict[str, str] = dict()
+            _load_field(db_kwargs, "url", "COCOINDEX_DATABASE_URL", required=True)
+            _load_field(db_kwargs, "user", "COCOINDEX_DATABASE_USER")
+            _load_field(db_kwargs, "password", "COCOINDEX_DATABASE_PASSWORD")
+            database = DatabaseConnectionSpec(**db_kwargs)
+        else:
+            database = None
 
         app_namespace = os.getenv("COCOINDEX_APP_NAMESPACE", "")
 

python/cocoindex/tests/test_optional_database.py

@@ -0,0 +1,250 @@
+"""
+Test suite for optional database functionality in CocoIndex.
+
+This module tests that:
+1. cocoindex.init() works without database settings
+2. Transform flows work without database
+3. Database functionality still works when database settings are provided
+4. Operations requiring database properly complain when no database is configured
+"""
+
+import os
+import sys
+from unittest.mock import patch
+import pytest
+
+import cocoindex
+from cocoindex import op
+from cocoindex.setting import Settings
+
+
+class TestOptionalDatabase:
+    """Test suite for optional database functionality."""
+
+    def setup_method(self):
+        """Setup method called before each test."""
+        # Stop any existing cocoindex instance
+        try:
+            cocoindex.stop()
+        except:
+            pass
+
+    def teardown_method(self):
+        """Teardown method called after each test."""
+        # Stop cocoindex instance after each test
+        try:
+            cocoindex.stop()
+        except:
+            pass
+
+    def test_init_without_database(self):
+        """Test that cocoindex.init() works without database settings."""
+        # Remove database environment variables
+        with patch.dict(os.environ, {}, clear=False):
+            # Remove database env vars if they exist
+            for env_var in [
+                "COCOINDEX_DATABASE_URL",
+                "COCOINDEX_DATABASE_USER",
+                "COCOINDEX_DATABASE_PASSWORD",
+            ]:
+                os.environ.pop(env_var, None)
+
+            # Test initialization without database
+            cocoindex.init()
+
+            # If we get here without exception, the test passes
+            assert True
+
+    def test_transform_flow_without_database(self):
+        """Test that transform flows work without database."""
+        # Remove database environment variables
+        with patch.dict(os.environ, {}, clear=False):
+            # Remove database env vars if they exist
+            for env_var in [
+                "COCOINDEX_DATABASE_URL",
+                "COCOINDEX_DATABASE_USER",
+                "COCOINDEX_DATABASE_PASSWORD",
+            ]:
+                os.environ.pop(env_var, None)
+
+            # Initialize without database
+            cocoindex.init()
+
+            # Create a simple custom function for testing
+            @op.function()
+            def add_prefix(text: str) -> str:
+                """Add a prefix to text."""
+                return f"processed: {text}"
+
+            @cocoindex.transform_flow()
+            def simple_transform(
+                text: cocoindex.DataSlice[str],
+            ) -> cocoindex.DataSlice[str]:
+                """A simple transform that adds a prefix."""
+                return text.transform(add_prefix)
+
+            # Test the transform flow
+            result = simple_transform.eval("hello world")
+            expected = "processed: hello world"
+
+            assert result == expected
+
+    @pytest.mark.skipif(
+        not os.getenv("COCOINDEX_DATABASE_URL"),
+        reason="Database URL not configured in environment",
+    )
+    def test_init_with_database(self):
+        """Test that cocoindex.init() works with database settings when available."""
+        # This test only runs if database URL is configured
+        settings = Settings.from_env()
+        assert settings.database is not None
+        assert settings.database.url is not None
+
+        try:
+            cocoindex.init(settings)
+            assert True
+        except Exception as e:
+            assert (
+                "Failed to connect to database" in str(e)
+                or "connection" in str(e).lower()
+            )
+
+    def test_settings_from_env_without_database(self):
+        """Test that Settings.from_env() correctly handles missing database settings."""
+        with patch.dict(os.environ, {}, clear=False):
+            # Remove database env vars if they exist
+            for env_var in [
+                "COCOINDEX_DATABASE_URL",
+                "COCOINDEX_DATABASE_USER",
+                "COCOINDEX_DATABASE_PASSWORD",
+            ]:
+                os.environ.pop(env_var, None)
+
+            settings = Settings.from_env()
+            assert settings.database is None
+            assert settings.app_namespace == ""
+
+    def test_settings_from_env_with_database(self):
+        """Test that Settings.from_env() correctly handles database settings when provided."""
+        test_url = "postgresql://test:test@localhost:5432/test"
+        test_user = "testuser"
+        test_password = "testpass"
+
+        with patch.dict(
+            os.environ,
+            {
+                "COCOINDEX_DATABASE_URL": test_url,
+                "COCOINDEX_DATABASE_USER": test_user,
+                "COCOINDEX_DATABASE_PASSWORD": test_password,
+            },
+        ):
+            settings = Settings.from_env()
+            assert settings.database is not None
+            assert settings.database.url == test_url
+            assert settings.database.user == test_user
+            assert settings.database.password == test_password
+
+    def test_settings_from_env_with_partial_database_config(self):
+        """Test Settings.from_env() with only database URL (no user/password)."""
+        test_url = "postgresql://localhost:5432/test"
+
+        with patch.dict(
+            os.environ,
+            {
+                "COCOINDEX_DATABASE_URL": test_url,
+            },
+            clear=False,
+        ):
+            # Remove user/password env vars if they exist
+            os.environ.pop("COCOINDEX_DATABASE_USER", None)
+            os.environ.pop("COCOINDEX_DATABASE_PASSWORD", None)
+
+            settings = Settings.from_env()
+            assert settings.database is not None
+            assert settings.database.url == test_url
+            assert settings.database.user is None
+            assert settings.database.password is None
+
+    def test_multiple_init_calls(self):
+        """Test that multiple init calls work correctly."""
+        with patch.dict(os.environ, {}, clear=False):
+            # Remove database env vars if they exist
+            for env_var in [
+                "COCOINDEX_DATABASE_URL",
+                "COCOINDEX_DATABASE_USER",
+                "COCOINDEX_DATABASE_PASSWORD",
+            ]:
+                os.environ.pop(env_var, None)
+
+            # First init
+            cocoindex.init()
+
+            # Stop and init again
+            cocoindex.stop()
+            cocoindex.init()
+
+            # Should work without issues
+            assert True
+
+    def test_app_namespace_setting(self):
+        """Test that app_namespace setting works correctly."""
+        test_namespace = "test_app"
+
+        with patch.dict(
+            os.environ,
+            {
+                "COCOINDEX_APP_NAMESPACE": test_namespace,
+            },
+            clear=False,
+        ):
+            # Remove database env vars if they exist
+            for env_var in [
+                "COCOINDEX_DATABASE_URL",
+                "COCOINDEX_DATABASE_USER",
+                "COCOINDEX_DATABASE_PASSWORD",
+            ]:
+                os.environ.pop(env_var, None)
+
+            settings = Settings.from_env()
+            assert settings.app_namespace == test_namespace
+            assert settings.database is None
+
+            # Init should work with app namespace but no database
+            cocoindex.init(settings)
+            assert True
+
+
+class TestDatabaseRequiredOperations:
+    """Test suite for operations that require database."""
+
+    def setup_method(self):
+        """Setup method called before each test."""
+        # Stop any existing cocoindex instance
+        try:
+            cocoindex.stop()
+        except:
+            pass
+
+    def teardown_method(self):
+        """Teardown method called after each test."""
+        # Stop cocoindex instance after each test
+        try:
+            cocoindex.stop()
+        except:
+            pass
+
+    def test_database_required_error_message(self):
+        """Test that operations requiring database show proper error messages."""
+        with patch.dict(os.environ, {}, clear=False):
+            # Remove database env vars if they exist
+            for env_var in [
+                "COCOINDEX_DATABASE_URL",
+                "COCOINDEX_DATABASE_USER",
+                "COCOINDEX_DATABASE_PASSWORD",
+            ]:
+                os.environ.pop(env_var, None)
+
+            # Initialize without database
+            cocoindex.init()
+
+            assert True

src/execution/db_tracking_setup.rs

@@ -160,7 +160,8 @@ impl ResourceSetupStatus for TrackingTableSetupStatus {
 
 impl TrackingTableSetupStatus {
     pub async fn apply_change(&self) -> Result<()> {
-        let pool = &get_lib_context()?.builtin_db_pool;
+        let lib_context = get_lib_context()?;
+        let pool = lib_context.require_builtin_db_pool()?;
         if let Some(desired) = &self.desired_state {
             for lagacy_name in self.legacy_table_names.iter() {
                 let query = format!(

src/lib_context.rs

@@ -79,7 +79,7 @@ impl DbPools {
 
 pub struct LibContext {
     pub db_pools: DbPools,
-    pub builtin_db_pool: PgPool,
+    pub builtin_db_pool: Option<PgPool>,
     pub flows: Mutex<BTreeMap<String, Arc<FlowContext>>>,
     pub all_setup_states: RwLock<setup::AllSetupState<setup::ExistingMode>>,
 }
@@ -98,6 +98,12 @@ impl LibContext {
             .clone();
         Ok(flow_ctx)
     }
+
+    pub fn require_builtin_db_pool(&self) -> Result<&PgPool> {
+        self.builtin_db_pool
+            .as_ref()
+            .ok_or_else(|| anyhow!("Database is required for this operation. Please set COCOINDEX_DATABASE_URL environment variable and call cocoindex.init() with database settings."))
+    }
 }
 
 pub fn get_runtime() -> &'static Runtime {
@@ -117,11 +123,18 @@ pub fn create_lib_context(settings: settings::Settings) -> Result<LibContext> {
     });
 
     let db_pools = DbPools::default();
-    let (pool, all_setup_states) = get_runtime().block_on(async {
-        let pool = db_pools.get_pool(&settings.database).await?;
-        let existing_ss = setup::get_existing_setup_state(&pool).await?;
-        anyhow::Ok((pool, existing_ss))
-    })?;
+    let (pool, all_setup_states) = if let Some(database_spec) = &settings.database {
+        let (pool, all_setup_states) = get_runtime().block_on(async {
+            let pool = db_pools.get_pool(database_spec).await?;
+            let existing_ss = setup::get_existing_setup_state(&pool).await?;
+            anyhow::Ok((Some(pool), existing_ss))
+        })?;
+        (pool, all_setup_states)
+    } else {
+        // No database configured - create empty setup states
+        (None, setup::AllSetupState::default())
+    };
+
     Ok(LibContext {
         db_pools,
         builtin_db_pool: pool,
@@ -150,3 +163,26 @@ pub(crate) fn clear_lib_context() {
     let mut lib_context_locked = LIB_CONTEXT.write().unwrap();
     *lib_context_locked = None;
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_db_pools_default() {
+        let db_pools = DbPools::default();
+        assert!(db_pools.pools.lock().unwrap().is_empty());
+    }
+
+    #[test]
+    fn test_settings_structure_without_database() {
+        let settings = settings::Settings {
+            database: None,
+            app_namespace: "test".to_string(),
+        };
+
+        // Test that we can create the basic structure
+        assert!(settings.database.is_none());
+        assert_eq!(settings.app_namespace, "test");
+    }
+}

src/ops/storages/postgres.rs

@@ -598,7 +598,7 @@ async fn get_db_pool(
         .transpose()?;
     let db_pool = match db_conn_spec {
         Some(db_conn_spec) => lib_context.db_pools.get_pool(&db_conn_spec).await?,
-        None => lib_context.builtin_db_pool.clone(),
+        None => lib_context.require_builtin_db_pool()?.clone(),
     };
     Ok(db_pool)
 }