feat: strip U+0000 in writing to Postgres to overcome its limitation (#937)

* feat: strip U+0000 in writing to Postgres to overcome its limitation

* docs: document the behavior of stripping 0

Author: georgeh0

docs/docs/ops/targets.md

@@ -39,6 +39,13 @@ For all other vector types, we map them to `jsonb` columns.
 
 :::
 
+:::info U+0000 (NUL) characters in strings
+
+U+0000 (NUL) is a valid character in Unicode, but Postgres has a limitation that strings (including `text`-like types and strings in `jsonb`) cannot contain them.
+CocoIndex automatically strips U+0000 (NUL) characters from strings before exporting to Postgres. For example, if you have a string `"Hello\0World"`, it will be exported as `"HelloWorld"`.
+
+:::
+
 #### Spec
 
 The spec takes the following fields:

src/ops/targets/postgres.rs

@@ -43,7 +43,7 @@ fn bind_value_field<'arg>(
                 builder.push_bind(&**v);
             }
             BasicValue::Str(v) => {
-                builder.push_bind(&**v);
+                builder.push_bind(utils::str_sanitize::ZeroCodeStrippedEncode(v.as_ref()));
             }
             BasicValue::Bool(v) => {
                 builder.push_bind(v);
@@ -82,7 +82,9 @@ fn bind_value_field<'arg>(
                 builder.push_bind(v);
             }
             BasicValue::Json(v) => {
-                builder.push_bind(sqlx::types::Json(&**v));
+                builder.push_bind(sqlx::types::Json(
+                    utils::str_sanitize::ZeroCodeStrippedSerialize(&**v),
+                ));
             }
             BasicValue::Vector(v) => match &field_schema.value_type.typ {
                 ValueType::Basic(BasicValueType::Vector(vs)) if convertible_to_pgvector(vs) => {
@@ -104,20 +106,24 @@ fn bind_value_field<'arg>(
                 }
             },
             BasicValue::UnionVariant { .. } => {
-                builder.push_bind(sqlx::types::Json(TypedValue {
-                    t: &field_schema.value_type.typ,
-                    v: value,
-                }));
+                builder.push_bind(sqlx::types::Json(
+                    utils::str_sanitize::ZeroCodeStrippedSerialize(TypedValue {
+                        t: &field_schema.value_type.typ,
+                        v: value,
+                    }),
+                ));
             }
         },
         Value::Null => {
             builder.push("NULL");
         }
         v => {
-            builder.push_bind(sqlx::types::Json(TypedValue {
-                t: &field_schema.value_type.typ,
-                v,
-            }));
+            builder.push_bind(sqlx::types::Json(
+                utils::str_sanitize::ZeroCodeStrippedSerialize(TypedValue {
+                    t: &field_schema.value_type.typ,
+                    v,
+                }),
+            ));
         }
     };
     Ok(())

src/utils/mod.rs

@@ -3,4 +3,5 @@ pub mod db;
 pub mod fingerprint;
 pub mod immutable;
 pub mod retryable;
+pub mod str_sanitize;
 pub mod yaml_ser;