fix: also strip \x00

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

Author: mudler

rag/engine/postgres.go

@@ -481,16 +481,16 @@ func (p *PostgresDB) StoreDocuments(s []string, metadata map[string]string) ([]R
 
 	// Insert documents
 	for i, content := range s {
-		// Sanitize content to prevent invalid UTF-8 from reaching PostgreSQL
-		content = strings.ToValidUTF8(content, " ")
+		// Sanitize content to prevent invalid UTF-8 and null bytes from reaching PostgreSQL
+		content = strings.ReplaceAll(strings.ToValidUTF8(content, " "), "\x00", "")
 
 		embedding := resp.Data[i].Embedding
 		embeddingStr := formatVector(embedding)
 
 		// Extract title from metadata if available
-		title := strings.ToValidUTF8(metadata["title"], " ")
+		title := strings.ReplaceAll(strings.ToValidUTF8(metadata["title"], " "), "\x00", "")
 		if title == "" {
-			title = strings.ToValidUTF8(metadata["source"], " ")
+			title = strings.ReplaceAll(strings.ToValidUTF8(metadata["source"], " "), "\x00", "")
 		}
 
 		// Calculate word count

rag/persistency.go

@@ -476,7 +476,9 @@ func fileToText(fpath string) (string, error) {
 		buf.ReadFrom(b)
 		// PDF extraction can produce invalid UTF-8 byte sequences that PostgreSQL rejects.
 		// Sanitize by replacing invalid sequences with the Unicode replacement character.
-		return strings.ToValidUTF8(buf.String(), " "), nil
+		text := strings.ToValidUTF8(buf.String(), " ")
+		text = strings.ReplaceAll(text, "\x00", "")
+		return text, nil
 	case ".txt", ".md":
 		f, err := os.Open(fpath)
 		if err != nil {