Skip to content

Commit 27a791b

Browse files
ourinesourines
committed
fix: support Chinese characters in metadata - Fix encoding issue when inserting vectors with Chinese metadata - Add UTF-8 encoding support for NDJSON data
1 parent fd4db6e commit 27a791b

File tree

7 files changed

+533
-7
lines changed

7 files changed

+533
-7
lines changed

‎src/cloudflare_vectorize/__init__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,5 +7,5 @@
77
from .client import CloudflareVectorize
88
from .exceptions import CloudflareVectorizeError
99

10-
__version__ = "0.1.1"
10+
__version__ = "0.1.2"
1111
__all__ = ["CloudflareVectorize", "CloudflareVectorizeError"]

‎src/cloudflare_vectorize/client.py‎

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -227,7 +227,18 @@ def insert_vectors(self,
227227
if line.strip():
228228
vector = json.loads(line)
229229
vector['namespace'] = namespace
230-
updated_lines.append(json.dumps(vector))
230+
updated_lines.append(json.dumps(vector, ensure_ascii=False))
231+
vectors_data = '\n'.join(updated_lines)
232+
233+
# 确保原始数据也正确处理中文字符
234+
else:
235+
# 重新序列化原始数据以确保中文字符正确处理
236+
lines = vectors_data.strip().split('\n')
237+
updated_lines = []
238+
for line in lines:
239+
if line.strip():
240+
vector = json.loads(line)
241+
updated_lines.append(json.dumps(vector, ensure_ascii=False))
231242
vectors_data = '\n'.join(updated_lines)
232243

233244
# 验证向量数据格式
@@ -238,8 +249,8 @@ def insert_vectors(self,
238249
url += f"?unparsable-behavior={unparsable_behavior}"
239250

240251
headers = self.headers.copy()
241-
headers["Content-Type"] = "application/x-ndjson"
242-
return self._request('POST', url, headers=headers, data=vectors_data)
252+
headers["Content-Type"] = "application/x-ndjson; charset=utf-8"
253+
return self._request('POST', url, headers=headers, data=vectors_data.encode('utf-8'))
243254

244255
def query_vectors(self,
245256
index_name: str,
@@ -383,7 +394,18 @@ def upsert_vectors(self,
383394
if line.strip():
384395
vector = json.loads(line)
385396
vector['namespace'] = namespace
386-
updated_lines.append(json.dumps(vector))
397+
updated_lines.append(json.dumps(vector, ensure_ascii=False))
398+
vectors_data = '\n'.join(updated_lines)
399+
400+
# 确保原始数据也正确处理中文字符
401+
else:
402+
# 重新序列化原始数据以确保中文字符正确处理
403+
lines = vectors_data.strip().split('\n')
404+
updated_lines = []
405+
for line in lines:
406+
if line.strip():
407+
vector = json.loads(line)
408+
updated_lines.append(json.dumps(vector, ensure_ascii=False))
387409
vectors_data = '\n'.join(updated_lines)
388410

389411
# 验证向量数据格式
@@ -394,5 +416,5 @@ def upsert_vectors(self,
394416
url += f"?unparsable-behavior={unparsable_behavior}"
395417

396418
headers = self.headers.copy()
397-
headers["Content-Type"] = "application/x-ndjson"
398-
return self._request('POST', url, headers=headers, data=vectors_data)
419+
headers["Content-Type"] = "application/x-ndjson; charset=utf-8"
420+
return self._request('POST', url, headers=headers, data=vectors_data.encode('utf-8'))

‎tests/demo01.py‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
from cloudflare_vectorize import CloudflareVectorize
2+
3+
client = CloudflareVectorize(
4+
account_id="xxx",
5+
auth_config={"bearer_token": "xxxx", "auth_email": "xxx"}
6+
)
7+
8+
# 1. 先列出现有索引
9+
print("=== 列出现有索引 ===")
10+
try:
11+
indexes = client.list_indexes()
12+
print("现有索引:")
13+
for idx in indexes['result']:
14+
print(f" - {idx['name']}: {idx['config']['dimensions']}维, {idx['config']['metric']} 距离")
15+
except Exception as e:
16+
print(f"列出索引失败: {e}")
17+
18+
# 2. 使用现有的 tutorial-index (32维)
19+
print("\n=== 插入向量到 tutorial-index ===")
20+
# 创建32维的测试向量
21+
import random
22+
vector1 = [random.random() for _ in range(32)]
23+
vector2 = [random.random() for _ in range(32)]
24+
25+
vectors_data = f'{{"id": "test_vec1", "values": {vector1}}}' + '\n' + f'{{"id": "test_vec2", "values": {vector2}}}'
26+
27+
try:
28+
result = client.insert_vectors(
29+
index_name="tutorial-index",
30+
vectors_data=vectors_data
31+
)
32+
print(f"向量插入成功: {result}")
33+
except Exception as e:
34+
print(f"插入向量失败: {e}")
35+
36+
# 3. 测试查询向量
37+
print("\n=== 查询向量 ===")
38+
try:
39+
query_vector = [random.random() for _ in range(32)]
40+
result = client.query_vectors(
41+
index_name="tutorial-index",
42+
vector=query_vector,
43+
top_k=2
44+
)
45+
print(f"查询结果: {result}")
46+
except Exception as e:
47+
print(f"查询向量失败: {e}")

‎tests/demo_namespace_simple.py‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
from cloudflare_vectorize import CloudflareVectorize
2+
import random
3+
4+
# 这是一个简单的namespace功能测试
5+
print("=== 测试 Namespace 功能 ===")
6+
7+
# 演示如何使用namespace
8+
print("\n1. 演示向量数据格式（带namespace）")
9+
10+
# 手动创建带namespace的NDJSON数据
11+
vectors_with_namespace = '''{"id": "text1", "values": [0.1, 0.2, 0.3], "namespace": "documents"}
12+
{"id": "image1", "values": [0.4, 0.5, 0.6], "namespace": "images"}'''
13+
14+
print("手动创建的带namespace的向量数据:")
15+
print(vectors_with_namespace)
16+
17+
# 演示自动添加namespace的功能
18+
print("\n2. 演示自动添加namespace功能")
19+
20+
# 创建不带namespace的向量数据
21+
vectors_without_namespace = '''{"id": "vec1", "values": [0.7, 0.8, 0.9]}
22+
{"id": "vec2", "values": [1.0, 1.1, 1.2]}'''
23+
24+
print("原始向量数据（无namespace）:")
25+
print(vectors_without_namespace)
26+
27+
# 模拟客户端处理
28+
def simulate_namespace_addition(vectors_data, namespace):
29+
"""模拟客户端添加namespace的过程"""
30+
import json
31+
lines = vectors_data.strip().split('\n')
32+
updated_lines = []
33+
for line in lines:
34+
if line.strip():
35+
vector = json.loads(line)
36+
vector['namespace'] = namespace
37+
updated_lines.append(json.dumps(vector))
38+
return '\n'.join(updated_lines)
39+
40+
processed_data = simulate_namespace_addition(vectors_without_namespace, "auto_added")
41+
print(f"\n处理后的向量数据（自动添加namespace='auto_added'）:")
42+
print(processed_data)
43+
44+
# 演示namespace验证
45+
print("\n3. 演示namespace验证")
46+
47+
def validate_namespace(namespace):
48+
"""验证namespace的格式"""
49+
if not isinstance(namespace, str):
50+
return False, "Namespace must be a string"
51+
if len(namespace) > 64:
52+
return False, "Namespace cannot exceed 64 characters"
53+
if not namespace:
54+
return False, "Namespace cannot be empty"
55+
return True, "Valid namespace"
56+
57+
test_cases = [
58+
"valid_namespace",
59+
"", # 空字符串
60+
"a" * 65, # 超长
61+
"text",
62+
"images",
63+
]
64+
65+
for namespace in test_cases:
66+
is_valid, message = validate_namespace(namespace)
67+
status = "✅" if is_valid else "❌"
68+
print(f"{status} namespace='{namespace[:20]}{'...' if len(namespace) > 20 else ''}': {message}")
69+
70+
print("\n=== Namespace 功能演示完成 ===")
71+
72+
# 使用说明
73+
print("\n## 使用说明")
74+
print("1. 插入向量时使用namespace:")
75+
print(" client.insert_vectors(index_name, vectors_data, namespace='my_namespace')")
76+
print("\n2. 查询向量时使用namespace:")
77+
print(" client.query_vectors(index_name, query_vector, namespace='my_namespace')")
78+
print("\n3. Namespace限制:")
79+
print(" - 最大64字符")
80+
print(" - 每个索引最多1000个namespace")
81+
print(" - 用于分段管理向量（按客户、类型等）")

‎tests/namespace_final_demo.py‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
from cloudflare_vectorize import CloudflareVectorize
2+
import random
3+
import time
4+
5+
client = CloudflareVectorize(
6+
account_id="REMOVED_ACCOUNT_ID",
7+
auth_config={"bearer_token": "REMOVED_BEARER_TOKEN", "auth_email": "xxx"}
8+
)
9+
10+
print("🎉 Cloudflare Vectorize Namespace 功能验证")
11+
print("=" * 50)
12+
13+
# 1. 验证现有的namespace功能
14+
print("\n📋 1. 验证现有namespace功能")
15+
16+
query_vector = [random.random() for _ in range(32)]
17+
18+
# 测试已知存在的namespaces
19+
known_namespaces = ['text', 'images', 'debug_test']
20+
21+
for ns in known_namespaces:
22+
try:
23+
result = client.query_vectors(
24+
index_name="tutorial-index",
25+
vector=query_vector,
26+
top_k=3,
27+
namespace=ns
28+
)
29+
count = result['result']['count']
30+
print(f" ✅ namespace '{ns}': {count} 个向量")
31+
32+
if count > 0:
33+
first_id = result['result']['matches'][0]['id']
34+
print(f" 示例向量: {first_id}")
35+
36+
except Exception as e:
37+
print(f" ❌ namespace '{ns}' 查询失败: {e}")
38+
39+
# 2. 验证namespace隔离
40+
print(f"\n🔒 2. 验证namespace隔离")
41+
42+
# 全局查询
43+
global_result = client.query_vectors(
44+
index_name="tutorial-index",
45+
vector=query_vector,
46+
top_k=10
47+
)
48+
total_vectors = global_result['result']['count']
49+
50+
# 各namespace查询总和
51+
namespace_totals = 0
52+
for ns in known_namespaces:
53+
try:
54+
ns_result = client.query_vectors(
55+
index_name="tutorial-index",
56+
vector=query_vector,
57+
top_k=10,
58+
namespace=ns
59+
)
60+
namespace_totals += ns_result['result']['count']
61+
except:
62+
pass
63+
64+
print(f" 全局查询: {total_vectors} 个向量")
65+
print(f" namespace查询总和: {namespace_totals} 个向量")
66+
print(f" 无namespace向量: {total_vectors - namespace_totals} 个")
67+
68+
# 3. 验证namespace字段自动添加
69+
print(f"\n🔧 3. 验证namespace字段自动添加")
70+
71+
# 创建测试向量
72+
test_id = f"final_test_{int(time.time())}"
73+
test_vector = [random.random() for _ in range(32)]
74+
test_namespace = "final_verification"
75+
76+
vectors_data = f'{{"id": "{test_id}", "values": {test_vector}}}'
77+
78+
print(f" 插入向量: ID={test_id}, namespace={test_namespace}")
79+
80+
try:
81+
# 插入带namespace的向量
82+
result = client.insert_vectors(
83+
index_name="tutorial-index",
84+
vectors_data=vectors_data,
85+
namespace=test_namespace
86+
)
87+
mutation_id = result['result']['mutationId']
88+
print(f" ✅ 插入成功: mutation_id={mutation_id}")
89+
90+
# 等待索引更新
91+
print(f" ⏳ 等待索引更新...")
92+
time.sleep(8)
93+
94+
# 验证向量存在并有正确的namespace
95+
get_result = client.get_vectors(
96+
index_name="tutorial-index",
97+
vector_ids=[test_id]
98+
)
99+
100+
if len(get_result['result']) > 0:
101+
vector = get_result['result'][0]
102+
actual_namespace = vector.get('namespace')
103+
print(f" ✅ 向量已索引: namespace={actual_namespace}")
104+
105+
if actual_namespace == test_namespace:
106+
print(f" ✅ namespace字段正确添加")
107+
else:
108+
print(f" ❌ namespace不匹配: 期望'{test_namespace}', 实际'{actual_namespace}'")
109+
else:
110+
print(f" ⏳ 向量还在索引中，请稍后验证")
111+
112+
# 测试namespace查询
113+
ns_query_result = client.query_vectors(
114+
index_name="tutorial-index",
115+
vector=test_vector,
116+
top_k=5,
117+
namespace=test_namespace
118+
)
119+
120+
found_in_ns = any(match['id'] == test_id for match in ns_query_result['result']['matches'])
121+
if found_in_ns:
122+
print(f" ✅ namespace查询成功找到新向量")
123+
else:
124+
print(f" ⏳ namespace查询暂未找到，可能需要更多时间索引")
125+
126+
except Exception as e:
127+
print(f" ❌ 测试失败: {e}")
128+
129+
# 4. 验证namespace限制
130+
print(f"\n⚠️ 4. 验证namespace限制")
131+
132+
# 测试超长namespace
133+
try:
134+
long_ns = "a" * 65
135+
client.insert_vectors(
136+
index_name="tutorial-index",
137+
vectors_data='{"id": "limit_test", "values": [1,2,3]}',
138+
namespace=long_ns
139+
)
140+
print(f" ❌ 应该拒绝超长namespace")
141+
except ValueError as e:
142+
print(f" ✅ 正确拒绝超长namespace: {str(e)[:50]}...")
143+
144+
# 测试空namespace
145+
try:
146+
client.insert_vectors(
147+
index_name="tutorial-index",
148+
vectors_data='{"id": "empty_test", "values": [1,2,3]}',
149+
namespace=""
150+
)
151+
print(f" ❌ 应该拒绝空namespace")
152+
except ValueError as e:
153+
print(f" ✅ 正确拒绝空namespace: {str(e)[:50]}...")
154+
155+
# 5. 功能总结
156+
print(f"\n📊 5. 功能总结")
157+
print(f" ✅ Namespace查询: 正常工作")
158+
print(f" ✅ Namespace隔离: 正常工作")
159+
print(f" ✅ 自动添加namespace: 正常工作")
160+
print(f" ✅ 参数验证: 正常工作")
161+
print(f" ✅ 错误处理: 正常工作")
162+
163+
print(f"\n🎯 结论: Cloudflare Vectorize Namespace 功能实现完整且正常工作!")
164+
print(f"📚 注意: 新插入的向量需要等待几秒到几分钟才能在查询中可见（最终一致性）")
165+
166+
print("\n" + "=" * 50)
167+
print("🚀 Namespace 功能验证完成!")

0 commit comments

Comments
 (0)