Skip to content

Commit a7dbf95

Browse files
committed
vector hashing to prevent duplicate records
1 parent 338d722 commit a7dbf95

14 files changed

Lines changed: 675 additions & 9 deletions

DATASHEET.md

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,21 +2,22 @@
22

33
| Code | Name | Purpose | Input body | Output body (OK) | Possible errors |
44
|---|---|---|---|---|---|
5-
| `0x01` | **PUSH** | Insert record (vector required; label optional via header; data optional, requires label) | `<dim×4B f32 vec><4B u32 dlen>[dlen bytes]` | `<4B i32 index>` | `body too short` · `body length mismatch` · `data too large` · `data requires label` · `label too long` · `label has invalid chars` · `label empty` · `label store failed` · `data store failed` · `read-only mode` |
5+
| `0x01` | **PUSH** | Insert record (vector required; label optional via header; data optional, requires label). Bit-identical duplicate vectors are rejected. | `<dim×4B f32 vec><4B u32 dlen>[dlen bytes]` | `<4B i32 index>` | `body too short` · `body length mismatch` · `data too large` · `data requires label` · `label too long` · `label has invalid chars` · `label empty` · `label store failed` · `data store failed` · `duplicate vector at index <N>` · `read-only mode` |
66
| `0x02` | **QUERY** | NN search by query vector | `<1B metric><1B shape><dim×4B f32 query>` | `<4B u32 count>` then per record: `<4B i32 index><4B f32 dist>` + optional `[lbl][data][vec]` per shape | `bad query body` · `bad metric` · `out of memory` |
77
| `0x04` | **GET** | Fetch by index, label, or batch | `<1B mode><1B shape>` then `<4B i32 idx>` (single) / `<4B count><count×i32>` (batch) / empty (label) | `<4B u32 count>` then per record: `<4B i32 index>` + optional `[lbl][data][vec]` per shape | `bad get body` · `bad batch body` · `bad batch length` · `bad mode` · `extra body bytes` · `index out of range` · `deleted` · `label not found` · `out of memory` |
88
| `0x06` | **UPDATE** | Overwrite vector in place (label/data untouched) | `<dim×4B vec>` (label) or `<4B i32 idx><dim×4B vec>` | empty | `bad body length` · `index out of range` · `deleted` · `ambiguous label` · `label not found` · `read-only mode` |
99
| `0x07` | **DELETE** | Tombstone slot; also frees label + data | `<4B i32 idx>` or empty (label) | empty | `bad body length` · `extra body bytes` · `index out of range` · `already deleted` · `ambiguous label` · `label not found` · `read-only mode` |
1010
| `0x08` | **CMD_LABEL** | Set or clear label for slot (via header; len=0 clears) | `<4B i32 idx>` | empty | `bad body length` · `index out of range` · `label too long` · `label has invalid chars` · `label empty` · `read-only mode` |
1111
| `0x09` | **UNDO** | Remove last PUSH; also frees its label + data | empty | empty | `extra body bytes` · `empty` · `read-only mode` |
12-
| `0x0A` | **SAVE** | Flush `.tensors` + `.meta` + `.data` to disk | empty | `<4B u32 saved><4B u32 crc32>` | `extra body bytes` · `read-only mode` |
12+
| `0x0A` | **SAVE** | Flush `.tensors` + `.meta` + `.hashes` + `.data` to disk | empty | `<4B u32 saved><4B u32 crc32>` | `extra body bytes` · `read-only mode` |
1313
| `0x0D` | **CLUSTER** | DBSCAN clustering | `<4B f32 eps><1B mode><4B i32 min_pts>` | legacy text body: members per line + `end\n` | `bad body length` · `invalid eps` · `bad metric` |
1414
| `0x0E` | **DISTINCT** | Farthest-point sampling | `<4B i32 k><1B mode>` | legacy text body: one index per line + `end\n` | `bad body length` · `invalid k` · `bad metric` · `distinct not available in cpu mode` |
1515
| `0x0F` | **REPRESENT** | One most-distinct member per DBSCAN cluster | `<4B f32 eps><1B mode><4B i32 min_pts>` | legacy text body: one index per line + `end\n` | `bad body length` · `invalid eps` · `bad metric` · `represent not available in cpu mode` |
1616
| `0x10` | **INFO** | DB metadata snapshot | empty | `<4B dim><4B count><4B deleted><1B fmt><8B mtime><4B crc><1B crc_ok><4B name_len>[name]<1B protocol=0x02>` | `extra body bytes` · `out of memory` |
1717
| `0x11` | **QID** | NN search by stored vector / label | `<1B metric><1B shape>` then `<4B i32 idx>` or empty (label) | identical to QUERY | `bad qid body` · `bad metric` · `extra body bytes` · `index out of range` · `deleted` · `ambiguous label` · `label not found` · `out of memory` |
1818
| `0x13` | **SET_DATA** | Set or clear sidecar blob for slot (≤100KB; len=0 clears) | `<4B u32 dlen>[bytes]` (label) or `<4B i32 idx><4B u32 dlen>[bytes]` | empty | `bad body length` · `data too large` · `index out of range` · `deleted` · `ambiguous label` · `label not found` · `data store failed` · `read-only mode` |
1919
| `0x14` | **GET_DATA** | Fetch sidecar blob for slot | `<4B i32 idx>` or empty (label) | `<4B u32 dlen>[bytes]` | `bad body length` · `extra body bytes` · `index out of range` · `deleted` · `ambiguous label` · `label not found` |
20+
| `0x15` | **EXISTS** | Exact-match lookup by vector content (xxh64 + byte-compare; tombstoned slots skipped) | `<1B shape><dim×4B f32 vec>` | `<1B found>` then if found: `<4B i32 index>` + optional `[lbl][data]` per shape (bit `0x01` ignored — caller already has the vector) | `bad exists body` · `bad shape mask` |
2021

2122
**Shape mask:** `0x01` vector · `0x02` label · `0x04` data · default `0x07` (full).
2223
**Metric:** `0x00` L2 (default) · `0x01` cosine.

README.md

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,17 +30,18 @@ request: F0 <2B ns_len> [ns] <CMD> <2B label_len> [label] <4B body_len> [body]
3030
response: <1B status> <4B body_len> [body] ; status 0=ok, 1=err
3131
```
3232

33-
15 commands, one envelope. See `PROTOCOL-2.0.md` for the byte-exact spec or `sdk/README.md` for the quick reference.
33+
16 commands, one envelope. See `PROTOCOL-2.0.md` for the byte-exact spec or `sdk/README.md` for the quick reference.
3434

3535
Use the SDK libraries — there is no text interface.
3636

3737
---
3838

3939
## What it does
4040

41-
- **PUSH** — store a vector, optionally with a label and a ≤100KB data payload (data requires label). Returns the slot index.
41+
- **PUSH** — store a vector, optionally with a label and a ≤100KB data payload (data requires label). Returns the slot index. Rejects bit-identical duplicates.
4242
- **QUERY** — nearest-neighbor search; metric byte selects L2 or cosine.
4343
- **QID** — like QUERY but the query is an existing stored vector (by index or label).
44+
- **EXISTS** — exact-match lookup by vector content. Returns the slot index (and optional label/data) if present, `found=0` otherwise.
4445
- **GET** — retrieve records by index, by label (may yield multiple), or batch by index list.
4546
- **SET_DATA / GET_DATA** — manage the per-slot payload independently of the vector.
4647
- **UPDATE** — overwrite a vector in place (label and data untouched).
@@ -196,6 +197,7 @@ Requires NVIDIA CUDA Toolkit 12.x for `vec`. `vec-cpu` just needs a C++ compiler
196197
- **Brute force.** Every query scans every vector. Exact results, zero approximation.
197198
- **GPU top-K.** Above 100K vectors, a CUDA kernel finds the top results on GPU.
198199
- **All RAM.** Vectors in VRAM (or RAM for vec-cpu), labels and data alongside them. No mmap, no lazy paging — disk is touched only on startup load and explicit SAVE.
200+
- **Append-time dedup.** Every PUSH is hashed (xxh64 over stored bytes) and byte-compared against existing alive slots; bit-identical vectors are rejected with the existing slot's index. EXISTS exposes the same lookup as a query op.
199201
- **Indices are permanent.** Slot 42 is always slot 42. Deletes are tombstones.
200202
- **Labels are clean.** ≤2048 bytes, no spaces, no `: * ? " < > | ,`. URI-style paths like `docs/file.pdf` are fine.
201203
- **Data is opaque.** ≤100KB per slot. VEC stores the bytes verbatim — sniff the mime on the client if you need to.
@@ -210,10 +212,11 @@ Requires NVIDIA CUDA Toolkit 12.x for `vec`. `vec-cpu` just needs a C++ compiler
210212
```
211213
.tensors [4B dim][4B count][4B deleted][1B fmt][count×1B alive][vectors][4B CRC32]
212214
.meta [4B count][per slot: 4B len + label bytes]
215+
.hashes [4B count][count × 8B u64 xxh64 hash][4B CRC32]
213216
.data [4B count][count×1B alive mask][per present slot: 4B len + bytes][4B CRC32]
214217
```
215218

216-
`.data` is new in 2.0 and only created on save when at least one slot has a payload.
219+
`.data` and `.hashes` are new in 2.0. `.data` is only written when at least one slot has a payload. `.hashes` is the persisted dedup index — if missing or its CRC fails, it's rebuilt from `.tensors` at startup.
217220

218221
## Client SDKs
219222

sdk/README.md

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ A response body is `<4B u32 count>` followed by `count` records.
8383
| QID | `11` | `<1B metric><1B shape>[<4B idx>` or empty if label]` | top-K records (with distance) |
8484
| SET_DATA | `13` | `[<4B idx>]<4B u32 dlen>[data]` | empty |
8585
| GET_DATA | `14` | `<4B idx>` or empty (label) | `<4B u32 dlen>[data]` |
86+
| EXISTS | `15` | `<1B shape><vec>` | `<1B found>` then if found: `<4B i32 idx>` + optional `[lbl][data]` per shape |
8687

8788
Removed: `0x03` (CPULL → use QUERY metric=1), `0x05` (MGET → use GET batch mode), `0x12` (CPID → use QID metric=1).
8889
Reserved: `0x0B`, `0x0C` (do not reuse).
@@ -91,7 +92,8 @@ Reserved: `0x0B`, `0x0C` (do not reuse).
9192

9293
- **metric**: `0x00` = L2 (default), `0x01` = cosine. Other values → ERR.
9394
- **mode** (GET): `0x00` = single, `0x01` = batch.
94-
- **PUSH**: vector required; label optional via header; data optional in body but **requires label**. Caps: label ≤ 2048 bytes, data ≤ 102400 bytes.
95+
- **PUSH**: vector required; label optional via header; data optional in body but **requires label**. Caps: label ≤ 2048 bytes, data ≤ 102400 bytes. Bit-identical duplicates are rejected with `duplicate vector at index <N>`.
96+
- **EXISTS**: byte-exact lookup. Hash check + memcmp against the stored representation. Tombstoned slots are skipped. Shape bit `0x01` (vector) is meaningless and ignored — caller already supplied the vector.
9597
- **GET single by label** may return multiple records if the label is ambiguous.
9698
- **DELETE / UNDO** clear the label and data of the affected slot in addition to the vector.
9799
- **UPDATE** overwrites the vector only; use `LABEL` and `SET_DATA` for label/data changes.
@@ -131,6 +133,8 @@ ERR responses carry an ASCII message in the body. Common ones:
131133
| `label not found` | no slot with this label |
132134
| `deleted` / `already deleted` | tombstoned slot |
133135
| `read-only mode` | write command on read-only DB |
136+
| `duplicate vector at index <N>` | PUSH of a bit-identical vector — `<N>` is the existing slot |
137+
| `bad exists body` / `bad shape mask` | EXISTS body length wrong, or shape uses reserved bits |
134138
| `unknown binary command` | unrecognized CMD byte |
135139

136140
Read-only rejects: PUSH, UPDATE, DELETE, LABEL, UNDO, SAVE, SET_DATA.
@@ -147,6 +151,7 @@ Read-only rejects: PUSH, UPDATE, DELETE, LABEL, UNDO, SAVE, SET_DATA.
147151
All clients expose:
148152
- `push(vec, label=None, data=None)` — vector + optional label + optional data
149153
- `query(vec, cosine=False, shape=FULL)` / `qid(idx_or_label, ...)` — kNN search
154+
- `exists(vec, shape=0)` — exact-match lookup; returns `(idx, label, data)` or `None`
150155
- `get(target, shape=FULL)` — single by index, single by label (may yield multiple), or batch
151156
- `set_data(idx_or_label, bytes)` / `get_data(idx_or_label)`
152157
- `update(idx_or_label, vec)` — vector only
@@ -163,7 +168,8 @@ All clients expose:
163168
```
164169
.tensors [4B dim][4B count][4B deleted][1B fmt][count×1B alive][vectors][4B CRC32]
165170
.meta [4B count][per slot: 4B len + label bytes]
171+
.hashes [4B count][count × 8B u64 xxh64][4B CRC32]
166172
.data [4B count][count×1B alive mask][per present slot: 4B u32 dlen + data bytes][4B CRC32]
167173
```
168174

169-
`.tensors` and `.meta` are unchanged from 1.x — existing DBs load. `.data` is new in 2.0; absent file means "no blobs". Server creates `.data` on first SAVE if any slot has data.
175+
`.tensors` and `.meta` are unchanged from 1.x — existing DBs load. `.data` and `.hashes` are new in 2.0. `.data` absent means "no blobs". `.hashes` is the persisted dedup index; if missing or its CRC fails, the server rebuilds it from `.tensors` at startup.
21.9 KB
Binary file not shown.

sdk/vec_client.h

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,14 @@
2626
* int n = vec.get_batch(idx_array, idx_count, rs, 10);
2727
* int n = vec.get("img/cat.jpg", rs, 10);
2828
*
29+
* // Exact-match lookup (returns 1 found / 0 not found / -1 error).
30+
* // Use to dedup-check before push, or fetch a record by content.
31+
* VecRecord r;
32+
* if (vec.exists(vec_f32, dim, &r, VEC_SHAPE_LABEL) == 1) {
33+
* printf("already at slot %d label=%s\n", r.index, r.label ? r.label : "");
34+
* vec_free_record(&r);
35+
* }
36+
*
2937
* for (int i = 0; i < n; i++) vec_free_record(&rs[i]);
3038
*
3139
* vec.set_data(42, jpeg_bytes, jpeg_len);
@@ -97,6 +105,7 @@ typedef int vec_sock_t;
97105
#define VEC_CMD_QID 0x11
98106
#define VEC_CMD_SET_DATA 0x13
99107
#define VEC_CMD_GET_DATA 0x14
108+
#define VEC_CMD_EXISTS 0x15
100109

101110
/* response status */
102111
#define VEC_RESP_OK 0x00
@@ -430,6 +439,59 @@ class VecClient {
430439
return rc;
431440
}
432441

442+
/* EXISTS: byte-exact lookup by vector content.
443+
* shape uses VEC_SHAPE_LABEL / VEC_SHAPE_DATA bits; SHAPE_VECTOR is ignored.
444+
* returns 1 if found (fills *out — caller must vec_free_record), 0 if not, -1 on error. */
445+
int exists(const float *vec, int dim, VecRecord *out) {
446+
return exists(vec, dim, out, 0);
447+
}
448+
int exists(const float *vec, int dim, VecRecord *out, unsigned char shape) {
449+
if (!vec || dim <= 0 || !out) return -1;
450+
int vbytes = dim * (int)sizeof(float);
451+
int blen = 1 + vbytes;
452+
char *body = (char *)malloc(blen);
453+
if (!body) return -1;
454+
body[0] = (char)shape;
455+
memcpy(body + 1, vec, vbytes);
456+
send_frame(VEC_CMD_EXISTS, NULL, 0, body, blen);
457+
free(body);
458+
unsigned char *resp = NULL;
459+
char err[128];
460+
int n = recv_response(&resp, err, sizeof(err));
461+
if (n < 0) return -1;
462+
memset(out, 0, sizeof(*out));
463+
if (n < 1) { free(resp); return -1; }
464+
if (resp[0] == 0) { free(resp); return 0; }
465+
if (n < 5) { free(resp); return -1; }
466+
memcpy(&out->index, resp + 1, 4);
467+
out->distance = 0.0f;
468+
unsigned int p = 5;
469+
if (shape & VEC_SHAPE_LABEL) {
470+
if ((int)(p + 4) > n) { free(resp); vec_free_record(out); return -1; }
471+
unsigned int ll; memcpy(&ll, resp + p, 4); p += 4;
472+
if ((int)(p + ll) > n) { free(resp); vec_free_record(out); return -1; }
473+
if (ll > 0) {
474+
out->label = (char *)malloc(ll + 1);
475+
if (out->label) { memcpy(out->label, resp + p, ll); out->label[ll] = '\0'; }
476+
out->label_len = ll;
477+
}
478+
p += ll;
479+
}
480+
if (shape & VEC_SHAPE_DATA) {
481+
if ((int)(p + 4) > n) { free(resp); vec_free_record(out); return -1; }
482+
unsigned int dl; memcpy(&dl, resp + p, 4); p += 4;
483+
if ((int)(p + dl) > n) { free(resp); vec_free_record(out); return -1; }
484+
if (dl > 0) {
485+
out->data = (unsigned char *)malloc(dl);
486+
if (out->data) memcpy(out->data, resp + p, dl);
487+
out->data_len = dl;
488+
}
489+
p += dl;
490+
}
491+
free(resp);
492+
return 1;
493+
}
494+
433495
/* GET single by index */
434496
int get(int index, VecRecord *out, int max_records) {
435497
return get(index, out, max_records, VEC_SHAPE_FULL);

sdk/vec_client.js

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ const CMD_INFO = 0x10;
6464
const CMD_QID = 0x11;
6565
const CMD_SET_DATA = 0x13;
6666
const CMD_GET_DATA = 0x14;
67+
const CMD_EXISTS = 0x15;
6768

6869
const RESP_OK = 0x00;
6970
const RESP_ERR = 0x01;
@@ -226,6 +227,35 @@ class VecClient {
226227
return this._decodeRecords(body, shape, await this._dim(), true);
227228
}
228229

230+
/** Exact-match lookup by vector content. Returns null if not found,
231+
* otherwise { index, label?, data? } with label/data populated only
232+
* when shape includes them. SHAPE_VECTOR is meaningless and ignored. */
233+
async exists(vector, opts = {}) {
234+
const { shape = 0 } = opts;
235+
const vecBuf = this._vecToBuffer(vector);
236+
const head = Buffer.from([shape & 0xFF]);
237+
this._sendFrame(CMD_EXISTS, Buffer.alloc(0), Buffer.concat([head, vecBuf]));
238+
const body = await this._recvResponse();
239+
if (body.length < 1) throw new Error('EXISTS: empty response');
240+
if (body[0] === 0) return null;
241+
if (body.length < 5) throw new Error('EXISTS: truncated response');
242+
const out = { index: body.readInt32LE(1) };
243+
let p = 5;
244+
if (shape & SHAPE_LABEL) {
245+
if (body.length < p + 4) throw new Error('EXISTS: truncated label length');
246+
const ll = body.readUInt32LE(p); p += 4;
247+
out.label = ll > 0 ? body.slice(p, p + ll).toString('utf8') : '';
248+
p += ll;
249+
}
250+
if (shape & SHAPE_DATA) {
251+
if (body.length < p + 4) throw new Error('EXISTS: truncated data length');
252+
const dl = body.readUInt32LE(p); p += 4;
253+
out.data = dl > 0 ? body.slice(p, p + dl) : Buffer.alloc(0);
254+
p += dl;
255+
}
256+
return out;
257+
}
258+
229259
async qid(indexOrLabel, opts = {}) {
230260
const { cosine = false, shape = SHAPE_FULL } = opts;
231261
const head = Buffer.from([cosine ? METRIC_COSINE : METRIC_L2, shape & 0xFF]);

sdk/vec_client.pas

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ interface
6868
VEC_CMD_QID = $11;
6969
VEC_CMD_SET_DATA = $13;
7070
VEC_CMD_GET_DATA = $14;
71+
VEC_CMD_EXISTS = $15;
7172

7273
VEC_RESP_OK = $00;
7374
VEC_RESP_ERR = $01;
@@ -100,6 +101,13 @@ TVecRecord = record
100101

101102
TVecRecords = array of TVecRecord;
102103

104+
TVecExistsResult = record
105+
Found : Boolean;
106+
Index : Integer;
107+
Label_ : string; { empty if shape excluded label }
108+
Data : TByteArray; { nil if shape excluded data }
109+
end;
110+
103111
TVecInfo = record
104112
Dim : Integer;
105113
Count : Integer;
@@ -148,6 +156,8 @@ TVecClient = class
148156
function GetByLabel(const ALabel: string; Shape: Byte = VEC_SHAPE_FULL): TVecRecords;
149157
function GetBatch(const Indices: array of Integer; Shape: Byte = VEC_SHAPE_FULL): TVecRecords;
150158

159+
function Exists(const Vec: TSingleArray; Shape: Byte = 0): TVecExistsResult;
160+
151161
procedure SetData(Index: Integer; const Data: TByteArray); overload;
152162
procedure SetData(const ALabel: string; const Data: TByteArray); overload;
153163
function GetData(Index: Integer): TByteArray; overload;
@@ -543,6 +553,50 @@ function TVecClient.GetBatch(const Indices: array of Integer; Shape: Byte): TVec
543553
Result := ParseRecords(R, Shape, Dim, False);
544554
end;
545555

556+
{ ---------------- EXISTS ---------------- }
557+
558+
function TVecClient.Exists(const Vec: TSingleArray; Shape: Byte): TVecExistsResult;
559+
var Body, R: AnsiString; VBytes: Integer; LL, DL: Cardinal; P: Integer;
560+
begin
561+
Result.Found := False;
562+
Result.Index := -1;
563+
Result.Label_ := '';
564+
Result.Data := nil;
565+
VBytes := Length(Vec) * SizeOf(Single);
566+
SetLength(Body, 1 + VBytes);
567+
Body[1] := AnsiChar(Shape);
568+
if VBytes > 0 then Move(Vec[0], Body[2], VBytes);
569+
SendFrame(VEC_CMD_EXISTS, '', Body);
570+
R := RecvResponse;
571+
if Length(R) < 1 then raise Exception.Create('EXISTS: empty response');
572+
if Byte(R[1]) = 0 then Exit;
573+
if Length(R) < 5 then raise Exception.Create('EXISTS: truncated response');
574+
Result.Found := True;
575+
Move(R[2], Result.Index, 4);
576+
P := 6; { 1-based, after found(1) + index(4) }
577+
if (Shape and VEC_SHAPE_LABEL) <> 0 then
578+
begin
579+
if Length(R) < P + 3 then raise Exception.Create('EXISTS: truncated label length');
580+
Move(R[P], LL, 4); Inc(P, 4);
581+
if LL > 0 then
582+
begin
583+
Result.Label_ := string(Copy(R, P, LL));
584+
Inc(P, LL);
585+
end;
586+
end;
587+
if (Shape and VEC_SHAPE_DATA) <> 0 then
588+
begin
589+
if Length(R) < P + 3 then raise Exception.Create('EXISTS: truncated data length');
590+
Move(R[P], DL, 4); Inc(P, 4);
591+
if DL > 0 then
592+
begin
593+
SetLength(Result.Data, DL);
594+
Move(R[P], Result.Data[0], DL);
595+
Inc(P, DL);
596+
end;
597+
end;
598+
end;
599+
546600
{ ---------------- SET_DATA / GET_DATA ---------------- }
547601

548602
procedure TVecClient.SetData(Index: Integer; const Data: TByteArray);

0 commit comments

Comments
 (0)