Skip to content

Commit 1cd7f5a

Browse files
DoneDeal0DoneDeal0
committed
chore: boost getlistdiff perf
1 parent e1838e0 commit 1cd7f5a

File tree

5 files changed

+240
-61
lines changed

5 files changed

+240
-61
lines changed

‎benchmark/index.ts‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import { runListBench100K, runListBench10K } from "./list-benchmark";
12
import {
23
runObjectBench10K,
34
runObjectBench100K,
@@ -6,8 +7,24 @@ import {
67

78
console.log("Running Superdiff benchmarks");
89

9-
runObjectBench10K();
10-
runObjectBench100K();
11-
runNestedObjectBench();
12-
// runListBench();
13-
// runTextBench();
10+
async function main() {
11+
console.log("=== SUPERDIFF BENCHMARKS ===");
12+
13+
// Objects
14+
// runObjectBench10K();
15+
// runObjectBench100K();
16+
// runNestedObjectBench();
17+
18+
// List
19+
runListBench10K();
20+
runListBench100K();
21+
// Lists (streaming)
22+
// await runListStreamBench();
23+
24+
console.log("\n=== BENCHMARK COMPLETE ===");
25+
}
26+
27+
main().catch((err) => {
28+
console.error(err);
29+
process.exit(1);
30+
});

‎benchmark/list-benchmark.ts‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
import arrDiff from "arr-diff";
2+
import deepDiff from "deep-diff";
3+
import { getListDiff } from "../src";
4+
// import { streamListDiff } from "../src/server";
5+
import { bench } from "./utils";
6+
7+
function generateList(size: number): number[] {
8+
return Array.from({ length: size }, (_, i) => i);
9+
}
10+
11+
function generateObjectList(
12+
size: number,
13+
randomize: boolean,
14+
): { id: number; value: number }[] {
15+
return Array.from({ length: size }, (_, i) => ({
16+
id: i,
17+
value: randomize && i % 1000 ? i + 10 : i,
18+
}));
19+
}
20+
21+
function mutateList(
22+
list: number[],
23+
updateRate: number,
24+
deleteRate: number,
25+
addRate: number,
26+
): number[] {
27+
const result: number[] = [];
28+
29+
for (let i = 0; i < list.length; i++) {
30+
// delete
31+
if (i % deleteRate === 0) continue;
32+
33+
// update
34+
if (i % updateRate === 0) {
35+
result.push(list[i] + 1_000_000);
36+
} else {
37+
result.push(list[i]);
38+
}
39+
40+
// add
41+
if (i % addRate === 0) {
42+
result.push(-i);
43+
}
44+
}
45+
46+
return result;
47+
}
48+
49+
export function runListBench10K() {
50+
const prev = generateList(10_000);
51+
const curr = mutateList(prev, 50, 200, 200);
52+
console.log("\nList diff – 10k items");
53+
54+
const deep = bench("deep-diff", 20, () => deepDiff.diff(prev, curr));
55+
const arrD = bench("arr-diff", 20, () => arrDiff(prev, curr));
56+
const superdiff = bench("Superdiff", 20, () => getListDiff(prev, curr));
57+
return { superdiff, deep, arrD };
58+
}
59+
60+
export function runListBench100K() {
61+
const prev = generateList(100_000);
62+
const curr = mutateList(prev, 20, 50, 50);
63+
console.log("\nList diff – 100k items");
64+
65+
const deep = bench("deep-diff", 20, () => deepDiff.diff(prev, curr));
66+
const arrD = bench("arr-diff", 20, () => arrDiff(prev, curr));
67+
const superdiff = bench("Superdiff", 20, () => getListDiff(prev, curr));
68+
return { superdiff, deep, arrD };
69+
}
70+
71+
// export async function runListStreamBench() {
72+
// const prev = generateObjectList(1_000_000, false);
73+
// const curr = generateObjectList(1_000_000, true);
74+
75+
// console.log("\nList diff – 1M items (streaming)");
76+
77+
// const start = performance.now();
78+
79+
// await new Promise<void>((resolve) => {
80+
// const stream = streamListDiff(prev, curr, "id", { useWorker: false });
81+
82+
// stream.on("data", () => {
83+
// // consume chunk (do nothing)
84+
// });
85+
86+
// stream.on("end", resolve);
87+
// stream.on("error", resolve);
88+
// });
89+
90+
// const duration = performance.now() - start;
91+
92+
// console.log(`Superdiff (stream): ${duration.toFixed(2)} ms`);
93+
94+
// return duration;
95+
// }

‎package-lock.json‎

Lines changed: 19 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,9 @@
9797
"@semantic-release/npm": "^12.0.1",
9898
"@swc/core": "^1.10.18",
9999
"@swc/jest": "^0.2.37",
100+
"@types/arr-diff": "^4.0.3",
100101
"@types/jest": "^29.5.14",
102+
"arr-diff": "^4.0.0",
101103
"blob-polyfill": "^9.0.20240710",
102104
"deep-diff": "^1.0.2",
103105
"deep-object-diff": "^1.1.9",

‎src/lib/list-diff/index.ts‎

Lines changed: 102 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,31 @@
1-
import { isEqual, isObject } from "@lib/utils";
1+
import { isObject } from "@lib/utils";
22
import {
33
DEFAULT_LIST_DIFF_OPTIONS,
44
ListStatus,
55
ListDiff,
66
ListDiffOptions,
77
} from "@models/list";
88

9+
function getDiffStatus(statusMap: Set<ListStatus>): ListDiff["status"] {
10+
if (statusMap.has(ListStatus.UPDATED)) return ListStatus.UPDATED;
11+
12+
const isUniqueStatus = (status: ListStatus) => {
13+
let isUnique = true;
14+
for (const value of statusMap) {
15+
if (value !== status) {
16+
isUnique = false;
17+
break;
18+
}
19+
}
20+
return isUnique;
21+
};
22+
23+
if (isUniqueStatus(ListStatus.ADDED)) return ListStatus.ADDED;
24+
if (isUniqueStatus(ListStatus.DELETED)) return ListStatus.DELETED;
25+
if (isUniqueStatus(ListStatus.EQUAL)) return ListStatus.EQUAL;
26+
return ListStatus.UPDATED;
27+
}
28+
929
function getLeanDiff(
1030
diff: ListDiff["diff"],
1131
showOnly = [] as ListDiffOptions["showOnly"],
@@ -39,17 +59,11 @@ function formatSingleListDiff<T>(
3959
};
4060
}
4161

42-
function getListStatus(listDiff: ListDiff["diff"]): ListStatus {
43-
return listDiff.some((value) => value.status !== ListStatus.EQUAL)
44-
? ListStatus.UPDATED
45-
: ListStatus.EQUAL;
46-
}
47-
4862
function isReferencedObject(
4963
value: unknown,
5064
referenceProperty: ListDiffOptions["referenceProperty"],
5165
): value is Record<string, unknown> {
52-
if (isObject(value) && !!referenceProperty) {
66+
if (!!referenceProperty && isObject(value)) {
5367
return Object.hasOwn(value, referenceProperty);
5468
}
5569
return false;
@@ -84,84 +98,116 @@ export const getListDiff = <T>(
8498
return formatSingleListDiff(prevList as T[], ListStatus.DELETED, options);
8599
}
86100
const diff: ListDiff["diff"] = [];
87-
const prevIndexMatches = new Set<number>();
101+
const previousMap = new Map<string, { indexes: number[]; value: T }>();
102+
const statusMap = new Set<ListStatus>();
88103

89-
nextList.forEach((nextValue, i) => {
90-
const prevIndex = prevList.findIndex((prevValue, prevIdx) => {
91-
if (prevIndexMatches.has(prevIdx)) {
92-
return false;
104+
prevList.forEach((prevValue, i) => {
105+
if (isReferencedObject(prevValue, options.referenceProperty)) {
106+
const reference = String(prevValue[options.referenceProperty || ""]);
107+
const match = previousMap.get(reference);
108+
if (match) {
109+
const nextIndexes = [...match.indexes, i];
110+
previousMap.set(reference, {
111+
value: match.value,
112+
indexes: nextIndexes,
113+
});
114+
} else {
115+
previousMap.set(reference, { value: prevValue, indexes: [i] });
93116
}
94-
if (isReferencedObject(prevValue, options.referenceProperty)) {
95-
if (isObject(nextValue)) {
96-
return isEqual(
97-
prevValue[options.referenceProperty as string],
98-
nextValue[options.referenceProperty as string],
99-
);
100-
}
101-
return false;
117+
} else {
118+
const reference = JSON.stringify(prevValue);
119+
const match = previousMap.get(reference);
120+
if (match) {
121+
const nextIndexes = [...match.indexes, i];
122+
previousMap.set(reference, {
123+
value: match.value,
124+
indexes: nextIndexes,
125+
});
126+
} else {
127+
previousMap.set(reference, { value: prevValue, indexes: [i] });
102128
}
103-
return isEqual(prevValue, nextValue);
104-
});
105-
if (prevIndex > -1) {
106-
prevIndexMatches.add(prevIndex);
107129
}
108-
const indexDiff = prevIndex === -1 ? null : i - prevIndex;
109-
if (indexDiff === 0 || options.ignoreArrayOrder) {
110-
let nextStatus = ListStatus.EQUAL;
111-
if (isReferencedObject(nextValue, options.referenceProperty)) {
112-
if (!isEqual(prevList[prevIndex], nextValue)) {
113-
nextStatus = ListStatus.UPDATED;
114-
}
115-
}
116-
return diff.push({
130+
});
131+
132+
const injectNextValueInDiff = (
133+
nextValue: T,
134+
previousMatch: { indexes: number[]; value: T } | undefined | undefined,
135+
i: number,
136+
reference: string,
137+
) => {
138+
if (previousMatch) {
139+
const prevIndex = previousMatch.indexes[0];
140+
const indexDiff = i - prevIndex;
141+
const nextStatus =
142+
indexDiff === 0 || options.ignoreArrayOrder
143+
? ListStatus.EQUAL
144+
: options.considerMoveAsUpdate
145+
? ListStatus.UPDATED
146+
: ListStatus.MOVED;
147+
diff.push({
117148
value: nextValue,
118149
prevIndex,
119150
newIndex: i,
120151
indexDiff,
121152
status: nextStatus,
122153
});
123-
}
124-
if (prevIndex === -1) {
125-
return diff.push({
154+
statusMap.add(nextStatus);
155+
const nextPrevIndexes = previousMatch.indexes.splice(1);
156+
if (nextPrevIndexes.length === 0) {
157+
previousMap.delete(reference);
158+
} else {
159+
previousMap.set(reference, {
160+
value: nextValue,
161+
indexes: nextPrevIndexes,
162+
});
163+
}
164+
} else {
165+
diff.push({
126166
value: nextValue,
127167
prevIndex: null,
128168
newIndex: i,
129-
indexDiff,
169+
indexDiff: null,
130170
status: ListStatus.ADDED,
131171
});
172+
statusMap.add(ListStatus.ADDED);
173+
}
174+
};
175+
176+
nextList.forEach((nextValue, i) => {
177+
if (!!options.referenceProperty && isObject(nextValue)) {
178+
const reference = String(nextValue[options.referenceProperty || ""]);
179+
const previousIndexes = previousMap.get(reference);
180+
return injectNextValueInDiff(nextValue, previousIndexes, i, reference);
181+
} else {
182+
const reference = JSON.stringify(nextValue);
183+
const previousIndexes = previousMap.get(reference);
184+
return injectNextValueInDiff(nextValue, previousIndexes, i, reference);
132185
}
133-
return diff.push({
134-
value: nextValue,
135-
prevIndex,
136-
newIndex: i,
137-
indexDiff,
138-
status: options.considerMoveAsUpdate
139-
? ListStatus.UPDATED
140-
: ListStatus.MOVED,
141-
});
142186
});
143187

144-
prevList.forEach((prevValue, i) => {
145-
if (!prevIndexMatches.has(i)) {
146-
return diff.push({
147-
value: prevValue,
188+
for (const data of previousMap.values()) {
189+
statusMap.add(ListStatus.DELETED);
190+
data.indexes.forEach((i) =>
191+
diff.push({
192+
value: data.value,
148193
prevIndex: i,
149194
newIndex: null,
150195
indexDiff: null,
151196
status: ListStatus.DELETED,
152-
});
153-
}
154-
});
197+
}),
198+
);
199+
}
200+
155201
if (options.showOnly && options?.showOnly?.length > 0) {
156202
return {
157203
type: "list",
158-
status: getListStatus(diff),
204+
status: getDiffStatus(statusMap),
159205
diff: getLeanDiff(diff, options.showOnly),
160206
};
161207
}
162208
return {
163209
type: "list",
164-
status: getListStatus(diff),
210+
status: getDiffStatus(statusMap),
165211
diff,
166212
};
167213
};

0 commit comments

Comments
 (0)