Skip to content

Commit 006d113

Browse files
scottn12ChumpChief
scottn12
and
ChumpChief
authored
feat(directory): Directory rollback across remote ops (Part 1 - storage ops only) (microsoft#25126)
## Description This PR is a prototype for rollback across remote ops for directory. This PR is based off of microsoft#25070 and implements rollback for storage ops (set, delete, clear). Note: Rollback acrros remote ops for subdirectories is **not** implemented in this PR and will be included in a follow up. ## Misc [AB#43738](https://dev.azure.com/fluidframework/235294da-091d-4c29-84fc-cdfc3d90890b/_workitems/edit/43738) Prototype: microsoft#25092 Will fix [AB#38522](https://dev.azure.com/fluidframework/235294da-091d-4c29-84fc-cdfc3d90890b/_workitems/edit/38522) --------- Co-authored-by: Matt Rakow <[email protected]>
1 parent 50f081a commit 006d113

File tree

8 files changed

+1367
-479
lines changed

8 files changed

+1367
-479
lines changed

‎packages/dds/map/src/directory.ts‎

Lines changed: 608 additions & 450 deletions
Large diffs are not rendered by default.

‎packages/dds/map/src/mapKernel.ts‎

Lines changed: 1 addition & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import {
2323
serializeValue,
2424
migrateIfSharedSerializable,
2525
} from "./localValues.js";
26+
import { findLast, findLastIndex } from "./utils.js";
2627

2728
/**
2829
* Defines the means to process and resubmit a given op on a map.
@@ -106,25 +107,6 @@ type PendingDataEntry = PendingKeyLifetime | PendingKeyDelete | PendingClear;
106107
*/
107108
type PendingLocalOpMetadata = PendingKeySet | PendingKeyDelete | PendingClear;
108109

109-
/**
110-
* Rough polyfill for Array.findLastIndex until we target ES2023 or greater.
111-
*/
112-
const findLastIndex = <T>(array: T[], callbackFn: (value: T) => boolean): number => {
113-
for (let i = array.length - 1; i >= 0; i--) {
114-
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
115-
if (callbackFn(array[i]!)) {
116-
return i;
117-
}
118-
}
119-
return -1;
120-
};
121-
122-
/**
123-
* Rough polyfill for Array.findLast until we target ES2023 or greater.
124-
*/
125-
const findLast = <T>(array: T[], callbackFn: (value: T) => boolean): T | undefined =>
126-
array[findLastIndex(array, callbackFn)];
127-
128110
/**
129111
* A SharedMap is a map-like distributed data structure.
130112
*/

‎packages/dds/map/src/test/mocha/directory.iteration.spec.ts‎

Lines changed: 202 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,202 @@
1+
/*!
2+
* Copyright (c) Microsoft Corporation and contributors. All rights reserved.
3+
* Licensed under the MIT License.
4+
*/
5+
6+
import { strict as assert } from "node:assert";
7+
8+
import { AttachState } from "@fluidframework/container-definitions";
9+
import {
10+
MockContainerRuntimeFactory,
11+
MockFluidDataStoreRuntime,
12+
MockStorage,
13+
type MockContainerRuntime,
14+
} from "@fluidframework/test-runtime-utils/internal";
15+
16+
import { DirectoryFactory } from "../../directoryFactory.js";
17+
import type { ISharedDirectory } from "../../interfaces.js";
18+
19+
interface TestParts {
20+
sharedDirectory: ISharedDirectory;
21+
dataStoreRuntime: MockFluidDataStoreRuntime;
22+
containerRuntimeFactory: MockContainerRuntimeFactory;
23+
containerRuntime: MockContainerRuntime;
24+
}
25+
26+
const directoryFactory = new DirectoryFactory();
27+
28+
function setupTest(): TestParts {
29+
const containerRuntimeFactory = new MockContainerRuntimeFactory({ flushMode: 1 }); // TurnBased
30+
const dataStoreRuntime = new MockFluidDataStoreRuntime({ clientId: "1" });
31+
const containerRuntime = containerRuntimeFactory.createContainerRuntime(dataStoreRuntime);
32+
const sharedDirectory = directoryFactory.create(dataStoreRuntime, "shared-directory-1");
33+
dataStoreRuntime.setAttachState(AttachState.Attached);
34+
sharedDirectory.connect({
35+
deltaConnection: dataStoreRuntime.createDeltaConnection(),
36+
objectStorage: new MockStorage(),
37+
});
38+
return {
39+
sharedDirectory,
40+
dataStoreRuntime,
41+
containerRuntimeFactory,
42+
containerRuntime,
43+
};
44+
}
45+
46+
// Helper to create another client attached to the same containerRuntimeFactory
47+
function createAdditionalClient(
48+
containerRuntimeFactory: MockContainerRuntimeFactory,
49+
id: string = "client-2",
50+
): {
51+
sharedDirectory: ISharedDirectory;
52+
dataStoreRuntime: MockFluidDataStoreRuntime;
53+
containerRuntime: MockContainerRuntime;
54+
} {
55+
const dataStoreRuntime = new MockFluidDataStoreRuntime({ clientId: id });
56+
const containerRuntime = containerRuntimeFactory.createContainerRuntime(dataStoreRuntime);
57+
const sharedDirectory = directoryFactory.create(dataStoreRuntime, `shared-directory-${id}`);
58+
dataStoreRuntime.setAttachState(AttachState.Attached);
59+
sharedDirectory.connect({
60+
deltaConnection: dataStoreRuntime.createDeltaConnection(),
61+
objectStorage: new MockStorage(),
62+
});
63+
return { sharedDirectory, dataStoreRuntime, containerRuntime };
64+
}
65+
66+
describe("SharedDirectory iteration", () => {
67+
it("should have eventually consistent iteration order between clients when simultaneous set", () => {
68+
const { sharedDirectory, containerRuntimeFactory, containerRuntime } = setupTest();
69+
// Create a second client
70+
const { sharedDirectory: sharedDirectory2, containerRuntime: containerRuntime2 } =
71+
createAdditionalClient(containerRuntimeFactory);
72+
73+
sharedDirectory.set("key1", "value1");
74+
sharedDirectory.set("key2", "value2");
75+
containerRuntime.flush();
76+
containerRuntimeFactory.processAllMessages();
77+
78+
sharedDirectory.set("key3", "value3");
79+
sharedDirectory2.set("key4", "value4");
80+
containerRuntime.flush();
81+
containerRuntime2.flush();
82+
containerRuntimeFactory.processAllMessages();
83+
84+
const keys1 = [...sharedDirectory.keys()];
85+
const keys2 = [...sharedDirectory2.keys()];
86+
87+
assert.deepStrictEqual(
88+
keys1,
89+
["key1", "key2", "key3", "key4"],
90+
"Keys should match expected",
91+
);
92+
assert.deepStrictEqual(keys1, keys2, "Keys should match between clients");
93+
});
94+
95+
it("should have eventually consistent iteration order between clients when suppressed delete", () => {
96+
const { sharedDirectory, containerRuntimeFactory, containerRuntime } = setupTest();
97+
// Create a second client
98+
const { sharedDirectory: sharedDirectory2, containerRuntime: containerRuntime2 } =
99+
createAdditionalClient(containerRuntimeFactory);
100+
101+
sharedDirectory.set("key1", "value1");
102+
sharedDirectory.set("key2", "value2");
103+
containerRuntime.flush();
104+
containerRuntimeFactory.processAllMessages();
105+
106+
// sharedDirectory2 won't observe the delete of key1, it is suppressed since there is a pending set of the
107+
// same key. But it should move to the end of the iteration order since the sequenced perspective
108+
// is that it was deleted and re-added.
109+
sharedDirectory.delete("key1");
110+
sharedDirectory2.set("key1", "otherValue1");
111+
containerRuntime.flush();
112+
containerRuntime2.flush();
113+
containerRuntimeFactory.processAllMessages();
114+
115+
const keys1 = [...sharedDirectory.keys()];
116+
const keys2 = [...sharedDirectory2.keys()];
117+
118+
assert.deepStrictEqual(keys1, ["key2", "key1"], "Keys should match expected");
119+
assert.deepStrictEqual(keys1, keys2, "Keys should match between clients");
120+
});
121+
122+
it("should have eventually consistent iteration order between clients when clear", () => {
123+
const { sharedDirectory, containerRuntimeFactory, containerRuntime } = setupTest();
124+
// Create a second client
125+
const { sharedDirectory: sharedDirectory2, containerRuntime: containerRuntime2 } =
126+
createAdditionalClient(containerRuntimeFactory);
127+
128+
sharedDirectory.set("key1", "value1");
129+
sharedDirectory.set("key2", "value2");
130+
containerRuntime.flush();
131+
containerRuntimeFactory.processAllMessages();
132+
133+
// sharedDirectory's clear() changes the effect of sharedDirectory2's set() of key1 (it becomes a
134+
// re-add instead of a modification). Its new iteration order should reflect the new
135+
// lifetime created by the re-add.
136+
sharedDirectory.clear();
137+
sharedDirectory2.set("key3", "value3");
138+
sharedDirectory2.set("key1", "otherValue1");
139+
sharedDirectory2.set("key4", "value4");
140+
containerRuntime.flush();
141+
containerRuntime2.flush();
142+
containerRuntimeFactory.processAllMessages();
143+
144+
const keys1 = [...sharedDirectory.keys()];
145+
const keys2 = [...sharedDirectory2.keys()];
146+
147+
assert.deepStrictEqual(keys1, ["key3", "key1", "key4"], "Keys should match expected");
148+
assert.deepStrictEqual(keys1, keys2, "Keys should match between clients");
149+
});
150+
151+
it("should have eventually consistent iteration order with nested subdirectory operations", () => {
152+
const { sharedDirectory, containerRuntimeFactory, containerRuntime } = setupTest();
153+
const { sharedDirectory: sharedDirectory2, containerRuntime: containerRuntime2 } =
154+
createAdditionalClient(containerRuntimeFactory);
155+
156+
sharedDirectory.set("rootKey1", "rootValue1");
157+
const subDir1 = sharedDirectory.createSubDirectory("subdir");
158+
subDir1.set("subKey1", "subValue1");
159+
sharedDirectory.set("rootKey2", "rootValue2");
160+
containerRuntime.flush();
161+
containerRuntimeFactory.processAllMessages();
162+
163+
const subDir2 = sharedDirectory2.getSubDirectory("subdir");
164+
assert(subDir2 !== undefined, "Subdirectory should exist on second client");
165+
166+
sharedDirectory.set("rootKey3", "rootValue3");
167+
subDir1.set("subKey2", "subValue2");
168+
sharedDirectory2.set("rootKey4", "rootValue4");
169+
subDir2.set("subKey3", "subValue3");
170+
sharedDirectory.delete("rootKey1");
171+
subDir1.delete("subKey1");
172+
sharedDirectory2.set("rootKey1", "newRootValue1");
173+
174+
containerRuntime.flush();
175+
containerRuntime2.flush();
176+
containerRuntimeFactory.processAllMessages();
177+
178+
const rootKeys1 = [...sharedDirectory.keys()];
179+
const rootKeys2 = [...sharedDirectory2.keys()];
180+
181+
assert.deepStrictEqual(
182+
rootKeys1,
183+
["rootKey2", "rootKey3", "rootKey4", "rootKey1"],
184+
"Root keys should match expected order",
185+
);
186+
assert.deepStrictEqual(rootKeys1, rootKeys2, "Root keys should match between clients");
187+
188+
const subKeys1 = [...subDir1.keys()];
189+
const subKeys2 = [...subDir2.keys()];
190+
191+
assert.deepStrictEqual(
192+
subKeys1,
193+
["subKey2", "subKey3"],
194+
"Subdirectory keys should match expected order",
195+
);
196+
assert.deepStrictEqual(
197+
subKeys1,
198+
subKeys2,
199+
"Subdirectory keys should match between clients",
200+
);
201+
});
202+
});

0 commit comments

Comments
 (0)