Skip to content

Commit 83642fa

Browse files
authored
Merge pull request #847 from MatrixAI/feature-multiple-vault-resource
Allow vault `efs` resource acquisition to operate on multiple vaults in parallel
2 parents 9276357 + 7491144 commit 83642fa

File tree

17 files changed

+551
-160
lines changed

17 files changed

+551
-160
lines changed

package-lock.json

Lines changed: 4 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@
8080
"@matrixai/mdns": "^1.2.6",
8181
"@matrixai/quic": "^1.3.1",
8282
"@matrixai/resources": "^1.1.5",
83-
"@matrixai/rpc": "^0.6.0",
83+
"@matrixai/rpc": "^0.6.2",
8484
"@matrixai/timer": "^1.1.3",
8585
"@matrixai/workers": "^1.3.7",
8686
"@matrixai/ws": "^1.1.7",

src/client/errors.ts

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,16 @@ class ErrorClientAuthDenied<T> extends ErrorClient<T> {
1818
exitCode = sysexits.NOPERM;
1919
}
2020

21+
class ErrorClientInvalidHeader<T> extends ErrorClient<T> {
22+
static description = 'The header message does not match the expected type';
23+
exitCode = sysexits.USAGE;
24+
}
25+
26+
class ErrorClientProtocolError<T> extends ErrorClient<T> {
27+
static description = 'Data does not match the protocol requirements';
28+
exitCode = sysexits.USAGE;
29+
}
30+
2131
class ErrorClientService<T> extends ErrorClient<T> {}
2232

2333
class ErrorClientServiceRunning<T> extends ErrorClientService<T> {
@@ -45,6 +55,8 @@ export {
4555
ErrorClientAuthMissing,
4656
ErrorClientAuthFormat,
4757
ErrorClientAuthDenied,
58+
ErrorClientInvalidHeader,
59+
ErrorClientProtocolError,
4860
ErrorClientService,
4961
ErrorClientServiceRunning,
5062
ErrorClientServiceNotRunning,

src/client/handlers/VaultsSecretsCat.ts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,9 @@ class VaultsSecretsCat extends DuplexHandler<
2323
ClientRPCResponseResult<ContentOrErrorMessage>
2424
> {
2525
public handle = async function* (
26-
input: AsyncIterable<ClientRPCRequestParams<SecretIdentifierMessage>>,
26+
input: AsyncIterableIterator<
27+
ClientRPCRequestParams<SecretIdentifierMessage>
28+
>,
2729
): AsyncGenerator<ClientRPCResponseResult<ContentOrErrorMessage>> {
2830
const { db, vaultManager }: { db: DB; vaultManager: VaultManager } =
2931
this.container;

src/client/handlers/VaultsSecretsEnv.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import { DuplexHandler } from '@matrixai/rpc';
1010
import * as vaultsUtils from '../../vaults/utils';
1111
import * as vaultsErrors from '../../vaults/errors';
1212

13-
class VaultsSecretsList extends DuplexHandler<
13+
class VaultsSecretsEnv extends DuplexHandler<
1414
{
1515
db: DB;
1616
vaultManager: VaultManager;
@@ -86,4 +86,4 @@ class VaultsSecretsList extends DuplexHandler<
8686
};
8787
}
8888

89-
export default VaultsSecretsList;
89+
export default VaultsSecretsEnv;

src/client/handlers/VaultsSecretsMkdir.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ class VaultsSecretsMkdir extends DuplexHandler<
2121
ClientRPCResponseResult<SuccessOrErrorMessage>
2222
> {
2323
public handle = async function* (
24-
input: AsyncIterable<ClientRPCRequestParams<SecretDirMessage>>,
24+
input: AsyncIterableIterator<ClientRPCRequestParams<SecretDirMessage>>,
2525
): AsyncGenerator<ClientRPCResponseResult<SuccessOrErrorMessage>> {
2626
const { db, vaultManager }: { db: DB; vaultManager: VaultManager } =
2727
this.container;

src/client/handlers/VaultsSecretsRemove.ts

Lines changed: 100 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -1,99 +1,129 @@
11
import type { DB } from '@matrixai/db';
2+
import type { ResourceAcquire } from '@matrixai/resources';
23
import type {
34
ClientRPCRequestParams,
45
ClientRPCResponseResult,
5-
SecretIdentifierMessage,
6+
SecretsRemoveHeaderMessage,
7+
SecretIdentifierMessageTagged,
68
SuccessOrErrorMessage,
79
} from '../types';
810
import type VaultManager from '../../vaults/VaultManager';
11+
import type { FileSystemWritable } from '../../vaults/types';
12+
import { withG } from '@matrixai/resources';
913
import { DuplexHandler } from '@matrixai/rpc';
1014
import * as vaultsUtils from '../../vaults/utils';
1115
import * as vaultsErrors from '../../vaults/errors';
16+
import * as clientErrors from '../errors';
1217

1318
class VaultsSecretsRemove extends DuplexHandler<
1419
{
1520
db: DB;
1621
vaultManager: VaultManager;
1722
},
18-
ClientRPCRequestParams<SecretIdentifierMessage>,
23+
ClientRPCRequestParams<
24+
SecretsRemoveHeaderMessage | SecretIdentifierMessageTagged
25+
>,
1926
ClientRPCResponseResult<SuccessOrErrorMessage>
2027
> {
2128
public handle = async function* (
22-
input: AsyncIterable<ClientRPCRequestParams<SecretIdentifierMessage>>,
29+
input: AsyncIterableIterator<
30+
ClientRPCRequestParams<
31+
SecretsRemoveHeaderMessage | SecretIdentifierMessageTagged
32+
>
33+
>,
2334
): AsyncGenerator<ClientRPCResponseResult<SuccessOrErrorMessage>> {
2435
const { db, vaultManager }: { db: DB; vaultManager: VaultManager } =
2536
this.container;
26-
// Create a record of secrets to be removed, grouped by vault names
27-
const vaultGroups: Record<string, Array<string>> = {};
28-
const secretNames: Array<[string, string]> = [];
29-
let metadata: any = undefined;
30-
for await (const secretRemoveMessage of input) {
31-
if (metadata == null) metadata = secretRemoveMessage.metadata ?? {};
32-
secretNames.push([
33-
secretRemoveMessage.nameOrId,
34-
secretRemoveMessage.secretName,
35-
]);
37+
// Extract the header message from the iterator
38+
const headerMessagePair = await input.next();
39+
const headerMessage:
40+
| SecretsRemoveHeaderMessage
41+
| SecretIdentifierMessageTagged = headerMessagePair.value;
42+
// Testing if the header is of the expected format
43+
if (
44+
headerMessagePair.done ||
45+
headerMessage.type !== 'VaultNamesHeaderMessage'
46+
) {
47+
throw new clientErrors.ErrorClientInvalidHeader();
3648
}
37-
secretNames.forEach(([vaultName, secretName]) => {
38-
if (vaultGroups[vaultName] == null) {
39-
vaultGroups[vaultName] = [];
49+
// Create an array of write acquires
50+
const vaultAcquires = await db.withTransactionF(async (tran) => {
51+
const vaultAcquires: Array<ResourceAcquire<FileSystemWritable>> = [];
52+
for (const vaultName of headerMessage.vaultNames) {
53+
const vaultIdFromName = await vaultManager.getVaultId(vaultName, tran);
54+
const vaultId = vaultIdFromName ?? vaultsUtils.decodeVaultId(vaultName);
55+
if (vaultId == null) {
56+
throw new vaultsErrors.ErrorVaultsVaultUndefined(
57+
`Vault ${vaultName} does not exist`,
58+
);
59+
}
60+
const acquire = await vaultManager.withVaults(
61+
[vaultId],
62+
async (vault) => vault.acquireWrite(),
63+
);
64+
vaultAcquires.push(acquire);
4065
}
41-
vaultGroups[vaultName].push(secretName);
66+
return vaultAcquires;
4267
});
43-
// Now, all the paths will be removed for a vault within a single commit
44-
yield* db.withTransactionG(
45-
async function* (tran): AsyncGenerator<SuccessOrErrorMessage> {
46-
for (const [vaultName, secretNames] of Object.entries(vaultGroups)) {
47-
const vaultIdFromName = await vaultManager.getVaultId(
48-
vaultName,
49-
tran,
50-
);
51-
const vaultId =
52-
vaultIdFromName ?? vaultsUtils.decodeVaultId(vaultName);
53-
if (vaultId == null) {
54-
throw new vaultsErrors.ErrorVaultsVaultUndefined();
68+
// Acquire all locks in parallel and perform all operations at once
69+
yield* withG(
70+
vaultAcquires,
71+
async function* (efses): AsyncGenerator<SuccessOrErrorMessage> {
72+
// Creating the vault name to efs map for easy access
73+
const vaultMap = new Map<string, FileSystemWritable>();
74+
for (let i = 0; i < efses.length; i++) {
75+
vaultMap.set(headerMessage!.vaultNames[i], efses[i]);
76+
}
77+
let loopRan = false;
78+
for await (const message of input) {
79+
loopRan = true;
80+
// Header messages should not be seen anymore
81+
if (message.type === 'VaultNamesHeaderMessage') {
82+
throw new clientErrors.ErrorClientProtocolError(
83+
'The header message cannot be sent multiple times',
84+
);
5585
}
56-
yield* vaultManager.withVaultsG(
57-
[vaultId],
58-
async function* (vault): AsyncGenerator<SuccessOrErrorMessage> {
59-
yield* vault.writeG(
60-
async function* (efs): AsyncGenerator<SuccessOrErrorMessage> {
61-
for (const secretName of secretNames) {
62-
try {
63-
const stat = await efs.stat(secretName);
64-
if (stat.isDirectory()) {
65-
await efs.rmdir(secretName, {
66-
recursive: metadata?.options?.recursive,
67-
});
68-
} else {
69-
await efs.unlink(secretName);
70-
}
71-
yield {
72-
type: 'success',
73-
success: true,
74-
};
75-
} catch (e) {
76-
if (
77-
e.code === 'ENOENT' ||
78-
e.code === 'ENOTEMPTY' ||
79-
e.code === 'EINVAL'
80-
) {
81-
// INVAL can be triggered if removing the root of the
82-
// vault is attempted.
83-
yield {
84-
type: 'error',
85-
code: e.code,
86-
reason: secretName,
87-
};
88-
} else {
89-
throw e;
90-
}
91-
}
92-
}
93-
},
94-
);
95-
},
96-
tran,
86+
const efs = vaultMap.get(message.nameOrId);
87+
if (efs == null) {
88+
throw new vaultsErrors.ErrorVaultsVaultUndefined(
89+
`Vault ${message.nameOrId} was not present in the header message`,
90+
);
91+
}
92+
try {
93+
const stat = await efs.stat(message.secretName);
94+
if (stat.isDirectory()) {
95+
await efs.rmdir(message.secretName, {
96+
recursive: headerMessage.recursive,
97+
});
98+
} else {
99+
await efs.unlink(message.secretName);
100+
}
101+
yield {
102+
type: 'success',
103+
success: true,
104+
};
105+
} catch (e) {
106+
if (
107+
e.code === 'ENOENT' ||
108+
e.code === 'ENOTEMPTY' ||
109+
e.code === 'EINVAL'
110+
) {
111+
// EINVAL can be triggered if removing the root of the
112+
// vault is attempted.
113+
yield {
114+
type: 'error',
115+
code: e.code,
116+
reason: message.secretName,
117+
};
118+
} else {
119+
throw e;
120+
}
121+
}
122+
}
123+
// Content messages must follow header messages
124+
if (!loopRan) {
125+
throw new clientErrors.ErrorClientProtocolError(
126+
'No content messages followed header message',
97127
);
98128
}
99129
},

src/client/types.ts

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -360,6 +360,19 @@ type SecretStatMessage = {
360360
};
361361
};
362362

363+
type SecretIdentifierMessageTagged = SecretIdentifierMessage & {
364+
type: 'SecretIdentifierMessage';
365+
};
366+
367+
type VaultNamesHeaderMessage = {
368+
type: 'VaultNamesHeaderMessage';
369+
vaultNames: Array<string>;
370+
};
371+
372+
type SecretsRemoveHeaderMessage = VaultNamesHeaderMessage & {
373+
recursive?: boolean;
374+
};
375+
363376
// Type casting for tricky handlers
364377

365378
type OverrideRPClientType<T extends RPCClient<ClientManifest>> = Omit<
@@ -435,6 +448,9 @@ export type {
435448
SecretRenameMessage,
436449
SecretFilesMessage,
437450
SecretStatMessage,
451+
SecretIdentifierMessageTagged,
452+
VaultNamesHeaderMessage,
453+
SecretsRemoveHeaderMessage,
438454
SignatureMessage,
439455
OverrideRPClientType,
440456
AuditMetricGetTypeOverride,

src/git/utils.ts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -218,7 +218,9 @@ async function listObjects({
218218
}
219219
return;
220220
default:
221-
utils.never();
221+
utils.never(
222+
`type must be one of "commit", "tree", "blob", or "tag", got "${type}"`,
223+
);
222224
}
223225
}
224226

src/utils/utils.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ function getDefaultNodePath(): string | undefined {
4848
return p;
4949
}
5050

51-
function never(message?: string): never {
51+
function never(message: string): never {
5252
throw new utilsErrors.ErrorUtilsUndefinedBehaviour(message);
5353
}
5454

0 commit comments

Comments
 (0)