Skip to content

Commit 18bd3d9

Browse files
u12206050u12206050
committed
Performance updates
1 parent f73c15c commit 18bd3d9

File tree

5 files changed

+92
-84
lines changed

5 files changed

+92
-84
lines changed

‎CHANGELOG.md‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,8 @@
1-
# Version 3.0.7
1+
# Version 3.0.8
2+
3+
- **Fixed** Slight performance improvement when importing data.
4+
5+
## Version 3.0.7
26

37
- **Fixed** SCHEMA_SYNC_DATA_ONLY being the opposite of what it should be.
48

‎src/collectionExporter.ts‎

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -82,24 +82,25 @@ class CollectionExporter implements IExporter {
8282
public export = () => this._persistQueue();
8383

8484
public async load(merge = false) {
85-
if (await ExportHelper.fileExists(this.filePath)) {
86-
const json = await readFile(this.filePath, { encoding: 'utf8' });
87-
88-
if (!json) {
89-
throw new Error(`Collection ${this.name} has invalid content: ${json}`);
90-
}
91-
const parsedJSON = JSON.parse(json) as Array<Item> | PARTIAL_CONFIG;
85+
let json;
86+
try {
87+
json = await readFile(this.filePath, { encoding: 'utf8' });
88+
} catch (e) {
89+
return null;
90+
}
9291

93-
if (Array.isArray(parsedJSON)) {
94-
return this.loadItems(parsedJSON, merge);
95-
} else if (!parsedJSON.partial) {
96-
throw new Error(`Collection ${this.name} has invalid JSON: ${json}`);
97-
}
92+
if (!json) {
93+
throw new Error(`Collection ${this.name} has invalid content: ${json}`);
94+
}
95+
const parsedJSON = JSON.parse(json) as Array<Item> | PARTIAL_CONFIG;
9896

99-
return await this.loadGroupedItems(parsedJSON, merge);
97+
if (Array.isArray(parsedJSON)) {
98+
return this.loadItems(parsedJSON, merge);
99+
} else if (!parsedJSON.partial) {
100+
throw new Error(`Collection ${this.name} has invalid JSON: ${json}`);
100101
}
101102

102-
return null;
103+
return await this.loadGroupedItems(parsedJSON, merge);
103104
}
104105

105106
protected exportCollectionToFile = async () => {

‎src/index.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
import { SchemaOverview } from '@directus/types';
2-
import { HookConfig } from '@directus/extensions';
1+
import type { HookConfig } from '@directus/extensions';
2+
import type { SchemaOverview } from '@directus/types';
33
import { condenseAction } from './condenseAction';
44
import { copyConfig } from './copyConfig';
55
import { ExportManager } from './exportManager';

‎src/schemaExporter.ts‎

Lines changed: 67 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,7 @@ export class SchemaExporter implements IExporter {
2929
} else {
3030
// Clean up old schema files
3131
const files = await glob(this.schemaFilesPath('*'));
32-
for (const file of files) {
33-
await rm(file);
34-
}
32+
await Promise.all(files.map(file => rm(file)));
3533
}
3634
};
3735

@@ -50,71 +48,74 @@ export class SchemaExporter implements IExporter {
5048
*/
5149
public load = async () => {
5250
const svc = this._getSchemaService();
53-
if (await ExportHelper.fileExists(this._filePath)) {
54-
const json = await readFile(this._filePath, { encoding: 'utf8' });
55-
if (json) {
56-
const schemaParsed = JSON.parse(json);
57-
// For older versions, the snapshot was stored under the key `snapshot`
58-
const { partial, hash, ...snapshot } = (
59-
(schemaParsed as any).snapshot
60-
? Object.assign((schemaParsed as any).snapshot, { hash: schemaParsed.hash })
61-
: schemaParsed
62-
) as Snapshot & { partial?: boolean; hash: string };
63-
64-
if (partial) {
65-
snapshot.collections = [];
66-
snapshot.fields = [];
67-
snapshot.relations = [];
68-
69-
let found = 0;
70-
const files = await glob(this.schemaFilesPath('*'));
71-
for (const file of files) {
72-
const collectionJson = await readFile(file, { encoding: 'utf8' });
73-
const { fields, relations, ...collectionInfo } = JSON.parse(collectionJson) as Collection & {
74-
fields: SnapshotField[];
75-
relations: SnapshotRelation[];
76-
};
77-
++found;
78-
79-
// Only add collection if it has a meta definition (actual table or group)
80-
if (collectionInfo.meta) {
81-
snapshot.collections.push(collectionInfo);
82-
}
83-
84-
for (const field of fields) {
85-
snapshot.fields.push(Object.assign({ collection: collectionInfo.collection }, field));
86-
}
87-
for (const relation of relations) {
88-
snapshot.relations.push(Object.assign({ collection: collectionInfo.collection }, relation));
89-
}
51+
let json;
52+
try {
53+
json = await readFile(this._filePath, { encoding: 'utf8' });
54+
} catch (e) {
55+
return;
56+
}
57+
if (json) {
58+
const schemaParsed = JSON.parse(json);
59+
// For older versions, the snapshot was stored under the key `snapshot`
60+
const { partial, hash, ...snapshot } = (
61+
(schemaParsed as any).snapshot
62+
? Object.assign((schemaParsed as any).snapshot, { hash: schemaParsed.hash })
63+
: schemaParsed
64+
) as Snapshot & { partial?: boolean; hash: string };
65+
66+
if (partial) {
67+
snapshot.collections = [];
68+
snapshot.fields = [];
69+
snapshot.relations = [];
70+
71+
let found = 0;
72+
const files = await glob(this.schemaFilesPath('*'));
73+
await Promise.all(files.map(async (file) => {
74+
const collectionJson = await readFile(file, { encoding: 'utf8' });
75+
const { fields, relations, ...collectionInfo } = JSON.parse(collectionJson) as Collection & {
76+
fields: SnapshotField[];
77+
relations: SnapshotRelation[];
78+
};
79+
++found;
80+
81+
// Only add collection if it has a meta definition (actual table or group)
82+
if (collectionInfo.meta) {
83+
snapshot.collections.push(collectionInfo);
9084
}
9185

92-
if (found === 0) {
93-
this.logger.error('No schema files found in schema directory');
94-
return;
86+
for (const field of fields) {
87+
snapshot.fields.push(Object.assign({ collection: collectionInfo.collection }, field));
9588
}
89+
for (const relation of relations) {
90+
snapshot.relations.push(Object.assign({ collection: collectionInfo.collection }, relation));
91+
}
92+
}));
9693

97-
this.logger.info(`Stitched ${found} partial schema files`);
94+
if (found === 0) {
95+
this.logger.error('No schema files found in schema directory');
96+
return;
97+
}
9898

99-
snapshot.collections.sort((a, b) => a.collection.localeCompare(b.collection));
100-
// Sort non-table collections to the start
101-
snapshot.collections.sort((a, b) => String(!!a.schema).localeCompare(String(!!b.schema)));
99+
this.logger.info(`Stitched ${found} partial schema files`);
102100

103-
// Sort fields and relations by collection
104-
snapshot.fields.sort((a, b) => a.collection.localeCompare(b.collection));
105-
snapshot.relations.sort((a, b) => a.collection.localeCompare(b.collection));
106-
}
101+
snapshot.collections.sort((a, b) => a.collection.localeCompare(b.collection));
102+
// Sort non-table collections to the start
103+
snapshot.collections.sort((a, b) => String(!!a.schema).localeCompare(String(!!b.schema)));
107104

108-
const currentSnapshot = await svc.snapshot();
109-
const currentHash = svc.getHashedSnapshot(currentSnapshot).hash;
110-
if (currentHash === hash) {
111-
this.logger.debug('Schema is already up-to-date');
112-
return;
113-
}
114-
const diff = await svc.diff(snapshot, { currentSnapshot, force: true });
115-
if (diff !== null) {
116-
await svc.apply({ diff, hash: currentHash });
117-
}
105+
// Sort fields and relations by collection
106+
snapshot.fields.sort((a, b) => a.collection.localeCompare(b.collection));
107+
snapshot.relations.sort((a, b) => a.collection.localeCompare(b.collection));
108+
}
109+
110+
const currentSnapshot = await svc.snapshot();
111+
const currentHash = svc.getHashedSnapshot(currentSnapshot).hash;
112+
if (currentHash === hash) {
113+
this.logger.debug('Schema is already up-to-date');
114+
return;
115+
}
116+
const diff = await svc.diff(snapshot, { currentSnapshot, force: true });
117+
if (diff !== null) {
118+
await svc.apply({ diff, hash: currentHash });
118119
}
119120
}
120121
};
@@ -170,10 +171,11 @@ export class SchemaExporter implements IExporter {
170171
await writeFile(this._filePath, schemaJson);
171172

172173
// Save all collections with fields as individual files
173-
for (const [collection, item] of Object.entries(map)) {
174-
const itemJson = JSON.stringify(item, null, 2);
175-
await writeFile(this.schemaFilesPath(collection), itemJson);
176-
}
174+
await Promise.all(
175+
Object.entries(map).map(([collection, item]) =>
176+
writeFile(this.schemaFilesPath(collection), JSON.stringify(item, null, 2))
177+
)
178+
);
177179
} else {
178180
const schemaJson = JSON.stringify(Object.assign({ hash }, snapshot), null, 2);
179181
await writeFile(this._filePath, schemaJson);

‎src/utils.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ export class ExportHelper {
6060
}
6161

6262
static async getExportMeta() {
63-
if (await this.fileExists(this.hashFile)) {
63+
try {
6464
const content = await readFile(this.hashFile, { encoding: 'utf8' });
6565
const [hash, ts] = content.split('@');
6666

@@ -70,8 +70,9 @@ export class ExportHelper {
7070
ts,
7171
};
7272
}
73+
} catch {
74+
// ignore
7375
}
74-
7576
return null;
7677
}
7778
}

0 commit comments

Comments
 (0)