Skip to content

Commit ce5ec15

Browse files
satyakighsatyakigh
authored
Create databases on start, increase database size and improve logging… (#261)
1 parent 09d8b0f commit ce5ec15

File tree

14 files changed

+145
-276
lines changed

14 files changed

+145
-276
lines changed

‎src/app/standalone.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import { arch, platform } from 'os';
12
import { createConnection, ProposedFeatures } from 'vscode-languageserver/node';
23
import { InitializedParams } from 'vscode-languageserver-protocol';
34
import { LspCapabilities } from '../protocol/LspCapabilities';
@@ -33,6 +34,11 @@ async function onInitialize(params: ExtendedInitializeParams) {
3334
},
3435
`${ExtensionName} initializing...`,
3536
);
37+
getLogger().info({
38+
Machine: `${platform()}-${arch()}`,
39+
Process: `${process.platform}-${process.arch}`,
40+
Runtime: `node=${process.versions.node} v8=${process.versions.v8} uv=${process.versions.uv} modules=${process.versions.modules}`,
41+
});
3642
LoggerFactory.initialize(AwsMetadata);
3743
TelemetryService.initialize(ClientInfo, AwsMetadata);
3844

@@ -46,8 +52,8 @@ async function onInitialize(params: ExtendedInitializeParams) {
4652
}
4753

4854
function onInitialized(params: InitializedParams) {
49-
getLogger().info(`${ExtensionName} initialized`);
5055
(server as any).initialized(params);
56+
getLogger().info(`${ExtensionName} initialized`);
5157
}
5258

5359
function onShutdown() {

‎src/datastore/DataStore.ts‎

Lines changed: 22 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,13 @@ export enum Persistence {
77
local = 'local',
88
}
99

10+
export enum StoreName {
11+
public_schemas = 'public_schemas',
12+
sam_schemas = 'sam_schemas',
13+
private_schemas = 'private_schemas',
14+
combined_schemas = 'combined_schemas',
15+
}
16+
1017
export interface DataStore {
1118
get<T>(key: string): T | undefined;
1219

@@ -22,26 +29,24 @@ export interface DataStore {
2229
}
2330

2431
export interface DataStoreFactory extends Closeable {
25-
getOrCreate(store: string): DataStore;
32+
get(store: StoreName): DataStore;
2633

2734
storeNames(): ReadonlyArray<string>;
28-
29-
stats(): unknown;
3035
}
3136

3237
export interface DataStoreFactoryProvider extends Closeable {
33-
get(store: string, persistence: Persistence): DataStore;
38+
get(store: StoreName, persistence: Persistence): DataStore;
3439
}
3540

3641
export class MemoryDataStoreFactoryProvider implements DataStoreFactoryProvider {
3742
private readonly memoryStoreFactory = new MemoryStoreFactory();
3843

39-
get(store: string, _persistence: Persistence): DataStore {
44+
get(store: StoreName, _persistence: Persistence): DataStore {
4045
return this.getMemoryStore(store);
4146
}
4247

43-
getMemoryStore(store: string): DataStore {
44-
return this.memoryStoreFactory.getOrCreate(store);
48+
getMemoryStore(store: StoreName): DataStore {
49+
return this.memoryStoreFactory.get(store);
4550
}
4651

4752
close(): Promise<void> {
@@ -50,14 +55,19 @@ export class MemoryDataStoreFactoryProvider implements DataStoreFactoryProvider
5055
}
5156

5257
export class MultiDataStoreFactoryProvider implements DataStoreFactoryProvider {
53-
private readonly memoryStoreFactory = new MemoryStoreFactory();
54-
private readonly lmdbStoreFactory = new LMDBStoreFactory();
58+
private readonly memoryStoreFactory: MemoryStoreFactory;
59+
private readonly lmdbStoreFactory: LMDBStoreFactory;
60+
61+
constructor(lmdbStore?: LMDBStoreFactory, memStore?: MemoryStoreFactory) {
62+
this.lmdbStoreFactory = lmdbStore ?? new LMDBStoreFactory();
63+
this.memoryStoreFactory = memStore ?? new MemoryStoreFactory();
64+
}
5565

56-
get(store: string, persistence: Persistence): DataStore {
66+
get(store: StoreName, persistence: Persistence): DataStore {
5767
if (persistence === Persistence.memory) {
58-
return this.memoryStoreFactory.getOrCreate(store);
68+
return this.memoryStoreFactory.get(store);
5969
}
60-
return this.lmdbStoreFactory.getOrCreate(store);
70+
return this.lmdbStoreFactory.get(store);
6171
}
6272

6373
close(): Promise<void> {

‎src/datastore/LMDB.ts‎

Lines changed: 47 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import { ScopedTelemetry } from '../telemetry/ScopedTelemetry';
66
import { Telemetry } from '../telemetry/TelemetryDecorator';
77
import { TelemetryService } from '../telemetry/TelemetryService';
88
import { pathToArtifact } from '../utils/ArtifactsDir';
9-
import { DataStore, DataStoreFactory } from './DataStore';
9+
import { DataStore, DataStoreFactory, StoreName } from './DataStore';
1010
import { encryptionStrategy } from './lmdb/Utils';
1111

1212
const log = LoggerFactory.getLogger('LMDB');
@@ -15,7 +15,7 @@ export class LMDBStore implements DataStore {
1515
private readonly telemetry: ScopedTelemetry;
1616

1717
constructor(
18-
private readonly name: string,
18+
public readonly name: StoreName,
1919
private readonly store: Database<unknown, string>,
2020
) {
2121
this.telemetry = TelemetryService.instance.get(`LMDB.${name}`);
@@ -55,20 +55,31 @@ export class LMDBStoreFactory implements DataStoreFactory {
5555
private readonly timeout: NodeJS.Timeout;
5656
private readonly env: RootDatabase;
5757

58-
private readonly stores = new Map<string, LMDBStore>();
58+
private readonly stores = new Map<StoreName, LMDBStore>();
5959

60-
constructor(private readonly rootDir: string = pathToArtifact('lmdb')) {
61-
log.info(`Initializing LMDB at ${rootDir} and version ${Version}`);
60+
constructor(
61+
private readonly rootDir: string = pathToArtifact('lmdb'),
62+
storeNames: StoreName[] = [StoreName.public_schemas, StoreName.sam_schemas],
63+
) {
6264
this.storePath = join(rootDir, Version);
65+
6366
this.env = open({
6467
path: this.storePath,
65-
maxDbs: 10, // 10 max databases
66-
mapSize: 100 * 1024 * 1024, // 100MB max size
68+
maxDbs: 10,
69+
mapSize: TotalMaxDbSize,
70+
remapChunks: true,
71+
pageSize: 8192,
6772
encoding: Encoding,
6873
encryptionKey: encryptionStrategy(Version),
6974
});
7075

71-
log.info('Setup LMDB guages');
76+
for (const store of storeNames) {
77+
const database = this.env.openDB<unknown, string>({
78+
name: store,
79+
encoding: Encoding,
80+
});
81+
this.stores.set(store, new LMDBStore(store, database));
82+
}
7283
this.registerLMDBGauges();
7384

7485
this.timeout = setTimeout(
@@ -77,44 +88,22 @@ export class LMDBStoreFactory implements DataStoreFactory {
7788
},
7889
2 * 60 * 1000,
7990
);
91+
92+
log.info(`Initialized LMDB ${Version} at ${rootDir}`);
8093
}
8194

82-
getOrCreate(store: string): DataStore {
83-
let val = this.stores.get(store);
95+
get(store: StoreName): DataStore {
96+
const val = this.stores.get(store);
8497
if (val === undefined) {
85-
let database;
86-
this.env.transactionSync(() => {
87-
database = this.env.openDB<unknown, string>({
88-
name: store,
89-
encoding: Encoding,
90-
});
91-
});
92-
93-
if (database === undefined) {
94-
throw new Error(`Failed to open LMDB store ${store}`);
95-
}
96-
val = new LMDBStore(store, database);
97-
this.stores.set(store, val);
98+
throw new Error(`Store ${store} not found. Available stores: ${[...this.stores.keys()].join(', ')}`);
9899
}
99-
100100
return val;
101101
}
102102

103103
storeNames(): ReadonlyArray<string> {
104104
return [...this.stores.keys()];
105105
}
106106

107-
stats(): Record<string, StoreStatsType> {
108-
const result: Record<string, StoreStatsType> = {};
109-
result['global'] = stats(this.env);
110-
111-
for (const [key, value] of this.stores.entries()) {
112-
result[key] = value.stats();
113-
}
114-
115-
return result;
116-
}
117-
118107
async close(): Promise<void> {
119108
// Clear the stores map but don't close individual stores
120109
// LMDB will close them when we close the environment
@@ -139,14 +128,31 @@ export class LMDBStoreFactory implements DataStoreFactory {
139128
}
140129

141130
private registerLMDBGauges(): void {
131+
let totalMb = 0;
132+
const globalStat = stats(this.env);
142133
this.telemetry.registerGaugeProvider('version', () => VersionNumber);
143-
this.telemetry.registerGaugeProvider('global.size_mb', () => stats(this.env).totalSizeMB, { unit: 'MB' });
144-
this.telemetry.registerGaugeProvider('global.max_size_mb', () => stats(this.env).maxSizeMB, {
134+
this.telemetry.registerGaugeProvider('global.size', () => globalStat.totalSizeMB, { unit: 'MB' });
135+
this.telemetry.registerGaugeProvider('global.max.size', () => globalStat.maxSizeMB, {
145136
unit: 'MB',
146137
});
147-
this.telemetry.registerGaugeProvider('global.entries', () => stats(this.env).entries);
148-
this.telemetry.registerGaugeProvider('global.readers', () => stats(this.env).numReaders);
149-
this.telemetry.registerGaugeProvider('stores.count', () => this.stores.size);
138+
this.telemetry.registerGaugeProvider('global.entries', () => globalStat.entries);
139+
totalMb += globalStat.totalSizeMB;
140+
141+
for (const [name, store] of this.stores.entries()) {
142+
const stat = store.stats();
143+
totalMb += stat.totalSizeMB;
144+
145+
this.telemetry.registerGaugeProvider(`store.${name}.size`, () => stat.totalSizeMB, {
146+
unit: 'MB',
147+
});
148+
this.telemetry.registerGaugeProvider(`store.${name}.entries`, () => stat.entries, {
149+
unit: 'MB',
150+
});
151+
}
152+
153+
this.telemetry.registerGaugeProvider('global.usage', () => totalMb / TotalMaxDbSize, {
154+
unit: '%',
155+
});
150156
}
151157
}
152158

@@ -157,6 +163,7 @@ function bytesToMB(bytes: number) {
157163
const VersionNumber = 2;
158164
const Version = `v${VersionNumber}`;
159165
const Encoding: 'msgpack' | 'json' | 'string' | 'binary' | 'ordered-binary' = 'msgpack';
166+
const TotalMaxDbSize = 250 * 1024 * 1024; // 250MB max size
160167

161168
function stats(store: RootDatabase | Database): StoreStatsType {
162169
const stats = store.getStats() as Record<string, number>;

‎src/datastore/MemoryStore.ts‎

Lines changed: 3 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { ScopedTelemetry } from '../telemetry/ScopedTelemetry';
22
import { Telemetry } from '../telemetry/TelemetryDecorator';
33
import { TelemetryService } from '../telemetry/TelemetryService';
4-
import { DataStore, DataStoreFactory } from './DataStore';
4+
import { DataStore, DataStoreFactory, StoreName } from './DataStore';
55

66
export class MemoryStore implements DataStore {
77
private readonly store = new Map<string, unknown>();
@@ -47,13 +47,13 @@ export class MemoryStore implements DataStore {
4747
export class MemoryStoreFactory implements DataStoreFactory {
4848
@Telemetry({ scope: 'MemoryStore.Global' }) private readonly telemetry!: ScopedTelemetry;
4949

50-
private readonly stores = new Map<string, MemoryStore>();
50+
private readonly stores = new Map<StoreName, MemoryStore>();
5151

5252
constructor() {
5353
this.registerMemoryStoreGauges();
5454
}
5555

56-
getOrCreate(store: string): DataStore {
56+
get(store: StoreName): DataStore {
5757
let val = this.stores.get(store);
5858
if (val === undefined) {
5959
val = new MemoryStore(store);
@@ -67,21 +67,6 @@ export class MemoryStoreFactory implements DataStoreFactory {
6767
return [...this.stores.keys()];
6868
}
6969

70-
stats(): unknown {
71-
const keys = [];
72-
const stores: Record<string, number> = {};
73-
for (const [key, value] of this.stores.entries()) {
74-
keys.push(key);
75-
stores[key] = value.keys().length;
76-
}
77-
78-
return {
79-
numStores: keys.length,
80-
storeNames: keys.sort(),
81-
stores,
82-
};
83-
}
84-
8570
close(): Promise<void> {
8671
return Promise.resolve();
8772
}

‎src/schema/CombinedSchemas.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ export class CombinedSchemas {
2828
const samSchema = samSchemas === undefined ? undefined : SamSchemas.from(samSchemas);
2929

3030
CombinedSchemas.log.info(
31-
`Schemas from ${regionalSchemas?.schemas.length} public schemas, ${privateSchema?.schemas.size} private schemas and ${samSchema?.schemas.size} sam schemas`,
31+
`Combined schemas from public=${regionalSchemas?.schemas.length}, private=${privateSchema?.schemas.size}, SAM=${samSchema?.schemas.size}`,
3232
);
3333
return new CombinedSchemas(regionalSchema, privateSchema, samSchema);
3434
}

‎src/schema/GetSamSchemaTask.ts‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,18 @@
1+
import { Logger } from 'pino';
12
import { DataStore } from '../datastore/DataStore';
2-
import { LoggerFactory } from '../telemetry/LoggerFactory';
33
import { Measure } from '../telemetry/TelemetryDecorator';
44
import { downloadJson } from '../utils/RemoteDownload';
55
import { GetSchemaTask } from './GetSchemaTask';
66
import { SamSchemas, SamSchemasType, SamStoreKey } from './SamSchemas';
77
import { CloudFormationResourceSchema, SamSchema, SamSchemaTransformer } from './SamSchemaTransformer';
88

9-
const logger = LoggerFactory.getLogger('GetSamSchemaTask');
10-
119
export class GetSamSchemaTask extends GetSchemaTask {
1210
constructor(private readonly getSamSchemas: () => Promise<Map<string, CloudFormationResourceSchema>>) {
1311
super();
1412
}
1513

1614
@Measure({ name: 'getSchemas' })
17-
override async runImpl(dataStore: DataStore): Promise<void> {
15+
protected override async runImpl(dataStore: DataStore, logger?: Logger): Promise<void> {
1816
try {
1917
const resourceSchemas = await this.getSamSchemas();
2018

@@ -34,9 +32,9 @@ export class GetSamSchemaTask extends GetSchemaTask {
3432

3533
await dataStore.put(SamStoreKey, samSchemasData);
3634

37-
logger.info(`Downloaded and stored ${resourceSchemas.size} SAM resource schemas`);
35+
logger?.info(`${resourceSchemas.size} SAM schemas downloaded and stored`);
3836
} catch (error) {
39-
logger.error(error, 'Failed to download SAM schema');
37+
logger?.error(error, 'Failed to download SAM schema');
4038
throw error;
4139
}
4240
}

‎src/schema/GetSchemaTask.ts‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ export class GetPublicSchemaTask extends GetSchemaTask {
3535
}
3636

3737
@Measure({ name: 'getSchemas' })
38-
override async runImpl(dataStore: DataStore, logger?: Logger) {
38+
protected override async runImpl(dataStore: DataStore, logger?: Logger) {
3939
if (this.attempts >= GetPublicSchemaTask.MaxAttempts) {
4040
logger?.error(`Reached max attempts for retrieving schemas for ${this.region} without success`);
4141
return;
@@ -53,7 +53,7 @@ export class GetPublicSchemaTask extends GetSchemaTask {
5353
};
5454

5555
await dataStore.put<RegionalSchemasType>(this.region, value);
56-
logger?.info(`${schemas.length} resource schemas retrieved for ${this.region}`);
56+
logger?.info(`${schemas.length} public schemas retrieved for ${this.region}`);
5757
}
5858
}
5959

@@ -68,7 +68,7 @@ export class GetPrivateSchemasTask extends GetSchemaTask {
6868
}
6969

7070
@Measure({ name: 'getSchemas' })
71-
override async runImpl(dataStore: DataStore, logger?: Logger) {
71+
protected override async runImpl(dataStore: DataStore, logger?: Logger) {
7272
try {
7373
const profile = this.getProfile();
7474
if (this.processedProfiles.has(profile)) {
@@ -88,11 +88,7 @@ export class GetPrivateSchemasTask extends GetSchemaTask {
8888
await dataStore.put<PrivateSchemasType>(profile, value);
8989

9090
this.processedProfiles.add(profile);
91-
if (schemas.length > 0) {
92-
void logger?.info(`${schemas.length} private registry schemas retrieved for profile: ${profile}`);
93-
} else {
94-
logger?.info(`No private registry schemas found for profile: ${profile}`);
95-
}
91+
logger?.info(`${schemas.length} private schemas retrieved for profile: ${profile}`);
9692
} catch (error) {
9793
logger?.error(error, `Failed to get private schemas`);
9894
throw error;

0 commit comments

Comments
 (0)