Skip to content

chore: figure out all the types we need at runtime, bundle them as giant strings, serve them to the language server MONGOSH-2350 #555

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Jul 2, 2025
Merged

chore: figure out all the types we need at runtime, bundle them as giant strings, serve them to the language server MONGOSH-2350 #555

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
d5525d9
figure out all the types we need at runtime
lerouxb Jun 30, 2025
8bd8285
bundle the js library and node types
lerouxb Jul 1, 2025
04386cc
cleanup
lerouxb Jul 1, 2025
3ce74ee
pass in an optional fallback servicesHost from tests
lerouxb Jul 2, 2025
0e52a50
deal with the optimisation that some files aren't included a bit better
lerouxb Jul 2, 2025
3b7da4f
s/fixtures/types
lerouxb Jul 2, 2025
36ec85d
nits
lerouxb Jul 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
202 changes: 199 additions & 3 deletions packages/mongodb-ts-autocomplete/scripts/extract-types.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,19 +1,196 @@
/* eslint-disable no-console */
import * as ts from 'typescript';
import { promises as fs } from 'fs';
import path from 'path';
import { replaceImports } from '../src/utils';

async function loadSources(sources: Record<string, string>) {
const result: Record<string, string> = {};
const result: Record<string, string | true> = {};

for (const [key, filepath] of Object.entries(sources)) {
result[key] = replaceImports(await fs.readFile(filepath, 'utf8'));
for (const [key, filePath] of Object.entries(sources)) {
// for .js filepaths we're never going to read them, so just make the
// value true as an optimisation so we can still know that they should
// exist during the language server's module resolution.
try {
const data = filePath.endsWith('.js')
? true
: replaceImports(await fs.readFile(filePath, 'utf8'));
result[key] = data;
} catch (err: any) {
if (err.code !== 'ENOENT') {
console.error(`Error reading file ${filePath}:`, err);
throw err;
}
}
}

return result;
}

function resolve(moduleName: string) {
const result = ts.resolveModuleName(
moduleName,
__filename,
{},
{
fileExists: (path: string) => ts.sys.fileExists(path),
readFile: (path: string) => ts.sys.readFile(path),
},
);

return result;
}

const deps: Record<string, string[]> = {
'@mongodb-js/mongodb-ts-autocomplete': [
// the module resolution won't be addressing this module by name, but we'll
// feed it this package.json as a fallback when it tries to find itself
'package.json',
],
'@types/node': [
'package.json',
'assert.d.ts',
'assert/strict.d.ts',
'async_hooks.d.ts',
'buffer.buffer.d.ts',
'buffer.d.ts',
'child_process.d.ts',
'cluster.d.ts',
'compatibility/disposable.d.ts',
'compatibility/index.d.ts',
'compatibility/indexable.d.ts',
'compatibility/iterators.d.ts',
'console.d.ts',
'constants.d.ts',
'crypto.d.ts',
'dgram.d.ts',
'diagnostics_channel.d.ts',
'dns.d.ts',
'dns/promises.d.ts',
'dom-events.d.ts',
'domain.d.ts',
'events.d.ts',
'fs.d.ts',
'fs/promises.d.ts',
'globals.d.ts',
'globals.typedarray.d.ts',
'http.d.ts',
'http2.d.ts',
'https.d.ts',
'index.d.ts',
'inspector.d.ts',
'module.d.ts',
'net.d.ts',
'os.d.ts',
'path.d.ts',
'perf_hooks.d.ts',
'process.d.ts',
'punycode.d.ts',
'querystring.d.ts',
'readline.d.ts',
'readline/promises.d.ts',
'repl.d.ts',
'sea.d.ts',
'sqlite.d.ts',
'stream.d.ts',
'stream/consumers.d.ts',
'stream/promises.d.ts',
'stream/web.d.ts',
'string_decoder.d.ts',
'test.d.ts',
'timers.d.ts',
'timers/promises.d.ts',
'tls.d.ts',
'trace_events.d.ts',
'tty.d.ts',
'url.d.ts',
'util.d.ts',
'v8.d.ts',
'vm.d.ts',
'wasi.d.ts',
'worker_threads.d.ts',
'zlib.d.ts',
],
assert: [
'package.json',
'assert.js', // exists only
],
buffer: ['package.json', 'index.d.ts'],
events: ['package.json'],
Comment on lines +139 to +144
Copy link
Collaborator

@gribnoysup gribnoysup Jul 2, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure how important this is, probably not much, but I think these (and a couple of other packages you have here) should be resolving to node built-ins and not to the npm packaged reimplementations of those, just wondering why they are getting here

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also not sure and I wondered about that myself. Some special-case handling?

punycode: [
'package.json',
'punycode.js', // exists only
],
querystring: [
'package.json',
'index.js', // exists only
],
string_decoder: [
'package.json',
'lib/string_decoder.js', // exists only
],
typescript: [
'package.json',
'lib/es2023.ts',
'lib/lib.decorators.d.ts',
'lib/lib.decorators.legacy.d.ts',
'lib/lib.es2015.collection.d.ts',
'lib/lib.es2015.core.d.ts',
'lib/lib.es2015.d.ts',
'lib/lib.es2015.generator.d.ts',
'lib/lib.es2015.iterable.d.ts',
'lib/lib.es2015.promise.d.ts',
'lib/lib.es2015.proxy.d.ts',
'lib/lib.es2015.reflect.d.ts',
'lib/lib.es2015.symbol.d.ts',
'lib/lib.es2015.symbol.wellknown.d.ts',
'lib/lib.es2016.array.include.d.ts',
'lib/lib.es2016.d.ts',
'lib/lib.es2016.intl.d.ts',
'lib/lib.es2017.arraybuffer.d.ts',
'lib/lib.es2017.d.ts',
'lib/lib.es2017.date.d.ts',
'lib/lib.es2017.intl.d.ts',
'lib/lib.es2017.object.d.ts',
'lib/lib.es2017.sharedmemory.d.ts',
'lib/lib.es2017.string.d.ts',
'lib/lib.es2017.typedarrays.d.ts',
'lib/lib.es2018.asyncgenerator.d.ts',
'lib/lib.es2018.asynciterable.d.ts',
'lib/lib.es2018.d.ts',
'lib/lib.es2018.intl.d.ts',
'lib/lib.es2018.promise.d.ts',
'lib/lib.es2018.regexp.d.ts',
'lib/lib.es2019.array.d.ts',
'lib/lib.es2019.d.ts',
'lib/lib.es2019.intl.d.ts',
'lib/lib.es2019.object.d.ts',
'lib/lib.es2019.string.d.ts',
'lib/lib.es2019.symbol.d.ts',
'lib/lib.es2020.bigint.d.ts',
'lib/lib.es2020.d.ts',
'lib/lib.es2020.date.d.ts',
'lib/lib.es2020.intl.d.ts',
'lib/lib.es2020.number.d.ts',
'lib/lib.es2020.promise.d.ts',
'lib/lib.es2020.sharedmemory.d.ts',
'lib/lib.es2020.string.d.ts',
'lib/lib.es2020.symbol.wellknown.d.ts',
'lib/lib.es5.d.ts',
],
'undici-types': ['package.json', 'index.d.ts'],
url: [
'package.json',
'url.js', // exists only
],
util: [
'package.json',
'util.js', // exists only
],
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm sure you've had a similar thought, but is there something we can do to make this list a bit easier to maintain (i.e. auto-generate it)?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Best idea I've had so far (and I thought about this for a couple of days, trying different things) is the fallback servicesHost that gathers the ones we're missing and fails a test. That's exactly how I build this list.

I don't expect this to break at all unless we bump typescript or @types/node and even then probably rarely. Personally I think we can see how it goes and optimise that at a later stage if it becomes a problem.

Whatever we do I'd like to keep a human in the loop because automatically shipping everything typescript decides to touch during module resolution is probably at least a little bit dangerous.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But I'm definitely open to suggestions and happy to build whatever people come up with!

Copy link
Collaborator Author

@lerouxb lerouxb Jul 2, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I considered globbing and/or grepping everything, but then (especially for the js lib) we'd include way too much stuff. So we'd have to understand how it works and end up with some custom module resolution implementation (we'd at least need all the ways those files reference each other, for example) and I'm trying to avoid going down that path.

In the end what we have to do is make the language server's existing algorithm where it calls service host happy otherwise it wouldn't work. ie. we basically have to respond the same as if it resolved this from disk.

We could probably approach this from the opposite end and run typescript with noLib, then just manually make it load the files we want, but that also feels flaky to me because typescript controls the structure of the lib folder and the library files are split over many. It isn't that 2023 is all prefixed by 2023 even. And I imagine those could change over time. I mean it would work, whether it is a better solution than this I don't know.

};

async function run() {
// TODO: switch require.resolve() to resolve()
const input: Record<string, string> = {
// mql imports bson but right now so does shell-api. We could bake the types
// those use into the files we generate using api-extractor, but maybe
Expand All @@ -38,6 +215,25 @@ async function run() {
'schema.d.ts',
),
};
for (const [moduleName, filePaths] of Object.entries(deps)) {
const { resolvedModule } = resolve(moduleName);
if (!resolvedModule || !resolvedModule.packageId) {
throw new Error(`Could not resolve module: ${moduleName}`);
}

const basePath = resolvedModule.resolvedFileName.slice(
0,
-resolvedModule.packageId.subModuleName.length,
);
//console.log({ basePath});
for (const filePath of filePaths) {
const fullPath = path.join(basePath, filePath);
//console.log({ fullPath });
// these are in the format import of typescript imports
input[`${moduleName}/${filePath}`] = fullPath;
}
}

const files = await loadSources(input);
const code = `
const files = ${JSON.stringify(files)};
Expand Down
12 changes: 12 additions & 0 deletions packages/mongodb-ts-autocomplete/src/autocompleter.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,18 @@ describe('MongoDBAutocompleter', function () {
result: 'db.foo.find({ foo',
},
]);

// this is what tells us what we're missing in extract-types.ts
const encounteredPaths = autocompleter.listEncounteredPaths();
if (
encounteredPaths.getScriptSnapshot.length ||
encounteredPaths.fileExists.length ||
encounteredPaths.readFile.length
) {
// eslint-disable-next-line no-console
console.log(JSON.stringify(encounteredPaths, null, 2));
expect.fail('There should be no encountered paths left over');
}
}

// then hit a different collection to make sure the caching works
Expand Down
11 changes: 8 additions & 3 deletions packages/mongodb-ts-autocomplete/src/autocompleter.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -210,9 +210,10 @@ export type ConnectionMQLDocument = ${schemaType};

getConnectionShellAPICode(connectionId: string): string {
return `
import {ConnectionMQLQuery, ConnectionMQLPipeline, ConnectionMQLDocument} from '/${connectionId}-schema.ts';
${adjustShellApiForConnection(ShellApiText as string)}
`;
/// <reference types="node" />
import {ConnectionMQLQuery, ConnectionMQLPipeline, ConnectionMQLDocument} from '/${connectionId}-schema.ts';
${adjustShellApiForConnection(ShellApiText as string)}
`;
}

getCurrentGlobalsCode(connectionId: string, databaseName: string): string {
Expand Down Expand Up @@ -332,6 +333,10 @@ declare global {

return this.autocompleter.autocomplete(code);
}

listEncounteredPaths() {
return this.autocompleter.listEncounteredPaths();
}
}

function adjustShellApiForConnection(ShellApiText: string): string {
Expand Down
1 change: 1 addition & 0 deletions packages/mongodb-ts-autocomplete/src/utils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
export function replaceImports(code: string) {
// mql uses these and we have to make sure the language server finds our
// copies at runtime.

return code
.replace(/'bson'/g, "'/bson.ts'")
.replace(/'mongodb'/g, "'/mongodb.ts'");
Expand Down
13 changes: 10 additions & 3 deletions packages/ts-autocomplete/src/index.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
/* eslint-disable no-console */
import path from 'path';
import { promises as fs } from 'fs';
import Autocompleter from './index';
Expand Down Expand Up @@ -39,7 +40,13 @@ describe('Autocompleter', function () {
});

const completions = autoCompleter.autocomplete('doesNotExist');
expect(completions.length).to.be.gt(100);
expect(completions.length).to.equal(69); // mostly keywords
const encounteredPaths = autoCompleter.listEncounteredPaths();
expect(encounteredPaths).to.deep.equal({
getScriptSnapshot: [],
fileExists: [],
readFile: [],
});
});

it('returns completions for global variables', function () {
Expand All @@ -50,7 +57,7 @@ describe('Autocompleter', function () {
// this is just the entire global scope
const completions = autoCompleter.autocomplete('myGlobalFunct');

expect(completions.length).to.be.gt(100);
expect(completions.length).to.equal(69);

// one of them is the myGlobalFunction() function
expect(
Expand Down Expand Up @@ -140,7 +147,7 @@ describe('Autocompleter', function () {

const completions = autoCompleter.autocomplete('doesNotExist({');

expect(completions.length).to.be.gt(100);
expect(completions.length).to.equal(69);
});

it('returns matches for object parameters of a function that exists', function () {
Expand Down
Loading
Loading