Skip to content

chore: figure out all the types we need at runtime, bundle them as giant strings, serve them to the language server MONGOSH-2350 #555

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Jul 2, 2025
Merged

chore: figure out all the types we need at runtime, bundle them as giant strings, serve them to the language server MONGOSH-2350 #555

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
d5525d9
figure out all the types we need at runtime
lerouxb Jun 30, 2025
8bd8285
bundle the js library and node types
lerouxb Jul 1, 2025
04386cc
cleanup
lerouxb Jul 1, 2025
3ce74ee
pass in an optional fallback servicesHost from tests
lerouxb Jul 2, 2025
0e52a50
deal with the optimisation that some files aren't included a bit better
lerouxb Jul 2, 2025
3b7da4f
s/fixtures/types
lerouxb Jul 2, 2025
36ec85d
nits
lerouxb Jul 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions packages/mongodb-ts-autocomplete/.depcheckrc
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
ignores:
- '@mongodb-js/mql-typescript'
- '@mongodb-js/prettier-config-devtools'
- '@mongodb-js/tsconfig-devtools'
- '@types/chai'
Expand Down
248 changes: 231 additions & 17 deletions packages/mongodb-ts-autocomplete/scripts/extract-types.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,43 +1,257 @@
/* eslint-disable no-console */
import * as ts from 'typescript';
import { promises as fs } from 'fs';
import path from 'path';
import { replaceImports } from '../src/utils';

async function loadSources(sources: Record<string, string>) {
const result: Record<string, string> = {};
const result: Record<string, string | boolean> = {};

for (const [key, filepath] of Object.entries(sources)) {
result[key] = replaceImports(await fs.readFile(filepath, 'utf8'));
for (const [key, filePath] of Object.entries(sources)) {
// for .js filepaths we're never going to read them, so just make the
// value true as an optimisation so we can still know that they should
// exist during the language server's module resolution.
try {
const data = filePath.endsWith('.js')
? true
: replaceImports(await fs.readFile(filePath, 'utf8'));
result[key] = data;
} catch (err: any) {
if (err.code !== 'ENOENT') {
console.error(`Error reading file ${filePath}:`, err);
throw err;
}
}
}

return result;
}

function resolve(moduleName: string) {
if (moduleName === '@mongodb-js/mongodb-ts-autocomplete') {
// There's a chicken-and-egg problem here: we can't compile this module
// itself without the extracted types and we can't extract it before it is
// compiled because the module is not compiled, so its dist/index.js does
// not exist yet. With this workaround we're resolving the package.json
// which definitely already exists.
const result = {
resolvedModule: {
resolvedFileName: path.resolve(__dirname, '..', 'package.json'),
packageId: {
subModuleName: 'package.json',
},
},
};

return result;
}
const result = ts.resolveModuleName(
moduleName,
__filename,
{},
{
fileExists: (path: string) => ts.sys.fileExists(path),
readFile: (path: string) => ts.sys.readFile(path),
},
);

return result;
}

function resolveModulePath(moduleName: string): string {
const { resolvedModule } = resolve(moduleName);
if (!resolvedModule) {
throw new Error(`Could not resolve module: ${moduleName}`);
}
return resolvedModule.resolvedFileName;
}

const deps: Record<string, string[]> = {
'@mongodb-js/mongodb-ts-autocomplete': [
// the module resolution won't be addressing this module by name, but we'll
// feed it this package.json as a fallback when it tries to find itself
'package.json',
],
'@types/node': [
'package.json',
'assert.d.ts',
'assert/strict.d.ts',
'async_hooks.d.ts',
'buffer.buffer.d.ts',
'buffer.d.ts',
'child_process.d.ts',
'cluster.d.ts',
'compatibility/disposable.d.ts',
'compatibility/index.d.ts',
'compatibility/indexable.d.ts',
'compatibility/iterators.d.ts',
'console.d.ts',
'constants.d.ts',
'crypto.d.ts',
'dgram.d.ts',
'diagnostics_channel.d.ts',
'dns.d.ts',
'dns/promises.d.ts',
'dom-events.d.ts',
'domain.d.ts',
'events.d.ts',
'fs.d.ts',
'fs/promises.d.ts',
'globals.d.ts',
'globals.typedarray.d.ts',
'http.d.ts',
'http2.d.ts',
'https.d.ts',
'index.d.ts',
'inspector.d.ts',
'module.d.ts',
'net.d.ts',
'os.d.ts',
'path.d.ts',
'perf_hooks.d.ts',
'process.d.ts',
'punycode.d.ts',
'querystring.d.ts',
'readline.d.ts',
'readline/promises.d.ts',
'repl.d.ts',
'sea.d.ts',
'sqlite.d.ts',
'stream.d.ts',
'stream/consumers.d.ts',
'stream/promises.d.ts',
'stream/web.d.ts',
'string_decoder.d.ts',
'test.d.ts',
'timers.d.ts',
'timers/promises.d.ts',
'tls.d.ts',
'trace_events.d.ts',
'tty.d.ts',
'url.d.ts',
'util.d.ts',
'v8.d.ts',
'vm.d.ts',
'wasi.d.ts',
'worker_threads.d.ts',
'zlib.d.ts',
],
assert: [
'package.json',
'assert.js', // exists only
],
buffer: ['package.json', 'index.d.ts'],
events: ['package.json'],
Comment on lines +139 to +144
Copy link
Collaborator

@gribnoysup gribnoysup Jul 2, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure how important this is, probably not much, but I think these (and a couple of other packages you have here) should be resolving to node built-ins and not to the npm packaged reimplementations of those, just wondering why they are getting here

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also not sure and I wondered about that myself. Some special-case handling?

punycode: [
'package.json',
'punycode.js', // exists only
],
querystring: [
'package.json',
'index.js', // exists only
],
string_decoder: [
'package.json',
'lib/string_decoder.js', // exists only
],
typescript: [
'package.json',
'lib/es2023.ts',
'lib/lib.decorators.d.ts',
'lib/lib.decorators.legacy.d.ts',
'lib/lib.es2015.collection.d.ts',
'lib/lib.es2015.core.d.ts',
'lib/lib.es2015.d.ts',
'lib/lib.es2015.generator.d.ts',
'lib/lib.es2015.iterable.d.ts',
'lib/lib.es2015.promise.d.ts',
'lib/lib.es2015.proxy.d.ts',
'lib/lib.es2015.reflect.d.ts',
'lib/lib.es2015.symbol.d.ts',
'lib/lib.es2015.symbol.wellknown.d.ts',
'lib/lib.es2016.array.include.d.ts',
'lib/lib.es2016.d.ts',
'lib/lib.es2016.intl.d.ts',
'lib/lib.es2017.arraybuffer.d.ts',
'lib/lib.es2017.d.ts',
'lib/lib.es2017.date.d.ts',
'lib/lib.es2017.intl.d.ts',
'lib/lib.es2017.object.d.ts',
'lib/lib.es2017.sharedmemory.d.ts',
'lib/lib.es2017.string.d.ts',
'lib/lib.es2017.typedarrays.d.ts',
'lib/lib.es2018.asyncgenerator.d.ts',
'lib/lib.es2018.asynciterable.d.ts',
'lib/lib.es2018.d.ts',
'lib/lib.es2018.intl.d.ts',
'lib/lib.es2018.promise.d.ts',
'lib/lib.es2018.regexp.d.ts',
'lib/lib.es2019.array.d.ts',
'lib/lib.es2019.d.ts',
'lib/lib.es2019.intl.d.ts',
'lib/lib.es2019.object.d.ts',
'lib/lib.es2019.string.d.ts',
'lib/lib.es2019.symbol.d.ts',
'lib/lib.es2020.bigint.d.ts',
'lib/lib.es2020.d.ts',
'lib/lib.es2020.date.d.ts',
'lib/lib.es2020.intl.d.ts',
'lib/lib.es2020.number.d.ts',
'lib/lib.es2020.promise.d.ts',
'lib/lib.es2020.sharedmemory.d.ts',
'lib/lib.es2020.string.d.ts',
'lib/lib.es2020.symbol.wellknown.d.ts',
'lib/lib.es5.d.ts',
],
'undici-types': ['package.json', 'index.d.ts'],
url: [
'package.json',
'url.js', // exists only
],
util: [
'package.json',
'util.js', // exists only
],
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm sure you've had a similar thought, but is there something we can do to make this list a bit easier to maintain (i.e. auto-generate it)?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Best idea I've had so far (and I thought about this for a couple of days, trying different things) is the fallback servicesHost that gathers the ones we're missing and fails a test. That's exactly how I build this list.

I don't expect this to break at all unless we bump typescript or @types/node and even then probably rarely. Personally I think we can see how it goes and optimise that at a later stage if it becomes a problem.

Whatever we do I'd like to keep a human in the loop because automatically shipping everything typescript decides to touch during module resolution is probably at least a little bit dangerous.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But I'm definitely open to suggestions and happy to build whatever people come up with!

Copy link
Collaborator Author

@lerouxb lerouxb Jul 2, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I considered globbing and/or grepping everything, but then (especially for the js lib) we'd include way too much stuff. So we'd have to understand how it works and end up with some custom module resolution implementation (we'd at least need all the ways those files reference each other, for example) and I'm trying to avoid going down that path.

In the end what we have to do is make the language server's existing algorithm where it calls service host happy otherwise it wouldn't work. ie. we basically have to respond the same as if it resolved this from disk.

We could probably approach this from the opposite end and run typescript with noLib, then just manually make it load the files we want, but that also feels flaky to me because typescript controls the structure of the lib folder and the library files are split over many. It isn't that 2023 is all prefixed by 2023 even. And I imagine those could change over time. I mean it would work, whether it is a better solution than this I don't know.

};

async function run() {
const mqlPath = path.join(
path.dirname(resolveModulePath('@mongodb-js/mql-typescript')),
'..',
'out',
'schema.d.ts',
);

const input: Record<string, string> = {
// mql imports bson but right now so does shell-api. We could bake the types
// those use into the files we generate using api-extractor, but maybe
// including it just once is not so bad.
'/bson.ts': path.join(require.resolve('bson'), '..', '..', 'bson.d.ts'),
'/bson.ts': resolveModulePath('bson'),
// mql imports the mongodb driver. We could also use api-extractor there to
// bake the few mongodb types we use into the schema.
'/mongodb.ts': path.join(
require.resolve('mongodb'),
'..',
'..',
'mongodb.d.ts',
),
'/mongodb.ts': resolveModulePath('mongodb'),
// We wouldn't have to include mql if we used it straight from shell-api,
// but since we're using it straight here for now to bypass the complicated
// generics on the shell-api side it is included here for now.
'/mql.ts': path.join(
require.resolve('@mongodb-js/mql-typescript'),
'..',
'..',
'out',
'schema.d.ts',
),
'/mql.ts': mqlPath,
};
for (const [moduleName, filePaths] of Object.entries(deps)) {
const { resolvedModule } = resolve(moduleName);
if (!resolvedModule || !resolvedModule.packageId) {
throw new Error(`Could not resolve module: ${moduleName}`);
}

const basePath = resolvedModule.resolvedFileName.slice(
0,
-resolvedModule.packageId.subModuleName.length,
);
for (const filePath of filePaths) {
const fullPath = path.join(basePath, filePath);
// these are in the format import of typescript imports
input[`${moduleName}/${filePath}`] = fullPath;
}
}

const files = await loadSources(input);
const code = `
const files = ${JSON.stringify(files)};
Expand Down
16 changes: 16 additions & 0 deletions packages/mongodb-ts-autocomplete/src/autocompleter.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,12 @@ describe('MongoDBAutocompleter', function () {
let autocompleterContext: AutocompletionContext;
let autocompleter: MongoDBAutocompleter;

before(function () {
// make sure that we fall back to the default ts.sys file methods so that
// encounteredPaths will be filled
process.env.CI = 'true';
});

beforeEach(function () {
autocompleterContext = {
currentDatabaseAndConnection: () => ({
Expand Down Expand Up @@ -72,6 +78,16 @@ describe('MongoDBAutocompleter', function () {
});
});

afterEach(function () {
// this is what tells us what we're missing in extract-types.ts
const encounteredPaths = autocompleter.listEncounteredPaths();
expect(encounteredPaths).to.deep.equal({
fileExists: [],
getScriptSnapshot: [],
readFile: [],
});
});

it('deals with no connection', async function () {
// The body of tests are all wrapped in loops so that we exercise the
// caching logic in the autocompleter.
Expand Down
11 changes: 8 additions & 3 deletions packages/mongodb-ts-autocomplete/src/autocompleter.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -210,9 +210,10 @@ export type ConnectionMQLDocument = ${schemaType};

getConnectionShellAPICode(connectionId: string): string {
return `
import {ConnectionMQLQuery, ConnectionMQLPipeline, ConnectionMQLDocument} from '/${connectionId}-schema.ts';
${adjustShellApiForConnection(ShellApiText as string)}
`;
/// <reference types="node" />
import {ConnectionMQLQuery, ConnectionMQLPipeline, ConnectionMQLDocument} from '/${connectionId}-schema.ts';
${adjustShellApiForConnection(ShellApiText as string)}
`;
}

getCurrentGlobalsCode(connectionId: string, databaseName: string): string {
Expand Down Expand Up @@ -332,6 +333,10 @@ declare global {

return this.autocompleter.autocomplete(code);
}

listEncounteredPaths() {
return this.autocompleter.listEncounteredPaths();
}
}

function adjustShellApiForConnection(ShellApiText: string): string {
Expand Down
11 changes: 8 additions & 3 deletions packages/ts-autocomplete/src/index.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
/* eslint-disable no-console */
import path from 'path';
import { promises as fs } from 'fs';
import Autocompleter from './index';
Expand All @@ -20,6 +21,10 @@ describe('Autocompleter', function () {
let CODE_TS: string;

before(async function () {
// make sure that we fall back to the default ts.sys file methods so that ts
// will load the lib files from disk
process.env.CI = 'true';

CODE_TS = await fs.readFile(
path.resolve(__dirname, '..', 'test', 'fixtures', 'code.ts'),
'utf8',
Expand All @@ -39,7 +44,7 @@ describe('Autocompleter', function () {
});

const completions = autoCompleter.autocomplete('doesNotExist');
expect(completions.length).to.be.gt(100);
expect(completions.length).to.equal(69); // mostly keywords
});

it('returns completions for global variables', function () {
Expand All @@ -50,7 +55,7 @@ describe('Autocompleter', function () {
// this is just the entire global scope
const completions = autoCompleter.autocomplete('myGlobalFunct');

expect(completions.length).to.be.gt(100);
expect(completions.length).to.equal(69);

// one of them is the myGlobalFunction() function
expect(
Expand Down Expand Up @@ -140,7 +145,7 @@ describe('Autocompleter', function () {

const completions = autoCompleter.autocomplete('doesNotExist({');

expect(completions.length).to.be.gt(100);
expect(completions.length).to.equal(69);
});

it('returns matches for object parameters of a function that exists', function () {
Expand Down
Loading