feat: use textmate service override

remove monarch fallback

Author: CGNonofr

src/languages/index.ts

@@ -1,10 +1,12 @@
 import * as monaco from 'monaco-editor'
-import { getOrCreateTextMateTokensProvider } from './textMate'
+import { setLanguages } from 'vscode/service-override/languages'
+import { setLanguageConfiguration } from 'vscode/service-override/languageConfiguration'
+import { StandaloneServices, ILanguageService } from 'vscode/services'
+import './textMate'
 import textMateLanguages from './extensions/languages.json'
-import { languageLoader as monarchLanguageLoader } from './monarch'
-import languageConfigurationLoader, { RawLanguageConfiguration } from './extensions/languageConfigurationLoader'
+import languageConfigurationLoader from './extensions/languageConfigurationLoader'
 import './snippets'
-import { addCustomFoldingMarkers } from '../hacks'
+import { addCustomFoldingMarkers, ILanguageConfiguration } from '../hacks'
 
 const customAliases: Partial<Record<string, string[]>> = {
   csharp: ['c#'],
@@ -19,95 +21,27 @@ const customAliases: Partial<Record<string, string[]>> = {
   postgres: ['postgresql', 'postgres', 'pg', 'postgre']
 }
 
+setLanguages(textMateLanguages.map(language => ({
+  ...language,
+  configuration: languageConfigurationLoader[language.id] != null ? `./${language.id}-configuration.json` : undefined
+})))
+
 for (const [languageId, aliases] of Object.entries(customAliases)) {
   monaco.languages.register({
     id: languageId,
     aliases
   })
 }
 
-const languagesIds = Array.from(new Set([
-  ...Object.keys(monarchLanguageLoader),
-  ...textMateLanguages.map(rawLanguage => rawLanguage.id)
-]))
-
 for (const textMateLanguage of textMateLanguages) {
-  monaco.languages.register(textMateLanguage)
-}
-
-for (const languageId of languagesIds) {
-  monaco.languages.setTokenizationSupportFactory(languageId, {
-    createTokenizationSupport: async () => {
-      return getOrCreateTextMateTokensProvider(languageId).catch(async (error: Error) => {
-        const monarchLoader = monarchLanguageLoader[languageId]
-        if (monarchLoader != null) {
-          monaco.errorHandler.onUnexpectedError(new Error(`Failed to load TextMate grammar for language ${languageId}, fallback to monarch`, {
-            cause: error
-          }))
-          try {
-            monaco.languages.setMonarchTokensProvider(languageId, (await monarchLoader()).language)
-          } catch (error) {
-            monaco.errorHandler.onUnexpectedError(new Error(`Failed to load Monarch grammar for language ${languageId}`, {
-              cause: error as Error
-            }))
-          }
-        } else {
-          monaco.errorHandler.onUnexpectedError(new Error(`Failed to load TextMate grammar for language ${languageId} and no fallback monarch`, {
-            cause: error
-          }))
-        }
-        return null
-      })
-    }
-  })
-}
-
-/**
- * Type is wrong
- * see https://github.com/microsoft/vscode/blob/cfad2543487c4a8e8f53b4451dbccdc1c2036f41/src/vs/workbench/contrib/codeEditor/browser/languageConfigurationExtensionPoint.ts#L381
- */
-function parseLanguageConfiguration (config: RawLanguageConfiguration): monaco.extra.ILanguageConfiguration {
-  const markers = config.folding?.markers
-  return {
-    ...config,
-    folding: config.folding != null
-      ? {
-          ...config.folding,
-          markers: (markers != null) ? { start: new RegExp(markers.start), end: new RegExp(markers.end) } : undefined
-        }
-      : undefined
+  const configurationLoader = languageConfigurationLoader[textMateLanguage.id]
+  if (configurationLoader != null) {
+    setLanguageConfiguration(`/${textMateLanguage.id}-configuration.json`, async () => JSON.stringify(addCustomFoldingMarkers((await configurationLoader()) as ILanguageConfiguration)))
   }
 }
 
-async function loadLanguageConfiguration (languageId: string) {
-  const loader = languageConfigurationLoader[languageId]
-  if (loader != null) {
-    const configuration = await loader()
-    monaco.extra.handleLanguageConfiguration(
-      languageId,
-      addCustomFoldingMarkers(parseLanguageConfiguration(configuration))
-    )
-  }
-}
-
-setTimeout(() => {
-  // In a timeout so the service can be overriden
-  const languageService = monaco.extra.StandaloneServices.get(monaco.languages.ILanguageService)
-  languageService.onDidEncounterLanguage(async (languageId) => {
-    if (languageId === 'plaintext') {
-      return
-    }
-
-    loadLanguageConfiguration(languageId).catch(error => {
-      monaco.errorHandler.onUnexpectedError(new Error(`Unable to load language configuration for language ${languageId}`, {
-        cause: error
-      }))
-    })
-  })
-})
-
 function getMonacoLanguage (languageOrModeId: string): string {
-  const languageService = monaco.extra.StandaloneServices.get(monaco.languages.ILanguageService)
+  const languageService = StandaloneServices.get(ILanguageService)
   const modeId = languageService.getLanguageIdByLanguageName(languageOrModeId.toLowerCase())
   if (modeId != null) {
     return modeId
@@ -123,10 +57,7 @@ function getMonacoLanguage (languageOrModeId: string): string {
 }
 
 async function loadLanguage (languageId: string): Promise<void> {
-  await Promise.all([
-    loadLanguageConfiguration(languageId),
-    getOrCreateTextMateTokensProvider(languageId)
-  ])
+  StandaloneServices.get(ILanguageService).createById(languageId)
 }
 
 export {

src/languages/monarch/index.ts

@@ -1,63 +1,9 @@
-import * as monaco from 'monaco-editor'
-import onigFile from 'vscode-oniguruma/release/onig.wasm'
-import CGTMGrammarFactory from './CGTMGrammarFactory'
-import CGTMTokenizationSupport from './CGTMTokenizationSupport'
+import { setGrammars, ITMSyntaxExtensionPoint } from 'vscode/service-override/textmate'
 import rawGrammars from '../extensions/grammars.json'
 import grammarLoader from '../extensions/grammarLoader'
 import './semanticTokens'
 
-function createGrammarFactory (): CGTMGrammarFactory {
-  const languageService = monaco.extra.StandaloneServices.get(monaco.languages.ILanguageService)
-  const parsedGrammars = (rawGrammars as unknown as Omit<monaco.extra.ITMSyntaxExtensionPoint, 'path'>[])
-    .map(grammar => ({
-      ...monaco.extra.parseTextMateGrammar(grammar as monaco.extra.ITMSyntaxExtensionPoint, languageService),
-      location: monaco.Uri.from({
-        scheme: 'browser',
-        path: grammar.scopeName + '.json'
-      })
-    }))
-
-  return new CGTMGrammarFactory(
-    parsedGrammars,
-    grammarLoader,
-    onigFile
-  )
-}
-
-let textMateGrammarFactory: CGTMGrammarFactory | null = null
-function getOrCreateGrammarFactory (): CGTMGrammarFactory {
-  if (textMateGrammarFactory == null) {
-    textMateGrammarFactory = createGrammarFactory()
-  }
-  return textMateGrammarFactory
-}
-
-async function createTextMateTokensProvider (languageId: string): Promise<monaco.languages.ITokenizationSupport | null> {
-  const languageService = monaco.extra.StandaloneServices.get(monaco.languages.ILanguageService)
-  const grammarFactory = getOrCreateGrammarFactory()
-  const encodedLanguageId = languageService.languageIdCodec.encodeLanguageId(languageId)
-  if (!grammarFactory.has(languageId)) {
-    return null
-  }
-  const { grammar, initialState, containsEmbeddedLanguages } = await grammarFactory.createGrammar(languageId, encodedLanguageId)
-  if (grammar == null) {
-    return null
-  }
-  const tokenization = new monaco.extra.TMTokenization(grammar, initialState, containsEmbeddedLanguages)
-  tokenization.onDidEncounterLanguage((encodedLanguageId) => {
-    // Force monaco to load this language and trigger the global `onDidEncounterLanguage`
-    languageService.createById(languageService.languageIdCodec.decodeLanguageId(encodedLanguageId))
-  })
-  return new CGTMTokenizationSupport(languageId, encodedLanguageId, tokenization, grammar)
-}
-
-const tokenizationSupports = new Map<string, Promise<monaco.languages.ITokenizationSupport | null>>()
-
-export function getOrCreateTextMateTokensProvider (languageId: string): Promise<monaco.languages.ITokenizationSupport | null> {
-  let tokenizationSupportPromise = tokenizationSupports.get(languageId)
-  if (tokenizationSupportPromise == null) {
-    tokenizationSupportPromise = createTextMateTokensProvider(languageId)
-    tokenizationSupports.set(languageId, tokenizationSupportPromise)
-  }
-  return tokenizationSupportPromise
-}
+setGrammars((rawGrammars as unknown as Omit<ITMSyntaxExtensionPoint, 'path'>[]).map(grammar => ({
+  ...grammar,
+  path: grammar.scopeName + '.json'
+})), async (grammar) => JSON.stringify(await grammarLoader[grammar.scopeName]!()))