custom languages

Author: mProjectsCode

exampleVault/customLanguages/odin.json

@@ -104,3 +104,37 @@ Inline code
 
 `{jsx} <button role="button" />`
 
+```Odin
+package main
+
+import "core:fmt"
+
+main :: proc() {
+	program := "+ + * 😃 - /"
+	accumulator := 0
+
+	for token in program {
+		switch token {
+		case '+': accumulator += 1
+		case '-': accumulator -= 1
+		case '*': accumulator *= 2
+		case '/': accumulator /= 2
+		case '😃': accumulator *= accumulator
+		case: // Ignore everything else
+		}
+	}
+
+	fmt.printf("The program \"%s\" calculates the value %d\n",
+	           program, accumulator)
+}
+```
+
+
+```cpp
+#include <iostream>
+
+int main() {
+    std::cout << "Hello World!";
+    return 0;
+}
+```

exampleVault/index.md

@@ -5,17 +5,15 @@ export class CodeBlock extends MarkdownRenderChild {
 	plugin: ShikiPlugin;
 	source: string;
 	language: string;
-	languageName: string;
 	ctx: MarkdownPostProcessorContext;
 	cachedMetaString: string;
 
-	constructor(plugin: ShikiPlugin, containerEl: HTMLElement, source: string, language: string, languageName: string, ctx: MarkdownPostProcessorContext) {
+	constructor(plugin: ShikiPlugin, containerEl: HTMLElement, source: string, language: string, ctx: MarkdownPostProcessorContext) {
 		super(containerEl);
 
 		this.plugin = plugin;
 		this.source = source;
 		this.language = language;
-		this.languageName = languageName;
 		this.ctx = ctx;
 		this.cachedMetaString = '';
 	}
@@ -31,7 +29,7 @@ export class CodeBlock extends MarkdownRenderChild {
 		const startLine = lines[sectionInfo.lineStart];
 
 		// regexp to match the text after the code block language
-		const regex = new RegExp('^[^`~]*?(```+|~~~+)' + this.languageName + ' (.*)', 'g');
+		const regex = new RegExp('^[^`~]*?(```+|~~~+)' + this.language + ' (.*)', 'g');
 		const match = regex.exec(startLine);
 		if (match !== null) {
 			return match[2];

src/CodeBlock.ts

@@ -1,7 +1,6 @@
 import { ExpressiveCodeEngine, ExpressiveCodeTheme } from '@expressive-code/core';
 import type ShikiPlugin from 'src/main';
-import { LoadedLanguage } from 'src/LoadedLanguage';
-import { bundledLanguages, createHighlighter, type Highlighter } from 'shiki/index.mjs';
+import { bundledLanguages, createHighlighter, type DynamicImportLanguageRegistration, type LanguageRegistration, type Highlighter } from 'shiki/index.mjs';
 import { ThemeMapper } from 'src/themes/ThemeMapper';
 import { pluginShiki } from '@expressive-code/plugin-shiki';
 import { pluginCollapsibleSections } from '@expressive-code/plugin-collapsible-sections';
@@ -30,9 +29,10 @@ export class CodeHighlighter {
 
 	ec!: ExpressiveCodeEngine;
 	ecElements!: HTMLElement[];
-	loadedLanguages!: Map<string, LoadedLanguage>;
+	loadedLanguages!: string[];
 	shiki!: Highlighter;
 	customThemes!: CustomTheme[];
+	customLanguages!: LanguageRegistration[];
 
 	constructor(plugin: ShikiPlugin) {
 		this.plugin = plugin;
@@ -41,62 +41,46 @@ export class CodeHighlighter {
 
 	async load(): Promise<void> {
 		await this.loadCustomThemes();
-
-		await this.loadLanguages();
+		await this.loadCustomLanguages();
 
 		await this.loadEC();
 		await this.loadShiki();
+
+		this.loadedLanguages = this.shiki.getLoadedLanguages();
 	}
 
 	async unload(): Promise<void> {
 		this.unloadEC();
 	}
 
-	async loadLanguages(): Promise<void> {
-		this.loadedLanguages = new Map();
+	async loadCustomLanguages(): Promise<void> {
+		this.customLanguages = [];
 
-		for (const [shikiLanguage, registration] of Object.entries(bundledLanguages)) {
-			// the last element of the array is seemingly the most recent version of the language
-			const language = (await registration()).default.at(-1);
-			const shikiLanguageName = shikiLanguage as keyof typeof bundledLanguages;
+		if (!this.plugin.loadedSettings.customLanguageFolder) return;
 
-			if (language === undefined) {
-				continue;
-			}
-
-			for (const alias of [language.name, ...(language.aliases ?? [])]) {
-				if (languageNameBlacklist.has(alias)) {
-					continue;
-				}
+		const languageFolder = normalizePath(this.plugin.loadedSettings.customLanguageFolder);
+		if (!(await this.plugin.app.vault.adapter.exists(languageFolder))) {
+			new Notice(`${this.plugin.manifest.name}\nUnable to open custom languages folder: ${languageFolder}`, 5000);
+			return;
+		}
 
-				if (!this.loadedLanguages.has(alias)) {
-					const newLanguage = new LoadedLanguage(alias);
-					newLanguage.addLanguage(shikiLanguageName);
+		const languageList = await this.plugin.app.vault.adapter.list(languageFolder);
+		const languageFiles = languageList.files.filter(f => f.toLowerCase().endsWith('.json'));
 
-					this.loadedLanguages.set(alias, newLanguage);
+		for (const languageFile of languageFiles) {
+			try {
+				const language = JSON.parse(await this.plugin.app.vault.adapter.read(languageFile)) as LanguageRegistration;
+				// validate that language file JSON can be parsed and contains at a minimum a scopeName
+				if (!language.name) {
+					throw Error('Invalid JSON language file is missing a name property.');
 				}
 
-				this.loadedLanguages.get(alias)!.addLanguage(shikiLanguageName);
-			}
-		}
-
-		for (const [alias, language] of this.loadedLanguages) {
-			if (language.languages.length === 1) {
-				language.setDefaultLanguage(language.languages[0]);
-			} else {
-				const defaultLanguage = language.languages.find(lang => lang === alias);
-				if (defaultLanguage !== undefined) {
-					language.setDefaultLanguage(defaultLanguage);
-				} else {
-					console.warn(`No default language found for ${alias}, using the first language in the list`);
-					language.setDefaultLanguage(language.languages[0]);
-				}
+				this.customLanguages.push(language);
+			} catch (e) {
+				new Notice(`${this.plugin.manifest.name}\nUnable to load custom language: ${languageFile}`, 5000);
+				console.warn(`Unable to load custom language: ${languageFile}`, e);
 			}
 		}
-
-		for (const disabledLanguage of this.plugin.loadedSettings.disabledLanguages) {
-			this.loadedLanguages.delete(disabledLanguage);
-		}
 	}
 
 	async loadCustomThemes(): Promise<void> {
@@ -150,7 +134,7 @@ export class CodeHighlighter {
 			themes: [new ExpressiveCodeTheme(await this.themeMapper.getThemeForEC())],
 			plugins: [
 				pluginShiki({
-					langs: Object.values(bundledLanguages),
+					langs: this.getLoadedLanguageRegistrations(),
 				}),
 				pluginCollapsibleSections(),
 				pluginTextMarkers(),
@@ -186,7 +170,7 @@ export class CodeHighlighter {
 	async loadShiki(): Promise<void> {
 		this.shiki = await createHighlighter({
 			themes: [await this.themeMapper.getTheme()],
-			langs: Object.keys(bundledLanguages),
+			langs: this.getLoadedLanguageRegistrations(),
 		});
 	}
 
@@ -206,4 +190,12 @@ export class CodeHighlighter {
 
 		container.innerHTML = toHtml(this.themeMapper.fixAST(result.renderedGroupAst));
 	}
+
+	getLoadedLanguageRegistrations(): (DynamicImportLanguageRegistration | LanguageRegistration)[] {
+		return [...Object.values(bundledLanguages), ...this.customLanguages];
+	}
+
+	obsidianSafeLanguageNames(): string[] {
+		return this.loadedLanguages.filter(lang => !languageNameBlacklist.has(lang));
+	}
 }

src/Highlighter.ts

@@ -1,5 +1,5 @@
 import { loadPrism, Plugin, TFile, type MarkdownPostProcessor } from 'obsidian';
-import { type ThemedToken, type TokensResult } from 'shiki';
+import { type BundledLanguage, type ThemedToken, type TokensResult } from 'shiki';
 import { CodeBlock } from 'src/CodeBlock';
 import { createCm6Plugin } from 'src/codemirror/Cm6_ViewPlugin';
 import { DEFAULT_SETTINGS, type Settings } from 'src/settings/Settings';
@@ -80,27 +80,27 @@ export default class ShikiPlugin extends Plugin {
 	}
 
 	registerCodeBlockProcessors(): void {
-		for (const [alias, language] of this.highlighter.loadedLanguages) {
+		for (const language of this.highlighter.obsidianSafeLanguageNames()) {
 			try {
 				this.registerMarkdownCodeBlockProcessor(
-					alias,
+					language,
 					async (source, el, ctx) => {
 						// this is so that we leave the hidden frontmatter code block in reading mode alone
-						if (alias === 'yaml' && ctx.frontmatter) {
+						if (language === 'yaml' && ctx.frontmatter) {
 							const sectionInfo = ctx.getSectionInfo(el);
 							if (sectionInfo && sectionInfo.lineStart === 0) {
 								el.addClass('shiki-hide-in-reading-mode');
 							}
 						}
 
-						const codeBlock = new CodeBlock(this, el, source, language.getDefaultLanguage(), alias, ctx);
+						const codeBlock = new CodeBlock(this, el, source, language, ctx);
 
 						ctx.addChild(codeBlock);
 					},
 					-1,
 				);
 			} catch (e) {
-				console.warn(`Failed to register code block processor for ${alias}`, e);
+				console.warn(`Failed to register code block processor for ${language}`, e);
 			}
 		}
 	}
@@ -154,14 +154,12 @@ export default class ShikiPlugin extends Plugin {
 	}
 
 	getHighlightTokens(code: string, lang: string): TokensResult | undefined {
-		const shikiLanguage = this.highlighter.loadedLanguages.get(lang);
-
-		if (shikiLanguage === undefined) {
+		if (!this.highlighter.obsidianSafeLanguageNames().includes(lang)) {
 			return undefined;
 		}
 
 		return this.highlighter.shiki.codeToTokens(code, {
-			lang: shikiLanguage.getDefaultLanguage(),
+			lang: lang as BundledLanguage,
 			theme: this.settings.theme,
 		});
 	}

src/LoadedLanguage.ts

@@ -1,6 +1,7 @@
 export interface Settings {
 	disabledLanguages: string[];
 	customThemeFolder: string;
+	customLanguageFolder: string;
 	theme: string;
 	preferThemeColors: boolean;
 	inlineHighlighting: boolean;
@@ -9,6 +10,7 @@ export interface Settings {
 export const DEFAULT_SETTINGS: Settings = {
 	disabledLanguages: [],
 	customThemeFolder: '',
+	customLanguageFolder: '',
 	theme: 'obsidian-theme',
 	preferThemeColors: true,
 	inlineHighlighting: true,