Merge pull request #32 from andrewbranch/extension-dir

Add option to customize download location of extensions

Author: andrewbranch

README.md

@@ -15,6 +15,7 @@ Includes OS dark mode support 🌙
   - [Languages](#languages)
   - [Themes](#themes)
 - [Using languages and themes from an extension](#using-languages-and-themes-from-an-extension)
+  - [Dealing with rate limiting in CI](#dealing-with-rate-limiting-in-ci)
 - [Styles](#styles)
   - [Class names](#class-names)
   - [Variables](#variables)
@@ -56,17 +57,19 @@ Add to your `gatsby-config.js` (all options are optional; defaults shown here):
         // All options are optional. Defaults shown here.
         options: {
           colorTheme: 'Dark+ (default dark)', // Read on for list of included themes. Also accepts object and function forms.
-          wrapperClassName: '',  // Additional class put on 'pre' tag
-          injectStyles: true,    // Injects (minimal) additional CSS for layout and scrolling
-          extensions: [],        // Extensions to download from the marketplace to provide more languages and themes
-          languageAliases: {},   // Map of custom/unknown language codes to standard/known language codes
-          replaceColor: x => x,  // Function allowing replacement of a theme color with another. Useful for replacing hex colors with CSS variables.
-          getLineClassName: ({   // Function allowing dynamic setting of additional class names on individual lines
-            content,             //   - the string content of the line
-            index,               //   - the zero-based index of the line within the code fence
-            language,            //   - the language specified for the code fence
-            codeFenceOptions     //   - any options set on the code fence alongside the language (more on this later)
-          }) => ''
+          wrapperClassName: '',   // Additional class put on 'pre' tag
+          injectStyles: true,     // Injects (minimal) additional CSS for layout and scrolling
+          extensions: [],         // Extensions to download from the marketplace to provide more languages and themes
+          languageAliases: {},    // Map of custom/unknown language codes to standard/known language codes
+          replaceColor: x => x,   // Function allowing replacement of a theme color with another. Useful for replacing hex colors with CSS variables.
+          getLineClassName: ({    // Function allowing dynamic setting of additional class names on individual lines
+            content,              //   - the string content of the line
+            index,                //   - the zero-based index of the line within the code fence
+            language,             //   - the language specified for the code fence
+            codeFenceOptions      //   - any options set on the code fence alongside the language (more on this later)
+          }) => '',
+          extensionDataDirectory: // Absolute path to the directory where extensions will be downloaded. Defaults to inside node_modules.
+            path.resolve('extensions'),
         }
       }]
     }
@@ -211,7 +214,13 @@ Add those strings to the `extensions` option in your plugin configuration in `ga
   }]
 ```
 
-Next time you `gatsby develop` or `gatsby build`, the extension will be downloaded and Scala code fences will be highlighted. Extensions are downloaded to `node_modules/gatsby-remark-vscode/lib/extensions`, so they remain cached on disk as long as `gastsby-remark-vscode` does.
+Next time you `gatsby develop` or `gatsby build`, the extension will be downloaded and Scala code fences will be highlighted. Extensions are downloaded to `node_modules/gatsby-remark-vscode/lib/extensions` by default (but can go elsewhere by setting the `extensionDataDirectory` option), so they remain cached on disk as long as `gastsby-remark-vscode` does.
+
+### Dealing with rate limiting in CI
+
+Anonymous requests to the Visual Studio Marketplace are rate limited, so if you’re downloading a lot of extensions or running builds in quick succession in an environment where the extensions aren’t already cached on disk (like on a build server), you might see failed requests.
+
+As a workaround, you can set the `extensionDataDirectory` plugin option to an absolute path pointing to a folder that you check into source control. After running a build locally, any extensions you’ve requested will appear in that directory. Then, in CI, gatsby-remark-vscode will check that directory and determine if anything needs to be downloaded. By checking including the extensions alongside your own source code, you can avoid making requests to the Visual Studio Marketplace in CI entirely.
 
 ## Styles
 

scripts/scrapeBuiltinExtensions.js

@@ -17,7 +17,7 @@ const grammarPath = /** @param {string} basename */ basename => path.join(gramma
 const themePath = /** @param {string} basename */ basename => path.join(themeDestDir, basename);
 let languageId = 0;
 
-glob(path.resolve(__dirname, '../vscode/extensions/**/package.json'), async (err, packages) => {
+glob(path.resolve(__dirname, '../vscode/extensions/*/package.json'), async (err, packages) => {
   try {
     if (err) throw err;
     await tryMkdir(grammarDestDir);

src/downloadExtension.js

@@ -35,9 +35,10 @@ async function mergeCache(cache, key, value) {
 /**
  * @param {import('.').ExtensionDemand} extensionDemand
  * @param {*} cache
+ * @param {string} extensionDir
  */
-async function syncExtensionData({ identifier }, cache) {
-  const packageJsonPath = path.join(getExtensionPath(identifier), 'package.json');
+async function syncExtensionData({ identifier }, cache, extensionDir) {
+  const packageJsonPath = path.join(getExtensionPath(identifier, extensionDir), 'package.json');
   const { grammars, themes } = await processExtension(packageJsonPath);
   Object.keys(grammars).forEach(scopeName => (grammars[scopeName].languageId = languageId++));
   await mergeCache(cache, 'grammars', grammars);
@@ -47,8 +48,9 @@ async function syncExtensionData({ identifier }, cache) {
 /**
  * @param {import('.').ExtensionDemand} extensionDemand
  * @param {*} cache
+ * @param {string} extensionDir
  */
-async function downloadExtension(extensionDemand, cache) {
+async function downloadExtension(extensionDemand, cache, extensionDir) {
   const { identifier, version } = extensionDemand;
   const { publisher, name } = parseExtensionIdentifier(identifier);
   const url = `https://marketplace.visualstudio.com/_apis/public/gallery/publishers/${publisher}/vsextensions/${name}/${version}/vspackage`;
@@ -73,37 +75,43 @@ async function downloadExtension(extensionDemand, cache) {
     });
   });
 
-  const extensionPath = getExtensionBasePath(identifier);
+  const extensionPath = getExtensionBasePath(identifier, extensionDir);
   await decompress(archive, extensionPath);
-  await syncExtensionData(extensionDemand, cache);
+  await syncExtensionData(extensionDemand, cache, extensionDir);
   return extensionPath;
 }
 
 /**
- * @param {'grammar' | 'theme'} type
- * @param {string} name
- * @param {import('.').ExtensionDemand[]} extensions
- * @param {*} cache
- * @param {Record<string, string>} languageAliases
+ * @typedef {object} DownloadExtensionOptions
+ * @property {'grammar' | 'theme'} type
+ * @property {string} name
+ * @property {import('.').ExtensionDemand[]} extensions
+ * @property {*} cache
+ * @property {Record<string, string>} languageAliases
+ * @property {string} extensionDir
+ */
+
+/**
+ * @param {DownloadExtensionOptions} options
  */
-async function downloadExtensionIfNeeded(type, name, extensions, cache, languageAliases) {
+async function downloadExtensionIfNeeded({ type, name, extensions, cache, languageAliases, extensionDir }) {
   extensions = extensions.slice();
   const extensionExists = type === 'grammar' ? grammarExists : themeExists;
   while (extensions.length && !(await extensionExists(name))) {
     const extensionDemand = extensions.shift();
     const { identifier, version } = extensionDemand;
-    const extensionPath = getExtensionBasePath(identifier);
+    const extensionPath = getExtensionBasePath(identifier, extensionDir);
     if (!fs.existsSync(extensionPath)) {
-      await downloadExtension(extensionDemand, cache);
+      await downloadExtension(extensionDemand, cache, extensionDir);
       continue;
     }
-    const packageJson = getExtensionPackageJson(identifier);
+    const packageJson = getExtensionPackageJson(identifier, extensionDir);
     if (packageJson.version !== version) {
-      await downloadExtension(extensionDemand, cache);
+      await downloadExtension(extensionDemand, cache, extensionDir);
       continue;
     }
 
-    await syncExtensionData(extensionDemand, cache);
+    await syncExtensionData(extensionDemand, cache, extensionDir);
   }
 
   /** @param {string} languageName */

src/index.js

@@ -119,6 +119,7 @@ function getStylesFromSettings(settings) {
  * @property {(line: LineData) => string=} getLineClassName
  * @property {boolean=} injectStyles
  * @property {(colorValue: string, theme: string) => string=} replaceColor
+ * @property {string=} extensionDataDirectory
  */
 
 function createPlugin() {
@@ -138,7 +139,8 @@ function createPlugin() {
       extensions = [],
       getLineClassName = () => '',
       injectStyles = true,
-      replaceColor = x => x
+      replaceColor = x => x,
+      extensionDataDirectory = path.resolve(__dirname, '../lib/extensions')
     } = {}
   ) {
     /** @type {Record<string, string>} */
@@ -153,7 +155,14 @@ function createPlugin() {
       const text = node.value || (node.children && node.children[0] && node.children[0].value);
       if (!text) continue;
       const { languageName, options } = parseCodeFenceHeader(node.lang ? node.lang.toLowerCase() : '');
-      await downloadExtensionIfNeeded('grammar', languageName, extensions, cache, languageAliases);
+      await downloadExtensionIfNeeded({
+        type: 'grammar',
+        name: languageName,
+        extensions,
+        cache,
+        languageAliases,
+        extensionDir: extensionDataDirectory
+      });
 
       const grammarCache = await cache.get('grammars');
       const scope = getScope(languageName, grammarCache, languageAliases);
@@ -166,68 +175,75 @@ function createPlugin() {
         warnMissingLanguageFile(missingScopeName, scope);
       });
 
-      const colorThemeValue =
-        typeof colorTheme === 'function'
-          ? colorTheme({
-              markdownNode,
-              codeFenceNode: node,
-              parsedOptions: options,
-              language: languageName
-            })
-          : colorTheme;
-      const colorThemeSettings = createColorThemeSettings(colorThemeValue);
-      const themeClassNames = createThemeClassNames(colorThemeSettings);
-      for (const setting in colorThemeSettings) {
-        const colorThemeIdentifier = colorThemeSettings[setting];
-        if (!colorThemeIdentifier) continue;
-        await downloadExtensionIfNeeded('theme', colorThemeIdentifier, extensions, cache, languageAliases);
+      try {
+        const colorThemeValue =
+          typeof colorTheme === 'function'
+            ? colorTheme({
+                markdownNode,
+                codeFenceNode: node,
+                parsedOptions: options,
+                language: languageName
+              })
+            : colorTheme;
+        const colorThemeSettings = createColorThemeSettings(colorThemeValue);
+        const themeClassNames = createThemeClassNames(colorThemeSettings);
+        for (const setting in colorThemeSettings) {
+          const colorThemeIdentifier = colorThemeSettings[setting];
+          if (!colorThemeIdentifier) continue;
+          await downloadExtensionIfNeeded({
+            type: 'theme',
+            name: colorThemeIdentifier,
+            extensions,
+            cache,
+            languageAliases,
+            extensionDir: extensionDataDirectory
+          });
 
-        const themeClassName = themeClassNames[setting];
-        const themeCache = await cache.get('themes');
-        const colorThemePath =
-          getThemeLocation(colorThemeIdentifier, themeCache) ||
-          path.resolve(markdownNode.fileAbsolutePath, colorThemeIdentifier);
+          const themeClassName = themeClassNames[setting];
+          const themeCache = await cache.get('themes');
+          const colorThemePath =
+            getThemeLocation(colorThemeIdentifier, themeCache) ||
+            path.resolve(markdownNode.fileAbsolutePath, colorThemeIdentifier);
 
-        const { resultRules: tokenColors, resultColors: settings } = loadColorTheme(colorThemePath);
-        const defaultTokenColors = {
-          settings: {
-            foreground: settings['editor.foreground'] || settings.foreground,
-            background: settings['editor.background'] || settings.background
-          }
-        };
+          const { resultRules: tokenColors, resultColors: settings } = loadColorTheme(colorThemePath);
+          const defaultTokenColors = {
+            settings: {
+              foreground: settings['editor.foreground'] || settings.foreground,
+              background: settings['editor.background'] || settings.background
+            }
+          };
 
-        registry.setTheme({ settings: [defaultTokenColors, ...tokenColors] });
-        if (!stylesheets[themeClassName]) {
-          const rules = [
-            renderRule(themeClassName, getStylesFromSettings(settings)),
-            ...(scope
-              ? prefixRules(
-                  generateTokensCSSForColorMap(
-                    registry.getColorMap().map(color => replaceColor(color, colorThemeIdentifier))
-                  ).split('\n'),
-                  `.${themeClassName} `
-                )
-              : [])
-          ];
+          registry.setTheme({ settings: [defaultTokenColors, ...tokenColors] });
+          if (!stylesheets[themeClassName]) {
+            const rules = [
+              renderRule(themeClassName, getStylesFromSettings(settings)),
+              ...(scope
+                ? prefixRules(
+                    generateTokensCSSForColorMap(
+                      registry.getColorMap().map(color => replaceColor(color, colorThemeIdentifier))
+                    ).split('\n'),
+                    `.${themeClassName} `
+                  )
+                : [])
+            ];
 
-          if (setting === 'prefersDarkTheme') {
-            stylesheets[themeClassName] = prefersDark(rules);
-          } else if (setting === 'prefersLightTheme') {
-            stylesheets[themeClassName] = prefersLight(rules);
-          } else {
-            stylesheets[themeClassName] = rules.join('\n');
+            if (setting === 'prefersDarkTheme') {
+              stylesheets[themeClassName] = prefersDark(rules);
+            } else if (setting === 'prefersLightTheme') {
+              stylesheets[themeClassName] = prefersLight(rules);
+            } else {
+              stylesheets[themeClassName] = rules.join('\n');
+            }
           }
         }
-      }
 
-      const rawLines = text.split(/\r?\n/);
-      const htmlLines = [];
-      /** @type {import('vscode-textmate').ITokenTypeMap} */
-      let tokenTypes = {};
-      /** @type {number} */
-      let languageId;
+        const rawLines = text.split(/\r?\n/);
+        const htmlLines = [];
+        /** @type {import('vscode-textmate').ITokenTypeMap} */
+        let tokenTypes = {};
+        /** @type {number} */
+        let languageId;
 
-      try {
         if (scope) {
           const grammarData = getGrammar(scope, grammarCache);
           languageId = grammarData.languageId;
@@ -269,19 +285,19 @@ function createPlugin() {
 
           htmlLines.push([`<span class="${className}">`, htmlLine, `</span>`].join(''));
         }
+
+        const className = joinClassNames(wrapperClassName, joinThemeClassNames(themeClassNames), 'vscode-highlight');
+        node.type = 'html';
+        node.value = [
+          `<pre class="${className}" data-language="${languageName}">`,
+          `<code class="vscode-highlight-code">`,
+          htmlLines.join('\n'),
+          `</code>`,
+          `</pre>`
+        ].join('');
       } finally {
         unlockRegistry();
       }
-
-      const className = joinClassNames(wrapperClassName, joinThemeClassNames(themeClassNames), 'vscode-highlight');
-      node.type = 'html';
-      node.value = [
-        `<pre class="${className}" data-language="${languageName}">`,
-        `<code class="vscode-highlight-code">`,
-        htmlLines.join('\n'),
-        `</code>`,
-        `</pre>`
-      ].join('');
     }
 
     const themeNames = Object.keys(stylesheets);

src/utils.js

@@ -22,26 +22,29 @@ function parseExtensionIdentifier(identifier) {
 /**
  * Gets the absolute path to the download path of a downloaded extension.
  * @param {string} identifier
+ * @param {string} extensionDir
  */
-function getExtensionBasePath(identifier) {
-  return path.resolve(__dirname, '../lib/extensions', identifier);
+function getExtensionBasePath(identifier, extensionDir) {
+  return path.join(extensionDir, identifier);
 }
 
 /**
  * Gets the absolute path to the data directory of a downloaded extension.
  * @param {string} identifier
+ * @param {string} extensionDir
  */
-function getExtensionPath(identifier) {
-  return path.resolve(getExtensionBasePath(identifier), 'extension');
+function getExtensionPath(identifier, extensionDir) {
+  return path.join(getExtensionBasePath(identifier, extensionDir), 'extension');
 }
 
 /**
  * Gets the package.json of an extension as a JavaScript object.
  * @param {string} identifier
+ * @param {string} extensionDir
  * @returns {object}
  */
-function getExtensionPackageJson(identifier) {
-  return require(path.join(getExtensionPath(identifier), 'package.json'));
+function getExtensionPackageJson(identifier, extensionDir) {
+  return require(path.join(getExtensionPath(identifier, extensionDir), 'package.json'));
 }
 
 /**