Merge pull request #41 from andrewbranch/bug/xml

Support themes published in plist format

Author: andrewbranch

package-lock.json

@@ -58,6 +58,7 @@
     "loglevel": "^1.6.3",
     "oniguruma": "^7.2.0",
     "plist": "^3.0.1",
+    "rimraf": "^3.0.0",
     "unist-util-visit": "^1.4.0",
     "vscode-textmate": "^4.1.0"
   },

package.json

@@ -4,7 +4,7 @@ const util = require('util');
 const glob = require('glob');
 const path = require('path');
 const processExtension = require('../src/processExtension');
-const { requireGrammar } = require('../src/utils');
+const { requirePlistOrJson } = require('../src/utils');
 
 const copyFile = util.promisify(fs.copyFile);
 const writeFile = util.promisify(fs.writeFile);
@@ -30,13 +30,13 @@ glob(path.resolve(__dirname, '../vscode/extensions/*/package.json'), async (err,
         ...await newManifestPromise,
         ...await Object.keys(extension.grammars).reduce(async (newManifestPromise, scopeName) => {
           const grammar = extension.grammars[scopeName];
-          const content = await requireGrammar(grammar.path);
+          const content = await requirePlistOrJson(grammar.path);
           let destPath = grammarPath(path.basename(grammar.path));
           const newContent = processTmJson(content);
           // Different languages are sometimes named the same thing
           languageId++;
           if (await exists(destPath)) {
-            const existingGrammar = await requireGrammar(destPath);
+            const existingGrammar = await requirePlistOrJson(destPath);
             if (existingGrammar.scopeName !== scopeName) {
               const [baseName, ...ext] = path.basename(grammar.path).split('.');
               const renamed = [baseName, languageId, ...ext].join('.');

scripts/scrapeBuiltinExtensions.js

@@ -3,8 +3,6 @@ const fs = require('fs');
 const path = require('path');
 const zlib = require('zlib');
 const util = require('util');
-const request = require('request');
-const decompress = require('decompress');
 const processExtension = require('./processExtension');
 const { highestBuiltinLanguageId } = require('./storeUtils');
 const {
@@ -42,34 +40,23 @@ async function syncExtensionData({ identifier }, cache, extensionDir) {
  * @param {import('.').ExtensionDemand} extensionDemand
  * @param {*} cache
  * @param {string} extensionDir
+ * @param {import('./host').Host} host
  */
-async function downloadExtension(extensionDemand, cache, extensionDir) {
+async function downloadExtension(extensionDemand, cache, extensionDir, host) {
   const { identifier, version } = extensionDemand;
   const { publisher, name } = parseExtensionIdentifier(identifier);
   const url = `https://marketplace.visualstudio.com/_apis/public/gallery/publishers/${publisher}/vsextensions/${name}/${version}/vspackage`;
-  const archive = await new Promise((resolve, reject) => {
-    request.get(url, { encoding: null }, (error, res, body) => {
-      if (error) {
-        return reject(error);
-      }
-      if (res.statusCode === 404) {
-        return reject(
-          new Error(`Could not find extension with publisher '${publisher}', name '${name}', and verion '${version}'.`)
-        );
-      }
-      if (res.statusCode !== 200) {
-        return reject(new Error(`Failed to download extension ${identifier} with status code ${res.statusCode}`));
-      }
-      if (res.headers['content-encoding'] === 'gzip') {
-        return gunzip(body).then(resolve, reject);
-      }
-
-      resolve(body);
-    });
-  });
+  const response = await host.fetch(url, { encoding: null });
+  if (response.statusCode === 404) {
+    throw new Error(`Could not find extension with publisher '${publisher}', name '${name}', and verion '${version}'.`);
+  }
+  if (response.statusCode !== 200) {
+    const details = response.body ? `. Response body:\n\n${response.body.toString('utf8')}` : '';
+    throw new Error(`Failed to download extension ${identifier} with status code ${response.statusCode}${details}`);
+  }
 
   const extensionPath = getExtensionBasePath(identifier, extensionDir);
-  await decompress(archive, extensionPath);
+  await host.decompress(response.body, extensionPath);
   await syncExtensionData(extensionDemand, cache, extensionDir);
   return extensionPath;
 }
@@ -79,24 +66,25 @@ async function downloadExtension(extensionDemand, cache, extensionDir) {
  * @property {import('.').ExtensionDemand[]} extensions
  * @property {*} cache
  * @property {string} extensionDir
+ * @property {import('./host').Host} host
  */
 
 /**
  * @param {DownloadExtensionOptions} options
  */
-async function downloadExtensionsIfNeeded({ extensions, cache, extensionDir }) {
+async function downloadExtensionsIfNeeded({ extensions, cache, extensionDir, host }) {
   extensions = extensions.slice();
   while (extensions.length) {
     const extensionDemand = extensions.shift();
     const { identifier, version } = extensionDemand;
     const extensionPath = getExtensionBasePath(identifier, extensionDir);
     if (!fs.existsSync(extensionPath)) {
-      await downloadExtension(extensionDemand, cache, extensionDir);
+      await downloadExtension(extensionDemand, cache, extensionDir, host);
       continue;
     }
     const packageJson = getExtensionPackageJson(identifier, extensionDir);
     if (packageJson.version !== version) {
-      await downloadExtension(extensionDemand, cache, extensionDir);
+      await downloadExtension(extensionDemand, cache, extensionDir, host);
       continue;
     }
 

src/downloadExtension.js

@@ -0,0 +1,42 @@
+const request = require('request');
+const decompress = require('decompress');
+const zlib = require('zlib');
+const util = require('util');
+const gunzip = util.promisify(zlib.gunzip);
+
+/**
+ * @typedef {object} Response
+ * @property {Buffer | undefined} body
+ * @property {number} statusCode
+ */
+
+/**
+ * @typedef {object} Host
+ * @property {(url: string, options: request.CoreOptions) => Promise<Response>} fetch
+ * @property {(input: string | Buffer, output: string) => Promise<unknown>} decompress
+ */
+
+/** @type {Host} */
+const host = {
+  fetch: (url, options) =>
+    new Promise((resolve, reject) => {
+      request.get(url, options, async (error, res, body) => {
+        if (error) {
+          return reject(error);
+        }
+        if (res.statusCode !== 200) {
+          return resolve({ body, statusCode: res.statusCode });
+        }
+        if (res.headers['content-encoding'] === 'gzip') {
+          const unzipped = await gunzip(body);
+          return resolve({ body: unzipped, statusCode: res.statusCode });
+        }
+
+        resolve({ body, statusCode: res.statusCode });
+      });
+    }),
+
+  decompress
+};
+
+module.exports = host;

src/host.js

@@ -2,6 +2,7 @@
 const fs = require('fs');
 const path = require('path');
 const logger = require('loglevel');
+const defaultHost = require('./host');
 const visit = require('unist-util-visit');
 const escapeHTML = require('lodash.escape');
 const lineHighlighting = require('./lineHighlighting');
@@ -122,6 +123,7 @@ function getStylesFromSettings(settings) {
  * @property {(colorValue: string, theme: string) => string=} replaceColor
  * @property {string=} extensionDataDirectory
  * @property {'trace' | 'debug' | 'info' | 'warn' | 'error'=} logLevel
+ * @property {import('./host').Host=} host
  */
 
 function createPlugin() {
@@ -143,7 +145,8 @@ function createPlugin() {
       injectStyles = true,
       replaceColor = x => x,
       extensionDataDirectory = path.resolve(__dirname, '../lib/extensions'),
-      logLevel = 'error'
+      logLevel = 'error',
+      host = defaultHost
     } = {}
   ) {
     logger.setLevel(logLevel);
@@ -162,7 +165,8 @@ function createPlugin() {
       await downloadExtensionsIfNeeded({
         extensions,
         cache,
-        extensionDir: extensionDataDirectory
+        extensionDir: extensionDataDirectory,
+        host
       });
 
       const grammarCache = await cache.get('grammars');

src/index.js

@@ -1,6 +1,6 @@
 // @ts-check
 const path = require('path');
-const { getLanguageNames, requireJson, requireGrammar } = require('./utils');
+const { getLanguageNames, requireJson, requirePlistOrJson } = require('./utils');
 
 async function processExtension(packageJsonPath) {
   const packageJson = requireJson(packageJsonPath);
@@ -10,7 +10,7 @@ async function processExtension(packageJsonPath) {
     const manifest = await Promise.all(
       packageJson.contributes.grammars.map(async grammar => {
         const sourcePath = path.resolve(path.dirname(packageJsonPath), grammar.path);
-        const content = await requireGrammar(sourcePath);
+        const content = await requirePlistOrJson(sourcePath);
         const { scopeName } = content;
         const languageRegistration = packageJson.contributes.languages.find(l => l.id === grammar.language);
 
@@ -44,7 +44,7 @@ async function processExtension(packageJsonPath) {
     const manifest = await Promise.all(
       packageJson.contributes.themes.map(async theme => {
         const sourcePath = path.resolve(path.dirname(packageJsonPath), theme.path);
-        const themeContents = requireJson(sourcePath);
+        const themeContents = await requirePlistOrJson(sourcePath);
         return {
           id: theme.id || path.basename(theme.path).split('.')[0],
           path: sourcePath,

src/processExtension.js

@@ -72,7 +72,7 @@ function sanitizeForClassName(str) {
 
 const readFile = util.promisify(fs.readFile);
 const requireJson = /** @param {string} pathName */ pathName => JSON5.parse(fs.readFileSync(pathName, 'utf8'));
-const requireGrammar = /** @param {string} pathName */ async pathName =>
+const requirePlistOrJson = /** @param {string} pathName */ async pathName =>
   path.extname(pathName) === '.json' ? requireJson(pathName) : plist.parse(await readFile(pathName, 'utf8'));
 
 module.exports = {
@@ -83,5 +83,5 @@ module.exports = {
   getLanguageNames,
   sanitizeForClassName,
   requireJson,
-  requireGrammar
+  requirePlistOrJson
 };