add yaml intelligence for _extension.yml (#1746)

* add schema for _extension.yml and enforce it at render time
* update tests
* changelog

Author: cscheid

news/changelog-1.1.md

@@ -76,6 +76,7 @@
 ## Extensions
 
 - Properly copy `format-resources` for HTML based formats
+- Extension YAML files `_extension.yml` are now validated at render time. (#1268)
 
 ## Publishing
 

src/command/list/cmd.ts

@@ -62,7 +62,7 @@ async function outputExtensions(
 ) {
   // Provide the with with a list
   const project = await projectContext(path);
-  const extensions = extensionContext.extensions(path, project);
+  const extensions = await extensionContext.extensions(path, project);
   if (extensions.length === 0) {
     info(
       `No extensions are installed in this ${

src/command/remove/cmd.ts

@@ -25,7 +25,6 @@ import {
   selectTool,
 } from "../../tools/tools-console.ts";
 import { haveArrowKeys } from "../../core/platform.ts";
-import { join } from "path/mod.ts";
 
 export const removeCommand = new Command()
   .hidden()
@@ -78,12 +77,12 @@ export const removeCommand = new Command()
           // Not provided, give the user a list to select from
           const workingDir = Deno.cwd();
 
-          const resolveTargetDir = () => {
+          const resolveTargetDir = async () => {
             if (options.embed) {
               // We're removing an embedded extension, lookup the extension
               // and use its path
               const context = createExtensionContext();
-              const extension = context.extension(
+              const extension = await context.extension(
                 options.embed,
                 workingDir,
               );
@@ -97,12 +96,12 @@ export const removeCommand = new Command()
               return workingDir;
             }
           };
-          const targetDir = resolveTargetDir();
+          const targetDir = await resolveTargetDir();
 
           // Process extension
           if (target) {
             // explicitly provided
-            const extensions = extensionContext.find(target, targetDir);
+            const extensions = await extensionContext.find(target, targetDir);
             if (extensions.length > 0) {
               await removeExtensions(extensions.slice(), options.prompt);
             } else {
@@ -111,7 +110,10 @@ export const removeCommand = new Command()
           } else {
             // Provide the with with a list
             const project = await projectContext(targetDir);
-            const extensions = extensionContext.extensions(targetDir, project);
+            const extensions = await extensionContext.extensions(
+              targetDir,
+              project,
+            );
 
             // Show a list
             if (extensions.length > 0) {

src/command/render/defaults.ts

@@ -39,7 +39,7 @@ import {
 import { TempContext } from "../../core/temp.ts";
 import { authorsFilter } from "./authors.ts";
 
-export function generateDefaults(
+export async function generateDefaults(
   options: PandocOptions,
 ): Promise<FormatPandoc | undefined> {
   let allDefaults: FormatPandoc | undefined;
@@ -48,7 +48,7 @@ export function generateDefaults(
     allDefaults = options.format.pandoc || {};
 
     // resolve filters
-    const resolvedFilters = resolveFilters(
+    const resolvedFilters = await resolveFilters(
       [
         ...(allDefaults[kFilters] || []),
       ],
@@ -67,9 +67,9 @@ export function generateDefaults(
       }
     }
 
-    return Promise.resolve(allDefaults);
+    return allDefaults;
   } else {
-    return Promise.resolve(undefined);
+    return undefined;
   }
 }
 

src/command/render/filters.ts

@@ -53,7 +53,6 @@ import {
 import { layoutFilter, layoutFilterParams } from "./layout.ts";
 import { pandocMetadataPath } from "./render-paths.ts";
 import { removePandocArgs } from "./flags.ts";
-import * as ld from "../../core/lodash.ts";
 import { mergeConfigs } from "../../core/config.ts";
 import { projectType } from "../../project/types/project-types.ts";
 import { readCodePage } from "../../core/windows.ts";
@@ -74,7 +73,7 @@ const kTimingFile = "timings-file";
 
 const kHasBootstrap = "has-bootstrap";
 
-export function filterParamsJson(
+export async function filterParamsJson(
   args: string[],
   options: PandocOptions,
   defaults: FormatPandoc | undefined,
@@ -105,7 +104,7 @@ export function filterParamsJson(
     ...ipynbFilterParams(options),
     ...projectFilterParams(options),
     ...quartoColumnParams,
-    ...quartoFilterParams(options, defaults),
+    ...await quartoFilterParams(options, defaults),
     ...crossrefFilterParams(options, defaults),
     ...layoutFilterParams(options.format),
     ...languageFilterParams(options.format.language),
@@ -262,7 +261,7 @@ function extractSmartIncludeInternal(
     if (value === undefined) {
       return [];
     }
-    if (ld.isArray(value)) {
+    if (Array.isArray(value)) {
       const contents = value.filter(isContent);
       const nonContents = value.filter((v) => !isContent(v));
       obj[name] = nonContents;
@@ -289,7 +288,7 @@ function extractIncludeVariables(obj: { [key: string]: unknown }) {
     delete obj[name];
     if (!value) {
       return [];
-    } else if (ld.isArray(value)) {
+    } else if (Array.isArray(value)) {
       return value as unknown[];
     } else {
       return [value];
@@ -414,7 +413,7 @@ function ipynbFilterParams(options: PandocOptions) {
   };
 }
 
-function quartoFilterParams(
+async function quartoFilterParams(
   options: PandocOptions,
   defaults?: FormatPandoc,
 ) {
@@ -449,7 +448,7 @@ function quartoFilterParams(
   if (shortcodes !== undefined) {
     params[kShortcodes] = shortcodes;
   }
-  const extShortcodes = extensionShortcodes(options);
+  const extShortcodes = await extensionShortcodes(options);
   if (extShortcodes) {
     params[kShortcodes] = params[kShortcodes] || [];
     (params[kShortcodes] as string[]).push(...extShortcodes);
@@ -477,10 +476,10 @@ function quartoFilterParams(
   return params;
 }
 
-function extensionShortcodes(options: PandocOptions) {
+async function extensionShortcodes(options: PandocOptions) {
   const extensionShortcodes: string[] = [];
   if (options.extension) {
-    const allExtensions = options.extension?.extensions(
+    const allExtensions = await options.extension?.extensions(
       options.source,
       options.project,
     );
@@ -508,10 +507,10 @@ function initFilterParams(dependenciesFile: string) {
 const kQuartoFilterMarker = "quarto";
 const kQuartoCiteProcMarker = "citeproc";
 
-export function resolveFilters(
+export async function resolveFilters(
   filters: QuartoFilter[],
   options: PandocOptions,
-): QuartoFilter[] | undefined {
+): Promise<QuartoFilter[] | undefined> {
   // build list of quarto filters
 
   // The default order of filters will be
@@ -532,7 +531,7 @@ export function resolveFilters(
   quartoFilters.push(quartoPostFilter());
 
   // Resolve any filters that are provided by an extension
-  filters = resolveFilterExtension(options, filters);
+  filters = await resolveFilterExtension(options, filters);
 
   // if 'quarto' is in the filters, inject our filters at that spot,
   // otherwise inject them at the beginning so user filters can take
@@ -613,20 +612,21 @@ function pdfEngine(options: PandocOptions): string {
 const kQuartoExtOrganization = "quarto-ext";
 const kQuartoExtBuiltIn = ["code-filename", "grouped-tabsets"];
 
-function resolveFilterExtension(
+async function resolveFilterExtension(
   options: PandocOptions,
   filters: QuartoFilter[],
-): QuartoFilter[] {
+): Promise<QuartoFilter[]> {
   // Resolve any filters that are provided by an extension
-  const results = filters.flatMap((filter) => {
+  const results: (QuartoFilter | QuartoFilter[])[] = [];
+  const getFilter = async (filter: QuartoFilter) => {
     // Look for extension names in the filter list and result them
     // into the filters provided by the extension
     if (
       filter !== kQuartoFilterMarker && filter !== kQuartoCiteProcMarker &&
       typeof (filter) === "string" &&
       !existsSync(filter)
     ) {
-      let extensions = options.extension?.find(
+      let extensions = await options.extension?.find(
         filter,
         options.source,
         "filters",
@@ -672,6 +672,10 @@ function resolveFilterExtension(
     } else {
       return filter;
     }
-  });
-  return results;
+  };
+  for (const filter of filters) {
+    const r = await getFilter(filter);
+    results.push(r);
+  }
+  return results.flat();
 }

src/command/render/pandoc.ts

@@ -224,7 +224,7 @@ export async function runPandoc(
   cleanMetadataForPrinting(printMetadata);
 
   // generate defaults and capture defaults to be printed
-  let allDefaults = await generateDefaults(options) || {};
+  let allDefaults = (await generateDefaults(options)) || {};
   let printAllDefaults = ld.cloneDeep(allDefaults) as FormatPandoc;
 
   // capture any filterParams in the FormatExtras
@@ -654,7 +654,7 @@ export async function runPandoc(
   // set parameters required for filters (possibily mutating all of it's arguments
   // to pull includes out into quarto parameters so they can be merged)
   let pandocArgs = args;
-  const paramsJson = filterParamsJson(
+  const paramsJson = await filterParamsJson(
     pandocArgs,
     options,
     allDefaults,

src/command/render/render-contexts.ts

@@ -572,7 +572,7 @@ const readExtensionFormat = async (
   // Read the format file and populate this
   if (formatDesc.extension) {
     // Find the yaml file
-    const extension = extensionContext.extension(
+    const extension = await extensionContext.extension(
       formatDesc.extension,
       file,
       project,

src/command/use/commands/template.ts

@@ -73,7 +73,7 @@ async function useTemplate(
       // Copy extensions
       const extDir = join(stagedDir, kExtensionDir);
       if (existsSync(extDir)) {
-        copyExtensions(source, extDir, outputDirectory);
+        await copyExtensions(source, extDir, outputDirectory);
       }
 
       for (const fileToCopy of filesToCopy) {

src/core/lib/yaml-intelligence/yaml-intelligence.ts

@@ -65,7 +65,10 @@ import {
 import { getFormatAliases } from "../yaml-schema/format-aliases.ts";
 import { getFrontMatterSchema } from "../yaml-schema/front-matter.ts";
 import { getEngineOptionsSchema } from "../yaml-schema/chunk-metadata.ts";
-import { getProjectConfigSchema } from "../yaml-schema/project-config.ts";
+import {
+  getExtensionConfigSchema,
+  getProjectConfigSchema,
+} from "../yaml-schema/project-config.ts";
 import {
   getYamlIntelligenceResource,
   setYamlIntelligenceResources,
@@ -1041,25 +1044,58 @@ function exportSmokeTest(
   console.error(JSON.stringify({ kind, context }, null, 2));
 }
 
+const determineSchema = async (context: YamlIntelligenceContext): Promise<{
+  schema: ConcreteSchema | undefined;
+  schemaName: string | undefined;
+}> => {
+  const extension = context.path === null
+    ? ""
+    : (context.path.split(".").pop() || "");
+
+  if (context.filetype !== "yaml") {
+    return {
+      schema: undefined,
+      schemaName: undefined,
+    };
+  }
+
+  if (extension === "qmd") {
+    const frontMatterSchema = await getFrontMatterSchema();
+    return {
+      schema: frontMatterSchema,
+      schemaName: "front-matter",
+    };
+  }
+  const extensionConfigNames = [
+    "_extension.yml",
+    "_extension.yaml",
+  ];
+  if (
+    context.path &&
+    extensionConfigNames.some((name) => context.path!.endsWith(name))
+  ) {
+    const extensionConfigSchema = await getExtensionConfigSchema();
+    return {
+      schema: extensionConfigSchema,
+      schemaName: "extension-config",
+    };
+  } else {
+    const projectConfigSchema = await getProjectConfigSchema();
+    return {
+      schema: projectConfigSchema,
+      schemaName: "project-config",
+    };
+  }
+};
+
 export async function getAutomation(
   kind: AutomationKind,
   context: YamlIntelligenceContext,
 ) {
-  const extension = context.path === null
-    ? ""
-    : (context.path.split(".").pop() || "");
-  const frontMatterSchema = await getFrontMatterSchema();
-  const projectConfigSchema = await getProjectConfigSchema();
-  const schema = ({
-    "yaml": extension === "qmd" ? frontMatterSchema : projectConfigSchema,
-    "markdown": undefined, // can't be known ahead of time
-    "script": undefined,
-  })[context.filetype];
-  const schemaName = ({
-    "yaml": extension === "qmd" ? "front-matter" : "config",
-    "markdown": undefined, // can't be known ahead of time
-    "script": undefined,
-  })[context.filetype];
+  const {
+    schema,
+    schemaName,
+  } = await determineSchema(context);
 
   const result = await automationFileTypeDispatch(context.filetype, kind, {
     ...context,

src/core/lib/yaml-schema/project-config.ts

@@ -42,6 +42,19 @@ export const getProjectConfigFieldsSchema = defineCached(
   "project-config-fields",
 );
 
+export const getExtensionConfigFieldsSchema = defineCached(
+  // deno-lint-ignore require-await
+  async () => {
+    return {
+      schema: objectSchemaFromFieldsObject(
+        getYamlIntelligenceResource("schema/extension.yml") as SchemaField[],
+      ),
+      errorHandlers: [],
+    };
+  },
+  "extension-config-fields",
+);
+
 function disallowTopLevelType(
   error: LocalizedError,
   parse: AnnotatedParse,
@@ -101,3 +114,17 @@ export const getProjectConfigSchema = defineCached(
   },
   "project-config",
 );
+
+export const getExtensionConfigSchema = defineCached(
+  async () => {
+    const extensionConfig = await getExtensionConfigFieldsSchema();
+    return {
+      schema: describeSchema(
+        extensionConfig,
+        "an extension configuration object",
+      ),
+      errorHandlers: [],
+    };
+  },
+  "extension-config",
+);