test: add tests for examples in dsl-reference

Signed-off-by: Matthias Pichler <[email protected]>

Author: matthias-pichler

.ci/validation/package-lock.json

@@ -25,6 +25,7 @@
   "dependencies": {
     "ajv": "^8.12.0",
     "ajv-formats": "^2.1.1",
-    "js-yaml": "^4.1.0"
+    "js-yaml": "^4.1.0",
+    "marked": "^13.0.0"
   }
 }

.ci/validation/package.json

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2023-Present The Serverless Workflow Specification Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { SWSchemaValidator } from "./index";
+import fs from "node:fs";
+import path from "node:path";
+import marked from "marked";
+
+SWSchemaValidator.prepareSchemas();
+
+const dslReferencePath = path.join(
+  __dirname,
+  "..",
+  "..",
+  "..",
+  "dsl-reference.md"
+);
+
+describe(`Verify every example in the dsl docs`, () => {
+  const workflows = marked
+    .lexer(fs.readFileSync(dslReferencePath, SWSchemaValidator.defaultEncoding))
+    .filter((item): item is marked.Tokens.Code => item.type === "code")
+    .filter((item) => item.lang === "yaml")
+    .map((item) => item.text)
+    .map((text) => SWSchemaValidator.yamlToJSON(text))
+    .filter((workflow) => typeof workflow === "object")
+    .filter((workflow) => "document" in workflow)
+    .filter((workflow) => "dsl" in workflow.document);
+
+  test.each(workflows)("$document.name", (workflow) => {
+    const results = SWSchemaValidator.validateSchema(workflow);
+    if (results?.errors) {
+      console.warn(
+        `Schema validation on workflow ${workflow.document.name} failed with: `,
+        JSON.stringify(results.errors, null, 2)
+      );
+    }
+    expect(results?.valid).toBeTruthy();
+  });
+});

.ci/validation/src/dsl.test.ts

@@ -34,7 +34,7 @@ describe(`Verify every example in the repository`, () => {
     .map((file) => file.name);
 
   test.each(examples)("Example %s", (file) => {
-    const workflow = SWSchemaValidator.toJSON(
+    const workflow = SWSchemaValidator.loadAsJSON(
       path.join(__dirname, `${examplePath}/${file}`)
     );
     const results = SWSchemaValidator.validateSchema(workflow);

.ci/validation/src/examples.test.ts

@@ -44,11 +44,15 @@ export module SWSchemaValidator {
   }
 
   function syncReadSchema(filename: string): any {
-    return toJSON(path.join(__dirname, `${schemaPath}/${filename}`));
+    return loadAsJSON(path.join(__dirname, `${schemaPath}/${filename}`));
   }
 
-  export function toJSON(filename: string): any {
-    const yamlObj = yaml.load(fs.readFileSync(filename, defaultEncoding), {
+  export function loadAsJSON(filename: string): any {
+    return yamlToJSON(fs.readFileSync(filename, defaultEncoding));
+  }
+
+  export function yamlToJSON(yamlStr: string): any {
+    const yamlObj = yaml.load(yamlStr, {
       json: true,
     });
     return structuredClone(yamlObj);

.ci/validation/src/index.ts

@@ -34,7 +34,7 @@ describe(`Check that invalid workflows are rejected`, () => {
     .map((file) => file.name);
 
   test.each(examples)("Example %s", (file) => {
-    const workflow = SWSchemaValidator.toJSON(
+    const workflow = SWSchemaValidator.loadAsJSON(
       path.join(__dirname, `${invalidPath}/${file}`)
     );
     const results = SWSchemaValidator.validateSchema(workflow);