feat: enhance tsed-flow-package-symbols with Markdown support, error handling, and schema updates

Added support for a `markdown_url` to enrich package symbols with Markdown-based documentation. Improved error handling in migration logic for presets and introduced several schema enhancements, including new fields in the `package_symbols` collection and updated display options.

Author: Romakita

extensions/tsed-flow-package-symbols/README.md

@@ -0,0 +1,109 @@
+# tsed-flow-package-symbols
+
+Directus operation that imports exported symbols of Ts.ED packages from the contractual JSON `https://tsed.dev/api.json` and writes them into the `package_symbols` collection.
+
+## How it works
+
+- The operation fetches `api.json` (or consumes a compatible payload provided by a previous step, see “Webhook/inline payload” below).
+- It iterates `modules["<package-name>"].symbols` and maps each entry to a `package_symbols` record via `src/mappers/mapApiSymbolToDirectus.ts`.
+- The corresponding package is ensured in the `packages` collection (created if missing) with `type: "official"`.
+- Upsert is performed by the symbol `id` coming from `api.json`. If a record already exists, its `versions` field is merged (union) with the new version from `api.json`.
+- Fields mapped per symbol:
+  - `name ← symbolName`
+  - `type ← symbolType`
+  - `doc_url ← origin(api.json) + path`
+  - `markdown_url ← markdown_base + path + ".md"`
+  - `deprecated ← status includes "deprecated"`
+  - `tags ← status without "deprecated"`
+  - `versions ← []` then the global `api.json.version` is appended during import
+
+## Build and run locally
+
+1) Install deps at the repo root
+```bash
+corepack enable
+yarn install --immutable
+```
+
+2) Build all extensions
+```bash
+yarn build
+```
+
+3) Start Directus (dev)
+```bash
+yarn start:dev
+```
+
+## Operation configuration (UI)
+
+Directus → Flows → Add operation → “Ts.ED Package Symbols importer”.
+
+Card options:
+- `url` (string) — `api.json` URL. Default in the card: `https://tsed.dev/api.json`.
+- `markdown_url` (string) — base URL where markdown files live. Default in the card: `https://tsed.dev/ai/references/api`.
+
+Notes about defaults:
+- The operation handler also has an internal fallback for `markdown_url` to `https://tsed.dev/ai/references` if the option isn’t provided by the flow. To get the expected final markdown path `…/api/<symbol>.md`, set the card option explicitly to `https://tsed.dev/ai/references/api` (as in the exported flows).
+
+Save the flow. The operation will use these options at runtime.
+
+## Trigger the flow via HTTP (Directus API)
+
+Manual trigger (flow id `569895a0-7a65-4cb2-94aa-67f77a776a08` in the sample exports):
+
+```bash
+CMS_API_URL="http://localhost:8055"
+CMS_API_TOKEN="<ADMIN_OR_ALLOWED_TOKEN>"
+FLOW_ID="569895a0-7a65-4cb2-94aa-67f77a776a08"
+
+curl -X POST \
+  "$CMS_API_URL/flows/trigger/$FLOW_ID" \
+  -H "Authorization: Bearer $CMS_API_TOKEN" \
+  -H "Content-Type: application/json" \
+  -d '{}'
+```
+
+### Webhook trigger and inline payload (optional)
+
+This repo also includes a webhook flow (`9039bef8-fdc3-4f31-b5ab-7fee31273921`). If you POST a body containing a payload compatible with `src/schema/ApiPayloadSchema.ts`, the operation will validate and use this body instead of fetching from `url`.
+
+Example (send the whole api.json as request body):
+
+```bash
+CMS_API_URL="http://localhost:8055"
+CMS_API_TOKEN="<ADMIN_OR_ALLOWED_TOKEN>"
+FLOW_ID="9039bef8-fdc3-4f31-b5ab-7fee31273921"
+
+curl -X POST \
+  "$CMS_API_URL/flows/trigger/$FLOW_ID" \
+  -H "Authorization: Bearer $CMS_API_TOKEN" \
+  -H "Content-Type: application/json" \
+  --data-binary @api.json
+```
+
+## Permissions required
+
+This operation runs with the flow runner’s role/policy (`accountability: "all"` in the exported flows). Ensure the role used to trigger the flow has at least:
+
+- `packages`: read, create, update (the operation may create missing packages)
+- `package_symbols`: read, create, update (the operation reads by `id`, then creates/updates symbols)
+
+If these permissions are missing, the flow will fail with `403 You don't have permission to access this.` during upsert.
+
+## Testing
+
+Run the unit tests for the mapper and the service:
+
+```bash
+yarn vitest run \
+  extensions/tsed-flow-package-symbols/src/mappers/mapApiSymbolToDirectus.spec.ts \
+  packages/usecases/package-symbols/PackageSymbolsService.spec.ts
+```
+
+Both tests should pass.
+
+## Notes
+
+- The `ApiPayloadSchema` strictly types the `api.json` format used by the operation.
+- The mapper intentionally keeps logic minimal and assumes the contract is stable.

extensions/tsed-flow-package-symbols/src/api.ts

@@ -3,10 +3,12 @@ import { inject } from "@tsed/di";
 import { wrapOperation } from "@tsed-cms/infra/bootstrap/directus.js";
 import type { Package } from "@tsed-cms/infra/directus/interfaces/DirectusSchema.js";
 import { HttpClient } from "@tsed-cms/infra/http/HttpClient.js";
+import { validate } from "@tsed-cms/infra/validators/validate.js";
 import { PackageSymbolsService } from "@tsed-cms/usecases/package-symbols/PackageSymbolsService.js";
 import { PackagesService } from "@tsed-cms/usecases/packages/PackagesService.js";
 
-import { type ApiJsonModule, type ApiJsonResponse, mapApiSymbolToDirectus } from "./mappers/mapApiSymbolToDirectus.js";
+import { mapApiSymbolToDirectus } from "./mappers/mapApiSymbolToDirectus.js";
+import { type ApiPayload, ApiPayloadSchema } from "./schema/ApiPayloadSchema.js";
 
 export type Options = {
   url?: string;
@@ -28,25 +30,42 @@ async function ensurePackage(pkgName: string): Promise<Package> {
 
 export default defineOperationApi<Options>({
   id: "tsed-flow-package-symbols",
-  handler: wrapOperation(async (opts) => {
+  handler: wrapOperation(async (opts, context) => {
     const startedAt = Date.now();
     const url = (opts?.url?.trim() || "https://tsed.dev/api.json").toString();
     const markdownUrl = (opts?.markdown_url?.trim() || "https://tsed.dev/ai/references").toString();
-
     const http = inject(HttpClient);
     const symbolsService = inject(PackageSymbolsService);
 
+    // Origin pour construire les URL de doc
+    const origin = new URL(url).origin;
+    let data: ApiPayload;
+
+    if ((context?.data?.["$last"] as any)?.body) {
+      try {
+        data = await validate((context?.data?.["$last"] as any)?.body, ApiPayloadSchema);
+      } catch (er) {
+        return {
+          url,
+          processed: 0,
+          upserted: 0,
+          durationMs: Date.now() - startedAt,
+          errors: [
+            {
+              error: er?.message || String(er)
+            }
+          ]
+        };
+      }
+    } else {
+      data = await http.get<ApiPayload>(url);
+    }
+
     let processed = 0;
     let upserted = 0;
     const errors: { name: string; pkg: string; error: string }[] = [];
 
-    // Fetch the JSON (typed)
-    const data = await http.get<ApiJsonResponse>(url);
-
-    // Origin pour construire les URL de doc
-    const origin = new URL(url).origin;
-
-    for (const [pkgName, mod] of Object.entries<ApiJsonModule>(data.modules)) {
+    for (const [pkgName, mod] of Object.entries(data.modules)) {
       for (const s of mod.symbols) {
         try {
           const pkg = await ensurePackage(pkgName);
@@ -59,7 +78,6 @@ export default defineOperationApi<Options>({
           upserted += 1;
           processed += 1;
         } catch (er: any) {
-          console.log(er);
           errors.push({
             name: s.symbolName,
             pkg: pkgName,

extensions/tsed-flow-package-symbols/src/mappers/mapApiSymbolToDirectus.spec.ts

@@ -1,21 +1,21 @@
-import { type ApiJsonModuleSymbol, mapApiSymbolToDirectus } from "./mapApiSymbolToDirectus.js";
+import type { ApiSymbol } from "../schema/ApiPayloadSchema.js";
+import { mapApiSymbolToDirectus } from "./mapApiSymbolToDirectus.js";
 
 describe("mapApiSymbolToDirectus", () => {
   const baseOrigin = "https://tsed.dev";
   const markdownOrigin = "https://tsed.dev/ai/references";
-  const pkgId = "pkg-1";
 
   it("mappe les champs de base correctement", () => {
-    const input: ApiJsonModuleSymbol = {
+    const input: ApiSymbol = {
       id: "sym-1",
       path: "/api/core/Controller",
       symbolName: "Controller",
       module: "@tsed/core",
       symbolType: "class",
       status: []
-    };
+    } as any;
 
-    const out = mapApiSymbolToDirectus(pkgId, input, baseOrigin, markdownOrigin);
+    const out = mapApiSymbolToDirectus(input, baseOrigin, markdownOrigin);
 
     expect(out).toMatchObject({
       id: "sym-1",
@@ -25,38 +25,37 @@ describe("mapApiSymbolToDirectus", () => {
       doc_url: "https://tsed.dev/api/core/Controller",
       markdown_url: "https://tsed.dev/ai/references/api/core/Controller.md",
       versions: [],
-      package: pkgId,
       deprecated: false,
       tags: []
     });
   });
 
-  it("gère le statut deprecated → deprecated=true et tags=['deprecated']", () => {
-    const input: ApiJsonModuleSymbol = {
+  it("manages the deprecated status → deprecated=true and tags=[]", () => {
+    const input: ApiSymbol = {
       id: "sym-2",
       path: "/api/schema/UseJsonMapper.html",
       symbolName: "UseJsonMapper",
       module: "@tsed/schema",
       symbolType: "decorator",
       status: ["deprecated"]
-    };
+    } as any;
 
-    const out = mapApiSymbolToDirectus(pkgId, input, baseOrigin);
+    const out = mapApiSymbolToDirectus(input, baseOrigin);
 
     expect(out.deprecated).toBe(true);
-    expect(out.tags).toEqual(["deprecated"]);
+    expect(out.tags).toEqual([]);
   });
 
-  it("status non défini → deprecated=false et tags=[]", () => {
-    const input: ApiJsonModuleSymbol = {
+  it("status undefined → deprecated=false and tags=[]", () => {
+    const input: ApiSymbol = {
       id: "sym-3",
       path: "/api/schema/JsonEntityStore.html",
       symbolName: "JsonEntityStore",
       module: "@tsed/schema",
       symbolType: "class"
     } as any;
 
-    const out = mapApiSymbolToDirectus(pkgId, input, baseOrigin);
+    const out = mapApiSymbolToDirectus(input, baseOrigin);
 
     expect(out.deprecated).toBe(false);
     expect(out.tags).toEqual([]);

extensions/tsed-flow-package-symbols/src/mappers/mapApiSymbolToDirectus.ts

@@ -1,27 +1,6 @@
 import type { PackageSymbol } from "@tsed-cms/infra/directus/interfaces/DirectusSchema.js";
 
-// Typed response for https://tsed.dev/api.json (stable format)
-export type ApiJsonModuleSymbol = {
-  id: string;
-  path: string;
-  symbolName: string;
-  module: string;
-  symbolType: PackageSymbol["type"];
-  status?: string[];
-};
-
-export type ApiJsonModule = {
-  name: string;
-  symbols: ApiJsonModuleSymbol[];
-};
-
-export type ApiJsonResponse = {
-  version: string;
-  scope: string;
-  symbolTypes: { value: PackageSymbol["type"]; label: string; code: string }[];
-  symbolStatus: { value: string; label: string }[];
-  modules: Record<string, ApiJsonModule>;
-};
+import type { ApiSymbol } from "../schema/ApiPayloadSchema.js";
 
 /**
  * Build a valid PackageSymbol payload (without the `package` relation)
@@ -30,15 +9,15 @@ export type ApiJsonResponse = {
  * Responsibility: mapping fields and normalizing optional metadata only.
  * Caller is responsible for resolving/ensuring the `package` id.
  */
-export function mapApiSymbolToDirectus(symbol: ApiJsonModuleSymbol, baseOrigin: string, markdownUrl?: string) {
+export function mapApiSymbolToDirectus(symbol: ApiSymbol, baseOrigin: string, markdownUrl?: string) {
   const deprecated = symbol.status ? symbol.status.includes("deprecated") : false;
   const tags = (symbol.status || []).filter((t) => t !== "deprecated");
 
   return {
     id: symbol.id,
     status: "published",
     name: symbol.symbolName,
-    type: symbol.symbolType,
+    type: symbol.symbolType as PackageSymbol["type"],
     doc_url: `${baseOrigin}${symbol.path}`,
     markdown_url: `${markdownUrl}${symbol.path}.md`,
     additional_doc_url: "",

extensions/tsed-flow-package-symbols/src/schema/ApiPayloadSchema.ts

@@ -0,0 +1,41 @@
+import { s } from "@tsed/schema";
+
+const ApiSymbolTypeSchema = s
+  .object({
+    value: s.string().enum("decorator", "class", "enum", "function", "interface", "const", "service", "type").required(),
+    label: s.string().required(),
+    code: s.string().required()
+  })
+
+const ApiSymbolSchema = s
+  .object({
+    id: s.string().required(),
+    path: s.string().required(),
+    module: s.string().required(),
+    symbolName: s.string().required(),
+    symbolType: s.string().required(),
+    symbolCode: s.string().required(),
+    status: s.array(s.string()).required()
+  })
+
+export const ApiPayloadSchema = s
+  .object({
+    version: s.string().required(),
+    scope: s.string().required(),
+    symbolTypes: s.array(ApiSymbolTypeSchema).required(),
+    symbolStatus: s.array(
+      s.object({
+        label: s.string().required(),
+        value: s.string().required()
+      })
+    ),
+    modules: s.record(
+      s.object({
+        name: s.string().required(),
+        symbols: s.array(ApiSymbolSchema).required()
+      })
+    )
+  })
+
+export type ApiPayload = s.infer<typeof ApiPayloadSchema>;
+export type ApiSymbol = s.infer<typeof ApiSymbolSchema>;

migrations/data/directus_flows.json

@@ -154,5 +154,22 @@
     "operation": "c651e60f-159c-4b4d-b636-c9d50debe3de",
     "date_created": "2025-11-15T10:33:07.296Z",
     "user_created": "dc515c74-5ed5-4464-b522-6ea1f2f6d270"
+  },
+  {
+    "id": "9039bef8-fdc3-4f31-b5ab-7fee31273921",
+    "name": "Trigger update package symbols",
+    "icon": "bolt",
+    "color": null,
+    "description": null,
+    "status": "active",
+    "trigger": "webhook",
+    "accountability": "all",
+    "options": {
+      "async": true,
+      "method": "POST"
+    },
+    "operation": "a2391589-d955-4677-9ff5-1bb846cebf98",
+    "date_created": "2025-11-15T11:49:36.406Z",
+    "user_created": "dc515c74-5ed5-4464-b522-6ea1f2f6d270"
   }
 ]

migrations/data/directus_operations.json

@@ -123,5 +123,22 @@
     "flow": "569895a0-7a65-4cb2-94aa-67f77a776a08",
     "date_created": "2025-11-15T10:35:32.381Z",
     "user_created": "dc515c74-5ed5-4464-b522-6ea1f2f6d270"
+  },
+  {
+    "id": "a2391589-d955-4677-9ff5-1bb846cebf98",
+    "name": "Ts.ED Package Symbols importer",
+    "key": "tsed_flow_package_symbols_uvxzc",
+    "type": "tsed-flow-package-symbols",
+    "position_x": 19,
+    "position_y": 1,
+    "options": {
+      "url": "https://tsed.dev/api.json",
+      "markdown_url": "https://tsed.dev/ai/references/api"
+    },
+    "resolve": null,
+    "reject": null,
+    "flow": "9039bef8-fdc3-4f31-b5ab-7fee31273921",
+    "date_created": "2025-11-15T11:49:52.495Z",
+    "user_created": "dc515c74-5ed5-4464-b522-6ea1f2f6d270"
   }
 ]