Merge pull request #1 from tsedio/fix-improves-data

Fix improves data

Author: Romakita

extensions/env-info/src/endpoint/index.ts

@@ -7,7 +7,7 @@ import { wrapEndpoint } from "@tsed-cms/infra/bootstrap/directus.js";
 
 import { EnvInfoService } from "./EnvInfoService.js";
 
-configuration().set("pkg", JSON.parse(readFileSync(join(import.meta.dirname, "..", "package.json"), "utf8")));
+configuration().set("pkg", JSON.parse(readFileSync(join(import.meta.dirname, "..", "..", "..", "package.json"), "utf8")));
 configuration().set("branch", readFileSync(join(process.cwd(), "resources/release.info"), "utf8").trim());
 configuration().set("envs", process.env);
 

extensions/tsed-flow-package-symbols/README.md

@@ -0,0 +1,109 @@
+# tsed-flow-package-symbols
+
+Directus operation that imports exported symbols of Ts.ED packages from the contractual JSON `https://tsed.dev/api.json` and writes them into the `package_symbols` collection.
+
+## How it works
+
+- The operation fetches `api.json` (or consumes a compatible payload provided by a previous step, see “Webhook/inline payload” below).
+- It iterates `modules["<package-name>"].symbols` and maps each entry to a `package_symbols` record via `src/mappers/mapApiSymbolToDirectus.ts`.
+- The corresponding package is ensured in the `packages` collection (created if missing) with `type: "official"`.
+- Upsert is performed by the symbol `id` coming from `api.json`. If a record already exists, its `versions` field is merged (union) with the new version from `api.json`.
+- Fields mapped per symbol:
+  - `name ← symbolName`
+  - `type ← symbolType`
+  - `doc_url ← origin(api.json) + path`
+  - `markdown_url ← markdown_base + path + ".md"`
+  - `deprecated ← status includes "deprecated"`
+  - `tags ← status without "deprecated"`
+  - `versions ← []` then the global `api.json.version` is appended during import
+
+## Build and run locally
+
+1) Install deps at the repo root
+```bash
+corepack enable
+yarn install --immutable
+```
+
+2) Build all extensions
+```bash
+yarn build
+```
+
+3) Start Directus (dev)
+```bash
+yarn start:dev
+```
+
+## Operation configuration (UI)
+
+Directus → Flows → Add operation → “Ts.ED Package Symbols importer”.
+
+Card options:
+- `url` (string) — `api.json` URL. Default in the card: `https://tsed.dev/api.json`.
+- `markdown_url` (string) — base URL where markdown files live. Default in the card: `https://tsed.dev/ai/references/api`.
+
+Notes about defaults:
+- The operation handler also has an internal fallback for `markdown_url` to `https://tsed.dev/ai/references` if the option isn’t provided by the flow. To get the expected final markdown path `…/api/<symbol>.md`, set the card option explicitly to `https://tsed.dev/ai/references/api` (as in the exported flows).
+
+Save the flow. The operation will use these options at runtime.
+
+## Trigger the flow via HTTP (Directus API)
+
+Manual trigger (flow id `569895a0-7a65-4cb2-94aa-67f77a776a08` in the sample exports):
+
+```bash
+CMS_API_URL="http://localhost:8055"
+CMS_API_TOKEN="<ADMIN_OR_ALLOWED_TOKEN>"
+FLOW_ID="569895a0-7a65-4cb2-94aa-67f77a776a08"
+
+curl -X POST \
+  "$CMS_API_URL/flows/trigger/$FLOW_ID" \
+  -H "Authorization: Bearer $CMS_API_TOKEN" \
+  -H "Content-Type: application/json" \
+  -d '{}'
+```
+
+### Webhook trigger and inline payload (optional)
+
+This repo also includes a webhook flow (`9039bef8-fdc3-4f31-b5ab-7fee31273921`). If you POST a body containing a payload compatible with `src/schema/ApiPayloadSchema.ts`, the operation will validate and use this body instead of fetching from `url`.
+
+Example (send the whole api.json as request body):
+
+```bash
+CMS_API_URL="http://localhost:8055"
+CMS_API_TOKEN="<ADMIN_OR_ALLOWED_TOKEN>"
+FLOW_ID="9039bef8-fdc3-4f31-b5ab-7fee31273921"
+
+curl -X POST \
+  "$CMS_API_URL/flows/trigger/$FLOW_ID" \
+  -H "Authorization: Bearer $CMS_API_TOKEN" \
+  -H "Content-Type: application/json" \
+  --data-binary @api.json
+```
+
+## Permissions required
+
+This operation runs with the flow runner’s role/policy (`accountability: "all"` in the exported flows). Ensure the role used to trigger the flow has at least:
+
+- `packages`: read, create, update (the operation may create missing packages)
+- `package_symbols`: read, create, update (the operation reads by `id`, then creates/updates symbols)
+
+If these permissions are missing, the flow will fail with `403 You don't have permission to access this.` during upsert.
+
+## Testing
+
+Run the unit tests for the mapper and the service:
+
+```bash
+yarn vitest run \
+  extensions/tsed-flow-package-symbols/src/mappers/mapApiSymbolToDirectus.spec.ts \
+  packages/usecases/package-symbols/PackageSymbolsService.spec.ts
+```
+
+Both tests should pass.
+
+## Notes
+
+- The `ApiPayloadSchema` strictly types the `api.json` format used by the operation.
+- The mapper intentionally keeps logic minimal and assumes the contract is stable.

extensions/tsed-flow-package-symbols/package.json

@@ -0,0 +1,36 @@
+{
+  "name": "tsed-flow-package-symbols",
+  "description": "Flow operation to import Ts.ED exported symbols into Directus package_symbols collection",
+  "icon": "code",
+  "version": "1.0.0",
+  "keywords": [
+    "directus",
+    "directus-extension",
+    "directus-extension-operation"
+  ],
+  "type": "module",
+  "files": [
+    "dist"
+  ],
+  "directus:extension": {
+    "type": "operation",
+    "path": {
+      "app": "dist/app.js",
+      "api": "dist/api.cjs"
+    },
+    "source": {
+      "app": "src/app.ts",
+      "api": "src/api.ts"
+    },
+    "host": "^10.10.0"
+  },
+  "scripts": {
+    "build": "directus-extension build",
+    "dev": "directus-extension build --watch --no-minify",
+    "link": "directus-extension link",
+    "add": "directus-extension add"
+  },
+  "devDependencies": {
+    "@directus/extensions-sdk": "13.1.1"
+  }
+}

extensions/tsed-flow-package-symbols/src/api.ts

@@ -0,0 +1,98 @@
+import { defineOperationApi } from "@directus/extensions-sdk";
+import { inject } from "@tsed/di";
+import { wrapOperation } from "@tsed-cms/infra/bootstrap/directus.js";
+import type { Package } from "@tsed-cms/infra/directus/interfaces/DirectusSchema.js";
+import { HttpClient } from "@tsed-cms/infra/http/HttpClient.js";
+import { validate } from "@tsed-cms/infra/validators/validate.js";
+import { PackageSymbolsService } from "@tsed-cms/usecases/package-symbols/PackageSymbolsService.js";
+import { PackagesService } from "@tsed-cms/usecases/packages/PackagesService.js";
+
+import { mapApiSymbolToDirectus } from "./mappers/mapApiSymbolToDirectus.js";
+import { type ApiPayload, ApiPayloadSchema } from "./schema/ApiPayloadSchema.js";
+
+export type Options = {
+  url?: string;
+  markdown_url?: string;
+};
+
+async function ensurePackage(pkgName: string): Promise<Package> {
+  const packagesService = inject(PackagesService);
+
+  const existing = await packagesService.findByName(pkgName);
+
+  if (existing) {
+    return existing;
+  }
+
+  // Tous les modules listés dans api.json sont traités comme "official"
+  return packagesService.upsertOne({ name: pkgName, type: "official" });
+}
+
+export default defineOperationApi<Options>({
+  id: "tsed-flow-package-symbols",
+  handler: wrapOperation(async (opts, context) => {
+    const startedAt = Date.now();
+    const url = (opts?.url?.trim() || "https://tsed.dev/api.json").toString();
+    const markdownUrl = (opts?.markdown_url?.trim() || "https://tsed.dev/ai/references").toString();
+    const http = inject(HttpClient);
+    const symbolsService = inject(PackageSymbolsService);
+
+    // Origin pour construire les URL de doc
+    const origin = new URL(url).origin;
+    let data: ApiPayload;
+
+    if ((context?.data?.["$last"] as any)?.body) {
+      try {
+        data = await validate((context?.data?.["$last"] as any)?.body, ApiPayloadSchema);
+      } catch (er) {
+        return {
+          url,
+          processed: 0,
+          upserted: 0,
+          durationMs: Date.now() - startedAt,
+          errors: [
+            {
+              error: er?.message || String(er)
+            }
+          ]
+        };
+      }
+    } else {
+      data = await http.get<ApiPayload>(url);
+    }
+
+    let processed = 0;
+    let upserted = 0;
+    const errors: { name: string; pkg: string; error: string }[] = [];
+
+    for (const [pkgName, mod] of Object.entries(data.modules)) {
+      for (const s of mod.symbols) {
+        try {
+          const pkg = await ensurePackage(pkgName);
+          const mapped = mapApiSymbolToDirectus(s, origin, markdownUrl);
+
+          mapped.versions.push(data.version);
+
+          await symbolsService.upsertOne({ ...mapped, package: pkg.id });
+
+          upserted += 1;
+          processed += 1;
+        } catch (er: any) {
+          errors.push({
+            name: s.symbolName,
+            pkg: pkgName,
+            error: er?.message || String(er)
+          });
+        }
+      }
+    }
+
+    return {
+      url,
+      processed,
+      upserted,
+      durationMs: Date.now() - startedAt,
+      errors
+    };
+  })
+});

extensions/tsed-flow-package-symbols/src/app.ts

@@ -0,0 +1,42 @@
+import { defineOperationApp } from "@directus/extensions-sdk";
+
+export default defineOperationApp({
+  id: "tsed-flow-package-symbols",
+  name: "Ts.ED Package Symbols importer",
+  icon: "code",
+  description: "Retrieves exported symbols from Ts.ED packages from api.json and upserts them into package_symbols.",
+  overview: ({ url }) => [
+    {
+      label: "API URL",
+      text: url || "https://tsed.dev/api.json"
+    }
+  ],
+  options: [
+    {
+      field: "url",
+      name: "API URL",
+      type: "string",
+      meta: {
+        width: "full",
+        interface: "input",
+        note: "URL of JSON symbols (default: https://tsed.dev/api.json)"
+      },
+      schema: {
+        default_value: "https://tsed.dev/api.json"
+      }
+    },
+    {
+      field: "markdown_url",
+      name: "Markdown base URL",
+      type: "string",
+      meta: {
+        width: "full",
+        interface: "input",
+        note: "URL where the markdown contents of the symbols are stored (default: https://tsed.dev/ai/references/api)"
+      },
+      schema: {
+        default_value: "https://tsed.dev/ai/references/api"
+      }
+    }
+  ]
+});

extensions/tsed-flow-package-symbols/src/mappers/mapApiSymbolToDirectus.spec.ts

@@ -0,0 +1,63 @@
+import type { ApiSymbol } from "../schema/ApiPayloadSchema.js";
+import { mapApiSymbolToDirectus } from "./mapApiSymbolToDirectus.js";
+
+describe("mapApiSymbolToDirectus", () => {
+  const baseOrigin = "https://tsed.dev";
+  const markdownOrigin = "https://tsed.dev/ai/references";
+
+  it("mappe les champs de base correctement", () => {
+    const input: ApiSymbol = {
+      id: "sym-1",
+      path: "/api/core/Controller",
+      symbolName: "Controller",
+      module: "@tsed/core",
+      symbolType: "class",
+      status: []
+    } as any;
+
+    const out = mapApiSymbolToDirectus(input, baseOrigin, markdownOrigin);
+
+    expect(out).toMatchObject({
+      id: "sym-1",
+      status: "published",
+      name: "Controller",
+      type: "class",
+      doc_url: "https://tsed.dev/api/core/Controller",
+      markdown_url: "https://tsed.dev/ai/references/api/core/Controller.md",
+      versions: [],
+      deprecated: false,
+      tags: []
+    });
+  });
+
+  it("manages the deprecated status → deprecated=true and tags=[]", () => {
+    const input: ApiSymbol = {
+      id: "sym-2",
+      path: "/api/schema/UseJsonMapper.html",
+      symbolName: "UseJsonMapper",
+      module: "@tsed/schema",
+      symbolType: "decorator",
+      status: ["deprecated"]
+    } as any;
+
+    const out = mapApiSymbolToDirectus(input, baseOrigin);
+
+    expect(out.deprecated).toBe(true);
+    expect(out.tags).toEqual([]);
+  });
+
+  it("status undefined → deprecated=false and tags=[]", () => {
+    const input: ApiSymbol = {
+      id: "sym-3",
+      path: "/api/schema/JsonEntityStore.html",
+      symbolName: "JsonEntityStore",
+      module: "@tsed/schema",
+      symbolType: "class"
+    } as any;
+
+    const out = mapApiSymbolToDirectus(input, baseOrigin);
+
+    expect(out.deprecated).toBe(false);
+    expect(out.tags).toEqual([]);
+  });
+});

extensions/tsed-flow-package-symbols/src/mappers/mapApiSymbolToDirectus.ts

@@ -0,0 +1,28 @@
+import type { PackageSymbol } from "@tsed-cms/infra/directus/interfaces/DirectusSchema.js";
+
+import type { ApiSymbol } from "../schema/ApiPayloadSchema.js";
+
+/**
+ * Build a valid PackageSymbol payload (without the `package` relation)
+ * from a single ApiJsonModuleSymbol item.
+ *
+ * Responsibility: mapping fields and normalizing optional metadata only.
+ * Caller is responsible for resolving/ensuring the `package` id.
+ */
+export function mapApiSymbolToDirectus(symbol: ApiSymbol, baseOrigin: string, markdownUrl?: string) {
+  const deprecated = symbol.status ? symbol.status.includes("deprecated") : false;
+  const tags = (symbol.status || []).filter((t) => t !== "deprecated");
+
+  return {
+    id: symbol.id,
+    status: "published",
+    name: symbol.symbolName,
+    type: symbol.symbolType as PackageSymbol["type"],
+    doc_url: `${baseOrigin}${symbol.path}`,
+    markdown_url: `${markdownUrl}${symbol.path}.md`,
+    additional_doc_url: "",
+    versions: [] as string[],
+    deprecated,
+    tags
+  } satisfies Omit<PackageSymbol, "package" | "user_created" | "date_created" | "user_updated" | "date_updated">;
+}