breaking change multi source (#390)

Co-authored-by: Guillaume Bourdat <[email protected]>

Author: JeromeBu

.vscode/launch.json

@@ -22,7 +22,7 @@
             "cwd": "${workspaceFolder}/api",
             "runtimeArgs": [
                 "run",
-                "import"
+                "job:import"
             ],
         },
         {
@@ -33,7 +33,28 @@
             "cwd": "${workspaceFolder}/api",
             "runtimeArgs": [
                 "run",
-                "update"
+                "job:update"
+            ],
+        },
+        {
+            "type": "node",
+            "request": "launch",
+            "name": "Fullcheck",
+            "runtimeExecutable": "yarn",
+            "cwd": "${workspaceFolder}",
+            "runtimeArgs": [
+                "fullcheck"
+            ],
+        },
+        {
+            "type": "node",
+            "request": "launch",
+            "name": "Import From Inner Identifers",
+            "runtimeExecutable": "yarn",
+            "cwd": "${workspaceFolder}/api",
+            "runtimeArgs": [
+                "run",
+                "job:import-from-inner-identifiers"
             ],
         }
     ]

api/package.json

@@ -10,24 +10,28 @@
     "types": "dist/src/lib/index.d.ts",
     "scripts": {
         "migrate": "dotenv -e ../.env -- kysely migrate",
-        "migrate:down": "dotenv -e ../.env -- kysely migrate down",
         "db:up": "yarn migrate latest",
         "test": "vitest --watch=false --no-file-parallelism",
         "dev": "yarn build && yarn start",
         "generate-translation-schema": "node scripts/generate-translation-schema.js",
         "build": "tsc && cp -r src/customization dist/src/",
         "start": "yarn db:up && dotenv -e ../.env -- node dist/src/entrypoints/start-api.js",
-        "update": "dotenv -e ../.env -- node dist/src/entrypoints/update.js",
+        "job:import": "dotenv -e ../.env -- node dist/src/entrypoints/import.js",
+        "job:import-from-inner-identifiers": "dotenv -e ../.env -- node dist/src/entrypoints/import-from-inner-identifiers.js",
+        "job:update": "dotenv -e ../.env -- node dist/src/entrypoints/update.js",
         "update-then-wait": "./update-then-wait.sh",
-        "import": "dotenv -e ../.env -- node dist/src/entrypoints/import.js",
         "_format": "prettier \"**/*.{ts,tsx,json,md}\"",
         "format": "yarn run _format --write",
         "format:check": "yarn run _format --list-different",
         "link-in-web": "ts-node --skipProject scripts/link-in-app.ts sill-web",
         "db:seed": "yarn build && dotenv -e ../.env -- node dist/scripts/seed.js",
         "typecheck": "tsc --noEmit",
-        "dev:db": "docker compose -f ../docker-compose.resources.yml up -d",
-        "dev:reset": "docker compose -f ../docker-compose.resources.yml down && rm -rf ../docker-data && docker compose -f ../docker-compose.resources.yml up -d",
+        "dev:db:up": "docker compose -f ../docker-compose.resources.yml up -d",
+        "dev:db:down": "docker compose -f ../docker-compose.resources.yml down",
+        "dev:db:flush": "rm -rf ../docker-data",
+        "dev:db:reset": "yarn dev:db:down && yarn dev:db:flush && yarn dev:db:up",
+        "dev:dbs:mig:up": "dotenv -e ../.env -- kysely migrate up",
+        "dev:dbs:mig:down": "dotenv -e ../.env -- kysely migrate down",
         "knip": "knip"
     },
     "author": "DINUM",
@@ -73,7 +77,7 @@
         "deepmerge": "^4.3.1",
         "generate-schema": "^2.6.0",
         "jsdom": "^26.0.0",
-        "kysely": "^0.27.4",
+        "kysely": "^0.28.2",
         "kysely-ctl": "^0.12.0",
         "pg": "^8.11.5",
         "tsafe": "^1.6.6",

api/scripts/seed.ts

@@ -11,6 +11,7 @@ import { Database } from "../src/core/adapters/dbApi/kysely/kysely.database";
 import { createPgDialect } from "../src/core/adapters/dbApi/kysely/kysely.dialect";
 import { SoftwareFormData, Source } from "../src/lib/ApiTypes";
 import { OmitFromExisting } from "../src/core/utils";
+import { makeCreateSofware } from "../src/core/usecases/createSoftware";
 
 const seed = async () => {
     const dbUrl = process.env.DATABASE_URL;
@@ -35,7 +36,7 @@ const seed = async () => {
     console.info("Adding source");
     const source = {
         slug: "wikidata",
-        description: null,
+        description: undefined,
         url: "https://www.wikidata.org/",
         kind: "wikidata",
         priority: 1
@@ -50,6 +51,8 @@ const seed = async () => {
         sub: null
     };
 
+    const UCCreateSofware = makeCreateSofware(dbApi);
+
     console.info("Adding user");
     const userId = await dbApi.user.add(someUser);
 
@@ -63,7 +66,6 @@ const seed = async () => {
             },
             externalIdForSource: undefined,
             sourceSlug: "wikidata",
-            comptoirDuLibreId: undefined,
             softwareLicense: "MIT",
             softwareMinimalVersion: "18.0.0",
             similarSoftwareExternalDataIds: [],
@@ -82,7 +84,6 @@ const seed = async () => {
             },
             externalIdForSource: undefined,
             sourceSlug: "wikidata",
-            comptoirDuLibreId: undefined,
             softwareLicense: "GPL-2.0",
             softwareMinimalVersion: "2.0.0",
             similarSoftwareExternalDataIds: [],
@@ -102,7 +103,6 @@ const seed = async () => {
             },
             externalIdForSource: undefined,
             sourceSlug: "wikidata",
-            comptoirDuLibreId: undefined,
             softwareLicense: "Apache-2.0",
             softwareMinimalVersion: "4.1.0",
             similarSoftwareExternalDataIds: [],
@@ -121,7 +121,6 @@ const seed = async () => {
             },
             externalIdForSource: undefined,
             sourceSlug: "wikidata",
-            comptoirDuLibreId: undefined,
             softwareLicense: "GPL-2.0",
             softwareMinimalVersion: "3.0.0",
             similarSoftwareExternalDataIds: [],
@@ -140,7 +139,6 @@ const seed = async () => {
             },
             externalIdForSource: undefined,
             sourceSlug: "wikidata",
-            comptoirDuLibreId: undefined,
             softwareLicense: "GPL-3.0",
             softwareMinimalVersion: "2.10.0",
             similarSoftwareExternalDataIds: [],
@@ -159,7 +157,6 @@ const seed = async () => {
             },
             externalIdForSource: "Q110492908",
             sourceSlug: "wikidata",
-            comptoirDuLibreId: 461,
             softwareLicense: "MIT",
             softwareMinimalVersion: "0.26.25",
             similarSoftwareExternalDataIds: [],
@@ -173,7 +170,7 @@ const seed = async () => {
     ];
 
     for (const formData of softwarePackagesFormData) {
-        await dbApi.software.create({ userId, formData });
+        await UCCreateSofware({ userId, formData });
     }
 
     // Add instances for Onyxia

api/src/core/adapters/CNLL/getExternalData.ts

@@ -0,0 +1,75 @@
+// SPDX-FileCopyrightText: 2021-2025 DINUM <[email protected]>
+// SPDX-FileCopyrightText: 2024-2025 Université Grenoble Alpes
+// SPDX-License-Identifier: MIT
+
+import memoize from "memoizee";
+
+import { GetSoftwareExternalData, SoftwareExternalData } from "../../ports/GetSoftwareExternalData";
+import { Source } from "../../usecases/readWriteSillData";
+import { SchemaOrganization } from "../dbApi/kysely/kysely.database";
+import { identifersUtils } from "../../../tools/identifiersTools";
+import { getCnllPrestatairesSill } from "../getCnllPrestatairesSill";
+import { CnllPrestatairesSill } from "../../ports/GetCnllPrestatairesSill";
+
+export const getCNLLSoftwareExternalData: GetSoftwareExternalData = memoize(
+    async ({
+        externalId,
+        source
+    }: {
+        externalId: string;
+        source: Source;
+    }): Promise<SoftwareExternalData | undefined> => {
+        if (source.kind !== "CNLL") throw new Error("This source if not compatible with CNLL Adapter");
+
+        const cnllProviders = await getCnllPrestatairesSill();
+
+        const providersForExternalId = cnllProviders.find(element => element.sill_id.toString() === externalId);
+
+        if (!providersForExternalId) return undefined;
+
+        return formatCNLLProvidersToExternalData(providersForExternalId, source);
+    }
+);
+
+const cnllProviderToCMProdivers = (provider: CnllPrestatairesSill.Prestataire): SchemaOrganization => {
+    return {
+        "@type": "Organization" as const,
+        name: provider.nom,
+        url: provider.url ?? undefined,
+        identifiers: [
+            identifersUtils.makeSIRENIdentifier({
+                SIREN: provider.siren,
+                additionalType: "Organization"
+            })
+        ]
+    };
+};
+
+const formatCNLLProvidersToExternalData = (
+    cnllProdivers: CnllPrestatairesSill,
+    source: Source
+): SoftwareExternalData => ({
+    externalId: cnllProdivers.sill_id.toString(),
+    sourceSlug: source.slug,
+    developers: [],
+    label: { "fr": cnllProdivers.nom },
+    description: { "fr": "" },
+    isLibreSoftware: true,
+    logoUrl: undefined,
+    websiteUrl: undefined,
+    sourceUrl: undefined,
+    documentationUrl: undefined,
+    license: undefined,
+    softwareVersion: undefined,
+    keywords: [],
+    programmingLanguages: [],
+    applicationCategories: [],
+    publicationTime: undefined,
+    referencePublications: [],
+    identifiers: [
+        identifersUtils.makeCNLLIdentifier({
+            cNNLId: cnllProdivers.sill_id.toString()
+        })
+    ],
+    providers: cnllProdivers.prestataires.map(prodiver => cnllProviderToCMProdivers(prodiver))
+});

api/src/core/adapters/CNLL/index.ts

@@ -0,0 +1,14 @@
+// SPDX-FileCopyrightText: 2021-2025 DINUM <[email protected]>
+// SPDX-FileCopyrightText: 2024-2025 Université Grenoble Alpes
+// SPDX-License-Identifier: MIT
+
+import { SecondarySourceGateway } from "../../ports/SourceGateway";
+import { getCNLLSoftwareExternalData } from "./getExternalData";
+
+export const cnllSourceGateway: SecondarySourceGateway = {
+    sourceType: "ComptoirDuLibre",
+    sourceProfile: "Secondary",
+    softwareExternalData: {
+        getById: getCNLLSoftwareExternalData
+    }
+};

api/src/core/adapters/comptoirDuLibre/getCDLExternalData.ts

@@ -0,0 +1,112 @@
+// SPDX-FileCopyrightText: 2021-2025 DINUM <[email protected]>
+// SPDX-FileCopyrightText: 2024-2025 Université Grenoble Alpes
+// SPDX-License-Identifier: MIT
+
+import memoize from "memoizee";
+
+import { GetSoftwareExternalData, SoftwareExternalData } from "../../ports/GetSoftwareExternalData";
+import { Source } from "../../usecases/readWriteSillData";
+import { comptoirDuLibreApi } from "../comptoirDuLibreApi";
+import { ComptoirDuLibre } from "../../ports/ComptoirDuLibreApi";
+import { SchemaOrganization } from "../dbApi/kysely/kysely.database";
+import { identifersUtils } from "../../../tools/identifiersTools";
+
+export const getCDLSoftwareExternalData: GetSoftwareExternalData = memoize(
+    async ({
+        externalId,
+        source
+    }: {
+        externalId: string;
+        source: Source;
+    }): Promise<SoftwareExternalData | undefined> => {
+        const comptoirAPi = await comptoirDuLibreApi.getComptoirDuLibre();
+
+        const comptoirSoftware = comptoirAPi.softwares.find(softwareItem => softwareItem.id.toString() === externalId);
+
+        if (!comptoirSoftware) return undefined;
+
+        return formatCDLSoftwareToExternalData(comptoirSoftware, source);
+    },
+    {
+        maxAge: 3 * 3600 * 1000
+    }
+);
+
+const cdlProviderToCMProdivers = (provider: ComptoirDuLibre.Provider): SchemaOrganization => {
+    return {
+        "@type": "Organization" as const,
+        name: provider.name,
+        url: provider.external_resources.website ?? undefined,
+        identifiers: [
+            identifersUtils.makeCDLIdentifier({
+                cdlId: provider.id.toString(),
+                url: provider.url,
+                additionalType: "Organization"
+            })
+        ]
+    };
+};
+
+const formatCDLSoftwareToExternalData = (
+    cdlSoftwareItem: ComptoirDuLibre.Software,
+    source: Source
+): SoftwareExternalData => {
+    const splittedCNLLUrl = !Array.isArray(cdlSoftwareItem.external_resources.cnll)
+        ? cdlSoftwareItem.external_resources.cnll.url.split("/")
+        : undefined;
+
+    return {
+        externalId: cdlSoftwareItem.id.toString(),
+        sourceSlug: source.slug,
+        developers: [],
+        label: { "fr": cdlSoftwareItem.name },
+        description: { "fr": "" },
+        isLibreSoftware: true,
+        //
+        logoUrl: undefined, // Use scrapper ?
+        websiteUrl: cdlSoftwareItem.external_resources.website ?? undefined,
+        sourceUrl: cdlSoftwareItem.external_resources.repository ?? undefined,
+        documentationUrl: undefined,
+        license: cdlSoftwareItem.licence,
+        softwareVersion: undefined,
+        keywords: [],
+        programmingLanguages: [],
+        applicationCategories: [],
+        publicationTime: undefined,
+        referencePublications: [],
+        identifiers: [
+            identifersUtils.makeCDLIdentifier({
+                cdlId: cdlSoftwareItem.id.toString(),
+                url: cdlSoftwareItem.url,
+                additionalType: "Software"
+            }),
+            ...(!Array.isArray(cdlSoftwareItem.external_resources.cnll) && splittedCNLLUrl
+                ? [
+                      identifersUtils.makeCNLLIdentifier({
+                          cNNLId: splittedCNLLUrl[splittedCNLLUrl.length - 1],
+                          url: cdlSoftwareItem.external_resources.cnll.url
+                      })
+                  ]
+                : []),
+            ...(!Array.isArray(cdlSoftwareItem.external_resources.framalibre)
+                ? [
+                      identifersUtils.makeFramaIndentifier({
+                          framaLibreId: cdlSoftwareItem.external_resources.framalibre.slug,
+                          url: cdlSoftwareItem.external_resources.framalibre.url,
+                          additionalType: "Software"
+                      })
+                  ]
+                : []),
+            ...(!Array.isArray(cdlSoftwareItem.external_resources.wikidata)
+                ? [
+                      identifersUtils.makeWikidataIdentifier({
+                          wikidataId: cdlSoftwareItem.external_resources.wikidata.id,
+                          url: cdlSoftwareItem.external_resources.wikidata.url,
+                          additionalType: "Software"
+                      })
+                  ]
+                : [])
+        ],
+        providers: cdlSoftwareItem.providers.map(prodiver => cdlProviderToCMProdivers(prodiver))
+    };
+};