[PARKED] feature: Image Description With AI

package.json

@@ -268,6 +268,8 @@
     }
   },
   "dependencies": {
+    "@ai-sdk/mistral": "^1.1.17",
+    "@ai-sdk/openai": "^1.2.5",
     "@radix-ui/react-alert-dialog": "^1.1.6",
     "@radix-ui/react-dialog": "^1.1.6",
     "@radix-ui/react-popover": "^1.1.6",
@@ -278,6 +280,7 @@
     "@react-hook/resize-observer": "^2.0.2",
     "@redux-saga/core": "^1.3.0",
     "@xmldom/xmldom": "^0.9.8",
+    "ai": "^4.1.61",
     "ajv": "^8.17.1",
     "ajv-formats": "^3.0.1",
     "classnames": "^2.5.1",
@@ -293,6 +296,7 @@
     "jsdom": "^26.0.0",
     "lunr": "^2.3.9",
     "lunr-languages": "^1.14.0",
+    "marked": "^15.0.7",
     "match-sorter": "^8.0.0",
     "mathjax": "^3.2.2",
     "mime-types": "^2.1.35",

src/common/aisdkModelOptions.ts

@@ -0,0 +1,40 @@
+// ==LICENSE-BEGIN==
+// Copyright 2017 European Digital Reading Lab. All rights reserved.
+// Licensed to the Readium Foundation under one or more contributor license agreements.
+// Use of this source code is governed by a BSD-style license
+// that can be found in the LICENSE file exposed on Github (readium) in the project repository.
+// ==LICENSE-END==
+
+export const DEFAULT_SYSTEM_PROMPT = "Your goal is to describe the image, you should not answer on a topic other than this image. Answer all requests in {{languages}} unless I explicitly ask you otherwise.";
+const ADVANCED_SYSTEM_PROMPT = {
+    goal: "describe the image, you should not answer on a topic other than this image. Answer all requests in {{languages}} unless I explicitly ask you otherwise",
+    context: {
+        title: "{{title}}",
+        author: "{{author}}",
+        publisher: "{{publisher}}",
+        languages: "{{languages}}",
+        text_before: "{{beforeText}}",
+        text_after: "{{afterText}}",
+    },
+};
+
+const ADVANCED_SYSTEM_PROMPT_STRING = JSON.stringify(ADVANCED_SYSTEM_PROMPT);
+
+export interface IaiSdkModel { id: string, name: string, systemPrompt: string };
+export const aiSDKModelOptions: Array<IaiSdkModel> = [
+    {
+        id: "openai__!__gpt-4o-mini__!__default-prompt",
+        name: "openAI gpt-4o-mini (default)",
+        systemPrompt: DEFAULT_SYSTEM_PROMPT,
+    },
+    {
+        id: "openai__!__gpt-4o-mini__!__specific-prompt",
+        name: "openAI gpt-4o-mini (advanced)",
+        systemPrompt: ADVANCED_SYSTEM_PROMPT_STRING,
+    },
+    {
+        id: "mistralai__!__pixtral-12b-2409",
+        name: "mistralAI Pixtral 12B",
+        systemPrompt: DEFAULT_SYSTEM_PROMPT,
+    },
+];

src/common/redux/actions/api_key/index.ts

@@ -0,0 +1,15 @@
+// ==LICENSE-BEGIN==
+// Copyright 2017 European Digital Reading Lab. All rights reserved.
+// Licensed to the Readium Foundation under one or more contributor license agreements.
+// Use of this source code is governed by a BSD-style license
+// that can be found in the LICENSE file exposed on Github (readium) in the project repository.
+// ==LICENSE-END==
+
+// import * as enable from "./enable";
+import * as set from "./setKey";
+import * as removeKey from "./removeKey";
+
+export {
+    set,
+    removeKey,
+};

src/common/redux/actions/api_key/removeKey.ts

@@ -0,0 +1,27 @@
+// ==LICENSE-BEGIN==
+// Copyright 2017 European Digital Reading Lab. All rights reserved.
+// Licensed to the Readium Foundation under one or more contributor license agreements.
+// Use of this source code is governed by a BSD-style license
+// that can be found in the LICENSE file exposed on Github (readium) in the project repository.
+// ==LICENSE-END==
+
+import { Action } from "readium-desktop/common/models/redux";
+import { IApiKey } from "../../states/api_key";
+
+export const ID = "API_KEY_REMOVE";
+
+// eslint-disable-next-line @typescript-eslint/no-empty-interface
+// interface IPayload extends Pick<IAnnotationState, "uuid"> {
+interface IPayload extends IApiKey {
+}
+
+export function build(param: IApiKey):
+    Action<typeof ID, IPayload> {
+
+    return {
+        type: ID,
+        payload: {...param},
+    };
+}
+build.toString = () => ID;
+export type TAction = ReturnType<typeof build>;

src/common/redux/actions/api_key/setKey.ts

@@ -0,0 +1,28 @@
+// ==LICENSE-BEGIN==
+// Copyright 2017 European Digital Reading Lab. All rights reserved.
+// Licensed to the Readium Foundation under one or more contributor license agreements.
+// Use of this source code is governed by a BSD-style license
+// that can be found in the LICENSE file exposed on Github (readium) in the project repository.
+// ==LICENSE-END==
+
+import { Action } from "readium-desktop/common/models/redux";
+import { IApiKey } from "../../states/api_key";
+
+export const ID = "API_KEY_SET";
+
+export interface Payload extends IApiKey {
+}
+
+export function build(key: string, provider: string, submitted: boolean): Action<typeof ID, Payload> {
+
+    return {
+        type: ID,
+        payload: {
+            key,
+            provider,
+            submitted,
+        },
+    };
+}
+build.toString = () => ID;
+export type TAction = ReturnType<typeof build>;

src/common/redux/actions/index.ts

@@ -28,6 +28,7 @@ import * as versionUpdateActions from "./version-update";
 import * as annotationActions from "./annotation";
 import * as creatorActions from "./creator";
 import * as settingsActions from "./settings";
+import * as apiKeysActions from "./api_key";
 
 export {
     historyActions,
@@ -53,4 +54,5 @@ export {
     annotationActions,
     creatorActions,
     settingsActions,
+    apiKeysActions,
 };

src/common/redux/reducers/api_key.ts

@@ -0,0 +1,40 @@
+// ==LICENSE-BEGIN==
+// Copyright 2017 European Digital Reading Lab. All rights reserved.
+// Licensed to the Readium Foundation under one or more contributor license agreements.
+// Use of this source code is governed by a BSD-style license
+// that can be found in the LICENSE file exposed on Github (readium) in the project repository.
+// ==LICENSE-END==
+
+import { type Reducer } from "redux";
+
+import { apiKeysActions } from "../actions";
+import { IApiKeysArray } from "../states/api_key";
+
+const initialState: IApiKeysArray = [];
+
+function apiKeysReducer_(
+    state = initialState,
+    action: apiKeysActions.set.TAction | apiKeysActions.removeKey.TAction,
+): IApiKeysArray {
+
+    switch (action.type) {
+        case apiKeysActions.set.ID:
+            return [
+                ...state,
+                {
+                    provider: action.payload.provider,
+                    key: action.payload.key,
+                    submitted: action.payload.submitted,
+                },
+            ];
+        case apiKeysActions.removeKey.ID:
+            return state.filter(
+                key => key.provider !== action.payload.provider || key.key !== action.payload.key,
+            );
+        default:
+            return state;
+    }
+}
+
+
+export const apiKeysReducer = apiKeysReducer_ as Reducer<ReturnType<typeof apiKeysReducer_>>;

src/common/redux/states/api_key.ts

@@ -0,0 +1,14 @@
+// ==LICENSE-BEGIN==
+// Copyright 2017 European Digital Reading Lab. All rights reserved.
+// Licensed to the Readium Foundation under one or more contributor license agreements.
+// Use of this source code is governed by a BSD-style license
+// that can be found in the LICENSE file exposed on Github (readium) in the project repository.
+// ==LICENSE-END==
+
+export interface IApiKey {
+    key: string,
+    provider: string,
+    submitted: boolean,
+}
+
+export type IApiKeysArray = IApiKey[]

src/common/redux/states/commonRootState.ts

@@ -15,6 +15,7 @@ import { INoteCreator } from "./creator";
 import { I18NState } from "readium-desktop/common/redux/states/i18n";
 import { TFIFOState } from "readium-desktop/utils/redux-reducers/fifo.reducer";
 import { IAnnotationPreParsingState } from "./renderer/annotation";
+import { IApiKeysArray } from "readium-desktop/common/redux/states/api_key";
 
 export interface ICommonRootState {
     i18n: I18NState;
@@ -28,4 +29,5 @@ export interface ICommonRootState {
     theme: ITheme;
     creator: INoteCreator;
     annotationImportQueue: TFIFOState<IAnnotationPreParsingState>;
+    apiKeys: IApiKeysArray;
 }

src/main/redux/middleware/persistence.ts

@@ -44,6 +44,7 @@ export const reduxPersistMiddleware: Middleware
                     settings: prevState.settings,
                     creator: prevState.creator,
                     annotationImportQueue: prevState.annotationImportQueue,
+                    apiKeys: prevState.apiKeys,
                 };
 
                 const persistNextState: PersistRootState = {
@@ -64,6 +65,7 @@ export const reduxPersistMiddleware: Middleware
                     settings: nextState.settings,
                     creator: nextState.creator,
                     annotationImportQueue: nextState.annotationImportQueue,
+                    apiKeys: nextState.apiKeys,
                 };
 
                 // RangeError: Maximum call stack size exceeded

src/main/redux/middleware/sync.ts

@@ -13,7 +13,7 @@ import {
     publicationActions, themeActions,
     readerActions, sessionActions, toastActions, versionUpdateActions,
     creatorActions,
-    annotationActions,
+    annotationActions, apiKeysActions,
 } from "readium-desktop/common/redux/actions";
 import { ActionSerializer } from "readium-desktop/common/services/serializer";
 import { getLibraryWindowFromDi, getReaderWindowFromDi } from "readium-desktop/main/di";
@@ -94,6 +94,9 @@ const SYNCHRONIZABLE_ACTIONS: string[] = [
     annotationActions.shiftFromAnnotationImportQueue.ID,
 
     readerActions.setTheLock.ID,
+
+    apiKeysActions.set.ID,
+    apiKeysActions.removeKey.ID,
 ];
 
 export const reduxSyncMiddleware: Middleware

src/main/redux/reducers/index.ts

@@ -33,6 +33,7 @@ import { creatorReducer } from "readium-desktop/common/redux/reducers/creator";
 import { settingsReducer } from "readium-desktop/common/redux/reducers/settings";
 import { fifoReducer } from "readium-desktop/utils/redux-reducers/fifo.reducer";
 import { IAnnotationPreParsingState } from "readium-desktop/common/redux/states/renderer/annotation";
+import { apiKeysReducer } from "readium-desktop/common/redux/reducers/api_key";
 
 export const rootReducer = combineReducers({ // RootState
     versionUpdate: versionUpdateReducer,
@@ -122,5 +123,5 @@ export const rootReducer = combineReducers({ // RootState
             },
         },
     ),
-
+    apiKeys: apiKeysReducer,
 });

src/main/redux/sagas/persist.ts

@@ -44,6 +44,7 @@ const persistStateToFs = async (nextState: RootState) => {
         settings: nextState.settings,
         creator: nextState.creator,
         annotationImportQueue: nextState.annotationImportQueue,
+        apiKeys: nextState.apiKeys,
     };
 
     await fsp.writeFile(stateFilePath, JSON.stringify(value), {encoding: "utf8"});

src/main/redux/sagas/win/library.ts

@@ -114,6 +114,7 @@ function* winOpen(action: winActions.library.openSucess.TAction) {
         },
         creator: state.creator,
         settings: state.settings,
+        apiKeys: state.apiKeys,
     };
     try {
         const publication = yield* callTyped(getCatalog);

src/main/redux/sagas/win/reader.ts

@@ -48,6 +48,7 @@ function* winOpen(action: winActions.reader.openSucess.TAction) {
     const transientConfigMerge = {...readerConfigInitialState, ...config};
     const creator = yield* selectTyped((_state: RootState) => _state.creator);
     const annotationImportQueue = yield* selectTyped((_state: RootState) => _state.annotationImportQueue);
+    const apiKeys = yield* selectTyped((_state: RootState) => _state.apiKeys);
 
     const publicationRepository = diMainGet("publication-repository");
     let tag: string[] = [];
@@ -109,6 +110,7 @@ function* winOpen(action: winActions.reader.openSucess.TAction) {
                 tag,
             },
             annotationImportQueue,
+            apiKeys,
         },
     } as readerIpc.EventPayload);
 }

src/main/redux/states/index.ts

@@ -21,6 +21,7 @@ import { IDictWinSessionReaderState } from "./win/session/reader";
 import { ICommonRootState } from "readium-desktop/common/redux/states/commonRootState";
 import { IWizardState } from "readium-desktop/common/redux/states/wizard";
 import { ISettingsState } from "readium-desktop/common/redux/states/settings";
+import { IApiKeysArray } from "readium-desktop/common/redux/states/api_key";
 
 export interface RootState extends ICommonRootState {
     app: AppState;
@@ -51,6 +52,7 @@ export interface RootState extends ICommonRootState {
     version: string;
     wizard: IWizardState;
     settings: ISettingsState;
+    apiKeys: IApiKeysArray;
 }
 
-export type PersistRootState = Pick<RootState, "win" | "publication" | "reader" | "session" | "i18n" | "opds" | "version" | "theme" | "wizard" | "settings" | "creator" | "annotationImportQueue">;
+export type PersistRootState = Pick<RootState, "win" | "publication" | "reader" | "session" | "i18n" | "opds" | "version" | "theme" | "wizard" | "settings" | "creator" | "annotationImportQueue" | "apiKeys">;