diff --git a/docs/tutorials/gemini-live-integration.mdx b/docs/tutorials/gemini-live-integration.mdx index 7e3c497a..80606a3e 100644 --- a/docs/tutorials/gemini-live-integration.mdx +++ b/docs/tutorials/gemini-live-integration.mdx @@ -136,8 +136,7 @@ Create a Fishjam agent configured to match the audio format that the Google clie ```python - from fishjam.peer import SubscribeOptions, SubscribeOptionsAudioSampleRate - from fishjam.agent import OutgoingAudioTrackOptions, TrackEncoding + from fishjam import AgentOptions from fishjam.integrations.gemini import GeminiIntegration room = fishjam_client.create_room() @@ -178,12 +177,11 @@ Fishjam handles raw bytes, while Google GenAI SDKs often expect Base64 strings. output: GI.geminiInputAudioSettings, }); - enum Modality { - AUDIO = 'AUDIO' - } + // ---cut--- import GeminiIntegration from '@fishjam-cloud/js-server-sdk/gemini'; + import { Modality } from '@google/genai'; const GEMINI_MODEL = 'gemini-2.5-flash-native-audio-preview-12-2025' diff --git a/package.json b/package.json index a7f9fd10..74cf2195 100644 --- a/package.json +++ b/package.json @@ -55,6 +55,7 @@ "@docusaurus/module-type-aliases": "^3.8.1", "@docusaurus/tsconfig": "^3.8.1", "@docusaurus/types": "^3.8.1", + "@google/genai": "^1.35.0", "@shikijs/twoslash": "^3.6.0", "cspell": "^9.1.2", "docusaurus-plugin-typedoc": "1.4.0", diff --git a/versioned_docs/version-0.23.0/api/server-python/fishjam.md b/versioned_docs/version-0.23.0/api/server-python/fishjam.md index 2123b2ea..38bf4e17 100644 --- a/versioned_docs/version-0.23.0/api/server-python/fishjam.md +++ b/versioned_docs/version-0.23.0/api/server-python/fishjam.md @@ -403,7 +403,7 @@ True if livestream viewers can omit specifying a token. ```python class AgentOptions: ``` -Options specific to a WebRTC Peer. +Options specific to an Agent Peer. Attributes: - output: Configuration for the agent's output options. diff --git a/versioned_docs/version-0.23.0/tutorials/gemini-live-integration.mdx b/versioned_docs/version-0.23.0/tutorials/gemini-live-integration.mdx index 7e3c497a..4dcf9dd6 100644 --- a/versioned_docs/version-0.23.0/tutorials/gemini-live-integration.mdx +++ b/versioned_docs/version-0.23.0/tutorials/gemini-live-integration.mdx @@ -136,8 +136,7 @@ Create a Fishjam agent configured to match the audio format that the Google clie ```python - from fishjam.peer import SubscribeOptions, SubscribeOptionsAudioSampleRate - from fishjam.agent import OutgoingAudioTrackOptions, TrackEncoding + from fishjam import AgentOptions from fishjam.integrations.gemini import GeminiIntegration room = fishjam_client.create_room() @@ -161,6 +160,7 @@ Fishjam handles raw bytes, while Google GenAI SDKs often expect Base64 strings. Now we setup the callbacks. We need to forward incoming Fishjam audio to Google, and forward incoming Google audio to Fishjam. ```ts + // @errors: 2307 import { FishjamClient } from '@fishjam-cloud/js-server-sdk'; import GI from '@fishjam-cloud/js-server-sdk/gemini'; @@ -178,12 +178,11 @@ Fishjam handles raw bytes, while Google GenAI SDKs often expect Base64 strings. output: GI.geminiInputAudioSettings, }); - enum Modality { - AUDIO = 'AUDIO' - } + // ---cut--- import GeminiIntegration from '@fishjam-cloud/js-server-sdk/gemini'; + import { Modality } from '@google/genai'; const GEMINI_MODEL = 'gemini-2.5-flash-native-audio-preview-12-2025' diff --git a/versioned_docs/version-0.24.0/api/server-python/fishjam/index.md b/versioned_docs/version-0.24.0/api/server-python/fishjam/index.md index 2f9558e7..86967ac3 100644 --- a/versioned_docs/version-0.24.0/api/server-python/fishjam/index.md +++ b/versioned_docs/version-0.24.0/api/server-python/fishjam/index.md @@ -394,7 +394,7 @@ True if livestream viewers can omit specifying a token. ```python class AgentOptions: ``` -Options specific to a WebRTC Peer. +Options specific to an Agent Peer. Attributes: - output: Configuration for the agent's output options. diff --git a/versioned_docs/version-0.24.0/tutorials/gemini-live-integration.mdx b/versioned_docs/version-0.24.0/tutorials/gemini-live-integration.mdx index 7e3c497a..4dcf9dd6 100644 --- a/versioned_docs/version-0.24.0/tutorials/gemini-live-integration.mdx +++ b/versioned_docs/version-0.24.0/tutorials/gemini-live-integration.mdx @@ -136,8 +136,7 @@ Create a Fishjam agent configured to match the audio format that the Google clie ```python - from fishjam.peer import SubscribeOptions, SubscribeOptionsAudioSampleRate - from fishjam.agent import OutgoingAudioTrackOptions, TrackEncoding + from fishjam import AgentOptions from fishjam.integrations.gemini import GeminiIntegration room = fishjam_client.create_room() @@ -161,6 +160,7 @@ Fishjam handles raw bytes, while Google GenAI SDKs often expect Base64 strings. Now we setup the callbacks. We need to forward incoming Fishjam audio to Google, and forward incoming Google audio to Fishjam. ```ts + // @errors: 2307 import { FishjamClient } from '@fishjam-cloud/js-server-sdk'; import GI from '@fishjam-cloud/js-server-sdk/gemini'; @@ -178,12 +178,11 @@ Fishjam handles raw bytes, while Google GenAI SDKs often expect Base64 strings. output: GI.geminiInputAudioSettings, }); - enum Modality { - AUDIO = 'AUDIO' - } + // ---cut--- import GeminiIntegration from '@fishjam-cloud/js-server-sdk/gemini'; + import { Modality } from '@google/genai'; const GEMINI_MODEL = 'gemini-2.5-flash-native-audio-preview-12-2025' diff --git a/yarn.lock b/yarn.lock index 44ec2ae0..51fe628e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3585,6 +3585,21 @@ __metadata: languageName: node linkType: hard +"@google/genai@npm:^1.35.0": + version: 1.35.0 + resolution: "@google/genai@npm:1.35.0" + dependencies: + google-auth-library: "npm:^10.3.0" + ws: "npm:^8.18.0" + peerDependencies: + "@modelcontextprotocol/sdk": ^1.24.0 + peerDependenciesMeta: + "@modelcontextprotocol/sdk": + optional: true + checksum: 10c0/0ee12fb5b2e7cf0ea00df0697fe65964b9824a620d58193cd3837799139a21e2302e18947f8a59bc8d308fc6d90f51e627c4ff7a98e0a5c25e6de759487e657e + languageName: node + linkType: hard + "@hapi/hoek@npm:^9.0.0, @hapi/hoek@npm:^9.3.0": version: 9.3.0 resolution: "@hapi/hoek@npm:9.3.0" @@ -8067,7 +8082,7 @@ __metadata: languageName: node linkType: hard -"base64-js@npm:^1.3.1, base64-js@npm:^1.5.1": +"base64-js@npm:^1.3.0, base64-js@npm:^1.3.1, base64-js@npm:^1.5.1": version: 1.5.1 resolution: "base64-js@npm:1.5.1" checksum: 10c0/f23823513b63173a001030fae4f2dabe283b99a9d324ade3ad3d148e218134676f1ee8568c877cd79ec1c53158dcf2d2ba527a97c606618928ba99dd930102bf @@ -8095,6 +8110,13 @@ __metadata: languageName: node linkType: hard +"bignumber.js@npm:^9.0.0": + version: 9.3.1 + resolution: "bignumber.js@npm:9.3.1" + checksum: 10c0/61342ba5fe1c10887f0ecf5be02ff6709271481aff48631f86b4d37d55a99b87ce441cfd54df3d16d10ee07ceab7e272fc0be430c657ffafbbbf7b7d631efb75 + languageName: node + linkType: hard + "binary-extensions@npm:^2.0.0": version: 2.3.0 resolution: "binary-extensions@npm:2.3.0" @@ -8236,6 +8258,13 @@ __metadata: languageName: node linkType: hard +"buffer-equal-constant-time@npm:^1.0.1": + version: 1.0.1 + resolution: "buffer-equal-constant-time@npm:1.0.1" + checksum: 10c0/fb2294e64d23c573d0dd1f1e7a466c3e978fe94a4e0f8183937912ca374619773bef8e2aceb854129d2efecbbc515bbd0cc78d2734a3e3031edb0888531bbc8e + languageName: node + linkType: hard + "buffer-from@npm:^1.0.0": version: 1.1.2 resolution: "buffer-from@npm:1.1.2" @@ -9975,6 +10004,13 @@ __metadata: languageName: node linkType: hard +"data-uri-to-buffer@npm:^4.0.0": + version: 4.0.1 + resolution: "data-uri-to-buffer@npm:4.0.1" + checksum: 10c0/20a6b93107597530d71d4cb285acee17f66bcdfc03fd81040921a81252f19db27588d87fc8fc69e1950c55cfb0bf8ae40d0e5e21d907230813eb5d5a7f9eb45b + languageName: node + linkType: hard + "dayjs@npm:^1.11.13": version: 1.11.13 resolution: "dayjs@npm:1.11.13" @@ -10379,6 +10415,15 @@ __metadata: languageName: node linkType: hard +"ecdsa-sig-formatter@npm:1.0.11, ecdsa-sig-formatter@npm:^1.0.11": + version: 1.0.11 + resolution: "ecdsa-sig-formatter@npm:1.0.11" + dependencies: + safe-buffer: "npm:^5.0.1" + checksum: 10c0/ebfbf19d4b8be938f4dd4a83b8788385da353d63307ede301a9252f9f7f88672e76f2191618fd8edfc2f24679236064176fab0b78131b161ee73daa37125408c + languageName: node + linkType: hard + "ee-first@npm:1.1.1": version: 1.1.1 resolution: "ee-first@npm:1.1.1" @@ -10946,7 +10991,7 @@ __metadata: languageName: node linkType: hard -"extend@npm:^3.0.0": +"extend@npm:^3.0.0, extend@npm:^3.0.2": version: 3.0.2 resolution: "extend@npm:3.0.2" checksum: 10c0/73bf6e27406e80aa3e85b0d1c4fd987261e628064e170ca781125c0b635a3dabad5e05adbf07595ea0cf1e6c5396cacb214af933da7cbaf24fe75ff14818e8f9 @@ -11125,6 +11170,16 @@ __metadata: languageName: node linkType: hard +"fetch-blob@npm:^3.1.2, fetch-blob@npm:^3.1.4": + version: 3.2.0 + resolution: "fetch-blob@npm:3.2.0" + dependencies: + node-domexception: "npm:^1.0.0" + web-streams-polyfill: "npm:^3.0.3" + checksum: 10c0/60054bf47bfa10fb0ba6cb7742acec2f37c1f56344f79a70bb8b1c48d77675927c720ff3191fa546410a0442c998d27ab05e9144c32d530d8a52fbe68f843b69 + languageName: node + linkType: hard + "figures@npm:^3.2.0": version: 3.2.0 resolution: "figures@npm:3.2.0" @@ -11281,6 +11336,7 @@ __metadata: "@fishjam-cloud/js-server-sdk": "link:./packages/js-server-sdk/packages/js-server-sdk" "@fishjam-cloud/react-client": "link:./packages/web-client-sdk/packages/react-client" "@fishjam-cloud/react-native-client": "link:./packages/mobile-client-sdk/packages/react-native-client" + "@google/genai": "npm:^1.35.0" "@mdx-js/react": "npm:^3.1.0" "@shikijs/rehype": "npm:^3.6.0" "@shikijs/transformers": "npm:^3.6.0" @@ -11407,6 +11463,15 @@ __metadata: languageName: node linkType: hard +"formdata-polyfill@npm:^4.0.10": + version: 4.0.10 + resolution: "formdata-polyfill@npm:4.0.10" + dependencies: + fetch-blob: "npm:^3.1.2" + checksum: 10c0/5392ec484f9ce0d5e0d52fb5a78e7486637d516179b0eb84d81389d7eccf9ca2f663079da56f761355c0a65792810e3b345dc24db9a8bbbcf24ef3c8c88570c6 + languageName: node + linkType: hard + "forwarded@npm:0.2.0": version: 0.2.0 resolution: "forwarded@npm:0.2.0" @@ -11512,6 +11577,29 @@ __metadata: languageName: node linkType: hard +"gaxios@npm:^7.0.0": + version: 7.1.3 + resolution: "gaxios@npm:7.1.3" + dependencies: + extend: "npm:^3.0.2" + https-proxy-agent: "npm:^7.0.1" + node-fetch: "npm:^3.3.2" + rimraf: "npm:^5.0.1" + checksum: 10c0/a4a1cdf9a392c0c22e9734a40dca5a77a2903f505b939a50f1e68e312458b1289b7993d2f72d011426e89657cae77a3aa9fc62fb140e8ba90a1faa31fdbde4d2 + languageName: node + linkType: hard + +"gcp-metadata@npm:^8.0.0": + version: 8.1.2 + resolution: "gcp-metadata@npm:8.1.2" + dependencies: + gaxios: "npm:^7.0.0" + google-logging-utils: "npm:^1.0.0" + json-bigint: "npm:^1.0.0" + checksum: 10c0/15a61231a9410dc11c2828d2c9fdc8b0a939f1af746195c44edc6f2ffea0acab52cef3a7b9828069a36fd5d68bda730f7328a415fe42a01258f6e249dfba6908 + languageName: node + linkType: hard + "gensequence@npm:^7.0.0": version: 7.0.0 resolution: "gensequence@npm:7.0.0" @@ -11630,6 +11718,22 @@ __metadata: languageName: node linkType: hard +"glob@npm:^10.3.7": + version: 10.5.0 + resolution: "glob@npm:10.5.0" + dependencies: + foreground-child: "npm:^3.1.0" + jackspeak: "npm:^3.1.2" + minimatch: "npm:^9.0.4" + minipass: "npm:^7.1.2" + package-json-from-dist: "npm:^1.0.0" + path-scurry: "npm:^1.11.1" + bin: + glob: dist/esm/bin.mjs + checksum: 10c0/100705eddbde6323e7b35e1d1ac28bcb58322095bd8e63a7d0bef1a2cdafe0d0f7922a981b2b48369a4f8c1b077be5c171804534c3509dfe950dde15fbe6d828 + languageName: node + linkType: hard + "glob@npm:^7.1.1, glob@npm:^7.1.3, glob@npm:^7.1.4": version: 7.2.3 resolution: "glob@npm:7.2.3" @@ -11743,6 +11847,28 @@ __metadata: languageName: node linkType: hard +"google-auth-library@npm:^10.3.0": + version: 10.5.0 + resolution: "google-auth-library@npm:10.5.0" + dependencies: + base64-js: "npm:^1.3.0" + ecdsa-sig-formatter: "npm:^1.0.11" + gaxios: "npm:^7.0.0" + gcp-metadata: "npm:^8.0.0" + google-logging-utils: "npm:^1.0.0" + gtoken: "npm:^8.0.0" + jws: "npm:^4.0.0" + checksum: 10c0/49d3931d20b1f4a4d075216bf5518e2b3396dcf441a8f1952611cf3b6080afb1261c3d32009609047ee4a1cc545269a74b4957e6bba9cce840581df309c4b145 + languageName: node + linkType: hard + +"google-logging-utils@npm:^1.0.0": + version: 1.1.3 + resolution: "google-logging-utils@npm:1.1.3" + checksum: 10c0/e65201c7e96543bd1423b9324013736646b9eed60941e0bfa47b9bfd146d2f09cf3df1c99ca60b7d80a726075263ead049ee72de53372cb8458c3bc55c2c1e59 + languageName: node + linkType: hard + "gopd@npm:^1.0.1, gopd@npm:^1.2.0": version: 1.2.0 resolution: "gopd@npm:1.2.0" @@ -11795,6 +11921,16 @@ __metadata: languageName: node linkType: hard +"gtoken@npm:^8.0.0": + version: 8.0.0 + resolution: "gtoken@npm:8.0.0" + dependencies: + gaxios: "npm:^7.0.0" + jws: "npm:^4.0.0" + checksum: 10c0/058538e5bbe081d30ada5f1fd34d3a8194357c2e6ecbf7c8a98daeefbf13f7e06c15649c7dace6a1d4cc3bc6dc5483bd484d6d7adc5852021896d7c05c439f37 + languageName: node + linkType: hard + "gzip-size@npm:^6.0.0": version: 6.0.0 resolution: "gzip-size@npm:6.0.0" @@ -13220,6 +13356,15 @@ __metadata: languageName: node linkType: hard +"json-bigint@npm:^1.0.0": + version: 1.0.0 + resolution: "json-bigint@npm:1.0.0" + dependencies: + bignumber.js: "npm:^9.0.0" + checksum: 10c0/e3f34e43be3284b573ea150a3890c92f06d54d8ded72894556357946aeed9877fd795f62f37fe16509af189fd314ab1104d0fd0f163746ad231b9f378f5b33f4 + languageName: node + linkType: hard + "json-buffer@npm:3.0.1": version: 3.0.1 resolution: "json-buffer@npm:3.0.1" @@ -13314,6 +13459,27 @@ __metadata: languageName: node linkType: hard +"jwa@npm:^2.0.1": + version: 2.0.1 + resolution: "jwa@npm:2.0.1" + dependencies: + buffer-equal-constant-time: "npm:^1.0.1" + ecdsa-sig-formatter: "npm:1.0.11" + safe-buffer: "npm:^5.0.1" + checksum: 10c0/ab3ebc6598e10dc11419d4ed675c9ca714a387481466b10e8a6f3f65d8d9c9237e2826f2505280a739cf4cbcf511cb288eeec22b5c9c63286fc5a2e4f97e78cf + languageName: node + linkType: hard + +"jws@npm:^4.0.0": + version: 4.0.1 + resolution: "jws@npm:4.0.1" + dependencies: + jwa: "npm:^2.0.1" + safe-buffer: "npm:^5.0.1" + checksum: 10c0/6be1ed93023aef570ccc5ea8d162b065840f3ef12f0d1bb3114cade844de7a357d5dc558201d9a65101e70885a6fa56b17462f520e6b0d426195510618a154d0 + languageName: node + linkType: hard + "katex@npm:^0.16.9": version: 0.16.22 resolution: "katex@npm:0.16.22" @@ -15315,6 +15481,13 @@ __metadata: languageName: node linkType: hard +"node-domexception@npm:^1.0.0": + version: 1.0.0 + resolution: "node-domexception@npm:1.0.0" + checksum: 10c0/5e5d63cda29856402df9472335af4bb13875e1927ad3be861dc5ebde38917aecbf9ae337923777af52a48c426b70148815e890a5d72760f1b4d758cc671b1a2b + languageName: node + linkType: hard + "node-emoji@npm:^2.1.0": version: 2.2.0 resolution: "node-emoji@npm:2.2.0" @@ -15327,6 +15500,17 @@ __metadata: languageName: node linkType: hard +"node-fetch@npm:^3.3.2": + version: 3.3.2 + resolution: "node-fetch@npm:3.3.2" + dependencies: + data-uri-to-buffer: "npm:^4.0.0" + fetch-blob: "npm:^3.1.4" + formdata-polyfill: "npm:^4.0.10" + checksum: 10c0/f3d5e56190562221398c9f5750198b34cf6113aa304e34ee97c94fd300ec578b25b2c2906edba922050fce983338fde0d5d34fcb0fc3336ade5bd0e429ad7538 + languageName: node + linkType: hard + "node-forge@npm:^1": version: 1.3.1 resolution: "node-forge@npm:1.3.1" @@ -18346,6 +18530,17 @@ __metadata: languageName: node linkType: hard +"rimraf@npm:^5.0.1": + version: 5.0.10 + resolution: "rimraf@npm:5.0.10" + dependencies: + glob: "npm:^10.3.7" + bin: + rimraf: dist/esm/bin.mjs + checksum: 10c0/7da4fd0e15118ee05b918359462cfa1e7fe4b1228c7765195a45b55576e8c15b95db513b8466ec89129666f4af45ad978a3057a02139afba1a63512a2d9644cc + languageName: node + linkType: hard + "robust-predicates@npm:^3.0.2": version: 3.0.2 resolution: "robust-predicates@npm:3.0.2" @@ -18395,7 +18590,7 @@ __metadata: languageName: node linkType: hard -"safe-buffer@npm:5.2.1, safe-buffer@npm:>=5.1.0, safe-buffer@npm:^5.1.0, safe-buffer@npm:~5.2.0": +"safe-buffer@npm:5.2.1, safe-buffer@npm:>=5.1.0, safe-buffer@npm:^5.0.1, safe-buffer@npm:^5.1.0, safe-buffer@npm:~5.2.0": version: 5.2.1 resolution: "safe-buffer@npm:5.2.1" checksum: 10c0/6501914237c0a86e9675d4e51d89ca3c21ffd6a31642efeba25ad65720bce6921c9e7e974e5be91a786b25aa058b5303285d3c15dbabf983a919f5f630d349f3 @@ -20333,6 +20528,13 @@ __metadata: languageName: node linkType: hard +"web-streams-polyfill@npm:^3.0.3": + version: 3.3.3 + resolution: "web-streams-polyfill@npm:3.3.3" + checksum: 10c0/64e855c47f6c8330b5436147db1c75cb7e7474d924166800e8e2aab5eb6c76aac4981a84261dd2982b3e754490900b99791c80ae1407a9fa0dcff74f82ea3a7f + languageName: node + linkType: hard + "webpack-bundle-analyzer@npm:^4.10.2": version: 4.10.2 resolution: "webpack-bundle-analyzer@npm:4.10.2" @@ -20691,6 +20893,21 @@ __metadata: languageName: node linkType: hard +"ws@npm:^8.18.0": + version: 8.19.0 + resolution: "ws@npm:8.19.0" + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: ">=5.0.2" + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + checksum: 10c0/4741d9b9bc3f9c791880882414f96e36b8b254e34d4b503279d6400d9a4b87a033834856dbdd94ee4b637944df17ea8afc4bce0ff4a1560d2166be8855da5b04 + languageName: node + linkType: hard + "xdg-basedir@npm:^5.0.1, xdg-basedir@npm:^5.1.0": version: 5.1.0 resolution: "xdg-basedir@npm:5.1.0"