From a169efc5c9a6c94994687ef506bbd2c3c7e7e4dd Mon Sep 17 00:00:00 2001 From: Guiners Date: Mon, 17 Nov 2025 17:06:51 +0100 Subject: [PATCH 1/5] adding count token samples --- ...nttoken-localtokenizer-compute-with-txt.js | 55 +++++++++++++++++++ .../counttoken-localtokenizer-with-txt.js | 48 ++++++++++++++++ ...en-localtokenizer-compute-with-txt.test.js | 32 +++++++++++ ...counttoken-localtokenizer-with-txt.test.js | 32 +++++++++++ 4 files changed, 167 insertions(+) create mode 100644 genai/count-tokens/counttoken-localtokenizer-compute-with-txt.js create mode 100644 genai/count-tokens/counttoken-localtokenizer-with-txt.js create mode 100644 genai/test/counttoken-localtokenizer-compute-with-txt.test.js create mode 100644 genai/test/counttoken-localtokenizer-with-txt.test.js diff --git a/genai/count-tokens/counttoken-localtokenizer-compute-with-txt.js b/genai/count-tokens/counttoken-localtokenizer-compute-with-txt.js new file mode 100644 index 0000000000..88a2785486 --- /dev/null +++ b/genai/count-tokens/counttoken-localtokenizer-compute-with-txt.js @@ -0,0 +1,55 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or written agreement, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the License for the License. +// limitations under the License. + +'use strict'; + +// [START googlegenaisdk_counttoken_localtokenizer_compute_with_txt] +const {GoogleGenAI} = require('@google/genai'); + +const GOOGLE_CLOUD_PROJECT = process.env.GOOGLE_CLOUD_PROJECT; +const GOOGLE_CLOUD_LOCATION = process.env.GOOGLE_CLOUD_LOCATION || 'global'; + +async function counttokenLocalTokenizerCompute() { + const client = new GoogleGenAI({ + vertexai: true, + project: GOOGLE_CLOUD_PROJECT, + location: GOOGLE_CLOUD_LOCATION, + httpOptions: {apiVersion: 'v1'}, + }); + + const response = await client.models.computeTokens({ + model: 'gemini-2.5-flash', + contents: "What's the longest word in the English language?", + }); + + console.log(response.tokensInfo); + + // Example output: + // { + // tokensInfo: [ + // { + // role: 'user', + // tokenIds: [...], + // tokens: [...], + // } + // ] + // } + + return response.tokensInfo; +} +// [END googlegenaisdk_counttoken_localtokenizer_compute_with_txt] + +module.exports = { + counttokenLocalTokenizerCompute, +}; diff --git a/genai/count-tokens/counttoken-localtokenizer-with-txt.js b/genai/count-tokens/counttoken-localtokenizer-with-txt.js new file mode 100644 index 0000000000..e0afb574b6 --- /dev/null +++ b/genai/count-tokens/counttoken-localtokenizer-with-txt.js @@ -0,0 +1,48 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +// [START googlegenaisdk_counttoken_localtokenizer_with_txt] + +const {GoogleGenAI} = require('@google/genai'); + +const GOOGLE_CLOUD_PROJECT = process.env.GOOGLE_CLOUD_PROJECT; +const GOOGLE_CLOUD_LOCATION = process.env.GOOGLE_CLOUD_LOCATION || 'global'; + +async function counttokenLocalTokenizer() { + const client = new GoogleGenAI({ + vertexai: true, + project: GOOGLE_CLOUD_PROJECT, + location: GOOGLE_CLOUD_LOCATION, + httpOptions: {apiVersion: 'v1'}, + }); + + const response = await client.models.countTokens({ + model: 'gemini-2.5-flash', + contents: "What's the highest mountain in Africa?", + }); + + console.log(response.totalTokens); + + // Example output: + // { totalTokens: 10 } + + return response.totalTokens; +} +// [END googlegenaisdk_counttoken_localtokenizer_with_txt] + +module.exports = { + counttokenLocalTokenizer, +}; diff --git a/genai/test/counttoken-localtokenizer-compute-with-txt.test.js b/genai/test/counttoken-localtokenizer-compute-with-txt.test.js new file mode 100644 index 0000000000..e1cd5515a1 --- /dev/null +++ b/genai/test/counttoken-localtokenizer-compute-with-txt.test.js @@ -0,0 +1,32 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +const {assert} = require('chai'); +const {describe, it} = require('mocha'); + +const projectId = process.env.CAIP_PROJECT_ID; +const sample = require('../count-tokens/counttoken-localtokenizer-compute-with-txt.js'); +const {delay} = require('./util'); + +describe('counttoken-localtokenizer-compute-with-txt', () => { + it('should return totalTokens from text prompt', async function () { + this.timeout(18000); + this.retries(4); + await delay(this.test); + const output = await sample.counttokenLocalTokenizerCompute(projectId); + assert(output.length > 0); + }); +}); diff --git a/genai/test/counttoken-localtokenizer-with-txt.test.js b/genai/test/counttoken-localtokenizer-with-txt.test.js new file mode 100644 index 0000000000..e3f11205ea --- /dev/null +++ b/genai/test/counttoken-localtokenizer-with-txt.test.js @@ -0,0 +1,32 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +'use strict'; + +const {assert} = require('chai'); +const {describe, it} = require('mocha'); + +const projectId = process.env.CAIP_PROJECT_ID; +const sample = require('../count-tokens/counttoken-localtokenizer-with-txt.js'); +const {delay} = require('./util'); + +describe('counttoken-localtokenizer-with-txt', () => { + it('should return totalTokens from text prompt', async function () { + this.timeout(18000); + this.retries(4); + await delay(this.test); + const output = await sample.counttokenLocalTokenizer(projectId); + assert(output > 0); + }); +}); From c105b5b3c7255863ec004a5e3d418c4d99052af9 Mon Sep 17 00:00:00 2001 From: Guiners Date: Fri, 5 Dec 2025 11:02:51 +0100 Subject: [PATCH 2/5] adding count token --- .../counttoken-localtokenizer-compute-with-txt.js | 4 ++-- genai/test/counttoken-localtokenizer-compute-with-txt.test.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/genai/count-tokens/counttoken-localtokenizer-compute-with-txt.js b/genai/count-tokens/counttoken-localtokenizer-compute-with-txt.js index 88a2785486..24df160b82 100644 --- a/genai/count-tokens/counttoken-localtokenizer-compute-with-txt.js +++ b/genai/count-tokens/counttoken-localtokenizer-compute-with-txt.js @@ -20,7 +20,7 @@ const {GoogleGenAI} = require('@google/genai'); const GOOGLE_CLOUD_PROJECT = process.env.GOOGLE_CLOUD_PROJECT; const GOOGLE_CLOUD_LOCATION = process.env.GOOGLE_CLOUD_LOCATION || 'global'; -async function counttokenLocalTokenizerCompute() { +async function countTokenLocalTokenizerCompute() { const client = new GoogleGenAI({ vertexai: true, project: GOOGLE_CLOUD_PROJECT, @@ -51,5 +51,5 @@ async function counttokenLocalTokenizerCompute() { // [END googlegenaisdk_counttoken_localtokenizer_compute_with_txt] module.exports = { - counttokenLocalTokenizerCompute, + countTokenLocalTokenizerCompute, }; diff --git a/genai/test/counttoken-localtokenizer-compute-with-txt.test.js b/genai/test/counttoken-localtokenizer-compute-with-txt.test.js index e1cd5515a1..1057d047d8 100644 --- a/genai/test/counttoken-localtokenizer-compute-with-txt.test.js +++ b/genai/test/counttoken-localtokenizer-compute-with-txt.test.js @@ -26,7 +26,7 @@ describe('counttoken-localtokenizer-compute-with-txt', () => { this.timeout(18000); this.retries(4); await delay(this.test); - const output = await sample.counttokenLocalTokenizerCompute(projectId); + const output = await sample.countTokenLocalTokenizerCompute(projectId); assert(output.length > 0); }); }); From 1ed2e27698a43ddbe73d63ce93a0ba9384e5d847 Mon Sep 17 00:00:00 2001 From: Guiners Date: Fri, 5 Dec 2025 11:03:44 +0100 Subject: [PATCH 3/5] adding count token --- genai/count-tokens/counttoken-localtokenizer-with-txt.js | 4 ++-- genai/test/counttoken-localtokenizer-with-txt.test.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/genai/count-tokens/counttoken-localtokenizer-with-txt.js b/genai/count-tokens/counttoken-localtokenizer-with-txt.js index e0afb574b6..1f2807c4a4 100644 --- a/genai/count-tokens/counttoken-localtokenizer-with-txt.js +++ b/genai/count-tokens/counttoken-localtokenizer-with-txt.js @@ -21,7 +21,7 @@ const {GoogleGenAI} = require('@google/genai'); const GOOGLE_CLOUD_PROJECT = process.env.GOOGLE_CLOUD_PROJECT; const GOOGLE_CLOUD_LOCATION = process.env.GOOGLE_CLOUD_LOCATION || 'global'; -async function counttokenLocalTokenizer() { +async function countTokenLocalTokenizer() { const client = new GoogleGenAI({ vertexai: true, project: GOOGLE_CLOUD_PROJECT, @@ -44,5 +44,5 @@ async function counttokenLocalTokenizer() { // [END googlegenaisdk_counttoken_localtokenizer_with_txt] module.exports = { - counttokenLocalTokenizer, + countTokenLocalTokenizer, }; diff --git a/genai/test/counttoken-localtokenizer-with-txt.test.js b/genai/test/counttoken-localtokenizer-with-txt.test.js index e3f11205ea..d538190e45 100644 --- a/genai/test/counttoken-localtokenizer-with-txt.test.js +++ b/genai/test/counttoken-localtokenizer-with-txt.test.js @@ -26,7 +26,7 @@ describe('counttoken-localtokenizer-with-txt', () => { this.timeout(18000); this.retries(4); await delay(this.test); - const output = await sample.counttokenLocalTokenizer(projectId); + const output = await sample.countTokenLocalTokenizer(projectId); assert(output > 0); }); }); From 9a2c208a768672190e8262976787281e4c4069b5 Mon Sep 17 00:00:00 2001 From: Robert Kozak <50328216+Guiners@users.noreply.github.com> Date: Fri, 5 Dec 2025 11:04:23 +0100 Subject: [PATCH 4/5] Update genai/test/counttoken-localtokenizer-compute-with-txt.test.js Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- genai/test/counttoken-localtokenizer-compute-with-txt.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/genai/test/counttoken-localtokenizer-compute-with-txt.test.js b/genai/test/counttoken-localtokenizer-compute-with-txt.test.js index 1057d047d8..eb9f902614 100644 --- a/genai/test/counttoken-localtokenizer-compute-with-txt.test.js +++ b/genai/test/counttoken-localtokenizer-compute-with-txt.test.js @@ -22,7 +22,7 @@ const sample = require('../count-tokens/counttoken-localtokenizer-compute-with-t const {delay} = require('./util'); describe('counttoken-localtokenizer-compute-with-txt', () => { - it('should return totalTokens from text prompt', async function () { + it('should return tokensInfo from text prompt', async function () { this.timeout(18000); this.retries(4); await delay(this.test); From cdf0f41f9f8021407c14248cd058569ca0251d54 Mon Sep 17 00:00:00 2001 From: Guiners Date: Fri, 5 Dec 2025 16:07:46 +0100 Subject: [PATCH 5/5] adding count token --- genai/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/genai/package.json b/genai/package.json index 2c370bd447..d4a806e544 100644 --- a/genai/package.json +++ b/genai/package.json @@ -13,7 +13,7 @@ "test": "c8 mocha -p -j 2 --timeout 2400000 test/*.test.js test/**/*.test.js" }, "dependencies": { - "@google/genai": "1.20.0", + "@google/genai": "1.30.0", "axios": "^1.6.2", "google-auth-library": "^10.3.0", "luxon": "^3.7.1",